def create_object_test():
"""Verifies file_io's object manipulation methods ."""
starttime = int(round(time.time() * 1000))
dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime)
print("Creating dir %s." % dir_name)
file_io.create_dir(dir_name)
# Create a file in this directory.
file_name = "%s/test_file.txt" % dir_name
print("Creating file %s." % file_name)
file_io.write_string_to_file(file_name, "test file creation.")
list_files_pattern = "%s/test_file*.txt" % dir_name
print("Getting files matching pattern %s." % list_files_pattern)
files_list = file_io.get_matching_files(list_files_pattern)
print(files_list)
assert len(files_list) == 1
assert files_list[0] == file_name
# Cleanup test files.
print("Deleting file %s." % file_name)
file_io.delete_file(file_name)
# Delete directory.
print("Deleting directory %s." % dir_name)
file_io.delete_recursively(dir_name)
def _save_and_write_assets(self, assets_collection_to_add=None):
"""Saves asset to the meta graph and writes asset files to disk.
Args:
assets_collection_to_add: The collection where the asset paths are setup.
"""
asset_source_filepath_list = self._save_assets(assets_collection_to_add)
# Return if there are no assets to write.
if len(asset_source_filepath_list) is 0:
tf_logging.info("No assets to write.")
return
assets_destination_dir = os.path.join(
compat.as_bytes(self._export_dir),
compat.as_bytes(constants.ASSETS_DIRECTORY))
if not file_io.file_exists(assets_destination_dir):
file_io.create_dir(assets_destination_dir)
# Copy each asset from source path to destination path.
for asset_source_filepath in asset_source_filepath_list:
asset_source_filename = os.path.basename(asset_source_filepath)
asset_destination_filepath = os.path.join(
compat.as_bytes(assets_destination_dir),
compat.as_bytes(asset_source_filename))
file_io.copy(
asset_source_filepath, asset_destination_filepath, overwrite=True)
tf_logging.info("Assets written to: %s", assets_destination_dir)
def save(self, as_text=False):
"""Writes a `SavedModel` protocol buffer to disk.
The function writes the SavedModel protocol buffer to the export directory
in serialized format.
Args:
as_text: Writes the SavedModel protocol buffer in text format to disk.
Returns:
The path to which the SavedModel protocol buffer was written.
"""
if not file_io.file_exists(self._export_dir):
file_io.create_dir(self._export_dir)
if as_text:
path = os.path.join(
compat.as_bytes(self._export_dir),
compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT))
file_io.write_string_to_file(path, str(self._saved_model))
else:
path = os.path.join(
compat.as_bytes(self._export_dir),
compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))
file_io.write_string_to_file(path, self._saved_model.SerializeToString())
tf_logging.info("SavedModel written to: %s", path)
return path
def create_dir_test():
"""Verifies file_io directory handling methods ."""
starttime = int(round(time.time() * 1000))
dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime)
print("Creating dir %s" % dir_name)
file_io.create_dir(dir_name)
elapsed = int(round(time.time() * 1000)) - starttime
print("Created directory in: %d milliseconds" % elapsed)
# Check that the directory exists.
dir_exists = file_io.is_directory(dir_name)
print("%s directory exists: %s" % (dir_name, dir_exists))
# List contents of just created directory.
print("Listing directory %s." % dir_name)
starttime = int(round(time.time() * 1000))
print(file_io.list_directory(dir_name))
elapsed = int(round(time.time() * 1000)) - starttime
print("Listed directory %s in %s milliseconds" % (dir_name, elapsed))
# Delete directory.
print("Deleting directory %s." % dir_name)
starttime = int(round(time.time() * 1000))
file_io.delete_recursively(dir_name)
elapsed = int(round(time.time() * 1000)) - starttime
print("Deleted directory %s in %s milliseconds" % (dir_name, elapsed))
def setUpClass(cls):
# Set up dirs.
cls.working_dir = tempfile.mkdtemp()
cls.source_dir = os.path.join(cls.working_dir, 'source')
cls.analysis_dir = os.path.join(cls.working_dir, 'analysis')
cls.output_dir = os.path.join(cls.working_dir, 'output')
file_io.create_dir(cls.source_dir)
# Make test image files.
img1_file = os.path.join(cls.source_dir, 'img1.jpg')
image1 = Image.new('RGB', size=(300, 300), color=(155, 0, 0))
image1.save(img1_file)
img2_file = os.path.join(cls.source_dir, 'img2.jpg')
image2 = Image.new('RGB', size=(50, 50), color=(125, 240, 0))
image2.save(img2_file)
img3_file = os.path.join(cls.source_dir, 'img3.jpg')
image3 = Image.new('RGB', size=(800, 600), color=(33, 55, 77))
image3.save(img3_file)
# Download inception checkpoint. Note that gs url doesn't work because
# we may not have gcloud signed in when running the test.
url = ('https://storage.googleapis.com/cloud-ml-data/img/' +
'flower_photos/inception_v3_2016_08_28.ckpt')
checkpoint_path = os.path.join(cls.working_dir, "checkpoint")
response = urlopen(url)
with open(checkpoint_path, 'wb') as f:
f.write(response.read())
# Make csv input file
cls.csv_input_filepath = os.path.join(cls.source_dir, 'input.csv')
file_io.write_string_to_file(
cls.csv_input_filepath,
'1,Monday,23.0,red blue,%s\n' % img1_file +
'0,Friday,18.0,green,%s\n' % img2_file +
'0,Sunday,12.0,green red blue green,%s\n' % img3_file)
# Call analyze.py to create analysis results.
schema = [{'name': 'target_col', 'type': 'FLOAT'},
{'name': 'cat_col', 'type': 'STRING'},
{'name': 'num_col', 'type': 'FLOAT'},
{'name': 'text_col', 'type': 'STRING'},
{'name': 'img_col', 'type': 'STRING'}]
schema_file = os.path.join(cls.source_dir, 'schema.json')
file_io.write_string_to_file(schema_file, json.dumps(schema))
features = {'target_col': {'transform': 'target'},
'cat_col': {'transform': 'one_hot'},
'num_col': {'transform': 'identity'},
'text_col': {'transform': 'multi_hot'},
'img_col': {'transform': 'image_to_vec', 'checkpoint': checkpoint_path}}
features_file = os.path.join(cls.source_dir, 'features.json')
file_io.write_string_to_file(features_file, json.dumps(features))
cmd = ['python ' + os.path.join(CODE_PATH, 'analyze.py'),
'--output=' + cls.analysis_dir,
'--csv=' + cls.csv_input_filepath,
'--schema=' + schema_file,
'--features=' + features_file]
subprocess.check_call(' '.join(cmd), shell=True)
def testGetMatchingFiles(self):
dir_path = os.path.join(self._base_dir, "temp_dir")
file_io.create_dir(dir_path)
files = ["file1.txt", "file2.txt", "file3.txt"]
for name in files:
file_path = os.path.join(dir_path, name)
file_io.FileIO(file_path, mode="w").write("testing")
expected_match = [os.path.join(dir_path, name) for name in files]
self.assertItemsEqual(file_io.get_matching_files(os.path.join(dir_path, "file*.txt")), expected_match)
file_io.delete_recursively(dir_path)
self.assertFalse(file_io.file_exists(os.path.join(dir_path, "file3.txt")))
def testIsDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
# Failure for a non-existing dir.
with self.assertRaises(errors.NotFoundError):
file_io.is_directory(dir_path)
file_io.create_dir(dir_path)
self.assertTrue(file_io.is_directory(dir_path))
file_path = os.path.join(dir_path, "test_file")
file_io.FileIO(file_path, mode="w").write("test")
# False for a file.
self.assertFalse(file_io.is_directory(file_path))
def setUpClass(cls):
# Set up dirs.
cls.working_dir = tempfile.mkdtemp()
cls.source_dir = os.path.join(cls.working_dir, 'source')
cls.analysis_dir = os.path.join(cls.working_dir, 'analysis')
cls.output_dir = os.path.join(cls.working_dir, 'output')
file_io.create_dir(cls.source_dir)
# Make test image files.
img1_file = os.path.join(cls.source_dir, 'img1.jpg')
image1 = Image.new('RGBA', size=(300, 300), color=(155, 0, 0))
image1.save(img1_file)
img2_file = os.path.join(cls.source_dir, 'img2.jpg')
image2 = Image.new('RGBA', size=(50, 50), color=(125, 240, 0))
image2.save(img2_file)
img3_file = os.path.join(cls.source_dir, 'img3.jpg')
image3 = Image.new('RGBA', size=(800, 600), color=(33, 55, 77))
image3.save(img3_file)
# Make csv input file
cls.csv_input_filepath = os.path.join(cls.source_dir, 'input.csv')
file_io.write_string_to_file(
cls.csv_input_filepath,
'1,1,Monday,23.0,%s\n' % img1_file +
'2,0,Friday,18.0,%s\n' % img2_file +
'3,0,Sunday,12.0,%s\n' % img3_file)
# Call analyze.py to create analysis results.
schema = [{'name': 'key_col', 'type': 'INTEGER'},
{'name': 'target_col', 'type': 'FLOAT'},
{'name': 'cat_col', 'type': 'STRING'},
{'name': 'num_col', 'type': 'FLOAT'},
{'name': 'img_col', 'type': 'STRING'}]
schema_file = os.path.join(cls.source_dir, 'schema.json')
file_io.write_string_to_file(schema_file, json.dumps(schema))
features = {'key_col': {'transform': 'key'},
'target_col': {'transform': 'target'},
'cat_col': {'transform': 'one_hot'},
'num_col': {'transform': 'identity'},
'img_col': {'transform': 'image_to_vec'}}
features_file = os.path.join(cls.source_dir, 'features.json')
file_io.write_string_to_file(features_file, json.dumps(features))
cmd = ['python ' + os.path.join(CODE_PATH, 'analyze.py'),
'--output=' + cls.analysis_dir,
'--csv=' + cls.csv_input_filepath,
'--schema=' + schema_file,
'--features=' + features_file]
subprocess.check_call(' '.join(cmd), shell=True)
# Setup a temp GCS bucket.
cls.bucket_root = 'gs://temp_mltoolbox_test_%s' % uuid.uuid4().hex
subprocess.check_call('gsutil mb %s' % cls.bucket_root, shell=True)
def end(self, session=None):
super(ExportLastModelMonitor, self).end(session)
file_io.recursive_create_dir(self._dest)
_recursive_copy(self.last_export_dir, self._dest)
if self._additional_assets:
# TODO(rhaertel): use the actual assets directory. For now, metadata.yaml
# must be a sibling of the export.meta file.
assets_dir = self._dest
file_io.create_dir(assets_dir)
_copy_all(self._additional_assets, assets_dir)
def testListDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
file_io.create_dir(dir_path)
files = [b"file1.txt", b"file2.txt", b"file3.txt"]
for name in files:
file_path = os.path.join(dir_path, compat.as_str_any(name))
file_io.write_string_to_file(file_path, "testing")
subdir_path = os.path.join(dir_path, "sub_dir")
file_io.create_dir(subdir_path)
subdir_file_path = os.path.join(subdir_path, "file4.txt")
file_io.write_string_to_file(subdir_file_path, "testing")
dir_list = file_io.list_directory(dir_path)
self.assertItemsEqual(files + [b"sub_dir"], dir_list)
def testIsDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
# Failure for a non-existing dir.
self.assertFalse(file_io.is_directory(dir_path))
file_io.create_dir(dir_path)
self.assertTrue(file_io.is_directory(dir_path))
file_path = os.path.join(dir_path, "test_file")
file_io.FileIO(file_path, mode="w").write("test")
# False for a file.
self.assertFalse(file_io.is_directory(file_path))
# Test that the value returned from `stat()` has `is_directory` set.
file_statistics = file_io.stat(dir_path)
self.assertTrue(file_statistics.is_directory)
def testListDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
file_io.create_dir(dir_path)
files = ["file1.txt", "file2.txt", "file3.txt"]
for name in files:
file_path = os.path.join(dir_path, name)
file_io.FileIO(file_path, mode="w").write("testing")
subdir_path = os.path.join(dir_path, "sub_dir")
file_io.create_dir(subdir_path)
subdir_file_path = os.path.join(subdir_path, "file4.txt")
file_io.FileIO(subdir_file_path, mode="w").write("testing")
dir_list = file_io.list_directory(dir_path)
self.assertItemsEqual(files + ["sub_dir"], dir_list)
def end(self, session=None):
super(ExportLastModelMonitor, self).end(session)
# Recursively copy the last location export dir from the exporter into the
# main export location.
file_io.recursive_create_dir(self._final_model_location)
_recursive_copy(self.last_export_dir, self._final_model_location)
if self._additional_assets:
# TODO(rhaertel): use the actual assets directory. For now, metadata.json
# must be a sibling of the export.meta file.
assets_dir = self._final_model_location
file_io.create_dir(assets_dir)
_copy_all(self._additional_assets, assets_dir)
def testGetMatchingFiles(self):
dir_path = os.path.join(self.get_temp_dir(), "temp_dir")
file_io.create_dir(dir_path)
files = ["file1.txt", "file2.txt", "file3.txt"]
for name in files:
file_path = os.path.join(dir_path, name)
file_io.write_string_to_file(file_path, "testing")
expected_match = [os.path.join(dir_path, name) for name in files]
self.assertItemsEqual(file_io.get_matching_files(os.path.join(dir_path,
"file*.txt")),
expected_match)
for name in files:
file_path = os.path.join(dir_path, name)
file_io.delete_file(file_path)
def _recursive_copy(src_dir, dest_dir):
"""Copy the contents of src_dir into the folder dest_dir.
When called, dest_dir should exist.
"""
for dir_name, sub_dirs, leaf_files in file_io.walk(src_dir):
# copy all the files over
for leaf_file in leaf_files:
leaf_file_path = os.path.join(dir_name, leaf_file)
_copy_all([leaf_file_path], dest_dir)
# Now make all the folders.
for sub_dir in sub_dirs:
file_io.create_dir(os.path.join(dest_dir, sub_dir))
def __init__(self, export_dir):
self._saved_model = saved_model_pb2.SavedModel()
self._saved_model.saved_model_schema_version = (
constants.SAVED_MODEL_SCHEMA_VERSION)
self._export_dir = export_dir
if not file_io.file_exists(export_dir):
file_io.create_dir(self._export_dir)
# Boolean to track whether variables and assets corresponding to the
# SavedModel have been saved. Specifically, the first meta graph to be added
# MUST use the add_meta_graph_and_variables() API. Subsequent add operations
# on the SavedModel MUST use the add_meta_graph() API which does not save
# weights.
self._has_saved_variables = False
def _setupWalkDirectories(self, dir_path):
# Creating a file structure as follows
# test_dir -> file: file1.txt; dirs: subdir1_1, subdir1_2, subdir1_3
# subdir1_1 -> file: file3.txt
# subdir1_2 -> dir: subdir2
file_io.create_dir(dir_path)
file_io.FileIO(os.path.join(dir_path, "file1.txt"), mode="w").write("testing")
sub_dirs1 = ["subdir1_1", "subdir1_2", "subdir1_3"]
for name in sub_dirs1:
file_io.create_dir(os.path.join(dir_path, name))
file_io.FileIO(os.path.join(dir_path, "subdir1_1/file2.txt"), mode="w").write("testing")
file_io.create_dir(os.path.join(dir_path, "subdir1_2/subdir2"))
def testMatchingFilesPermission(self):
# Create top level directory test_dir.
dir_path = os.path.join(self._base_dir, "test_dir")
file_io.create_dir(dir_path)
# Create second level directories `noread` and `any`.
noread_path = os.path.join(dir_path, "noread")
file_io.create_dir(noread_path)
any_path = os.path.join(dir_path, "any")
file_io.create_dir(any_path)
files = ["file1.txt", "file2.txt", "file3.txt"]
for name in files:
file_path = os.path.join(any_path, name)
file_io.FileIO(file_path, mode="w").write("testing")
file_path = os.path.join(noread_path, "file4.txt")
file_io.FileIO(file_path, mode="w").write("testing")
# Change noread to noread access.
os.chmod(noread_path, 0)
expected_match = [os.path.join(any_path, name) for name in files]
self.assertItemsEqual(
file_io.get_matching_files(os.path.join(dir_path, "*", "file*.txt")),
expected_match)
# Change noread back so that it could be cleaned during tearDown.
os.chmod(noread_path, 0o777)
def setUp(self):
self._base_dir = os.path.join(self.get_temp_dir(), "base_dir")
file_io.create_dir(self._base_dir)
def train_and_evaluate(args):
print(args)
try:
os.makedirs(args.job_dir)
except:
pass
data_dir = args.data_dir
preprocessed_dir = args.preprocessed_dir
if args.data_dir.startswith('gs://'): # OpenCV does not support Google storage. We need to copy them to local
tempDataDir = "temp_data_dir"
file_io.create_dir(tempDataDir)
copyDir(args.data_dir, tempDataDir)
# Unzip the file train_images.zip
print("Extract zip file: train_images.zip")
with ZipFile(os.path.join(tempDataDir, "train_images.zip"), 'r') as zipObj:
# Extract all the contents of zip file in different directory
zipObj.extractall(os.path.join(tempDataDir, "train_images"))
print("Extract zip file completed: train_images.zip")
data_dir = tempDataDir
preprocessed_dir = os.path.join(data_dir, "preprocessed")
model_dir = args.model_dir
if args.model_dir.startswith('gs://'): # H5py does not support Google storage. We need to copy them to local
tempModelDir = "temp_model_dir"
file_io.create_dir(tempModelDir)
copyFile(os.path.join(args.model_dir, args.model_name + ".h5"), tempModelDir)
model_dir = tempModelDir
sd_util.initContext(data_dir, model_dir, preprocessed_dir)
sd_util.FAST_VERIFICATION = args.fast_verification
sd_util.SHOW_PROGRESS = args.show_progress
bigLoop = 1
epochs = args.num_epochs
if epochs > 3:
bigLoop = int(epochs // 3 + 1)
epochs = 3
label_df = sd_util.loadLabels(os.path.join(sd_util.DATA_DIR, "train.csv"))
trainFiles, validFiles = sd_training.splitTrainingDataSet(os.path.join(sd_util.PREPROCESSED_FOLDER, "AllFiles.txt"))
# This two line for fast run training to verify code logic.
if sd_util.FAST_VERIFICATION:
trainFiles = trainFiles[:30]
validFiles = validFiles[:30]
print("Train files: {}".format(trainFiles.shape[0]))
print("Valid files: {}".format(validFiles.shape[0]))
optimizer = keras.optimizers.Adam()
continueTraining = args.continue_training
for _ in range(bigLoop):
print(args)
_, modelName = sd_training.train(args.model_name, label_df, trainFiles, validFiles, shrink = 2,
batch_size = args.batch_size, epoch = epochs, lossfunc = sd_losses.diceBCELoss,
optimizer = optimizer, continueTraining = continueTraining,
unlockResnet = args.unlock_backbone)
continueTraining = True
if args.model_dir.startswith('gs://'):
modelFileName = modelName + ".h5"
copyFile(os.path.join(model_dir, modelFileName), args.model_dir)
print("Copied H5 file to: {}".format(os.path.join(args.model_dir, modelFileName)))
def setUpClass(cls):
# Set up dirs.
cls.working_dir = tempfile.mkdtemp()
cls.source_dir = os.path.join(cls.working_dir, 'source')
cls.analysis_dir = os.path.join(cls.working_dir, 'analysis')
cls.output_dir = os.path.join(cls.working_dir, 'output')
file_io.create_dir(cls.source_dir)
# Make test image files.
img1_file = os.path.join(cls.source_dir, 'img1.jpg')
image1 = Image.new('RGBA', size=(300, 300), color=(155, 0, 0))
image1.save(img1_file)
img2_file = os.path.join(cls.source_dir, 'img2.jpg')
image2 = Image.new('RGBA', size=(50, 50), color=(125, 240, 0))
image2.save(img2_file)
img3_file = os.path.join(cls.source_dir, 'img3.jpg')
image3 = Image.new('RGBA', size=(800, 600), color=(33, 55, 77))
image3.save(img3_file)
# Make csv input file
cls.csv_input_filepath = os.path.join(cls.source_dir, 'input.csv')
file_io.write_string_to_file(
cls.csv_input_filepath, '1,1,Monday,23.0,%s\n' % img1_file +
'2,0,Friday,18.0,%s\n' % img2_file +
'3,0,Sunday,12.0,%s\n' % img3_file)
# Call analyze_data.py to create analysis results.
schema = [{
'name': 'key_col',
'type': 'INTEGER'
}, {
'name': 'target_col',
'type': 'FLOAT'
}, {
'name': 'cat_col',
'type': 'STRING'
}, {
'name': 'num_col',
'type': 'FLOAT'
}, {
'name': 'img_col',
'type': 'STRING'
}]
schema_file = os.path.join(cls.source_dir, 'schema.json')
file_io.write_string_to_file(schema_file, json.dumps(schema))
features = {
'key_col': {
'transform': 'key'
},
'target_col': {
'transform': 'target'
},
'cat_col': {
'transform': 'one_hot'
},
'num_col': {
'transform': 'identity'
},
'img_col': {
'transform': 'image_to_vec'
}
}
features_file = os.path.join(cls.source_dir, 'features.json')
file_io.write_string_to_file(features_file, json.dumps(features))
cmd = [
'python ' + os.path.join(CODE_PATH, 'analyze_data.py'),
'--output-dir=' + cls.analysis_dir,
'--csv-file-pattern=' + cls.csv_input_filepath,
'--csv-schema-file=' + schema_file,
'--features-file=' + features_file
]
subprocess.check_call(' '.join(cmd), shell=True)
# Setup a temp GCS bucket.
cls.bucket_root = 'gs://temp_mltoolbox_test_%s' % uuid.uuid4().hex
subprocess.check_call('gsutil mb %s' % cls.bucket_root, shell=True)
def create_dir_test():
"""Verifies file_io directory handling methods."""
# Test directory creation.
starttime_ms = int(round(time.time() * 1000))
dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms)
print("Creating dir %s" % dir_name)
file_io.create_dir(dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Created directory in: %d milliseconds" % elapsed_ms)
# Check that the directory exists.
dir_exists = file_io.is_directory(dir_name)
assert dir_exists
print("%s directory exists: %s" % (dir_name, dir_exists))
# Test recursive directory creation.
starttime_ms = int(round(time.time() * 1000))
recursive_dir_name = "%s/%s/%s" % (dir_name,
"nested_dir1",
"nested_dir2")
print("Creating recursive dir %s" % recursive_dir_name)
file_io.recursive_create_dir(recursive_dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Created directory recursively in: %d milliseconds" % elapsed_ms)
# Check that the directory exists.
recursive_dir_exists = file_io.is_directory(recursive_dir_name)
assert recursive_dir_exists
print("%s directory exists: %s" % (recursive_dir_name, recursive_dir_exists))
# Create some contents in the just created directory and list the contents.
num_files = 10
files_to_create = ["file_%d.txt" % n for n in range(num_files)]
for file_num in files_to_create:
file_name = "%s/%s" % (dir_name, file_num)
print("Creating file %s." % file_name)
file_io.write_string_to_file(file_name, "test file.")
print("Listing directory %s." % dir_name)
starttime_ms = int(round(time.time() * 1000))
directory_contents = file_io.list_directory(dir_name)
print(directory_contents)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Listed directory %s in %s milliseconds" % (dir_name, elapsed_ms))
assert set(directory_contents) == set(files_to_create + ["nested_dir1/"])
# Test directory renaming.
dir_to_rename = "%s/old_dir" % dir_name
new_dir_name = "%s/new_dir" % dir_name
file_io.create_dir(dir_to_rename)
assert file_io.is_directory(dir_to_rename)
assert not file_io.is_directory(new_dir_name)
starttime_ms = int(round(time.time() * 1000))
print("Will try renaming directory %s to %s" % (dir_to_rename, new_dir_name))
file_io.rename(dir_to_rename, new_dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Renamed directory %s to %s in %s milliseconds" % (
dir_to_rename, new_dir_name, elapsed_ms))
assert not file_io.is_directory(dir_to_rename)
assert file_io.is_directory(new_dir_name)
# Test Delete directory recursively.
print("Deleting directory recursively %s." % dir_name)
starttime_ms = int(round(time.time() * 1000))
file_io.delete_recursively(dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
dir_exists = file_io.is_directory(dir_name)
assert not dir_exists
print("Deleted directory recursively %s in %s milliseconds" % (
dir_name, elapsed_ms))
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
data_sets = input_data.read_data_sets(tempfile.mkdtemp(), FLAGS.fake_data)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
placeholders = placeholder_inputs()
keys_placeholder, images_placeholder, labels_placeholder = placeholders
inputs = {'key': keys_placeholder.name, 'image': images_placeholder.name}
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a Graph that computes predictions from the inference model.
logits = mnist.inference(images_placeholder,
FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = mnist.loss(logits, labels_placeholder)
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
prediction = tf.argmax(logits, 1)
scores = tf.nn.softmax(logits)
# Mark the outputs.
outputs = {'key': keys.name,
'prediction': prediction.name,
'scores': scores.name}
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = mnist.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = mnist.evaluation(logits, labels_placeholder)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Add the variable initializer Op.
init = tf.initialize_all_variables()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train,
images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.train_dir, 'checkpoint')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.train)
# Evaluate against the validation set.
print('Validation Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.validation)
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.test)
# Export the model so that it can be loaded and used later for predictions.
file_io.create_dir(FLAGS.model_dir)
saver.save(sess, os.path.join(FLAGS.model_dir, 'export'))
def create_dir(cls, dirname):
return file_io.create_dir(dirname)
def create_object_test():
"""Verifies file_io's object manipulation methods ."""
starttime_ms = int(round(time.time() * 1000))
dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms)
print("Creating dir %s." % dir_name)
file_io.create_dir(dir_name)
num_files = 5
# Create files of 2 different patterns in this directory.
files_pattern_1 = [
"%s/test_file_%d.txt" % (dir_name, n) for n in range(num_files)
]
files_pattern_2 = [
"%s/testfile%d.txt" % (dir_name, n) for n in range(num_files)
]
starttime_ms = int(round(time.time() * 1000))
files_to_create = files_pattern_1 + files_pattern_2
for file_name in files_to_create:
print("Creating file %s." % file_name)
file_io.write_string_to_file(file_name, "test file creation.")
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Created %d files in %s milliseconds" %
(len(files_to_create), elapsed_ms))
# Listing files of pattern1.
list_files_pattern = "%s/test_file*.txt" % dir_name
print("Getting files matching pattern %s." % list_files_pattern)
starttime_ms = int(round(time.time() * 1000))
files_list = file_io.get_matching_files(list_files_pattern)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Listed files in %s milliseconds" % elapsed_ms)
print(files_list)
assert set(files_list) == set(files_pattern_1)
# Listing files of pattern2.
list_files_pattern = "%s/testfile*.txt" % dir_name
print("Getting files matching pattern %s." % list_files_pattern)
starttime_ms = int(round(time.time() * 1000))
files_list = file_io.get_matching_files(list_files_pattern)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Listed files in %s milliseconds" % elapsed_ms)
print(files_list)
assert set(files_list) == set(files_pattern_2)
# Test renaming file.
file_to_rename = "%s/oldname.txt" % dir_name
file_new_name = "%s/newname.txt" % dir_name
file_io.write_string_to_file(file_to_rename, "test file.")
assert file_io.file_exists(file_to_rename)
assert not file_io.file_exists(file_new_name)
print("Will try renaming file %s to %s" % (file_to_rename, file_new_name))
starttime_ms = int(round(time.time() * 1000))
file_io.rename(file_to_rename, file_new_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("File %s renamed to %s in %s milliseconds" %
(file_to_rename, file_new_name, elapsed_ms))
assert not file_io.file_exists(file_to_rename)
assert file_io.file_exists(file_new_name)
# Delete directory.
print("Deleting directory %s." % dir_name)
file_io.delete_recursively(dir_name)
def create_dir_test():
"""Verifies file_io directory handling methods."""
# Test directory creation.
starttime_ms = int(round(time.time() * 1000))
dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms)
print("Creating dir %s" % dir_name)
file_io.create_dir(dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Created directory in: %d milliseconds" % elapsed_ms)
# Check that the directory exists.
dir_exists = file_io.is_directory(dir_name)
assert dir_exists
print("%s directory exists: %s" % (dir_name, dir_exists))
# Test recursive directory creation.
starttime_ms = int(round(time.time() * 1000))
recursive_dir_name = "%s/%s/%s" % (dir_name, "nested_dir1", "nested_dir2")
print("Creating recursive dir %s" % recursive_dir_name)
file_io.recursive_create_dir(recursive_dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Created directory recursively in: %d milliseconds" % elapsed_ms)
# Check that the directory exists.
recursive_dir_exists = file_io.is_directory(recursive_dir_name)
assert recursive_dir_exists
print("%s directory exists: %s" %
(recursive_dir_name, recursive_dir_exists))
# Create some contents in the just created directory and list the contents.
num_files = 10
files_to_create = ["file_%d.txt" % n for n in range(num_files)]
for file_num in files_to_create:
file_name = "%s/%s" % (dir_name, file_num)
print("Creating file %s." % file_name)
file_io.write_string_to_file(file_name, "test file.")
print("Listing directory %s." % dir_name)
starttime_ms = int(round(time.time() * 1000))
directory_contents = file_io.list_directory(dir_name)
print(directory_contents)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Listed directory %s in %s milliseconds" % (dir_name, elapsed_ms))
assert set(directory_contents) == set(files_to_create + ["nested_dir1/"])
# Test directory renaming.
dir_to_rename = "%s/old_dir" % dir_name
new_dir_name = "%s/new_dir" % dir_name
file_io.create_dir(dir_to_rename)
assert file_io.is_directory(dir_to_rename)
assert not file_io.is_directory(new_dir_name)
starttime_ms = int(round(time.time() * 1000))
print("Will try renaming directory %s to %s" %
(dir_to_rename, new_dir_name))
file_io.rename(dir_to_rename, new_dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Renamed directory %s to %s in %s milliseconds" %
(dir_to_rename, new_dir_name, elapsed_ms))
assert not file_io.is_directory(dir_to_rename)
assert file_io.is_directory(new_dir_name)
# Test Delete directory recursively.
print("Deleting directory recursively %s." % dir_name)
starttime_ms = int(round(time.time() * 1000))
file_io.delete_recursively(dir_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
dir_exists = file_io.is_directory(dir_name)
assert not dir_exists
print("Deleted directory recursively %s in %s milliseconds" %
(dir_name, elapsed_ms))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
if not file_io.file_exists(self.train_dir):
file_io.create_dir(self.train_dir)
model_flags_dict = {
"model": FLAGS.model,
"feature_sizes": FLAGS.feature_sizes,
"feature_names": FLAGS.feature_names,
"frame_features": FLAGS.frame_features,
"label_loss": FLAGS.label_loss,
}
flags_json_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if file_io.file_exists(flags_json_path):
existing_flags = json.loads(
file_io.FileIO(flags_json_path, "r").read())
if existing_flags != model_flags_dict:
logging.error(
"Model flags do not match existing file %s. Please "
"delete the file, change --train_dir, or pass flag "
"--start_new_model", flags_json_path)
logging.error("Ran model with flags: %s",
str(model_flags_dict))
logging.error("Previously ran with flags: %s",
str(existing_flags))
exit(1)
else:
# Write the file.
with file_io.FileIO(flags_json_path, mode="w") as fout:
fout.write(json.dumps(model_flags_dict))
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model(self.model, self.reader)
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
num_frames = tf.get_collection("num_frames")[0]
init_op = tf.global_variables_initializer()
sv = tf.train.Supervisor(graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session**.",
task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
try:
logging.info("%s: Entering training loop.",
task_as_string(self.task))
while (not sv.should_stop()) and (not self.max_steps_reached):
batch_start_time = time.time()
_, global_step_val, loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, predictions, labels])
seconds_per_batch = time.time() - batch_start_time
examples_per_second = labels_val.shape[
0] / seconds_per_batch
if self.max_steps and self.max_steps <= global_step_val:
self.max_steps_reached = True
if self.is_master and global_step_val % 10 == 0 and self.train_dir:
eval_start_time = time.time()
hit_at_one = eval_util.calculate_hit_at_one(
predictions_val, labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val,
labels_val)
eval_end_time = time.time()
eval_time = eval_end_time - eval_start_time
logging.info("training step " + str(global_step_val) +
" | Loss: " + ("%.2f" % loss_val) +
" Examples/sec: " +
("%.2f" % examples_per_second) +
" | Hit@1: " + ("%.2f" % hit_at_one) +
" PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1",
hit_at_one), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second),
global_step_val)
sv.summary_writer.flush()
# Exporting the model every x steps
time_to_export = (
(self.last_model_export_step == 0)
or (global_step_val - self.last_model_export_step
>= self.export_model_steps))
if self.is_master and time_to_export:
self.export_model(global_step_val, sv.saver,
sv.save_path, sess)
self.last_model_export_step = global_step_val
else:
logging.info("training step " + str(global_step_val) +
" | Loss: " + ("%.2f" % loss_val) +
" Examples/sec: " +
("%.2f" % examples_per_second))
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def setUpClass(cls):
# Set up dirs.
cls.working_dir = tempfile.mkdtemp()
cls.source_dir = os.path.join(cls.working_dir, 'source')
cls.analysis_dir = os.path.join(cls.working_dir, 'analysis')
cls.output_dir = os.path.join(cls.working_dir, 'output')
file_io.create_dir(cls.source_dir)
# Make test image files.
img1_file = os.path.join(cls.source_dir, 'img1.jpg')
image1 = Image.new('RGBA', size=(300, 300), color=(155, 0, 0))
image1.save(img1_file)
img2_file = os.path.join(cls.source_dir, 'img2.jpg')
image2 = Image.new('RGBA', size=(50, 50), color=(125, 240, 0))
image2.save(img2_file)
img3_file = os.path.join(cls.source_dir, 'img3.jpg')
image3 = Image.new('RGBA', size=(800, 600), color=(33, 55, 77))
image3.save(img3_file)
# Download inception checkpoint. Note that gs url doesn't work because
# we may not have gcloud signed in when running the test.
url = ('https://storage.googleapis.com/cloud-ml-data/img/' +
'flower_photos/inception_v3_2016_08_28.ckpt')
checkpoint_path = os.path.join(cls.working_dir, "checkpoint")
response = urlopen(url)
with open(checkpoint_path, 'wb') as f:
f.write(response.read())
# Make csv input file
cls.csv_input_filepath = os.path.join(cls.source_dir, 'input.csv')
file_io.write_string_to_file(
cls.csv_input_filepath, '1,1,Monday,23.0,%s\n' % img1_file +
'2,0,Friday,18.0,%s\n' % img2_file +
'3,0,Sunday,12.0,%s\n' % img3_file)
# Call analyze.py to create analysis results.
schema = [{
'name': 'key_col',
'type': 'INTEGER'
}, {
'name': 'target_col',
'type': 'FLOAT'
}, {
'name': 'cat_col',
'type': 'STRING'
}, {
'name': 'num_col',
'type': 'FLOAT'
}, {
'name': 'img_col',
'type': 'STRING'
}]
schema_file = os.path.join(cls.source_dir, 'schema.json')
file_io.write_string_to_file(schema_file, json.dumps(schema))
features = {
'key_col': {
'transform': 'key'
},
'target_col': {
'transform': 'target'
},
'cat_col': {
'transform': 'one_hot'
},
'num_col': {
'transform': 'identity'
},
'img_col': {
'transform': 'image_to_vec',
'checkpoint': checkpoint_path
}
}
features_file = os.path.join(cls.source_dir, 'features.json')
file_io.write_string_to_file(features_file, json.dumps(features))
cmd = [
'python ' + os.path.join(CODE_PATH, 'analyze.py'),
'--output=' + cls.analysis_dir, '--csv=' + cls.csv_input_filepath,
'--schema=' + schema_file, '--features=' + features_file
]
subprocess.check_call(' '.join(cmd), shell=True)
def setUp(self):
self._base_dir = os.path.join(self.get_temp_dir(), 'base_dir')
file_io.create_dir(self._base_dir)
super(IoUtilsTest, self).setUp()
def run_training():
#Read the training data
training_examples = read_im_list('tf_train.csv')
val_examples = read_im_list('tf_val.csv')
#np.random.seed(42) #shuffle the same way each time for consistency
np.random.shuffle(training_examples)
#fetcher = Fetcher(examples)
fetcher = Fetcher(training_examples)
#images, labels = fetcher.load_batch_balanced(FLAGS.batch_size)
#print images.shape, labels.shape
file_io.create_dir(os.path.join(FLAGS.model_dir))
file_io.create_dir(os.path.join(FLAGS.train_output_dir))
#FLAGS.im_size = 256
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
x = tf.placeholder(tf.float32, shape=(None, FLAGS.im_size,FLAGS.im_size,3))
y_ = tf.placeholder(tf.float32, shape=(None, n_classes))
# See "Using instance keys": https://cloud.google.com/ml/docs/how-tos/preparing-models
# for why we have keys_placeholder
keys_placeholder = tf.placeholder(tf.int64, shape=(None,))
# IMPORTANT: Do not change the input map
inputs = {'key': keys_placeholder.name, 'image': x.name}
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a the network
#net = network(x)
#net = alexnet(x)
net = overfeat(x)
# Add to the Graph the Ops for loss calculation.
loss = slim.losses.softmax_cross_entropy(net, y_)
tf.scalar_summary(loss.op.name, loss) # keep track of value for TensorBoard
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
# DO NOT REMOVE OR CHANGE VARIABLE NAMES - used when predicting with a model
prediction = tf.argmax(net, 1)
scores = tf.nn.softmax(net)
# Mark the outputs.
outputs = {'key': keys.name,
'prediction': prediction.name,
'scores': scores.name}
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = tf.train.AdamOptimizer(FLAGS.Adam_lr).minimize(loss) # lr = 1e-4
#train_op = tf.train.AdamOptimizer(FLAGS.Adam_lr, beta1 = FLAGS.Adam_beta1,
# beta2=FLAGS.Adam_beta2, epsilon=FLAGS.Adam_eps).minimize(loss)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Add the variable initializer Op.
#init = tf.initialize_all_variables()
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver(max_to_keep = 20)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_output_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
lossf = open(os.path.join(FLAGS.model_dir,'loss_acc.txt'),'w')
lossf.write('step, loss\n')
lossf.close()
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
#images, labels = fetcher.load_batch(FLAGS.batch_size)
images, labels = fetcher.load_batch_balanced(FLAGS.batch_size)
# images, labels = load_cv_batch(x_train,y_train,step,FLAGS.batch_size)
feed_dict = {x: images, y_: labels}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if (step % 5) == 0:
# Print status to stdout.
print('Step %d: loss = %g (%.3f sec)' % (step, loss_value, duration))
sys.stdout.flush()
with open(os.path.join(FLAGS.model_dir,'loss_acc.txt'),'a') as lossf:
lossf.write('%d, %g\n' %(step, loss_value) )
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
if (step%25 == 0 and (step > 1)) or (step == FLAGS.max_steps-1):
input_alias_map = json.loads(sess.graph.get_collection('inputs')[0])
output_alias_map = json.loads(sess.graph.get_collection('outputs')[0])
aliases, tensor_names = zip(*output_alias_map.items())
y_true = []
y_pred = []
for (label, files) in val_examples:
#channels = [ misc.imread(file_io.FileIO(f,'r')) for f in files]
#channels = [misc.imresize(channels[j],(im_size,im_size)) for j in xrange(len(channels))]
channels = misc.imread(files)
#channels = misc.imresize(channels,[FLAGS.im_size, FLAGS.im_size])
#image = np.dstack(channels)
image = misc.imresize(channels,[FLAGS.im_size, FLAGS.im_size])
feed_dict = {input_alias_map['image']: [image]}
predict, scores = sess.run(fetches=[output_alias_map['prediction'],
output_alias_map['scores']], feed_dict=feed_dict)
y_true.append(np.argmax(label))
y_pred.append(predict[0])
accuracy = metrics.accuracy_score(y_true,y_pred)
#f1macro = metrics.f1_score(y_true,y_pred,average='macro')
#f1micro = metrics.f1_score(y_true,y_pred,average='micro')
#print('Val Accuracy: %g, Val F1macro: %g, Val F1micro:%g\n' %(accuracy, f1macro, f1micro))
#print('Confusion matrix is')
#print(metrics.confusion_matrix(y_true, y_pred))
#with open(os.path.join(FLAGS.model_dir,'loss_acc.txt'),'a') as lossf:
# lossf.write('%d, %g, %g, %g, %g\n' %(step, loss_value, accuracy, f1macro, f1micro))
f1 = metrics.f1_score(y_true, y_pred)
cohen_kappa = metrics.cohen_kappa_score(y_true, y_pred)
print('Val Accuracy: %g, F1: %g, cohen_kappa: %g\n' %(accuracy, f1, cohen_kappa))
print('Confusion matrix is')
print(metrics.confusion_matrix(y_true, y_pred))
with open(os.path.join(FLAGS.model_dir,'loss_acc.txt'),'a') as lossf:
lossf.write('%d, %g, %g, %g %g\n' %(step, loss_value, accuracy, f1, cohen_kappa))
if ((step%1000 == 0) and (step>1)) or (step == FLAGS.max_steps-1): # and (step > 1):
# Export the model so that it can be loaded and used later for predictions.
file_io.create_dir(os.path.join(FLAGS.model_dir, str(step)))
saver.save(sess, os.path.join(FLAGS.model_dir, str(step),'export' ))
def train(args):
data_loader = TextLoader(args.data_dir, args.batch_size, args.seq_length)
args.vocab_size = data_loader.vocab_size
# check compatibility if training is continued from previously saved model
if args.init_from is not None:
# check if all necessary files exist
# note to self: file_io.file_exists should be able to replace these when working with gcs, or there's also file_io.is_path, if we really want to tailor things specifically.
assert file_io.file_exists(
args.init_from), " %s must be a path" % args.init_from
assert file_io.file_exists(
os.path.join(args.init_from, "config.pkl")
), "config.pkl file does not exist in path %s" % args.init_from
assert file_io.file_exists(
os.path.join(args.init_from, "chars_vocab.pkl")
), "chars_vocab.pkl.pkl file does not exist in path %s" % args.init_from
ckpt = tf.train.get_checkpoint_state(args.init_from)
assert ckpt, "No checkpoint found"
assert ckpt.model_checkpoint_path, "No model path found in checkpoint"
# open old config and check if models are compatible
# see above note; remember to use file_io.FileIO
with file_io.FileIO(os.path.join(args.init_from, 'config.pkl'),
'rb') as f:
saved_model_args = cPickle.load(f)
need_be_same = ["model", "rnn_size", "num_layers", "seq_length"]
for checkme in need_be_same:
assert vars(saved_model_args)[checkme] == vars(
args
)[checkme], "Command line argument and saved model disagree on '%s' " % checkme
# open saved vocab/dict and check if vocabs/dicts are compatible
with file_io.FileIO(os.path.join(args.init_from, 'chars_vocab.pkl'),
'rb') as f:
saved_chars, saved_vocab = cPickle.load(f)
assert saved_chars == data_loader.chars, "Data and loaded model disagree on character set!"
assert saved_vocab == data_loader.vocab, "Data and loaded model disagree on dictionary mappings!"
if not file_io.is_path(args.save_dir):
file_io.create_dir(
args.save_dir) # file_io.create_dir? also: file_io.FileIO below
with file_io.FileIO(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
cPickle.dump(args, f)
with file_io.FileIO(os.path.join(args.save_dir, 'chars_vocab.pkl'),
'wb') as f:
cPickle.dump((data_loader.chars, data_loader.vocab), f)
model = Model(args)
with tf.Session() as sess:
# instrument for tensorboard
summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter(
os.path.join(args.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S")))
writer.add_graph(
sess.graph
) ## this'll get you the graph in tensorboard, which is helpful
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
# restore model
if args.init_from is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
for e in range(args.num_epochs):
sess.run(
tf.assign(model.lr, args.learning_rate * (args.decay_rate**e)))
data_loader.reset_batch_pointer()
state = sess.run(model.initial_state)
for b in range(data_loader.num_batches):
start = time.time()
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.targets: y}
for i, (c, h) in enumerate(model.initial_state):
## just, to avoid idiocy later, (c, h) = (hidden state, output)
feed[c] = state[i].c
feed[h] = state[i].h
# instrument for tensorboard
summ, train_loss, state, _ = sess.run(
[summaries, model.cost, model.final_state, model.train_op],
feed)
writer.add_summary(summ, e * data_loader.num_batches + b)
end = time.time()
print(
"{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
.format(e * data_loader.num_batches + b,
args.num_epochs * data_loader.num_batches, e,
train_loss, end - start))
if (e * data_loader.num_batches + b) % args.save_every == 0\
or (e == args.num_epochs-1 and
b == data_loader.num_batches-1):
# save for the last result
checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
saver.save(sess,
checkpoint_path,
global_step=e * data_loader.num_batches + b)
print("model saved to {}".format(checkpoint_path))
def create_object_test():
"""Verifies file_io's object manipulation methods ."""
starttime_ms = int(round(time.time() * 1000))
dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms)
print("Creating dir %s." % dir_name)
file_io.create_dir(dir_name)
num_files = 5
# Create files of 2 different patterns in this directory.
files_pattern_1 = ["%s/test_file_%d.txt" % (dir_name, n)
for n in range(num_files)]
files_pattern_2 = ["%s/testfile%d.txt" % (dir_name, n)
for n in range(num_files)]
starttime_ms = int(round(time.time() * 1000))
files_to_create = files_pattern_1 + files_pattern_2
for file_name in files_to_create:
print("Creating file %s." % file_name)
file_io.write_string_to_file(file_name, "test file creation.")
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Created %d files in %s milliseconds" %
(len(files_to_create), elapsed_ms))
# Listing files of pattern1.
list_files_pattern = "%s/test_file*.txt" % dir_name
print("Getting files matching pattern %s." % list_files_pattern)
starttime_ms = int(round(time.time() * 1000))
files_list = file_io.get_matching_files(list_files_pattern)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Listed files in %s milliseconds" % elapsed_ms)
print(files_list)
assert set(files_list) == set(files_pattern_1)
# Listing files of pattern2.
list_files_pattern = "%s/testfile*.txt" % dir_name
print("Getting files matching pattern %s." % list_files_pattern)
starttime_ms = int(round(time.time() * 1000))
files_list = file_io.get_matching_files(list_files_pattern)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("Listed files in %s milliseconds" % elapsed_ms)
print(files_list)
assert set(files_list) == set(files_pattern_2)
# Test renaming file.
file_to_rename = "%s/oldname.txt" % dir_name
file_new_name = "%s/newname.txt" % dir_name
file_io.write_string_to_file(file_to_rename, "test file.")
assert file_io.file_exists(file_to_rename)
assert not file_io.file_exists(file_new_name)
print("Will try renaming file %s to %s" % (file_to_rename, file_new_name))
starttime_ms = int(round(time.time() * 1000))
file_io.rename(file_to_rename, file_new_name)
elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
print("File %s renamed to %s in %s milliseconds" % (
file_to_rename, file_new_name, elapsed_ms))
assert not file_io.file_exists(file_to_rename)
assert file_io.file_exists(file_new_name)
# Delete directory.
print("Deleting directory %s." % dir_name)
file_io.delete_recursively(dir_name)
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
data_sets = input_data.read_data_sets(tempfile.mkdtemp(), FLAGS.fake_data)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
placeholders = placeholder_inputs()
keys_placeholder, images_placeholder, labels_placeholder = placeholders
inputs = {'key': keys_placeholder.name, 'image': images_placeholder.name}
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a Graph that computes predictions from the inference model.
logits = mnist.inference(images_placeholder,
FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = mnist.loss(logits, labels_placeholder)
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
prediction = tf.argmax(logits, 1)
scores = tf.nn.softmax(logits)
# Mark the outputs.
outputs = {'key': keys.name,
'prediction': prediction.name,
'scores': scores.name}
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = mnist.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = mnist.evaluation(logits, labels_placeholder)
# Build the summary operation based on the TF collection of Summaries.
# TODO(b/33420312): remove the if once 0.12 is fully rolled out to prod.
if tf.__version__ < '0.12':
summary_op = tf.merge_all_summaries()
else:
summary_op = tf.contrib.deprecated.merge_all_summaries()
# Add the variable initializer Op.
init = tf.initialize_all_variables()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train,
images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.train_dir, 'checkpoint')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.train)
# Evaluate against the validation set.
print('Validation Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.validation)
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.test)
# Export the model so that it can be loaded and used later for predictions.
file_io.create_dir(FLAGS.model_dir)
saver.save(sess, os.path.join(FLAGS.model_dir, 'export'))
def _get_filepath(self, id, mkdir=False):
path = 'embeddings/%d' % int(id / 10000)
if mkdir and not file_io.file_exists(path):
file_io.create_dir(path)
return '%s/%d.emb' % (path, id)
def run_training():
file_io.create_dir(FLAGS.model_dir)
np.set_printoptions(suppress=True)
input_shape = (300, 300, 3)
prior_filename = os.path.basename(FLAGS.prior_path)
file_io.copy(FLAGS.prior_path, prior_filename)
priors = pickle.load(open(prior_filename, 'rb'))
bbox_util = BBoxUtility(FLAGS.num_classes, priors)
annotation_filename = os.path.basename(FLAGS.annotation_path)
file_io.copy(FLAGS.annotation_path, annotation_filename)
gt = pickle.load(open(annotation_filename, 'rb'))
keys = sorted(gt.keys())
num_train = int(round(0.8 * len(keys)))
train_keys = keys[:num_train]
val_keys = keys[num_train:]
num_val = len(val_keys)
images_filename = os.path.basename(FLAGS.images_path)
file_io.copy(FLAGS.images_path, images_filename)
tar = tarfile.open(images_filename)
tar.extractall()
tar.close()
path_prefix = images_filename.split('.')[0] + '/'
gen = Generator(gt, bbox_util, 4, path_prefix,
train_keys, val_keys,
(input_shape[0], input_shape[1]), do_crop=False)
net, model = SSD300(input_shape, num_classes=FLAGS.num_classes)
weight_filename = os.path.basename(FLAGS.weight_path)
file_io.copy(FLAGS.weight_path, weight_filename)
model.load_weights(weight_filename, by_name=True)
freeze = ['input_1', 'conv1_1', 'conv1_2', 'pool1',
'conv2_1', 'conv2_2', 'pool2',
'conv3_1', 'conv3_2', 'conv3_3', 'pool3',
'conv4_1', 'conv4_2', 'conv4_3', 'pool4']
for L in model.layers:
if L.name in freeze:
L.trainable = False
base_lr = 3e-4
optim = keras.optimizers.Adam(lr=base_lr)
model.compile(optimizer=optim,
loss=MultiboxLoss(FLAGS.num_classes, neg_pos_ratio=2.0).compute_loss)
# train
model.fit_generator(gen.generate(True), gen.train_batches,
FLAGS.epoch,
validation_data=gen.generate(False),
nb_val_samples=gen.val_batches,
nb_worker=1)
# define prediction layer
keys_placeholder = tf.placeholder(tf.string, shape=[None])
keep_top_k_placeholder = tf.placeholder(dtype='int32', shape=(None))
original_size_placeholder = tf.placeholder(dtype='float32', shape=(None, 2))
confidence_threshold_placeholder = tf.placeholder(dtype='float32', shape=(None))
detection_out = Lambda(bbox_util.detection_out, arguments={
'keep_top_k': keep_top_k_placeholder,
'confidence_threshold': confidence_threshold_placeholder,
'original_size': original_size_placeholder
})(net['predictions'])
# export
inputs = {'key': keys_placeholder,
'data': model.input,
'keep_top_k': keep_top_k_placeholder,
'confidence_threshold': confidence_threshold_placeholder,
'original_size': original_size_placeholder}
outputs = {'key': tf.identity(keys_placeholder), 'objects': detection_out}
export(get_session(), inputs, outputs, FLAGS.model_dir)
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST. If input_path is specified, download the data from GCS to
# the folder expected by read_data_sets.
data_dir = tempfile.mkdtemp()
if FLAGS.input_path:
files = [
os.path.join(FLAGS.input_path, file_name)
for file_name in INPUT_FILES
]
subprocess.check_call(['gsutil', '-m', '-q', 'cp', '-r'] + files +
[data_dir])
data_sets = input_data.read_data_sets(data_dir, FLAGS.fake_data)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
placeholders = placeholder_inputs()
keys_placeholder, images_placeholder, labels_placeholder = placeholders
inputs = {
'key': keys_placeholder.name,
'image': images_placeholder.name
}
input_signatures = {}
for key, val in inputs.iteritems():
predict_input_tensor = meta_graph_pb2.TensorInfo()
predict_input_tensor.name = val
for placeholder in placeholders:
if placeholder.name == val:
predict_input_tensor.dtype = placeholder.dtype.as_datatype_enum
input_signatures[key] = predict_input_tensor
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a Graph that computes predictions from the inference model.
logits = mnist.inference(images_placeholder, FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = mnist.loss(logits, labels_placeholder)
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
prediction = tf.argmax(logits, 1)
scores = tf.nn.softmax(logits)
# Mark the outputs.
outputs = {
'key': keys.name,
'prediction': prediction.name,
'scores': scores.name
}
output_signatures = {}
for key, val in outputs.iteritems():
predict_output_tensor = meta_graph_pb2.TensorInfo()
predict_output_tensor.name = val
for placeholder in [keys, prediction, scores]:
if placeholder.name == val:
predict_output_tensor.dtype = placeholder.dtype.as_datatype_enum
output_signatures[key] = predict_output_tensor
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = mnist.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = mnist.evaluation(logits, labels_placeholder)
# Build the summary operation based on the TF collection of Summaries.
# Remove this if once Tensorflow 0.12 is standard.
try:
summary_op = tf.contrib.deprecated.merge_all_summaries()
except AttributeError:
summary_op = tf.merge_all_summaries()
# Add the variable initializer Op.
init = tf.initialize_all_variables()
# Create a saver for writing legacy training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
# Remove this if once Tensorflow 0.12 is standard.
try:
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
except AttributeError:
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train, images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.train_dir, 'checkpoint')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
do_eval(sess, eval_correct, images_placeholder,
labels_placeholder, data_sets.train)
# Evaluate against the validation set.
print('Validation Data Eval:')
do_eval(sess, eval_correct, images_placeholder,
labels_placeholder, data_sets.validation)
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess, eval_correct, images_placeholder,
labels_placeholder, data_sets.test)
file_io.create_dir(FLAGS.model_dir)
predict_signature_def = signature_def_utils.build_signature_def(
input_signatures, output_signatures,
signature_constants.PREDICT_METHOD_NAME)
# Create a saver for writing SavedModel training checkpoints.
build = builder.SavedModelBuilder(
os.path.join(FLAGS.model_dir, 'saved_model'))
logging.debug('Saved model path %s',
os.path.join(FLAGS.model_dir, 'saved_model'))
build.add_meta_graph_and_variables(
sess, [tag_constants.SERVING],
signature_def_map={
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
predict_signature_def
},
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS))
build.save()
def run_training():
#Read the training data
examples, n_classes = read_training_list() #TODO:Replace this
np.random.seed(42) #shuffle the same way each time for consistency
np.random.shuffle(examples)
fetcher = Fetcher()
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
x = tf.placeholder(tf.float32, shape=(None, 512, 512, 3))
y_ = tf.placeholder(tf.float32, shape=(None, n_classes))
# See "Using instance keys": https://cloud.google.com/ml/docs/how-tos/preparing-models
# for why we have keys_placeholder
keys_placeholder = tf.placeholder(tf.int64, shape=(None, ))
# IMPORTANT: Do not change the input map
inputs = {'key': keys_placeholder.name, 'image': x.name}
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a the network
net = network(x)
# Add to the Graph the Ops for loss calculation.
loss = slim.losses.softmax_cross_entropy(net, y_)
tf.scalar_summary(loss.op.name,
loss) # keep track of value for TensorBoard
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
# DO NOT REMOVE OR CHANGE VARIABLE NAMES - used when predicting with a model
prediction = tf.argmax(net, 1)
scores = tf.nn.softmax(net)
# Mark the outputs.
outputs = {
'key': keys.name,
'prediction': prediction.name,
'scores': scores.name
}
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = tf.train.AdamOptimizer(1e-4).minimize(loss)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Add the variable initializer Op.
init = tf.initialize_all_variables()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_output_dir,
sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
images, labels = fetcher.load_batch(FLAGS.batch_size)
feed_dict = {x: images, y_: labels}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 1 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
sys.stdout.flush()
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Export the model so that it can be loaded and used later for predictions.
file_io.create_dir(FLAGS.model_dir)
saver.save(sess, os.path.join(FLAGS.model_dir, 'export'))
#make world readable for submission to evaluation server
if FLAGS.model_dir.startswith('gs://'):
subprocess.call(
['gsutil', 'acl', 'ch', '-u', 'AllUsers:R', FLAGS.model_dir])
def setUp(self): self._base_dir = os.path.join(self.get_temp_dir(), "base_dir") file_io.create_dir(self._base_dir)
def run_training():
#Read the training data
training_examples = read_im_list('tf_train.csv')
val_examples = read_im_list('tf_val.csv')
#np.random.seed(42) #shuffle the same way each time for consistency
np.random.shuffle(training_examples)
#fetcher = Fetcher(examples)
fetcher = Fetcher(training_examples)
#images, labels = fetcher.load_batch_balanced(FLAGS.batch_size)
#print images.shape, labels.shape
file_io.create_dir(os.path.join(FLAGS.model_dir))
file_io.create_dir(os.path.join(FLAGS.train_output_dir))
#FLAGS.im_size = 256
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
x = tf.placeholder(tf.float32,
shape=(None, FLAGS.im_size, FLAGS.im_size, 3))
y_ = tf.placeholder(tf.float32, shape=(None, n_classes))
# See "Using instance keys": https://cloud.google.com/ml/docs/how-tos/preparing-models
# for why we have keys_placeholder
keys_placeholder = tf.placeholder(tf.int64, shape=(None, ))
# IMPORTANT: Do not change the input map
inputs = {'key': keys_placeholder.name, 'image': x.name}
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a the network
#net = network(x)
net = alexnet(x)
# Add to the Graph the Ops for loss calculation.
loss = slim.losses.softmax_cross_entropy(net, y_)
tf.scalar_summary(loss.op.name,
loss) # keep track of value for TensorBoard
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
# DO NOT REMOVE OR CHANGE VARIABLE NAMES - used when predicting with a model
prediction = tf.argmax(net, 1)
scores = tf.nn.softmax(net)
# Mark the outputs.
outputs = {
'key': keys.name,
'prediction': prediction.name,
'scores': scores.name
}
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = tf.train.AdamOptimizer(FLAGS.Adam_lr).minimize(
loss) # lr = 1e-4
#train_op = tf.train.AdamOptimizer(FLAGS.Adam_lr, beta1 = FLAGS.Adam_beta1,
# beta2=FLAGS.Adam_beta2, epsilon=FLAGS.Adam_eps).minimize(loss)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Add the variable initializer Op.
#init = tf.initialize_all_variables()
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver(max_to_keep=20)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_output_dir,
sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
lossf = open(os.path.join(FLAGS.model_dir, 'loss_acc.txt'), 'w')
lossf.write('step, loss\n')
lossf.close()
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
#images, labels = fetcher.load_batch(FLAGS.batch_size)
images, labels = fetcher.load_batch_balanced(FLAGS.batch_size)
# images, labels = load_cv_batch(x_train,y_train,step,FLAGS.batch_size)
feed_dict = {x: images, y_: labels}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if (step % 5) == 0:
# Print status to stdout.
print('Step %d: loss = %g (%.3f sec)' %
(step, loss_value, duration))
sys.stdout.flush()
with open(os.path.join(FLAGS.model_dir, 'loss_acc.txt'),
'a') as lossf:
lossf.write('%d, %g\n' % (step, loss_value))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
if (step % 25 == 0 and
(step > 1)) or (step == FLAGS.max_steps - 1):
input_alias_map = json.loads(
sess.graph.get_collection('inputs')[0])
output_alias_map = json.loads(
sess.graph.get_collection('outputs')[0])
aliases, tensor_names = zip(*output_alias_map.items())
y_true = []
y_pred = []
for (label, files) in val_examples:
#channels = [ misc.imread(file_io.FileIO(f,'r')) for f in files]
#channels = [misc.imresize(channels[j],(im_size,im_size)) for j in xrange(len(channels))]
channels = misc.imread(files)
#channels = misc.imresize(channels,[FLAGS.im_size, FLAGS.im_size])
#image = np.dstack(channels)
image = misc.imresize(channels,
[FLAGS.im_size, FLAGS.im_size])
feed_dict = {input_alias_map['image']: [image]}
predict, scores = sess.run(fetches=[
output_alias_map['prediction'],
output_alias_map['scores']
],
feed_dict=feed_dict)
y_true.append(np.argmax(label))
y_pred.append(predict[0])
accuracy = metrics.accuracy_score(y_true, y_pred)
#f1macro = metrics.f1_score(y_true,y_pred,average='macro')
#f1micro = metrics.f1_score(y_true,y_pred,average='micro')
#print('Val Accuracy: %g, Val F1macro: %g, Val F1micro:%g\n' %(accuracy, f1macro, f1micro))
f1 = metrics.f1_score(y_true, y_pred)
cohen_kappa = metrics.cohen_kappa_score(y_true, y_pred)
print('Val Accuracy: %g, F1: %g, cohen_kappa: %g\n' %
(accuracy, f1, cohen_kappa))
print('Confusion matrix is')
print(metrics.confusion_matrix(y_true, y_pred))
with open(os.path.join(FLAGS.model_dir, 'loss_acc.txt'),
'a') as lossf:
lossf.write('%d, %g, %g, %g %g\n' %
(step, loss_value, accuracy, f1, cohen_kappa))
if ((step % 1000 == 0) and
(step > 1)) or (step
== FLAGS.max_steps - 1): # and (step > 1):
# Export the model so that it can be loaded and used later for predictions.
file_io.create_dir(os.path.join(FLAGS.model_dir, str(step)))
saver.save(sess,
os.path.join(FLAGS.model_dir, str(step), 'export'))
