def testRename(self): file_path = os.path.join(self._base_dir, "temp_file") file_io.FileIO(file_path, mode="w").write("testing") rename_path = os.path.join(self._base_dir, "rename_file") file_io.rename(file_path, rename_path) self.assertTrue(file_io.file_exists(rename_path)) self.assertFalse(file_io.file_exists(file_path))
def test_table(self): initializer = lookup_ops.TextFileInitializer( self._vocab_path, key_dtype=dtypes.string, key_index=lookup_ops.TextFileIndex.WHOLE_LINE, value_dtype=dtypes.int64, value_index=lookup_ops.TextFileIndex.LINE_NUMBER) root = util.Checkpoint(table=lookup_ops.HashTable( initializer, default_value=-1)) root.table_user = def_function.function( root.table.lookup, input_signature=[tensor_spec.TensorSpec(None, dtypes.string)]) self.assertEqual( 2, self.evaluate(root.table_user(constant_op.constant("gamma")))) save_dir = os.path.join(self.get_temp_dir(), "saved_model") save.save(root, save_dir) file_io.delete_file(self._vocab_path) self.assertAllClose( {"output_0": [2, 0]}, _import_and_infer(save_dir, {"keys": ["gamma", "alpha"]})) second_dir = os.path.join(self.get_temp_dir(), "second_dir") # Asset paths should track the location the SavedModel is loaded from. file_io.rename(save_dir, second_dir) self.assertAllClose( {"output_0": [2, 1]}, _import_and_infer(second_dir, {"keys": ["gamma", "beta"]}))
def test_table(self): initializer = lookup_ops.TextFileInitializer( self._vocab_path, key_dtype=dtypes.string, key_index=lookup_ops.TextFileIndex.WHOLE_LINE, value_dtype=dtypes.int64, value_index=lookup_ops.TextFileIndex.LINE_NUMBER) root = util.Checkpoint( table=lookup_ops.HashTable(initializer, default_value=-1)) root.table_user = def_function.function( root.table.lookup, input_signature=[tensor_spec.TensorSpec(None, dtypes.string)]) self.assertEqual( 2, self.evaluate(root.table_user(constant_op.constant("gamma")))) save_dir = os.path.join(self.get_temp_dir(), "saved_model") save.save(root, save_dir) file_io.delete_file(self._vocab_path) self.assertAllClose({"output_0": [2, 0]}, _import_and_infer(save_dir, {"keys": ["gamma", "alpha"]})) second_dir = os.path.join(self.get_temp_dir(), "second_dir") # Asset paths should track the location the SavedModel is loaded from. file_io.rename(save_dir, second_dir) self.assertAllClose({"output_0": [2, 1]}, _import_and_infer(second_dir, {"keys": ["gamma", "beta"]}))
def testRename(self): file_path = os.path.join(self._base_dir, "temp_file") file_io.write_string_to_file(file_path, "testing") rename_path = os.path.join(self._base_dir, "rename_file") file_io.rename(file_path, rename_path) self.assertTrue(file_io.file_exists(rename_path)) self.assertFalse(file_io.file_exists(file_path))
def testRenameOverwrite(self): file_path = os.path.join(self.get_temp_dir(), "temp_file") file_io.write_string_to_file(file_path, "testing") rename_path = os.path.join(self.get_temp_dir(), "rename_file") file_io.write_string_to_file(rename_path, "rename") file_io.rename(file_path, rename_path, overwrite=True) self.assertTrue(file_io.file_exists(rename_path)) self.assertFalse(file_io.file_exists(file_path)) file_io.delete_file(rename_path)
def testRenameOverwriteFalse(self): file_path = os.path.join(self._base_dir, "temp_file") file_io.FileIO(file_path, mode="w").write("testing") rename_path = os.path.join(self._base_dir, "rename_file") file_io.FileIO(rename_path, mode="w").write("rename") with self.assertRaises(errors.AlreadyExistsError): file_io.rename(file_path, rename_path, overwrite=False) self.assertTrue(file_io.file_exists(rename_path)) self.assertTrue(file_io.file_exists(file_path))
def test_asset_path_returned(self): root = tracking.AutoTrackable() root.path = tracking.Asset(self._vocab_path) save_dir = os.path.join(self.get_temp_dir(), "saved_model") root.get_asset = def_function.function(lambda: root.path.asset_path) save.save(root, save_dir, signatures=root.get_asset.get_concrete_function()) second_dir = os.path.join(self.get_temp_dir(), "second_dir") file_io.rename(save_dir, second_dir) imported_path = _import_and_infer(second_dir, {})["output_0"] self.assertIn( compat.as_str_any(second_dir), compat.as_str_any(imported_path))
def testRenameOverwriteFalse(self): file_path = os.path.join(self.get_temp_dir(), "temp_file") file_io.write_string_to_file(file_path, "testing") rename_path = os.path.join(self.get_temp_dir(), "rename_file") file_io.write_string_to_file(rename_path, "rename") with self.assertRaises(errors.AlreadyExistsError): file_io.rename(file_path, rename_path, overwrite=False) self.assertTrue(file_io.file_exists(rename_path)) self.assertTrue(file_io.file_exists(file_path)) file_io.delete_file(rename_path) file_io.delete_file(file_path)
def test_asset_path_returned(self): root = tracking.AutoTrackable() root.path = tracking.TrackableAsset(self._vocab_path) save_dir = os.path.join(self.get_temp_dir(), "saved_model") root.get_asset = def_function.function(lambda: root.path.asset_path) save.save(root, save_dir, signatures=root.get_asset.get_concrete_function()) second_dir = os.path.join(self.get_temp_dir(), "second_dir") file_io.rename(save_dir, second_dir) imported_path = _import_and_infer(second_dir, {})["output_0"] self.assertIn(compat.as_str_any(second_dir), compat.as_str_any(imported_path))
def write_model_config(model_dir: str, config_str: str): """Writes the config to the model's directory.""" model_dir = root_dir(model_dir) model_config_path = get_model_config_path(model_dir) # if the model config file already exists, rename it if file_io.file_exists(model_config_path): prev_config_filename = '%s_%d.yaml' % (model_config_path.split('.')[0], int(time())) file_io.rename(model_config_path, prev_config_filename) logging.info('Previous model config file was renamed: %s' % prev_config_filename) # save the config file to the model directory if not is_s3_path(model_dir): os.makedirs(model_dir, exist_ok=True) file_io.write_string_to_file(model_config_path, config_str)
def Import(self, request, context): def get_mtime(filepath): if file_io.file_exists(filepath): return file_io.stat(filepath).mtime_nsec return None def is_new_emb(id, filepath): origin_mtime = get_mtime(self._get_filepath(id)) if origin_mtime is None: return True new_mtime = get_mtime(filepath) return origin_mtime < new_mtime logging.info("Importing..") all_filepaths = list(glob.iglob('%s/*.emb' % request.path)) total_count = len(all_filepaths) if total_count <= 0: logging.info("No files for importing!") return pb2.SimpleReponse(message='No files for importing!') logging.info("Importing files count: %d" % total_count) pos = len(request.path) + 1 def path_to_id(filepath): return int(filepath[pos:-4]) for filepaths in chunks(all_filepaths, 10000): t0 = time.time() ids = map(path_to_id, filepaths) ids_filepaths = [(id, filepath) for id, filepath in zip(ids, filepaths) if is_new_emb(id, filepath)] xb = self._path_to_xb([filepath for _, filepath in ids_filepaths]) ids = np.array([id for id, _ in ids_filepaths], dtype=np.int64) self.faiss_index.replace(xb, ids) for id, filepath in ids_filepaths: file_io.rename(filepath, self._get_filepath(id, mkdir=True), overwrite=True) logging.info("%d embeddings added %.3f s", xb.shape[0], time.time() - t0) return pb2.SimpleReponse(message='Imported, %d!' % total_count)
def test_assets(self, cycles): file1 = self._make_asset("contents 1") file2 = self._make_asset("contents 2") root = tracking.AutoTrackable() root.asset1 = tracking.TrackableAsset(file1) root.asset2 = tracking.TrackableAsset(file2) save_dir = os.path.join(self.get_temp_dir(), "save_dir") save.save(root, save_dir) file_io.delete_file(file1) file_io.delete_file(file2) load_dir = os.path.join(self.get_temp_dir(), "load_dir") file_io.rename(save_dir, load_dir) imported = load.load(load_dir) with open(self.evaluate(imported.asset1.asset_path), "r") as f: self.assertEqual("contents 1", f.read()) with open(self.evaluate(imported.asset2.asset_path), "r") as f: self.assertEqual("contents 2", f.read())
def test_assets(self): file1 = self._make_asset("contents 1") file2 = self._make_asset("contents 2") root = tracking.Checkpointable() root.asset1 = tracking.TrackableAsset(file1) root.asset2 = tracking.TrackableAsset(file2) save_dir = os.path.join(self.get_temp_dir(), "save_dir") save.save(root, save_dir, signatures={}) file_io.delete_file(file1) file_io.delete_file(file2) load_dir = os.path.join(self.get_temp_dir(), "load_dir") file_io.rename(save_dir, load_dir) imported = load.load(load_dir) with open(imported.asset1.asset_path.numpy(), "r") as f: self.assertEquals("contents 1", f.read()) with open(imported.asset2.asset_path.numpy(), "r") as f: self.assertEquals("contents 2", f.read())
def test_assets(self): file1 = self._make_asset("contents 1") file2 = self._make_asset("contents 2") root = tracking.AutoCheckpointable() root.asset1 = tracking.TrackableAsset(file1) root.asset2 = tracking.TrackableAsset(file2) save_dir = os.path.join(self.get_temp_dir(), "save_dir") save.save(root, save_dir, signatures={}) file_io.delete_file(file1) file_io.delete_file(file2) load_dir = os.path.join(self.get_temp_dir(), "load_dir") file_io.rename(save_dir, load_dir) imported = load.load(load_dir) with open(imported.asset1.asset_path.numpy(), "r") as f: self.assertEquals("contents 1", f.read()) with open(imported.asset2.asset_path.numpy(), "r") as f: self.assertEquals("contents 2", f.read())
def test_assets_import(self): file1 = self._make_asset("contents 1") file2 = self._make_asset("contents 2") root = tracking.Checkpointable() root.f = def_function.function( lambda x: 2. * x, input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)]) root.asset1 = tracking.TrackableAsset(file1) root.asset2 = tracking.TrackableAsset(file2) save_dir = os.path.join(self.get_temp_dir(), "save_dir") save.save(root, save_dir) file_io.delete_file(file1) file_io.delete_file(file2) load_dir = os.path.join(self.get_temp_dir(), "load_dir") file_io.rename(save_dir, load_dir) imported = load.load(load_dir) with open(imported.asset1.asset_path.numpy(), "r") as f: self.assertEquals("contents 1", f.read()) with open(imported.asset2.asset_path.numpy(), "r") as f: self.assertEquals("contents 2", f.read())
def create_dir_test(): """Verifies file_io directory handling methods.""" # Test directory creation. starttime_ms = int(round(time.time() * 1000)) dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms) print("Creating dir %s" % dir_name) file_io.create_dir(dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Created directory in: %d milliseconds" % elapsed_ms) # Check that the directory exists. dir_exists = file_io.is_directory(dir_name) assert dir_exists print("%s directory exists: %s" % (dir_name, dir_exists)) # Test recursive directory creation. starttime_ms = int(round(time.time() * 1000)) recursive_dir_name = "%s/%s/%s" % (dir_name, "nested_dir1", "nested_dir2") print("Creating recursive dir %s" % recursive_dir_name) file_io.recursive_create_dir(recursive_dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Created directory recursively in: %d milliseconds" % elapsed_ms) # Check that the directory exists. recursive_dir_exists = file_io.is_directory(recursive_dir_name) assert recursive_dir_exists print("%s directory exists: %s" % (recursive_dir_name, recursive_dir_exists)) # Create some contents in the just created directory and list the contents. num_files = 10 files_to_create = ["file_%d.txt" % n for n in range(num_files)] for file_num in files_to_create: file_name = "%s/%s" % (dir_name, file_num) print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file.") print("Listing directory %s." % dir_name) starttime_ms = int(round(time.time() * 1000)) directory_contents = file_io.list_directory(dir_name) print(directory_contents) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Listed directory %s in %s milliseconds" % (dir_name, elapsed_ms)) assert set(directory_contents) == set(files_to_create + ["nested_dir1/"]) # Test directory renaming. dir_to_rename = "%s/old_dir" % dir_name new_dir_name = "%s/new_dir" % dir_name file_io.create_dir(dir_to_rename) assert file_io.is_directory(dir_to_rename) assert not file_io.is_directory(new_dir_name) starttime_ms = int(round(time.time() * 1000)) print("Will try renaming directory %s to %s" % (dir_to_rename, new_dir_name)) file_io.rename(dir_to_rename, new_dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Renamed directory %s to %s in %s milliseconds" % ( dir_to_rename, new_dir_name, elapsed_ms)) assert not file_io.is_directory(dir_to_rename) assert file_io.is_directory(new_dir_name) # Test Delete directory recursively. print("Deleting directory recursively %s." % dir_name) starttime_ms = int(round(time.time() * 1000)) file_io.delete_recursively(dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms dir_exists = file_io.is_directory(dir_name) assert not dir_exists print("Deleted directory recursively %s in %s milliseconds" % ( dir_name, elapsed_ms))
def rename(cls, oldname, newname, overwrite=False): file_io.rename(oldname, newname, overwrite)
def create_object_test(): """Verifies file_io's object manipulation methods .""" starttime_ms = int(round(time.time() * 1000)) dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms) print("Creating dir %s." % dir_name) file_io.create_dir(dir_name) num_files = 5 # Create files of 2 different patterns in this directory. files_pattern_1 = ["%s/test_file_%d.txt" % (dir_name, n) for n in range(num_files)] files_pattern_2 = ["%s/testfile%d.txt" % (dir_name, n) for n in range(num_files)] starttime_ms = int(round(time.time() * 1000)) files_to_create = files_pattern_1 + files_pattern_2 for file_name in files_to_create: print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file creation.") elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Created %d files in %s milliseconds" % (len(files_to_create), elapsed_ms)) # Listing files of pattern1. list_files_pattern = "%s/test_file*.txt" % dir_name print("Getting files matching pattern %s." % list_files_pattern) starttime_ms = int(round(time.time() * 1000)) files_list = file_io.get_matching_files(list_files_pattern) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Listed files in %s milliseconds" % elapsed_ms) print(files_list) assert set(files_list) == set(files_pattern_1) # Listing files of pattern2. list_files_pattern = "%s/testfile*.txt" % dir_name print("Getting files matching pattern %s." % list_files_pattern) starttime_ms = int(round(time.time() * 1000)) files_list = file_io.get_matching_files(list_files_pattern) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Listed files in %s milliseconds" % elapsed_ms) print(files_list) assert set(files_list) == set(files_pattern_2) # Test renaming file. file_to_rename = "%s/oldname.txt" % dir_name file_new_name = "%s/newname.txt" % dir_name file_io.write_string_to_file(file_to_rename, "test file.") assert file_io.file_exists(file_to_rename) assert not file_io.file_exists(file_new_name) print("Will try renaming file %s to %s" % (file_to_rename, file_new_name)) starttime_ms = int(round(time.time() * 1000)) file_io.rename(file_to_rename, file_new_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("File %s renamed to %s in %s milliseconds" % ( file_to_rename, file_new_name, elapsed_ms)) assert not file_io.file_exists(file_to_rename) assert file_io.file_exists(file_new_name) # Delete directory. print("Deleting directory %s." % dir_name) file_io.delete_recursively(dir_name)