def _select_run_subfolder(
project_dir: str,
run_dir: str,
run_name: Optional[str] = None,
allowed_content_for_run: Optional[List[str]] = None) -> str:
allowed_content_for_run = allowed_content_for_run or []
path_to_runs = os.path.join(project_dir, run_dir)
if run_name:
run_folder_full_path = os.path.join(path_to_runs, run_name)
if not os.path.exists(run_folder_full_path):
return run_name
not_allowed_subdirs = _get_not_allowed_content(
run_folder_full_path, allowed_content_for_run)
if not_allowed_subdirs:
msg = ("Folder {} has not allowed content {}. Please remove it "
"or select other run").format(run_folder_full_path,
not_allowed_subdirs)
raise ValueError(msg)
return run_name
all_runs = file_utils.get_existing_fnames_with_index(
os.path.join(path_to_runs, _RUN_DIR_IDENTIFIER), project_dir)
if not all_runs:
run_folder = '-'.join([_RUN_DIR_IDENTIFIER, "1"])
return run_folder
run_folder_full_path = all_runs[-1]
if _get_not_allowed_content(run_folder_full_path, allowed_content_for_run):
run_folder_full_path = file_utils.get_incremented_path(
os.path.join(project_dir, run_folder_full_path))
run_folder = os.path.relpath(run_folder_full_path, path_to_runs)
return run_folder
def test_get_incremented_path(self):
self.assertEqual(
"file/one.ext",
file_utils.get_incremented_path("file/one.ext", add_index=False))
self.assertEqual(
"file/one-1.ext",
file_utils.get_incremented_path("file/one.ext", add_index=True))
self.assertEqual(
"file/one-1.ext",
file_utils.get_incremented_path("file/one-1.ext", add_index=False))
self.assertEqual(
"file/one-1-1.ext",
file_utils.get_incremented_path("file/one-1.ext", add_index=True))
_, file_name = tempfile.mkstemp(suffix='.ext')
file_name_incremented1 = file_utils.add_suffix(file_name, '-1')
file_name_incremented1_1 = file_utils.add_suffix(file_name, '-1-1')
file_name_incremented2 = file_utils.add_suffix(file_name, '-2')
file_name_incremented1000 = file_utils.add_suffix(file_name, '-1000')
file_name_incremented1001 = file_utils.add_suffix(file_name, '-1001')
self.assertEqual(
file_name_incremented1,
file_utils.get_incremented_path(file_name, add_index=True))
self.assertEqual(
file_name_incremented1,
file_utils.get_incremented_path(file_name, add_index=False))
with open(file_name_incremented1, 'w') as f:
f.write('')
self.assertEqual(
file_name_incremented1_1,
file_utils.get_incremented_path(file_name_incremented1,
add_index=True))
self.assertEqual(
file_name_incremented2,
file_utils.get_incremented_path(file_name_incremented1,
add_index=False))
with open(file_name_incremented1000, 'w') as f:
f.write('')
self.assertEqual(
file_name_incremented1001,
file_utils.get_incremented_path(file_name_incremented1000,
add_index=False))
def test_get_incremented_path_with_existing_index(self):
_, file_name = tempfile.mkstemp(suffix='-1.ext')
file_name_incremented1 = file_utils.add_suffix(file_name, '-1')
file_name_incremented2 = file_utils.add_suffix(file_name, '-2')
file_name_incremented1000 = file_utils.add_suffix(file_name, '-1000')
file_name_incremented1001 = file_utils.add_suffix(file_name, '-1001')
with open(file_name_incremented1, 'w') as f:
f.write('')
file_name = file_name_incremented1
self.assertEqual(
file_name_incremented2,
file_utils.get_incremented_path(file_name, add_index=False))
self.assertEqual(
file_name_incremented2,
file_utils.get_incremented_path(file_name_incremented1,
add_index=False))
with open(file_name_incremented1000, 'w') as f:
f.write('')
self.assertEqual(
file_name_incremented1001,
file_utils.get_incremented_path(file_name_incremented1000,
add_index=False))
def create_new_tfrecords_writer(self):
"""
Creates new file_writer and closes old one
Also if needed, increments the file name
"""
logger = logging.getLogger(__name__)
if self._file_locker is None:
self._create_file_locker()
self._file_locker.lock_or_wait()
self._maybe_close_tfrecords_writer()
writer_file_name = file_utils.get_incremented_path(
self.save_name, True)
if self.compression_type:
options = tf.python_io.TFRecordOptions(
getattr(tf.python_io.TFRecordCompressionType,
self.compression_type))
else:
options = []
logger.info("Create new tfrecords writer for file %s",
writer_file_name)
self._tfrecords_writer = tf.python_io.TFRecordWriter(
writer_file_name, options=options)
self._file_locker.release()
def _get_save_file_name(self) -> str:
file_name = "{}.json".format(self.name)
file_name = os.path.join(self.save_dir, file_name)
file_name = file_utils.get_incremented_path(file_name)
return file_name