def create_backup(self):
io.log_info ("Creating backup...", end='\r')
if not self.autobackups_path.exists():
self.autobackups_path.mkdir(exist_ok=True)
bckp_filename_list = [ self.get_strpath_storage_for_file(filename) for _, filename in self.get_model_filename_list() ]
bckp_filename_list += [ str(self.get_summary_path()), str(self.model_data_path) ]
# Create new backup
session_suffix = f'_{self.session_name}' if self.session_name else ''
idx_str = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') + session_suffix
idx_backup_path = self.autobackups_path / idx_str
idx_backup_path.mkdir()
for filename in bckp_filename_list:
shutil.copy(str(filename), str(idx_backup_path / Path(filename).name))\
previews = self.get_previews()
# Generate previews and save in new backup
plist = []
for i in range(len(previews)):
name, bgr = previews[i]
plist += [ (bgr, idx_backup_path / ( ('preview_%s.jpg') % (name)) ) ]
if len(plist) != 0:
self.get_preview_history_writer().post(plist, self.loss_history, self.iter)
# Check if we've exceeded the max number of backups
if self.maximum_n_backups != 0:
all_backups = sorted([x for x in self.autobackups_path.iterdir() if x.is_dir()])
while len(all_backups) > self.maximum_n_backups:
oldest_backup = all_backups.pop(0)
pathex.delete_all_files(oldest_backup)
oldest_backup.rmdir()
def create_backup(self):
io.log_info("Creating backup...", end='\r')
if not self.autobackups_path.exists():
self.autobackups_path.mkdir(exist_ok=True)
bckp_filename_list = [
self.get_strpath_storage_for_file(filename)
for _, filename in self.get_model_filename_list()
]
bckp_filename_list += [
str(self.get_summary_path()),
str(self.model_data_path)
]
for i in range(24, 0, -1):
idx_str = '%.2d' % i
next_idx_str = '%.2d' % (i + 1)
idx_backup_path = self.autobackups_path / idx_str
next_idx_packup_path = self.autobackups_path / next_idx_str
if idx_backup_path.exists():
if i == 24:
pathex.delete_all_files(idx_backup_path)
else:
next_idx_packup_path.mkdir(exist_ok=True)
pathex.move_all_files(idx_backup_path,
next_idx_packup_path)
if i == 1:
idx_backup_path.mkdir(exist_ok=True)
for filename in bckp_filename_list:
shutil.copy(str(filename),
str(idx_backup_path / Path(filename).name))
previews = self.get_previews()
plist = []
for i in range(len(previews)):
name, bgr = previews[i]
plist += [
(bgr, idx_backup_path / (('preview_%s.jpg') % (name)))
]
for preview, filepath in plist:
preview_lh = ModelBase.get_loss_history_preview(
self.loss_history, self.iter, preview.shape[1],
preview.shape[2])
img = (np.concatenate([preview_lh, preview], axis=0) *
255).astype(np.uint8)
cv2_imwrite(filepath, img)
def create_backup(self):
io.log_info("Creating backup...", end='\r')
if not self.autobackups_path.exists():
self.autobackups_path.mkdir(exist_ok=True)
bckp_filename_list = [
self.get_strpath_storage_for_file(filename)
for _, filename in self.get_model_filename_list()
]
bckp_filename_list += [
str(self.get_summary_path()),
str(self.model_data_path)
]
for i in range(24, 0, -1):
idx_str = '%.2d' % i
next_idx_str = '%.2d' % (i + 1)
idx_backup_path = self.autobackups_path / idx_str
next_idx_packup_path = self.autobackups_path / next_idx_str
if idx_backup_path.exists():
if i == 24:
pathex.delete_all_files(idx_backup_path)
else:
next_idx_packup_path.mkdir(exist_ok=True)
pathex.move_all_files(idx_backup_path,
next_idx_packup_path)
if i == 1:
idx_backup_path.mkdir(exist_ok=True)
for filename in bckp_filename_list:
shutil.copy(str(filename),
str(idx_backup_path / Path(filename).name))
previews = self.get_previews()
plist = []
for i in range(len(previews)):
name, bgr = previews[i]
plist += [
(bgr, idx_backup_path / (('preview_%s.jpg') % (name)))
]
if len(plist) != 0:
self.get_preview_history_writer().post(
plist, self.loss_history, self.iter)
def save(self):
summary_path = self.get_strpath_storage_for_file('summary.txt')
Path(summary_path).write_text(self.get_summary_text())
self.onSave()
model_data = {
'iter': self.iter,
'options': self.options,
'loss_history': self.loss_history,
'sample_for_preview': self.sample_for_preview,
'choosed_gpu_indexes': self.choosed_gpu_indexes,
}
pathex.write_bytes_safe(self.model_data_path, pickle.dumps(model_data))
if self.autobackup:
bckp_filename_list = [
self.get_strpath_storage_for_file(filename)
for _, filename in self.get_model_filename_list()
]
bckp_filename_list += [
str(summary_path),
str(self.model_data_path)
]
current_hour = time.localtime().tm_hour
if self.autobackup_current_hour != current_hour:
self.autobackup_current_hour = current_hour
for i in range(15, 0, -1):
idx_str = '%.2d' % i
next_idx_str = '%.2d' % (i + 1)
idx_backup_path = self.autobackups_path / idx_str
next_idx_packup_path = self.autobackups_path / next_idx_str
if idx_backup_path.exists():
if i == 15:
pathex.delete_all_files(idx_backup_path)
else:
next_idx_packup_path.mkdir(exist_ok=True)
pathex.move_all_files(idx_backup_path,
next_idx_packup_path)
if i == 1:
idx_backup_path.mkdir(exist_ok=True)
for filename in bckp_filename_list:
shutil.copy(
str(filename),
str(idx_backup_path / Path(filename).name))
previews = self.get_previews()
plist = []
for i in range(len(previews)):
name, bgr = previews[i]
plist += [(bgr, idx_backup_path /
(('preview_%s.jpg') % (name)))]
for preview, filepath in plist:
preview_lh = ModelBase.get_loss_history_preview(
self.loss_history, self.iter, preview.shape[1],
preview.shape[2])
img = (
np.concatenate([preview_lh, preview], axis=0) *
255).astype(np.uint8)
cv2_imwrite(filepath, img)
