def create_backup(self): io.log_info ("Creating backup...", end='\r') if not self.autobackups_path.exists(): self.autobackups_path.mkdir(exist_ok=True) bckp_filename_list = [ self.get_strpath_storage_for_file(filename) for _, filename in self.get_model_filename_list() ] bckp_filename_list += [ str(self.get_summary_path()), str(self.model_data_path) ] # Create new backup session_suffix = f'_{self.session_name}' if self.session_name else '' idx_str = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') + session_suffix idx_backup_path = self.autobackups_path / idx_str idx_backup_path.mkdir() for filename in bckp_filename_list: shutil.copy(str(filename), str(idx_backup_path / Path(filename).name))\ previews = self.get_previews() # Generate previews and save in new backup plist = [] for i in range(len(previews)): name, bgr = previews[i] plist += [ (bgr, idx_backup_path / ( ('preview_%s.jpg') % (name)) ) ] if len(plist) != 0: self.get_preview_history_writer().post(plist, self.loss_history, self.iter) # Check if we've exceeded the max number of backups if self.maximum_n_backups != 0: all_backups = sorted([x for x in self.autobackups_path.iterdir() if x.is_dir()]) while len(all_backups) > self.maximum_n_backups: oldest_backup = all_backups.pop(0) pathex.delete_all_files(oldest_backup) oldest_backup.rmdir()
def create_backup(self): io.log_info("Creating backup...", end='\r') if not self.autobackups_path.exists(): self.autobackups_path.mkdir(exist_ok=True) bckp_filename_list = [ self.get_strpath_storage_for_file(filename) for _, filename in self.get_model_filename_list() ] bckp_filename_list += [ str(self.get_summary_path()), str(self.model_data_path) ] for i in range(24, 0, -1): idx_str = '%.2d' % i next_idx_str = '%.2d' % (i + 1) idx_backup_path = self.autobackups_path / idx_str next_idx_packup_path = self.autobackups_path / next_idx_str if idx_backup_path.exists(): if i == 24: pathex.delete_all_files(idx_backup_path) else: next_idx_packup_path.mkdir(exist_ok=True) pathex.move_all_files(idx_backup_path, next_idx_packup_path) if i == 1: idx_backup_path.mkdir(exist_ok=True) for filename in bckp_filename_list: shutil.copy(str(filename), str(idx_backup_path / Path(filename).name)) previews = self.get_previews() plist = [] for i in range(len(previews)): name, bgr = previews[i] plist += [ (bgr, idx_backup_path / (('preview_%s.jpg') % (name))) ] for preview, filepath in plist: preview_lh = ModelBase.get_loss_history_preview( self.loss_history, self.iter, preview.shape[1], preview.shape[2]) img = (np.concatenate([preview_lh, preview], axis=0) * 255).astype(np.uint8) cv2_imwrite(filepath, img)
def create_backup(self): io.log_info("Creating backup...", end='\r') if not self.autobackups_path.exists(): self.autobackups_path.mkdir(exist_ok=True) bckp_filename_list = [ self.get_strpath_storage_for_file(filename) for _, filename in self.get_model_filename_list() ] bckp_filename_list += [ str(self.get_summary_path()), str(self.model_data_path) ] for i in range(24, 0, -1): idx_str = '%.2d' % i next_idx_str = '%.2d' % (i + 1) idx_backup_path = self.autobackups_path / idx_str next_idx_packup_path = self.autobackups_path / next_idx_str if idx_backup_path.exists(): if i == 24: pathex.delete_all_files(idx_backup_path) else: next_idx_packup_path.mkdir(exist_ok=True) pathex.move_all_files(idx_backup_path, next_idx_packup_path) if i == 1: idx_backup_path.mkdir(exist_ok=True) for filename in bckp_filename_list: shutil.copy(str(filename), str(idx_backup_path / Path(filename).name)) previews = self.get_previews() plist = [] for i in range(len(previews)): name, bgr = previews[i] plist += [ (bgr, idx_backup_path / (('preview_%s.jpg') % (name))) ] if len(plist) != 0: self.get_preview_history_writer().post( plist, self.loss_history, self.iter)
def save(self): summary_path = self.get_strpath_storage_for_file('summary.txt') Path(summary_path).write_text(self.get_summary_text()) self.onSave() model_data = { 'iter': self.iter, 'options': self.options, 'loss_history': self.loss_history, 'sample_for_preview': self.sample_for_preview, 'choosed_gpu_indexes': self.choosed_gpu_indexes, } pathex.write_bytes_safe(self.model_data_path, pickle.dumps(model_data)) if self.autobackup: bckp_filename_list = [ self.get_strpath_storage_for_file(filename) for _, filename in self.get_model_filename_list() ] bckp_filename_list += [ str(summary_path), str(self.model_data_path) ] current_hour = time.localtime().tm_hour if self.autobackup_current_hour != current_hour: self.autobackup_current_hour = current_hour for i in range(15, 0, -1): idx_str = '%.2d' % i next_idx_str = '%.2d' % (i + 1) idx_backup_path = self.autobackups_path / idx_str next_idx_packup_path = self.autobackups_path / next_idx_str if idx_backup_path.exists(): if i == 15: pathex.delete_all_files(idx_backup_path) else: next_idx_packup_path.mkdir(exist_ok=True) pathex.move_all_files(idx_backup_path, next_idx_packup_path) if i == 1: idx_backup_path.mkdir(exist_ok=True) for filename in bckp_filename_list: shutil.copy( str(filename), str(idx_backup_path / Path(filename).name)) previews = self.get_previews() plist = [] for i in range(len(previews)): name, bgr = previews[i] plist += [(bgr, idx_backup_path / (('preview_%s.jpg') % (name)))] for preview, filepath in plist: preview_lh = ModelBase.get_loss_history_preview( self.loss_history, self.iter, preview.shape[1], preview.shape[2]) img = ( np.concatenate([preview_lh, preview], axis=0) * 255).astype(np.uint8) cv2_imwrite(filepath, img)