def test_unpack(self): fsm = FileStructManager(base_dir=self.base_dir, is_continue=False) cm = CheckpointsManager(fsm) f = open(cm.weights_file(), 'w') f.write('1') f.close() f = open(cm.optimizer_state_file(), 'w') f.write('2') f = open(cm.trainer_file(), 'w') f.write('3') f.close() cm.pack() try: cm.unpack() except Exception as err: self.fail('Exception on unpacking') for i, f in enumerate( [cm.weights_file(), cm.optimizer_state_file(), cm.trainer_file()]): if not (os.path.exists(f) and os.path.isfile(f)): self.fail("File '{}' doesn't remove after pack".format(f)) with open(f, 'r') as file: if file.read() != str(i + 1): self.fail("File content corrupted")
def test_pack(self): fsm = FileStructManager(base_dir=self.base_dir, is_continue=False) cm = CheckpointsManager(fsm) with self.assertRaises(CheckpointsManager.SMException): cm.pack() os.mkdir(cm.weights_file()) os.mkdir(cm.optimizer_state_file()) with self.assertRaises(CheckpointsManager.SMException): cm.pack() shutil.rmtree(cm.weights_file()) shutil.rmtree(cm.optimizer_state_file()) f = open(cm.weights_file(), 'w') f.close() f = open(cm.optimizer_state_file(), 'w') f.close() f = open(cm.trainer_file(), 'w') f.close() try: cm.pack() except Exception as err: self.fail('Exception on packing files: [{}]'.format(err)) for f in [cm.weights_file(), cm.optimizer_state_file()]: if os.path.exists(f) and os.path.isfile(f): self.fail("File '{}' doesn't remove after pack".format(f)) result = os.path.join( fsm.get_path(cm, check=False, create_if_non_exists=False), 'last_checkpoint.zip') self.assertTrue(os.path.exists(result) and os.path.isfile(result)) f = open(cm.weights_file(), 'w') f.close() f = open(cm.optimizer_state_file(), 'w') f.close() f = open(cm.trainer_file(), 'w') f.close() try: cm.pack() result = os.path.join( fsm.get_path(cm, check=False, create_if_non_exists=False), 'last_checkpoint.zip.old') self.assertTrue(os.path.exists(result) and os.path.isfile(result)) except Exception as err: self.fail('Fail to pack with existing previous state file')
def _resume(self) -> int: if self._resume_from == 'last': ckpts_manager = self._checkpoint_manager elif self._checkpoint_manager == 'best': ckpts_manager = CheckpointsManager(self._fsm, 'best') else: raise NotImplementedError("Resume parameter may be only 'last' or 'best' not {}".format(self._resume_from)) ckpts_manager.unpack() self._data_processor.load() with open(ckpts_manager.trainer_file(), 'r') as file: start_epoch_idx = json.load(file)['last_epoch'] + 1 ckpts_manager.pack() return start_epoch_idx