def step(self, iteration: int, **kwargs: Any) -> None: """ Perform the appropriate action at the given iteration. Args: iteration (int): the current iteration, ranged in [0, max_iter-1]. kwargs (Any): extra data to save, same as in :meth:`Checkpointer.save`. """ iteration = int(iteration) additional_state = {"iteration": iteration} additional_state.update(kwargs) if (iteration + 1) % self.period == 0: self.checkpointer.save("model_{:07d}".format(iteration), **additional_state) if self.max_to_keep is not None: self.recent_checkpoints.append(self.checkpointer.get_checkpoint_file()) # pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`. # pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`. if len(self.recent_checkpoints) > self.max_to_keep: file_to_delete = self.recent_checkpoints.pop(0) if PathManager.exists( file_to_delete ) and not file_to_delete.endswith("model_final.pth"): PathManager.rm(file_to_delete) if iteration >= self.max_iter - 1: # pyre-ignore self.checkpointer.save("model_final", **additional_state)
def test_bad_args(self) -> None: with self.assertRaises(NotImplementedError): PathManager.copy( self._remote_uri, self._remote_uri, foo="foo" # type: ignore ) with self.assertRaises(NotImplementedError): PathManager.exists(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.get_local_path( self._remote_uri, foo="foo" # type: ignore ) with self.assertRaises(NotImplementedError): PathManager.isdir(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.isfile(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.ls(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.mkdirs(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.open(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.rm(self._remote_uri, foo="foo") # type: ignore PathManager.set_strict_kwargs_checking(False) PathManager.get_local_path(self._remote_uri, foo="foo") # type: ignore f = PathManager.open(self._remote_uri, foo="foo") # type: ignore f.close() PathManager.set_strict_kwargs_checking(True)
def step(self, iteration: int, **kwargs: Any): """ Perform the appropriate action at the given iteration. Args: iteration (int): the current iteration, ranged in [0, max_iter-1]. kwargs (Any): extra data to save, same as in :meth:`Checkpointer.save`. """ iteration = int(iteration) additional_state = {"iteration": iteration} additional_state.update(kwargs) if (iteration + 1) % self.period == 0: self.checkpointer.save("model_{:07d}".format(iteration), **additional_state) if self.max_to_keep is not None: all_checkpoint_files = ( self.checkpointer.get_all_checkpoint_files()) all_checkpoint_files = [ item for item in all_checkpoint_files if not item.endswith("model_final.pth") ] all_checkpoint_files.sort() files_to_delete = all_checkpoint_files[:-self.max_to_keep] for file in files_to_delete: if PathManager.exists(file): PathManager.rm(file) if iteration >= self.max_iter - 1: self.checkpointer.save("model_final", **additional_state)
def test_rm(self): with open(os.path.join(self._tmpdir, "test_rm.txt"), "w") as f: rm_file = f.name f.write(self._tmpfile_contents) f.flush() self.assertTrue(PathManager.exists(rm_file)) self.assertTrue(PathManager.isfile(rm_file)) PathManager.rm(rm_file) self.assertFalse(PathManager.exists(rm_file)) self.assertFalse(PathManager.isfile(rm_file))
def clean_up_checkpoints(checkpoint_dir, n_keep=1): checkpoints = glob(os.path.join(checkpoint_dir, "model_*.pth")) checkpoints.sort() checkpoints_to_delete = checkpoints[:-n_keep] for checkpoint in checkpoints_to_delete: try: PathManager.rm(checkpoint) except FileNotFoundError: pass
def create_file_symlink(file1, file2): """ Simply create the symlinks for a given file1 to file2. Useful during model checkpointing to symlinks to the latest successful checkpoint. """ try: if PathManager.exists(file2): PathManager.rm(file2) PathManager.symlink(file1, file2) except Exception as e: logging.info(f"Could NOT create symlink. Error: {e}")
def test_bad_args(self) -> None: # TODO (T58240718): Replace with dynamic checks with self.assertRaises(ValueError): PathManager.copy( self._tmpfile, self._tmpfile, foo="foo" # type: ignore ) with self.assertRaises(ValueError): PathManager.exists(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.get_local_path(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.isdir(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.isfile(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.ls(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.mkdirs(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.open(self._tmpfile, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.rm(self._tmpfile, foo="foo") # type: ignore PathManager.set_strict_kwargs_checking(False) PathManager.copy( self._tmpfile, self._tmpfile, foo="foo" # type: ignore ) PathManager.exists(self._tmpfile, foo="foo") # type: ignore PathManager.get_local_path(self._tmpfile, foo="foo") # type: ignore PathManager.isdir(self._tmpfile, foo="foo") # type: ignore PathManager.isfile(self._tmpfile, foo="foo") # type: ignore PathManager.ls(self._tmpdir, foo="foo") # type: ignore PathManager.mkdirs(self._tmpdir, foo="foo") # type: ignore f = PathManager.open(self._tmpfile, foo="foo") # type: ignore f.close() # pyre-ignore with open(os.path.join(self._tmpdir, "test_rm.txt"), "w") as f: rm_file = f.name f.write(self._tmpfile_contents) f.flush() PathManager.rm(rm_file, foo="foo") # type: ignore
def rm(path: str) -> None: if FVCorePathManager: return FVCorePathManager.rm(path) os.remove(path)
# Populate the img_paths and img_labels based on torchvision image folder file structure. for label in label_paths: label_path = os.path.join(split_path, label) images = PathManager.ls(os.path.join(split_path, label)) print(f"{len(images)} examples found for { label }, { split }.") total_split_examples += len(images) for image in images: img_path = os.path.join(label_path, image) img_paths.append(img_path) img_labels.append(label) dataset_summary[split]["num_examples"] = total_split_examples print(f"{ total_split_examples } found for { split } split \n") # Remove the split .npy filelist if they exist and resave them.. image_path = os.path.join(args.output, f"{split}_images.npy") PathManager.rm(image_path) save_file(img_paths, image_path) print(f"Saved { image_path }") label_path = os.path.join(args.output, f"{split}_labels.npy") PathManager.rm(label_path) save_file(img_labels, label_path) print(f"Saved { label_path }") # Save dataset summary. dataset_summary_path = os.path.join(args.output, "dataset_summary.json") PathManager.rm(dataset_summary_path) save_file(dataset_summary, dataset_summary_path)
def save_filelist_data(input_data, output_filepath): # Remove the split .npy filelist if they exist and resave them. if PathManager.exists(output_filepath): PathManager.rm(output_filepath) save_file(input_data, output_filepath)