def __init__(
self,
model: nn.Module,
save_dir: str = "",
*,
save_to_disk: bool = True,
**checkpointables: object,
) -> None:
"""
Args:
model (nn.Module): model.
save_dir (str): a directory to save and find checkpoints.
save_to_disk (bool): if True, save checkpoint to disk, otherwise
disable saving for this checkpointer.
checkpointables (object): any checkpointable objects, i.e., objects
that have the `state_dict()` and `load_state_dict()` method. For
example, it can be used like
`Checkpointer(model, "dir", optimizer=optimizer)`.
"""
if isinstance(model, (DistributedDataParallel, DataParallel)):
model = model.module
self.model = model
self.checkpointables = copy.copy(checkpointables) # pyre-ignore
self.logger = logging.getLogger(__name__) # pyre-ignore
self.save_dir = save_dir
self.save_to_disk = save_to_disk
# Default PathManager, support HTTP URLs (for backward compatibility in open source).
# A user may want to use a different project-specific PathManager
self.path_manager: PathManager = PathManager()
self.path_manager.register_handler(HTTPURLHandler())
def setUpClass(cls) -> None:
# Use a unique pid based cache directory name.
pid = os.getpid()
cls._cache_dir: str = os.path.join(get_cache_dir(), f"{__name__}_{pid}")
cls._pathmgr.register_handler(HTTPURLHandler())
if os.path.exists(cls._cache_dir):
shutil.rmtree(cls._cache_dir)
os.makedirs(cls._cache_dir, exist_ok=True)
def test_open_new_path_manager(self) -> None:
with self._patch_download():
path_manager = PathManager()
with self.assertRaises(OSError): # no handler registered
f = path_manager.open(self._remote_uri, "rb")
path_manager.register_handler(HTTPURLHandler())
with path_manager.open(self._remote_uri, "rb") as f:
self.assertTrue(os.path.isfile(f.name))
self.assertTrue(f.read() != "")
handler. By default, checking is enabled since it is innately unsafe:
multiple `PathHandler`s could reuse arguments with different semantic
meanings or types.
Args:
enable (bool)
"""
self._native_path_handler._strict_kwargs_check = enable
for handler in self._path_handlers.values():
handler._strict_kwargs_check = enable
PathManager = PathManagerBase()
"""
A global PathManager.
Any sufficiently complicated/important project should create their own
PathManager instead of using the global PathManager, to avoid conflicts
when multiple projects have conflicting PathHandlers.
History: at first, PathManager is part of detectron2 *only*, and therefore
does not consider cross-projects conflict issues. It is later used by more
projects and moved to fvcore to faciliate more use across projects and lead
to some conflicts.
Now the class `PathManagerBase` is added to help create per-project path manager,
and this global is still named "PathManager" to keep backward compatibility.
"""
PathManager.register_handler(HTTPURLHandler())
PathManager.register_handler(OneDrivePathHandler())
def register_http_url_handler():
"""
support reading file from url starting with "http://", "https://", "ftp://"
"""
PathManager.register_handler(HTTPURLHandler(), allow_override=True)
def setUp(self) -> None:
self._pathmgr.register_handler(HTTPURLHandler())
if os.path.exists(self._cache_dir):
shutil.rmtree(self._cache_dir)
os.makedirs(self._cache_dir, exist_ok=True)
def __init__(self):
path_manager = PathManager()
path_manager.register_handler(HTTPURLHandler())
path_manager.register_handler(GoogleDrivePathHandler())
self.path_manager = path_manager
