def create_path_manager() -> PathManager: # TODO: move this inline import out after AIRStore OSS public released from airstore.client.airstore_tabular import AIRStorePathHandler pathmanager = PathManager() pathmanager.register_handler(AIRStorePathHandler()) pathmanager.set_strict_kwargs_checking(False) return pathmanager
def test_open_new_path_manager(self) -> None: with self._patch_download(): path_manager = PathManager() with self.assertRaises(OSError): # no handler registered f = path_manager.open(self._remote_uri, "rb") path_manager.register_handler(HTTPURLHandler()) with path_manager.open(self._remote_uri, "rb") as f: self.assertTrue(os.path.isfile(f.name)) self.assertTrue(f.read() != "")
def get_skateboard_data( avoid_manifold: bool = False, silence_logs: bool = False ) -> Generator[Tuple[str, PathManager], None, None]: """ Context manager for accessing Co3D dataset by tests, at least for the first 5 skateboards. Internally, we want this to exercise the normal way to access the data directly manifold, but on an RE worker this is impossible so we use a workaround. Args: avoid_manifold: Use the method used by RE workers even locally. silence_logs: Whether to reduce log output from iopath library. Yields: dataset_root: (str) path to dataset root. path_manager: path_manager to access it with. """ path_manager = PathManager() if silence_logs: logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL) logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL) if not os.environ.get("FB_TEST", False): if os.getenv("FAIR_ENV_CLUSTER", "") == "": raise unittest.SkipTest("Unknown environment. Data not available.") yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", path_manager elif avoid_manifold or os.environ.get("INSIDE_RE_WORKER", False): from libfb.py.parutil import get_file_path par_path = "skateboard_first_5" source = get_file_path(par_path) assert Path(source).is_file() with tempfile.TemporaryDirectory() as dest: with ZipFile(source) as f: f.extractall(dest) yield os.path.join(dest, "extracted"), path_manager else: from iopath.fb.manifold import ManifoldPathHandler path_manager.register_handler(ManifoldPathHandler()) yield "manifold://co3d/tree/extracted", path_manager
class LayoutParserHandler(PathHandler): """ Resolve anything that's in LayoutParser model zoo. """ PREFIX = "lp://" def _get_supported_prefixes(self): return [self.PREFIX] def _get_local_path(self, path, **kwargs): model_name = path[len(self.PREFIX):] dataset_name, *model_name, data_type = model_name.split('/') if data_type == 'weight': model_url = MODEL_CATALOG[dataset_name]['/'.join(model_name)] elif data_type == 'config': model_url = CONFIG_CATALOG[dataset_name]['/'.join(model_name)] else: raise ValueError(f"Unknown data_type {data_type}") return PathManager.get_local_path(model_url, **kwargs) def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs) PathManager = PathManagerBase() PathManager.register_handler(DropboxHandler()) PathManager.register_handler(LayoutParserHandler())
PathManager = PathManagerBase() """ This is a detectron2 project-specific PathManager. We try to stay away from global PathManager in fvcore as it introduces potential conflicts among other libraries. """ class Detectron2Handler(PathHandler): """ Resolve anything that's hosted under detectron2's namespace. """ PREFIX = "detectron2://" S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" def _get_supported_prefixes(self): return [self.PREFIX] def _get_local_path(self, path): name = path[len(self.PREFIX):] return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name) def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs) PathManager.register_handler(HTTPURLHandler()) PathManager.register_handler(OneDrivePathHandler()) PathManager.register_handler(Detectron2Handler())
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from iopath.common.file_io import HTTPURLHandler from iopath.common.file_io import PathManager as PathManagerBase # A trick learned from https://github.com/facebookresearch/detectron2/blob/65faeb4779e4c142484deeece18dc958c5c9ad18/detectron2/utils/file_io.py#L3 class DropboxHandler(HTTPURLHandler): """ Supports download and file check for dropbox links """ def _get_supported_prefixes(self): return ["https://www.dropbox.com"] def _isfile(self, path): return path in self.cache_map PathManager = PathManagerBase() PathManager.register_handler(DropboxHandler())
import mobile_cv.lut.lib.pt.flops_utils as flops_utils import torch from d2go.config import temp_defrost, CfgNode from d2go.export.api import convert_and_export_predictor from d2go.setup import ( basic_argument_parser, prepare_for_launch, setup_after_launch, ) from iopath.common.file_io import PathManager from iopath.fb.manifold import ManifoldPathHandler from mobile_cv.common.misc.py import post_mortem_if_fail path_manager = PathManager() path_manager.register_handler(ManifoldPathHandler()) logger = logging.getLogger("d2go.tools.export") INFERNCE_CONFIG_FILENAME = "inference_config.json" MOBILE_OPTIMIZED_BUNDLE_FILENAME = "mobile_optimized_bundled.ptl" def write_model_with_config(output_dir: str, model_jit_path: str, inference_config: Optional[CfgNode] = None): """ Writes the sdk inference config along with model file and saves the model with configuration at ${output_dir}/mobile_optimized_bundled.ptl """ model_jit_local_path = path_manager.get_local_path(model_jit_path)
def __init__(self): path_manager = PathManager() path_manager.register_handler(HTTPURLHandler()) path_manager.register_handler(GoogleDrivePathHandler()) self.path_manager = path_manager
cache_dir: Optional[str] = None, **kwargs: Any, ) -> str: destination = kwargs["destination"] local_path = self.path_manager.get_local_path(path, force) shutil.move(local_path, destination) return destination def _open(self, path: str, mode: str = "r", buffering: int = -1, **kwargs: Any) -> Union[IO[str], IO[bytes]]: self._check_kwargs(kwargs) assert mode in ("r", "rb"), "{} does not support open with {} mode".format( self.__class__.__name__, mode) assert ( buffering == -1 ), f"{self.__class__.__name__} does not support the `buffering` argument" local_path = self._get_local_path(path, force=False) return open(local_path, mode) _DATASET_DOWNLOAD_MANAGER = PathManager() _DATASET_DOWNLOAD_MANAGER.register_handler(CombinedInternalPathhandler())
""" This is a herbarium project-specific PathManager. We try to stay away from global PathManager in fvcore as it introduces potential conflicts among other libraries. """ class HerbariumHandler(PathHandler): """ Resolve anything that's hosted under herbarium's namespace. """ PREFIX = "herbarium://" S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/herbarium/" def _get_supported_prefixes(self): return [self.PREFIX] def _get_local_path(self, path, **kwargs): name = path[len(self.PREFIX):] return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name, **kwargs) def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs) PathManager.register_handler(HTTPURLHandler()) PathManager.register_handler(OneDrivePathHandler()) PathManager.register_handler(HerbariumHandler())