def test_download(self) -> None: download( "https://www.facebook.com", ".", filename=self._filename, progress=False, ) self.assertTrue(os.path.isfile(self._filename))
def _get_local_path(self, path: str, **kwargs: Any) -> str: """ This implementation downloads the remote resource and caches it locally. The resource will only be downloaded if not previously requested. """ self._check_kwargs(kwargs) if path not in self.cache_map or not os.path.exists( self.cache_map[path] ): logger = logging.getLogger(__name__) parsed_url = urlparse(path) dirname = os.path.join( get_cache_dir(), os.path.dirname(parsed_url.path.lstrip("/")) ) filename = path.split("/")[-1] cached = os.path.join(dirname, filename) print() print("Path in PM:") print(path) print() print("dirname in PM") print(dirname) dirname = "/content" with file_lock(cached): if not os.path.isfile(cached): logger.info("Downloading {} ...".format(path)) cached = download(path, dirname, filename=filename) logger.info("URL {} cached in {}".format(path, cached)) self.cache_map[path] = cached return self.cache_map[path]
def prepare_panoptic_coco(): dataset_dir = os.path.join(os.getenv("DETECTRON2_DATASETS", "datasets"), "coco") for s in ["val2017", "train2017"]: separate_coco_semantic_from_panoptic( os.path.join(dataset_dir, "annotations/panoptic_{}.json".format(s)), os.path.join(dataset_dir, "panoptic_{}".format(s)), os.path.join(dataset_dir, "panoptic_stuff_{}".format(s)), COCO_CATEGORIES, ) # Prepare val2017_100 for quick testing: dest_dir = os.path.join(dataset_dir, "annotations/") URL_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" download(URL_PREFIX + "annotations/coco/panoptic_val2017_100.json", dest_dir) with open(os.path.join(dest_dir, "panoptic_val2017_100.json")) as f: obj = json.load(f) def link_val100(dir_full, dir_100): print("Creating " + dir_100 + " ...") os.makedirs(dir_100, exist_ok=True) for img in obj["images"]: basename = os.path.splitext(img["file_name"])[0] src = os.path.join(dir_full, basename + ".png") dst = os.path.join(dir_100, basename + ".png") src = os.path.relpath(src, start=dir_100) os.symlink(src, dst) link_val100( os.path.join(dataset_dir, "panoptic_val2017"), os.path.join(dataset_dir, "panoptic_val2017_100"), ) link_val100( os.path.join(dataset_dir, "panoptic_stuff_val2017"), os.path.join(dataset_dir, "panoptic_stuff_val2017_100"), )
def cache_url(url: str, cache_dir: str) -> str: """ This implementation downloads the remote resource and caches it locally. The resource will only be downloaded if not previously requested. """ parsed_url = urlparse(url) dirname = os.path.join(cache_dir, os.path.dirname(parsed_url.path.lstrip("/"))) makedir(dirname) filename = url.split("/")[-1] cached = os.path.join(dirname, filename) with file_lock(cached): if not os.path.isfile(cached): logging.info(f"Downloading {url} to {cached} ...") cached = download(url, dirname, filename=filename) logging.info(f"URL {url} cached in {cached}") return cached
def get(config_path: str, pretrained: bool = False): r""" Get a model specified by relative path under Detectron2's official ``configs/`` directory. Args: config_path: Name of config file relative to ``configs/`` directory under project root. (E.g. ``width_ablations/bicaptioning_R_50_L1_H2048.yaml``) pretrained: If ``True``, will initialize the model with the pretrained weights. If ``False``, the weights will be initialized randomly. """ # Get the original path to config file (shipped with inside the package). _pkg_config_path = pkg_resources.resource_filename( "virtex.model_zoo", os.path.join("configs", config_path) ) if not os.path.exists(_pkg_config_path): raise RuntimeError("{} not available in Model Zoo!".format(config_path)) _C = Config(_pkg_config_path) model = PretrainingModelFactory.from_config(_C) if pretrained: # Get URL for the checkpoint for this config path. if config_path in _ModelZooUrls.CONFIG_PATH_TO_DB_ID: dropbox_id = _ModelZooUrls.CONFIG_PATH_TO_DB_ID[config_path] filename = os.path.basename(config_path).replace(".yaml", ".pth") checkpoint_url = f"{_ModelZooUrls.URL_PREFIX}/{dropbox_id}/{filename}?dl=1" else: raise RuntimeError("{} not available in Model Zoo!".format(config_path)) # Download the pretrained model weights and save with a sensible name. # This will be downloaded only if it does not exist. checkpoint_path = download( checkpoint_url, dir=os.path.expanduser("~/.torch/virtex_cache"), filename=os.path.basename(config_path).replace(".yaml", ".pth") ) CheckpointManager(model=model).load(checkpoint_path) return model
if __name__ == "__main__": dataset_dir = os.path.join(os.path.dirname(__file__), "coco") for s in ["val2017", "train2017"]: separate_coco_semantic_from_panoptic( os.path.join(dataset_dir, "annotations/panoptic_{}.json".format(s)), os.path.join(dataset_dir, "panoptic_{}".format(s)), os.path.join(dataset_dir, "panoptic_stuff_{}".format(s)), COCO_CATEGORIES, ) # Prepare val2017_100 for quick testing: dest_dir = os.path.join(dataset_dir, "annotations/") URL_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" download(URL_PREFIX + "annotations/coco/panoptic_val2017_100.json", dest_dir) with open(os.path.join(dest_dir, "panoptic_val2017_100.json")) as f: obj = json.load(f) def link_val100(dir_full, dir_100): print("Creating " + dir_100 + " ...") os.makedirs(dir_100, exist_ok=True) for img in obj["images"]: basename = os.path.splitext(img["file_name"])[0] src = os.path.join(dir_full, basename + ".png") dst = os.path.join(dir_100, basename + ".png") src = os.path.relpath(src, start=dir_100) os.symlink(src, dst) link_val100( os.path.join(dataset_dir, "panoptic_val2017"),