def from_pretrained(self): checkpoint_path = module_path + "/{}.pt".format(self.model_name) if not os.path.exists(checkpoint_path): checkpoint_url = f'https://drive.google.com/uc?id={model_urls[self.model_name]}' gdown.cached_download(checkpoint_url, checkpoint_path) self.model.load_state_dict(torch.load(checkpoint_path, map_location=self.device)) return self
def setup_repository(): # Downloading, extracting models. models_url = 'https://drive.google.com/uc?id=1QJZWF9CzgOiYzjzsRSu2LOkrzi2S6j_U' models_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'resources', 'models') os.makedirs(models_path, exist_ok=True) md5 = '8920cc50fee3505e958307fa11088c0d' models_archive_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'models.zip') gdown.cached_download(url=models_url, path=models_archive_path, md5=md5) gdown.extractall(path=models_archive_path, to=models_path) os.remove(models_archive_path) # Setting up the data folder with runtime_config.ini file data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'resources', 'data') os.makedirs(data_path, exist_ok=True) runtime_config_path = os.path.join(data_path, 'runtime_config.ini') if os.path.exists(runtime_config_path): os.remove(runtime_config_path) pfile = open(runtime_config_path, 'w') pfile.write("[Predictions]\n") pfile.write("non_overlapping=true\n") pfile.write( "reconstruction_method=probabilities #probabilities, thresholding\n") pfile.write( "reconstruction_order=resample_first #resample_first, resample_second\n" ) pfile.write("probability_threshold=0.4\n") pfile.close()
def download_and_load_model(model_files) -> RecursiveScriptModule: """ Downloads and torch.jit.load the model from google drive, the downloaded model is saved in /tmp since in heroku we get /tmp to save all our stuff, if the app is not running in production the model must be saved in load storage, hence the model is directly loaded Args: model_files: the dict containing the model information Returns: (RecursiveScriptModule): the loaded torch.jit model """ if "PRODUCTION" in os.environ: logger.info( f"=> Downloading Model {model_files['model_file']} from {model_files['model_url']}" ) # heroku gives you `/tmp` to store files, which can be cached model_path: Path = Path("/tmp") / f"{model_files['model_file']}.pt" if not model_path.exists(): gdown.cached_download(url=model_files["model_url"], path=model_path) logger.info(f"=> Loading {model_files['model_file']} from download_cache") model: RecursiveScriptModule = torch.jit.load(str(model_path)) else: logger.info(f"=> Loading {model_files['model_file']} from Local") model = torch.jit.load( str((Path("models") / (model_files["model_file"] + ".pt"))) ) return model
def creator(path): train_path = gdown.cached_download(train_url) dev_path = gdown.cached_download(dev_url) test_path = gdown.cached_download(test_url) dataset = {} for split in ("train", "dev", "test"): data_path = { "train": train_path, "dev": dev_path, "test": test_path }[split] with io.open(data_path, "rt", encoding="utf-8") as f: data = [json.loads(line) for line in f.readlines()] temp = [] for x in data: answer_key = x["answerKey"] if split != "test" else "" options = { choice["label"]: choice["text"] for choice in x["question"]["choices"] } stem = x["question"]["stem"] temp.append({ "id": x["id"], "answer_key": answer_key, "options": options, "stem": stem }) dataset[split] = temp with io.open(path, "wb") as f: pickle.dump(dataset, f) return dataset
def setup_repository(): # Downloading, extracting models. models_url = 'https://drive.google.com/uc?id=1DBIl8JyXEo6YdM9uNyo3vrv5T2WsYSXT' models_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'resources', 'models') os.makedirs(models_path, exist_ok=True) md5 = '434775bebd64910e01f4198eab251666' models_archive_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'models.zip') gdown.cached_download(url=models_url, path=models_archive_path, md5=md5) gdown.extractall(path=models_archive_path, to=models_path) os.remove(models_archive_path) # Setting up the data folder with runtime_config.ini file data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'resources', 'data') os.makedirs(data_path, exist_ok=True) runtime_config_path = os.path.join(data_path, 'runtime_config.ini') if os.path.exists(runtime_config_path): os.remove(runtime_config_path) pfile = open(runtime_config_path, 'w') pfile.write("[Predictions]\n") pfile.write("non_overlapping=true\n") pfile.write( "reconstruction_method=probabilities #probabilities, thresholding\n") pfile.write( "reconstruction_order=resample_first #resample_first, resample_second\n" ) pfile.write("probability_threshold=0.4\n") pfile.close()
def get_bag_file(bag_id): if bag_id == "dynamic.desk": bag_file = gdown.cached_download( url= "https://drive.google.com/uc?id=1vaPyJERDNFY7W8VUBT3JvPNXHpZ6Bqun", # NOQA md5="5bfb6eb7f80773dd2b8a6b16a93823e9", ) elif bag_id == "static.robot": bag_file = gdown.cached_download( url= "https://drive.google.com/uc?id=1DwSQLhrtciKCO566R2TqdPc81QOi7tlN", # NOQA md5="ec155c5eacab3d90e72d7b1286c7aff3", ) elif bag_id == "static.desk": bag_file = gdown.cached_download( url= "https://drive.google.com/uc?id=1mArVTWl2f0Uws_mRoCtDzttV2s6AahZa", # NOQA md5="a70a792577447a414c8ac7fe5f4aa316", ) elif bag_id == "static.desk.topdown": bag_file = gdown.cached_download( url= "https://drive.google.com/uc?id=1UxUg4IozQvQNrCALXkzk23v3fBjpFCqZ", # NOQA md5="3625c11cd130a06557f38b8ff390882e", ) else: raise ValueError(f"Unknown bag_id: {bag_id}") return bag_file
def model_init_app(reset=False): model_path = current_app.config["MODEL_DIRECTORY"] # check if model path exists if not os.path.exists(model_path): # check if parent directory exists parent_dir = os.path.dirname(model_path) if not os.path.exists(parent_dir): os.mkdir(parent_dir) current_app.logger.info(f"Created parent dir {parent_dir}") # download and unzip to correct path url = 'https://drive.google.com/uc?id=1XwjMGcYIg2qwEKHsC7uSmZayHvnEFhyg' output = os.path.join(parent_dir, 'warpgan_pretrained.zip') current_app.logger.debug("Downloading pretrained model") gdown.cached_download(url, output, quiet=False, postprocess=gdown.extractall) current_app.logger.debug("Download complete") os.remove(output) # remove downloaded zip after extraction else: current_app.logger.info("Pretrained model found - loading local file") if reset: pass network = WarpGAN() network.load_model(model_path) return network
def download(self): gdown.cached_download( url= "https://drive.google.com/uc?id=1l0ki7dX1WxcmV5Tfm41FPW-yk-wKUfne", # NOQA path=self.root_dir + ".zip", postprocess=gdown.extractall, )
def load_dataset(): """Load and return the dataset Returns ------- data : TODO -------- """ destination = os.path.join(DATA_DIR, 'dataset.jsons.gz') gdown.cached_download(dataset_url, destination, md5='a8f860b2dc400e14d4e7083775c1308a', quiet=False) if not os.path.isfile(destination): # Data not cached, download urllib.request.urlretrieve(dataset_url, destination) source_file = tf.constant([destination]) dataset = tf.data.TextLineDataset(filenames=source_file, compression_type='GZIP') dataset = dataset.map( lambda x: tf.py_function(_serialize, [x], [tf.string])) dataset = dataset.map(_parse) return dataset
def pr2_urdfpath(): gdown.cached_download( url='https://drive.google.com/uc?id=1zy4C665o6efPko7eMk4XBdHbvgFfdC-6', path=osp.join(download_dir, 'pr2_description.tar.gz'), md5='e4fb915accdb3568a5524c92e9c35c9a', postprocess=gdown.extractall, ) return osp.join(download_dir, 'pr2_description/pr2.urdf')
def panda_urdfpath(): gdown.cached_download( url='https://drive.google.com/uc?id=1h6ib9jpEUNa1xB2DNrnRQtqpSD2Rj9bz', path=osp.join(download_dir, 'franka_description.tar.gz'), md5='3de5bd15262b519e3beb88f1422032ac', postprocess=gdown.extractall, ) return osp.join(download_dir, 'franka_description/panda.urdf')
def main(): # __ https://drive.google.com/file/d/1zo4BkS8wqqbc7gxYQMedhqCiQUb51LAp/view?usp=sharing url = "https://drive.google.com/uc?id=0B9P1L--7Wd2vNm9zMTJWOGxobkU" output = "20150428_collected_images.tgz" gdown.download(url, output, quiet=False) md5 = "fa837a88f0c40c513d975104edf3da17" gdown.cached_download(url, output, md5=md5, postprocess=gdown.extractall) gdown.download(url)
def contrib_file(download_dir): """Download the contributon file """ contrib_file = download_dir / 'contrib.deeplift.h5' url = 'https://drive.google.com/uc?id=1-70VlFvcOCwwt4SrEXoqkaXyBQPnlQGZ' md5 = '56e456f0d1aeffc9d3fcdfead0520c17' gdown.cached_download(url, str(contrib_file), md5=md5) return contrib_file
def fetch_urdfpath(): gdown.cached_download( url='https://drive.google.com/uc?id=1y7Jc3QoVW6J072CrSNupfKpyLp4NNxuH', path=osp.join(download_dir, 'fetch_description.tar.gz'), md5='fbe29ab5f3d029d165a625175b43a265', postprocess=gdown.extractall, ) return osp.join(download_dir, 'fetch_description/fetch.urdf')
def gdrive_download(url, folder, filename): cache_dir = os.path.join(os.getenv("HOME"), ".cache", "torchfly", folder) os.makedirs(cache_dir, exist_ok=True) filepath = os.path.join(cache_dir, filename) gdown.cached_download(url, filepath, quiet=False) return filepath
def download(url, path=None, cached=True): import gdown # kitti_url = 'https://drive.google.com/uc?id=1QHvE8oHlHqXB97RHlulLuWCmleE0wZ8B' # kitti_out = 'kitti.tar.bz2' gdown.cached_download(url, path=path, quiet=False, proxy=False, postprocess=gdown.extractall)
def download(cls) -> None: url: str = "https://drive.google.com/uc?id=1if4VoEXNx9W3XCn0Y7Fp15B4GpcYbyYi" # NOQA md5 = None # 'c9122e177a766a9691cab13c5cda41a9' gdown.cached_download( url=url, path=cls._root_dir + ".zip", md5=md5, postprocess=gdown.extractall, )
def bunny_objpath(): target_path = osp.join(download_dir, 'mesh', 'bunny.obj') gdown.cached_download( url='https://drive.google.com/uc?id=18aAYzBglAGaSwes6oAENcufD8KC5XDph', path=target_path, md5='19bd31bde1fcf5242a8a82ed4ac03c72', quiet=True, ) return target_path
def download(self): logger.info( f"=> Downloading GDrive file {self.file_name} from {self.gdrive_url}" ) file_path: Path = get_temp_folder() / f"{self.file_name}" if not file_path.exists(): gdown.cached_download(url=self.gdrive_url, path=file_path) return file_path
def download_ckpt(data_dir, name, download_info): if os.path.exists(join(data_dir, 'checkpoints', name)): print('\nINFO: {} model already downloaded.'.format(name)) else: print('\nINFO: Downloading {} model...'.format(name)) url = 'https://drive.google.com/uc?id=' + download_info['url'] zip_output = join(data_dir, 'checkpoints', name + '.zip') md5 = download_info['md5'] gdown.cached_download(url, zip_output, md5=md5, postprocess=gdown.extractall, quiet=True) os.remove(zip_output)
def _download_unity(self): url = "https://drive.google.com/uc?id=" + APP_GDRIVE_ID[platform] os.makedirs("binary", exist_ok=True) # zip_path = os.path.join("binary", APP_FILE_NAME[platform]) zip_path = APP_FILE_NAME[platform] if os.path.exists(zip_path): logger.info("%s is already downloaded.", zip_path) else: logger.info("Downloading Unity app from %s", url) gdown.cached_download(url, zip_path, postprocess=gdown.extractall)
def download(cls) -> None: url: str = "https://drive.google.com/uc?id=1PKEJ8SVIVLukvmeIBexp6_XmJetHoOf2" # NOQA md5: str = "540c37435e4a16546850a83690b2db9b" gdown.cached_download( url=url, path=cls._root_dir + ".zip", md5=md5, postprocess=gdown.extractall, )
def _SSD_VGG_VOC_weights(): import gdown url = 'https://drive.google.com/uc?id=1GP6i9lcEqK3mOgr7soxruLhWpITxShge' output = Path.home() / '.anchors-jax/ssd-coco.jax' output.parent.mkdir(exist_ok=True, parents=True) gdown.cached_download(url, str(output)) params = pickle.load(output.open('rb')) return _split_bb_ssd_params(params)
def get_path(cls, quiet=True): """ returns the mnist folder, downloading the mnist data if needed """ if cls._path is None: url = 'https://drive.google.com/uc?id=1im9_agM-hHl8dKQU5uWh054rScjPINtz' # jeff hykin's google drive copy of mnist output = f'{temp_folder_path}/mnist.zip' gdown.cached_download(url, output, postprocess=gdown.extractall, quiet=quiet) cls._path = temp_folder_path+"/mnist" return cls._path
def creator(path): dataset = {} for split in ('train', 'dev', 'test'): en_path = gdown.cached_download(en_url.format(split)) ja_path = gdown.cached_download(ja_url.format(split)) with io.open(en_path, 'rt') as en, io.open(ja_path, 'rt') as ja: dataset[split] = [(x.rstrip(os.linesep), y.rstrip(os.linesep)) for x, y in zip(en, ja)] with io.open(path, 'wb') as f: pickle.dump(dataset, f) return dataset
def setup_repository(): # Downloading, extracting models. models_url = 'https://drive.google.com/uc?id=1ga08d8QQfAHOgTSKiPpIN7f_owuicNUA' models_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'resources', 'models') os.makedirs(models_path, exist_ok=True) md5 = '55c66e000de9077e483635029f740901' models_archive_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'models.zip') gdown.cached_download(url=models_url, path=models_archive_path, md5=md5) gdown.extractall(path=models_archive_path, to=models_path) os.remove(models_archive_path)
def __init__(self): root_dir = morefusion.utils.get_data_path( "wkentaro/morefusion/ycb_video/real_data/20191212_163242.566559922" ) gdown.cached_download( url= "https://drive.google.com/uc?id=1llWN7MOLzJZnaRDD4XGSmRWAFBtP3P9z", # NOQA md5="a773bb947377811b2b66ab9bc17f4d8d", path=root_dir + ".zip", ) self._dataset = morefusion.datasets.MyRealRGBDPoseEstimationDataset( root_dir=root_dir, )
def prepare_data(self): if os.path.exists(join(self.root_dir, self.DIR_NAME)) and os.path.isdir( join(self.root_dir, self.DIR_NAME)): if get_dir_md5hash(join(self.root_dir, self.DIR_NAME)) == self.DIR_HASH: return True raise ValueError('Wrong checksum, delete %s dir' % self.root_dir) gdown.cached_download( 'https://drive.google.com/uc?id=1-yZki-hyVcHKWB4VfqAUaAfBazPyemqN', os.path.join(self.root_dir, 'acdc.tar.gz'), md5=self.TAR_HASH, postprocess=gdown.extractall) return True
def _download_dataset(self) -> None: base_url = self.module.base_gdrive_url all_name_url_md5 = [(item[0], base_url + item[1], item[2]) for item in self.module.name_gdriveid_md5] for name, gdrive_url, md5 in all_name_url_md5: if self.verbose: print("Downloading " + name + "...") filepath = self.root / name gdown.download(gdrive_url, str(filepath), quiet=False) gdown.cached_download(gdrive_url, str(filepath), md5=md5) self._extract_archive(filepath, remove_archive=True)
def load_state_dict_from_url(url, name): # moslty code from torch.hub torch_home = _get_torch_home() model_dir = os.path.join(torch_home, "checkpoints") os.makedirs(model_dir, exist_ok=True) cached_file = os.path.join(model_dir, f"{name}.pth") gdown.cached_download(url, path=cached_file, quiet=False) state_dict = torch.load(cached_file)["state_dict"] state_dict = OrderedDict( {key[7:]: value for key, value in state_dict.items()}) return state_dict