def home(self) -> Path:
""" @:return the current dataset home """
ds_home = LibUtil.datasets_home()
ds_name = type(self).__name__
ds_path = ds_home.joinpath(ds_name)
ds_path.mkdir(exist_ok=True)
return ds_path
def download_model(cls, model_name: str) -> str:
available_models = cls.available_models() # get the lis
assert model_name in available_models, "Invalid model name {}".format(model_name)
checkpoint_model_path = LibUtil.models_home(sub_folder="tf").joinpath(model_name)
model_uri = available_models[model_name]
if not checkpoint_model_path.exists():
FileUtil.download_file(model_uri, checkpoint_model_path, unzip=True, show_progress=True)
return checkpoint_model_path
def download_pipeline(cls, model_name: str) -> str:
available_pipelines = cls.available_pipelines()
assert model_name in available_pipelines, \
"there is not a pipeline available for the model {}".format(model_name)
pipeline_uri = available_pipelines[model_name]
filename = Path(urlparse(pipeline_uri).path).name
pipeline_model_path = LibUtil.pipelines_home(subfolder="tf").joinpath(filename)
if not pipeline_model_path.exists():
pipeline_model_path = FileUtil.download_file(pipeline_uri, pipeline_model_path.parent, show_progress=True)
return pipeline_model_path
def _clone_repo(self):
try:
assert self._repo_uri and validators.url(
self._repo_uri), "Invalid repo uri"
repo_name = self.get_repo_name_from_url(self._repo_uri)
self._repo_folder = LibUtil.repos_home().joinpath(repo_name)
self._repo_folder.mkdir(exist_ok=True)
if not self._repo_folder.joinpath(".git").is_dir():
Repo.clone_from(self._repo_uri,
self._repo_folder,
depth=1,
branch='master',
progress=CloneProgress(repo_name))
except git.exc.InvalidGitRepositoryError as ex:
logger.error("Error cloning the repository from {} : {}".format(
self._repo_uri, ex))
except git.exc.GitError as ex:
logger.error("Error cloning the repository from {} : {}".format(
self._repo_uri, ex))