def read_single_project(dir, project_class=Project):
projects_in_dir = get_subdirs(dir)
if len(projects_in_dir) != 1:
raise RuntimeError('Found {} dirs instead of 1'.format(
len(projects_in_dir)))
project_dir = os.path.join(dir, projects_in_dir[0])
try:
project_fs = project_class(project_dir, OpenMode.READ)
except Exception as e:
projects_in_dir = get_subdirs(project_dir)
if len(projects_in_dir) != 1:
raise e
project_dir = os.path.join(project_dir, projects_in_dir[0])
project_fs = project_class(project_dir, OpenMode.READ)
return project_fs
def _read(self):
meta_json = load_json_file(self._get_project_meta_path())
self._meta = ProjectMeta.from_json(meta_json)
possible_datasets = get_subdirs(self.directory)
for ds_name in possible_datasets:
current_dataset = Dataset(os.path.join(self.directory, ds_name),
OpenMode.READ)
self._datasets = self._datasets.add(current_dataset)
if self.total_items == 0:
raise RuntimeError('Project is empty')
def read_single_project(dir, project_class=Project):
'''
Read project from given ditectory. Generate exception error if given dir contains more than one subdirectory
:param dir: str
:param project_class: Project class object type
:return: Project class object
'''
projects_in_dir = get_subdirs(dir)
if len(projects_in_dir) != 1:
raise RuntimeError('Found {} dirs instead of 1'.format(len(projects_in_dir)))
project_dir = os.path.join(dir, projects_in_dir[0])
try:
project_fs = project_class(project_dir, OpenMode.READ)
except Exception as e:
projects_in_dir = get_subdirs(project_dir)
if len(projects_in_dir) != 1:
raise e
project_dir = os.path.join(project_dir, projects_in_dir[0])
project_fs = project_class(project_dir, OpenMode.READ)
return project_fs
def _read(self):
'''
Download project from given project directory. Checks item and annotation directoris existing and dataset not empty.
Consistency checks. Every image must have an annotation, and the correspondence must be one to one.
'''
meta_json = load_json_file(self._get_project_meta_path())
self._meta = ProjectMeta.from_json(meta_json)
possible_datasets = get_subdirs(self.directory)
for ds_name in possible_datasets:
current_dataset = self.dataset_class(os.path.join(self.directory, ds_name), OpenMode.READ)
self._datasets = self._datasets.add(current_dataset)
if self.total_items == 0:
raise RuntimeError('Project is empty')
def download_or_get_repo(self):
git_url = self.app_info["githubUrl"]
version = self.app_info.get("version", "master")
already_downloaded = False
path_cache = None
if version != "master":
path_cache = os.path.join(
constants.APPS_STORAGE_DIR(),
*Path(git_url.replace(".git", "")).parts[1:], version)
already_downloaded = sly.fs.dir_exists(path_cache)
if already_downloaded is False:
self.logger.info("Git repo will be downloaded")
api = Api(self.info['server_address'], self.info['api_token'])
tar_path = os.path.join(self.dir_task_src, 'repo.tar.gz')
api.app.download_git_archive(self.app_info["moduleId"],
self.app_info["id"],
version,
tar_path,
log_progress=True,
ext_logger=self.logger)
with tarfile.open(tar_path) as archive:
archive.extractall(self.dir_task_src)
subdirs = get_subdirs(self.dir_task_src)
if len(subdirs) != 1:
raise RuntimeError(
"Repo is downloaded and extracted, but resulting directory not found"
)
extracted_path = os.path.join(self.dir_task_src, subdirs[0])
for filename in os.listdir(extracted_path):
shutil.move(os.path.join(extracted_path, filename),
os.path.join(self.dir_task_src, filename))
remove_dir(extracted_path)
silent_remove(tar_path)
#git.download(git_url, self.dir_task_src, github_token, version)
if path_cache is not None:
shutil.copytree(self.dir_task_src, path_cache)
else:
self.logger.info("Git repo already exists")
shutil.copytree(path_cache, self.dir_task_src)
def download(github_url,
dest_dir,
github_token=None,
version="master",
log_progress=True):
tar_path = os.path.join(dest_dir, 'repo.tar.gz')
download_tar(github_url, tar_path, github_token, version, log_progress)
with tarfile.open(tar_path) as archive:
archive.extractall(dest_dir)
subdirs = get_subdirs(dest_dir)
if len(subdirs) != 1:
raise RuntimeError(
"Repo is downloaded and extracted, but resulting directory not found"
)
extracted_path = os.path.join(dest_dir, subdirs[0])
for filename in os.listdir(extracted_path):
shutil.move(os.path.join(extracted_path, filename),
os.path.join(dest_dir, filename))
remove_dir(extracted_path)
silent_remove(tar_path)
def read_single_project(dir):
projects_in_dir = get_subdirs(dir)
if len(projects_in_dir) != 1:
raise RuntimeError('Found {} dirs instead of 1'.format(
len(projects_in_dir)))
return Project(os.path.join(dir, projects_in_dir[0]), OpenMode.READ)