def _get_component_descr(self, component, which="model"):
component = os.path.normpath(component)
self.assert_is_component(component, which)
if self._is_nongroup_component(component, which):
# component has an explicit yaml file
# TODO - move into the component directory when loading
return load_component_descr(
os.path.join(self.local_path, component), which)
elif self._is_python_component(component, which):
return load_python_component_descr(component, which)
else:
k = self.get_group_name(component, which)
if k is None:
raise ValueError("{} {} doesn't exist".format(
which, component))
else:
if self.component_group_list is not None:
# already cached
return self.component_group_list[which][
k]._get_component_descr(relative_path(component, k))
else:
grp = LocalComponentGroup.load(
os.path.join(self.local_path, k), which)
return grp._get_component_descr(relative_path(
component, k))
def list_softlink_dependencies(component_dir, source_path):
"""List dependencies of a directory
Returns a set
"""
return {relative_path(f, source_path) if os.path.isdir(f)
else relative_path(os.path.dirname(f), source_path)
for f in list_softlink_realpaths(component_dir)
if is_subdir(f, source_path)}
def _pull_component(self, component, which="model"):
if not self._pulled and self.auto_update:
self.pull_source()
component_dir = self.local_source._get_component_dir(component, which)
if self.use_lfs:
# the only call to git-lfs -> pulling specific sub-files
lfs_installed(raise_exception=True)
# get a list of directories to source (relative to the local_path)
softlink_dirs = list(
list_softlink_dependencies(component_dir, self.local_path))
# pull these softlinks
for pull_dir in [
component,
relative_path(component_dir, self.local_path)
] + softlink_dirs:
cmd = ["git-lfs", "pull", "-I {0}/**".format(pull_dir)]
logger.info(" ".join(cmd))
subprocess.call(cmd,
cwd=self.local_path,
env=dict(os.environ, GIT_LFS_SKIP_SMUDGE="1"))
return self.local_source._pull_component(component, which)
def _get_component_download_dir(self, component, which='model', name=None):
component = os.path.normpath(component)
if name is None:
name = which
insert_path = os.path.join("downloaded", '{}_files'.format(name))
# special case: component can be outside of the root directory
if self._is_nongroup_component(component,
which) or self._is_python_component(
component, which):
return os.path.join(self.local_path, os.path.normpath(component),
insert_path)
else:
k = self.get_group_name(component, which)
if k is None and which == 'dataloader':
# fallback: get model's download directory
return self._get_component_download_dir(component,
which='model',
name='dataloader')
if k is None:
raise ValueError("Couldn't get {} download_dir. Model"
" or group doesn't exist for {}".format(
which, component))
return os.path.join(self.local_path, k, insert_path,
relative_path(component, k))
def get_group_name(self, component, which='model'):
component = os.path.normpath(component)
if self.component_group_list is not None:
# already cached
for k in self.component_group_list[which]:
if component.startswith(os.path.join(k, "")):
return k
return None
else:
group_path = LocalComponentGroup.group_path(os.path.join(self.local_path, component), which)
if group_path is None:
return None
else:
return relative_path(group_path, self.local_path)
def _is_component(self, component, which="model"):
component = os.path.normpath(component)
if self._is_nongroup_component(component, which):
# it contains a {which}.y?ml
return True
else:
# it's present in one of the groups
k = self.get_group_name(component, which)
if k is not None:
# check that it's indeed found in one of the components
if self.component_group_list is not None:
# already cached
return component in self._list_components(which)
else:
grp = LocalComponentGroup.load(os.path.join(self.local_path, k), which)
return grp._is_component(relative_path(component, k))
else:
return False