def populate_latest_versions(self):
conda3_pkgs: list = list()
conda2_pkgs: list = list()
normal_pkgs: list = list()
for key in self.keys:
# Repack key data
manager, package = key.split('&')
if manager == 'conda2':
conda2_pkgs.append([package, key])
elif manager == 'conda3':
conda3_pkgs.append([package, key])
else:
normal_pkgs.append([manager, package, key])
if conda2_pkgs:
# load all versions at once
all_pkgs = [x[0] for x in conda2_pkgs]
mgr = get_package_manager('conda2')
try:
versions = mgr.latest_versions(all_pkgs,
labbook=self.labbook,
username=self.username)
# save
for version, pkg in zip(versions, conda2_pkgs):
self.latest_versions[pkg[1]] = version
except ValueError as err:
logger.warning(
f"An error occurred while looking up conda2 latest versions: {err}"
)
if conda3_pkgs:
# load all versions at once
all_pkgs = [x[0] for x in conda3_pkgs]
mgr = get_package_manager('conda3')
try:
versions = mgr.latest_versions(all_pkgs,
labbook=self.labbook,
username=self.username)
# save
for version, pkg in zip(versions, conda3_pkgs):
self.latest_versions[pkg[1]] = version
except ValueError as err:
logger.warning(
f"An error occurred while looking up conda3 latest versions: {err}"
)
if normal_pkgs:
# For these package managers, look up each latest version individually
for pkg in normal_pkgs:
mgr = get_package_manager(pkg[0])
self.latest_versions[pkg[2]] = mgr.latest_version(
pkg[1], labbook=self.labbook, username=self.username)
def _load_packages(self) -> List[str]:
"""Load packages from yaml files in expected location in directory tree. """
root_dir = os.path.join(self.labbook.root_dir, '.gigantum', 'env',
'package_manager')
package_files = [
os.path.join(root_dir, n) for n in os.listdir(root_dir)
if 'yaml' in n
]
docker_lines = ['## Adding individual packages']
apt_updated = False
for package in sorted(package_files):
pkg_fields: Dict[str, Any] = {}
with open(package) as package_content:
pkg_fields.update(yaml.safe_load(package_content))
# Generate the appropriate docker command for the given package info
pkg_info = {
"name": str(pkg_fields['package']),
"version": str(pkg_fields.get('version'))
}
if not pkg_fields.get('from_base'):
if pkg_fields['manager'] == 'apt' and not apt_updated:
docker_lines.append('RUN apt-get -y update')
apt_updated = True
docker_lines.extend(
get_package_manager(
pkg_fields['manager']).generate_docker_install_snippet(
[pkg_info]))
return docker_lines
def populate_results(self) -> None:
# Repack key data
packages: Dict[str, List[Tuple[str, str, str]]] = {
'conda2': list(),
'conda3': list(),
'pip': list(),
'apt': list()
}
for key in self.keys:
manager, package = key.split('&')
packages[manager].append((manager, package, key))
for manager in packages.keys():
package_names = [x[1] for x in packages[manager]]
if package_names:
mgr = get_package_manager(manager)
try:
metadata = mgr.get_packages_metadata(
package_names,
labbook=self.labbook,
username=self.username)
# save
for pkg_input, metadata in zip(packages[manager],
metadata):
self.results[pkg_input[2]] = metadata
except ValueError as err:
logger.warning(
f"An error occurred while looking up {manager} latest versions and metadata: {err}"
)
def populate_latest_versions(self):
conda3_pkgs = list()
conda2_pkgs = list()
normal_pkgs = list()
for key in self.keys:
# Repack key data
manager, package = key.split('&')
if manager == 'conda2':
conda2_pkgs.append([package, key])
elif manager == 'conda3':
conda3_pkgs.append([package, key])
else:
normal_pkgs.append([manager, package, key])
if conda2_pkgs:
# load all versions at once
all_pkgs = [x[0] for x in conda2_pkgs]
mgr = get_package_manager('conda2')
versions = mgr.latest_versions(all_pkgs,
labbook=self.labbook,
username=self.username)
# save
for version, pkg in zip(versions, conda2_pkgs):
self.latest_versions[pkg[1]] = version
if conda3_pkgs:
# load all versions at once
all_pkgs = [x[0] for x in conda3_pkgs]
mgr = get_package_manager('conda3')
versions = mgr.latest_versions(all_pkgs,
labbook=self.labbook,
username=self.username)
# save
for version, pkg in zip(versions, conda3_pkgs):
self.latest_versions[pkg[1]] = version
if normal_pkgs:
# For these package managers, look up each latest version individually
for pkg in normal_pkgs:
mgr = get_package_manager(pkg[0])
self.latest_versions[pkg[2]] = mgr.latest_version(
pkg[1], labbook=self.labbook, username=self.username)
def helper_resolve_packages(labbook, package_input):
"""Helper to return a PackageComponent object"""
manager = list(set([x['manager'] for x in package_input]))
if len(manager) > 1:
raise ValueError("Only batch add via 1 package manager at a time.")
# Instantiate appropriate package manager
mgr = get_package_manager(manager[0])
# Validate packages
pkg_result = mgr.validate_packages(package_input, labbook, get_logged_in_username())
# Return object
return [PackageComponent(manager=manager[0],
package=pkg.package,
version=pkg.version,
is_valid=not pkg.error) for pkg in pkg_result]
def helper_resolve_check_packages(labbook, package_input):
"""Helper to return a list of PackageComponent objects that have been validated"""
manager = list(set([x['manager'] for x in package_input]))
if len(manager) > 1:
raise ValueError("Only batch add via 1 package manager at a time.")
# Instantiate appropriate package manager
mgr = get_package_manager(manager[0])
# Validate packages
pkg_result = mgr.validate_packages(package_input, labbook, get_logged_in_username())
# Create dataloader
keys = [f"{manager[0]}&{pkg.package}" for pkg in pkg_result]
vd = PackageDataloader(keys, labbook, get_logged_in_username())
# Return object
return [PackageComponent(_dataloader=vd,
manager=manager[0],
package=pkg.package,
version=pkg.version,
is_valid=not pkg.error) for pkg in pkg_result]
def test_get_invalid(self):
with pytest.raises(ValueError):
get_package_manager("asdfasdf")
def test_get_apt(self):
assert type(get_package_manager('apt')) == AptPackageManager
assert type(get_package_manager('apt-get')) == AptPackageManager
def test_get_conda(self):
assert type(get_package_manager('conda2')) == Conda2PackageManager
assert type(get_package_manager('conda3')) == Conda3PackageManager
def test_get_pip(self):
assert type(get_package_manager('pip')) == PipPackageManager
assert type(get_package_manager('pip2')) == PipPackageManager