def create_package(self, input_folder: Path, output_file: Path, store_extenal_info: bool = True, tar_extra_args: list = None, validate_only: bool = False):
"""
Create a leaf artifact from given folder containing a manifest.json
"""
mffile = input_folder / LeafFiles.MANIFEST
infofile = self.find_external_info_file(output_file)
if not mffile.exists():
raise LeafException("Cannot find manifest: {file}".format(file=mffile))
manifest = Manifest.parse(mffile)
manifest.validate_model()
if not validate_only:
if is_latest_package(manifest.identifier):
raise LeafException("Invalid version for manifest {mf} ({kw} is a reserved keyword)".format(mf=mffile, kw=LeafConstants.LATEST))
self.logger.print_default("Found package {mf.identifier} in {folder}".format(mf=manifest, folder=input_folder))
# Check if external info file exists
if not store_extenal_info and infofile.exists():
raise LeafException(
"A previous info file ({file}) exists for your package".format(file=infofile),
hints="You should remove it with 'rm {file}'".format(file=infofile),
)
self.__exec_tar(output_file, input_folder, extra_args=tar_extra_args)
self.logger.print_default("Leaf package created: {file}".format(file=output_file))
if store_extenal_info:
self.logger.print_default("Write info to {file}".format(file=infofile))
jwritefile(infofile, self.__build_pkg_node(output_file, manifest=manifest), pp=True)
def build_packages_environment(self, items: list, ipmap=None):
"""
Get the env vars declared by given packages
@param items: a list of InstalledPackage or PackageIdentifier
"""
ipmap = ipmap or self.list_installed_packages()
out = Environment()
for item in items:
ip = None
if isinstance(item, InstalledPackage):
ip = item
elif isinstance(item, PackageIdentifier):
ip = None
if is_latest_package(item):
ip = find_manifest(item, ipmap)
else:
ip = ipmap.get(item)
if ip is None:
raise InvalidPackageNameException(item)
else:
raise InvalidPackageNameException(item)
vr = VariableResolver(ip, ipmap.values())
out.append(ip.build_environment(vr=vr.resolve))
return out
def generate_manifest(self,
output_file: Path,
fragment_files: list = None,
info_map: dict = None,
resolve_envvars: bool = False):
"""
Used to create a manifest.json file
"""
model = OrderedDict()
# Load fragments
if fragment_files is not None:
for fragment_file in fragment_files:
self.logger.print_default(
"Use json fragment: {fragment}".format(
fragment=fragment_file))
jlayer_update(model,
jloadfile(fragment_file),
list_append=True)
# Load model
manifest = Manifest(model)
info = manifest.jsonget(JsonConstants.INFO, default=OrderedDict())
# Set the common info
if info_map is not None:
for key in (
JsonConstants.INFO_NAME,
JsonConstants.INFO_VERSION,
JsonConstants.INFO_DESCRIPTION,
JsonConstants.INFO_MASTER,
JsonConstants.INFO_DATE,
JsonConstants.INFO_REQUIRES,
JsonConstants.INFO_DEPENDS,
JsonConstants.INFO_TAGS,
JsonConstants.INFO_LEAF_MINVER,
JsonConstants.INFO_AUTOUPGRADE,
):
if key in info_map:
value = info_map[key]
if value is not None:
if key in (JsonConstants.INFO_REQUIRES,
JsonConstants.INFO_DEPENDS,
JsonConstants.INFO_TAGS):
# Handle lists
model_list = manifest.jsonpath(
[JsonConstants.INFO, key], default=[])
for motif in value:
if motif not in model_list:
if key == JsonConstants.INFO_DEPENDS:
# Try to parse as a conditional package
# identifier
ConditionalPackageIdentifier.parse(
motif)
elif key == JsonConstants.INFO_REQUIRES:
# Try to parse as a package identifier
PackageIdentifier.parse(motif)
self.logger.print_verbose(
"Add '{motif}' to '{key}' list".format(
motif=motif, key=key))
model_list.append(motif)
else:
self.logger.print_verbose(
"Set '{key}' = '{value}'".format(key=key,
value=value))
info[key] = value
# String replacement
jsonstr = jtostring(manifest.json, pp=True)
if resolve_envvars:
for var in set(
re.compile(r"#\{([a-zA-Z0-9_]+)\}").findall(jsonstr)):
value = os.environ.get(var)
if value is None:
raise LeafException(
"Cannot find '{var}' in env".format(var=var),
hints="Set the variable with 'export {var}=xxx'".
format(var=var))
self.logger.print_default("Replace {key} --> {value}".format(
key=var, value=value))
jsonstr = jsonstr.replace("#{{{var}}}".format(var=var), value)
if is_latest_package(manifest.identifier):
raise LeafException(
"Invalid version ({word} is a reserved keyword)".format(
word=LeafConstants.LATEST))
self.logger.print_default(
"Save '{mf.identifier}' manifest to {file}".format(
mf=manifest, file=output_file))
with output_file.open("w") as fp:
fp.write(jsonstr)
def generate_index(
self,
index_file: Path,
artifacts: list,
name: str = None,
description: str = None,
use_external_info: bool = True,
use_extra_tags: bool = True,
prettyprint: bool = False,
resolve: bool = True,
):
"""
Create an index.json referencing all given artifacts
"""
if not index_file.exists():
index_file.touch()
if resolve:
index_file = index_file.resolve()
try:
# Create the "info" node
info_node = OrderedDict()
if name is not None:
info_node[JsonConstants.REMOTE_NAME] = name
if description is not None:
info_node[JsonConstants.REMOTE_DESCRIPTION] = description
info_node[JsonConstants.REMOTE_DATE] = self.__get_date_now()
packages_map = OrderedDict()
# Resolve artifacts if needed
if resolve:
artifacts = [a.resolve() for a in artifacts]
for artifact in artifacts:
artifact_node = None
if use_external_info:
infofile = self.find_external_info_file(artifact)
if infofile.exists():
self.logger.print_default(
"Reading info from {file}".format(file=infofile))
artifact_node = jloadfile(infofile)
if artifact_node is None:
self.logger.print_default(
"Compute info for {artifact}".format(
artifact=artifact))
artifact_node = self.__build_pkg_node(artifact)
ap = AvailablePackage(artifact_node)
pi = ap.identifier
if is_latest_package(pi):
raise LeafException(
"Invalid version for package {artifact} ({word} is a reserved keyword)"
.format(artifact=artifact, word=LeafConstants.LATEST))
if pi in packages_map:
self.logger.print_default(
"Artifact already present: {pi}".format(pi=pi))
if ap.hashsum != AvailablePackage(
packages_map[pi]).hashsum:
raise LeafException(
"Artifact {pi} has multiple different artifacts for same version"
.format(pi=pi))
else:
# Read extra tags
extratags_file = artifact.parent / (artifact.name +
".tags")
if use_extra_tags and extratags_file.exists():
with extratags_file.open() as fp:
for tag in filter(
None, map(str.strip,
fp.read().splitlines())):
if tag not in ap.tags:
self.logger.print_default(
"Add extra tag {tag}".format(tag=tag))
ap.tags.append(tag)
self.logger.print_default("Add package {pi}".format(pi=pi))
try:
relative_path = artifact.relative_to(index_file.parent)
artifact_node[JsonConstants.REMOTE_PACKAGE_FILE] = str(
relative_path)
except ValueError:
raise LeafException(
"Artifact {a} must be relative to {i.parent}".
format(a=artifact, i=index_file))
packages_map[pi] = artifact_node
# Create the json structure
root_node = OrderedDict()
root_node[JsonConstants.INFO] = info_node
root_node[JsonConstants.REMOTE_PACKAGES] = list(
packages_map.values())
jwritefile(index_file, root_node, pp=prettyprint)
self.logger.print_default(
"Index created: {index}".format(index=index_file))
except BaseException as e:
# Clean the invalid index file
if index_file.exists():
index_file.unlink()
raise e