def create_archive(
filename: str,
from_dir: str,
dest: str,
force_extension: Optional[str] = None,
from_dir_rename: Optional[str] = None,
no_root_dir: bool = False,
) -> None:
"""Create an archive file (.tgz, .tar.gz, .tar or .zip).
On Windows, if the python tarfile and zipfile modules are available, the
python implementation is used to create the archive. On others system,
create_archive spawn tar, gzip or zip as it is twice faster that the python
implementation. If the tar, gzip or zip binary is not found, the python
implementation is used.
:param filename: archive to create
:param from_dir: directory to pack (full path)
:param dest: destination directory (should exist)
:param force_extension: specify the archive extension if not in the
filename. If filename has no extension and force_extension is None
create_archive will fail.
:param from_dir_rename: name of root directory in the archive.
:param no_root_dir: create archive without the root dir (zip only)
:raise ArchiveError: if an error occurs
"""
# Check extension
from_dir = from_dir.rstrip("/")
filepath = os.path.abspath(os.path.join(dest, filename))
ext = check_type(filename, force_extension=force_extension)
if from_dir_rename is None:
from_dir_rename = os.path.basename(from_dir)
if ext == "zip":
zip_archive = zipfile.ZipFile(filepath, "w", zipfile.ZIP_DEFLATED)
for root, _, files in os.walk(from_dir):
relative_root = os.path.relpath(
os.path.abspath(root), os.path.abspath(from_dir)
)
for f in files:
zip_file_path = os.path.join(from_dir_rename, relative_root, f)
if no_root_dir:
zip_file_path = os.path.join(relative_root, f)
zip_archive.write(os.path.join(root, f), zip_file_path)
zip_archive.close()
else:
if ext == "tar":
tar_format = "w"
elif ext == "tar.gz":
tar_format = "w:gz"
elif ext == "tar.bz2":
tar_format = "w:bz2"
else:
assert_never()
with closing(tarfile.open(filepath, tar_format)) as tar_archive:
tar_archive.add(name=from_dir, arcname=from_dir_rename, recursive=True)
def update(
self,
vcs: Literal["git"] | Literal["svn"] | Literal["external"],
url: str,
revision: Optional[str] = None,
) -> ReturnValue:
"""Update content of the working directory.
:param vcs: vcs kind
:param url: repository url, when vcs is external the url is the path
to the source directory
:param revision: revision
Note that when vcs is set to git or svn, the version control ignore
setting is taken into account. Additionally, when the vcs is
external and the source directory contains a .git subdirectory then
git ignore setting is taken into account.
"""
# Reset changelog file
if os.path.isfile(self.changelog_file):
rm(self.changelog_file)
update: Callable[
[str, Optional[str]], tuple[ReturnValue, Optional[str], Optional[str]]
]
if vcs == "git":
update = self.update_git
elif vcs == "svn":
update = self.update_svn
elif vcs == "external":
update = self.update_external
else:
assert_never()
result, old_commit, new_commit = update(url=url, revision=revision)
with open(self.metadata_file, "w") as fd:
json.dump(
{
"name": self.name,
"url": url,
"old_commit": old_commit,
"new_commit": new_commit,
"revision": revision,
},
fd,
)
return result
def add_spec(
self,
name: str,
env: BaseEnv,
primitive: PRIMITIVE,
qualifier: Optional[str] = None,
source_packages: Optional[list[str]] = None,
expand_build: bool = True,
source_name: Optional[str] = None,
plan_line: Optional[str] = None,
plan_args: Optional[dict] = None,
sandbox: Optional[SandBox] = None,
upload: bool = False,
force_download: bool = False,
) -> Build | CreateSources | CreateSource | Install | Test:
"""Expand an anod action into a tree (internal).
:param name: spec name
:param env: context in which to load the spec
:param primitive: spec primitive
:param qualifier: qualifier
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:param expand_build: should build primitive be expanded
:param source_name: source name associated with the source
primitive
:param plan_line: corresponding line:linenumber in the plan
:param plan_args: action args after plan execution, taking into
account plan context (such as with defaults(XXX):)
:param sandbox: if not None, anod instance are automatically bind to
the given sandbox
:param upload: if True consider uploads to the store (sources and
binaries)
:param force_download: if True force a download
"""
def add_action(data: Action,
connect_with: Optional[Action] = None) -> None:
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
def add_dep(spec_instance: Anod, dep: Dependency,
dep_instance: Anod) -> None:
"""Add a new dependency in an Anod instance dependencies dict.
:param spec_instance: an Anod instance
:param dep: the dependency we want to add
:param dep_instance: the Anod instance loaded for that dependency
"""
if dep.local_name in spec_instance.deps:
raise AnodError(
origin="expand_spec",
message="The spec {} has two dependencies with the same "
"local_name attribute ({})".format(spec_instance.name,
dep.local_name),
)
spec_instance.deps[dep.local_name] = dep_instance
# Initialize a spec instance
e3.log.debug("add spec: name:{}, qualifier:{}, primitive:{}".format(
name, qualifier, primitive))
spec = self.load(
name,
qualifier=qualifier,
env=env,
kind=primitive,
sandbox=sandbox,
source_name=source_name,
)
result: Build | CreateSources | CreateSource | Install | Test
# Initialize the resulting action based on the primitive name
if primitive == "source":
if not has_primitive(spec, "source"):
raise SchedulingError(
f"spec {name} does not support primitive source")
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
if TYPE_CHECKING:
# When creating sources we know that the
# source_pkg_build attribute is set
assert spec.source_pkg_build is not None
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb.name not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(
name=name,
env=env,
primitive="source",
source_name=sb.name,
plan_line=plan_line,
plan_args=plan_args,
sandbox=sandbox,
upload=upload,
)
self.connect(result, sub_result)
elif primitive == "build":
if not has_primitive(spec, "build"):
raise SchedulingError(
f"spec {name} does not support primitive build for"
" platform {env.platform} and qualifier '{qualifier}'")
result = Build(spec)
elif primitive == "test":
result = Test(spec)
elif primitive == "install":
result = Install(spec)
else:
assert_never()
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if primitive == "install" and force_download:
# We have an "download" dependency explicit in the plan
# Make sure to record the BuildOrDownload decision
if result in self:
action_preds = self.predecessors(result)
if action_preds:
dec = action_preds[0]
if isinstance(dec, BuildOrDownload):
dec.set_decision(which=BuildOrDownload.INSTALL,
decision_maker=plan_line)
elif (primitive == "install" and not spec.has_package
and has_primitive(spec, "build") and not force_download):
if plan_line is not None and plan_args is not None:
# We have an explicit call to install() in the plan but the
# spec has no binary package to download.
raise SchedulingError(
f"error in plan at {plan_line}: "
"install should be replaced by build - "
f"the spec {spec.name} has a build primitive "
"but does not define a package")
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(
name,
env,
"build",
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
if expand_build and primitive == "build" and spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(
name,
env,
"install",
qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError(
f"spec {name} does not support primitive {primitive}")
# Add the action in the DAG
add_action(result)
if primitive == "install":
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, "build"):
build_action = self.add_spec(
name=name,
env=env,
primitive="build",
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
decision = self.add_decision(BuildOrDownload, result,
build_action, download_action)
if force_download:
decision.set_decision(which=BuildOrDownload.INSTALL,
decision_maker=plan_line)
else:
self.connect(result, download_action)
elif primitive == "source":
if source_name is not None:
# Also add an UploadSource action
if upload:
upload_src = UploadSource(spec, source_name)
self.add(upload_src)
# Link the upload to the current context
if plan_line is not None and plan_args is not None:
self.link_to_plan(
vertex_id=upload_src.uid,
plan_line=plan_line,
plan_args=plan_args,
)
self.connect(self.root, upload_src)
self.connect(upload_src, result)
if TYPE_CHECKING:
# When creating sources we know that the
# source_pkg_build attribute is set
assert spec.source_pkg_build is not None
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
raise SchedulingError(
origin="add_spec",
message=f"unknown repository {checkout}",
)
co = Checkout(checkout, self.repo.repos[checkout])
add_action(co, result)
# Look for dependencies. Consider that "None" means "no dependency".
spec_dependencies = list(
fetch_attr(spec, f"{primitive}_deps", None) or [])
source_spec_dependencies_names = {
d.name
for d in spec_dependencies if d.kind == "source"
}
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == "source":
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(
e.name,
kind="source",
env=self.default_env,
qualifier=None,
sandbox=sandbox,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_instance)
self.dependencies[spec.uid][e.local_name] = (
e,
spec.deps[e.local_name],
)
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind if e.kind != "download" else "install",
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
force_download=e.kind == "download",
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_action.anod_instance)
self.dependencies[spec.uid][e.local_name] = (
e, spec.deps[e.local_name])
if e.kind == "build" and self[
child_action.uid].data.kind == "install":
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
child_action_preds = self.predecessors(child_action)
if child_action_preds:
dec = child_action_preds[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result,
BuildOrDownload.BUILD,
plan_line
if plan_line is not None else "unknown line",
)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
source_list = fetch_attr(spec, f"{primitive}_source_list", None)
if source_list is not None:
for s in source_list:
# set source builder
if s.name in self.sources:
sb_spec, sb = self.sources[s.name]
if (sb_spec != spec.name
and sb_spec not in source_spec_dependencies_names
# ignore unmanaged source builders which do not
# create many issues (no need to find the source
# builder to apply patches, update repositories, ...)
# and this creates too many warnings in production that
# we do not have time to fix
and not isinstance(sb, UnmanagedSourceBuilder)):
logger.warning(
f"{spec.name}.anod ({primitive}): source {s.name}"
f" coming from {sb_spec} but there is no"
f" source_pkg dependency for {sb_spec} in {primitive}_deps",
)
s.set_builder(sb)
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = (result.uid.rsplit(".", 1)[0] +
".source_install." + s.name)
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(
origin="expand_spec",
message="source %s does not exist "
"(referenced by %s)" % (s.name, result.uid),
)
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(
name=spec_decl,
env=self.default_env,
primitive="source",
plan_args=None,
plan_line=plan_line,
source_name=s.name,
sandbox=sandbox,
upload=upload,
)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos[repo])
add_action(r, connect_with=source_action)
self.add_decision(
CreateSourceOrDownload,
src_get_action,
source_action,
src_download_action,
)
return result
def add_anod_action(
self,
name: str,
env: BaseEnv,
primitive: PRIMITIVE,
qualifier: Optional[str] = None,
source_packages: Optional[list[str]] = None,
upload: bool = True,
plan_line: Optional[str] = None,
plan_args: Optional[dict] = None,
sandbox: Optional[SandBox] = None,
) -> Action:
"""Add an Anod action to the context (internal function).
Note that using add_anod_action should be avoided when possible
and replaced by a call to add_plan_action.
:param name: spec name
:param env: context in which to load the spec
:param primitive: spec primitive
:param qualifier: qualifier
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:param upload: if True consider uploading to the store
:param plan_line: corresponding line:linenumber in the plan
:param plan_args: action args after plan execution, taking into
account plan context (such as with defaults(XXX):)
:param sandbox: the SandBox object that will be used to run commands
:return: the root added action
"""
# First create the subtree for the spec
result = self.add_spec(
name,
env,
primitive,
qualifier,
source_packages=source_packages,
plan_line=plan_line,
plan_args=plan_args,
sandbox=sandbox,
upload=upload,
)
# Resulting subtree should be connected to the root node
self.connect(self.root, result)
# Ensure decision is set in case of explicit build or install
if primitive == "build":
build_action = None
for el in self.predecessors(result):
if isinstance(el, BuildOrDownload):
el.set_decision(BuildOrDownload.BUILD, plan_line)
build_action = self[el.left]
if build_action is None and isinstance(result, Build):
build_action = result
# Create upload nodes
if build_action is not None:
spec = build_action.data
if spec.component is not None and upload:
upload_bin: UploadBinaryComponent | UploadSourceComponent
if spec.has_package:
upload_bin = UploadBinaryComponent(spec)
else:
upload_bin = UploadSourceComponent(spec)
self.add(upload_bin)
# ??? is it needed?
if plan_line is not None and plan_args is not None:
self.link_to_plan(
vertex_id=upload_bin.uid,
plan_line=plan_line,
plan_args=plan_args,
)
self.connect(self.root, upload_bin)
self.connect(upload_bin, build_action)
elif primitive == "install":
for el in self.predecessors(result):
if isinstance(el, BuildOrDownload):
el.set_decision(BuildOrDownload.INSTALL, plan_line)
elif primitive != "source" and primitive != "test":
assert_never()
return result
def __init__(
self,
name: str,
product_version: Optional[str] = None,
host: Optional[str] = None,
target: Optional[str] = None,
build: Optional[str] = None,
qualifier: Optional[str] = None,
local_name: Optional[str] = None,
require: Literal["build_tree"]
| Literal["installation"]
| Literal["download"]
| Literal["source_pkg"] = "build_tree",
track: bool = False,
**kwargs: Any,
) -> None:
"""Initialize a Dependency object.
:param name: basename of the Anod spec file (without .anod extension)
:param product_version: product version to force when loading the spec
:param host: can be either 'target' or 'build'. If set to 'target' then
it means that the dependency will be loaded with host set to
the current module target information. If set to 'build' then
similar mechanism is used with the current module 'build'
platform.
:param target: can be either 'host' or 'build'
:param build: build platform (if not the current build platform), the
special value "default" change the build platform to the default
build platform. Note that on windows, the default is always
x86-windows.
:param qualifier: qualifier to set when loading the spec
:param local_name: if not None, this name will be the dependency key
in deps and makedeps dictionaries. It allows importing twice
the same anod module with different qualifers or platforms
:param require: can be 'build_tree' (to force a local build),
'installation', 'download', or 'source_pkg'
:param track: if True, track all source packages metadata and include
them in the local metadata.
:param kwargs: other parameters valid in some API that we ignore now
"""
del kwargs
self.name = name
self.product_version = product_version
self.host = host
self.target = target
self.build = build
self.qualifier = qualifier
self.local_name = local_name if local_name is not None else name
if require not in (
"build_tree",
"download",
"installation",
"source_pkg",
):
raise e3.anod.error.SpecError(
f"require should be build_tree, download, installation,"
f" or source_pkg not {require}.")
if require == "build_tree":
self.kind = "build"
elif require == "download":
self.kind = "download"
elif require == "installation":
self.kind = "install"
elif require == "source_pkg":
self.kind = "source"
else:
assert_never()
self.track = track
def unpack_archive(
filename: str,
dest: str,
selected_files: Optional[Sequence[str]] = None,
remove_root_dir: RemoveRootDirType = False,
unpack_cmd: Optional[Callable[..., None]] = None,
force_extension: Optional[str] = None,
delete: bool = False,
ignore: Optional[list[str]] = None,
preserve_timestamps: bool = True,
tmp_dir_root: Optional[str] = None,
) -> None:
"""Unpack an archive file (.tgz, .tar.gz, .tar or .zip).
:param filename: archive to unpack
:param dest: destination directory (should exist)
:param selected_files: list of files to unpack (partial extraction). If
None all files are unpacked
:param remove_root_dir: if True then the root dir of the archive is
suppressed.
if set to 'auto' then the root dir of the archive is suppressed only
if it is possible. If not do not raise an exception in that case and
fallback on the other method.
:param unpack_cmd: command to run to unpack the archive, if None use
default methods or raise ArchiveError if archive format is not
supported. If unpack_cmd is not None, then remove_root_dir is ignored.
The unpack_cmd must raise ArchiveError in case of failure.
:param force_extension: specify the archive extension if not in the
filename. If filename has no extension and force_extension is None
unpack_archive will fail.
:param delete: if True and remove_root_dir is also True, remove files
from dest if they do not exist in the archive
:param ignore: a list of files/folders to keep when synchronizing with
the final destination directory.
:param preserve_timestamps: if False and remove_root_dir is True, and the
target directory exists, ensure that updated files get their timestamp
updated to current time.
:param tmp_dir_root: If not None the temporary directory used to extract the
archive will be created in tmp_dir_root directory. If None the temporary
directory is created in the destination directory. This argument only
has an effect when remove_root_dir is True.
:raise ArchiveError: in case of error
cygpath (win32) utilities might be needed when using remove_root_dir option
"""
logger.debug("unpack %s in %s", filename, dest)
# First do some checks such as archive existence or destination directory
# existence.
if not os.path.isfile(filename):
raise ArchiveError(origin="unpack_archive", message=f"cannot find {filename}")
if not os.path.isdir(dest):
raise ArchiveError(
origin="unpack_archive", message=f"dest dir {dest} does not exist"
)
if selected_files is None:
selected_files = []
# We need to resolve to an absolute path as the extraction related
# processes will be run in the destination directory
filename = os.path.abspath(filename)
if unpack_cmd is not None:
# Use user defined unpack command
if not selected_files:
return unpack_cmd(filename, dest)
else:
return unpack_cmd(filename, dest, selected_files=selected_files)
ext = check_type(filename, force_extension=force_extension)
# If remove_root_dir is set then extract to a temp directory first.
# Otherwise extract directly to the final destination
if remove_root_dir:
if tmp_dir_root is None:
tmp_dir_root = os.path.dirname(os.path.abspath(dest))
tmp_dest = tempfile.mkdtemp(prefix="", dir=tmp_dir_root)
else:
tmp_dest = dest
try:
if ext == "tar" or ext == "tar.bz2" or ext == "tar.gz":
try:
# Set the right mode
mode = "r:"
if ext.endswith("bz2"):
mode += "bz2"
elif ext.endswith("gz"):
mode += "gz"
# Extract tar files
with closing(tarfile.open(filename, mode=mode)) as fd:
check_selected = set(selected_files)
def is_match(name: str, files: Sequence[str]) -> bool:
"""check if name match any of the expression in files.
:param name: file name
:param files: list of patterns to test against
:return: True when the name is matched
"""
for pattern in files:
if fnmatch.fnmatch(name, pattern):
if pattern in check_selected:
check_selected.remove(pattern)
return True
return False
dirs: list[str] = []
# IMPORTANT: don't use the method extract. Always use the
# extractall function. Indeed extractall will set file
# permissions only once all selected members are unpacked.
# Using extract can lead to permission denied for example
# if a read-only directory is created.
if selected_files:
member_list = []
for tinfo in fd:
if is_match(
tinfo.name, selected_files
) or tinfo.name.startswith(tuple(dirs)):
# If dir then add it for recursive extracting
if tinfo.isdir() and not tinfo.name.startswith(
tuple(dirs)
):
dirs.append(tinfo.name)
member_list.append(tinfo)
if check_selected:
raise ArchiveError(
"unpack_archive", f"Cannot untar {filename} "
)
fd.extractall(path=tmp_dest, members=member_list)
else:
fd.extractall(path=tmp_dest)
except tarfile.TarError as e:
raise ArchiveError(
origin="unpack_archive",
message=f"Cannot untar {filename} ({e})",
).with_traceback(sys.exc_info()[2])
elif ext == "zip":
try:
with closing(E3ZipFile(filename, mode="r")) as zip_fd:
zip_fd.extractall(
tmp_dest, selected_files if selected_files else None
)
except zipfile.BadZipfile as e:
raise ArchiveError(
origin="unpack_archive",
message=f"Cannot unzip {filename} ({e})",
).with_traceback(sys.exc_info()[2])
else:
assert_never()
if remove_root_dir:
# First check that we have only one dir in our temp destination,
# and no other files or directories. If not raise an error.
nb_files = len(os.listdir(tmp_dest))
if nb_files == 0:
# Nothing to do...
return
if nb_files > 1:
if remove_root_dir != "auto":
raise ArchiveError(
origin="unpack_archive",
message="archive does not have a unique root dir",
)
# We cannot remove root dir but remove_root_dir is set to
# 'auto' so fallback on non remove_root_dir method
if not os.listdir(dest):
e3.fs.mv(os.path.join(tmp_dest, "*"), dest)
else:
e3.fs.sync_tree(
tmp_dest,
dest,
delete=delete,
ignore=ignore,
preserve_timestamps=preserve_timestamps,
)
else:
root_dir = os.path.join(tmp_dest, os.listdir(tmp_dest)[0])
# Now check if the destination directory is empty. If this is
# the case a simple move will work, otherwise we need to do a
# sync_tree (which cost more)
if not os.listdir(dest):
e3.fs.mv(
[os.path.join(root_dir, f) for f in os.listdir(root_dir)], dest
)
else:
e3.fs.sync_tree(
root_dir,
dest,
delete=delete,
ignore=ignore,
preserve_timestamps=preserve_timestamps,
)
finally:
# Always remove the temp directory before exiting
if remove_root_dir:
e3.fs.rm(tmp_dest, True)