def primitive_func(self, *args, **kwargs):
self.log.debug("%s %s starts", self.name, f.__name__)
result = False
# Ensure temporary directory is set to a directory local to
# the current sandbox. This avoid mainly to loose track of
# temporary files that are then accumulating on the
# filesystems.
if self.build_space is not None and self.build_space.initialized:
for tmp_var in ("TMP", "TEMP", "TMPDIR"):
os.environ[tmp_var] = self.build_space.tmp_dir
try:
# If there is a pre function call it
if pre is not None:
self._pre = getattr(self, pre)()
# Run the primitive
result = f(self, *args, **kwargs)
self.log.debug("%s %s ends", self.name, f.__name__)
# And return the result
return result
except AnodError:
self.log.exception("%s %s fails", self.name, f.__name__)
raise AnodError(
"%s %s fails (AnodError exception in primitive)" %
(self.name, f.__name__))
except Exception as e:
self.log.exception("%s %s fails", self.name, f.__name__)
raise AnodError("%s %s fails (got exception: %s)" %
(self.name, f.__name__, e))
def primitive_func(self, *args, **kwargs): # type: ignore
self.log.debug("%s %s starts", self.name, f.__name__)
result = False
# Ensure temporary directory is set to a directory local to
# the current sandbox. This avoid mainly to loose track of
# temporary files that are then accumulating on the
# filesystems.
# ??? Temporary fix for T409-012 ???
if self.__build_space is not None and self.build_space.initialized:
for tmp_var in ("TMP", "TEMP", "TMPDIR"):
os.environ[tmp_var] = self.build_space.tmp_dir
try:
# If there is a pre function call it
if pre is not None:
self._pre = getattr(self, pre)()
# Run the primitive
result = f(self, *args, **kwargs)
self.log.debug("%s %s ends", self.name, f.__name__)
# And return the result
return result
except Exception:
error_msg = f"{self.name} {f.__name__} fails"
self.log.exception(error_msg)
raise AnodError(error_msg) from None
def check_api_version(version):
"""Make sure there are no API mismatch."""
if version.strip() not in SUPPORTED_API:
raise AnodError(
origin='check_api_version',
message='API version mismatch. Anod specs are at %s but the '
'driver is only supporting %s' %
(version.strip(), ','.join(SUPPORTED_API)))
def check_api_version(version):
"""Make sure there are no API mismatch."""
if version.strip() not in SUPPORTED_API:
raise AnodError(
origin="check_api_version",
message="API version mismatch. Anod specs are at %s but the "
"driver is only supporting %s" %
(version.strip(), ",".join(SUPPORTED_API)),
)
def add(self, name, value):
"""Add a fingerprint element.
:param name: name of the new element
:type name: str
:param value: associated value (should be a string)
:type value: str | unicode
:raise: AnodError
"""
if isinstance(value, str) or isinstance(value, unicode):
self.elements[name] = value
else:
raise AnodError(
'value for %s should be a string got %s' % (name, value),
'fingerprint.add')
def add_dep(spec_instance: Anod, dep: Dependency,
dep_instance: Anod) -> None:
"""Add a new dependency in an Anod instance dependencies dict.
:param spec_instance: an Anod instance
:param dep: the dependency we want to add
:param dep_instance: the Anod instance loaded for that dependency
"""
if dep.local_name in spec_instance.deps:
raise AnodError(
origin="expand_spec",
message="The spec {} has two dependencies with the same "
"local_name attribute ({})".format(spec_instance.name,
dep.local_name),
)
spec_instance.deps[dep.local_name] = dep_instance
def primitive_func(self, *args, **kwargs):
# Check whether the instance has been activated
if self.build_space is None:
# Not yet activated, fail
raise AnodError('AnodDriver.activate() has not been run')
self.log.debug("%s %s starts", self.name, f.__name__)
success = False # Set to True if no exception
result = False
# First reset last build status and actual fingerprint. This
# ensure that even a crash will keep the module in a mode that
# force its rebuild.
self.build_space.update_status(self.kind)
# Ensure temporary directory is set to a directory local to
# the current sandbox. This avoid mainly to loose track of
# temporary files that are then accumulating on the
# filesystems.
for tmp_var in ('TMP', 'TEMP', 'TMPDIR'):
os.environ[tmp_var] = self.build_space.tmp_dir
try:
result = f(self, *args, **kwargs)
except AnodError as e:
self.log.error(e)
e += '{name} {action} fails'.format(name=self.name,
action=f.__name__)
raise
except Exception:
self.log.error("%s %s fails", self.name, f.__name__)
self.build_space.dump_traceback(self.name, f.__name__)
else:
self.log.debug("%s %s ends", self.name, f.__name__)
success = True
finally:
if success:
# Don't update status or fingerprint if the primitive
# is an installation outside the build space.
self.build_space.update_status(
kind=self.kind,
status=ReturnValue.success,
fingerprint=self.fingerprint)
return result
def dump_traceback(self, spec_name, kind):
"""Dump traceback in log dir and raise an AnodError with the last line.
Traceback info is collected using the information returned by
sys.exc_info().
:param spec_name: name of the anod spec
:type spec_name: str
:param kind: anod action ('build', 'test', 'install', ...)
:type kind: str
:raise: AnodError
"""
import traceback
tb_filename = os.path.join(self.log_dir, 'traceback_%s' % kind)
_, _, exc_traceback = sys.exc_info()
with open(tb_filename, 'w') as f_tb:
for l in traceback.format_tb(exc_traceback):
f_tb.write(l + '\n')
msg = traceback.format_tb(exc_traceback)[-1]
raise AnodError('%s failed with %s\n(see %s)' %
(msg if '.anod' in msg else spec_name, sys.exc_value,
tb_filename)), None, sys.exc_traceback
def add_spec(
self,
name,
env=None,
primitive=None,
qualifier=None,
source_packages=None,
expand_build=True,
source_name=None,
plan_line=None,
plan_args=None,
sandbox=None,
upload=False,
):
"""Expand an anod action into a tree (internal).
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:type source_packages: list[str] | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
:param plan_line: corresponding line:linenumber in the plan
:type plan_line: str
:param plan_args: action args after plan execution, taking into
account plan context (such as with defaults(XXX):)
:type plan_args: dict
:param sandbox: if not None, anod instance are automatically bind to
the given sandbox
:type sandbox: None | Sandbox
:param upload: if True consider uploads to the store (sources and
binaries)
:type upload: bool
"""
def add_action(data, connect_with=None):
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
def add_dep(spec_instance, dep, dep_instance):
"""Add a new dependency in an Anod instance dependencies dict.
:param spec_instance: an Anod instance
:type spec_instance: Anod
:param dep: the dependency we want to add
:type dep: Dependency
:param dep_instance: the Anod instance loaded for that dependency
:type dep_instance: Anod
"""
if dep.local_name in spec_instance.deps:
raise AnodError(
origin="expand_spec",
message="The spec {} has two dependencies with the same "
"local_name attribute ({})".format(spec_instance.name,
dep.local_name),
)
spec_instance.deps[dep.local_name] = dep_instance
# Initialize a spec instance
e3.log.debug("add spec: name:{}, qualifier:{}, primitive:{}".format(
name, qualifier, primitive))
spec = self.load(
name,
qualifier=qualifier,
env=env,
kind=primitive,
sandbox=sandbox,
source_name=source_name,
)
# Initialize the resulting action based on the primitive name
if primitive == "source":
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb.name not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(
name=name,
env=env,
primitive="source",
source_name=sb.name,
plan_line=plan_line,
plan_args=plan_args,
sandbox=sandbox,
upload=upload,
)
self.connect(result, sub_result)
elif primitive == "build":
result = Build(spec)
elif primitive == "test":
result = Test(spec)
elif primitive == "install":
result = Install(spec)
else: # defensive code
raise ValueError("add_spec error: %s is not known" % primitive)
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if (primitive == "install" and not spec.has_package
and has_primitive(spec, "build")):
if plan_line is not None and plan_args is not None:
# We have an explicit call to install() in the plan but the
# spec has no binary package to download.
raise SchedulingError(
"error in plan at {}: "
"install should be replaced by build".format(plan_line))
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(
name,
env,
"build",
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
if expand_build and primitive == "build" and spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(
name,
env,
"install",
qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError("spec %s does not support primitive %s" %
(name, primitive))
# Add the action in the DAG
add_action(result)
if primitive == "install":
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, "build"):
build_action = self.add_spec(
name=name,
env=env,
primitive="build",
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
self.add_decision(BuildOrDownload, result, build_action,
download_action)
else:
self.connect(result, download_action)
elif primitive == "source":
if source_name is not None:
# Also add an UploadSource action
if upload:
upload_src = UploadSource(spec, source_name)
self.add(upload_src)
# Link the upload to the current context
if plan_line is not None and plan_args is not None:
self.link_to_plan(
vertex_id=upload_src.uid,
plan_line=plan_line,
plan_args=plan_args,
)
self.connect(self.root, upload_src)
self.connect(upload_src, result)
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
raise SchedulingError(
origin="add_spec",
message="unknown repository {}".format(
checkout),
)
co = Checkout(checkout,
self.repo.repos.get(checkout))
add_action(co, result)
# Look for dependencies
spec_dependencies = []
if ("%s_deps" % primitive in dir(spec)
and getattr(spec, "%s_deps" % primitive) is not None):
spec_dependencies += getattr(spec, "%s_deps" % primitive)
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == "source":
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(
e.name,
kind="source",
env=self.default_env,
qualifier=None,
sandbox=sandbox,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_instance)
self.dependencies[spec.uid][e.local_name] = (
e,
spec.deps[e.local_name],
)
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind,
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_action.anod_instance)
self.dependencies[spec.uid][e.local_name] = (
e, spec.deps[e.local_name])
if e.kind == "build" and self[
child_action.uid].data.kind == "install":
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result,
BuildOrDownload.BUILD,
plan_line
if plan_line is not None else "unknown line",
)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if "%s_source_list" % primitive in dir(spec):
source_list = getattr(spec, "{}_source_list".format(primitive))
for s in source_list:
# set source builder
if s.name in self.sources:
s.set_builder(self.sources[s.name])
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = (result.uid.rsplit(".", 1)[0] +
".source_install." + s.name)
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(
origin="expand_spec",
message="source %s does not exist "
"(referenced by %s)" % (s.name, result.uid),
)
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(
name=spec_decl,
env=self.default_env,
primitive="source",
plan_args=None,
plan_line=plan_line,
source_name=s.name,
sandbox=sandbox,
upload=upload,
)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos.get(repo))
add_action(r, connect_with=source_action)
self.add_decision(
CreateSourceOrDownload,
src_get_action,
source_action,
src_download_action,
)
return result
def build_space(self) -> BuildSpace:
if self.__build_space is None:
raise AnodError("build space not set")
return self.__build_space
def add_spec(
self,
name: str,
env: BaseEnv,
primitive: PRIMITIVE,
qualifier: Optional[str] = None,
source_packages: Optional[list[str]] = None,
expand_build: bool = True,
source_name: Optional[str] = None,
plan_line: Optional[str] = None,
plan_args: Optional[dict] = None,
sandbox: Optional[SandBox] = None,
upload: bool = False,
force_download: bool = False,
) -> Build | CreateSources | CreateSource | Install | Test:
"""Expand an anod action into a tree (internal).
:param name: spec name
:param env: context in which to load the spec
:param primitive: spec primitive
:param qualifier: qualifier
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:param expand_build: should build primitive be expanded
:param source_name: source name associated with the source
primitive
:param plan_line: corresponding line:linenumber in the plan
:param plan_args: action args after plan execution, taking into
account plan context (such as with defaults(XXX):)
:param sandbox: if not None, anod instance are automatically bind to
the given sandbox
:param upload: if True consider uploads to the store (sources and
binaries)
:param force_download: if True force a download
"""
def add_action(data: Action,
connect_with: Optional[Action] = None) -> None:
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
def add_dep(spec_instance: Anod, dep: Dependency,
dep_instance: Anod) -> None:
"""Add a new dependency in an Anod instance dependencies dict.
:param spec_instance: an Anod instance
:param dep: the dependency we want to add
:param dep_instance: the Anod instance loaded for that dependency
"""
if dep.local_name in spec_instance.deps:
raise AnodError(
origin="expand_spec",
message="The spec {} has two dependencies with the same "
"local_name attribute ({})".format(spec_instance.name,
dep.local_name),
)
spec_instance.deps[dep.local_name] = dep_instance
# Initialize a spec instance
e3.log.debug("add spec: name:{}, qualifier:{}, primitive:{}".format(
name, qualifier, primitive))
spec = self.load(
name,
qualifier=qualifier,
env=env,
kind=primitive,
sandbox=sandbox,
source_name=source_name,
)
result: Build | CreateSources | CreateSource | Install | Test
# Initialize the resulting action based on the primitive name
if primitive == "source":
if not has_primitive(spec, "source"):
raise SchedulingError(
f"spec {name} does not support primitive source")
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
if TYPE_CHECKING:
# When creating sources we know that the
# source_pkg_build attribute is set
assert spec.source_pkg_build is not None
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb.name not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(
name=name,
env=env,
primitive="source",
source_name=sb.name,
plan_line=plan_line,
plan_args=plan_args,
sandbox=sandbox,
upload=upload,
)
self.connect(result, sub_result)
elif primitive == "build":
if not has_primitive(spec, "build"):
raise SchedulingError(
f"spec {name} does not support primitive build for"
" platform {env.platform} and qualifier '{qualifier}'")
result = Build(spec)
elif primitive == "test":
result = Test(spec)
elif primitive == "install":
result = Install(spec)
else:
assert_never()
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if primitive == "install" and force_download:
# We have an "download" dependency explicit in the plan
# Make sure to record the BuildOrDownload decision
if result in self:
action_preds = self.predecessors(result)
if action_preds:
dec = action_preds[0]
if isinstance(dec, BuildOrDownload):
dec.set_decision(which=BuildOrDownload.INSTALL,
decision_maker=plan_line)
elif (primitive == "install" and not spec.has_package
and has_primitive(spec, "build") and not force_download):
if plan_line is not None and plan_args is not None:
# We have an explicit call to install() in the plan but the
# spec has no binary package to download.
raise SchedulingError(
f"error in plan at {plan_line}: "
"install should be replaced by build - "
f"the spec {spec.name} has a build primitive "
"but does not define a package")
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(
name,
env,
"build",
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
if expand_build and primitive == "build" and spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(
name,
env,
"install",
qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError(
f"spec {name} does not support primitive {primitive}")
# Add the action in the DAG
add_action(result)
if primitive == "install":
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, "build"):
build_action = self.add_spec(
name=name,
env=env,
primitive="build",
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
decision = self.add_decision(BuildOrDownload, result,
build_action, download_action)
if force_download:
decision.set_decision(which=BuildOrDownload.INSTALL,
decision_maker=plan_line)
else:
self.connect(result, download_action)
elif primitive == "source":
if source_name is not None:
# Also add an UploadSource action
if upload:
upload_src = UploadSource(spec, source_name)
self.add(upload_src)
# Link the upload to the current context
if plan_line is not None and plan_args is not None:
self.link_to_plan(
vertex_id=upload_src.uid,
plan_line=plan_line,
plan_args=plan_args,
)
self.connect(self.root, upload_src)
self.connect(upload_src, result)
if TYPE_CHECKING:
# When creating sources we know that the
# source_pkg_build attribute is set
assert spec.source_pkg_build is not None
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
raise SchedulingError(
origin="add_spec",
message=f"unknown repository {checkout}",
)
co = Checkout(checkout, self.repo.repos[checkout])
add_action(co, result)
# Look for dependencies. Consider that "None" means "no dependency".
spec_dependencies = list(
fetch_attr(spec, f"{primitive}_deps", None) or [])
source_spec_dependencies_names = {
d.name
for d in spec_dependencies if d.kind == "source"
}
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == "source":
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(
e.name,
kind="source",
env=self.default_env,
qualifier=None,
sandbox=sandbox,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_instance)
self.dependencies[spec.uid][e.local_name] = (
e,
spec.deps[e.local_name],
)
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind if e.kind != "download" else "install",
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
force_download=e.kind == "download",
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_action.anod_instance)
self.dependencies[spec.uid][e.local_name] = (
e, spec.deps[e.local_name])
if e.kind == "build" and self[
child_action.uid].data.kind == "install":
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
child_action_preds = self.predecessors(child_action)
if child_action_preds:
dec = child_action_preds[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result,
BuildOrDownload.BUILD,
plan_line
if plan_line is not None else "unknown line",
)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
source_list = fetch_attr(spec, f"{primitive}_source_list", None)
if source_list is not None:
for s in source_list:
# set source builder
if s.name in self.sources:
sb_spec, sb = self.sources[s.name]
if (sb_spec != spec.name
and sb_spec not in source_spec_dependencies_names
# ignore unmanaged source builders which do not
# create many issues (no need to find the source
# builder to apply patches, update repositories, ...)
# and this creates too many warnings in production that
# we do not have time to fix
and not isinstance(sb, UnmanagedSourceBuilder)):
logger.warning(
f"{spec.name}.anod ({primitive}): source {s.name}"
f" coming from {sb_spec} but there is no"
f" source_pkg dependency for {sb_spec} in {primitive}_deps",
)
s.set_builder(sb)
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = (result.uid.rsplit(".", 1)[0] +
".source_install." + s.name)
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(
origin="expand_spec",
message="source %s does not exist "
"(referenced by %s)" % (s.name, result.uid),
)
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(
name=spec_decl,
env=self.default_env,
primitive="source",
plan_args=None,
plan_line=plan_line,
source_name=s.name,
sandbox=sandbox,
upload=upload,
)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos[repo])
add_action(r, connect_with=source_action)
self.add_decision(
CreateSourceOrDownload,
src_get_action,
source_action,
src_download_action,
)
return result
def add_spec(self,
name,
env=None,
primitive=None,
qualifier=None,
source_packages=None,
expand_build=True,
source_name=None,
plan_line=None,
plan_args=None,
force_source_deps=None):
"""Expand an anod action into a tree (internal).
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:type source_packages: list[str] | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
:param force_source_deps: whether to force loading the source deps
:type force_source_deps: bool
"""
def add_action(data, connect_with=None):
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
# Initialize a spec instance
e3.log.debug('name:{}, qualifier:{}, primitive:{}'.format(
name, qualifier, primitive))
spec = self.load(name, qualifier=qualifier, env=env, kind=primitive)
# Initialize the resulting action based on the primitive name
if primitive == 'source':
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(name=name,
env=env,
primitive='source',
source_name=sb.name,
plan_line=plan_line,
plan_args=None)
self.connect(result, sub_result)
elif primitive == 'build':
result = Build(spec)
elif primitive == 'test':
result = Test(spec)
elif primitive == 'install':
result = Install(spec)
else: # defensive code
raise ValueError('add_spec error: %s is not known' % primitive)
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if primitive == 'install' and \
not (spec.has_package and spec.component is not None) and \
has_primitive(spec, 'build'):
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(name,
env,
'build',
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
force_source_deps=force_source_deps)
if expand_build and primitive == 'build' and \
(spec.has_package and spec.component is not None):
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(name,
env,
'install',
qualifier,
plan_args=None,
plan_line=plan_line,
force_source_deps=force_source_deps)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError('spec %s does not support primitive %s' %
(name, primitive))
# Add the action in the DAG
add_action(result)
if primitive == 'install':
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, 'build'):
build_action = self.add_spec(
name=name,
env=env,
primitive='build',
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
force_source_deps=force_source_deps)
self.add_decision(BuildOrDownload, result, build_action,
download_action)
else:
self.connect(result, download_action)
elif primitive == 'source':
if source_name is not None:
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
logger.warning('unknown repository %s',
checkout)
co = Checkout(checkout,
self.repo.repos.get(checkout))
add_action(co, result)
# Look for dependencies
spec_dependencies = []
if '%s_deps' % primitive in dir(spec) and \
getattr(spec, '%s_deps' % primitive) is not None:
spec_dependencies += getattr(spec, '%s_deps' % primitive)
if force_source_deps and primitive != 'source':
if 'source_deps' in dir(spec) and \
getattr(spec, 'source_deps') is not None:
spec_dependencies += getattr(spec, 'source_deps')
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == 'source':
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(e.name,
kind='source',
env=BaseEnv(),
qualifier=None)
spec.deps[e.local_name] = child_instance
if force_source_deps:
# When in force_source_deps we also want to add
# source_deps of all "source_pkg" dependencies.
if 'source_deps' in dir(child_instance) and \
getattr(child_instance,
'source_deps') is not None:
spec_dependencies += child_instance.source_deps
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind,
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
force_source_deps=force_source_deps)
spec.deps[e.local_name] = child_action.anod_instance
if e.kind == 'build' and \
self[child_action.uid].data.kind == 'install':
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result, BuildOrDownload.BUILD, plan_line
if plan_line is not None else 'unknown line')
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if '%s_source_list' % primitive in dir(spec):
source_list = getattr(spec, '{}_source_list'.format(primitive))
for s in source_list:
# set source builder
if s.name in self.sources:
s.set_builder(self.sources[s.name])
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = result.uid.rsplit('.', 1)[0] + \
'.source_install.' + s.name
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(origin='expand_spec',
message='source %s does not exist '
'(referenced by %s)' %
(s.name, result.uid))
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(name=spec_decl,
env=BaseEnv(),
primitive='source',
plan_args=None,
plan_line=plan_line,
source_name=s.name)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos.get(repo))
add_action(r, connect_with=source_action)
self.add_decision(CreateSourceOrDownload, src_get_action,
source_action, src_download_action)
return result
def add_spec(self,
name,
env=None,
primitive=None,
qualifier=None,
expand_build=True,
source_name=None):
"""Expand an anod action into a tree (internal).
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
"""
# Initialize a spec instance
spec = self.load(name, qualifier=qualifier, env=env, kind=primitive)
# Initialize the resulting action based on the primitive name
if primitive == 'source':
result = CreateSource(spec, source_name)
elif primitive == 'build':
result = Build(spec)
elif primitive == 'test':
result = Test(spec)
elif primitive == 'install':
result = Install(spec)
else:
raise Exception(primitive)
if not spec.has_package and primitive == 'install' and \
has_primitive(spec, 'build'):
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(name,
env,
'build',
qualifier,
expand_build=False)
if expand_build and primitive == 'build' and \
spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(name, env, 'install', qualifier)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
# Add the action in the DAG
self.add(result)
if primitive == 'install':
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
self.add(download_action)
if has_primitive(spec, 'build'):
build_action = self.add_spec(name,
env,
'build',
qualifier,
expand_build=False)
self.add_decision(BuildOrInstall, result, build_action,
download_action)
else:
self.connect(result, download_action)
# Look for dependencies
if '%s_deps' % primitive in dir(spec) and \
getattr(spec, '%s_deps' % primitive) is not None:
for e in getattr(spec, '%s_deps' % primitive):
if isinstance(e, Dependency):
if e.kind == 'source':
# A source dependency does not create a new node but
# ensure that sources associated with it are available
self.load(e.name,
kind='source',
env=BaseEnv(),
qualifier=None)
continue
child_action = self.add_spec(e.name,
e.env(spec, self.default_env),
e.kind, e.qualifier)
spec.deps[e.local_name] = result.anod_instance
if e.kind == 'build' and \
self[child_action.uid].data.kind == 'install':
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrInstall):
dec.add_trigger(result, BuildOrInstall.BUILD)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if '%s_source_list' % primitive in dir(spec):
for s in getattr(spec, '%s_source_list' % primitive):
# set source builder
if s.name in self.sources:
s.set_builder(self.sources[s.name])
# add source install node
src_install_uid = result.uid.rsplit('.', 1)[0] + \
'.source_install.' + s.name
src_install_action = InstallSource(src_install_uid, spec, s)
self.add(src_install_action)
self.connect(result, src_install_action)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(origin='expand_spec',
message='source %s does not exist '
'(referenced by %s)' %
(s.name, result.uid))
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
self.add(src_get_action)
self.connect(src_install_action, src_get_action)
src_download_action = DownloadSource(obj)
self.add(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(spec_decl,
BaseEnv(),
'source',
None,
source_name=s.name)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos[repo])
self.add(r)
self.connect(source_action, r)
self.add_decision(CreateSourceOrDownload, src_get_action,
source_action, src_download_action)
return result
