def test_primitive():
class NoPrimitive(Anod):
def build(self):
return 2
no_primitive = NoPrimitive("", "build")
assert has_primitive(no_primitive, "build") is False
class WithPrimitive(Anod):
build_qualifier_format = (("error", False),)
package = Anod.Package(prefix="mypackage", version=lambda: "42")
@Anod.primitive()
def build(self):
if "error" in self.parsed_qualifier:
raise ValueError(self.parsed_qualifier["error"])
elif "error2" in self.parsed_qualifier:
self.shell(sys.executable, "-c", "import sys; sys.exit(2)")
else:
hello = self.shell(
sys.executable, "-c", 'print("world")', output=subprocess.PIPE
)
return hello.out.strip()
with_primitive = WithPrimitive("", "build")
with_primitive2 = WithPrimitive("error=foobar", "build")
with_primitive3 = WithPrimitive("error2", "build")
with_primitive4 = WithPrimitive("error3", "build")
Anod.sandbox = SandBox(root_dir=os.getcwd())
Anod.sandbox.spec_dir = os.path.join(os.path.dirname(__file__), "data")
Anod.sandbox.create_dirs()
# Activate the logging
AnodDriver(anod_instance=with_primitive, store=None).activate(Anod.sandbox, None)
AnodDriver(anod_instance=with_primitive2, store=None).activate(Anod.sandbox, None)
AnodDriver(anod_instance=with_primitive3, store=None).activate(Anod.sandbox, None)
AnodDriver(anod_instance=with_primitive4, store=None) # don't activate
with_primitive.build_space.create()
assert has_primitive(with_primitive, "build") is True
assert with_primitive.build() == "world"
with_primitive2.build_space.create()
with pytest.raises(AnodError):
with_primitive2.build()
assert with_primitive2.package.name.startswith("mypackage")
# Check __getitem__
# PKG_DIR returns the path to the pkg directory
assert with_primitive2["PKG_DIR"].endswith("pkg")
with_primitive3.build_space.create()
with pytest.raises(AnodError):
with_primitive3.build()
def test_primitive():
class NoPrimitive(Anod):
def build(self):
return 2
no_primitive = NoPrimitive('', 'build')
assert has_primitive(no_primitive, 'build') is False
class WithPrimitive(Anod):
build_qualifier_format = (
('error', False),)
@Anod.primitive()
def build(self):
if 'error' in self.parsed_qualifier:
raise ValueError(self.parsed_qualifier['error'])
return 3
with_primitive = WithPrimitive('', 'build')
with_primitive2 = WithPrimitive('error=foobar', 'build')
tempd = tempfile.mkdtemp()
try:
Anod.sandbox = SandBox()
Anod.sandbox.root_dir = tempd
Anod.sandbox.spec_dir = os.path.join(
os.path.dirname(__file__),
'data')
Anod.sandbox.create_dirs()
# Activate the logging
AnodDriver(anod_instance=with_primitive, store=None).activate()
AnodDriver(anod_instance=with_primitive2, store=None).activate()
with_primitive.build_space.create()
assert has_primitive(with_primitive, 'build') is True
assert with_primitive.build() == 3
with_primitive2.build_space.create()
with pytest.raises(AnodError) as err:
with_primitive2.build()
assert 'foobar' in err.message
finally:
e3.fs.rm(tempd, True)
def test_primitive():
class NoPrimitive(Anod):
def build(self):
return 2
no_primitive = NoPrimitive("", "build")
assert has_primitive(no_primitive, "build") is False
class WithPrimitive(Anod):
build_qualifier_format = (("error", False),)
@Anod.primitive()
def build(self):
if "error" in self.parsed_qualifier:
raise ValueError(self.parsed_qualifier["error"])
return 3
with_primitive = WithPrimitive("", "build")
with_primitive2 = WithPrimitive("error=foobar", "build")
Anod.sandbox = SandBox()
Anod.sandbox.root_dir = os.getcwd()
Anod.sandbox.spec_dir = os.path.join(os.path.dirname(__file__), "data")
Anod.sandbox.create_dirs()
# Activate the logging
AnodDriver(anod_instance=with_primitive, store=None).activate()
AnodDriver(anod_instance=with_primitive2, store=None).activate()
with_primitive.build_space.create()
assert has_primitive(with_primitive, "build") is True
assert with_primitive.build() == 3
with_primitive2.build_space.create()
with pytest.raises(AnodError) as err:
with_primitive2.build()
assert "foobar" in str(err.value)
def add_spec(self,
name,
env=None,
primitive=None,
qualifier=None,
expand_build=True,
source_name=None):
"""Internal function.
The function expand an anod action into a tree
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
"""
# Initialize a spec instance
spec = self.load(name, qualifier=qualifier, env=env, kind=primitive)
# Initialize the resulting action based on the primitive name
if primitive == 'source':
result = CreateSource(spec, source_name)
elif primitive == 'build':
result = Build(spec)
elif primitive == 'test':
result = Test(spec)
elif primitive == 'install':
result = Install(spec)
else:
raise Exception(primitive)
if not spec.has_package and primitive == 'install' and \
has_primitive(spec, 'build'):
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(name, env, 'build',
qualifier,
expand_build=False)
if expand_build and primitive == 'build' and \
spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(name, env, 'install', qualifier)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
# Add the action in the DAG
self.add(result)
if primitive == 'install':
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
self.add(download_action)
if has_primitive(spec, 'build'):
build_action = self.add_spec(name,
env,
'build',
qualifier,
expand_build=False)
self.add_decision(BuildOrInstall,
result,
build_action,
download_action)
else:
self.connect(result, download_action)
# Look for dependencies
if '%s_deps' % primitive in dir(spec):
for e in getattr(spec, '%s_deps' % primitive):
if isinstance(e, Dependency):
if e.kind == 'source':
# A source dependency does not create a new node but
# ensure that sources associated with it are available
self.load(e.name, kind='source',
env=BaseEnv(), qualifier=None)
continue
child_action = self.add_spec(e.name,
e.env(spec, self.default_env),
e.kind,
e.qualifier)
if e.kind == 'build' and \
self[child_action.uid].data.kind == 'install':
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrInstall):
dec.add_trigger(result, BuildOrInstall.BUILD)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if '%s_source_list' % primitive in dir(spec):
for s in getattr(spec, '%s_source_list' % primitive):
# add source install node
src_install_uid = result.uid.rsplit('.', 1)[0] + \
'.source_install.' + s.name
src_install_action = InstallSource(src_install_uid, s)
self.add(src_install_action)
self.connect(result, src_install_action)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(
origin='expand_spec',
message='source %s does not exist '
'(referenced by %s)' % (s.name, result.uid))
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
self.add(src_get_action)
self.connect(src_install_action, src_get_action)
src_download_action = DownloadSource(obj)
self.add(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(spec_decl,
BaseEnv(),
'source',
None,
source_name=s.name)
for repo in obj.checkout:
r = Checkout(repo)
self.add(r)
self.connect(source_action, r)
self.add_decision(CreateSourceOrDownload,
src_get_action,
source_action,
src_download_action)
return result
def add_spec(
self,
name,
env=None,
primitive=None,
qualifier=None,
source_packages=None,
expand_build=True,
source_name=None,
plan_line=None,
plan_args=None,
sandbox=None,
upload=False,
):
"""Expand an anod action into a tree (internal).
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:type source_packages: list[str] | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
:param plan_line: corresponding line:linenumber in the plan
:type plan_line: str
:param plan_args: action args after plan execution, taking into
account plan context (such as with defaults(XXX):)
:type plan_args: dict
:param sandbox: if not None, anod instance are automatically bind to
the given sandbox
:type sandbox: None | Sandbox
:param upload: if True consider uploads to the store (sources and
binaries)
:type upload: bool
"""
def add_action(data, connect_with=None):
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
def add_dep(spec_instance, dep, dep_instance):
"""Add a new dependency in an Anod instance dependencies dict.
:param spec_instance: an Anod instance
:type spec_instance: Anod
:param dep: the dependency we want to add
:type dep: Dependency
:param dep_instance: the Anod instance loaded for that dependency
:type dep_instance: Anod
"""
if dep.local_name in spec_instance.deps:
raise AnodError(
origin="expand_spec",
message="The spec {} has two dependencies with the same "
"local_name attribute ({})".format(spec_instance.name,
dep.local_name),
)
spec_instance.deps[dep.local_name] = dep_instance
# Initialize a spec instance
e3.log.debug("add spec: name:{}, qualifier:{}, primitive:{}".format(
name, qualifier, primitive))
spec = self.load(
name,
qualifier=qualifier,
env=env,
kind=primitive,
sandbox=sandbox,
source_name=source_name,
)
# Initialize the resulting action based on the primitive name
if primitive == "source":
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb.name not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(
name=name,
env=env,
primitive="source",
source_name=sb.name,
plan_line=plan_line,
plan_args=plan_args,
sandbox=sandbox,
upload=upload,
)
self.connect(result, sub_result)
elif primitive == "build":
result = Build(spec)
elif primitive == "test":
result = Test(spec)
elif primitive == "install":
result = Install(spec)
else: # defensive code
raise ValueError("add_spec error: %s is not known" % primitive)
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if (primitive == "install" and not spec.has_package
and has_primitive(spec, "build")):
if plan_line is not None and plan_args is not None:
# We have an explicit call to install() in the plan but the
# spec has no binary package to download.
raise SchedulingError(
"error in plan at {}: "
"install should be replaced by build".format(plan_line))
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(
name,
env,
"build",
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
if expand_build and primitive == "build" and spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(
name,
env,
"install",
qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError("spec %s does not support primitive %s" %
(name, primitive))
# Add the action in the DAG
add_action(result)
if primitive == "install":
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, "build"):
build_action = self.add_spec(
name=name,
env=env,
primitive="build",
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
self.add_decision(BuildOrDownload, result, build_action,
download_action)
else:
self.connect(result, download_action)
elif primitive == "source":
if source_name is not None:
# Also add an UploadSource action
if upload:
upload_src = UploadSource(spec, source_name)
self.add(upload_src)
# Link the upload to the current context
if plan_line is not None and plan_args is not None:
self.link_to_plan(
vertex_id=upload_src.uid,
plan_line=plan_line,
plan_args=plan_args,
)
self.connect(self.root, upload_src)
self.connect(upload_src, result)
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
raise SchedulingError(
origin="add_spec",
message="unknown repository {}".format(
checkout),
)
co = Checkout(checkout,
self.repo.repos.get(checkout))
add_action(co, result)
# Look for dependencies
spec_dependencies = []
if ("%s_deps" % primitive in dir(spec)
and getattr(spec, "%s_deps" % primitive) is not None):
spec_dependencies += getattr(spec, "%s_deps" % primitive)
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == "source":
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(
e.name,
kind="source",
env=self.default_env,
qualifier=None,
sandbox=sandbox,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_instance)
self.dependencies[spec.uid][e.local_name] = (
e,
spec.deps[e.local_name],
)
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind,
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_action.anod_instance)
self.dependencies[spec.uid][e.local_name] = (
e, spec.deps[e.local_name])
if e.kind == "build" and self[
child_action.uid].data.kind == "install":
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result,
BuildOrDownload.BUILD,
plan_line
if plan_line is not None else "unknown line",
)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if "%s_source_list" % primitive in dir(spec):
source_list = getattr(spec, "{}_source_list".format(primitive))
for s in source_list:
# set source builder
if s.name in self.sources:
s.set_builder(self.sources[s.name])
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = (result.uid.rsplit(".", 1)[0] +
".source_install." + s.name)
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(
origin="expand_spec",
message="source %s does not exist "
"(referenced by %s)" % (s.name, result.uid),
)
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(
name=spec_decl,
env=self.default_env,
primitive="source",
plan_args=None,
plan_line=plan_line,
source_name=s.name,
sandbox=sandbox,
upload=upload,
)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos.get(repo))
add_action(r, connect_with=source_action)
self.add_decision(
CreateSourceOrDownload,
src_get_action,
source_action,
src_download_action,
)
return result
def test_primitive():
class NoPrimitive(Anod):
def build(self):
return 2
no_primitive = NoPrimitive('', 'build')
assert has_primitive(no_primitive, 'build') is False
class WithPrimitive(Anod):
build_qualifier_format = (('error', False), )
package = Anod.Package(prefix='mypackage', version=lambda: '42')
@Anod.primitive()
def build(self):
if 'error' in self.parsed_qualifier:
raise ValueError(self.parsed_qualifier['error'])
elif 'error2' in self.parsed_qualifier:
self.shell(sys.executable, '-c', 'import sys; sys.exit(2)')
else:
hello = self.shell(sys.executable,
'-c',
'print("world")',
output=subprocess.PIPE)
return hello.out.strip()
with_primitive = WithPrimitive('', 'build')
with_primitive2 = WithPrimitive('error=foobar', 'build')
with_primitive3 = WithPrimitive('error2', 'build')
with_primitive4 = WithPrimitive('error3', 'build')
Anod.sandbox = SandBox()
Anod.sandbox.root_dir = os.getcwd()
Anod.sandbox.spec_dir = os.path.join(os.path.dirname(__file__), 'data')
Anod.sandbox.create_dirs()
# Activate the logging
AnodDriver(anod_instance=with_primitive, store=None).activate()
AnodDriver(anod_instance=with_primitive2, store=None).activate()
AnodDriver(anod_instance=with_primitive3, store=None).activate()
AnodDriver(anod_instance=with_primitive4, store=None) # don't activate
with_primitive.build_space.create()
assert has_primitive(with_primitive, 'build') is True
assert with_primitive.build() == 'world'
assert with_primitive.has_nsis is False
with_primitive2.build_space.create()
with pytest.raises(AnodError) as err:
with_primitive2.build()
assert 'foobar' in str(err.value)
assert with_primitive2.package.name.startswith('mypackage')
# Check __getitem__
# PKG_DIR returns the path to the pkg directory
assert with_primitive2['PKG_DIR'].endswith('pkg')
# Check access to build_space config dict directly in Anod instance
with_primitive2.build_space.config['config-key'] = 'config-value'
assert with_primitive2['config-key'] == 'config-value'
with_primitive3.build_space.create()
with pytest.raises(ShellError) as err:
with_primitive3.build()
assert 'build fails' in str(err.value)
with_primitive3.build_space.set_logging()
with pytest.raises(ShellError) as err:
with_primitive3.build()
assert 'build fails' in str(err.value)
with open(with_primitive3.build_space.log_file) as f:
assert 'import sys; sys.exit(2)' in f.read()
with_primitive3.build_space.end()
with pytest.raises(AnodError) as err:
with_primitive4.build()
assert 'AnodDriver.activate() has not been run' in str(err)
def add_spec(
self,
name: str,
env: BaseEnv,
primitive: PRIMITIVE,
qualifier: Optional[str] = None,
source_packages: Optional[list[str]] = None,
expand_build: bool = True,
source_name: Optional[str] = None,
plan_line: Optional[str] = None,
plan_args: Optional[dict] = None,
sandbox: Optional[SandBox] = None,
upload: bool = False,
force_download: bool = False,
) -> Build | CreateSources | CreateSource | Install | Test:
"""Expand an anod action into a tree (internal).
:param name: spec name
:param env: context in which to load the spec
:param primitive: spec primitive
:param qualifier: qualifier
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:param expand_build: should build primitive be expanded
:param source_name: source name associated with the source
primitive
:param plan_line: corresponding line:linenumber in the plan
:param plan_args: action args after plan execution, taking into
account plan context (such as with defaults(XXX):)
:param sandbox: if not None, anod instance are automatically bind to
the given sandbox
:param upload: if True consider uploads to the store (sources and
binaries)
:param force_download: if True force a download
"""
def add_action(data: Action,
connect_with: Optional[Action] = None) -> None:
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
def add_dep(spec_instance: Anod, dep: Dependency,
dep_instance: Anod) -> None:
"""Add a new dependency in an Anod instance dependencies dict.
:param spec_instance: an Anod instance
:param dep: the dependency we want to add
:param dep_instance: the Anod instance loaded for that dependency
"""
if dep.local_name in spec_instance.deps:
raise AnodError(
origin="expand_spec",
message="The spec {} has two dependencies with the same "
"local_name attribute ({})".format(spec_instance.name,
dep.local_name),
)
spec_instance.deps[dep.local_name] = dep_instance
# Initialize a spec instance
e3.log.debug("add spec: name:{}, qualifier:{}, primitive:{}".format(
name, qualifier, primitive))
spec = self.load(
name,
qualifier=qualifier,
env=env,
kind=primitive,
sandbox=sandbox,
source_name=source_name,
)
result: Build | CreateSources | CreateSource | Install | Test
# Initialize the resulting action based on the primitive name
if primitive == "source":
if not has_primitive(spec, "source"):
raise SchedulingError(
f"spec {name} does not support primitive source")
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
if TYPE_CHECKING:
# When creating sources we know that the
# source_pkg_build attribute is set
assert spec.source_pkg_build is not None
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb.name not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(
name=name,
env=env,
primitive="source",
source_name=sb.name,
plan_line=plan_line,
plan_args=plan_args,
sandbox=sandbox,
upload=upload,
)
self.connect(result, sub_result)
elif primitive == "build":
if not has_primitive(spec, "build"):
raise SchedulingError(
f"spec {name} does not support primitive build for"
" platform {env.platform} and qualifier '{qualifier}'")
result = Build(spec)
elif primitive == "test":
result = Test(spec)
elif primitive == "install":
result = Install(spec)
else:
assert_never()
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if primitive == "install" and force_download:
# We have an "download" dependency explicit in the plan
# Make sure to record the BuildOrDownload decision
if result in self:
action_preds = self.predecessors(result)
if action_preds:
dec = action_preds[0]
if isinstance(dec, BuildOrDownload):
dec.set_decision(which=BuildOrDownload.INSTALL,
decision_maker=plan_line)
elif (primitive == "install" and not spec.has_package
and has_primitive(spec, "build") and not force_download):
if plan_line is not None and plan_args is not None:
# We have an explicit call to install() in the plan but the
# spec has no binary package to download.
raise SchedulingError(
f"error in plan at {plan_line}: "
"install should be replaced by build - "
f"the spec {spec.name} has a build primitive "
"but does not define a package")
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(
name,
env,
"build",
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
if expand_build and primitive == "build" and spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(
name,
env,
"install",
qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError(
f"spec {name} does not support primitive {primitive}")
# Add the action in the DAG
add_action(result)
if primitive == "install":
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, "build"):
build_action = self.add_spec(
name=name,
env=env,
primitive="build",
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
)
decision = self.add_decision(BuildOrDownload, result,
build_action, download_action)
if force_download:
decision.set_decision(which=BuildOrDownload.INSTALL,
decision_maker=plan_line)
else:
self.connect(result, download_action)
elif primitive == "source":
if source_name is not None:
# Also add an UploadSource action
if upload:
upload_src = UploadSource(spec, source_name)
self.add(upload_src)
# Link the upload to the current context
if plan_line is not None and plan_args is not None:
self.link_to_plan(
vertex_id=upload_src.uid,
plan_line=plan_line,
plan_args=plan_args,
)
self.connect(self.root, upload_src)
self.connect(upload_src, result)
if TYPE_CHECKING:
# When creating sources we know that the
# source_pkg_build attribute is set
assert spec.source_pkg_build is not None
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
raise SchedulingError(
origin="add_spec",
message=f"unknown repository {checkout}",
)
co = Checkout(checkout, self.repo.repos[checkout])
add_action(co, result)
# Look for dependencies. Consider that "None" means "no dependency".
spec_dependencies = list(
fetch_attr(spec, f"{primitive}_deps", None) or [])
source_spec_dependencies_names = {
d.name
for d in spec_dependencies if d.kind == "source"
}
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == "source":
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(
e.name,
kind="source",
env=self.default_env,
qualifier=None,
sandbox=sandbox,
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_instance)
self.dependencies[spec.uid][e.local_name] = (
e,
spec.deps[e.local_name],
)
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind if e.kind != "download" else "install",
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
sandbox=sandbox,
upload=upload,
force_download=e.kind == "download",
)
add_dep(spec_instance=spec,
dep=e,
dep_instance=child_action.anod_instance)
self.dependencies[spec.uid][e.local_name] = (
e, spec.deps[e.local_name])
if e.kind == "build" and self[
child_action.uid].data.kind == "install":
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
child_action_preds = self.predecessors(child_action)
if child_action_preds:
dec = child_action_preds[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result,
BuildOrDownload.BUILD,
plan_line
if plan_line is not None else "unknown line",
)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
source_list = fetch_attr(spec, f"{primitive}_source_list", None)
if source_list is not None:
for s in source_list:
# set source builder
if s.name in self.sources:
sb_spec, sb = self.sources[s.name]
if (sb_spec != spec.name
and sb_spec not in source_spec_dependencies_names
# ignore unmanaged source builders which do not
# create many issues (no need to find the source
# builder to apply patches, update repositories, ...)
# and this creates too many warnings in production that
# we do not have time to fix
and not isinstance(sb, UnmanagedSourceBuilder)):
logger.warning(
f"{spec.name}.anod ({primitive}): source {s.name}"
f" coming from {sb_spec} but there is no"
f" source_pkg dependency for {sb_spec} in {primitive}_deps",
)
s.set_builder(sb)
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = (result.uid.rsplit(".", 1)[0] +
".source_install." + s.name)
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(
origin="expand_spec",
message="source %s does not exist "
"(referenced by %s)" % (s.name, result.uid),
)
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(
name=spec_decl,
env=self.default_env,
primitive="source",
plan_args=None,
plan_line=plan_line,
source_name=s.name,
sandbox=sandbox,
upload=upload,
)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos[repo])
add_action(r, connect_with=source_action)
self.add_decision(
CreateSourceOrDownload,
src_get_action,
source_action,
src_download_action,
)
return result
def add_spec(self,
name,
env=None,
primitive=None,
qualifier=None,
source_packages=None,
expand_build=True,
source_name=None,
plan_line=None,
plan_args=None,
force_source_deps=None):
"""Expand an anod action into a tree (internal).
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param source_packages: if not empty only create the specified list of
source packages and not all source packages defined in the anod
specification file
:type source_packages: list[str] | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
:param force_source_deps: whether to force loading the source deps
:type force_source_deps: bool
"""
def add_action(data, connect_with=None):
self.add(data)
if connect_with is not None:
self.connect(connect_with, data)
# Initialize a spec instance
e3.log.debug('name:{}, qualifier:{}, primitive:{}'.format(
name, qualifier, primitive))
spec = self.load(name, qualifier=qualifier, env=env, kind=primitive)
# Initialize the resulting action based on the primitive name
if primitive == 'source':
if source_name is not None:
result = CreateSource(spec, source_name)
else:
# Create the root node
result = CreateSources(spec)
# A consequence of calling add_action here
# will result in skipping dependencies parsing.
add_action(result)
# Then one node for each source package
for sb in spec.source_pkg_build:
if source_packages and sb not in source_packages:
# This source package is defined in the spec but
# explicitly excluded in the plan
continue
if isinstance(sb, UnmanagedSourceBuilder):
# do not create source package for unmanaged source
continue
sub_result = self.add_spec(name=name,
env=env,
primitive='source',
source_name=sb.name,
plan_line=plan_line,
plan_args=None)
self.connect(result, sub_result)
elif primitive == 'build':
result = Build(spec)
elif primitive == 'test':
result = Test(spec)
elif primitive == 'install':
result = Install(spec)
else: # defensive code
raise ValueError('add_spec error: %s is not known' % primitive)
# If this action is directly linked with a plan line make sure
# to register the link between the action and the plan even
# if the action has already been added via another dependency
if plan_line is not None and plan_args is not None:
self.link_to_plan(vertex_id=result.uid,
plan_line=plan_line,
plan_args=plan_args)
if primitive == 'install' and \
not (spec.has_package and spec.component is not None) and \
has_primitive(spec, 'build'):
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(name,
env,
'build',
qualifier,
expand_build=False,
plan_args=plan_args,
plan_line=plan_line,
force_source_deps=force_source_deps)
if expand_build and primitive == 'build' and \
(spec.has_package and spec.component is not None):
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(name,
env,
'install',
qualifier,
plan_args=None,
plan_line=plan_line,
force_source_deps=force_source_deps)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
if not has_primitive(spec, primitive):
raise SchedulingError('spec %s does not support primitive %s' %
(name, primitive))
# Add the action in the DAG
add_action(result)
if primitive == 'install':
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
add_action(download_action)
if has_primitive(spec, 'build'):
build_action = self.add_spec(
name=name,
env=env,
primitive='build',
qualifier=qualifier,
expand_build=False,
plan_args=None,
plan_line=plan_line,
force_source_deps=force_source_deps)
self.add_decision(BuildOrDownload, result, build_action,
download_action)
else:
self.connect(result, download_action)
elif primitive == 'source':
if source_name is not None:
for sb in spec.source_pkg_build:
if sb.name == source_name:
for checkout in sb.checkout:
if checkout not in self.repo.repos:
logger.warning('unknown repository %s',
checkout)
co = Checkout(checkout,
self.repo.repos.get(checkout))
add_action(co, result)
# Look for dependencies
spec_dependencies = []
if '%s_deps' % primitive in dir(spec) and \
getattr(spec, '%s_deps' % primitive) is not None:
spec_dependencies += getattr(spec, '%s_deps' % primitive)
if force_source_deps and primitive != 'source':
if 'source_deps' in dir(spec) and \
getattr(spec, 'source_deps') is not None:
spec_dependencies += getattr(spec, 'source_deps')
for e in spec_dependencies:
if isinstance(e, Dependency):
if e.kind == 'source':
# A source dependency does not create a new node but
# ensure that sources associated with it are available
child_instance = self.load(e.name,
kind='source',
env=BaseEnv(),
qualifier=None)
spec.deps[e.local_name] = child_instance
if force_source_deps:
# When in force_source_deps we also want to add
# source_deps of all "source_pkg" dependencies.
if 'source_deps' in dir(child_instance) and \
getattr(child_instance,
'source_deps') is not None:
spec_dependencies += child_instance.source_deps
continue
child_action = self.add_spec(
name=e.name,
env=e.env(spec, self.default_env),
primitive=e.kind,
qualifier=e.qualifier,
plan_args=None,
plan_line=plan_line,
force_source_deps=force_source_deps)
spec.deps[e.local_name] = child_action.anod_instance
if e.kind == 'build' and \
self[child_action.uid].data.kind == 'install':
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrDownload):
dec.add_trigger(
result, BuildOrDownload.BUILD, plan_line
if plan_line is not None else 'unknown line')
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if '%s_source_list' % primitive in dir(spec):
source_list = getattr(spec, '{}_source_list'.format(primitive))
for s in source_list:
# set source builder
if s.name in self.sources:
s.set_builder(self.sources[s.name])
# set other sources to compute source ignore
s.set_other_sources(source_list)
# add source install node
src_install_uid = result.uid.rsplit('.', 1)[0] + \
'.source_install.' + s.name
src_install_action = InstallSource(src_install_uid, spec, s)
add_action(src_install_action, connect_with=result)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(origin='expand_spec',
message='source %s does not exist '
'(referenced by %s)' %
(s.name, result.uid))
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
add_action(src_get_action, connect_with=src_install_action)
src_download_action = DownloadSource(obj)
add_action(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(name=spec_decl,
env=BaseEnv(),
primitive='source',
plan_args=None,
plan_line=plan_line,
source_name=s.name)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos.get(repo))
add_action(r, connect_with=source_action)
self.add_decision(CreateSourceOrDownload, src_get_action,
source_action, src_download_action)
return result
def wrapper(self, *args, **kwargs):
if not has_primitive(self.anod_instance, func.__name__):
raise AnodError('no primitive %s' % func.__name__)
elif self.anod_instance.anod_id is None:
raise AnodError('.activate() has not been called')
return func(self, *args, **kwargs)
def add_spec(self,
name,
env=None,
primitive=None,
qualifier=None,
expand_build=True,
source_name=None):
"""Expand an anod action into a tree (internal).
:param name: spec name
:type name: str
:param env: spec environment
:type env: BaseEnv | None
:param primitive: spec primitive
:type primitive: str
:param qualifier: qualifier
:type qualifier: str | None
:param expand_build: should build primitive be expanded
:type expand_build: bool
:param source_name: source name associated with the source
primitive
:type source_name: str | None
"""
# Initialize a spec instance
spec = self.load(name, qualifier=qualifier, env=env, kind=primitive)
# Initialize the resulting action based on the primitive name
if primitive == 'source':
result = CreateSource(spec, source_name)
elif primitive == 'build':
result = Build(spec)
elif primitive == 'test':
result = Test(spec)
elif primitive == 'install':
result = Install(spec)
else:
raise Exception(primitive)
if not spec.has_package and primitive == 'install' and \
has_primitive(spec, 'build'):
# Case in which we have an install dependency but no install
# primitive. In that case the real dependency is a build tree
# dependency. In case there is no build primitive and no
# package keep the install primitive (usually this means there
# is an overloaded download procedure).
return self.add_spec(name,
env,
'build',
qualifier,
expand_build=False)
if expand_build and primitive == 'build' and \
spec.has_package:
# A build primitive is required and the spec defined a binary
# package. In that case the implicit post action of the build
# will be a call to the install primitive
return self.add_spec(name, env, 'install', qualifier)
# Add this stage if the action is already in the DAG, then it has
# already been added.
if result in self:
return result
# Add the action in the DAG
self.add(result)
if primitive == 'install':
# Expand an install node to
# install --> decision --> build
# \-> download binary
download_action = DownloadBinary(spec)
self.add(download_action)
if has_primitive(spec, 'build'):
build_action = self.add_spec(name,
env,
'build',
qualifier,
expand_build=False)
self.add_decision(BuildOrInstall, result, build_action,
download_action)
else:
self.connect(result, download_action)
# Look for dependencies
if '%s_deps' % primitive in dir(spec) and \
getattr(spec, '%s_deps' % primitive) is not None:
for e in getattr(spec, '%s_deps' % primitive):
if isinstance(e, Dependency):
if e.kind == 'source':
# A source dependency does not create a new node but
# ensure that sources associated with it are available
self.load(e.name,
kind='source',
env=BaseEnv(),
qualifier=None)
continue
child_action = self.add_spec(e.name,
e.env(spec, self.default_env),
e.kind, e.qualifier)
spec.deps[e.local_name] = result.anod_instance
if e.kind == 'build' and \
self[child_action.uid].data.kind == 'install':
# We have a build tree dependency that produced a
# subtree starting with an install node. In that case
# we expect the user to choose BUILD as decision.
dec = self.predecessors(child_action)[0]
if isinstance(dec, BuildOrInstall):
dec.add_trigger(result, BuildOrInstall.BUILD)
# Connect child dependency
self.connect(result, child_action)
# Look for source dependencies (i.e sources needed)
if '%s_source_list' % primitive in dir(spec):
for s in getattr(spec, '%s_source_list' % primitive):
# set source builder
if s.name in self.sources:
s.set_builder(self.sources[s.name])
# add source install node
src_install_uid = result.uid.rsplit('.', 1)[0] + \
'.source_install.' + s.name
src_install_action = InstallSource(src_install_uid, spec, s)
self.add(src_install_action)
self.connect(result, src_install_action)
# Then add nodes to create that source (download or creation
# using anod source and checkouts)
if s.name in self.sources:
spec_decl, obj = self.sources[s.name]
else:
raise AnodError(origin='expand_spec',
message='source %s does not exist '
'(referenced by %s)' %
(s.name, result.uid))
src_get_action = GetSource(obj)
if src_get_action in self:
self.connect(src_install_action, src_get_action)
continue
self.add(src_get_action)
self.connect(src_install_action, src_get_action)
src_download_action = DownloadSource(obj)
self.add(src_download_action)
if isinstance(obj, UnmanagedSourceBuilder):
# In that case only download is available
self.connect(src_get_action, src_download_action)
else:
source_action = self.add_spec(spec_decl,
BaseEnv(),
'source',
None,
source_name=s.name)
for repo in obj.checkout:
r = Checkout(repo, self.repo.repos[repo])
self.add(r)
self.connect(source_action, r)
self.add_decision(CreateSourceOrDownload, src_get_action,
source_action, src_download_action)
return result
def wrapper(self, *args, **kwargs):
if not has_primitive(self.anod_instance, func.__name__):
raise AnodError("no primitive %s" % func.__name__)
elif self.anod_instance.build_space is None:
raise AnodError(".activate() has not been called")
return func(self, *args, **kwargs)