def load(self, repository):
"""Load an anod specification.
:param repository: the anod spec repository of the spec file
:type repository: AnodSpecRepository
:raise SandBoxError: in case of failure
"""
if self.is_loaded:
return self.anod_class
logger.debug('loading anod spec: %s', self.name)
# Create a new module
mod_name = 'anod_' + self.name
anod_module = imp.new_module(mod_name)
try:
with open(self.path) as fd:
# Inject the prolog into the new module dict
anod_module.__dict__.update(repository.prolog_dict)
# Exec spec code
code = compile(fd.read(), self.path, 'exec')
exec(code, anod_module.__dict__)
except Exception as e:
logger.error('exception: %s', e)
logger.error('cannot load code of %s', self.name)
raise SandBoxError(
origin='load',
message='invalid spec code for %s' % self.name), \
None, sys.exc_traceback
# At this stage we have loaded completely the module. Now we need to
# look for a subclass of Anod. Use python inspection features to
# achieve this.
for members in inspect.getmembers(anod_module):
_, value = members
# Return the first subclass of Anod defined in this module
if inspect.isclass(value) and value.__module__ == mod_name \
and 'Anod' in [k.__name__ for k in value.__mro__]:
# This class is a child of Anod so register it.
# Note that even if we won't use directly the
# module we need to keep a reference on it in order
# to avoid garbage collector issues.
value.spec_checksum = self.checksum
# Give a name to our Anod class: the basename of the
# anod spec file (without the .anod extension)
value.name = self.name
self.anod_class = value
self.module = anod_module
self.anod_class.data_files = self.data
self.anod_class.spec_dir = os.path.dirname(self.path)
return self.anod_class
logger.error('spec %s does not contains an Anod subclass', self.name)
raise SandBoxError('cannot find Anod subclass in %s' % self.path,
'load')
def __init__(self, spec_dir, spec_config=None):
"""Initialize an AnodSpecRepository.
:param spec_dir: directory containing the anod specs.
:type spec_dir: str
:param spec_config: dictionary containing the configuration for this
AnodSpecRepository
:type spec_config: dict
"""
logger.debug('initialize spec repository (%s)', spec_dir)
if not os.path.isdir(spec_dir):
raise SandBoxError('spec directory %s does not exist' % spec_dir)
self.spec_dir = spec_dir
self.api_version = None
self.specs = {}
self.repos = {}
# Look for all spec files and data files
spec_list = {
os.path.basename(k)[:-5]: {
'path': k,
'data': []
}
for k in ls(os.path.join(self.spec_dir, '*.anod'),
emit_log_record=False)
}
logger.debug('found %s specs', len(spec_list))
data_list = [
os.path.basename(k)[:-5]
for k in ls(os.path.join(self.spec_dir, '*.yaml'),
emit_log_record=False)
]
logger.debug('found %s yaml files', len(data_list))
# Match yaml files with associated specifications
for data in data_list:
candidate_specs = [
spec_file for spec_file in spec_list
if data.startswith(spec_file)
]
# We pick the longuest spec name
candidate_specs.sort(key=lambda x: len(x))
if candidate_specs:
spec_list[candidate_specs[-1]]['data'].append(data)
# Create AnodModule objects
for name, value in spec_list.iteritems():
self.specs[name] = AnodModule(name, **value)
# Declare spec prolog
prolog_file = os.path.join(spec_dir, 'prolog.py')
self.prolog_dict = {
'spec_config': spec_config,
'__spec_repository': self
}
if os.path.exists(prolog_file):
with open(prolog_file) as f:
exec(compile(f.read(), prolog_file, 'exec'), self.prolog_dict)
def root_dir(self) -> str:
"""Root path of the sandbox.
:raise SandBoxError: when the sandbox is not initialized
"""
if self.__root_dir is None:
raise SandBoxError(
origin="root_dir",
message="sandbox not loaded. Please call load()")
return self.__root_dir
def root_dir(self):
"""Root path of the sandbox.
:raise SandBoxError: when the sandbox is not initialized
:rtype: str
"""
if self.__root_dir is None:
raise SandBoxError(
origin='root_dir',
message='sandbox not loaded. Please call load()')
return self.__root_dir
def main(get_argument_parser=False):
"""Manipulate an Anod sandbox.
This function creates the main code for the entry-point e3-sandbox. To
create new actions it is possible to create new sandbox plugins. e.g. to
add a new plugin ``foo`` from a package ``e3-contrib``, derives the
class :class:`SandBoxAction` and register the extension by adding in
:file:`e3-contrib/setup.py`::
entry_points={
'e3.anod.sandbox.sandbox_action': [
'foo = e3_contrib.sandbox_actions.SandBoxFoo']
}
:param get_argument_parser: return e3.main.Main argument_parser instead
of running the action.
:type get_argument_parser: bool
"""
m = Main()
m.parse_args(known_args_only=True)
subparsers = m.argument_parser.add_subparsers(title="action",
description="valid actions")
# Load all sandbox actions plugins
ext = stevedore.ExtensionManager(
namespace="e3.anod.sandbox.sandbox_action",
invoke_on_load=True,
invoke_args=(subparsers, ),
)
if len(ext.names()) != len(ext.entry_points_names()):
raise SandBoxError(
"an error occured when loading sandbox_action entry points %s" %
",".join(ext.entry_points_names())) # defensive code
if get_argument_parser:
return m.argument_parser
args = m.argument_parser.parse_args()
e3.log.debug("sandbox action plugins loaded: %s", ",".join(ext.names()))
# An action has been selected, run it
try:
ext[args.action].obj.run(args)
except SandBoxError as err:
logger.error(err)
sys.exit(1)
def __init__(self, spec_dir: str, spec_config: Optional[dict] = None):
"""Initialize an AnodSpecRepository.
:param spec_dir: directory containing the anod specs.
:param spec_config: dictionary containing the configuration for this
AnodSpecRepository
"""
logger.debug("initialize spec repository (%s)", spec_dir)
if not os.path.isdir(spec_dir):
raise SandBoxError("spec directory %s does not exist" % spec_dir)
self.spec_dir = spec_dir
self.api_version = __version__
self.specs = {}
self.repos: Dict[str, Dict[str, str]] = {}
# Look for all spec files and data files
spec_list = {
os.path.basename(os.path.splitext(k)[0]): {
"path": k,
"data": []
}
for k in ls(os.path.join(self.spec_dir, "*.anod"),
emit_log_record=False)
}
logger.debug("found %s specs", len(spec_list))
# API == 1.4
yaml_files = ls(os.path.join(self.spec_dir, "*.yaml"),
emit_log_record=False)
data_list = [os.path.basename(k)[:-5] for k in yaml_files]
logger.debug("found %s yaml files API 1.4 compatible", len(data_list))
# Match yaml files with associated specifications
for data in data_list:
candidate_specs = [
spec_file for spec_file in spec_list
if data.startswith(spec_file)
]
# We pick the longuest spec name
candidate_specs.sort(key=len)
if candidate_specs:
spec_list[candidate_specs[-1]]["data"].append(
data) # type: ignore
# Find yaml files that are API >= 1.5 compatible
new_yaml_files = ls(os.path.join(self.spec_dir, "*", "*.yaml"),
emit_log_record=False)
for yml_f in new_yaml_files:
associated_spec = os.path.basename(os.path.dirname(yml_f))
# Keep only the yaml files associated with an .anod file
if associated_spec in spec_list:
# We're recording the relative path without the extension
suffix, _ = os.path.splitext(os.path.basename(yml_f))
spec_list[associated_spec]["data"].append( # type: ignore
os.path.join(associated_spec, suffix))
# Create AnodModule objects
for name, value in spec_list.items():
self.specs[name] = AnodModule(name, **value) # type: ignore
# Load config/repositories.yaml
repo_file = os.path.join(self.spec_dir, "config", "repositories.yaml")
if os.path.isfile(repo_file):
with open(repo_file) as fd:
self.repos = yaml.safe_load(fd)
# Declare spec prolog
prolog_file = os.path.join(spec_dir, "prolog.py")
self.prolog_dict = {
"spec_config": spec_config,
"__spec_repository": self
}
if os.path.exists(prolog_file):
with open(prolog_file) as f:
exec(compile(f.read(), prolog_file, "exec"), self.prolog_dict)
def load(self, repository: AnodSpecRepository) -> Callable[..., Anod]:
"""Load an anod specification and return the corresponding Anod class.
:param repository: the anod spec repository of the spec file
:raise SandBoxError: in case of failure
"""
if self.is_loaded:
if TYPE_CHECKING:
assert self.anod_class is not None
return self.anod_class
logger.debug("loading anod spec: %s", self.name)
# Create a new module
mod_name = "anod_" + self.name
anod_module = types.ModuleType(mod_name)
try:
with open(self.path) as fd:
# Inject the prolog into the new module dict
anod_module.__dict__.update(repository.prolog_dict)
# Exec spec code
code = compile(fd.read(), self.path, "exec")
exec(code, anod_module.__dict__)
except Exception as e:
logger.error("exception: %s", e)
logger.error("cannot load code of %s", self.name)
raise SandBoxError(origin="load",
message="invalid spec code for %s" %
self.name).with_traceback(sys.exc_info()[2])
# At this stage we have loaded completely the module. Now we need to
# look for a subclass of Anod. Use python inspection features to
# achieve this.
for members in inspect.getmembers(anod_module):
_, value = members
# Return the first subclass of Anod defined in this module
if (inspect.isclass(value) and value.__module__ == mod_name
and "Anod" in (k.__name__ for k in value.__mro__)):
# This class is a child of Anod so register it.
# Note that even if we won't use directly the
# module we need to keep a reference on it in order
# to avoid garbage collector issues.
value.spec_checksum = self.checksum
# Give a name to our Anod class: the basename of the
# anod spec file (without the .anod extension)
value.name = self.name
self.anod_class = value
self.module = anod_module
self.anod_class.data_files = self.data # type: ignore
self.anod_class.spec_dir = os.path.dirname(
self.path) # type: ignore
self.anod_class.api_version = repository.api_version # type: ignore
return value
logger.error("spec %s does not contains an Anod subclass", self.name)
raise SandBoxError("cannot find Anod subclass in %s" % self.path,
"load")
def __init__(
self,
spec_dir: str,
spec_config: Any = None,
# Ideally should be spec_config: Optional[SpecConfig] = None,
# We keep it to Any to avoid mypy issues on other projects
extra_repositories_config: Optional[dict] = None,
):
"""Initialize an AnodSpecRepository.
:param spec_dir: directory containing the anod specs.
:param spec_config: dictionary containing the configuration for this
AnodSpecRepository
:param extra_repositories_config: first read the configuration from
<spec_dir>/config/repositories.yaml and update the result with
extra_repositories_config
"""
logger.debug("initialize spec repository (%s)", spec_dir)
if not os.path.isdir(spec_dir):
raise SandBoxError(f"spec directory {spec_dir} does not exist")
self.spec_dir = spec_dir
self.api_version = __version__
self.specs = {}
self.repos: Dict[str, Dict[str, str]] = {}
# Look for all spec files and data files
spec_list = {
os.path.basename(os.path.splitext(k)[0]): {
"path": k,
"data": []
}
for k in ls(os.path.join(self.spec_dir, "*.anod"),
emit_log_record=False)
}
logger.debug("found %s specs", len(spec_list))
# API == 1.4
yaml_files = ls(os.path.join(self.spec_dir, "*.yaml"),
emit_log_record=False)
data_list = [os.path.basename(k)[:-5] for k in yaml_files]
logger.debug("found %s yaml files API 1.4 compatible", len(data_list))
# Match yaml files with associated specifications
for data in data_list:
candidate_specs = [
spec_file for spec_file in spec_list
if data.startswith(spec_file)
]
# We pick the longuest spec name
candidate_specs.sort(key=len)
if candidate_specs:
spec_list[candidate_specs[-1]]["data"].append(
data) # type: ignore
# Find yaml files that are API >= 1.5 compatible
new_yaml_files = ls(os.path.join(self.spec_dir, "*", "*.yaml"),
emit_log_record=False)
for yml_f in new_yaml_files:
associated_spec = os.path.basename(os.path.dirname(yml_f))
# Keep only the yaml files associated with an .anod file
if associated_spec in spec_list:
# We're recording the relative path without the extension
suffix, _ = os.path.splitext(os.path.basename(yml_f))
spec_list[associated_spec]["data"].append( # type: ignore
os.path.join(associated_spec, suffix))
# Create AnodModule objects
for name, value in spec_list.items():
self.specs[name] = AnodModule(name, **value) # type: ignore
# Load config/repositories.yaml
repo_file = os.path.join(self.spec_dir, "config", "repositories.yaml")
if os.path.isfile(repo_file):
with open(repo_file) as fd:
self.repos = yaml.safe_load(fd)
if extra_repositories_config:
for repo_name, repo_data in extra_repositories_config.items():
if repo_name in self.repos:
self.repos[repo_name].update(repo_data)
else:
self.repos[repo_name] = repo_data
# Make sure that all revision are strings and not floats
for repo_conf in self.repos.values():
if "revision" in repo_conf:
repo_conf["revision"] = str(repo_conf["revision"])
if spec_config is None:
spec_config = SpecConfig()
spec_config.spec_dir = self.spec_dir
spec_config.repositories = self.repos
# Declare spec prolog
prolog_file = os.path.join(spec_dir, "prolog.py")
self.prolog_dict = {
"spec_config": spec_config,
"__spec_repository": self
}
if os.path.exists(prolog_file):
with open(prolog_file) as f:
exec(compile(f.read(), prolog_file, "exec"), self.prolog_dict)
def run(self, args):
sandbox = SandBox(root_dir=args.sandbox)
if args.specs_dir:
sandbox.specs_dir = args.specs_dir
if args.create_sandbox:
sandbox.create_dirs()
if args.create_sandbox and args.spec_git_url:
mkdir(sandbox.specs_dir)
g = GitRepository(sandbox.specs_dir)
if e3.log.default_output_stream is not None:
g.log_stream = e3.log.default_output_stream
g.init()
g.update(args.spec_git_url, args.spec_git_branch, force=True)
sandbox.dump_configuration()
sandbox.write_scripts()
asr = AnodSpecRepository(sandbox.specs_dir)
check_api_version(asr.api_version)
# Load plan content if needed
if args.plan:
if not os.path.isfile(args.plan):
raise SandBoxError(f"plan file {args.plan} does not exist",
origin="SandBoxExec.run")
with open(args.plan) as plan_fd:
plan_content = ["def main_entry_point():"]
plan_content += [
f" {line}" for line in plan_fd.read().splitlines()
]
plan_content = "\n".join(plan_content)
env = BaseEnv()
cm = PlanContext(server=env)
store = None
resolver = getattr(
AnodContext,
str(args.resolver),
AnodContext.always_create_source_resolver,
)
logger.debug("Using resolver %s", resolver.__name__)
# Declare available actions and their signature
def anod_action(module,
build=None,
host=None,
target=None,
qualifier=None):
pass # all: no cover
for a in ("anod_install", "anod_build", "anod_test"):
cm.register_action(a, anod_action)
# Load the plan and execute
plan = Plan(data={})
plan.load_chunk(plan_content)
actions = cm.execute(plan, "main_entry_point")
ac = AnodContext(asr, default_env=env)
for action in actions:
ac.add_anod_action(
action.module,
action,
action.action.replace("anod_", "", 1),
action.qualifier,
)
# Check if machine plan is locally schedulable
action_list = ac.schedule(resolver)
e = ElectrolytJobFactory(sandbox, asr, store, dry_run=args.dry_run)
e.run(action_list)
def error(message: str) -> NoReturn:
raise SandBoxError(message)
def run(self, args):
if args.api_version != "1.5":
raise SandBoxError("Only 1.5 is supported for now")
cd(args.spec_dir)
migrate_v1_5()
def __init__(self, spec_dir, spec_config=None):
"""Initialize an AnodSpecRepository.
:param spec_dir: directory containing the anod specs.
:type spec_dir: str
:param spec_config: dictionary containing the configuration for this
AnodSpecRepository
:type spec_config: dict | SandboxConfig
"""
logger.debug('initialize spec repository (%s)', spec_dir)
if not os.path.isdir(spec_dir):
raise SandBoxError(
'spec directory %s does not exist' % spec_dir)
self.spec_dir = spec_dir
self.api_version = __version__
self.specs = {}
self.repos = {}
# Look for all spec files and data files
spec_list = {os.path.basename(os.path.splitext(k)[0]): {'path': k,
'data': []}
for k in ls(os.path.join(self.spec_dir, '*.anod'),
emit_log_record=False)}
logger.debug('found %s specs', len(spec_list))
# API == 1.4
yaml_files = ls(os.path.join(self.spec_dir, '*.yaml'),
emit_log_record=False)
data_list = [os.path.basename(k)[:-5] for k in yaml_files]
logger.debug('found %s yaml files API 1.4 compatible', len(data_list))
# Match yaml files with associated specifications
for data in data_list:
candidate_specs = [spec_file for spec_file in spec_list
if data.startswith(spec_file)]
# We pick the longuest spec name
candidate_specs.sort(key=len)
if candidate_specs:
spec_list[candidate_specs[-1]]['data'].append(data)
# Find yaml files that are API >= 1.5 compatible
new_yaml_files = ls(os.path.join(self.spec_dir, '*', '*.yaml'),
emit_log_record=False)
for yml_f in new_yaml_files:
associated_spec = os.path.basename(os.path.dirname(yml_f))
# Keep only the yaml files associated with an .anod file
if associated_spec in spec_list:
# We're recording the relative path without the extension
suffix, _ = os.path.splitext(os.path.basename(yml_f))
spec_list[associated_spec]['data'].append(
os.path.join(associated_spec, suffix))
# Create AnodModule objects
for name, value in spec_list.iteritems():
self.specs[name] = AnodModule(name, **value)
# Declare spec prolog
prolog_file = os.path.join(spec_dir, 'prolog.py')
self.prolog_dict = {'spec_config': spec_config,
'__spec_repository': self}
if os.path.exists(prolog_file):
with open(prolog_file) as f:
exec(compile(f.read(), prolog_file, 'exec'),
self.prolog_dict)
