def __init__(self, experiment):
self.experiment = experiment
self.dir = ''
self.resources = []
self.linked_resources = []
self.env_vars = {}
self.new_files = []
self.commands = OrderedDict()
self.optional_output = []
self.required_output = []
self.properties = tools.Properties()
if not valid_module_name(name):
tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
# Create a directory for the new package.
pkg_path = spack.db.filename_for_package_name(name)
if os.path.exists(pkg_path) and not args.force:
tty.die("%s already exists." % pkg_path)
else:
mkdirp(os.path.dirname(pkg_path))
versions = spack.package.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = {version: url}
elif len(versions) > 1:
tty.msg(
"Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5,
abort='q')
if not valid_module_name(name):
tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
# Create a directory for the new package.
pkg_path = spack.db.filename_for_package_name(name)
if os.path.exists(pkg_path) and not args.force:
tty.die("%s already exists." % pkg_path)
else:
mkdirp(os.path.dirname(pkg_path))
versions = spack.package.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = {version: url}
elif len(versions) > 1:
tty.msg(
"Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(["%-10s%s" % (v, u) for v, u in versions.iteritems()])
)
print
archives_to_fetch = tty.get_number("Include how many checksums in the package file?", default=5, abort="q")
if not archives_to_fetch:
tty.msg("Aborted.")
class Run(object):
"""
A Task can consist of one or multiple Runs
"""
def __init__(self, experiment):
self.experiment = experiment
self.dir = ''
self.resources = []
self.linked_resources = []
self.env_vars = {}
self.new_files = []
self.commands = OrderedDict()
self.optional_output = []
self.required_output = []
self.properties = tools.Properties()
def set_property(self, name, value):
"""
Add a key-value property to a run. These can be used later for
evaluation
Example:
>>> run.set_property('domain', 'gripper')
"""
self.properties[name] = value
def require_resource(self, resource_name):
"""
Some resources can be used by linking to the resource in the
experiment directory without copying it into each run
In the argo cluster however, requiring a resource implies copying it
into the task directory.
Example:
>>> run.require_resource('PLANNER')
Make the planner resource available for this run
In environments like the argo cluster, this implies
copying the planner into each task. For the gkigrid, we merely
need to set up the PLANNER environment variable.
"""
self.linked_resources.append(resource_name)
def add_resource(self, resource_name, source, dest, required=True,
symlink=False):
"""
Example:
>>> run.add_resource('DOMAIN', '../benchmarks/gripper/domain.pddl',
'domain.pddl')
Copy "../benchmarks/gripper/domain.pddl" into the run
directory under name "domain.pddl" and make it available as
resource "DOMAIN" (usable as environment variable $DOMAIN).
"""
self.resources.append((source, dest, required, symlink))
self.env_vars[resource_name] = dest
def add_command(self, name, command, **kwargs):
"""Adds a command to the run.
"name" is the command's name.
"command" has to be a list of strings.
The items in kwargs are passed to the calls.call.Call() class. You can
find the valid keys there.
kwargs can also contain a value for "abort_on_failure" which makes the
run abort if the command does not return 0.
The remaining items in kwargs are passed to subprocess.Popen()
The allowed parameters can be found at
http://docs.python.org/library/subprocess.html
Examples:
>>> run.add_command('translate', [run.translator.shell_name,
'domain.pddl', 'problem.pddl'])
>>> run.add_command('preprocess', [run.preprocessor.shell_name],
{'stdin': 'output.sas'})
>>> run.add_command('validate', ['VALIDATE', 'DOMAIN', 'PROBLEM',
'sas_plan'])
"""
assert type(name) is str, 'The command name must be a string'
assert type(command) in (tuple, list), 'The command must be a list'
name = name.replace(' ', '_')
self.commands[name] = (command, kwargs)
def declare_optional_output(self, file_glob):
"""
Example:
>>> run.declare_optional_output('plan.soln*')
Specifies that all files names "plan.soln*" (using
shell-style glob patterns) are part of the experiment output.
"""
self.optional_output.append(file_glob)
def declare_required_output(self, filename):
"""
Declare output files that must be present at the end or we have an
error. A specification like this is e.g. necessary for the Argo
cluster. On the gkigrid, this wouldn't do anything, although
the declared outputs should be stored somewhere so that we
can later verify that all went according to plan.
"""
self.required_output.append(filename)
def build(self):
"""
After having made all the necessary adjustments with the methods above,
this method can be used to write everything to the disk.
"""
assert self.dir
tools.overwrite_dir(self.dir)
# We need to build the linked resources before the run script.
# Only this way we have all resources in self.resources
# (linked ones too)
self._build_linked_resources()
self._build_run_script()
self._build_resources()
self._build_properties_file()
def _build_run_script(self):
if not self.commands:
msg = 'Please add at least one command via run.add_command()'
raise SystemExit(msg)
self.experiment.env_vars.update(self.env_vars)
self.env_vars = self.experiment.env_vars.copy()
run_script = open(os.path.join(tools.DATA_DIR, 'run-template.py')).read()
def make_call(name, cmd, kwargs):
abort_on_failure = kwargs.pop('abort_on_failure',
DEFAULT_ABORT_ON_FAILURE)
if not type(cmd) is list:
logging.error('Commands have to be lists of strings. '
'The command <%s> is not a list.' % cmd)
sys.exit(1)
if not cmd:
logging.error('Command "%s" cannot be empty' % name)
sys.exit(1)
# Support running globally installed binaries
def format_arg(arg):
return arg if arg in self.env_vars else '"%s"' % arg
def format_key_value_pair(key, val):
return '%s=%s' % (key, val if val in self.env_vars else repr(val))
cmd_string = '[%s]' % ', '.join([format_arg(arg) for arg in cmd])
kwargs_string = ', '.join(format_key_value_pair(key, value)
for key, value in kwargs.items())
parts = [cmd_string]
if kwargs_string:
parts.append(kwargs_string)
call = ('retcode = Call(%s, **redirects).wait()\n'
'save_returncode("%s", retcode)\n') % (', '.join(parts), name)
if abort_on_failure:
call += ('if not retcode == 0:\n'
' print_(driver_log, "%s returned %%s" %% retcode)\n'
' sys.exit(1)\n' % name)
return call
calls_text = '\n'.join(make_call(name, cmd, kwargs)
for name, (cmd, kwargs) in self.commands.items())
if self.env_vars:
env_vars_text = ''
for var, filename in sorted(self.env_vars.items()):
abs_filename = self._get_abs_path(filename)
rel_filename = self._get_rel_path(abs_filename)
env_vars_text += ('%s = "%s"\n' % (var, rel_filename))
else:
env_vars_text = '"Here you would find variable declarations"'
for old, new in [('VARIABLES', env_vars_text), ('CALLS', calls_text)]:
run_script = run_script.replace('"""%s"""' % old, new)
self.new_files.append(('run', run_script))
return
def _build_linked_resources(self):
"""
If we are building an argo experiment, add all linked resources to
the resources list
"""
self.experiment.environment.build_linked_resources(self)
def _build_resources(self):
for name, content in self.new_files:
filename = self._get_abs_path(name)
with open(filename, 'w') as file:
logging.debug('Writing file "%s"' % filename)
file.write(content)
if name == 'run':
# Make run script executable
os.chmod(filename, 0755)
for source, dest, required, symlink in self.resources:
if required and not os.path.exists(source):
logging.error('The required resource can not be found: %s' %
source)
sys.exit(1)
dest = self._get_abs_path(dest)
if symlink:
source = self._get_rel_path(source)
os.symlink(source, dest)
logging.debug('Linking from %s to %s' % (source, dest))
continue
logging.debug('Copying %s to %s' % (source, dest))
tools.copy(source, dest, required)
def _build_properties_file(self):
# Check correctness of id property
run_id = self.properties.get('id')
if run_id is None:
logging.error('Each run must have an id')
sys.exit(1)
if not type(run_id) is list:
logging.error('id must be a list, but %s is not' % run_id)
sys.exit(1)
self.properties['id'] = [str(item) for item in run_id]
self.properties.filename = self._get_abs_path('properties')
self.properties.write()
def _get_abs_path(self, rel_path):
"""
Example:
>>> _get_abs_path('run')
/home/user/mytestjob/runs-00001-00100/run
"""
return os.path.join(self.dir, rel_path)
def _get_rel_path(self, abs_path):
return os.path.relpath(abs_path, start=self.dir)
from external.ordereddict import OrderedDict
from llnl.util.lang import memoized
import spack.error
__all__ = [
'SpackConfigParser', 'get_config', 'SpackConfigurationError',
'InvalidConfigurationScopeError', 'InvalidSectionNameError',
'ReadOnlySpackConfigError', 'ConfigParserError', 'NoOptionError',
'NoSectionError']
_named_section_re = r'([^ ]+) "([^"]+)"'
"""Names of scopes and their corresponding configuration files."""
_scopes = OrderedDict({
'site' : os.path.join(spack.etc_path, 'spackconfig'),
'user' : os.path.expanduser('~/.spackconfig')
})
_field_regex = r'^([\w-]*)' \
r'(?:\.(.*(?=.)))?' \
r'(?:\.([\w-]+))?$'
_section_regex = r'^([\w-]*)\s*' \
r'\"([^"]*\)\"$'
# Cache of configs -- we memoize this for performance.
_config = {}
def get_config(scope=None, **kwargs):
"""Get a Spack configuration object, which can be used to set options.
