def execute(statement, **kwargs):
'''execute a statement locally.
This method implements the same parameter interpolation
as the function :func:`run`.
Arguments
---------
statement : string
Command line statement to run.
Returns
-------
stdout : string
Data sent to standard output by command
stderr : string
Data sent to standard error by command
'''
if not kwargs:
kwargs = get_caller_locals()
kwargs = dict(list(get_params().items()) + list(kwargs.items()))
logger = get_logger()
logger.info("running %s" % (statement % kwargs))
if "cwd" not in kwargs:
cwd = get_params()["work_dir"]
else:
cwd = kwargs["cwd"]
# cleaning up of statement
# remove new lines and superfluous spaces and tabs
statement = " ".join(re.sub("\t+", " ", statement).split("\n")).strip()
if statement.endswith(";"):
statement = statement[:-1]
# always use bash
os.environ.update(
{'BASH_ENV': os.path.join(os.environ['HOME'], '.bashrc')})
process = subprocess.Popen(statement % kwargs,
cwd=cwd,
shell=True,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
env=os.environ.copy(),
executable="/bin/bash")
# process.stdin.close()
stdout, stderr = process.communicate()
if process.returncode != 0:
raise OSError(
"Child was terminated by signal %i: \n"
"The stderr was: \n%s\n%s\n" %
(-process.returncode, stderr, statement))
return stdout, stderr
def get_parameters(filenames=None,
defaults=None,
site_ini=True,
user=True,
only_import=None):
'''read one or more config files and build global PARAMS configuration
dictionary.
Arguments
---------
filenames : list
List of filenames of the configuration files to read.
defaults : dict
Dictionary with default values. These will be overwrite
any hard-coded parameters, but will be overwritten by user
specified parameters in the configuration files.
user : bool
If set, configuration files will also be read from a
file called :file:`.cgat.yml` in the user`s
home directory.
only_import : bool
If set to a boolean, the parameter dictionary will be a
defaultcollection. This is useful for pipelines that are
imported (for example for documentation generation) but not
executed as there might not be an appropriate .yml file
available. If `only_import` is None, it will be set to the
default, which is to raise an exception unless the calling
script is imported or the option ``--is-test`` has been passed
at the command line.
Returns
-------
params : dict
Global configuration dictionary.
'''
global PARAMS, HAVE_INITIALIZED
# only execute function once
if HAVE_INITIALIZED:
return PARAMS
if filenames is None:
filenames = ["pipeline.yml"]
if isinstance(filenames, str):
filenames = [filenames]
old_id = id(PARAMS)
caller_locals = get_caller_locals()
# check if this is only for import
if only_import is None:
only_import = is_test() or "__name__" not in caller_locals or \
caller_locals["__name__"] != "__main__"
# important: only update the PARAMS variable as
# it is referenced in other modules. Thus the type
# needs to be fixed at import. Raise error where this
# is not the case.
# Note: Parameter sharing in the pipeline module needs
# to be reorganized.
if only_import:
# turn on default dictionary
TriggeredDefaultFactory.with_default = True
# check if the pipeline is in testing mode
found = False
if 'argv' in caller_locals and caller_locals['argv'] is not None:
for e in caller_locals['argv']:
if 'template_pipeline.py' in e:
found = True
PARAMS['testing'] = 'self' in caller_locals or found
if site_ini:
# read configuration from /etc/cgat/pipeline.yml
fn = "/etc/cgat/pipeline.yml"
if os.path.exists(fn):
filenames.insert(0, fn)
if user:
# read configuration from a users home directory
fn = os.path.join(os.path.expanduser("~"),
".cgat.yml")
if os.path.exists(fn):
if 'pipeline.yml' in filenames:
index = filenames.index('pipeline.yml')
filenames.insert(index, fn)
else:
filenames.append(fn)
filenames = [x.strip() for x in filenames if os.path.exists(x)]
# save list of config files
PARAMS["pipeline_yml"] = filenames
# update with hard-coded PARAMS
nested_update(PARAMS, HARDCODED_PARAMS)
if defaults:
nested_update(PARAMS, defaults)
# reset working directory. Set in PARAMS to prevent repeated calls to
# os.getcwd() failing if network is busy
PARAMS["start_dir"] = os.path.abspath(os.getcwd())
# location of pipelines - set via location of top frame (cgatflow command)
if '__file__' in caller_locals:
PARAMS["pipelinedir"] = os.path.dirname(caller_locals["__file__"])
else:
PARAMS["pipelinedir"] = 'unknown'
for filename in filenames:
if not os.path.exists(filename):
continue
get_logger().info("reading config from file {}".format(
filename))
with open(filename) as inf:
p = yaml.load(inf)
if p:
nested_update(PARAMS, p)
# for backwards compatibility - normalize dictionaries
p = {}
for k, v in PARAMS.items():
if isinstance(v, collections.Mapping):
for kk, vv in v.items():
new_key = "{}_{}".format(k, kk)
if new_key in p:
raise ValueError(
"key {} does already exist".format(new_key))
p[new_key] = vv
nested_update(PARAMS, p)
# interpolate some params with other parameters
for param in INTERPOLATE_PARAMS:
try:
PARAMS[param] = PARAMS[param] % PARAMS
except TypeError as msg:
raise TypeError('could not interpolate %s: %s' %
(PARAMS[param], msg))
# expand directory pathnames
for param, value in list(PARAMS.items()):
if (param.endswith("dir") and isinstance(value, str) and value.startswith(".")):
PARAMS[param] = os.path.abspath(value)
# make sure that the dictionary reference has not changed
assert id(PARAMS) == old_id
HAVE_INITIALIZED = True
return PARAMS
def peek_parameters(workingdir,
pipeline,
on_error_raise=None,
prefix=None,
update_interface=False,
restrict_interface=False):
'''peek configuration parameters from external pipeline.
As the paramater dictionary is built at runtime, this method
executes the pipeline in workingdir, dumping its configuration
values and reading them into a dictionary.
If either `pipeline` or `workingdir` are not found, an error is
raised. This behaviour can be changed by setting `on_error_raise`
to False. In that case, an empty dictionary is returned.
Arguments
---------
workingdir : string
Working directory. This is the directory that the pipeline
was executed in.
pipeline : string
Name of the pipeline script. The pipeline is assumed to live
in the same directory as the current pipeline.
on_error_raise : Bool
If set to a boolean, an error will be raised (or not) if there
is an error during parameter peeking, for example if
`workingdir` can not be found. If `on_error_raise` is None, it
will be set to the default, which is to raise an exception
unless the calling script is imported or the option
``--is-test`` has been passed at the command line.
prefix : string
Add a prefix to all parameters. This is useful if the paramaters
are added to the configuration dictionary of the calling pipeline.
update_interface : bool
If True, this method will prefix any options in the
``[interface]`` section with `workingdir`. This allows
transparent access to files in the external pipeline.
restrict_interface : bool
If True, only interface parameters will be imported.
Returns
-------
config : dict
Dictionary of configuration values.
'''
caller_locals = get_caller_locals()
# check if we should raise errors
if on_error_raise is None:
on_error_raise = not is_test() and \
"__name__" in caller_locals and \
caller_locals["__name__"] == "__main__"
# patch - if --help or -h in command line arguments,
# do not peek as there might be no config file.
if "--help" in sys.argv or "-h" in sys.argv:
return {}
if workingdir == "":
workingdir = os.path.abspath(".")
# patch for the "config" target - use default
# pipeline directory if directory is not specified
# working dir is set to "?!"
if ("config" in sys.argv or "check" in sys.argv
or "clone" in sys.argv and workingdir == "?!"):
workingdir = os.path.join(get_params()["pipelinedir"],
"pipeline_" + pipeline)
if not os.path.exists(workingdir):
if on_error_raise:
raise ValueError("can't find working dir %s" % workingdir)
else:
return {}
statement = "cgatflow {} -v 0 dump".format(pipeline)
os.environ.update(
{'BASH_ENV': os.path.join(os.environ['HOME'], '.bashrc')})
process = subprocess.Popen(statement,
cwd=workingdir,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
# process.stdin.close()
stdout, stderr = process.communicate()
if process.returncode != 0:
raise OSError(
("Child was terminated by signal %i: \n"
"Statement: %s\n"
"The stderr was: \n%s\n"
"Stdout: %s") % (-process.returncode, statement, stderr, stdout))
# subprocess only accepts encoding argument in py >= 3.6 so
# decode here.
stdout = stdout.decode("utf-8").splitlines()
# remove any log messages
stdout = [x for x in stdout if x.startswith("{")]
if len(stdout) > 1:
raise ValueError("received multiple configurations")
dump = json.loads(stdout[0])
# update interface
if update_interface:
for key, value in list(dump.items()):
if key.startswith("interface"):
if isinstance(value, str):
dump[key] = os.path.join(workingdir, value)
elif isinstance(value, collections.Mapping):
for kkey, vvalue in list(value.items()):
value[key] = os.path.join(workingdir, vvalue)
# keep only interface if so required
if restrict_interface:
dump = dict([(k, v) for k, v in dump.items()
if k.startswith("interface")])
# prefix all parameters
if prefix is not None:
dump = dict([("%s%s" % (prefix, x), y) for x, y in list(dump.items())])
return dump
def run(statement, **kwargs):
"""run a command line statement.
This function runs a single or multiple statements either locally
or on the cluster using drmaa. How a statement is executed or how
it is modified depends on the context.
The context is provided by keyword arguments provided as named
function arguments ('kwargs') but also from defaults (see
below). The following keyword arguments are recognized:
job_memory
memory to use for the job per thread. Memory specification should be in a
format that is accepted by the job scheduler. Note that memory
is per thread. If you have 6 threads and the total memory is
6Gb, use 1G as job_memory.
job_total_memory
total memory to use for a job. This will be divided by the number of
threads.
job_threads
number of threads to request for the job.
job_options
options to the job scheduler.
job_condaenv
conda environment to use for the job.
job_array
if set, run statement as an array job. Job_array should be
tuple with start, end, and increment.
In addition, any additional variables will be used to interpolate
the command line string using python's '%' string interpolation
operator.
The context is build in a hierarchical manner with successive
operations overwriting previous values.
1. Global variables
The context is initialized
with system-wide defaults stored in the global PARAMS
singleton.
2. Context of caller
The context of the calling function is examined
and any local variables defined in this context are added.
3. kwargs
Any options given explicitely as options to the run() method
are added.
4. params
If the context of the calling function contains a params
variable, its contents are added to the context. This permits
setting variables in configuration files in TaskLibrary
functions.
By default, a job is sent to the cluster, unless:
* ``to_cluster`` is present and set to None.
* ``without_cluster`` is True.
* ``--local`` has been specified on the command line
and the option ``without_cluster`` has been set as
a result.
* no libdrmaa is present
* the global session is not initialized (GLOBAL_SESSION is
None)
Troubleshooting:
1. DRMAA creates sessions and their is a limited number
of sessions available. If there are two many or sessions
become not available after failed jobs, use ``qconf -secl``
to list sessions and ``qconf -kec #`` to delete sessions.
2. Memory: 1G of free memory can be requested using the job_memory
variable: ``job_memory = "1G"``
If there are error messages like "no available queue", then the
problem could be that a particular complex attribute has
not been defined (the code should be ``hc`` for ``host:complex``
and not ``hl`` for ``host:local``. Note that qrsh/qsub directly
still works.
The job will be executed within PARAMS["work_dir"], unless
PARAMS["work_dir"] is not local. In that case, the job will
be executed in a shared temporary directory.
Arguments
---------
statement : string or list of strings
A command line statement or a list of command line statements
to be executed.
kwargs : dictionary
Context for job. The context is used to interpolate the command
line statement.
"""
logger = get_logger()
# combine options using priority
options = dict(list(get_params().items()))
caller_options = get_caller_locals()
options.update(list(caller_options.items()))
if "self" in options:
del options["self"]
options.update(list(kwargs.items()))
# inject params named tuple from TaskLibrary functions into option
# dict. This allows overriding options set in the code with options set
# in a .yml file
if "params" in options:
try:
options.update(options["params"]._asdict())
except AttributeError:
pass
# insert parameters supplied through simplified interface such
# as job_memory, job_options, job_queue
options['cluster']['options'] = options.get('job_options',
options['cluster']['options'])
options['cluster']['queue'] = options.get('job_queue',
options['cluster']['queue'])
options['without_cluster'] = options.get('without_cluster')
# SGE compatible job_name
name_substrate = str(options.get("outfile", "cgatcore"))
if os.path.basename(name_substrate).startswith("result"):
name_substrate = os.path.basename(os.path.dirname(name_substrate))
else:
name_substrate = os.path.basename(name_substrate)
options["job_name"] = re.sub("[:]", "_", name_substrate)
try:
calling_module = get_caller().__name__
except AttributeError:
calling_module = "unknown"
options["task_name"] = calling_module + "." + get_calling_function()
# build statements using parameter interpolation
if isinstance(statement, list):
statement_list = []
for stmt in statement:
statement_list.append(interpolate_statement(stmt, options))
else:
statement_list = [interpolate_statement(statement, options)]
if len(statement_list) == 0:
logger.warn("no statements found - no execution")
return []
if options.get("dryrun", False):
for statement in statement_list:
logger.info("dry-run: {}".format(statement))
return []
# execute statement list
runner = make_runner(**options)
with runner as r:
benchmark_data = r.run(statement_list)
# log benchmark_data
for data in benchmark_data:
logger.info(json.dumps(data))
BenchmarkData = collections.namedtuple('BenchmarkData',
sorted(benchmark_data[0]))
return [BenchmarkData(**d) for d in benchmark_data]