def __init__(self,
namespace="gmxapi",
operation=None,
params=None,
depends=()):
self._namespace = str(to_string(namespace))
# We can add an operations submodule to validate these. E.g. self.operation = gmx.workflow.operations.normalize(operation)
if operation is not None:
self._operation = str(to_string(operation))
else:
raise exceptions.UsageError("Invalid argument type for operation.")
# Note: Nothing currently prevents attribute updates by assignment after adding the element to a workspec,
# but this protocol will be clarified with https://github.com/kassonlab/gmxapi/issues/92
if params is None:
self.params = GmxMap()
elif isinstance(params, dict):
self.params = GmxMap(
{to_string(name): params[name]
for name in params})
else:
raise exceptions.UsageError(
"If provided, params must be a dictionary of keyword arguments"
)
self.depends = []
for d in depends:
if isinstance(d, (list, tuple)):
self.depends.append([str(name) for name in d])
else:
self.depends.append(str(d))
# The Python class for work elements keeps a strong reference to a WorkSpec object containing its description
self._name = None
self._workspec = None
def __init__(self, filename: str = None, mode: str = 'r'):
"""Open a TPR file.
File access mode is indicated by 'r' for read-only access.
Args:
filename (str): Path to a run input file (e.g. 'myfile.tpr')
mode (str): File access mode.
Note:
Currently, TPR files are read-only from the Python interface.
Example:
>>> import gmxapi as gmx
>>> filehandle = gmx.TprFile(filename, 'r')
"""
if filename is None:
raise exceptions.UsageError("TprFile objects must be associated with a file.")
if mode != 'r':
raise exceptions.UsageError("TPR files only support read-only access.")
self.mode = mode
self.filename = filename
self._tprFileHandle = None
def read_tpr(tprfile: typing.Union[str, TprFile]):
"""
Get a simulation input object from a TPR run input file.
Arguments:
tprfile : TPR input object or filename
Returns:
simulation input object
The returned object may be inspected by the user. Simulation input parameters
may be extracted through the `parameters` attribute.
Example:
>>> sim_input = gmx.fileio.read_tpr(tprfile=tprfilename)
>>> params = sim_input.parameters.extract()
>>> print(params['init-step'])
0
Supports the `read_tpr` gmxapi work graph operation. (not yet implemented)
"""
if not isinstance(tprfile, TprFile):
try:
tprfile = TprFile(os.fsencode(tprfile), mode='r')
except Exception as e:
raise exceptions.UsageError("TPR object or file name is required.") from e
return _SimulationInput(tprfile)
def add_element(self, element):
"""Add an element to a work specification if possible.
Adding an element to a WorkSpec must preserve the validity of the workspec, which involves several checks.
We do not yet check for element uniqueness beyond a string name.
If an element is added that was previously in another WorkSpec, it must first be removed from the
other WorkSpec.
"""
if hasattr(element, "namespace") and hasattr(
element, "operation") and hasattr(element, "serialize"):
if not hasattr(element, "name") or element.name is None or len(
str(element.name)) < 1:
raise exceptions.UsageError(
"Only named elements may be added to a WorkSpec.")
if element.name in self.elements:
raise exceptions.UsageError(
"Elements in WorkSpec must be uniquely identifiable.")
if hasattr(element, "depends"):
for dependency in element.depends:
if not dependency in self.elements:
raise exceptions.UsageError(
"Element dependencies must already be specified before an Element may be added."
)
# Okay, it looks like we have an element we can add
if hasattr(
element, "workspec"
) and element.workspec is not None and element.workspec is not self:
raise exceptions.Error(
"Element must be removed from its current WorkSpec to be added to this WorkSpec, but element "
"removal is not yet implemented.")
self.elements[element.name] = element.serialize()
element.workspec = self
else:
raise exceptions.ValueError(
"Provided object does not appear to be compatible with gmx.workflow.WorkElement."
)
logger.info("Added element {} to workspec.".format(element.name))
def scoped_communicator(original_comm, requested_size: int = None):
from gmxapi.simulation.context import _acquire_communicator, _get_ensemble_communicator
if requested_size is None:
communicator = _acquire_communicator(communicator=original_comm)
else:
if original_comm is None or not hasattr(original_comm, 'Get_size'):
raise exceptions.UsageError('A valid communicator must be provided when requesting a specific size.')
original_comm_size = original_comm.Get_size()
if original_comm_size < requested_size:
raise exceptions.FeatureNotAvailableError(
'Cannot produce a subcommunicator of size {} from a communicator of size {}.'.format(
requested_size,
original_comm_size
))
assert original_comm_size >= requested_size
communicator = _get_ensemble_communicator(original_comm, requested_size)
try:
yield communicator
finally:
communicator.Free()
def cli(command: NDArray,
shell: bool,
output: OutputCollectionDescription,
stdin: str = ''):
"""Execute a command line program in a subprocess.
Configure an executable in a subprocess. Executes when run in an execution
Context, as part of a work graph or via gmx.run(). Runs in the current
working directory.
Shell processing is not enabled, but can be considered for a future version.
This means that shell expansions such as environment variables, globbing (`*`),
and other special symbols (like `~` for home directory) are not available.
This allows a simpler and more robust implementation, as well as a better
ability to uniquely identify the effects of a command line operation. If you
think this disallows important use cases, please let us know.
Arguments:
command: a tuple (or list) to be the subprocess arguments, including `executable`
output: mapping of command line flags to output filename arguments
shell: unused (provides forward-compatibility)
stdin (str): String input to send to STDIN (terminal input) of the executable.
Multi-line text sent to *stdin* should be joined into a single string
(e.g. ``'\n'.join(list_of_strings) + '\n'``).
If multiple strings are provided to *stdin*, gmxapi will assume an ensemble,
and will run one operation for each provided string.
Only string input (:py:func:str) to *stdin* is currently supported.
If you have a use case that requires streaming input or binary input,
please open an issue or contact the author(s).
Arguments are iteratively added to the command line with standard Python
iteration, so you should use a tuple or list even if you have only one parameter.
I.e. If you provide a string with `arguments="asdf"` then it will be passed as
`... "a" "s" "d" "f"`. To pass a single string argument, `arguments=("asdf")`
or `arguments=["asdf"]`.
`input` and `output` should be a dictionary with string keys, where the keys
name command line "flags" or options.
Example:
Execute a command named `exe` that takes a flagged option for file name
(stored in a local Python variable `my_filename`) and an `origin` flag
that uses the next three arguments to define a vector.
>>> my_filename = "somefilename"
>>> result = cli(('exe', '--origin', 1.0, 2.0, 3.0, '-f', my_filename), shell=False)
>>> assert hasattr(result, 'file')
>>> assert hasattr(result, 'erroroutput')
>>> assert hasattr(result, 'returncode')
Returns:
A data structure with attributes for each of the results `file`, `erroroutput`, and `returncode`
Result object attributes:
* `file`: the mapping of CLI flags to filename strings resulting from the `output` kwarg
* `erroroutput`: A string of error output (if any) if the process failed.
* `returncode`: return code of the subprocess.
"""
# Note: we could make provisions for stdio filehandles in a future version. E.g.
# * STDOUT is available if a consuming operation is bound to `output.stdout`.
# * STDERR is available if a consuming operation is bound to `output.stderr`.
# * Otherwise, STDOUT and/or STDERR is(are) closed when command is called.
# In the operation implementation, we expect the `shell` parameter to be intercepted by the
# wrapper and set to False.
if shell:
raise exceptions.UsageError(
"Operation does not support shell processing.")
if stdin == '':
stdin = None
if isinstance(command, (str, bytes)):
command = [command]
command = list([arg for arg in command])
executable = shutil.which(command[0])
if executable is None:
raise exceptions.ValueError(
'"{}" is not found or not executable.'.format(command[0]))
command[0] = executable
# TODO: (FR9) Can OS input/output filehandles be a responsibility of
# the code providing 'resources'?
erroroutput = ''
logger.debug('executing subprocess')
try:
completed_process = subprocess.run(command,
shell=shell,
input=stdin,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding='utf-8',
universal_newlines=True)
returncode = completed_process.returncode
# TODO: Resource management code should manage a safe data object for `output`.
for line in completed_process.stdout.split('\n'):
logger.debug(line)
except subprocess.CalledProcessError as e:
logger.info(
"commandline operation had non-zero return status when calling {}".
format(e.cmd))
erroroutput = e.output
returncode = e.returncode
# Publish outputs.
output.erroroutput = erroroutput
output.returncode = returncode
def cli(command: NDArray, shell: bool, output: OutputCollectionDescription):
"""Execute a command line program in a subprocess.
Configure an executable in a subprocess. Executes when run in an execution
Context, as part of a work graph or via gmx.run(). Runs in the current
working directory.
Shell processing is not enabled, but can be considered for a future version.
This means that shell expansions such as environment variables, globbing (`*`),
and other special symbols (like `~` for home directory) are not available.
This allows a simpler and more robust implementation, as well as a better
ability to uniquely identify the effects of a command line operation. If you
think this disallows important use cases, please let us know.
Arguments:
command : a tuple (or list) to be the subprocess arguments, including `executable`
output : mapping of command line flags to output filename arguments
shell : unused (provides forward-compatibility)
Arguments are iteratively added to the command line with standard Python
iteration, so you should use a tuple or list even if you have only one parameter.
I.e. If you provide a string with `arguments="asdf"` then it will be passed as
`... "a" "s" "d" "f"`. To pass a single string argument, `arguments=("asdf")`
or `arguments=["asdf"]`.
`input` and `output` should be a dictionary with string keys, where the keys
name command line "flags" or options.
Example:
Execute a command named `exe` that takes a flagged option for file name
(stored in a local Python variable `my_filename`) and an `origin` flag
that uses the next three arguments to define a vector.
>>> my_filename = "somefilename"
>>> result = cli(('exe', '--origin', 1.0, 2.0, 3.0, '-f', my_filename), shell=False)
>>> assert hasattr(result, 'file')
>>> assert hasattr(result, 'erroroutput')
>>> assert hasattr(result, 'returncode')
Returns:
A data structure with attributes for each of the results `file`, `erroroutput`, and `returncode`
Result object attributes:
* `file`: the mapping of CLI flags to filename strings resulting from the `output` kwarg
* `erroroutput`: A string of error output (if any) if the process failed.
* `returncode`: return code of the subprocess.
"""
# Note: we could make provisions for stdio filehandles in a future version. E.g.
# * STDOUT is available if a consuming operation is bound to `output.stdout`.
# * STDERR is available if a consuming operation is bound to `output.stderr`.
# * Otherwise, STDOUT and/or STDERR is(are) closed when command is called.
#
# Warning:
# Commands relying on STDIN cannot be used and is closed when command is called.
# In the operation implementation, we expect the `shell` parameter to be intercepted by the
# wrapper and set to False.
if shell:
raise exceptions.UsageError("Operation does not support shell processing.")
if isinstance(command, (str, bytes)):
command = [command]
command = list([arg for arg in command])
try:
command[0] = shutil.which(command[0])
except Exception:
raise exceptions.ValueError('command argument could not be resolved to an executable file path.')
# TODO: (FR9) Can OS input/output filehandles be a responsibility of
# the code providing 'resources'?
erroroutput = ''
logger.debug('executing subprocess')
try:
# TODO: If Python >=3.5 is required, switch to subprocess.run()
command_output = subprocess.check_output(command,
shell=shell,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
)
returncode = 0
# TODO: Resource management code should manage a safe data object for `output`.
# WARNING: We have no reason to assume the output is utf-8 encoded text!!!
for line in command_output.decode('utf-8').split('\n'):
logger.debug(line)
except subprocess.CalledProcessError as e:
logger.info("commandline operation had non-zero return status when calling {}".format(e.cmd))
erroroutput = e.output.decode('utf-8')
returncode = e.returncode
# resources.output.erroroutput.publish(erroroutput)
# resources.output.returncode.publish(returncode)
# `publish` is descriptive, but redundant. Access to the output data handler is
# assumed to coincide with publishing, and we assume data is published when the
# handler is released. A class with a single `publish` method is overly complex
# since we can just use the assignment operator.
output.erroroutput = erroroutput
output.returncode = returncode
def from_tpr(input=None, **kwargs):
"""Create a WorkSpec from a (list of) tpr file(s).
Generates a work specification based on the provided simulation input and returns a handle to the
MD simulation element of the workflow. Key word arguments can override simulation behavior from
``input``.
If the MD operation discovers artifacts from a previous simulation that was launched from the same input,
the simulation resumes from the last checkpointed step. If ``append_output`` is set ``False``, existing
artifacts are kept separate from new output with the standard file naming convention,
and new output begins from the last checkpointed step, if any.
Setting ``end_time`` redefines the end point of the simulation trajectory from what was provided in
``input``. It is equivalent to changing the number of steps requested in the MDP (or TPR) input, but
the time is provided as picoseconds instead of a number of time steps.
.. deprecated:: 0.0.7
If ``steps=N`` is provided and N is an integer
greater than or equal to 1, the MD operation advances the trajectory by ``N`` steps, regardless of the number
of simulation steps specified in ``input`` or ``end_time``. For convenience, setting ``steps=None`` does not override
``input``.
Note that when it is not ``None``, ``steps`` takes precedence over ``end_time`` and ``input``, but can still be
superceded by a signal, such as if an MD plugin or other code has a simulation completion condition that occurs
before ``N`` additional steps have run.
Where key word arguments correspond to ``gmx mdrun`` command line options, the corresponding flags are noted below.
Keyword Arguments:
input (str): *Required* string or list of strings giving the filename(s) of simulation input
append_output (bool): Append output for continuous trajectories if True, truncate existing output data if False. (default True)
end_time (float): Specify the final time in the simulation trajectory, overriding input read from TPR.
grid (tuple): Domain decomposition grid divisions (nx, ny, nz). (-dd)
max_hours (float): Terminate after 0.99 times this many hours if simulation is still running. (-maxh)
pme_ranks (int): number of separate ranks to be used for PME electrostatics. (-npme)
threads_per_pme_rank (int): Number of OpenMP threads per PME rank. (-ntomp_pme)
steps (int): Override input files and run for this many steps. (-nsteps; deprecated)
threads (int): Total number of threads to start. (-nt)
threads_per_rank (int): number of OpenMP threads to start per MPI rank. (-ntomp)
tmpi (int): number of thread-MPI ranks to start. (-ntmpi)
.. versionchanged:: 0.1
*pme_threads_per_rank* renamed to *threads_per_pme_rank*.
Returns:
simulation member of a gmx.workflow.WorkSpec object
Produces a WorkSpec with the following data::
version: gmxapi_workspec_0_1
elements:
tpr_input:
namespace: gromacs
operation: load_tpr
params: {'input': ['tpr_filename1', 'tpr_filename2', ...]}
md_sim:
namespace: gmxapi
operation: md
depends: ['tpr_input']
params: {'kw1': arg1, 'kw2': arg2, ...}
Bugs: version 0.0.6
* There is not a way to programatically check the current step number on disk.
See https://github.com/kassonlab/gmxapi/issues/56 and https://github.com/kassonlab/gmxapi/issues/85
"""
import os
usage = "argument to from_tpr() should be a valid filename or list of filenames, followed by optional key word arguments."
# Normalize to tuple input type.
if isinstance(input, list) or isinstance(input, tuple):
tpr_list = tuple([to_string(element) for element in input])
else:
try:
tpr_list = (to_string(input), )
except:
raise exceptions.UsageError(usage)
# Check for valid filenames
for arg in tpr_list:
if not (os.path.exists(arg) and os.path.isfile(arg)):
arg_path = os.path.abspath(arg)
raise exceptions.UsageError(usage + " Got {}".format(arg_path))
# Note: These are runner parameters, not MD parameters, and should be in the call to gmx.run() instead of here.
# Reference https://github.com/kassonlab/gmxapi/issues/95
params = {}
for arg_key in kwargs:
if arg_key == 'grid' or arg_key == 'dd':
params['grid'] = tuple(kwargs[arg_key])
elif arg_key == 'pme_ranks' or arg_key == 'npme':
params['pme_ranks'] = int(kwargs[arg_key])
elif arg_key == 'threads' or arg_key == 'nt':
params['threads'] = int(kwargs[arg_key])
elif arg_key == 'tmpi' or arg_key == 'ntmpi':
params['tmpi'] = int(kwargs[arg_key])
elif arg_key == 'threads_per_rank' or arg_key == 'ntomp':
params['threads_per_rank'] = int(kwargs[arg_key])
elif arg_key == 'pme_threads_per_rank' or arg_key == 'threads_per_pme_rank' or arg_key == 'ntomp_pme':
# TODO: Remove this temporary accommodation.
assert not gmx.version.api_is_at_least(0, 2)
if arg_key == 'pme_threads_per_rank':
warnings.warn(
"Key word pme_threads_per_rank has been renamed to threads_per_pme_rank.",
DeprecationWarning)
params['threads_per_pme_rank'] = int(kwargs[arg_key])
elif arg_key == 'steps' or arg_key == 'nsteps':
if kwargs[arg_key] is None:
# None means "don't override the input" which is indicated by a parameter value of -2 in GROMACS 2019
steps = -2
else:
# Otherwise we require steps to be a positive integer
try:
steps = int(kwargs[arg_key])
if steps < 1:
raise exceptions.ValueError(
'steps to run must be at least 1')
except (TypeError, ValueError) as e:
# steps is not an integer.
raise exceptions.TypeError(
'"steps" could not be interpreted as an integer.')
# The "nsteps" command line flag will be removed in GROMACS 2020
# and so "steps" is deprecated in gmxapi 0.0.7
warnings.warn(
"`steps` keyword argument is deprecated. Consider `end_time` instead.",
DeprecationWarning)
params['steps'] = steps
elif arg_key == 'max_hours' or arg_key == 'maxh':
params['max_hours'] = float(kwargs[arg_key])
elif arg_key == 'append_output':
# Try not to encourage confusion with the `mdrun` `-noappend` flag, which would be a confusing double negative if represented as a bool.
params['append_output'] = bool(kwargs[arg_key])
elif arg_key == 'end_time':
params[arg_key] = float(kwargs[arg_key])
else:
raise exceptions.UsageError(
"Invalid key word argument: {}. {}".format(arg_key, usage))
# Create an empty WorkSpec
workspec = WorkSpec()
# Create and add the Element for the tpr file(s)
inputelement = WorkElement(namespace='gromacs',
operation='load_tpr',
params={'input': tpr_list})
inputelement.name = 'tpr_input'
if inputelement.name not in workspec.elements:
# Operations such as this need to be replaced with accessors or properties that can check the validity of the WorkSpec
workspec.elements[inputelement.name] = inputelement.serialize()
inputelement.workspec = workspec
# Create and add the simulation element
# We can add smarter handling of the `depends` argument, but it is only critical to check when adding the element
# to a WorkSpec.
mdelement = WorkElement(operation='md',
depends=[inputelement.name],
params=params)
mdelement.name = 'md_sim'
# Check that the element has not already been added, but that its dependency has.
workspec.add_element(mdelement)
return mdelement
def __init__(self, **kwargs):
"""Initialization defines the unique input requirements of a work graph node.
Initialization parameters map to the parameters of the wrapped function with
addition(s) to support gmxapi data flow and deferred execution.
If provided, an ``input`` keyword argument is interpreted as a parameter pack
of base input. Inputs also present as standalone keyword arguments override
values in ``input``.
Inputs that are handles to gmxapi operations or outputs induce data flow
dependencies that the framework promises to satisfy before the Operation
executes and produces output.
"""
#
# Define the unique identity and data flow constraints of this work graph node.
#
# TODO: (FR4) generalize
input_dependencies = []
# TODO: Make allowed input strongly specified in the Operation definition.
# TODO: Resolve execution dependencies at run() and make non-data
# execution `dependencies` just another input that takes the default
# output of an operation and doesn't do anything with it.
# If present, kwargs['input'] is treated as an input "pack" providing _default_ values.
input_kwargs = {}
if 'input' in kwargs:
provided_input = kwargs.pop('input')
if provided_input is not None:
# Try to determine what 'input' is.
# TODO: (FR5+) handling should be related to Context.
# The process of accepting input arguments includes resolving placement in
# a work graph and resolving the Context responsibilities for graph nodes.
if hasattr(provided_input, 'run'):
input_dependencies.append(provided_input)
else:
# Assume a parameter pack is provided.
for key, value in provided_input.items():
input_kwargs[key] = value
assert 'input' not in kwargs
assert 'input' not in input_kwargs
# Merge kwargs and kwargs['input'] (keyword parameters versus parameter pack)
for key in kwargs:
if key in self.signature.parameters:
input_kwargs[key] = kwargs[key]
else:
raise exceptions.UsageError('Unexpected keyword argument: {}'.format(key))
# TODO: (FR4) Check input types
self.__input = PyFuncInput(args=[],
kwargs=input_kwargs,
dependencies=input_dependencies)
# TODO: (FR5+) Split the definition of the resource structure
# and the resource initialization.
# Resource structure definition logic can be moved to the level
# of the class definition. We need knowledge of the inputs to
# uniquely identify the resources for this operation instance.
# Implementation suggestion: Context-provided metaclass defines
# resource manager interface for this Operation. Factory function
# initializes compartmentalized resource management at object creation.
self.__resource_manager = get_resource_manager(self)
