def executable(context, task: scalems.subprocess.Subprocess):
"""Implement scalems.executable for the RPWorkflowContext.
Provide the awaitable result for the Subprocess Future behavior.
TODO: Tie return value to SubprocessResult.
TODO: Manage the state of the Subprocess instance.
TODO: Finish implementing Future.
TODO: Move Future base to asyncio.Future.
"""
if not isinstance(context, scalems.radical.RPWorkflowContext):
raise DispatchError('This resource factory is only valid for RADICAL Pilot workflow contexts.')
task_input = task.input_collection()
args = list([arg for arg in task_input.argv])
# TODO: stream based input with PIPE.
kwargs = {
'stdin': None,
'stdout': None,
'stderr': None,
'env': None
}
# Construct the RP executable task description.
# Ref: https://radicalpilot.readthedocs.io/en/stable/apidoc.html#radical.pilot.ComputeUnit
task_description = {'executable': args[0],
'cpu_processes': 1}
task = context.umgr.submit_units(context.rp.ComputeUnitDescription(task_description))
task_ref = weakref.ref(task)
# TODO: The Context should be in charge of creating the Future.
try:
future = RPFuture(task_ref)
except TypeError as e:
raise InternalError('Failed to get a reference to a new RADICAL Pilot task.') from e
except Exception as e:
raise InternalError('Unknown error occurred in RADICAL Pilot connector.') from e
def cb(obj, state):
# Where is the state enumeration?
# TODO: assert state in [...]
if task_ref().exit_code is not None:
future.set_result(RPResult())
task.register_callback(cb)
async def coroutine():
# task.wait() just hangs. Using umgr.wait_units() instead...
# task.wait()
# TODO: Why does task.wait() not work?
# TODO: Can we at least wait on a specific task ID?
context.umgr.wait_units()
return future
return coroutine()
def _parse_option(arg: str) -> tuple:
if not isinstance(arg, str):
raise InternalError(
'Bug: This function should only be called with a str.')
if arg.count('=') != 1:
raise argparse.ArgumentTypeError(
'Expected a key/value pair delimited by "=".')
return tuple(arg.split('='))
def add_item(self, task_description) -> scalems.context.ItemView:
# # TODO: Resolve implementation details for *operation*.
# if operation != 'scalems.executable':
# raise MissingImplementationError('No implementation for {} in {}'.format(operation, repr(self)))
# # Copy a static copy of the input.
# # TODO: Dispatch tasks addition, allowing negotiation of Context capabilities and subscription
# # to resources owned by other Contexts.
# if not isinstance(bound_input, scalems.subprocess.SubprocessInput):
# raise ValueError('Only scalems.subprocess.SubprocessInput objects supported as input.')
if not isinstance(task_description, scalems.subprocess.Subprocess):
raise MissingImplementationError('Operation not supported.')
uid = task_description.uid()
if uid in self.task_map:
# TODO: Consider decreasing error level to `warning`.
raise DuplicateKeyError('Task already present in workflow.')
logger.debug('Adding {} to {}'.format(str(task_description), str(self)))
record = {
'uid': task_description.uid().hex(),
'type': task_description.resource_type().scoped_identifier(),
'input': {}
}
task_input = task_description.input_collection()
for field in dataclasses.fields(task_input):
name = field.name
try:
# TODO: Need serialization typing.
record['input'][name] = getattr(task_input, name)
except AttributeError as e:
raise InternalError('Unexpected missing field.') from e
record = json.dumps(record, cls=Encoder)
# TODO: Make sure there are no artifacts of shallow copies that may result in a user modifying nested objects unexpectedly.
item = scalems.context.Task(self, record)
# TODO: Check for ability to dispatch.
self.task_map[uid] = item
# TODO: Register task factory (dependent on executor).
# TODO: Register input factory (dependent on dispatcher and task factory / executor).
# TODO: Register results handler (dependent on dispatcher end points).
task_view = scalems.context.ItemView(context=self, uid=uid)
# TODO: Use an abstract event hook for `add_item` and other (decorated) methods.
# Internal functionality can probably explicitly register and unregister, accounting
# for the current details of thread safety. External access will need to be in
# terms of a concurrency framework, so we can use a scoped `async with event_subscription`
# to create an asynchronous iterator (with some means to externally end the subscription,
# either through the generator protocol directly or through logic in the provider of the iterator)
dispatcher_queue = self._queue
# self._queue may be removed by another thread before we add the item to it,
# but that is fine. There is nothing wrong with abandoning an unneeded queue.
if dispatcher_queue is not None:
logger.debug('Running dispatcher detected. Entering live dispatching hook.')
# Add the AddItem message to the queue.
assert isinstance(dispatcher_queue, queue.SimpleQueue)
dispatcher_queue.put({'add_item': task_description})
return task_view
def decode(cls,
obj) -> typing.Union[UnboundObject, BaseDecoded]: # noqa: C901
"""Create unbound SCALE-MS objects from their basic Python representations.
We assume this is called in a bottom-up manner as a nested record is deserialized.
Unrecognized objects are returned unaltered because they may be members
of an enclosing object with appropriate dispatching.
.. todo:: Consider where to register transcoders for compatible/virtual types.
E.g. Infer np.array(..., dtype=int) -> scalems.Integer
This is a small number of cases, since we can lean on the descriptors in the buffer protocol.
"""
if not isinstance(obj, dict):
# Probably don't have any special handling for such objects until we know what they are nested in.
...
else:
assert isinstance(obj, dict)
if 'schema' in obj:
# We currently have very limited schema processing.
try:
spec = obj['schema']['spec']
except KeyError:
spec = None
if not isinstance(spec, str) or spec != 'scalems.v0':
# That's fine...
logger.info('Unrecognized *schema* when decoding object.')
return obj
if 'name' not in obj['schema'] or not isinstance(
obj['schema']['name'], str):
raise InternalError('Invalid schema.')
else:
# schema = obj['schema']['name']
...
# Dispatch the object...
...
raise MissingImplementationError(
'We do not yet support dynamic type registration through the work record.'
)
if 'type' in obj:
# Dispatch the decoding according to the type.
try:
dispatch = cls.get_decoder(obj['type'])
except TypeError:
dispatch = BasicSerializable.decode
if dispatch is not None:
return dispatch(obj)
# Just return un-recognized objects unaltered.
return obj
def serialize(self) -> str:
"""Encode the task as a JSON record.
Input and Result will be serialized as references.
The caller is responsible for serializing existing records
for bound objects, if they exist.
"""
record = {}
record['uid'] = self.uid().hex()
# "label" not yet supported.
record['type'] = self.resource_type().scoped_identifier()
record['input'] = dataclasses.asdict(self._bound_input) # reference
record['result'] = dataclasses.asdict(self._result) # reference
try:
serialized = json.dumps(record, default=encode)
except TypeError as e:
logger.critical(
'Missing encoding logic for scalems data. Encoder says ' +
str(e))
raise InternalError('Missing serialization support.') from e
# raise MissingImplementationError('To do...')
return serialized
def uid(self) -> bytes:
if not isinstance(self._uid, bytes):
raise InternalError(
'Task._uid was stored as bytes. Implementation changed?')
return bytes(self._uid)
def __init__(self, fingerprint: bytes):
self._data = bytes(fingerprint)
# Expecting a 256-bit SHA256 hash digest
if len(self._data) != 32:
raise InternalError('Expected a 256-bit hash digest. Got '
f'{repr(fingerprint)}')
async def input_resource_scope(context, task_input: typing.Union[
scalems.subprocess.SubprocessInput,
typing.Awaitable[scalems.subprocess.SubprocessInput]]):
"""Manage the actual execution context of the asyncio.subprocess.Process.
Translate a scalems.subprocess.SubprocessInput to a local SubprocessInput instance.
InputResource factory for *subprocess* based implementations.
TODO: How should this be composed in terms of the context and (local) resource type?
"""
# Await the inputs.
if inspect.isawaitable(task_input):
task_input = await task_input
# TODO: Use an awaitable to deliver the SubprocessInput providing the arguments.
# Consider providing generic awaitable behavior through a decorator or base class
# so we don't need to do extra checks and can just await all scalems objects.
# TODO: What sort of validation or normalization do we want to do for the executable name?
if not isinstance(task_input, scalems.subprocess.SubprocessInput):
raise InternalError('Unexpected input type.')
program = pathlib.Path(shutil.which(task_input.argv[0]))
if not program.exists():
raise InternalError(
'Could not find executable. Input should be vetted before this point.'
)
args = list(str(arg) for arg in task_input.argv[1:])
# Warning: If subprocess.Popen receives *env* argument that is not None, it **replaces** the
# default environment (a duplicate of the caller's environment). We might want to provide
# fancier semantics to copy or reject select variables from the environment or to
# dynamically read the default environment at execution time (note that the results of
# such an operation would not represent a unique result!)
get_env = lambda: None
get_stdin = lambda: None
if task_input.stdin is not None:
# TODO: Strengthen the typing for stdin parameter.
if not isinstance(task_input.stdin, (os.PathLike, str, list, tuple)):
# Note: this is an internal error indicating a bug in SCALEMS.
raise InternalError('No handler for stdin argument of this form.' +
repr(task_input.stdin))
if isinstance(task_input.stdin, (list, tuple)):
# TODO: Assign appropriate responsibility for maintaining filesystem artifacts.
infile = 'stdin'
with open(infile, 'w') as fp:
# Normalize line endings for local environment.
fp.writelines([line.rstrip() for line in task_input.stdin])
else:
infile = task_input.stdin
get_stdin = lambda path=os.fsencode(infile): open(
os.path.abspath(path), 'r')
stdout = task_input.stdout
if stdout is None:
stdout = 'stdout'
if not isinstance(task_input.stdout, (os.PathLike, str)):
# Note: this is an internal error indicating a bug in SCALEMS.
raise InternalError('No handler for stdout argument of this form. ' +
repr(task_input.stdout))
if os.path.exists(stdout):
raise InternalError(
'Dirty working directory is not recoverable in current implementation.'
)
get_stdout = lambda path=os.path.abspath(stdout): open(path, 'w')
stderr = task_input.stderr
if stderr is None:
stderr = 'stderr'
if not isinstance(task_input.stderr, (os.PathLike, str)):
# Note: this is an internal error indicating a bug in SCALEMS.
raise InternalError('No handler for stderr argument of this form.' +
repr(task_input.stderr))
if os.path.exists(stderr):
raise InternalError(
'Dirty working directory is not recoverable in current implementation.'
)
get_stderr = lambda path=os.path.abspath(stderr): open(path, 'w')
# Create scoped resources. This depends somewhat on the input.
# For each non-None stdio stream, we need to provide an open file-like handle.
# Note: there may be a use case for scoped run-time determination of *env*,
# but we have not yet allowed for that.
with get_stdin() as fh_in, get_stdout() as fh_out, get_stderr() as fh_err:
try:
# TODO: Create SubprocessInput with a coroutine so that we can yield an awaitable.
subprocess_input = SubprocessInput(program,
args,
stdin=fh_in,
stdout=fh_out,
stderr=fh_err,
env=get_env())
# Provide resources to the implementation coroutine.
yield subprocess_input # needs to be an awaitable... ?
finally:
# Clean up scoped resources.
# Deliver output files?
...
def add_item(self, task_description) -> ItemView:
# # TODO: Resolve implementation details for *operation*.
# if operation != 'scalems.executable':
# raise MissingImplementationError('No implementation for {} in {}'.format(
# operation, repr(self)))
# # Copy a static copy of the input.
# # TODO: Dispatch tasks addition, allowing negotiation of Context capabilities
# and subscription
# # to resources owned by other Contexts.
# if not isinstance(bound_input, scalems.subprocess.SubprocessInput):
# raise ValueError('Only scalems.subprocess.SubprocessInput objects
# supported as input.')
# TODO: Replace with type-based dispatching or some normative interface test.
from .subprocess import Subprocess
if not isinstance(task_description, (Subprocess, dict)):
raise MissingImplementationError('Operation not supported.')
if hasattr(task_description, 'uid'):
uid: bytes = task_description.uid()
if uid in self.tasks:
# TODO: Consider decreasing error level to `warning`.
raise DuplicateKeyError('Task already present in workflow.')
logger.debug('Adding {} to {}'.format(str(task_description),
str(self)))
record = {
'uid': uid.hex(),
'type': task_description.resource_type().scoped_identifier(),
'input': {}
}
task_input = task_description.input_collection()
for field in dataclasses.fields(task_input):
name = field.name
try:
# TODO: Need serialization typing.
record['input'][name] = getattr(task_input, name)
except AttributeError as e:
raise InternalError('Unexpected missing field.') from e
else:
assert isinstance(task_description, dict)
assert 'uid' in task_description
uid = task_description['uid']
implementation_identifier = task_description.get(
'implementation', None)
if not isinstance(implementation_identifier, list):
raise DispatchError('Bug: bad schema checking?')
if uid in self.tasks:
# TODO: Consider decreasing error level to `warning`.
raise DuplicateKeyError('Task already present in workflow.')
logger.debug('Adding {} to {}'.format(str(task_description),
str(self)))
record = {
'uid': uid.hex(),
'type': tuple(implementation_identifier),
'input': task_description
}
serialized_record = json.dumps(record, default=encode)
# TODO: Make sure there are no artifacts of shallow copies that may result in
# a user modifying nested objects unexpectedly.
item = Task(self, serialized_record)
# TODO: Check for ability to dispatch.
# Note module dependencies, etc. and check in target execution environment
# (e.g. https://docs.python.org/3/library/importlib.html#checking-if-a-module
# -can-be-imported)
# TODO: Provide a data descriptor and possibly a more formal Workflow class.
# We do not yet check that the derived classes actually initialize self.tasks.
self.tasks[uid] = item
task_view = ItemView(manager=self, uid=uid)
# TODO: Register task factory (dependent on executor).
# TODO: Register input factory (dependent on dispatcher and task factory /
# executor).
# TODO: Register results handler (dependent on dispatcher end points).
# TODO: Consider an abstract event hook for `add_item` and other (decorated)
# methods.
# Internal functionality can probably explicitly register and unregister,
# accounting for the current details of thread safety.
# External access will need to be in terms of a concurrency framework,
# so we can use a scoped `async with event_subscription`
# to create an asynchronous iterator that a coroutine can use to receive
# add_item messages
# (with some means to externally end the subscription,
# either through the generator protocol directly or through logic in the
# provider of the iterator)
for callback in self._event_hooks['add_item']:
# TODO: Do we need to provide a contextvars.Context object to the callback?
logger.debug(f'Running dispatching hook for add_item subscriber '
f'{repr(callback)}.')
callback(_CommandQueueAddItem({'add_item': uid}))
return task_view
def executable(*args,
manager: scalems.workflow.WorkflowManager = None,
**kwargs):
"""Execute a command line program.
Configure an executable to run in one (or more) subprocess(es).
Executes when run in an execution Context, as part of a work graph.
Process environment and execution mechanism depends on the execution environment,
but is likely similar to (or implemented in terms of) the POSIX execvp system call.
Shell processing of *argv* is disabled to improve determinism.
This means that shell expansions such as environment variables, globbing (``*``),
and other special symbols (like ``~`` for home directory) are not available.
This allows a simpler and more robust implementation, as well as a better
ability to uniquely identify the effects of a command line operation. If you
think this disallows important use cases, please let us know.
Arguments:
manager: Workflow manager to which the work should be submitted.
args: a tuple (or list) to be the subprocess arguments, including the executable
*args* is required. Additional key words are optional.
Other Parameters:
outputs (Mapping): labeled output files, mapping command line flag to one (or
more) filenames.
inputs (Mapping): labeled input files, mapping command line flag to one (or
more) filenames.
environment (Mapping): environment variables to be set in the process
environment.
stdin (str): source for posix style standard input file handle (default None).
stdout (str): Capture standard out to a filesystem artifact, even if it is not
consumed in the workflow.
stderr (str): Capture standard error to a filesystem artifact, even if it is
not consumed in the workflow.
resources (Mapping): Name additional required resources, such as an MPI
environment.
.. todo:: Support POSIX sigaction / IPC traps?
.. todo:: Consider dataclasses.dataclass types to replace reusable/composable
function signatures.
Program arguments are iteratively added to the command line with standard Python
iteration, so you should use a tuple or list even if you have only one parameter.
I.e. If you provide a string with ``arguments="asdf"`` then it will be passed as
``... "a" "s" "d" "f"``. To pass a single string argument, ``arguments=("asdf")``
or ``arguments=["asdf"]``.
*inputs* and *outputs* should be a dictionary with string keys, where the keys
name command line "flags" or options.
Note that the Execution Context (e.g. RPContext, LocalContext, DockerContext)
determines the handling of *resources*. Typical values in *resources* may include
* procs_per_task (int): Number of processes to spawn for an instance of the *exec*.
* threads_per_proc (int): Number of threads to allocate for each process.
* gpus_per_task (int): Number of GPU devices to allocate for and instance of the
*exec*.
* launcher (str): Task launch mechanism, such as `mpiexec`.
Returns:
Output collection contains *exitcode*, *stdout*, *stderr*, *file*.
The *file* output has the same keys as the *outputs* key word argument.
Example:
Execute a command named ``exe`` that takes a flagged option for input
and output file names
(stored in a local Python variable ``my_filename`` and as the string literal
``'exe.out'``)
and an ``origin`` flag
that uses the next three arguments to define a vector.
>>> my_filename = "somefilename"
>>> command = scalems.executable(
... ('exe', '--origin', 1.0, 2.0, 3.0),
... inputs={'--infile': scalems.file(my_filename)},
... outputs={'--outfile': scalems.file('exe.out')})
>>> assert hasattr(command, 'file')
>>> import os
>>> assert os.path.exists(command.file['--outfile'].result())
>>> assert hasattr(command, 'exitcode')
TODO:
Consider input/output files that do not appear on the command line, but which
must figure into data flow.
"""
if manager is None:
manager = _context.get_context()
# TODO: Figure out a reasonable way to check and catch invalid input
# through a dispatcher.
# subprocess_input = context.add(Subprocess.input_type(), *args, **kwargs)
input_type = Subprocess.resource_type().input_type()
if not isinstance(input_type, type):
raise InternalError(
'Bug: {} is not coded correctly for the {}.input_type() interface.'
.format(__name__, str(type(Subprocess.resource_type()))))
# TODO: Add input separately. First, just add the Subprocess object.
# Note: static type checkers may not be able to resolve that `input_type is
# SubprocessInput` for argument checking. Provide local object to the context and
# replace local reference with a view to the workflow item.
# subprocess_input = _context.add_to_workflow(
# context, Subprocess.resource_type().input_type(), *args, **kwargs)
bound_input = SubprocessInput(*args, **kwargs)
director = scalems.workflow.workflow_item_director_factory(Subprocess,
manager=manager)
# Design note: at some point, dynamic workflows will require thread-safe
# workflow editing context. We could either block on acquiring the editor
# context, use an async context manager, or hide the possible async
# aspect by letting the return value of the director be awaitable.
# TODO: This would be more readable in a form like
# `workflow.add_item(Subprocess, bound_input)`
try:
task_view: scalems.workflow.ItemView = director(input=bound_input)
except TypeError as e:
logger.error('Invalid input in SubprocessInput: ' + str(e))
raise
except json.JSONDecodeError as e:
logger.critical('Malformed data: ' + e.msg)
raise InternalError(
'Bug: internal data is not being conditioned properly') from e
except Exception as e:
logger.critical('Unhandled ' + repr(e))
raise
# TODO: The returned value should be a TaskView provided by the Context with
# minimal state or ownership semantics.
return task_view
async def dispatch(self):
"""Start the executor task, then provide a scope for concurrent activity.
Provide the executor with any currently-managed work in a queue.
While the scope is active, new work added to the queue will be picked up
by the executor.
When leaving the `with` block, trigger the executor clean-up and wait for its task to complete.
.. todo:: Clarify re-entrance policy, thread-safety, etcetera, and enforce.
.. todo:: Allow an externally provided dispatcher factory, or even a running dispatcher?
"""
# 1. Install a hook to catch new calls to add_item (the dispatcher_queue) and try not to yield until the current workflow state is obtained.
# 2. Get snapshot of current workflow state with which to initialize the dispatcher. (It is now okay to yield.)
# 3. Bind a new executor to its queue.
# 4. Bind a dispatcher to the executor and the dispatcher_queue.
# 5. Allow the executor and dispatcher to start using the event loop.
# Avoid race conditions while checking for a running dispatcher.
async with self._dispatcher_lock:
# Dispatching state may be reentrant, but it does not make sense to re-enter through this call structure.
if self._dispatcher is not None:
raise ProtocolError('Already dispatching through {}.'.format(repr(self._dispatcher())))
# For an externally-provided dispatcher:
# else:
# self._dispatcher = weakref.ref(dispatcher)
# 1. Install a hook to catch new calls to add_item
if self._queue is not None:
raise ProtocolError('Found unexpected dispatcher queue.')
dispatcher_queue = queue.SimpleQueue()
self._queue = dispatcher_queue
# 2. Get snapshot of current workflow state with which to initialize the dispatcher.
# TODO: Topologically sort DAG!
initial_task_list = list(self.task_map.keys())
# It is now okay to yield.
# 3. Bind a new executor to its queue.
# Note: if there were a reason to decouple the executor lifetime from this scope,
# we could consider a more object-oriented interface with it.
executor_queue = asyncio.Queue()
for key in initial_task_list:
await executor_queue.put({'add_item': key})
executor = run_executor(source_context=self, command_queue=executor_queue)
# 4. Bind a dispatcher to the executor_queue and the dispatcher_queue.
# TODO: We should bind the dispatcher directly to the executor, but that requires
# that we make an Executor class with concurrency-safe methods.
# dispatcher = run_dispatcher(dispatcher_queue, executor_queue)
# self._dispatcher = weakref.ref(dispatcher)
# TODO: Toggle active dispatcher state.
# scalems.context._dispatcher.set(...)
# 5. Allow the executor and dispatcher to start using the event loop.
executor_task = asyncio.create_task(executor)
# asyncio.create_task(dispatcher)
try:
# We can surrender control here and leave the executor and dispatcher tasks running
# while evaluating a `with` block suite for the `dispatch` context manager.
yield
except Exception as e:
logger.exception('Uncaught exception while in dispatching context: {}'.format(str(e)))
raise e
finally:
async with self._dispatcher_lock:
self._dispatcher = None
self._queue = None
# dispatcher_queue.put({'control': 'stop'})
# await dispatcher
# TODO: Make sure the dispatcher hasn't died. Look for acknowledgement
# of receipt of the Stop command.
# TODO: Check status...
if not dispatcher_queue.empty():
logger.error('Dispatcher finished while items remain in dispatcher queue. Approximate size: {}'.format(dispatcher_queue.qsize()))
# Stop the executor.
executor_queue.put_nowait({'control': 'stop'})
await executor_task
if executor_task.exception() is not None:
raise executor_task.exception()
# Check that the queue drained.
# WARNING: The queue will never finish draining if executor_task fails.
# I.e. don't `await executor_queue.join()`
if not executor_queue.empty():
raise InternalError('Bug: Executor left tasks in the queue without raising an exception.')
logger.debug('Exiting {} dispatch context.'.format(type(self).__name__))
async def run_executor(source_context: AsyncWorkflowManager, command_queue: asyncio.Queue):
"""Process workflow messages until a stop message is received.
Initial implementation processes commands serially without regard for possible
concurrency.
Towards concurrency:
We can create all tasks without awaiting any of them.
Some tasks will be awaiting results from other tasks.
All tasks will be awaiting a asyncio.Lock or asyncio.Condition for each
required resource, but must do so indirectly.
To avoid dead-locks, we can't have a Lock object for each resource unless
they are managed by an intermediary that can do some serialization of requests.
In other words, we need a Scheduler that tracks the resource pool, packages
resource locks only when they can all be acquired without race conditions or blocking,
and which then notifies the Condition for each task that it is allowed to run.
It should not do so until the dependencies of the task are known to have
all of the resources they need to complete (running with any dynamic dependencies
also running) and, preferably, complete.
Alternatively, the Scheduler can operate in blocks, allocating all resources,
offering the locks to tasks, waiting for all resources to be released, then repeating.
We can allow some conditions to "wake up" the scheduler to back fill a block
of resources, but we should be careful with that.
(We still need to consider dynamic tasks that
generate other tasks. I think the only way to distinguish tasks which can't be
dynamic from those which might be would be with the `def` versus `async def` in
the implementing function declaration. If we abstract `await` with `scalems.wait`,
we can throw an exception at execution time after checking a ContextVar.
It may be better to just let implementers use `await` for dynamically created tasks,
but we need to make the same check if a function calls `.result()` or otherwise
tries to create a dependency on an item that was not allocated resources before
the function started executing. In a conservative first draft, we can simply
throw an exception if a non-`async def` function attempts to call a scalems workflow
command like add_item while in an executing context.)
"""
# Could also accept a "stop" Event object, but we would need some other way to yield
# on an empty queue.
while True:
command = await command_queue.get()
try:
logger.debug('Executor is handling {}'.format(repr(command)))
# TODO: Use formal RPC protocol.
if 'control' in command:
if command['control'] == 'stop':
return
else:
raise ProtocolError('Unknown command: {}'.format(command['control']))
if 'add_item' not in command:
raise MissingImplementationError('Executor has no implementation for {}'.format(str(command)))
key = command['add_item']
item = source_context.item(key)
if not isinstance(item, scalems.context.Task):
raise InternalError('Expected {}.item() to return a scalems.context.Task'.format(repr(source_context)))
# TODO: Ensemble handling
item_shape = item.description().shape()
if len(item_shape) != 1 or item_shape[0] != 1:
raise MissingImplementationError('Executor cannot handle multidimensional tasks yet.')
# TODO: Automatically resolve resource types.
task_type_identifier = item.description().type().identifier()
if task_type_identifier != 'scalems.subprocess.SubprocessTask':
raise MissingImplementationError('Executor does not have an implementation for {}'.format(str(task_type_identifier)))
task_type = scalems.subprocess.SubprocessTask()
# TODO: Use abstract input factory.
logger.debug('Resolving input for {}'.format(str(item)))
input_type = task_type.input_type()
input_record = input_type(**item.input)
input_resources = operations.input_resource_scope(context=source_context, task_input=input_record)
# We need to provide a scope in which we guarantee the availability of resources,
# such as temporary files provided for input, or other internally-generated
# asyncio entities.
async with input_resources as subprocess_input:
logger.debug('Creating coroutine for {}'.format(task_type.__class__.__name__))
# TODO: Use abstract task factory.
coroutine = operations.subprocessCoroutine(subprocess_input)
logger.debug('Creating asyncio Task for {}'.format(repr(coroutine)))
awaitable = asyncio.create_task(coroutine)
# TODO: Use abstract results handler.
logger.debug('Waiting for task to complete.')
result = await awaitable
subprocess_exception = awaitable.exception()
if subprocess_exception is not None:
logger.exception('subprocess task raised exception {}'.format(str(subprocess_exception)))
raise subprocess_exception
logger.debug('Setting result for {}'.format(str(item)))
item.set_result(result)
finally:
logger.debug('Releasing "{}" from command queue.'.format(str(command)))
command_queue.task_done()