async def target(self, event, pipeline, context):
command_name = await self.command_name.read(event, pipeline, context)
ebnf = await self.ebnf.read(event, pipeline, context)
# ---
if command_name:
try:
command = m42pl.command(command_name)
source = {
command_name: command
}
except Exception:
source = {}
else:
source = m42pl.commands.ALIASES
# ---
for alias, command in source.items():
yield derive(event, data={
'command': {
'alias': alias,
'aliases': command._aliases_,
'schema': command._schema_,
'about': command._about_,
'syntax': command._syntax_,
'type': list(filter(None, map(lambda t: issubclass(command, t) and t.__name__ or None, self.types)))[0],
'ebnf': ebnf is True and getattr(command, '_ebnf_', '') or ''
}
})
def run_pipeline(context: str, event: str, chan_read: Queue, chan_write: Queue,
chunk: int, chunks: int, modules: dict):
"""Runs a split pipeline.
:param context: Source context as JSON string
:param event: Source event as JSON string
:param chan_read: Multiprocessing queue output (read from);
:param chan_write: Multiprocessing queue input (write to);
:param chunk: Current chunk number (starts at 0)
:param chunks: Total number of chunks (i.e. number of parallel
split pipelines / process)
:param modules: Modules names and paths to load
"""
async def run(pipeline, context, event):
async with context.kvstore:
async for _ in pipeline(context, event):
pass
# Load missing modules
m42pl.load_modules(names=modules['names'], paths=modules['paths'])
# Build local context and event from seralized instances
context = Context.from_dict(json.loads(context))
# event = Event.from_dict(json.loads(event))
event = json.loads(event)
# Get main pipeline
pipeline = context.pipelines['main']
# ---
# Customize main pipeline to read & write to the multiprocessing pipe
# Intermediate or last command: read input from MPI pipe
if chan_read:
pipeline.commands = [
m42pl.command('mpi-receive')(chan_read),
] + pipeline.commands
# First or intermediate command: write output to MPI pipe
if chan_write:
pipeline.commands.append(m42pl.command('mpi-send')(chan_write))
# ---
# Rebuild and reconfigure pipeline
pipeline.build()
pipeline.set_chunk(chunk, chunks)
# ---
# Close stdout and run
if chan_write is not None:
sys.stdout = DevNull()
sys.stderr = DevNull()
asyncio.run(run(pipeline, context, event))
def __new__(cls, *args, **kwargs) -> tuple:
"""Returns a new (encode, send) commands tuple.
"""
self = super(MPISend, cls).__new__(cls)
self.__init__(*args, **kwargs)
return (
# m42pl.command('msgpack')(dest_field=self.msgpack_field),
m42pl.command('encode')(
codec='msgpack',
dest=self.msgpack_field
),
self
)
def __new__(cls, *args, **kwargs) -> tuple:
"""Returns a new (receive, decode) commands tuple.
"""
self = super(MPIReceive, cls).__new__(cls)
self.__init__(*args, **kwargs)
return (
self,
# m42pl.command('msgunpack')(src_field=self.msgpack_field)
m42pl.command('decode')(
codec='msgpack',
src=self.msgpack_field
)
)
def from_dict(cls, data: dict) -> object:
"""Returns a new :class:`Pipeline` from a :class:`dict`.
The source map ``data`` must match with the following map:
.. code-block:: python
{
'commands': [ # Commands list
{
'alias': '...', # Name (alias)
'args': [], # Arguments
'kwargs': {} # Keyword arguments
},
# ... # Other attrs are ignored
]
}
**Important**: This function does not invoke the commands'
`__new__` method: this means that :param:`data` must have been
generated from an already built pipeline, i.e. whoms commands
have been instanciated.
The underlying reason is that commands may overload their
`__new__` method to return more than one class instance.
:param data: Source pipeline as :class:`dict`
"""
# Build the commands list
commands = []
for command in data['commands']:
commands.append(object.__new__(m42pl.command(command['alias'])))
commands[-1].__init__(*command.get('args', []),
**command.get('kwargs', {}))
# Remove original commands list from dict
data.pop('commands')
# Builds and returns a new pipeline
# return cls( { **{'commands': commands}, **dc} )
return cls(commands=commands)
def command(self, items):
def configure_command(cmd, items):
if isinstance(cmd, (list, tuple)):
for c in cmd:
configure_command(c, items)
else:
cmd._lncol_ = items[0].line, items[0].column
# --- Depreciation warning ---
# cmd._offset_ = items[0].pos_in_stream
cmd._offset_ = items[0].start_pos
cmd._name_ = command_name
# Extract command name and body
command_name = str(items[0])
command_body = ' '.join(
filter(
None,
len(items) > 1 and items[1:] or []
)
)
# Instanciate new command
try:
cmd = m42pl.command(command_name).from_script(command_body)
except Exception as error:
raise errors.ScriptError(
line=items[0].line,
column=items[0].column,
# --- Depreciation warning ---
# offset=items[0].pos_in_stream,
offset=items[0].start_pos,
message=str(error)
)
# Setup command instance
configure_command(cmd, items)
# Done
return cmd
def __call__(self, args):
super().__call__(args)
with open(args.source) as fd:
print(m42pl.command(self.command)(fd.read())())
def __init_subclass__(cls):
def new_testcase(test_script: TestScript):
"""Generic test case factory.
Returns a new test case method from a :class:`TestScript`
instance.
"""
def testcase(self):
"""Tests command execution and results.
This generic test case method performs the following
operations and tests:
* Generate the test case source script
* Initialize M42PL
* Run the previously generated source script
* Assert the number of results match the expected
* Assert the content of results
"""
# ---
# Generate the test case source script by merging the
# test case class's script's snippets and the test
# script code.
source = dedent(f'''\
{cls.script_begin}
{test_script.source}
{cls.script_end}
''')
# ---
# Local copy the test script parameters
expected = test_script.expected
fields_in = test_script.fields_in
fields_out = test_script.fields_out
first_event = test_script.first_event
# ---
# Initialize M42PL
kvstore = m42pl.kvstore('local')()
dispatcher = m42pl.dispatcher('local_test')()
# ---
# Run the the test script
results = dispatcher(source, kvstore, first_event)
# ---
# Test results length
self.assertEqual(len(results), len(expected))
# ---
# Test results content
# Events are compared on-by-one with each other from
# the results set and the expected set.
for res, exp in zip(results, expected):
# Clean up all fields but the selected one
for dataset in (res, exp):
# Keep only fields named in fields_in
if len(fields_in):
for field in [
k for k, v in dataset['data'].items()
if k not in fields_in
]:
dataset['data'].pop(field)
# Remove fields named in fields_out
if len(fields_out):
for field in fields_out:
dataset['data'].pop(field)
# Assert equality between result and expected
self.assertDictEqual(res['data'], exp['data'])
# ---
# Return the generated test case function `testcase`
return testcase
# Generated tests methods
for test_scripts in [cls.expected_success, cls.expected_failure]:
for test_script in test_scripts:
setattr(cls, f'test_script_{test_script.name}',
new_testcase(test_script))
# Initialize M42PL and setup command class only if we're
# targetting a final test case class (having a command alias)
if len(cls.command_alias):
m42pl.load_modules()
cls.command = m42pl.command(cls.command_alias)
def __call__(self, args):
super().__call__(args)
print(m42pl.command(args.command)._ebnf_)