def testcase(self):
"""Tests command execution and results.
This generic test case method performs the following
operations and tests:
* Generate the test case source script
* Initialize M42PL
* Run the previously generated source script
* Assert the number of results match the expected
* Assert the content of results
"""
# ---
# Generate the test case source script by merging the
# test case class's script's snippets and the test
# script code.
source = dedent(f'''\
{cls.script_begin}
{test_script.source}
{cls.script_end}
''')
# ---
# Local copy the test script parameters
expected = test_script.expected
fields_in = test_script.fields_in
fields_out = test_script.fields_out
first_event = test_script.first_event
# ---
# Initialize M42PL
kvstore = m42pl.kvstore('local')()
dispatcher = m42pl.dispatcher('local_test')()
# ---
# Run the the test script
results = dispatcher(source, kvstore, first_event)
# ---
# Test results length
self.assertEqual(len(results), len(expected))
# ---
# Test results content
# Events are compared on-by-one with each other from
# the results set and the expected set.
for res, exp in zip(results, expected):
# Clean up all fields but the selected one
for dataset in (res, exp):
# Keep only fields named in fields_in
if len(fields_in):
for field in [
k for k, v in dataset['data'].items()
if k not in fields_in
]:
dataset['data'].pop(field)
# Remove fields named in fields_out
if len(fields_out):
for field in fields_out:
dataset['data'].pop(field)
# Assert equality between result and expected
self.assertDictEqual(res['data'], exp['data'])
def from_dict(cls, data: dict) -> 'Context':
"""Returns a new :class:`Context` from a :class:`dict`.
"""
return cls(pipelines=dict([
(name, m42pl.pipeline.Pipeline.from_dict(pipeline))
for name, pipeline in data['pipelines'].items()
]),
kvstore=m42pl.kvstore(data['kvstore']['alias'])(
*data['kvstore']['args'], data['kvstore']['kwargs']))
def __call__(self, args):
super().__call__(args)
dispatcher = m42pl.dispatcher(
args.dispatcher)(**args.dispatcher_kwargs)
kvstore = m42pl.kvstore(args.kvstore)(**args.kvstore_kwargs)
status = asyncio.run(dispatcher.status(kvstore, args.identifier))
# ---
if args.output == 'json':
self.ouptut_json(status)
elif args.output == 'list':
self.output_list(status)
def __call__(self, args):
super().__call__(args)
with open(args.source, 'r') as fd:
source = fd.read()
try:
# Select, instanciate and run dispatcher
m42pl.dispatcher(args.dispatcher)()(
source=source,
kvstore=m42pl.kvstore(args.kvstore)(),
event=Event.from_dict(json.loads(args.event)))
except Exception as error:
print(CLIErrorRender(error, source).render())
if args.raise_errors:
raise
def __call__(self, args):
super().__call__(args)
# Build aliases list for completer
self.aliases = [alias for alias, _ in m42pl.commands.ALIASES.items()]
# Select and instanciate dispatcher
# dispatcher = m42pl.dispatcher(args.dispatcher)(**args.dispatcher_kwargs)
# Select and connect KVStore
kvstore = m42pl.kvstore(args.kvstore)(**args.kvstore_kwargs)
# Read history file
if args.history:
self.history_file = Path(args.history)
if self.history_file.is_file():
readline.read_history_file(self.history_file)
# Print status
print(f'{len(self.aliases)} commands loaded')
# REPL loop
while True:
try:
# Register SINGINT (note the underlying pipelines will also
# register it; thats why we need regsiter it after each loop)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Read and cleanup script
source = input(self.prompt).lstrip(' ').rstrip(' ')
if len(source):
# Try to interpret source as builtin
rx = self.regex_builtins.match(source)
if rx:
getattr(self, f'builtin_{rx.groupdict()["name"]}')()
# Otherwise, interpret source as a M42PL pipeline
else:
if not self.dispatcher:
self.dispatcher = m42pl.dispatcher(
args.dispatcher)(**args.dispatcher_kwargs)
readline.write_history_file(self.history_file)
self.dispatcher(
source=source,
kvstore=kvstore,
# event=len(args.event) > 0 and Event(data=args.event) or None
event=Event(args.event))
except EOFError:
self.stop()
except Exception as error:
print(CLIErrorRender(error, source).render())
if args.raise_errors:
raise
def __call__(self, args):
super().__call__(args)
# Setup PromptToolkit
self.prompt = PromptSession(
PromptPrefix(args.prefix or '<bold>m42pl@{w}</bold>'),
multiline=True,
bottom_toolbar=self.prompt_bottom_toolbar,
prompt_continuation=self.prompt_continuation,
history=FileHistory(args.history or self.history_file),
completer=REPLCompleter(builtins=self.builtins.list_builtins()),
key_bindings=self.prompt_keys_bindings)
# Select and connect KVStore
kvstore = m42pl.kvstore(args.kvstore)(**args.kvstore_kwargs)
# REPL loop
while True:
try:
# Register SINGINT (note the underlying pipelines will also
# register it; thats why we need regsiter it after each loop)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Read and cleanup script
source = self.prompt.prompt().strip()
# Process
if len(source) > 0:
# Run builtins
source = self.builtins(source)
if source and len(source) > 0:
# Otherwise, interpret source as a M42PL pipeline
# else:
if not self.dispatcher:
self.dispatcher = m42pl.dispatcher(
args.dispatcher)(**args.dispatcher_kwargs)
self.dispatcher(source=source,
kvstore=kvstore,
event=Event(args.event))
except EOFError:
self.stop()
except Exception as error:
print(CLIErrorRender(error, source).render())
if args.raise_errors:
raise
import m42pl
from m42pl.event import Event
class PipelineRequest(BaseModel):
script: str
event: dict = {}
app = FastAPI()
m42pl.load_modules()
dispatcher = m42pl.dispatcher('local_detached')()
kvstore = m42pl.kvstore('redis')()
@app.get("/ping")
def ping():
return {"ping": "pong"}
@app.post('/run')
def run(pipeline: PipelineRequest):
"""Starts as new pipeline.
"""
try:
# ---
pid = dispatcher(source=pipeline.script,
kvstore=kvstore,
def __call__(self, args):
super().__call__(args)
# Init dispatcher
dispatcher = m42pl.dispatcher(args.dispatcher)(**args.dispatcher_kwargs)
# Init KVStore
kvstore = m42pl.kvstore(args.kvstore)(**args.kvstore_kwargs)
# Init Flask app
app = Flask('m42pl')
@app.route('/')
def index():
"""Returns the list of available endpoints.
"""
return {
'endpoints': {
'GET /': {
'description': 'Returns the list of API calls'
},
'POST /run': {
'description': 'Runs a pipeline',
'properties': {
'script': {
'description': 'M42PL script code',
'type': 'string'
},
'event': {
'description': 'Initial (source) event',
'type': 'object'
}
}
},
'GET /status/<pid>': {
'description': 'Returns the given pipeline status'
}
}
}
@app.route('/run', methods=['POST',])
def run():
"""Starts a new pipeline.
"""
try:
jsdata = request.get_json()
script = jsdata['script']
event = jsdata.get('event', {})
# ---
pid = dispatcher(
source=script,
kvstore=kvstore,
event=Event(event)
)
# ---
return {'pid': pid}
except Exception as error:
return {'error': str(error)}
raise error
@app.route('/status/<pid>')
def status(pid: int):
"""Returns a pipeline status.
"""
return {'status': asyncio.run(dispatcher.status_str(pid))}
# Start Flask app
app.run()