class Cut(unittest.TestCase, StreamingCommand):
"""Test unit for the `cut` command.
"""
command_alias = 'cut'
expected_success = [
TestScript(
name='comma',
source=dedent('''\
| eval data = 'one,two,three'
| cut data ','
'''),
expected=[Event({'data': ['one', 'two', 'three']})],
),
TestScript(
name='redundant_commas',
source=dedent('''\
| eval data = ',,,,,one,two,,three,,,,'
| cut data ','
'''),
expected=[Event({'data': ['one', 'two', 'three']})],
),
TestScript(
name='no_clean',
source=dedent('''\
| eval data = 'one,two,,four'
| cut data ',' clean=no
'''),
expected=[Event({'data': ['one', 'two', '', 'four']})],
),
]
class Buffer(unittest.TestCase, BufferingCommand):
"""Test unit for the `buffer` command.
"""
command_alias = 'buffer'
script_begin = dedent('''\
| make count=11
''')
expected_success = [
TestScript(
name='simple_one',
source=dedent('''\
| buffer size=5 showchunk=yes
'''),
expected=[
*[
Event({'chunk': 1}),
] * 5,
*[
Event({'chunk': 2}),
] * 5,
*[
Event({'chunk': 3}),
] * 1,
],
),
]
class Expand(unittest.TestCase, StreamingCommand):
"""Test unit for the `expand` command.
"""
command_alias = 'expand'
expected_success = [
TestScript(name='expand_list',
source=dedent('''\
| eval data = list(1, 'two')
| expand data
'''),
expected=[Event({'data': 1}),
Event({'data': 'two'})]),
TestScript(name='expand_single',
source=dedent('''\
| eval data = 1
| expand data
'''),
expected=[Event({'data': 1})]),
TestScript(name='expand_unexistent',
source=dedent('''\
| expand data
'''),
expected=[]),
]
class Encode(unittest.TestCase, StreamingCommand):
"""Test unit for the `encoding` command.
"""
command_alias = 'encode'
script_begin = dedent('''\
| make showinfo=yes count=1
''')
expected_success = [
TestScript(
name='encode_simplesyntax_event',
source=dedent('''\
| encode with 'msgpack'
'''),
expected=[
Event({
'encoded':
b'\x84\xa2id\x00\xa5chunk\x82\xa5chunk\x00\xa6chunks\x01\xa5count\x82\xa5begin\x00\xa3end\x01\xa8pipeline\x81\xa4name\xa4main'
})
],
fields_in=['encoded']),
TestScript(name='encode_simplesyntax_field',
source=dedent('''\
| encode pipeline with 'msgpack' as pipeline_encoded
'''),
expected=[
Event({
'pipeline_encoded':
b'\x81\xa8pipeline\x81\xa4name\xa4main'
})
],
fields_in=['pipeline_encoded']),
TestScript(
name='encode_longsyntax_event',
source=dedent('''\
| encode codec='msgpack'
'''),
expected=[
Event({
'encoded':
b'\x84\xa2id\x00\xa5chunk\x82\xa5chunk\x00\xa6chunks\x01\xa5count\x82\xa5begin\x00\xa3end\x01\xa8pipeline\x81\xa4name\xa4main'
})
],
fields_in=['encoded']),
TestScript(name='encode_longsyntax_field',
source=dedent('''\
| encode src="pipeline" codec='msgpack' dest="pipeline_encoded"
'''),
expected=[
Event({
'pipeline_encoded':
b'\x81\xa8pipeline\x81\xa4name\xa4main'
})
],
fields_in=['pipeline_encoded'])
]
async def target(self, event, pipeline, context):
keys = await self.keys.read(event, pipeline, context)
for i, item in enumerate(await self.field.read(event, pipeline,
context)):
if isinstance(item, dict):
yield Event(data=item)
elif isinstance(item, (list, tuple)):
if len(keys):
if i < len(keys):
yield Event(data={keys[i]: item})
else:
yield Event(
data={
await self.key.read(event, pipeline, context): item
})
class Buffer(unittest.TestCase, GeneratingCommand):
"""Test unit for the `buffer` command.
"""
command_alias = 'echo'
expected_success = [
TestScript(
name='echo',
source=dedent('''\
| echo
'''),
first_event=Event({'hello': 'world'}),
expected=[Event({'hello': 'world'})],
),
]
async def target(self, event, pipeline, context):
# ---
# Create new event which will holds the aggregated values
stated_event = Event(meta={'stats': {}})
# ---
# Prepare event signature
signature = blake2b()
# ---
# Read aggregation fields values and update signature
for field in self.aggr_fields:
# Read aggregation field value
value = await field.read(event, pipeline, context)
# Update signature
signature.update(str(value).encode())
# Write aggregation field value to new event
await field.write(stated_event, value)
# ---
# Update source event and new event signatures
event['sign'] = signature.hexdigest()
stated_event['sign'] = signature.hexdigest()
# ---
# Compute and add the stated fields to the new event
for field, function in self.stated_fields.items():
await field.write(stated_event, await
function(event, pipeline, context))
# ---
# Done
yield stated_event
async def keep(self, event):
"""Keep only the selected fields.
"""
# WTF: Creating a new event without `data={}` uses current
# event's data.
# TODO: Debug and resolve issue
_event = Event(data={}, sign=event['sign'])
for field in self.fields:
_event = await field.write(_event, await field.read(event))
return _event
class Eval(unittest.TestCase, StreamingCommand):
"""Test unit for the `eval` command.
"""
command_alias = 'eval'
expected_success = [
TestScript(name='single_value',
source=dedent('''\
| eval simple_int = 42
'''),
expected=[Event({'simple_int': 42})]),
TestScript(name='multi_values',
source=dedent('''\
| eval mv_foo = 1, mv_bar = 2
'''),
expected=[Event({
'mv_foo': 1,
'mv_bar': 2
})]),
TestScript(name='syntax_operator',
source=dedent('''\
| eval rounded = 40 + 2.0
'''),
expected=[Event({'rounded': 42.0})]),
TestScript(name='syntax_function',
source=dedent('''\
| eval rounded = round(42.21, 1)
'''),
expected=[Event({'rounded': 42.2})]),
TestScript(name='syntax_complex_functions',
source=dedent('''\
| eval result = tostring(toint(round(40 + 1.0, 1)) + 1) + ' is the answer'
'''),
expected=[Event({'result': '42 is the answer'})]),
TestScript(name='default_value_simple',
source=dedent('''\
| eval result = field(a, 42)
'''),
expected=[Event({'result': 42})]),
TestScript(name='default_value_nested_1',
source=dedent('''\
| eval result = field(a.b, 42)
'''),
expected=[Event({'result': 42})]),
TestScript(name='default_value_nested_2',
source=dedent('''\
| eval result = field(a.b.c, 42)
'''),
expected=[Event({'result': 42})])
]
async def __call__(self, event: dict, pipeline: Pipeline, context: Context,
*args, **kwargs) -> AsyncGenerator[dict | None, None]:
"""Runs the command.
:param event: Latest generated event or `None`
:param pipeline: Current pipeline instance
"""
try:
async for _event in self.target(event or Event(), pipeline,
context):
yield _event
except Exception as error:
raise CommandError(command=self, message=str(error)) from error
def __call__(self, args):
super().__call__(args)
with open(args.source, 'r') as fd:
source = fd.read()
try:
# Select, instanciate and run dispatcher
m42pl.dispatcher(args.dispatcher)()(
source=source,
kvstore=m42pl.kvstore(args.kvstore)(),
event=Event.from_dict(json.loads(args.event)))
except Exception as error:
print(CLIErrorRender(error, source).render())
if args.raise_errors:
raise
def run(pipeline: PipelineRequest):
"""Starts as new pipeline.
"""
try:
# ---
pid = dispatcher(source=pipeline.script,
kvstore=kvstore,
event=Event(pipeline.event))
# ---
return {'pid': pid}
except Exception as error:
raise HTTPException(400, str(error))
return {'error': str(error)}
raise error
def __call__(self, args):
super().__call__(args)
# Build aliases list for completer
self.aliases = [alias for alias, _ in m42pl.commands.ALIASES.items()]
# Select and instanciate dispatcher
# dispatcher = m42pl.dispatcher(args.dispatcher)(**args.dispatcher_kwargs)
# Select and connect KVStore
kvstore = m42pl.kvstore(args.kvstore)(**args.kvstore_kwargs)
# Read history file
if args.history:
self.history_file = Path(args.history)
if self.history_file.is_file():
readline.read_history_file(self.history_file)
# Print status
print(f'{len(self.aliases)} commands loaded')
# REPL loop
while True:
try:
# Register SINGINT (note the underlying pipelines will also
# register it; thats why we need regsiter it after each loop)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Read and cleanup script
source = input(self.prompt).lstrip(' ').rstrip(' ')
if len(source):
# Try to interpret source as builtin
rx = self.regex_builtins.match(source)
if rx:
getattr(self, f'builtin_{rx.groupdict()["name"]}')()
# Otherwise, interpret source as a M42PL pipeline
else:
if not self.dispatcher:
self.dispatcher = m42pl.dispatcher(
args.dispatcher)(**args.dispatcher_kwargs)
readline.write_history_file(self.history_file)
self.dispatcher(
source=source,
kvstore=kvstore,
# event=len(args.event) > 0 and Event(data=args.event) or None
event=Event(args.event))
except EOFError:
self.stop()
except Exception as error:
print(CLIErrorRender(error, source).render())
if args.raise_errors:
raise
class Assert(unittest.TestCase, StreamingCommand):
"""Test unit for the `assertion` command.
"""
command_alias = 'rename'
script_begin = dedent('''\
| make count=1 showinfo=yes
''')
expected_success = [
TestScript(name='valid',
source=dedent('''\
| assert id == 0
'''),
expected=[Event({'id': 0})],
fields_in=[
'id',
])
]
def run():
"""Starts a new pipeline.
"""
try:
jsdata = request.get_json()
script = jsdata['script']
event = jsdata.get('event', {})
# ---
pid = dispatcher(
source=script,
kvstore=kvstore,
event=Event(event)
)
# ---
return {'pid': pid}
except Exception as error:
return {'error': str(error)}
raise error
class Rename(unittest.TestCase, StreamingCommand):
"""Test unit for the `rename` command.
"""
command_alias = 'rename'
script_begin = dedent('''\
| make showinfo=yes
''')
expected_success = [
TestScript(
name='single_existing_field',
source=dedent('''\
| rename chunk as renamed_chunk
'''),
expected=[Event({'renamed_chunk': {
'chunk': 0,
'chunks': 1
}})],
fields_in=['renamed_chunk'])
]
class EvalFunctions(unittest.TestCase, StreamingCommand):
"""Test unit for the `eval` command functions.
"""
command_alias = 'eval'
expected_success = [
TestScript(name='single_value',
source=dedent('''\
| eval
test.misc.field = field(unexistent, 42),
test.cast.tostring = tostring(42),
test.cast.toint = toint('42'),
test.cast.tofloat = tofloat('42.21'),
test.string.clean = clean(' sp lit ed ! '),
test.path.basename = basename('/one/two/three'),
test.path.dirname = dirname('/one/two/three'),
test.path.joinpath = joinpath('one', 'two', 'three'),
'''),
expected=[
Event({
'test': {
'misc': {
'field': 42
},
'cast': {
'tostring': '42',
'toint': 42,
'tofloat': 42.21
},
'string': {
'clean': 'splited!'
},
'path': {
'basename': 'three',
'dirname': '/one/two',
'joinpath': 'one/two/three'
}
}
})
]),
]
def __call__(self, args):
super().__call__(args)
# Setup PromptToolkit
self.prompt = PromptSession(
PromptPrefix(args.prefix or '<bold>m42pl@{w}</bold>'),
multiline=True,
bottom_toolbar=self.prompt_bottom_toolbar,
prompt_continuation=self.prompt_continuation,
history=FileHistory(args.history or self.history_file),
completer=REPLCompleter(builtins=self.builtins.list_builtins()),
key_bindings=self.prompt_keys_bindings)
# Select and connect KVStore
kvstore = m42pl.kvstore(args.kvstore)(**args.kvstore_kwargs)
# REPL loop
while True:
try:
# Register SINGINT (note the underlying pipelines will also
# register it; thats why we need regsiter it after each loop)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Read and cleanup script
source = self.prompt.prompt().strip()
# Process
if len(source) > 0:
# Run builtins
source = self.builtins(source)
if source and len(source) > 0:
# Otherwise, interpret source as a M42PL pipeline
# else:
if not self.dispatcher:
self.dispatcher = m42pl.dispatcher(
args.dispatcher)(**args.dispatcher_kwargs)
self.dispatcher(source=source,
kvstore=kvstore,
event=Event(args.event))
except EOFError:
self.stop()
except Exception as error:
print(CLIErrorRender(error, source).render())
if args.raise_errors:
raise
async def __call__(self, request):
"""Handles an AIOHTTP request.
:param request: AIOHTTP request
"""
try:
jsdata = await request.json()
except Exception as error:
jsdata = {}
resp = []
# ---
# Process request in sub-pipeline (== handler's pipeline)
async for next_event in self.runner(
self.context,
Event(
data={
'request': {
'url': str(request.url),
'host': request.host,
'path': request.path,
'scheme': request.scheme,
'jsdata': jsdata,
'query_string': request.query_string,
'content_type': request.content_type,
'content_length': request.content_length
}
})):
if next_event is not None:
resp.append(next_event['data'])
# ---
# Format and return response
if len(resp) == 0:
return web.Response(text='{}')
elif len(resp) == 1:
return web.Response(text=json.dumps(resp[0]))
else:
return web.Response(text=json.dumps(resp))
async def __call__(self,
context: Context | None = None,
event: dict | None = None,
infinite: bool = False,
timeout: float = 0.0):
"""Runs the pipeline.
:param context: Current context
:param event: Initial event
:param infinite: ``True`` if the pipeline should run forever,
``False`` otherwise
:param timeout: Generator timeout to force pipeline wakeup
"""
# Setup context
self.context = context
# Setup signal handler
signal.signal(signal.SIGINT, self.stop)
# ---
# Setup commands
await self.setup_commands(event or Event())
# ---
# Enter pipeline context
async with AsyncExitStack() as stack:
self.logger.debug(f'entering commands contexts')
# Enter commands context
metas = [
await stack.enter_async_context(cmd)
for cmd in self.pipeline.metas
]
generator = self.pipeline.generator and await stack.enter_async_context(
self.pipeline.generator) or None
processors = [
await stack.enter_async_context(cmd)
for cmd in self.pipeline.processors
]
# Run pipeline metas
self.logger.info(f'running pipeline metas')
async for _ in self.run_commands(commands=metas,
event=event,
ending=False,
remain=0):
pass
# ---
# Setup the events iterator, i.e. the pipeline generator
#
# If the pipeline run in infinite mode, the initial event will
# be received later, and the iterator will be set at the same time.
if infinite:
self.trace(1, 'infinite mode: set iterator to None')
iterator = None
# Otherwise, set the iterator immediately.
else:
self.trace(1, 'standard mode: set iterator from generator')
iterator = generator and generator(
event=event, pipeline=self.pipeline,
context=self.context).__aiter__() or None
# ---
# Start pipeline loop
next_event = event
# while self._ready:
while self._ready:
self.trace(2, 'looping')
try:
# ---
# If pipeline runs in infinite mode and, it receive its
# next event from the calling function. The iterator is
# (re)set right after.
if infinite and not iterator:
self.trace(
3,
f'pipeline is infinite, iterator is None: yield for initial event'
)
next_event = yield
self.trace(
3,
f'initial event: {next_event and next_event["sign"] or None}'
)
if next_event:
# self.trace(4, f'initial event is not None, reset iterator on it')
iterator = generator and generator(
event=next_event,
pipeline=self.pipeline,
context=self.context).__aiter__() or None
else:
self.trace(
4,
f'initial event is None, pipeline should raise StopAsyncIteration right after'
)
# ---
# If the pipeline has a generating command derived into an
# iterator, it retrieves its next event from this iterator.
if iterator:
self.trace(3, f'iterator is set, await next event')
if timeout > 0.0:
task = asyncio.create_task(iterator.__anext__())
self.trace(4, f'task shiedled: {task}')
next_event = await asyncio.wait_for(
asyncio.shield(task), timeout)
else:
next_event = await iterator.__anext__()
# ---
# If neither the calling function nor the generator had
# yield an event, stop the iteration.
if next_event is None:
self.logger.info(f'next_event is None, breaking')
self.trace(
3, f'next event is None, raise StopAsyncIteration')
raise StopAsyncIteration()
# ---
# Timeout occurs when the iterator took too long to yield an
# event. Send None to the processors to 'wake-up' the
# buffering commands and process the buffered events.
except asyncio.TimeoutError:
self.logger.debug(
f'generator timeout, forcing pipeline wakeup')
self.trace(4, f'shielded task {task} -> wake up !')
async for e in self.run_commands(commands=processors,
event=None,
ending=False,
remain=0):
yield e
next_event = await task # type: ignore
# ---
# StopAsyncIteration occurs when either the iterator or the
# calling function have finished to produce events.
except StopAsyncIteration:
# self.trace(3, 'catched StopAsyncIteration')
# Always empty the buffered events
if len(processors):
self.logger.info(
f'received StopAsyncIteration, running pipeline processors in end mode'
)
# self.trace(4, f'running processors in end mode')
async for _event in self.run_commands(
commands=processors,
event=None,
ending=True,
remain=0):
# self.trace(5, f'yield event from processors in end mode: {_event.signature}')
yield _event
# If the pipeline runs in infinte mode, reset its iterator
# and yield None to indicate the end of the current loop.
# The iterator will be reset in the new loop.
if infinite:
self.logger.debug(
f'received StopAsyncIteration, reset pipeline loop'
)
self.trace(
4, 'inifite mote, reset iterator and yield None')
iterator = None
next_event = None
# yield None
# Otherwise, simply break the pipeline loop.
else:
self.logger.debug(
f'received StopAsyncIteration, breaking pipeline loop'
)
self.trace(4, 'standard mode, return')
return
# ---
# Process the received event.
if len(processors):
self.trace(
3,
f'running processors on event {next_event and next_event["sign"] or None}'
)
self.trace(3, f'processors: {processors}')
async for _event in self.run_commands(commands=processors,
event=next_event,
ending=False,
remain=0):
self.trace(
4,
f'yield event from processors: {_event["sign"]}')
yield _event
# Reinitialize next event
next_event = None
elif next_event:
yield next_event
async def target(self, pipeline):
async for event in super().target(pipeline):
self.stats_key('', event['data'])
yield Event(data=self.stats)
async def target(self, event, pipeline, context):
if self.field:
yield await self.field.write(Event(), event['data'])
else:
yield event