def __init__(self, fields: list):
"""
:param fields: Fields renaming tuple
Format is `[('name', 'new_name'), ...]`
"""
super().__init__(fields)
self.fields = [(Field(old), Field(new)) for old, new in fields]
def __init__(self, functions: list, fields: list, **kwargs):
"""
:param functions: Aggregation functions list
e.g. `[('function_name', 'source_field', 'dest_field')]`
:param fields: Aggregation fields list
"""
super().__init__(functions, fields)
# ---
# Aggregation fields
self.aggr_fields = [Field(f, default=None, seqn=False) for f in fields]
# ---
# Aggreation results
self.aggregates = {} # type: dict
# ---
# Stated fields map (aka. functions calling map)
self.stated_fields = {}
for function in functions:
# Extract and format function name, source field and destination field names
function_name, source_field, dest_field, *_ = chain(
function, [None] * 3)
source_field = Field(source_field)
dest_field = dest_field and Field(dest_field) or Field(
f'{function_name}({source_field.name or ""})')
# Add destination field and function call
if function_name in stats_functions.names:
self.stated_fields[dest_field] = stats_functions.names[
function_name](source_field, self.aggr_fields, dest_field,
self.aggregates)
else:
raise Exception(f'Unknown stats function: {function_name}')
def __init__(self, field: str, keys: list = []):
"""
:param field: Field to split on
:param keys: New field names
"""
super().__init__(field, keys)
self.field = Field(field, default=[], seqn=True)
self.keys = Field(keys, default=[], seqn=True)
def __init__(self, command: str, params: list = []):
"""
:param command: Command name (e.g. 'ls')
:param args: Command arguments
"""
super().__init__(command, params)
self.command = Field(command, default=command)
self.args = [Field(param) for param in params]
def __init__(self, path: str, field: str = 'line'):
"""
:param path: Source file path
:param dest: Destination field
"""
super().__init__(path, field)
self.path = Field(path)
self.field = Field(field, default='line')
def __init__(self, count, pipeline):
"""
:param count: Event count limit
:param pipeline: Pipeline reference
"""
super().__init__(count, pipeline)
self.count = Field(count)
self.pipeline = Field(pipeline)
def __init__(self, command: str = None, ebnf: bool = False):
"""
:param command: Returns only the given command information
:param ebnbf: Includes the command ENBF
"""
super().__init__(command)
self.command_name = Field(command, default=command)
self.ebnf = Field(ebnf, default=ebnf)
def __init__(self, key: str, value: Any):
"""
:param key: Key name
:param value: Key value
"""
super().__init__(key, value)
self.key = Field(key, default=key)
self.value = Field(value, default=value)
def __init__(self, key: str, dest):
"""
:param key: Key name
:param dest: Destination field
"""
super().__init__(key, dest)
self.key = Field(key, default=key)
self.dest = Field(dest, default=dest)
def __init__(self, field: str, pipeline: str):
"""
:param field: Conditional expression
:param pipeline: Pipeline ID
"""
super().__init__(field, pipeline)
self.field = Field(field)
self.pipeline = Field(pipeline)
def __init__(self,
size: int = 128,
showchunk: bool = False,
chunkfield: str = 'chunk'):
super().__init__(size)
self.size = Field(size, default=128)
self.showchunk = Field(showchunk, default=False)
self.chunkfield = Field(chunkfield, default='chunk')
self.chunks = 1
def __init__(self, name: str, macro_kwargs: dict = {}):
"""
:param name: Macro name.
"""
super().__init__(name)
self.name = Field(name, default=name)
self.params = FieldsMap(
**dict([(name, Field(field))
for name, field in macro_kwargs.items()]))
def __init__(self, mode: str = '+', fields: list = []):
"""
:param mode: If set to '+', keep only the specified fields
If set of '-', remove the specified fields
Defaults to '+'
:param fields: List of fields to keep or remove.
"""
super().__init__(mode, fields)
self.mode = Field(mode, default=mode)
self.fields = [Field(f) for f in fields]
def __init__(self, url: str = 'tcp://127.0.0.1:5555',
codec: str = 'msgpack', **kwargs):
"""
:param url: ZMQ url (including protocol, address and port)
:param codec: Encoder name (defaults to 'msgpack')
"""
self.args = FieldsMap(**{
'url': Field(url, default='tcp://127.0.0.1:5555'),
'codec': Field(codec, default='msgpack'),
})
def __init__(self, path: str, field: str = None):
"""
:param path: File path
:param field: Field to write; If None, write complete event
"""
super().__init__(path, field)
self.path = Field(path)
self.field = field and Field(field, default='') or None
self.formatter = formatters.Json()
self.cache = {} # type: dict[str, Any]
def __init__(self, name: str, pipeline: str, notes: str = None):
"""
:param name: Macro name
:param pipeline: Macro code
:param notes: Macro notes
"""
super().__init__(name, pipeline)
self.name = Field(name, default=name)
self.pipeline = Field(pipeline)
self.notes = Field(notes, default='')
def __init__(self, src: str, dest: str, searchpath: str = ''):
"""
:param src_field: Source field
:param dest_field: Destination field
:param searchpath: Templates default search path
"""
super().__init__(src, dest, searchpath)
self.src_field = Field(src)
self.dest_field = Field(dest)
self.searchpath = Field(searchpath, default=os.path.abspath('.'))
def __init__(self, field, expr: str, clean: bool = True):
"""
:param field: Source field to cut
:param expr: Regular expression
"""
super().__init__(field, expr, clean)
self.field = Field(field)
self.fields = FieldsMap(**{
'expr': Field(expr),
'clean': Field(clean, default=True)
})
def __init__(self, expression: str, src: str, dest: str):
"""
:param expression: XPath epxression
:param src: Source field
:param dest: Destination field
"""
super().__init__(expression, src, dest)
# Initialize fields
self.xpath = Field(expression)
self.src = Field(src)
self.dest = Field(dest)
# Initialize parser
self.parser = etree.HTMLParser()
def __init__(self,
expression: str,
src: str = None,
dest: str = 'jsonpath'):
"""
:param expression: JSONPath expression
:param src: Source field
:param dest: Destination field
"""
super().__init__(expression, src, dest)
# Initialize fields
self.jspath = Field(expression, default=expression)
self.src = Field(src, default=src)
self.dest = Field(dest, default=dest)
def __init__(self,
protocol: str = 'tcp',
host: str = 'host',
port: str = 'port'):
"""
:param protocol: Protocol to use, defaults to TCP
:param host: Host IP to bind, default to localhost / 127.0.0.1
:param port: Host port to bind, default to 9999
"""
self.fields = FieldsMap(
**{
'protocol': Field(protocol, default=protocol),
'host': Field(host, default='127.0.0.1'),
'port': Field(port, default=9999)
})
def __init__(self, buffer: int = 126):
"""
:param buffer: Internal buffer size
"""
super().__init__(buffer)
self.buffer = Field(buffer, default=126)
self.events: dict[str, Any] = {}
def __init__(self, path: str = 'path', encoder: str = 'encoder',
host: str = 'host', port: str|int = 'port'):
"""
:param path: Event's field name indicating on which path the
event should be broadcasted
:param encoder: Encoder name
:param host: Websocket server host
:param port: Websocket server port
"""
super().__init__(path, encoder, host, port)
self.path = Field(path, default='/')
self.fields = FieldsMap(**{
'encoder': Field(encoder, default='json'),
'host': Field(host, default='127.0.0.1'),
'port': Field(port, default=8888)
})
self.paths: dict[str, Any] = {}
def __init__(self,
expression: str,
src: str,
dest: str = None,
update: bool = False):
"""
:param expression: Regular expression with named groups
:param src: Source field name
:param dest: Destination field name
:param update: Update the source field instead
"""
super().__init__(expression, src, dest, update)
self.expression = Field(expression)
self.source_field = Field(src)
self.dest_field = dest and Field(dest).name or ''
self.update = Field(update, default=update)
self.compiled = None
class ExtractKV(StreamingCommand):
_about_ = 'Extract keys/values pairs from a given field'
_syntax_ = ('[field=]<source field> '
'[[kvdelim=]<key/value delimiter>] '
'[[pairdelim=]<key/value pairs delimiter>] '
'[[dest=]<dest field>]')
_aliases_ = ['extract_kv', 'extract_kvs']
_schema_ = {'properties': {}} # type: ignore
def __init__(self,
field,
kvdelim: str = '=',
pairdelim: str = ',',
dest: str = None):
"""
:param field: Source field
:param kvdelim: Key and value delimiter regex;
Defaults to an equal sign ``=``
:param pairdelim: Key/value pairs delimiter regex;
Defaults to a comma ``,``
:param dest: Destination field;
Defaults to the source field
"""
super().__init__(field, kvdelim, pairdelim, dest)
self.field = Field(field)
self.kvdelim = Field(kvdelim, type=str, default='=')
self.pairdelim = Field(pairdelim, type=str, default=',')
self.dest = dest and Field(dest) or self.field
async def setup(self, event, pipeline, context):
self.kvdelim = regex.compile(await
self.kvdelim.read(event, pipeline,
context))
self.pairdelim = regex.compile(await self.pairdelim.read(
event, pipeline, context))
async def target(self, event, pipeline, context):
line = await self.field.read(event, pipeline, context)
pairs = [
kv for kv in [
self.kvdelim.split(pair)
for pair in filter(None, self.pairdelim.split(line))
] if len(kv) == 2
]
yield await self.dest.write(event, dict(pairs))
class JSONPath(StreamingCommand):
_about_ = 'Evaluate a JSONPath expression'
_syntax_ = ('<[expression=]expression> <[field=]source field>'
'[[dest=]dest field]')
_aliases_ = ['jsonpath', 'jspath']
_schema_ = {'properties': {'{dest}': {'description': 'Extracted field'}}}
def __init__(self,
expression: str,
src: str = None,
dest: str = 'jsonpath'):
"""
:param expression: JSONPath expression
:param src: Source field
:param dest: Destination field
"""
super().__init__(expression, src, dest)
# Initialize fields
self.jspath = Field(expression, default=expression)
self.src = Field(src, default=src)
self.dest = Field(dest, default=dest)
async def setup(self, event, pipeline, context):
self.jspath = jsonpath_ng.parse(await self.jspath.read(
event, pipeline, context))
async def target(self, event, pipeline, context):
matched = []
# Read source field
field = await self.src.read(event, pipeline, context)
# Match JSONPath
if not field:
matched = self.jspath.find(event['data'])
elif isinstance(field, dict):
matched = self.jspath.find(field)
elif isinstance(field, str):
matched = self.jspath.find(json.load(field))
# Write matched items
if len(matched) == 1:
yield await self.dest.write(event, matched[0].value)
elif len(matched) > 1:
yield await self.dest.write(event,
[match.value for match in matched])
else:
yield event
def __init__(self,
urls: str,
method: str = 'GET',
headers: dict = {},
data: dict = None,
json: dict = None,
frequency: int = -1,
count: int = 1):
"""
:param urls: URLs to fetch
:param method: HTTP method (``GET``, ``POST``, ...)
:param headers: HTTP headers field
(defaults to an empty ``dict``)
:param data: Form field (defaults to ``None``)
Should be either ``None`` or a ``dict`` field
:param json: JSON payload (defaults to ``None``)
Should be either ``None`` or a ``dict`` field
:param frequency: Sleep time between each call
(defaults to -1, i.e. no sleep time)
:param count: Number of requests to performs
(defaults to 1, i.e. a single request)
"""
super().__init__(urls, method, headers, data, json, frequency, count)
self.fields = FieldsMap(
**{
'urls': Field(urls, seqn=True, default=[]),
'method': Field(method, default=method),
'headers': Field(headers, default=headers),
'data': Field(data, default=data),
'json': Field(json, default=json),
'frequency': Field(frequency, default=frequency),
'count': Field(count, default=count)
})
def __init__(self, field: str|list = [], *args, **kwargs):
"""
:param url: ZMQ URL as ``<protocol>://host:port``
:param codec: Encoder name; Defaults to ``msgpack``
:param field: Field(s) to send; Defaults to an empty list,
meaning the full event will be sent.
"""
for parent in [Base, StreamingCommand, MergingCommand]:
parent.__init__(self, *args, field=field, **kwargs)
self.fields = Field(field, default=[], seqn=True)
def __init__(self, topic: str | list = [], *args, **kwargs):
"""
:param topic: ZMQ topic name or list of names
"""
super().__init__(*args, topic=topic, **kwargs)
# NB: In PUB/SUB, the client socket must subscribe at least to an
# empty topic (''), meaning it will receive all messages (no filter).
self.args.update(**{'topic': Field(topic, default=[
'',
], seqn=True)})
class ExtractMap(StreamingCommand):
_about_ = 'Extract values from a given field.'
_syntax_ = ('[field=]<source field> '
'[[headers=](headers)] '
'[[prefix=]<key prefix>] '
'[[delim=]<values delimiter>] '
'[[dest=]<dest field>]')
_aliases_ = ['extract_map', 'extract_maps']
_schema_ = {'properties': {}} # type: ignore
def __init__(self,
field,
headers: list = [],
prefix: str = '_',
delim: str = ',',
dest: str = None):
"""
:param field: Source field
:param headers: Values keys (names)
:param prefix: Values keys prefix if no or not enough header
are provided; Defaults to 'the keys count
:param delim: Values delimiter regex;
Defaults to a comma ``,``
:param dest: Destination field;
Default to the source field
"""
super().__init__(field, headers, delim, dest)
self.field = Field(field)
self.headers = Field(headers, seqn=True, default=[])
self.prefix = Field(prefix, default='_')
self.delim = Field(delim, type=str, default='=')
self.dest = dest and Field(dest) or self.field
def get_header(self):
# First, yield headers
# `self.headers` have been solved in `setup` at this step
yield from self.headers
# Then, yield enumerated headers with prefix and count
# `self.prefix` have been solved in `setup` at this step
count = -1
while True:
count += 1
yield f'{self.prefix}{count}'
async def setup(self, event, pipeline, context):
self.headers = await self.headers.read(event, pipeline, context)
self.prefix = await self.prefix.read(event, pipeline, context)
self.delim = regex.compile(await
self.delim.read(event, pipeline, context))
async def target(self, event, pipeline, context):
line = await self.field.read(event, pipeline, context)
yield await self.dest.write(
event,
dict(zip(self.get_header(), filter(None, self.delim.split(line)))))
