def __init__(self, fields: list): """ :param fields: Fields renaming tuple Format is `[('name', 'new_name'), ...]` """ super().__init__(fields) self.fields = [(Field(old), Field(new)) for old, new in fields]
def __init__(self, functions: list, fields: list, **kwargs): """ :param functions: Aggregation functions list e.g. `[('function_name', 'source_field', 'dest_field')]` :param fields: Aggregation fields list """ super().__init__(functions, fields) # --- # Aggregation fields self.aggr_fields = [Field(f, default=None, seqn=False) for f in fields] # --- # Aggreation results self.aggregates = {} # type: dict # --- # Stated fields map (aka. functions calling map) self.stated_fields = {} for function in functions: # Extract and format function name, source field and destination field names function_name, source_field, dest_field, *_ = chain( function, [None] * 3) source_field = Field(source_field) dest_field = dest_field and Field(dest_field) or Field( f'{function_name}({source_field.name or ""})') # Add destination field and function call if function_name in stats_functions.names: self.stated_fields[dest_field] = stats_functions.names[ function_name](source_field, self.aggr_fields, dest_field, self.aggregates) else: raise Exception(f'Unknown stats function: {function_name}')
def __init__(self, field: str, keys: list = []): """ :param field: Field to split on :param keys: New field names """ super().__init__(field, keys) self.field = Field(field, default=[], seqn=True) self.keys = Field(keys, default=[], seqn=True)
def __init__(self, command: str, params: list = []): """ :param command: Command name (e.g. 'ls') :param args: Command arguments """ super().__init__(command, params) self.command = Field(command, default=command) self.args = [Field(param) for param in params]
def __init__(self, path: str, field: str = 'line'): """ :param path: Source file path :param dest: Destination field """ super().__init__(path, field) self.path = Field(path) self.field = Field(field, default='line')
def __init__(self, count, pipeline): """ :param count: Event count limit :param pipeline: Pipeline reference """ super().__init__(count, pipeline) self.count = Field(count) self.pipeline = Field(pipeline)
def __init__(self, command: str = None, ebnf: bool = False): """ :param command: Returns only the given command information :param ebnbf: Includes the command ENBF """ super().__init__(command) self.command_name = Field(command, default=command) self.ebnf = Field(ebnf, default=ebnf)
def __init__(self, key: str, value: Any): """ :param key: Key name :param value: Key value """ super().__init__(key, value) self.key = Field(key, default=key) self.value = Field(value, default=value)
def __init__(self, key: str, dest): """ :param key: Key name :param dest: Destination field """ super().__init__(key, dest) self.key = Field(key, default=key) self.dest = Field(dest, default=dest)
def __init__(self, field: str, pipeline: str): """ :param field: Conditional expression :param pipeline: Pipeline ID """ super().__init__(field, pipeline) self.field = Field(field) self.pipeline = Field(pipeline)
def __init__(self, size: int = 128, showchunk: bool = False, chunkfield: str = 'chunk'): super().__init__(size) self.size = Field(size, default=128) self.showchunk = Field(showchunk, default=False) self.chunkfield = Field(chunkfield, default='chunk') self.chunks = 1
def __init__(self, name: str, macro_kwargs: dict = {}): """ :param name: Macro name. """ super().__init__(name) self.name = Field(name, default=name) self.params = FieldsMap( **dict([(name, Field(field)) for name, field in macro_kwargs.items()]))
def __init__(self, mode: str = '+', fields: list = []): """ :param mode: If set to '+', keep only the specified fields If set of '-', remove the specified fields Defaults to '+' :param fields: List of fields to keep or remove. """ super().__init__(mode, fields) self.mode = Field(mode, default=mode) self.fields = [Field(f) for f in fields]
def __init__(self, url: str = 'tcp://127.0.0.1:5555', codec: str = 'msgpack', **kwargs): """ :param url: ZMQ url (including protocol, address and port) :param codec: Encoder name (defaults to 'msgpack') """ self.args = FieldsMap(**{ 'url': Field(url, default='tcp://127.0.0.1:5555'), 'codec': Field(codec, default='msgpack'), })
def __init__(self, path: str, field: str = None): """ :param path: File path :param field: Field to write; If None, write complete event """ super().__init__(path, field) self.path = Field(path) self.field = field and Field(field, default='') or None self.formatter = formatters.Json() self.cache = {} # type: dict[str, Any]
def __init__(self, name: str, pipeline: str, notes: str = None): """ :param name: Macro name :param pipeline: Macro code :param notes: Macro notes """ super().__init__(name, pipeline) self.name = Field(name, default=name) self.pipeline = Field(pipeline) self.notes = Field(notes, default='')
def __init__(self, src: str, dest: str, searchpath: str = ''): """ :param src_field: Source field :param dest_field: Destination field :param searchpath: Templates default search path """ super().__init__(src, dest, searchpath) self.src_field = Field(src) self.dest_field = Field(dest) self.searchpath = Field(searchpath, default=os.path.abspath('.'))
def __init__(self, field, expr: str, clean: bool = True): """ :param field: Source field to cut :param expr: Regular expression """ super().__init__(field, expr, clean) self.field = Field(field) self.fields = FieldsMap(**{ 'expr': Field(expr), 'clean': Field(clean, default=True) })
def __init__(self, expression: str, src: str, dest: str): """ :param expression: XPath epxression :param src: Source field :param dest: Destination field """ super().__init__(expression, src, dest) # Initialize fields self.xpath = Field(expression) self.src = Field(src) self.dest = Field(dest) # Initialize parser self.parser = etree.HTMLParser()
def __init__(self, expression: str, src: str = None, dest: str = 'jsonpath'): """ :param expression: JSONPath expression :param src: Source field :param dest: Destination field """ super().__init__(expression, src, dest) # Initialize fields self.jspath = Field(expression, default=expression) self.src = Field(src, default=src) self.dest = Field(dest, default=dest)
def __init__(self, protocol: str = 'tcp', host: str = 'host', port: str = 'port'): """ :param protocol: Protocol to use, defaults to TCP :param host: Host IP to bind, default to localhost / 127.0.0.1 :param port: Host port to bind, default to 9999 """ self.fields = FieldsMap( **{ 'protocol': Field(protocol, default=protocol), 'host': Field(host, default='127.0.0.1'), 'port': Field(port, default=9999) })
def __init__(self, buffer: int = 126): """ :param buffer: Internal buffer size """ super().__init__(buffer) self.buffer = Field(buffer, default=126) self.events: dict[str, Any] = {}
def __init__(self, path: str = 'path', encoder: str = 'encoder', host: str = 'host', port: str|int = 'port'): """ :param path: Event's field name indicating on which path the event should be broadcasted :param encoder: Encoder name :param host: Websocket server host :param port: Websocket server port """ super().__init__(path, encoder, host, port) self.path = Field(path, default='/') self.fields = FieldsMap(**{ 'encoder': Field(encoder, default='json'), 'host': Field(host, default='127.0.0.1'), 'port': Field(port, default=8888) }) self.paths: dict[str, Any] = {}
def __init__(self, expression: str, src: str, dest: str = None, update: bool = False): """ :param expression: Regular expression with named groups :param src: Source field name :param dest: Destination field name :param update: Update the source field instead """ super().__init__(expression, src, dest, update) self.expression = Field(expression) self.source_field = Field(src) self.dest_field = dest and Field(dest).name or '' self.update = Field(update, default=update) self.compiled = None
class ExtractKV(StreamingCommand): _about_ = 'Extract keys/values pairs from a given field' _syntax_ = ('[field=]<source field> ' '[[kvdelim=]<key/value delimiter>] ' '[[pairdelim=]<key/value pairs delimiter>] ' '[[dest=]<dest field>]') _aliases_ = ['extract_kv', 'extract_kvs'] _schema_ = {'properties': {}} # type: ignore def __init__(self, field, kvdelim: str = '=', pairdelim: str = ',', dest: str = None): """ :param field: Source field :param kvdelim: Key and value delimiter regex; Defaults to an equal sign ``=`` :param pairdelim: Key/value pairs delimiter regex; Defaults to a comma ``,`` :param dest: Destination field; Defaults to the source field """ super().__init__(field, kvdelim, pairdelim, dest) self.field = Field(field) self.kvdelim = Field(kvdelim, type=str, default='=') self.pairdelim = Field(pairdelim, type=str, default=',') self.dest = dest and Field(dest) or self.field async def setup(self, event, pipeline, context): self.kvdelim = regex.compile(await self.kvdelim.read(event, pipeline, context)) self.pairdelim = regex.compile(await self.pairdelim.read( event, pipeline, context)) async def target(self, event, pipeline, context): line = await self.field.read(event, pipeline, context) pairs = [ kv for kv in [ self.kvdelim.split(pair) for pair in filter(None, self.pairdelim.split(line)) ] if len(kv) == 2 ] yield await self.dest.write(event, dict(pairs))
class JSONPath(StreamingCommand): _about_ = 'Evaluate a JSONPath expression' _syntax_ = ('<[expression=]expression> <[field=]source field>' '[[dest=]dest field]') _aliases_ = ['jsonpath', 'jspath'] _schema_ = {'properties': {'{dest}': {'description': 'Extracted field'}}} def __init__(self, expression: str, src: str = None, dest: str = 'jsonpath'): """ :param expression: JSONPath expression :param src: Source field :param dest: Destination field """ super().__init__(expression, src, dest) # Initialize fields self.jspath = Field(expression, default=expression) self.src = Field(src, default=src) self.dest = Field(dest, default=dest) async def setup(self, event, pipeline, context): self.jspath = jsonpath_ng.parse(await self.jspath.read( event, pipeline, context)) async def target(self, event, pipeline, context): matched = [] # Read source field field = await self.src.read(event, pipeline, context) # Match JSONPath if not field: matched = self.jspath.find(event['data']) elif isinstance(field, dict): matched = self.jspath.find(field) elif isinstance(field, str): matched = self.jspath.find(json.load(field)) # Write matched items if len(matched) == 1: yield await self.dest.write(event, matched[0].value) elif len(matched) > 1: yield await self.dest.write(event, [match.value for match in matched]) else: yield event
def __init__(self, urls: str, method: str = 'GET', headers: dict = {}, data: dict = None, json: dict = None, frequency: int = -1, count: int = 1): """ :param urls: URLs to fetch :param method: HTTP method (``GET``, ``POST``, ...) :param headers: HTTP headers field (defaults to an empty ``dict``) :param data: Form field (defaults to ``None``) Should be either ``None`` or a ``dict`` field :param json: JSON payload (defaults to ``None``) Should be either ``None`` or a ``dict`` field :param frequency: Sleep time between each call (defaults to -1, i.e. no sleep time) :param count: Number of requests to performs (defaults to 1, i.e. a single request) """ super().__init__(urls, method, headers, data, json, frequency, count) self.fields = FieldsMap( **{ 'urls': Field(urls, seqn=True, default=[]), 'method': Field(method, default=method), 'headers': Field(headers, default=headers), 'data': Field(data, default=data), 'json': Field(json, default=json), 'frequency': Field(frequency, default=frequency), 'count': Field(count, default=count) })
def __init__(self, field: str|list = [], *args, **kwargs): """ :param url: ZMQ URL as ``<protocol>://host:port`` :param codec: Encoder name; Defaults to ``msgpack`` :param field: Field(s) to send; Defaults to an empty list, meaning the full event will be sent. """ for parent in [Base, StreamingCommand, MergingCommand]: parent.__init__(self, *args, field=field, **kwargs) self.fields = Field(field, default=[], seqn=True)
def __init__(self, topic: str | list = [], *args, **kwargs): """ :param topic: ZMQ topic name or list of names """ super().__init__(*args, topic=topic, **kwargs) # NB: In PUB/SUB, the client socket must subscribe at least to an # empty topic (''), meaning it will receive all messages (no filter). self.args.update(**{'topic': Field(topic, default=[ '', ], seqn=True)})
class ExtractMap(StreamingCommand): _about_ = 'Extract values from a given field.' _syntax_ = ('[field=]<source field> ' '[[headers=](headers)] ' '[[prefix=]<key prefix>] ' '[[delim=]<values delimiter>] ' '[[dest=]<dest field>]') _aliases_ = ['extract_map', 'extract_maps'] _schema_ = {'properties': {}} # type: ignore def __init__(self, field, headers: list = [], prefix: str = '_', delim: str = ',', dest: str = None): """ :param field: Source field :param headers: Values keys (names) :param prefix: Values keys prefix if no or not enough header are provided; Defaults to 'the keys count :param delim: Values delimiter regex; Defaults to a comma ``,`` :param dest: Destination field; Default to the source field """ super().__init__(field, headers, delim, dest) self.field = Field(field) self.headers = Field(headers, seqn=True, default=[]) self.prefix = Field(prefix, default='_') self.delim = Field(delim, type=str, default='=') self.dest = dest and Field(dest) or self.field def get_header(self): # First, yield headers # `self.headers` have been solved in `setup` at this step yield from self.headers # Then, yield enumerated headers with prefix and count # `self.prefix` have been solved in `setup` at this step count = -1 while True: count += 1 yield f'{self.prefix}{count}' async def setup(self, event, pipeline, context): self.headers = await self.headers.read(event, pipeline, context) self.prefix = await self.prefix.read(event, pipeline, context) self.delim = regex.compile(await self.delim.read(event, pipeline, context)) async def target(self, event, pipeline, context): line = await self.field.read(event, pipeline, context) yield await self.dest.write( event, dict(zip(self.get_header(), filter(None, self.delim.split(line)))))