def meta_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = reduce(reducer, group_rules, word)
return DotDict(merge([item, {field: replacement}]))
def meta_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = reduce(reducer, group_rules, word)
return DotDict(merge([item, {field: replacement}]))
def async_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = yield ait.coop_reduce(reducer, group_rules, word)
combined = merge([item, {field: replacement}])
return_value(DotDict(combined))
def async_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = yield ait.coop_reduce(reducer, group_rules, word)
combined = merge([item, {field: replacement}])
return_value(DotDict(combined))
def parser(item, rules, skip=False, **kwargs):
""" Parsers the pipe content
Args:
item (obj): The entry to process (a DotDict instance)
rules (List[obj]): the parsed rules (Objectify instances).
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
dict: The item
Examples:
>>> from meza.fntools import Objectify
>>>
>>> item = DotDict({'content': 'hello world', 'title': 'greeting'})
>>> match = r'(\w+)\s(\w+)'
>>> rule = {'field': 'content', 'match': match, 'replace': '$2wide'}
>>> conf = {'rule': rule, 'multi': False, 'convert': True}
>>> rules = [Objectify(rule)]
>>> kwargs = {'stream': item, 'conf': conf}
>>> regexed = parser(item, rules, **kwargs)
>>> regexed == {'content': 'worldwide', 'title': 'greeting'}
True
>>> conf['multi'] = True
>>> parser(item, rules, **kwargs) == regexed
True
"""
multi = kwargs['conf']['multi']
recompile = not multi
def meta_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = reduce(reducer, group_rules, word)
return DotDict(merge([item, {field: replacement}]))
if skip:
item = kwargs['stream']
else:
new_rules = [get_new_rule(r, recompile=recompile) for r in rules]
grouped = group_by(new_rules, 'field')
field_rules = [g[1] for g in grouped]
item = reduce(meta_reducer, field_rules, item)
return item
def parser(item, rules, skip=False, **kwargs):
""" Parsers the pipe content
Args:
item (obj): The entry to process (a DotDict instance)
rules (List[obj]): the parsed rules (Objectify instances).
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
dict: The item
Examples:
>>> from meza.fntools import Objectify
>>>
>>> item = DotDict({'content': 'hello world', 'title': 'greeting'})
>>> match = r'(\w+)\s(\w+)'
>>> rule = {'field': 'content', 'match': match, 'replace': '$2wide'}
>>> conf = {'rule': rule, 'multi': False, 'convert': True}
>>> rules = [Objectify(rule)]
>>> kwargs = {'stream': item, 'conf': conf}
>>> regexed = parser(item, rules, **kwargs)
>>> regexed == {'content': 'worldwide', 'title': 'greeting'}
True
>>> conf['multi'] = True
>>> parser(item, rules, **kwargs) == regexed
True
"""
multi = kwargs['conf']['multi']
recompile = not multi
def meta_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = reduce(reducer, group_rules, word)
return DotDict(merge([item, {field: replacement}]))
if skip:
item = kwargs['stream']
else:
new_rules = [get_new_rule(r, recompile=recompile) for r in rules]
grouped = group_by(new_rules, 'field')
field_rules = [g[1] for g in grouped]
item = reduce(meta_reducer, field_rules, item)
return item
def async_parser(item, rules, skip=False, **kwargs):
""" Asynchronously parsers the pipe content
Args:
item (obj): The entry to process (a DotDict instance)
rules (List[obj]): the parsed rules (Objectify instances).
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
Deferred: twisted.internet.defer.Deferred dict
Examples:
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> item = DotDict({'content': 'hello world', 'title': 'greeting'})
>>> match = r'(\w+)\s(\w+)'
>>> replace = '$2wide'
>>>
>>> def run(reactor):
... callback = lambda x: print(x['content'])
... rule = {'field': 'content', 'match': match, 'replace': replace}
... conf = {'rule': rule, 'multi': False, 'convert': True}
... rules = [Objectify(rule)]
... kwargs = {'stream': item, 'conf': conf}
... d = async_parser(item, rules, **kwargs)
... return d.addCallbacks(callback, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
worldwide
"""
multi = kwargs['conf']['multi']
recompile = not multi
@coroutine
def async_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = yield ait.coop_reduce(reducer, group_rules, word)
combined = merge([item, {field: replacement}])
return_value(DotDict(combined))
if skip:
item = kwargs['stream']
else:
new_rules = [get_new_rule(r, recompile=recompile) for r in rules]
grouped = group_by(new_rules, 'field')
field_rules = [g[1] for g in grouped]
item = yield ait.async_reduce(async_reducer, field_rules, item)
return_value(item)
def async_parser(item, rules, skip=False, **kwargs):
""" Asynchronously parsers the pipe content
Args:
item (obj): The entry to process (a DotDict instance)
rules (List[obj]): the parsed rules (Objectify instances).
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
Deferred: twisted.internet.defer.Deferred dict
Examples:
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> item = DotDict({'content': 'hello world', 'title': 'greeting'})
>>> match = r'(\w+)\s(\w+)'
>>> replace = '$2wide'
>>>
>>> def run(reactor):
... callback = lambda x: print(x['content'])
... rule = {'field': 'content', 'match': match, 'replace': replace}
... conf = {'rule': rule, 'multi': False, 'convert': True}
... rules = [Objectify(rule)]
... kwargs = {'stream': item, 'conf': conf}
... d = async_parser(item, rules, **kwargs)
... return d.addCallbacks(callback, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
worldwide
"""
multi = kwargs['conf']['multi']
recompile = not multi
@coroutine
def async_reducer(item, rules):
field = rules[0]['field']
word = item.get(field, **kwargs)
grouped = group_by(rules, 'flags')
group_rules = [g[1] for g in grouped] if multi else rules
reducer = multi_substitute if multi else substitute
replacement = yield ait.coop_reduce(reducer, group_rules, word)
combined = merge([item, {field: replacement}])
return_value(DotDict(combined))
if skip:
item = kwargs['stream']
else:
new_rules = [get_new_rule(r, recompile=recompile) for r in rules]
grouped = group_by(new_rules, 'field')
field_rules = [g[1] for g in grouped]
item = yield ait.async_reduce(async_reducer, field_rules, item)
return_value(item)
