def parser(item, rules, skip=False, **kwargs): """ Parsers the pipe content Args: item (obj): The entry to process (a DotDict instance) rules (List[obj]): the parsed rules (Objectify instances). skip (bool): Don't parse the content kwargs (dict): Keyword arguments Kwargs: stream (dict): The original item Returns: dict: The item Examples: >>> from meza.fntools import Objectify >>> >>> item = DotDict({'content': 'hello world', 'title': 'greeting'}) >>> match = r'(\w+)\s(\w+)' >>> rule = {'field': 'content', 'match': match, 'replace': '$2wide'} >>> conf = {'rule': rule, 'multi': False, 'convert': True} >>> rules = [Objectify(rule)] >>> kwargs = {'stream': item, 'conf': conf} >>> regexed = parser(item, rules, **kwargs) >>> regexed == {'content': 'worldwide', 'title': 'greeting'} True >>> conf['multi'] = True >>> parser(item, rules, **kwargs) == regexed True """ multi = kwargs['conf']['multi'] recompile = not multi def meta_reducer(item, rules): field = rules[0]['field'] word = item.get(field, **kwargs) grouped = group_by(rules, 'flags') group_rules = [g[1] for g in grouped] if multi else rules reducer = multi_substitute if multi else substitute replacement = reduce(reducer, group_rules, word) return DotDict(merge([item, {field: replacement}])) if skip: item = kwargs['stream'] else: new_rules = [get_new_rule(r, recompile=recompile) for r in rules] grouped = group_by(new_rules, 'field') field_rules = [g[1] for g in grouped] item = reduce(meta_reducer, field_rules, item) return item
def async_parser(item, rules, skip=False, **kwargs): """ Asynchronously parsers the pipe content Args: item (obj): The entry to process (a DotDict instance) rules (List[obj]): the parsed rules (Objectify instances). skip (bool): Don't parse the content kwargs (dict): Keyword arguments Kwargs: stream (dict): The original item Returns: Deferred: twisted.internet.defer.Deferred dict Examples: >>> from riko.bado import react >>> from riko.bado.mock import FakeReactor >>> from meza.fntools import Objectify >>> >>> item = DotDict({'content': 'hello world', 'title': 'greeting'}) >>> match = r'(\w+)\s(\w+)' >>> replace = '$2wide' >>> >>> def run(reactor): ... callback = lambda x: print(x['content']) ... rule = {'field': 'content', 'match': match, 'replace': replace} ... conf = {'rule': rule, 'multi': False, 'convert': True} ... rules = [Objectify(rule)] ... kwargs = {'stream': item, 'conf': conf} ... d = async_parser(item, rules, **kwargs) ... return d.addCallbacks(callback, logger.error) >>> >>> try: ... react(run, _reactor=FakeReactor()) ... except SystemExit: ... pass ... worldwide """ multi = kwargs['conf']['multi'] recompile = not multi @coroutine def async_reducer(item, rules): field = rules[0]['field'] word = item.get(field, **kwargs) grouped = group_by(rules, 'flags') group_rules = [g[1] for g in grouped] if multi else rules reducer = multi_substitute if multi else substitute replacement = yield ait.coop_reduce(reducer, group_rules, word) combined = merge([item, {field: replacement}]) return_value(DotDict(combined)) if skip: item = kwargs['stream'] else: new_rules = [get_new_rule(r, recompile=recompile) for r in rules] grouped = group_by(new_rules, 'field') field_rules = [g[1] for g in grouped] item = yield ait.async_reduce(async_reducer, field_rules, item) return_value(item)