def get_assignment(result, skip=False, **kwargs): # print(result) result = iter(listize(result)) if skip: return None, result try: first_result = next(result) except StopIteration: first_result = None try: second_result = next(result) except StopIteration: # pipe delivers one result, e.g., strconcat result = chain([first_result], result) multiple = False else: # pipe delivers multiple results, e.g., fetchpage/tokenizer result = chain([first_result], [second_result], result) multiple = True first = kwargs.get('count') == 'first' _all = kwargs.get('count') == 'all' one = first or not (multiple or _all) return one, iter([first_result]) if one else result
def _make_content(i, value=None, tag='content', append=True, strip=False): content = i.get(tag) try: value = value.strip() if value and strip else value except AttributeError: pass if content and value and append: content = listize(content) content.append(value) elif content and value: content = ''.join([content, value]) elif value: content = value return {tag: content} if content else {}
def _make_content(i, value=None, tag='content', append=True, strip=False): content = i.get(tag) try: value = value.strip() if value and strip else value except AttributeError: pass if content and value and append: content = listize(content) content.append(value) elif content and value: content = ''.join([content, value]) elif value: content = value return {tag: content} if content else {}
def get_broadcast_funcs(**kwargs): kw = Objectify(kwargs, conf={}) pieces = kw.conf[kw.extract] if kw.extract else kw.conf no_conf = remove_keys(kwargs, 'conf') noop = partial(cast, _type='none') if kw.listize: listed = listize(pieces) piece_defs = map(DotDict, listed) if kw.pdictize else listed parser = partial(parse_conf, **no_conf) pfuncs = [partial(parser, conf=conf) for conf in piece_defs] get_pieces = lambda item: broadcast(item, *pfuncs) elif kw.ptype != 'none': conf = DotDict(pieces) if kw.pdictize and pieces else pieces get_pieces = partial(parse_conf, conf=conf, **no_conf) else: get_pieces = noop ffunc = noop if kw.ftype == 'none' else partial(get_field, **kwargs) return (ffunc, get_pieces)
def get_broadcast_funcs(**kwargs): kw = Objectify(kwargs, conf={}) pieces = kw.conf[kw.extract] if kw.extract else kw.conf no_conf = remove_keys(kwargs, 'conf') noop = partial(cast, _type='none') if kw.listize: listed = listize(pieces) piece_defs = map(DotDict, listed) if kw.pdictize else listed parser = partial(parse_conf, **no_conf) pfuncs = [partial(parser, conf=conf) for conf in piece_defs] get_pieces = lambda item: broadcast(item, *pfuncs) elif kw.ptype != 'none': conf = DotDict(pieces) if kw.pdictize and pieces else pieces get_pieces = partial(parse_conf, conf=conf, **no_conf) else: get_pieces = noop ffunc = noop if kw.ftype == 'none' else partial(get_field, **kwargs) return (ffunc, get_pieces)
def get_skip(item, skip_if=None, **kwargs): item = item or {} if callable(skip_if): skip = skip_if(item) elif skip_if: skips = listize(skip_if) for _skip in skips: value = item.get(_skip['field'], '') text = _skip.get('text') op = _skip.get('op', 'contains') match = SKIP_SWITCH[op](text, value) if text else value skip = match if _skip.get('include') else not match if skip: break else: skip = False return skip
def get_skip(item, skip_if=None, **kwargs): item = item or {} if callable(skip_if): skip = skip_if(item) elif skip_if: skips = listize(skip_if) for _skip in skips: value = item.get(_skip['field'], '') text = _skip.get('text') op = _skip.get('op', 'contains') match = SKIP_SWITCH[op](text, value) if text else value skip = match if _skip.get('include') else not match if skip: break else: skip = False return skip
def get_assignment(result, skip=False, **kwargs): # print(result) result = iter(listize(result)) if skip: return None, result first_result = next(result) try: second_result = next(result) except StopIteration: # pipe delivers one result, e.g., strconcat result = chain([first_result], result) multiple = False else: # pipe delivers multiple results, e.g., fetchpage/tokenizer result = chain([first_result], [second_result], result) multiple = True first = kwargs.get('count') == 'first' _all = kwargs.get('count') == 'all' one = first or not (multiple or _all) return one, iter([first_result]) if one else result