def pipe_itembuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that builds an item. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipeforever pipe or an iterable of items
conf : {
'attrs': [
{'key': {'value': <'title'>}, 'value': {'value': <'chair'>}},
{'key': {'value': <'color'>}, 'value': {'value': <'red'>}}
]
}
Returns
------
_OUTPUT : generator of items
"""
funcs = get_splits(None, conf['attrs'], **cdicts(opts, kwargs))
finite = utils.finitize(_INPUT)
inputs = imap(DotDict, finite)
pieces = imap(funcs[0], inputs)
results = imap(utils.parse_params, pieces)
_OUTPUT = imap(DotDict, results)
return _OUTPUT
def get_splits(_INPUT, pieces=None, funcs=None, **kwargs):
finitize = kwargs.pop('finitize', False)
dictize = kwargs.pop('dictize', False)
finite = utils.finitize(_INPUT) if finitize and _INPUT else _INPUT
funcs = funcs or get_broadcast_funcs(pieces, **kwargs)
inputs = imap(DotDict, finite) if finite and dictize else finite
return utils.broadcast(inputs, *funcs) if inputs else funcs
def asyncPipeItembuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that asynchronously builds an item. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'attrs': [
{'key': {'value': 'title'}, 'value': {'value': 'new title'}},
{'key': {'value': 'desc.content'}, 'value': {'value': 'new desc'}}
]
}
Returns
------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
pkwargs = cdicts(opts, kwargs)
asyncFuncs = yield asyncGetSplits(None, conf['attrs'], **pkwargs)
_input = yield _INPUT
finite = utils.finitize(_input)
inputs = imap(DotDict, finite)
pieces = yield asyncImap(asyncFuncs[0], inputs)
results = imap(utils.parse_params, pieces)
_OUTPUT = imap(DotDict, results)
returnValue(_OUTPUT)
def asyncGetParsed(_INPUT, asyncFunc):
_input = yield _INPUT
finite = utils.finitize(_input)
confs = yield asyncImap(asyncFunc, finite)
splits = imap(parse_conf, confs)
asyncGetElement = partial(maybeDeferred, get_element)
asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
results = yield asyncDispatch(splits, *asyncFuncs)
returnValue(results)
def asyncGetParsed(_INPUT, asyncFunc):
_input = yield _INPUT
finite = utils.finitize(_input)
confs = yield asyncImap(asyncFunc, finite)
splits = imap(parse_conf, confs)
asyncGetElement = partial(maybeDeferred, get_element)
asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
results = yield asyncDispatch(splits, *asyncFuncs)
returnValue(results)
def asyncGetSplits(_INPUT, pieces=None, funcs=None, **kwargs):
_input = yield _INPUT
finitize = kwargs.pop('finitize', False)
dictize = kwargs.pop('dictize', False)
# asyncDict = partial(maybeDeferred, DotDict)
# inputs = yield asyncCmap(asyncDict, _input)
finite = utils.finitize(_input) if finitize and _input else _input
funcs = funcs or get_async_broadcast_funcs(pieces, **kwargs)
inputs = imap(DotDict, finite) if finite and dictize else finite
if inputs:
result = yield asyncBroadcast(inputs, *funcs)
else:
result = yield asyncReturn(funcs)
returnValue(result)
def pipe_rssitembuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that builds an rss item. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipeforever asyncPipe or an iterable of items or fields
conf : {
'mediaContentType': {'type': 'text', 'value': ''},
'mediaContentHeight': {'type': 'text', 'value': ''},
'mediaContentWidth': {'type': 'text', 'value': ''},
'mediaContentURL': {'type': 'text', 'value': 'url'},
'mediaThumbHeight': {'type': 'text', 'value': ''},
'mediaThumbWidth': {'type': 'text', 'value': ''},
'mediaThumbURL': {'type': 'text', 'value': 'url'},
'description': {'type': 'text', 'value': 'description'},
'pubdate': {'type': 'text', 'value': 'pubdate'},
'author': {'type': 'text', 'value': 'author'},
'title': {'type': 'text', 'value': 'title'},
'link': {'type': 'text', 'value': 'url'},
'guid': {'type': 'text', 'value': 'guid'},
}
Yields
------
_OUTPUT : items
"""
get_value = partial(utils.get_value, **kwargs)
pkwargs = utils.combine_dicts({'parse_func': get_value}, kwargs)
parse_conf = partial(utils.parse_conf, DotDict(conf), **pkwargs)
get_RSS = lambda key, value: (RSS.get(key, key), value)
get_YAHOO = lambda key, value: (YAHOO.get(key), value)
make_dict = lambda func, conf: dict(starmap(func, conf.iteritems()))
clean_dict = lambda d: dict(i for i in d.items() if all(i))
funcs = [partial(make_dict, get_RSS), partial(make_dict, get_YAHOO)]
finite = utils.finitize(_INPUT)
inputs = imap(DotDict, finite)
confs = imap(parse_conf, inputs)
splits = utils.broadcast(confs, *funcs)
combined = starmap(utils.combine_dicts, splits)
result = imap(clean_dict, combined)
_OUTPUT = imap(DotDict, result)
return _OUTPUT
def get_output(_INPUT, **kwargs):
others = (v for k, v in kwargs.iteritems() if k.startswith('_OTHER'))
others_items = utils.multiplex(others)
input_items = utils.finitize(_INPUT)
return chain(input_items, others_items)
def get_parsed(_INPUT, func):
finite = utils.finitize(_INPUT)
confs = imap(func, finite)
splits = imap(parse_conf, confs)
return utils.dispatch(splits, *get_dispatch_funcs('pass', get_element))
def get_parsed(_INPUT, func):
finite = utils.finitize(_INPUT)
confs = imap(func, finite)
splits = imap(parse_conf, confs)
return utils.dispatch(splits, *get_dispatch_funcs('pass', get_element))
