def asyncParseResult(urls, _, _pass):
# asyncParse = partial(deferToThread, speedparser.parse)
asyncParse = partial(maybeDeferred, speedparser.parse)
str_urls = get_urls(urls)
contents = yield asyncImap(getPage, str_urls)
parsed = yield asyncImap(asyncParse, contents)
entries = imap(utils.gen_entries, parsed)
items = utils.multiplex(entries)
returnValue(items)
def asyncParseResult(urls, _, _pass):
# asyncParse = partial(deferToThread, speedparser.parse)
asyncParse = partial(maybeDeferred, speedparser.parse)
str_urls = get_urls(urls)
contents = yield asyncImap(getPage, str_urls)
parsed = yield asyncImap(asyncParse, contents)
entries = imap(utils.gen_entries, parsed)
items = utils.multiplex(entries)
returnValue(items)
def asyncPipeItembuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that asynchronously builds an item. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'attrs': [
{'key': {'value': 'title'}, 'value': {'value': 'new title'}},
{'key': {'value': 'desc.content'}, 'value': {'value': 'new desc'}}
]
}
Returns
------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
pkwargs = cdicts(opts, kwargs)
asyncFuncs = yield asyncGetSplits(None, conf['attrs'], **pkwargs)
_input = yield _INPUT
finite = utils.finitize(_input)
inputs = imap(DotDict, finite)
pieces = yield asyncImap(asyncFuncs[0], inputs)
results = imap(utils.parse_params, pieces)
_OUTPUT = imap(DotDict, results)
returnValue(_OUTPUT)
def asyncGetParsed(_INPUT, asyncFunc):
_input = yield _INPUT
finite = utils.finitize(_input)
confs = yield asyncImap(asyncFunc, finite)
splits = imap(parse_conf, confs)
asyncGetElement = partial(maybeDeferred, get_element)
asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
results = yield asyncDispatch(splits, *asyncFuncs)
returnValue(results)
def asyncGetParsed(_INPUT, asyncFunc):
_input = yield _INPUT
finite = utils.finitize(_input)
confs = yield asyncImap(asyncFunc, finite)
splits = imap(parse_conf, confs)
asyncGetElement = partial(maybeDeferred, get_element)
asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
results = yield asyncDispatch(splits, *asyncFuncs)
returnValue(results)