예제 #1
0
파일: pipefetch.py 프로젝트: ganugapav/pipe
def asyncParseResult(urls, _, _pass):
    # asyncParse = partial(deferToThread, speedparser.parse)
    asyncParse = partial(maybeDeferred, speedparser.parse)
    str_urls = get_urls(urls)
    contents = yield asyncImap(getPage, str_urls)
    parsed = yield asyncImap(asyncParse, contents)
    entries = imap(utils.gen_entries, parsed)
    items = utils.multiplex(entries)
    returnValue(items)
예제 #2
0
def asyncParseResult(urls, _, _pass):
    # asyncParse = partial(deferToThread, speedparser.parse)
    asyncParse = partial(maybeDeferred, speedparser.parse)
    str_urls = get_urls(urls)
    contents = yield asyncImap(getPage, str_urls)
    parsed = yield asyncImap(asyncParse, contents)
    entries = imap(utils.gen_entries, parsed)
    items = utils.multiplex(entries)
    returnValue(items)
예제 #3
0
def asyncPipeItembuilder(context=None, _INPUT=None, conf=None, **kwargs):
    """A source that asynchronously builds an item. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : asyncPipe like object (twisted Deferred iterable of items)
    conf : {
        'attrs': [
            {'key': {'value': 'title'}, 'value': {'value': 'new title'}},
            {'key': {'value': 'desc.content'}, 'value': {'value': 'new desc'}}
        ]
    }

    Returns
    ------
    _OUTPUT : twisted.internet.defer.Deferred generator of items
    """
    pkwargs = cdicts(opts, kwargs)
    asyncFuncs = yield asyncGetSplits(None, conf['attrs'], **pkwargs)
    _input = yield _INPUT
    finite = utils.finitize(_input)
    inputs = imap(DotDict, finite)
    pieces = yield asyncImap(asyncFuncs[0], inputs)
    results = imap(utils.parse_params, pieces)
    _OUTPUT = imap(DotDict, results)
    returnValue(_OUTPUT)
예제 #4
0
def asyncGetParsed(_INPUT, asyncFunc):
    _input = yield _INPUT
    finite = utils.finitize(_input)
    confs = yield asyncImap(asyncFunc, finite)
    splits = imap(parse_conf, confs)
    asyncGetElement = partial(maybeDeferred, get_element)
    asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
    results = yield asyncDispatch(splits, *asyncFuncs)
    returnValue(results)
예제 #5
0
def asyncGetParsed(_INPUT, asyncFunc):
    _input = yield _INPUT
    finite = utils.finitize(_input)
    confs = yield asyncImap(asyncFunc, finite)
    splits = imap(parse_conf, confs)
    asyncGetElement = partial(maybeDeferred, get_element)
    asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
    results = yield asyncDispatch(splits, *asyncFuncs)
    returnValue(results)