示例#1
0
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously replaces text using regexes. Each
    has the general format: "In [field] replace [regex pattern] with [text]".
    Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'RULE': [
            {'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
            {'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
            {'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
    """
    splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
    first = partial(maybeDeferred, convert_func)
    asyncFuncs = get_async_dispatch_funcs(first=first)
    parsed = yield asyncDispatch(splits, *asyncFuncs)
    _OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
    returnValue(iter(_OUTPUT))
示例#2
0
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs):
    asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
    parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0])
    results = yield asyncStarMap(asyncParseResult, parsed)
    items = imap(utils.gen_items, results)
    _OUTPUT = utils.multiplex(items)
    returnValue(_OUTPUT)
示例#3
0
def asyncPipeStrreplace(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously replaces text. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'RULE': [
            {
                'param': {'value': <match type: 1=first, 2=last, 3=every>},
                'find': {'value': <text to find>},
                'replace': {'value': <replacement>}
            }
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
    """
    splits = yield asyncGetSplits(_INPUT, conf['RULE'], **kwargs)
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
    returnValue(iter(_OUTPUT))
示例#4
0
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously replaces text using regexes. Each
    has the general format: "In [field] replace [regex pattern] with [text]".
    Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'RULE': [
            {'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
            {'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
            {'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
    """
    splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
    first = partial(maybeDeferred, convert_func)
    asyncFuncs = get_async_dispatch_funcs(first=first)
    parsed = yield asyncDispatch(splits, *asyncFuncs)
    _OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
    returnValue(iter(_OUTPUT))
示例#5
0
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs):
    asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
    parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0])
    results = yield asyncStarMap(asyncParseResult, parsed)
    items = imap(utils.gen_items, results)
    _OUTPUT = utils.multiplex(items)
    returnValue(_OUTPUT)
示例#6
0
def asyncPipeLoop(context=None, _INPUT=None, conf=None, embed=None, **kwargs):
    """An operator that asynchronously loops over the input and performs the
    embedded submodule. Not loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : asyncPipe like object (twisted Deferred iterable of items)
    embed : the submodule, i.e., asyncPipe*(context, _INPUT, conf)
        Most modules, with the exception of User inputs and Operators can be
        sub-modules.

    conf : {
        'assign_part': {'value': <all or first>},
        'assign_to': {'value': <assigned field name>},
        'emit_part': {'value': <all or first>},
        'mode': {'value': <assign or EMIT>},
        'with': {'value': <looped field name or blank>},
        'embed': {'value': {'conf': <module conf>}}
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of items
    """
    cust_func = get_cust_func(context, conf, embed, parse_embed, **kwargs)
    opts.update({'cust_func': cust_func})
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    gathered = yield asyncStarMap(asyncParseResult, splits)
    _OUTPUT = utils.multiplex(gathered)
    returnValue(_OUTPUT)
示例#7
0
def asyncPipeStrreplace(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously replaces text. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'RULE': [
            {
                'param': {'value': <match type: 1=first, 2=last, 3=every>},
                'find': {'value': <text to find>},
                'replace': {'value': <replacement>}
            }
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
    """
    splits = yield asyncGetSplits(_INPUT, conf['RULE'], **kwargs)
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
    returnValue(iter(_OUTPUT))
示例#8
0
文件: pipehash.py 项目: mewt/pipe2py
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously hashes the given text. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#9
0
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously hashes the given text. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#10
0
def asyncPipeCurrencyformat(context=None, _INPUT=None, conf=None, **kwargs):
    """A number module that asynchronously formats a number to a given currency
    string. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or numbers
    conf : {'currency': {'value': <'USD'>}}

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of formatted currencies
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#11
0
def asyncPipeStrtransform(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously splits a string into tokens
    delimited by separators. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {'transformation': {value': <'swapcase'>}}

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of tokenized strings
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#12
0
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously returns a substring. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'from': {'type': 'number', value': <starting position>},
        'length': {'type': 'number', 'value': <count of characters to return>}
    }

    returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of substrings
    """
    conf['start'] = conf.pop('from', dict.get(conf, 'start'))
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#13
0
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
    """A number module that asynchronously performs basic arithmetic, such as
    addition and subtraction. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or numbers
    conf : {
        'OTHER': {'type': 'number', 'value': <'5'>},
        'OP': {'type': 'text', 'value': <'modulo'>}
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#14
0
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
    """A number module that asynchronously performs basic arithmetic, such as
    addition and subtraction. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or numbers
    conf : {
        'OTHER': {'type': 'number', 'value': <'5'>},
        'OP': {'type': 'text', 'value': <'modulo'>}
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#15
0
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously returns a substring. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'from': {'type': 'number', value': <starting position>},
        'length': {'type': 'number', 'value': <count of characters to return>}
    }

    returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of substrings
    """
    conf['start'] = conf.pop('from', dict.get(conf, 'start'))
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    returnValue(iter(_OUTPUT))
示例#16
0
def asyncPipeExchangerate(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously retrieves the current exchange rate
    for a given currency pair. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings (base currency)
    conf : {
        'quote': {'value': <'USD'>},
        'default': {'value': <'USD'>},
        'offline': {'type': 'bool', 'value': '0'},
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
    """
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    _OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
    returnValue(iter(_OUTPUT))
示例#17
0
def asyncPipeStrconcat(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously builds a string. Loopable. No direct
    input.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : asyncPipe like object (twisted Deferred iterable of items)
    conf : {
        'part': [
            {'value': <'<img src="'>},
            {'subkey': <'img.src'>},
            {'value': <'">'>}
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of joined strings
    """
    splits = yield asyncGetSplits(_INPUT, conf['part'], **cdicts(opts, kwargs))
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), splits)
    returnValue(iter(_OUTPUT))
示例#18
0
def asyncPipeStrconcat(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously builds a string. Loopable. No direct
    input.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : asyncPipe like object (twisted Deferred iterable of items)
    conf : {
        'part': [
            {'value': <'<img src="'>},
            {'subkey': <'img.src'>},
            {'value': <'">'>}
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of joined strings
    """
    splits = yield asyncGetSplits(_INPUT, conf['part'], **cdicts(opts, kwargs))
    _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), splits)
    returnValue(iter(_OUTPUT))
示例#19
0
def asyncPipeStringtokenizer(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously splits a string into tokens
    delimited by separators. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'to-str': {'value': <delimiter>},
        'dedupe': {'type': 'bool', value': <1>},
        'sort': {'type': 'bool', value': <1>}
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of items
    """
    conf['delimiter'] = conf.pop('to-str', dict.get(conf, 'delimiter'))
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    items = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    _OUTPUT = utils.multiplex(items)
    returnValue(_OUTPUT)
示例#20
0
def asyncPipeFetch(context=None, _INPUT=None, conf=None, **kwargs):
    """A source that asynchronously fetches and parses one or more feeds to
    return the feed entries. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : asyncPipe like object (twisted Deferred iterable of items)
    conf : {
        'URL': [
            {'type': 'url', 'value': <url1>},
            {'type': 'url', 'value': <url2>},
            {'type': 'url', 'value': <url3>},
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of items
    """
    splits = yield asyncGetSplits(_INPUT, conf['URL'], **cdicts(opts, kwargs))
    items = yield asyncStarMap(asyncParseResult, splits)
    _OUTPUT = utils.multiplex(items)
    returnValue(_OUTPUT)
def asyncPipeStringtokenizer(context=None, _INPUT=None, conf=None, **kwargs):
    """A string module that asynchronously splits a string into tokens
    delimited by separators. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : twisted Deferred iterable of items or strings
    conf : {
        'to-str': {'value': <delimiter>},
        'dedupe': {'type': 'bool', value': <1>},
        'sort': {'type': 'bool', value': <1>}
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of items
    """
    conf['delimiter'] = conf.pop('to-str', dict.get(conf, 'delimiter'))
    splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
    parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
    items = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
    _OUTPUT = utils.multiplex(items)
    returnValue(_OUTPUT)
示例#22
0
def asyncPipeFetch(context=None, _INPUT=None, conf=None, **kwargs):
    """A source that asynchronously fetches and parses one or more feeds to
    return the feed entries. Loopable.

    Parameters
    ----------
    context : pipe2py.Context object
    _INPUT : asyncPipe like object (twisted Deferred iterable of items)
    conf : {
        'URL': [
            {'type': 'url', 'value': <url1>},
            {'type': 'url', 'value': <url2>},
            {'type': 'url', 'value': <url3>},
        ]
    }

    Returns
    -------
    _OUTPUT : twisted.internet.defer.Deferred generator of items
    """
    splits = yield asyncGetSplits(_INPUT, conf['URL'], **cdicts(opts, kwargs))
    items = yield asyncStarMap(asyncParseResult, splits)
    _OUTPUT = utils.multiplex(items)
    returnValue(_OUTPUT)