def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text using regexes. Each
has the general format: "In [field] replace [regex pattern] with [text]".
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
{'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
{'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
first = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs(first=first)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs):
asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0])
results = yield asyncStarMap(asyncParseResult, parsed)
items = imap(utils.gen_items, results)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeStrreplace(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{
'param': {'value': <match type: 1=first, 2=last, 3=every>},
'find': {'value': <text to find>},
'replace': {'value': <replacement>}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **kwargs)
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text using regexes. Each
has the general format: "In [field] replace [regex pattern] with [text]".
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
{'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
{'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
first = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs(first=first)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs):
asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0])
results = yield asyncStarMap(asyncParseResult, parsed)
items = imap(utils.gen_items, results)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeLoop(context=None, _INPUT=None, conf=None, embed=None, **kwargs):
"""An operator that asynchronously loops over the input and performs the
embedded submodule. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
embed : the submodule, i.e., asyncPipe*(context, _INPUT, conf)
Most modules, with the exception of User inputs and Operators can be
sub-modules.
conf : {
'assign_part': {'value': <all or first>},
'assign_to': {'value': <assigned field name>},
'emit_part': {'value': <all or first>},
'mode': {'value': <assign or EMIT>},
'with': {'value': <looped field name or blank>},
'embed': {'value': {'conf': <module conf>}}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
cust_func = get_cust_func(context, conf, embed, parse_embed, **kwargs)
opts.update({'cust_func': cust_func})
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
gathered = yield asyncStarMap(asyncParseResult, splits)
_OUTPUT = utils.multiplex(gathered)
returnValue(_OUTPUT)
def asyncPipeStrreplace(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{
'param': {'value': <match type: 1=first, 2=last, 3=every>},
'find': {'value': <text to find>},
'replace': {'value': <replacement>}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **kwargs)
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeCurrencyformat(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously formats a number to a given currency
string. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {'currency': {'value': <'USD'>}}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of formatted currencies
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrtransform(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {'transformation': {value': <'swapcase'>}}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously performs basic arithmetic, such as
addition and subtraction. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {
'OTHER': {'type': 'number', 'value': <'5'>},
'OP': {'type': 'text', 'value': <'modulo'>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously performs basic arithmetic, such as
addition and subtraction. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {
'OTHER': {'type': 'number', 'value': <'5'>},
'OP': {'type': 'text', 'value': <'modulo'>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeExchangerate(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously retrieves the current exchange rate
for a given currency pair. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings (base currency)
conf : {
'quote': {'value': <'USD'>},
'default': {'value': <'USD'>},
'offline': {'type': 'bool', 'value': '0'},
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrconcat(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously builds a string. Loopable. No direct
input.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'part': [
{'value': <'<img src="'>},
{'subkey': <'img.src'>},
{'value': <'">'>}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of joined strings
"""
splits = yield asyncGetSplits(_INPUT, conf['part'], **cdicts(opts, kwargs))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), splits)
returnValue(iter(_OUTPUT))
def asyncPipeStrconcat(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously builds a string. Loopable. No direct
input.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'part': [
{'value': <'<img src="'>},
{'subkey': <'img.src'>},
{'value': <'">'>}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of joined strings
"""
splits = yield asyncGetSplits(_INPUT, conf['part'], **cdicts(opts, kwargs))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), splits)
returnValue(iter(_OUTPUT))
def asyncPipeStringtokenizer(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'to-str': {'value': <delimiter>},
'dedupe': {'type': 'bool', value': <1>},
'sort': {'type': 'bool', value': <1>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
conf['delimiter'] = conf.pop('to-str', dict.get(conf, 'delimiter'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
items = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeFetch(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that asynchronously fetches and parses one or more feeds to
return the feed entries. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'URL': [
{'type': 'url', 'value': <url1>},
{'type': 'url', 'value': <url2>},
{'type': 'url', 'value': <url3>},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['URL'], **cdicts(opts, kwargs))
items = yield asyncStarMap(asyncParseResult, splits)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeStringtokenizer(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'to-str': {'value': <delimiter>},
'dedupe': {'type': 'bool', value': <1>},
'sort': {'type': 'bool', value': <1>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
conf['delimiter'] = conf.pop('to-str', dict.get(conf, 'delimiter'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
items = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeFetch(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that asynchronously fetches and parses one or more feeds to
return the feed entries. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'URL': [
{'type': 'url', 'value': <url1>},
{'type': 'url', 'value': <url2>},
{'type': 'url', 'value': <url3>},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['URL'], **cdicts(opts, kwargs))
items = yield asyncStarMap(asyncParseResult, splits)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)