def asyncPipeRegex(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously replaces text in items using regexes.
Each has the general format: "In [field] replace [match] with [replace]".
Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'RULE': [
{
'field': {'value': <'search field'>},
'match': {'value': <'regex'>},
'replace': {'value': <'replacement'>},
'globalmatch': {'value': '1'},
'singlelinematch': {'value': '2'},
'multilinematch': {'value': '4'},
'casematch': {'value': '8'}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
asyncConvert = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs('pass', asyncConvert)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield maybeDeferred(parse_results, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text using regexes. Each
has the general format: "In [field] replace [regex pattern] with [text]".
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
{'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
{'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
first = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs(first=first)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeRegex(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously replaces text in items using regexes.
Each has the general format: "In [field] replace [match] with [replace]".
Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'RULE': [
{
'field': {'value': <'search field'>},
'match': {'value': <'regex'>},
'replace': {'value': <'replacement'>},
'globalmatch': {'value': '1'},
'singlelinematch': {'value': '2'},
'multilinematch': {'value': '4'},
'casematch': {'value': '8'}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
asyncConvert = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs('pass', asyncConvert)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield maybeDeferred(parse_results, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrreplace(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{
'param': {'value': <match type: 1=first, 2=last, 3=every>},
'find': {'value': <text to find>},
'replace': {'value': <replacement>}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **kwargs)
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrreplace(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{
'param': {'value': <match type: 1=first, 2=last, 3=every>},
'find': {'value': <text to find>},
'replace': {'value': <replacement>}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **kwargs)
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text using regexes. Each
has the general format: "In [field] replace [regex pattern] with [text]".
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
{'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
{'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
first = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs(first=first)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeExchangerate(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously retrieves the current exchange rate
for a given currency pair. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings (base currency)
conf : {
'quote': {'value': <'USD'>},
'default': {'value': <'USD'>},
'offline': {'type': 'bool', 'value': '0'},
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
offline = conf.get('offline', {}).get('value')
# TODO add async rate data fetching
rate_data = get_offline_rate_data() if offline else get_rate_data()
rates = parse_request(rate_data)
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = starmap(partial(parse_result, rates=rates), parsed)
returnValue(iter(_OUTPUT))
def asyncGetParsed(_INPUT, asyncFunc):
_input = yield _INPUT
finite = utils.finitize(_input)
confs = yield asyncImap(asyncFunc, finite)
splits = imap(parse_conf, confs)
asyncGetElement = partial(maybeDeferred, get_element)
asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
results = yield asyncDispatch(splits, *asyncFuncs)
returnValue(results)
def asyncGetParsed(_INPUT, asyncFunc):
_input = yield _INPUT
finite = utils.finitize(_input)
confs = yield asyncImap(asyncFunc, finite)
splits = imap(parse_conf, confs)
asyncGetElement = partial(maybeDeferred, get_element)
asyncFuncs = get_async_dispatch_funcs('pass', asyncGetElement)
results = yield asyncDispatch(splits, *asyncFuncs)
returnValue(results)
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeCurrencyformat(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously formats a number to a given currency
string. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {'currency': {'value': <'USD'>}}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of formatted currencies
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStrtransform(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {'transformation': {value': <'swapcase'>}}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously performs basic arithmetic, such as
addition and subtraction. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {
'OTHER': {'type': 'number', 'value': <'5'>},
'OP': {'type': 'text', 'value': <'modulo'>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously performs basic arithmetic, such as
addition and subtraction. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {
'OTHER': {'type': 'number', 'value': <'5'>},
'OP': {'type': 'text', 'value': <'modulo'>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeExchangerate(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously retrieves the current exchange rate
for a given currency pair. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings (base currency)
conf : {
'quote': {'value': <'USD'>},
'default': {'value': <'USD'>},
'offline': {'type': 'bool', 'value': '0'},
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeStringtokenizer(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'to-str': {'value': <delimiter>},
'dedupe': {'type': 'bool', value': <1>},
'sort': {'type': 'bool', value': <1>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
conf['delimiter'] = conf.pop('to-str', dict.get(conf, 'delimiter'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
items = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeStringtokenizer(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'to-str': {'value': <delimiter>},
'dedupe': {'type': 'bool', value': <1>},
'sort': {'type': 'bool', value': <1>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
conf['delimiter'] = conf.pop('to-str', dict.get(conf, 'delimiter'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
items = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
