def asyncPipeExchangerate(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously retrieves the current exchange rate
for a given currency pair. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings (base currency)
conf : {
'quote': {'value': <'USD'>},
'default': {'value': <'USD'>},
'offline': {'type': 'bool', 'value': '0'},
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
offline = conf.get('offline', {}).get('value')
# TODO add async rate data fetching
rate_data = get_offline_rate_data() if offline else get_rate_data()
rates = parse_request(rate_data)
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = starmap(partial(parse_result, rates=rates), parsed)
returnValue(iter(_OUTPUT))
def pipe_regex(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that replaces text in items using regexes. Each has the
general format: "In [field] replace [match] with [replace]". Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipe2py.modules pipe like object (iterable of items)
conf : {
'RULE': [
{
'field': {'value': <'search field'>},
'match': {'value': <'regex'>},
'replace': {'value': <'replacement'>},
'globalmatch': {'value': '1'},
'singlelinematch': {'value': '2'},
'multilinematch': {'value': '4'},
'casematch': {'value': '8'}
}
]
}
Returns
-------
_OUTPUT : generator of items
"""
splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs('pass', convert_func))
_OUTPUT = parse_results(parsed)
return _OUTPUT
def pipe_truncate(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that returns a specified number of items from the top of a
feed. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipe2py.modules pipe like object (iterable of items)
kwargs -- terminal, if the truncation value is wired in
conf : {
'start': {'type': 'number', value': <starting location>}
'count': {'type': 'number', value': <desired feed length>}
}
Returns
-------
_OUTPUT : generator of items
"""
funcs = get_splits(None, conf, **cdicts(opts, kwargs))
pieces, _pass = funcs[0](), funcs[2]()
if _pass:
_OUTPUT = _INPUT
else:
try:
start = int(pieces.start)
except AttributeError:
start = 0
stop = start + int(pieces.count)
_OUTPUT = islice(_INPUT, start, stop)
return _OUTPUT
def asyncPipeUniq(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously returns a specified number of items from
the top of a feed. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items
conf : {
'start': {'type': 'number', value': <starting location>}
'count': {'type': 'number', value': <desired feed length>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of unique items
"""
_input = yield _INPUT
asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
pieces = yield asyncFuncs[0]()
_pass = yield asyncFuncs[2]()
if _pass:
_OUTPUT = _input
else:
start = int(pieces.start)
stop = start + int(pieces.count)
_OUTPUT = islice(_input, start, stop)
returnValue(_OUTPUT)
def pipe_itembuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that builds an item. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipeforever pipe or an iterable of items
conf : {
'attrs': [
{'key': {'value': <'title'>}, 'value': {'value': <'chair'>}},
{'key': {'value': <'color'>}, 'value': {'value': <'red'>}}
]
}
Returns
------
_OUTPUT : generator of items
"""
funcs = get_splits(None, conf['attrs'], **cdicts(opts, kwargs))
finite = utils.finitize(_INPUT)
inputs = imap(DotDict, finite)
pieces = imap(funcs[0], inputs)
results = imap(utils.parse_params, pieces)
_OUTPUT = imap(DotDict, results)
return _OUTPUT
def asyncPipeItembuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that asynchronously builds an item. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'attrs': [
{'key': {'value': 'title'}, 'value': {'value': 'new title'}},
{'key': {'value': 'desc.content'}, 'value': {'value': 'new desc'}}
]
}
Returns
------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
pkwargs = cdicts(opts, kwargs)
asyncFuncs = yield asyncGetSplits(None, conf['attrs'], **pkwargs)
_input = yield _INPUT
finite = utils.finitize(_input)
inputs = imap(DotDict, finite)
pieces = yield asyncImap(asyncFuncs[0], inputs)
results = imap(utils.parse_params, pieces)
_OUTPUT = imap(DotDict, results)
returnValue(_OUTPUT)
def pipe_loop(context=None, _INPUT=None, conf=None, embed=None, **kwargs):
"""An operator that loops over the input and performs the embedded
submodule. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipe2py.modules pipe like object (iterable of items)
embed : the submodule, i.e., pipe_*(context, _INPUT, conf)
Most modules, with the exception of User inputs and Operators can be
sub-modules.
conf : {
'assign_part': {'value': <all or first>},
'assign_to': {'value': <assigned field name>},
'emit_part': {'value': <all or first>},
'mode': {'value': <assign or EMIT>},
'with': {'value': <looped field name or blank>},
'embed': {'value': {'conf': <module conf>}}
}
Returns
-------
_OUTPUT : generator of items
"""
cust_func = get_cust_func(context, conf, embed, parse_embed, **kwargs)
opts.update({'cust_func': cust_func})
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
gathered = starmap(parse_result, splits)
_OUTPUT = utils.multiplex(gathered)
return _OUTPUT
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text using regexes. Each
has the general format: "In [field] replace [regex pattern] with [text]".
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
{'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
{'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
first = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs(first=first)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def asyncPipeRename(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously renames or copies fields in the input
source. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'RULE': [
{
'op': {'value': 'rename or copy'},
'field': {'value': 'old field'},
'newval': {'value': 'new field'}
}
]
}
kwargs : other inputs, e.g., to feed terminals for rule values
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
_OUTPUT = yield maybeDeferred(parse_results, splits, **kwargs)
returnValue(_OUTPUT)
def asyncPipeRegex(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously replaces text in items using regexes.
Each has the general format: "In [field] replace [match] with [replace]".
Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'RULE': [
{
'field': {'value': <'search field'>},
'match': {'value': <'regex'>},
'replace': {'value': <'replacement'>},
'globalmatch': {'value': '1'},
'singlelinematch': {'value': '2'},
'multilinematch': {'value': '4'},
'casematch': {'value': '8'}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
asyncConvert = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs('pass', asyncConvert)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield maybeDeferred(parse_results, parsed)
returnValue(iter(_OUTPUT))
def pipe_urlbuilder(context=None, _INPUT=None, conf=None, **kwargs):
"""A url module that builds a url. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipeforever pipe or an iterable of items or fields
conf : {
'PARAM': [
{'key': {'value': <'order'>}, 'value': {'value': <'desc'>}},
{'key': {'value': <'page'>}, 'value': {'value': <'2'>}}
]
'PATH': {'type': 'text', 'value': <''>},
'BASE': {'type': 'text', 'value': <'http://site.com/feed.xml'>},
}
Yields
------
_OUTPUT : url
"""
pkwargs = cdicts(opts, kwargs)
get_params = get_funcs(conf.get('PARAM', []), **kwargs)[0]
get_paths = get_funcs(conf.get('PATH', []), **pkwargs)[0]
get_base = get_funcs(conf['BASE'], listize=False, **pkwargs)[0]
parse_params = utils.parse_params
splits = get_splits(_INPUT, funcs=[get_params, get_paths, get_base])
parsed = utils.dispatch(splits, *get_dispatch_funcs('pass', parse_params))
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously replaces text using regexes. Each
has the general format: "In [field] replace [regex pattern] with [text]".
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'RULE': [
{'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}},
{'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}},
{'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}},
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of replaced strings
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
first = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs(first=first)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield asyncStarMap(asyncParseResult, parsed)
returnValue(iter(_OUTPUT))
def pipe_rename(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that renames or copies fields in the input source.
Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipe2py.modules pipe like object (iterable of items)
conf : {
'RULE': [
{
'op': {'value': 'rename or copy'},
'field': {'value': 'old field'},
'newval': {'value': 'new field'}
}
]
}
kwargs : other inputs, e.g., to feed terminals for rule values
Returns
-------
_OUTPUT : generator of items
"""
splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
_OUTPUT = parse_results(splits, **kwargs)
return _OUTPUT
def pipe_strregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that replaces text using regexes. Each has the general
format: "In [field] replace [regex pattern] with [text]". Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
conf : {
'RULE': [
{
'match': {'value': <regex>},
'replace': {'value': <'replacement'>}
}
]
}
Returns
-------
_OUTPUT : generator of replaced strings
"""
splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs(first=convert_func))
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def pipe_exchangerate(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that retrieves the current exchange rate for a given
currency pair. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings (base currency)
conf : {
'quote': {'value': <'USD'>},
'default': {'value': <'USD'>},
'offline': {'type': 'bool', 'value': '0'},
}
Returns
-------
_OUTPUT : generator of hashed strings
"""
offline = conf.get('offline', {}).get('value')
rate_data = get_offline_rate_data(err=False) if offline else get_rate_data()
rates = parse_request(rate_data)
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs())
_OUTPUT = starmap(partial(parse_result, rates=rates), parsed)
return _OUTPUT
def pipe_regex(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that replaces text in items using regexes. Each has the
general format: "In [field] replace [match] with [replace]". Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipe2py.modules pipe like object (iterable of items)
conf : {
'RULE': [
{
'field': {'value': <'search field'>},
'match': {'value': <'regex'>},
'replace': {'value': <'replacement'>},
'globalmatch': {'value': '1'},
'singlelinematch': {'value': '2'},
'multilinematch': {'value': '4'},
'casematch': {'value': '8'}
}
]
}
Returns
-------
_OUTPUT : generator of items
"""
splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs('pass', convert_func))
_OUTPUT = parse_results(parsed)
return _OUTPUT
def asyncPipeRegex(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously replaces text in items using regexes.
Each has the general format: "In [field] replace [match] with [replace]".
Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
conf : {
'RULE': [
{
'field': {'value': <'search field'>},
'match': {'value': <'regex'>},
'replace': {'value': <'replacement'>},
'globalmatch': {'value': '1'},
'singlelinematch': {'value': '2'},
'multilinematch': {'value': '4'},
'casematch': {'value': '8'}
}
]
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
asyncConvert = partial(maybeDeferred, convert_func)
asyncFuncs = get_async_dispatch_funcs('pass', asyncConvert)
parsed = yield asyncDispatch(splits, *asyncFuncs)
_OUTPUT = yield maybeDeferred(parse_results, parsed)
returnValue(iter(_OUTPUT))
def pipe_strregex(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that replaces text using regexes. Each has the general
format: "In [field] replace [regex pattern] with [text]". Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
conf : {
'RULE': [
{
'match': {'value': <regex>},
'replace': {'value': <'replacement'>}
}
]
}
Returns
-------
_OUTPUT : generator of replaced strings
"""
splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs(first=convert_func))
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs):
asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0])
results = yield asyncStarMap(asyncParseResult, parsed)
items = imap(utils.gen_items, results)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def asyncPipeLoop(context=None, _INPUT=None, conf=None, embed=None, **kwargs):
"""An operator that asynchronously loops over the input and performs the
embedded submodule. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : asyncPipe like object (twisted Deferred iterable of items)
embed : the submodule, i.e., asyncPipe*(context, _INPUT, conf)
Most modules, with the exception of User inputs and Operators can be
sub-modules.
conf : {
'assign_part': {'value': <all or first>},
'assign_to': {'value': <assigned field name>},
'emit_part': {'value': <all or first>},
'mode': {'value': <assign or EMIT>},
'with': {'value': <looped field name or blank>},
'embed': {'value': {'conf': <module conf>}}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of items
"""
cust_func = get_cust_func(context, conf, embed, parse_embed, **kwargs)
opts.update({'cust_func': cust_func})
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
gathered = yield asyncStarMap(asyncParseResult, splits)
_OUTPUT = utils.multiplex(gathered)
returnValue(_OUTPUT)
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs):
asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0])
results = yield asyncStarMap(asyncParseResult, parsed)
items = imap(utils.gen_items, results)
_OUTPUT = utils.multiplex(items)
returnValue(_OUTPUT)
def get_cust_func(context, conf, embed, parse_func, **kwargs):
# Prepare the submodule to take parameters from the loop instead of from
# the user
embed_context = copy(context)
embed_context.submodule = True
copts = {'context': embed_context, 'conf': conf, 'embed': embed}
pkwargs = cdicts(copts, kwargs)
return partial(parse_func, **pkwargs)
def parse_embed(item, context=None, conf=None, embed=None, **kwargs):
parsed_conf = get_funcs(conf, **cdicts(opts, kwargs))[0](item)
try:
embedded = parsed_conf['embed'].get()
except TypeError:
embedded = parsed_conf['embed']
embedded_conf = embedded.get('conf', {})
pairs = [('pass_if', kwargs), ('pdictize', embedded)]
true_pairs = ((x, y[x]) for x, y in pairs if x in y)
ekwargs = dict(chain([('with', parsed_conf.get('with'))], true_pairs))
context.inputs = get_funcs(embedded_conf, **ekwargs)[0](item)
return embed(context, iter([item]), embedded_conf, **ekwargs)
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def pipe_hash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
Returns
-------
_OUTPUT : generator of hashed strings
"""
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs())
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of hashed strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def pipe_hash(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that hashes the given text. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
Returns
-------
_OUTPUT : generator of hashed strings
"""
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs())
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeCurrencyformat(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously formats a number to a given currency
string. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {'currency': {'value': <'USD'>}}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of formatted currencies
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def pipe_strtransform(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that splits a string into tokens delimited by
separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
conf : {'transformation': {value': <'swapcase'>}}
Returns
-------
_OUTPUT : generator of tokenized strings
"""
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs())
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def pipe_currencyformat(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that formats a number to a given currency string.
Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or numbers
conf : {'currency': {'value': <'USD'>}}
Returns
-------
_OUTPUT : generator of formatted currencies
"""
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs('num'))
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeStrtransform(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously splits a string into tokens
delimited by separators. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {'transformation': {value': <'swapcase'>}}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized strings
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def pipe_fetchdata(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that fetches and parses an XML or JSON file. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipeforever pipe or an iterable of items or fields
conf : {
'URL': {'value': <url>},
'path': {'value': <dot separated path to data list>}
}
Yields
------
_OUTPUT : items
Examples
--------
>>> from os import path as p
>>> from pipe2py.modules.pipeforever import pipe_forever
>>> parent = p.dirname(p.dirname(__file__))
>>> abspath = p.abspath(p.join(parent, 'data', 'gigs.json'))
>>> path = 'value.items'
>>> url = "file://%s" % abspath
>>> conf = {'URL': {'value': url}, 'path': {'value': path}}
>>> pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys()[:5]
[u'y:repeatcount', u'description', u'pubDate', u'title', u'y:published']
>>> abspath = p.abspath(p.join(parent, 'data', 'places.xml'))
>>> path = 'appointment'
>>> url = "file://%s" % abspath
>>> conf = {'URL': {'value': url}, 'path': {'value': path}}
>>> sorted(pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys())
['alarmTime', 'begin', 'duration', 'places', 'subject', 'uid']
>>> conf = {'URL': {'value': url}, 'path': {'value': ''}}
>>> sorted(pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys())
['appointment', 'reminder']
"""
# todo: iCal and KML
funcs = get_splits(None, conf, **cdicts(opts, kwargs))
parsed = get_parsed(_INPUT, funcs[0])
results = starmap(parse_result, parsed)
items = imap(utils.gen_items, results)
_OUTPUT = utils.multiplex(items)
return _OUTPUT
def pipe_fetchdata(context=None, _INPUT=None, conf=None, **kwargs):
"""A source that fetches and parses an XML or JSON file. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipeforever pipe or an iterable of items or fields
conf : {
'URL': {'value': <url>},
'path': {'value': <dot separated path to data list>}
}
Yields
------
_OUTPUT : items
Examples
--------
>>> from os import path as p
>>> from pipe2py.modules.pipeforever import pipe_forever
>>> parent = p.dirname(p.dirname(__file__))
>>> abspath = p.abspath(p.join(parent, 'data', 'gigs.json'))
>>> path = 'value.items'
>>> url = "file://%s" % abspath
>>> conf = {'URL': {'value': url}, 'path': {'value': path}}
>>> pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys()[:5]
[u'y:repeatcount', u'description', u'pubDate', u'title', u'y:published']
>>> abspath = p.abspath(p.join(parent, 'data', 'places.xml'))
>>> path = 'appointment'
>>> url = "file://%s" % abspath
>>> conf = {'URL': {'value': url}, 'path': {'value': path}}
>>> sorted(pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys())
['alarmTime', 'begin', 'duration', 'places', 'subject', 'uid']
>>> conf = {'URL': {'value': url}, 'path': {'value': ''}}
>>> sorted(pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys())
['appointment', 'reminder']
"""
# todo: iCal and KML
funcs = get_splits(None, conf, **cdicts(opts, kwargs))
parsed = get_parsed(_INPUT, funcs[0])
results = starmap(parse_result, parsed)
items = imap(utils.gen_items, results)
_OUTPUT = utils.multiplex(items)
return _OUTPUT
def pipe_uniq(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that filters out non unique items according to the specified
field. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : pipe2py.modules pipe like object (iterable of items)
kwargs -- other inputs, e.g. to feed terminals for rule values
conf : {'field': {'type': 'text', 'value': <field to be unique>}}
Returns
-------
_OUTPUT : generator of unique items
"""
funcs = get_splits(None, conf, **cdicts(opts, kwargs))
pieces, _pass = funcs[0](), funcs[2]()
_OUTPUT = _INPUT if _pass else unique_items(_INPUT, pieces.field)
return _OUTPUT
def asyncPipeUniq(context=None, _INPUT=None, conf=None, **kwargs):
"""An operator that asynchronously filters out non unique items according
to the specified field. Not loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items
conf : {'field': {'type': 'text', 'value': <field to be unique>}}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of unique items
"""
_input = yield _INPUT
asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs))
pieces = yield asyncFuncs[0]()
_pass = yield asyncFuncs[2]()
_OUTPUT = _input if _pass else unique_items(_input, pieces.field)
returnValue(_OUTPUT)
def pipe_substr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
Returns
-------
_OUTPUT : generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs())
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously performs basic arithmetic, such as
addition and subtraction. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {
'OTHER': {'type': 'number', 'value': <'5'>},
'OP': {'type': 'text', 'value': <'modulo'>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that asynchronously returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs())
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
def pipe_substr(context=None, _INPUT=None, conf=None, **kwargs):
"""A string module that returns a substring. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : iterable of items or strings
conf : {
'from': {'type': 'number', value': <starting position>},
'length': {'type': 'number', 'value': <count of characters to return>}
}
Returns
-------
_OUTPUT : generator of substrings
"""
conf['start'] = conf.pop('from', dict.get(conf, 'start'))
splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = utils.dispatch(splits, *get_dispatch_funcs())
_OUTPUT = starmap(parse_result, parsed)
return _OUTPUT
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs):
"""A number module that asynchronously performs basic arithmetic, such as
addition and subtraction. Loopable.
Parameters
----------
context : pipe2py.Context object
_INPUT : twisted Deferred iterable of items or numbers
conf : {
'OTHER': {'type': 'number', 'value': <'5'>},
'OP': {'type': 'text', 'value': <'modulo'>}
}
Returns
-------
_OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats
"""
splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs))
parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num'))
_OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed)
returnValue(iter(_OUTPUT))
