def asyncPipeExchangerate(context=None, _INPUT=None, conf=None, **kwargs): """A string module that asynchronously retrieves the current exchange rate for a given currency pair. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or strings (base currency) conf : { 'quote': {'value': <'USD'>}, 'default': {'value': <'USD'>}, 'offline': {'type': 'bool', 'value': '0'}, } Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of hashed strings """ offline = conf.get('offline', {}).get('value') # TODO add async rate data fetching rate_data = get_offline_rate_data() if offline else get_rate_data() rates = parse_request(rate_data) splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs()) _OUTPUT = starmap(partial(parse_result, rates=rates), parsed) returnValue(iter(_OUTPUT))
def pipe_regex(context=None, _INPUT=None, conf=None, **kwargs): """An operator that replaces text in items using regexes. Each has the general format: "In [field] replace [match] with [replace]". Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipe2py.modules pipe like object (iterable of items) conf : { 'RULE': [ { 'field': {'value': <'search field'>}, 'match': {'value': <'regex'>}, 'replace': {'value': <'replacement'>}, 'globalmatch': {'value': '1'}, 'singlelinematch': {'value': '2'}, 'multilinematch': {'value': '4'}, 'casematch': {'value': '8'} } ] } Returns ------- _OUTPUT : generator of items """ splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs('pass', convert_func)) _OUTPUT = parse_results(parsed) return _OUTPUT
def pipe_truncate(context=None, _INPUT=None, conf=None, **kwargs): """An operator that returns a specified number of items from the top of a feed. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipe2py.modules pipe like object (iterable of items) kwargs -- terminal, if the truncation value is wired in conf : { 'start': {'type': 'number', value': <starting location>} 'count': {'type': 'number', value': <desired feed length>} } Returns ------- _OUTPUT : generator of items """ funcs = get_splits(None, conf, **cdicts(opts, kwargs)) pieces, _pass = funcs[0](), funcs[2]() if _pass: _OUTPUT = _INPUT else: try: start = int(pieces.start) except AttributeError: start = 0 stop = start + int(pieces.count) _OUTPUT = islice(_INPUT, start, stop) return _OUTPUT
def asyncPipeUniq(context=None, _INPUT=None, conf=None, **kwargs): """An operator that asynchronously returns a specified number of items from the top of a feed. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items conf : { 'start': {'type': 'number', value': <starting location>} 'count': {'type': 'number', value': <desired feed length>} } returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of unique items """ _input = yield _INPUT asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs)) pieces = yield asyncFuncs[0]() _pass = yield asyncFuncs[2]() if _pass: _OUTPUT = _input else: start = int(pieces.start) stop = start + int(pieces.count) _OUTPUT = islice(_input, start, stop) returnValue(_OUTPUT)
def pipe_itembuilder(context=None, _INPUT=None, conf=None, **kwargs): """A source that builds an item. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipeforever pipe or an iterable of items conf : { 'attrs': [ {'key': {'value': <'title'>}, 'value': {'value': <'chair'>}}, {'key': {'value': <'color'>}, 'value': {'value': <'red'>}} ] } Returns ------ _OUTPUT : generator of items """ funcs = get_splits(None, conf['attrs'], **cdicts(opts, kwargs)) finite = utils.finitize(_INPUT) inputs = imap(DotDict, finite) pieces = imap(funcs[0], inputs) results = imap(utils.parse_params, pieces) _OUTPUT = imap(DotDict, results) return _OUTPUT
def asyncPipeItembuilder(context=None, _INPUT=None, conf=None, **kwargs): """A source that asynchronously builds an item. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : asyncPipe like object (twisted Deferred iterable of items) conf : { 'attrs': [ {'key': {'value': 'title'}, 'value': {'value': 'new title'}}, {'key': {'value': 'desc.content'}, 'value': {'value': 'new desc'}} ] } Returns ------ _OUTPUT : twisted.internet.defer.Deferred generator of items """ pkwargs = cdicts(opts, kwargs) asyncFuncs = yield asyncGetSplits(None, conf['attrs'], **pkwargs) _input = yield _INPUT finite = utils.finitize(_input) inputs = imap(DotDict, finite) pieces = yield asyncImap(asyncFuncs[0], inputs) results = imap(utils.parse_params, pieces) _OUTPUT = imap(DotDict, results) returnValue(_OUTPUT)
def pipe_loop(context=None, _INPUT=None, conf=None, embed=None, **kwargs): """An operator that loops over the input and performs the embedded submodule. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipe2py.modules pipe like object (iterable of items) embed : the submodule, i.e., pipe_*(context, _INPUT, conf) Most modules, with the exception of User inputs and Operators can be sub-modules. conf : { 'assign_part': {'value': <all or first>}, 'assign_to': {'value': <assigned field name>}, 'emit_part': {'value': <all or first>}, 'mode': {'value': <assign or EMIT>}, 'with': {'value': <looped field name or blank>}, 'embed': {'value': {'conf': <module conf>}} } Returns ------- _OUTPUT : generator of items """ cust_func = get_cust_func(context, conf, embed, parse_embed, **kwargs) opts.update({'cust_func': cust_func}) splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs)) gathered = starmap(parse_result, splits) _OUTPUT = utils.multiplex(gathered) return _OUTPUT
def asyncPipeStrregex(context=None, _INPUT=None, conf=None, **kwargs): """A string module that asynchronously replaces text using regexes. Each has the general format: "In [field] replace [regex pattern] with [text]". Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or strings conf : { 'RULE': [ {'match': {'value': <regex1>}, 'replace': {'value': <'text1'>}}, {'match': {'value': <regex2>}, 'replace': {'value': <'text2'>}}, {'match': {'value': <regex3>}, 'replace': {'value': <'text3'>}}, ] } Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of replaced strings """ splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs)) first = partial(maybeDeferred, convert_func) asyncFuncs = get_async_dispatch_funcs(first=first) parsed = yield asyncDispatch(splits, *asyncFuncs) _OUTPUT = yield asyncStarMap(asyncParseResult, parsed) returnValue(iter(_OUTPUT))
def asyncPipeRename(context=None, _INPUT=None, conf=None, **kwargs): """An operator that asynchronously renames or copies fields in the input source. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : asyncPipe like object (twisted Deferred iterable of items) conf : { 'RULE': [ { 'op': {'value': 'rename or copy'}, 'field': {'value': 'old field'}, 'newval': {'value': 'new field'} } ] } kwargs : other inputs, e.g., to feed terminals for rule values Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of items """ splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs)) _OUTPUT = yield maybeDeferred(parse_results, splits, **kwargs) returnValue(_OUTPUT)
def asyncPipeRegex(context=None, _INPUT=None, conf=None, **kwargs): """An operator that asynchronously replaces text in items using regexes. Each has the general format: "In [field] replace [match] with [replace]". Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : asyncPipe like object (twisted Deferred iterable of items) conf : { 'RULE': [ { 'field': {'value': <'search field'>}, 'match': {'value': <'regex'>}, 'replace': {'value': <'replacement'>}, 'globalmatch': {'value': '1'}, 'singlelinematch': {'value': '2'}, 'multilinematch': {'value': '4'}, 'casematch': {'value': '8'} } ] } Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of items """ splits = yield asyncGetSplits(_INPUT, conf['RULE'], **cdicts(opts, kwargs)) asyncConvert = partial(maybeDeferred, convert_func) asyncFuncs = get_async_dispatch_funcs('pass', asyncConvert) parsed = yield asyncDispatch(splits, *asyncFuncs) _OUTPUT = yield maybeDeferred(parse_results, parsed) returnValue(iter(_OUTPUT))
def pipe_urlbuilder(context=None, _INPUT=None, conf=None, **kwargs): """A url module that builds a url. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipeforever pipe or an iterable of items or fields conf : { 'PARAM': [ {'key': {'value': <'order'>}, 'value': {'value': <'desc'>}}, {'key': {'value': <'page'>}, 'value': {'value': <'2'>}} ] 'PATH': {'type': 'text', 'value': <''>}, 'BASE': {'type': 'text', 'value': <'http://site.com/feed.xml'>}, } Yields ------ _OUTPUT : url """ pkwargs = cdicts(opts, kwargs) get_params = get_funcs(conf.get('PARAM', []), **kwargs)[0] get_paths = get_funcs(conf.get('PATH', []), **pkwargs)[0] get_base = get_funcs(conf['BASE'], listize=False, **pkwargs)[0] parse_params = utils.parse_params splits = get_splits(_INPUT, funcs=[get_params, get_paths, get_base]) parsed = utils.dispatch(splits, *get_dispatch_funcs('pass', parse_params)) _OUTPUT = starmap(parse_result, parsed) return _OUTPUT
def pipe_rename(context=None, _INPUT=None, conf=None, **kwargs): """An operator that renames or copies fields in the input source. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipe2py.modules pipe like object (iterable of items) conf : { 'RULE': [ { 'op': {'value': 'rename or copy'}, 'field': {'value': 'old field'}, 'newval': {'value': 'new field'} } ] } kwargs : other inputs, e.g., to feed terminals for rule values Returns ------- _OUTPUT : generator of items """ splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs)) _OUTPUT = parse_results(splits, **kwargs) return _OUTPUT
def pipe_strregex(context=None, _INPUT=None, conf=None, **kwargs): """A string module that replaces text using regexes. Each has the general format: "In [field] replace [regex pattern] with [text]". Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : iterable of items or strings conf : { 'RULE': [ { 'match': {'value': <regex>}, 'replace': {'value': <'replacement'>} } ] } Returns ------- _OUTPUT : generator of replaced strings """ splits = get_splits(_INPUT, conf['RULE'], **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs(first=convert_func)) _OUTPUT = starmap(parse_result, parsed) return _OUTPUT
def pipe_exchangerate(context=None, _INPUT=None, conf=None, **kwargs): """A string module that retrieves the current exchange rate for a given currency pair. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : iterable of items or strings (base currency) conf : { 'quote': {'value': <'USD'>}, 'default': {'value': <'USD'>}, 'offline': {'type': 'bool', 'value': '0'}, } Returns ------- _OUTPUT : generator of hashed strings """ offline = conf.get('offline', {}).get('value') rate_data = get_offline_rate_data(err=False) if offline else get_rate_data() rates = parse_request(rate_data) splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs()) _OUTPUT = starmap(partial(parse_result, rates=rates), parsed) return _OUTPUT
def asyncPipeFetchdata(context=None, _INPUT=None, conf=None, **kwargs): asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs)) parsed = yield asyncGetParsed(_INPUT, asyncFuncs[0]) results = yield asyncStarMap(asyncParseResult, parsed) items = imap(utils.gen_items, results) _OUTPUT = utils.multiplex(items) returnValue(_OUTPUT)
def asyncPipeLoop(context=None, _INPUT=None, conf=None, embed=None, **kwargs): """An operator that asynchronously loops over the input and performs the embedded submodule. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : asyncPipe like object (twisted Deferred iterable of items) embed : the submodule, i.e., asyncPipe*(context, _INPUT, conf) Most modules, with the exception of User inputs and Operators can be sub-modules. conf : { 'assign_part': {'value': <all or first>}, 'assign_to': {'value': <assigned field name>}, 'emit_part': {'value': <all or first>}, 'mode': {'value': <assign or EMIT>}, 'with': {'value': <looped field name or blank>}, 'embed': {'value': {'conf': <module conf>}} } Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of items """ cust_func = get_cust_func(context, conf, embed, parse_embed, **kwargs) opts.update({'cust_func': cust_func}) splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) gathered = yield asyncStarMap(asyncParseResult, splits) _OUTPUT = utils.multiplex(gathered) returnValue(_OUTPUT)
def get_cust_func(context, conf, embed, parse_func, **kwargs): # Prepare the submodule to take parameters from the loop instead of from # the user embed_context = copy(context) embed_context.submodule = True copts = {'context': embed_context, 'conf': conf, 'embed': embed} pkwargs = cdicts(copts, kwargs) return partial(parse_func, **pkwargs)
def parse_embed(item, context=None, conf=None, embed=None, **kwargs): parsed_conf = get_funcs(conf, **cdicts(opts, kwargs))[0](item) try: embedded = parsed_conf['embed'].get() except TypeError: embedded = parsed_conf['embed'] embedded_conf = embedded.get('conf', {}) pairs = [('pass_if', kwargs), ('pdictize', embedded)] true_pairs = ((x, y[x]) for x, y in pairs if x in y) ekwargs = dict(chain([('with', parsed_conf.get('with'))], true_pairs)) context.inputs = get_funcs(embedded_conf, **ekwargs)[0](item) return embed(context, iter([item]), embedded_conf, **ekwargs)
def asyncPipeHash(context=None, _INPUT=None, conf=None, **kwargs): """A string module that asynchronously hashes the given text. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or strings Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of hashed strings """ splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs()) _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed) returnValue(iter(_OUTPUT))
def pipe_hash(context=None, _INPUT=None, conf=None, **kwargs): """A string module that hashes the given text. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : iterable of items or strings Returns ------- _OUTPUT : generator of hashed strings """ splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs()) _OUTPUT = starmap(parse_result, parsed) return _OUTPUT
def asyncPipeCurrencyformat(context=None, _INPUT=None, conf=None, **kwargs): """A number module that asynchronously formats a number to a given currency string. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or numbers conf : {'currency': {'value': <'USD'>}} Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of formatted currencies """ splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num')) _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed) returnValue(iter(_OUTPUT))
def pipe_strtransform(context=None, _INPUT=None, conf=None, **kwargs): """A string module that splits a string into tokens delimited by separators. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : iterable of items or strings conf : {'transformation': {value': <'swapcase'>}} Returns ------- _OUTPUT : generator of tokenized strings """ splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs()) _OUTPUT = starmap(parse_result, parsed) return _OUTPUT
def pipe_currencyformat(context=None, _INPUT=None, conf=None, **kwargs): """A number module that formats a number to a given currency string. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : iterable of items or numbers conf : {'currency': {'value': <'USD'>}} Returns ------- _OUTPUT : generator of formatted currencies """ splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs('num')) _OUTPUT = starmap(parse_result, parsed) return _OUTPUT
def asyncPipeStrtransform(context=None, _INPUT=None, conf=None, **kwargs): """A string module that asynchronously splits a string into tokens delimited by separators. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or strings conf : {'transformation': {value': <'swapcase'>}} Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of tokenized strings """ splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs()) _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed) returnValue(iter(_OUTPUT))
def pipe_fetchdata(context=None, _INPUT=None, conf=None, **kwargs): """A source that fetches and parses an XML or JSON file. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipeforever pipe or an iterable of items or fields conf : { 'URL': {'value': <url>}, 'path': {'value': <dot separated path to data list>} } Yields ------ _OUTPUT : items Examples -------- >>> from os import path as p >>> from pipe2py.modules.pipeforever import pipe_forever >>> parent = p.dirname(p.dirname(__file__)) >>> abspath = p.abspath(p.join(parent, 'data', 'gigs.json')) >>> path = 'value.items' >>> url = "file://%s" % abspath >>> conf = {'URL': {'value': url}, 'path': {'value': path}} >>> pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys()[:5] [u'y:repeatcount', u'description', u'pubDate', u'title', u'y:published'] >>> abspath = p.abspath(p.join(parent, 'data', 'places.xml')) >>> path = 'appointment' >>> url = "file://%s" % abspath >>> conf = {'URL': {'value': url}, 'path': {'value': path}} >>> sorted(pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys()) ['alarmTime', 'begin', 'duration', 'places', 'subject', 'uid'] >>> conf = {'URL': {'value': url}, 'path': {'value': ''}} >>> sorted(pipe_fetchdata(_INPUT=pipe_forever(), conf=conf).next().keys()) ['appointment', 'reminder'] """ # todo: iCal and KML funcs = get_splits(None, conf, **cdicts(opts, kwargs)) parsed = get_parsed(_INPUT, funcs[0]) results = starmap(parse_result, parsed) items = imap(utils.gen_items, results) _OUTPUT = utils.multiplex(items) return _OUTPUT
def pipe_uniq(context=None, _INPUT=None, conf=None, **kwargs): """An operator that filters out non unique items according to the specified field. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : pipe2py.modules pipe like object (iterable of items) kwargs -- other inputs, e.g. to feed terminals for rule values conf : {'field': {'type': 'text', 'value': <field to be unique>}} Returns ------- _OUTPUT : generator of unique items """ funcs = get_splits(None, conf, **cdicts(opts, kwargs)) pieces, _pass = funcs[0](), funcs[2]() _OUTPUT = _INPUT if _pass else unique_items(_INPUT, pieces.field) return _OUTPUT
def asyncPipeUniq(context=None, _INPUT=None, conf=None, **kwargs): """An operator that asynchronously filters out non unique items according to the specified field. Not loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items conf : {'field': {'type': 'text', 'value': <field to be unique>}} returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of unique items """ _input = yield _INPUT asyncFuncs = yield asyncGetSplits(None, conf, **cdicts(opts, kwargs)) pieces = yield asyncFuncs[0]() _pass = yield asyncFuncs[2]() _OUTPUT = _input if _pass else unique_items(_input, pieces.field) returnValue(_OUTPUT)
def pipe_substr(context=None, _INPUT=None, conf=None, **kwargs): """A string module that returns a substring. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : iterable of items or strings conf : { 'from': {'type': 'number', value': <starting position>}, 'length': {'type': 'number', 'value': <count of characters to return>} } Returns ------- _OUTPUT : generator of substrings """ conf['start'] = conf.pop('from', dict.get(conf, 'start')) splits = get_splits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = utils.dispatch(splits, *get_dispatch_funcs()) _OUTPUT = starmap(parse_result, parsed) return _OUTPUT
def asyncPipeSubstr(context=None, _INPUT=None, conf=None, **kwargs): """A string module that asynchronously returns a substring. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or strings conf : { 'from': {'type': 'number', value': <starting position>}, 'length': {'type': 'number', 'value': <count of characters to return>} } returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of substrings """ conf['start'] = conf.pop('from', dict.get(conf, 'start')) splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs()) _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed) returnValue(iter(_OUTPUT))
def asyncPipeSimplemath(context=None, _INPUT=None, conf=None, **kwargs): """A number module that asynchronously performs basic arithmetic, such as addition and subtraction. Loopable. Parameters ---------- context : pipe2py.Context object _INPUT : twisted Deferred iterable of items or numbers conf : { 'OTHER': {'type': 'number', 'value': <'5'>}, 'OP': {'type': 'text', 'value': <'modulo'>} } Returns ------- _OUTPUT : twisted.internet.defer.Deferred generator of tokenized floats """ splits = yield asyncGetSplits(_INPUT, conf, **cdicts(opts, kwargs)) parsed = yield asyncDispatch(splits, *get_async_dispatch_funcs('num')) _OUTPUT = yield asyncStarMap(partial(maybeDeferred, parse_result), parsed) returnValue(iter(_OUTPUT))