def pipe_urlbuilder(context, _INPUT, conf, **kwargs): """This source builds a url and yields it forever. Keyword arguments: context -- pipeline context _INPUT -- not used conf: BASE -- base PATH -- path elements PARAM -- query parameters Yields (_OUTPUT): url """ for item in _INPUT: #note: we could cache get_value results if item==True url = util.get_value(conf['BASE'], item, **kwargs) if not url.endswith('/'): url += '/' if 'PATH' in conf: path = conf['PATH'] if not isinstance(path, list): path = [path] path = [util.get_value(p, item, **kwargs) for p in path if p] url += "/".join(p for p in path if p) url = url.rstrip("/") #Ensure url is valid url = util.url_quote(url) param_defs = conf['PARAM'] if not isinstance(param_defs, list): param_defs = [param_defs] params = dict([(util.get_value(p['key'], item, **kwargs), util.get_value(p['value'], item, **kwargs)) for p in param_defs if p]) if params and params.keys() != [u'']: url += "?" + urllib.urlencode(params) yield url
def pipe_urlbuilder(context, _INPUT, conf, **kwargs): """This source builds a url and yields it forever. Keyword arguments: context -- pipeline context _INPUT -- not used conf: BASE -- base PATH -- path elements PARAM -- query parameters Yields (_OUTPUT): url """ for item in _INPUT: #note: we could cache get_value results if item==True url = util.get_value(conf['BASE'], item, **kwargs) if not url.endswith('/'): url += '/' if 'PATH' in conf: path = conf['PATH'] if not isinstance(path, list): path = [path] path = [util.get_value(p, item, **kwargs) for p in path if p] url += "/".join(str(p) for p in path if p) url = url.rstrip("/") #Ensure url is valid url = util.url_quote(url) param_defs = conf['PARAM'] if not isinstance(param_defs, list): param_defs = [param_defs] params = dict([(util.get_value(p['key'], item, **kwargs), util.get_value(p['value'], item, **kwargs)) for p in param_defs if p]) if params and params.keys() != [u'']: url += "?" + urllib.urlencode(params) yield url
def pipe_urlinput(context, _INPUT, conf, **kwargs): """This source prompts the user for a url and yields it forever. Keyword arguments: context -- pipeline context _INPUT -- not used conf: name -- input parameter name default -- default prompt -- prompt Yields (_OUTPUT): url """ value = util.get_input(context, conf) #Ensure url is valid value = util.url_quote(value) while True: yield value
def pipe_urlbuilder(context=None, _INPUT=None, conf=None, **kwargs): """This source builds a url and yields it forever. Keyword arguments: context -- pipeline context _INPUT -- not used conf: BASE -- base PATH -- path elements PARAM -- query parameters Yields (_OUTPUT): url """ conf = DotDict(conf) paths = util.listize(conf.get('PATH')) # use .get() incase 'PATH' isnt set param_defs = util.listize(conf['PARAM']) url = None for item in _INPUT: # if _INPUT is pipeforever and not a loop, get values from cache if not url: item = DotDict(item) forever = item.get('forever') url = conf.get('BASE', **kwargs) url += '/' if not url.endswith('/') else url url += "/".join(str(p) for p in paths if p) url = url.rstrip("/") url = util.url_quote(url) # Ensure url is valid params = dict(_gen_params(param_defs, item, **kwargs)) if params and params.keys() != [u'']: url += "?" + urllib.urlencode(params) yield url url = url if forever else None