Python schemesplit Examples

Example #1

File: scheme_raw.py Project: christofd/disco

def input_stream(fd, sze, url, params):
    """Opens a StringIO whose data is everything after the url scheme.

    For example, `raw://hello_world` would return `hello_world` when read by the task.
    """
    scheme, string = schemesplit(url)
    return (StringIO(string), len(string), url)

Example #2

def input_stream(fd, sze, url, params):
    """Opens a StringIO whose data is everything after the url scheme.

    For example, `raw://hello_world` would return `hello_world` when read by the task.
    """
    from cStringIO import StringIO
    from disco.util import schemesplit
    scheme, string = schemesplit(url)
    return (StringIO(string), len(string), url)

Example #3

File: scheme_raw.py Project: AlexArgus/disco

def input_stream(fd, sze, url, params):
    """Opens a StringIO whose data is everything after the url scheme.

    For example, `raw://hello_world` would return `hello_world` when read by the task.
    """
    from disco.compat import StringIO, bytes_to_str
    from disco.util import schemesplit
    scheme, string = schemesplit(url)
    ascii = bytes_to_str(string)
    return (StringIO(ascii), len(ascii), url)

Example #4

File: scheme_redis.py Project: AlexArgus/disco

 def __init__(self, path, redis_url):
     _redis_scheme, rest = schemesplit(redis_url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.redis.flushdb()
     self.cursor = '0'
     self.url = redis_url
     from collections import defaultdict
     self.Dict = defaultdict(list)
     super(AtomicDict, self).__init__(path, 'w')
     self.write(redis_url + "\n")

Example #5

 def __init__(self, path, redis_url):
     _redis_scheme, rest = schemesplit(redis_url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.redis.flushdb()
     self.cursor = '0'
     self.url = redis_url
     from collections import defaultdict
     self.Dict = defaultdict(list)
     super(AtomicDict, self).__init__(path, 'w')
     self.write(redis_url + "\n")

Example #6

def map_input_stream(stream, size, url, params):
    """
    An :func:`input_stream` which looks at the scheme of ``url``
    and tries to import a function named ``input_stream``
    from the module ``disco.schemes.scheme_SCHEME``,
    where SCHEME is the parsed scheme.
    If no scheme is found in the url, ``file`` is used.
    The resulting input stream is then used.
    """
    from disco.util import schemesplit
    scheme, _url = schemesplit(url)
    scheme_ = 'scheme_%s' % (scheme or 'file')
    mod = __import__('disco.schemes.%s' % scheme_, fromlist=[scheme_])
    Task.insert_globals([mod.input_stream])
    return mod.input_stream(stream, size, url, params)

Example #7

File: func.py Project: JensRantil/disco

def map_input_stream(stream, size, url, params):
    """
    An :func:`input_stream` which looks at the scheme of ``url``
    and tries to import a function named ``input_stream``
    from the module ``disco.schemes.scheme_SCHEME``,
    where SCHEME is the parsed scheme.
    If no scheme is found in the url, ``file`` is used.
    The resulting input stream is then used.
    """
    from disco.util import schemesplit
    scheme, _url = schemesplit(url)
    scheme_ = 'scheme_%s' % (scheme or 'file')
    mod = __import__('disco.schemes.%s' % scheme_, fromlist=[scheme_])
    Task.insert_globals([mod.input_stream])
    return mod.input_stream(stream, size, url, params)

Example #8

File: scheme_redis.py Project: AlexArgus/disco

 def __init__(self, url):
     _redis_scheme, rest = schemesplit(url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.cursor = '0'
     self.url = url

Example #9

def open_url(url, *args, **kwargs):
    from disco.util import schemesplit
    scheme, rest = schemesplit(url)
    if not scheme or scheme == 'file':
        return open_local(rest, *args, **kwargs)
    return open_remote(url, *args, **kwargs)

Example #10

File: __init__.py Project: Dieterbe/disco

def import_scheme(url):
    scheme, _rest = util.schemesplit(url)
    scheme = 'scheme_%s' % (scheme or 'file')
    return __import__('disco.schemes.%s' % scheme, fromlist=[scheme])

Example #11

File: scheme_file.py Project: jamesaimonetti/disco

def input_stream(fd, size, url, params):
    """Opens the url locally on the node."""
    scheme, path = schemesplit(url)
    return open_local(path, url)

Example #12

File: scheme_hdfs.py Project: nagyistge/discoproject.org-disco

def open(url, task=None):
    _hdfs_scheme, address = schemesplit(url)
    namenode_port, rest = schemesplit(address)
    http_url = 'http://' + namenode_port + '/webhdfs/v1/' + rest + '?op=OPEN'
    return comm.open_url(http_url)

Example #13

File: __init__.py Project: chinnurtb/disco_playground

def import_scheme(url):
    scheme, _rest = util.schemesplit(url)
    scheme = 'scheme_%s' % (scheme or 'file')
    return __import__('disco.schemes.%s' % scheme, fromlist=[scheme])

Example #14

File: __init__.py Project: yuj/disco

def import_scheme(url):
    scheme, _rest = util.schemesplit(url)
    scheme = 'scheme_{0}'.format((scheme or 'file'))
    return __import__('disco.schemes.{0}'.format(scheme), fromlist=[scheme])

Example #15

def input_stream(fd, size, url, params):
    """Opens the url locally on the node."""
    from disco.comm import open_local
    from disco.util import schemesplit
    scheme, path = schemesplit(url)
    return open_local(path)

Example #16

File: wordcount_hdfs.py Project: AlexArgus/disco

def getHdfsMaster(discoMaster):
    from disco.util import schemesplit
    _, rest = schemesplit(discoMaster)
    return rest.split(':')[0] + ':50070'

Example #17

File: comm.py Project: tsloughter/disco

def open_url(url, *args, **kwargs):
    from disco.util import schemesplit
    scheme, rest = schemesplit(url)
    if not scheme or scheme == 'file':
        return open_local(rest, *args, **kwargs)
    return open_remote(url, *args, **kwargs)

Example #18

File: wiki_reader.py Project: nagyistge/discoproject.org-disco

def getHdfsMaster(discoMaster):
    from disco.util import schemesplit
    _, rest = schemesplit(discoMaster)
    return rest.split(':')[0] + ':50070'

Example #19

File: scheme_hdfs.py Project: AlexArgus/disco

def open(url, task=None):
    _hdfs_scheme, address = schemesplit(url)
    namenode_port, rest = schemesplit(address)
    http_url = 'http://' + namenode_port + '/webhdfs/v1/' + rest + '?op=OPEN'
    return comm.open_url(http_url)

Example #20

 def __init__(self, url):
     _redis_scheme, rest = schemesplit(url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.cursor = '0'
     self.url = url