Example #1
0
def input_stream(fd, sze, url, params):
    """Opens a StringIO whose data is everything after the url scheme.

    For example, `raw://hello_world` would return `hello_world` when read by the task.
    """
    scheme, string = schemesplit(url)
    return (StringIO(string), len(string), url)
Example #2
0
def input_stream(fd, sze, url, params):
    """Opens a StringIO whose data is everything after the url scheme.

    For example, `raw://hello_world` would return `hello_world` when read by the task.
    """
    from cStringIO import StringIO
    from disco.util import schemesplit
    scheme, string = schemesplit(url)
    return (StringIO(string), len(string), url)
Example #3
0
def input_stream(fd, sze, url, params):
    """Opens a StringIO whose data is everything after the url scheme.

    For example, `raw://hello_world` would return `hello_world` when read by the task.
    """
    from disco.compat import StringIO, bytes_to_str
    from disco.util import schemesplit
    scheme, string = schemesplit(url)
    ascii = bytes_to_str(string)
    return (StringIO(ascii), len(ascii), url)
Example #4
0
 def __init__(self, path, redis_url):
     _redis_scheme, rest = schemesplit(redis_url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.redis.flushdb()
     self.cursor = '0'
     self.url = redis_url
     from collections import defaultdict
     self.Dict = defaultdict(list)
     super(AtomicDict, self).__init__(path, 'w')
     self.write(redis_url + "\n")
Example #5
0
 def __init__(self, path, redis_url):
     _redis_scheme, rest = schemesplit(redis_url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.redis.flushdb()
     self.cursor = '0'
     self.url = redis_url
     from collections import defaultdict
     self.Dict = defaultdict(list)
     super(AtomicDict, self).__init__(path, 'w')
     self.write(redis_url + "\n")
Example #6
0
def map_input_stream(stream, size, url, params):
    """
    An :func:`input_stream` which looks at the scheme of ``url``
    and tries to import a function named ``input_stream``
    from the module ``disco.schemes.scheme_SCHEME``,
    where SCHEME is the parsed scheme.
    If no scheme is found in the url, ``file`` is used.
    The resulting input stream is then used.
    """
    from disco.util import schemesplit
    scheme, _url = schemesplit(url)
    scheme_ = 'scheme_%s' % (scheme or 'file')
    mod = __import__('disco.schemes.%s' % scheme_, fromlist=[scheme_])
    Task.insert_globals([mod.input_stream])
    return mod.input_stream(stream, size, url, params)
Example #7
0
def map_input_stream(stream, size, url, params):
    """
    An :func:`input_stream` which looks at the scheme of ``url``
    and tries to import a function named ``input_stream``
    from the module ``disco.schemes.scheme_SCHEME``,
    where SCHEME is the parsed scheme.
    If no scheme is found in the url, ``file`` is used.
    The resulting input stream is then used.
    """
    from disco.util import schemesplit
    scheme, _url = schemesplit(url)
    scheme_ = 'scheme_%s' % (scheme or 'file')
    mod = __import__('disco.schemes.%s' % scheme_, fromlist=[scheme_])
    Task.insert_globals([mod.input_stream])
    return mod.input_stream(stream, size, url, params)
Example #8
0
 def __init__(self, url):
     _redis_scheme, rest = schemesplit(url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.cursor = '0'
     self.url = url
Example #9
0
def open_url(url, *args, **kwargs):
    from disco.util import schemesplit
    scheme, rest = schemesplit(url)
    if not scheme or scheme == 'file':
        return open_local(rest, *args, **kwargs)
    return open_remote(url, *args, **kwargs)
Example #10
0
def import_scheme(url):
    scheme, _rest = util.schemesplit(url)
    scheme = 'scheme_%s' % (scheme or 'file')
    return __import__('disco.schemes.%s' % scheme, fromlist=[scheme])
Example #11
0
def input_stream(fd, size, url, params):
    """Opens the url locally on the node."""
    scheme, path = schemesplit(url)
    return open_local(path, url)
def open(url, task=None):
    _hdfs_scheme, address = schemesplit(url)
    namenode_port, rest = schemesplit(address)
    http_url = 'http://' + namenode_port + '/webhdfs/v1/' + rest + '?op=OPEN'
    return comm.open_url(http_url)
Example #13
0
def import_scheme(url):
    scheme, _rest = util.schemesplit(url)
    scheme = 'scheme_%s' % (scheme or 'file')
    return __import__('disco.schemes.%s' % scheme, fromlist=[scheme])
Example #14
0
def import_scheme(url):
    scheme, _rest = util.schemesplit(url)
    scheme = 'scheme_{0}'.format((scheme or 'file'))
    return __import__('disco.schemes.{0}'.format(scheme), fromlist=[scheme])
Example #15
0
def input_stream(fd, size, url, params):
    """Opens the url locally on the node."""
    from disco.comm import open_local
    from disco.util import schemesplit
    scheme, path = schemesplit(url)
    return open_local(path)
Example #16
0
def getHdfsMaster(discoMaster):
    from disco.util import schemesplit
    _, rest = schemesplit(discoMaster)
    return rest.split(':')[0] + ':50070'
Example #17
0
def open_url(url, *args, **kwargs):
    from disco.util import schemesplit
    scheme, rest = schemesplit(url)
    if not scheme or scheme == 'file':
        return open_local(rest, *args, **kwargs)
    return open_remote(url, *args, **kwargs)
def getHdfsMaster(discoMaster):
    from disco.util import schemesplit
    _, rest = schemesplit(discoMaster)
    return rest.split(':')[0] + ':50070'
Example #19
0
def open(url, task=None):
    _hdfs_scheme, address = schemesplit(url)
    namenode_port, rest = schemesplit(address)
    http_url = 'http://' + namenode_port + '/webhdfs/v1/' + rest + '?op=OPEN'
    return comm.open_url(http_url)
Example #20
0
 def __init__(self, url):
     _redis_scheme, rest = schemesplit(url)
     host, port, dbid = rest.split(":")
     self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
     self.cursor = '0'
     self.url = url