def input_stream(fd, sze, url, params):
"""Opens a StringIO whose data is everything after the url scheme.
For example, `raw://hello_world` would return `hello_world` when read by the task.
"""
scheme, string = schemesplit(url)
return (StringIO(string), len(string), url)
def input_stream(fd, sze, url, params):
"""Opens a StringIO whose data is everything after the url scheme.
For example, `raw://hello_world` would return `hello_world` when read by the task.
"""
from cStringIO import StringIO
from disco.util import schemesplit
scheme, string = schemesplit(url)
return (StringIO(string), len(string), url)
def input_stream(fd, sze, url, params):
"""Opens a StringIO whose data is everything after the url scheme.
For example, `raw://hello_world` would return `hello_world` when read by the task.
"""
from disco.compat import StringIO, bytes_to_str
from disco.util import schemesplit
scheme, string = schemesplit(url)
ascii = bytes_to_str(string)
return (StringIO(ascii), len(ascii), url)
def __init__(self, path, redis_url):
_redis_scheme, rest = schemesplit(redis_url)
host, port, dbid = rest.split(":")
self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
self.redis.flushdb()
self.cursor = '0'
self.url = redis_url
from collections import defaultdict
self.Dict = defaultdict(list)
super(AtomicDict, self).__init__(path, 'w')
self.write(redis_url + "\n")
def __init__(self, path, redis_url):
_redis_scheme, rest = schemesplit(redis_url)
host, port, dbid = rest.split(":")
self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
self.redis.flushdb()
self.cursor = '0'
self.url = redis_url
from collections import defaultdict
self.Dict = defaultdict(list)
super(AtomicDict, self).__init__(path, 'w')
self.write(redis_url + "\n")
def map_input_stream(stream, size, url, params):
"""
An :func:`input_stream` which looks at the scheme of ``url``
and tries to import a function named ``input_stream``
from the module ``disco.schemes.scheme_SCHEME``,
where SCHEME is the parsed scheme.
If no scheme is found in the url, ``file`` is used.
The resulting input stream is then used.
"""
from disco.util import schemesplit
scheme, _url = schemesplit(url)
scheme_ = 'scheme_%s' % (scheme or 'file')
mod = __import__('disco.schemes.%s' % scheme_, fromlist=[scheme_])
Task.insert_globals([mod.input_stream])
return mod.input_stream(stream, size, url, params)
def map_input_stream(stream, size, url, params):
"""
An :func:`input_stream` which looks at the scheme of ``url``
and tries to import a function named ``input_stream``
from the module ``disco.schemes.scheme_SCHEME``,
where SCHEME is the parsed scheme.
If no scheme is found in the url, ``file`` is used.
The resulting input stream is then used.
"""
from disco.util import schemesplit
scheme, _url = schemesplit(url)
scheme_ = 'scheme_%s' % (scheme or 'file')
mod = __import__('disco.schemes.%s' % scheme_, fromlist=[scheme_])
Task.insert_globals([mod.input_stream])
return mod.input_stream(stream, size, url, params)
def __init__(self, url):
_redis_scheme, rest = schemesplit(url)
host, port, dbid = rest.split(":")
self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
self.cursor = '0'
self.url = url
def open_url(url, *args, **kwargs):
from disco.util import schemesplit
scheme, rest = schemesplit(url)
if not scheme or scheme == 'file':
return open_local(rest, *args, **kwargs)
return open_remote(url, *args, **kwargs)
def import_scheme(url):
scheme, _rest = util.schemesplit(url)
scheme = 'scheme_%s' % (scheme or 'file')
return __import__('disco.schemes.%s' % scheme, fromlist=[scheme])
def input_stream(fd, size, url, params):
"""Opens the url locally on the node."""
scheme, path = schemesplit(url)
return open_local(path, url)
def open(url, task=None):
_hdfs_scheme, address = schemesplit(url)
namenode_port, rest = schemesplit(address)
http_url = 'http://' + namenode_port + '/webhdfs/v1/' + rest + '?op=OPEN'
return comm.open_url(http_url)
def import_scheme(url):
scheme, _rest = util.schemesplit(url)
scheme = 'scheme_%s' % (scheme or 'file')
return __import__('disco.schemes.%s' % scheme, fromlist=[scheme])
def import_scheme(url):
scheme, _rest = util.schemesplit(url)
scheme = 'scheme_{0}'.format((scheme or 'file'))
return __import__('disco.schemes.{0}'.format(scheme), fromlist=[scheme])
def input_stream(fd, size, url, params):
"""Opens the url locally on the node."""
from disco.comm import open_local
from disco.util import schemesplit
scheme, path = schemesplit(url)
return open_local(path)
def getHdfsMaster(discoMaster):
from disco.util import schemesplit
_, rest = schemesplit(discoMaster)
return rest.split(':')[0] + ':50070'
def open_url(url, *args, **kwargs):
from disco.util import schemesplit
scheme, rest = schemesplit(url)
if not scheme or scheme == 'file':
return open_local(rest, *args, **kwargs)
return open_remote(url, *args, **kwargs)
def getHdfsMaster(discoMaster):
from disco.util import schemesplit
_, rest = schemesplit(discoMaster)
return rest.split(':')[0] + ':50070'
def open(url, task=None):
_hdfs_scheme, address = schemesplit(url)
namenode_port, rest = schemesplit(address)
http_url = 'http://' + namenode_port + '/webhdfs/v1/' + rest + '?op=OPEN'
return comm.open_url(http_url)
def __init__(self, url):
_redis_scheme, rest = schemesplit(url)
host, port, dbid = rest.split(":")
self.redis = redis.StrictRedis(host=host, port=port, db=dbid)
self.cursor = '0'
self.url = url