def stat_input_stream(fd, size, url, params):
from disco import util
from hustle.core.marble import MarbleStream
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
print "Error handling hustle_input_stream for %s. %s" % (url, e)
raise e
otab = None
try:
# print "FLurlG: %s" % url
fle = util.localize(rest, disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s" % fle
otab = MarbleStream(fle)
rows = otab.number_rows
frows = float(rows)
rval = {'_': rows, }
for field, (subdb, subindexdb, _, column, _) in otab.dbs.iteritems():
if subindexdb:
rval[field] = subindexdb.stat(otab.txn)['ms_entries'] / frows
yield '', rval
except Exception as e:
print "Gibbers: %s" % e
raise e
finally:
if otab:
otab.close()
def stat_input_stream(fd, size, url, params):
from disco import util
from hustle.core.marble import MarbleStream
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
print "Error handling hustle_input_stream for %s. %s" % (url, e)
raise e
otab = None
try:
# print "FLurlG: %s" % url
fle = util.localize(rest,
disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s" % fle
otab = MarbleStream(fle)
rows = otab.number_rows
frows = float(rows)
rval = {
'_': rows,
}
for field, (subdb, subindexdb, _, column, _) in otab.dbs.iteritems():
if subindexdb:
rval[field] = subindexdb.stat(otab.txn)['ms_entries'] / frows
yield '', rval
except Exception as e:
print "Gibbers: %s" % e
raise e
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index,
key_names):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
fle = util.localize(rest,
disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s %s" % (url, fle)
otab = None
try:
# import sys
# sys.path.append('/Library/Python/2.7/site-packages/pycharm-debug.egg')
# import pydevd
# pydevd.settrace('localhost', port=12999, stdoutToServer=True, stderrToServer=True)
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
row_iter = prefix_gen + \
[otab.mget(col, bitmap) if col is not None else repeat(None, blen)
for col in key_names[index]]
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index,
key_names):
from disco import util
from hustle.core.marble import Expr, MarbleStream
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
print "Error handling hustle_input_stream for %s. %s" % (url, e)
raise e
fle = util.localize(rest,
disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s %s" % (url, fle)
otab = None
try:
# import sys
# sys.path.append('/Library/Python/2.7/site-packages/pycharm-debug.egg')
# import pydevd
# pydevd.settrace('localhost', port=12999, stdoutToServer=True, stderrToServer=True)
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bitmaps[index] = where(otab)
else:
# it is either the table itself, or a partition expression. either way,
# return the entire table
bitmaps[index] = otab.iter_all()
for index, bitmap in bitmaps.iteritems():
prefix = [index] if gen_where_index else []
for row_id in bitmap:
record = [
otab.get(col, row_id) if col else None
for col in key_names[index]
]
# print "Gibbled: %s" % repr(record)
record[
0:
0] = prefix # this looks odd, but is faster than 'prefix + record'
yield tuple(record), empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index, key_names):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
print "Error handling hustle_input_stream for %s. %s" % (url, e)
raise e
fle = util.localize(rest, disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s %s" % (url, fle)
otab = None
try:
# import sys
# sys.path.append('/Library/Python/2.7/site-packages/pycharm-debug.egg')
# import pydevd
# pydevd.settrace('localhost', port=12999, stdoutToServer=True, stderrToServer=True)
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
row_iter = prefix_gen + \
[otab.mget(col, bitmap) if col is not None else repeat(None, blen)
for col in key_names[index]]
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index, key_names):
from disco import util
from hustle.core.marble import Expr, MarbleStream
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
print "Error handling hustle_input_stream for %s. %s" % (url, e)
raise e
fle = util.localize(rest, disco_data=params._task.disco_data, ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s %s" % (url, fle)
otab = None
try:
# import sys
# sys.path.append('/Library/Python/2.7/site-packages/pycharm-debug.egg')
# import pydevd
# pydevd.settrace('localhost', port=12999, stdoutToServer=True, stderrToServer=True)
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bitmaps[index] = where(otab)
else:
# it is either the table itself, or a partition expression. either way,
# return the entire table
bitmaps[index] = otab.iter_all()
for index, bitmap in bitmaps.iteritems():
prefix = [index] if gen_where_index else []
for row_id in bitmap:
record = [otab.get(col, row_id) if col else None for col in key_names[index]]
# print "Gibbled: %s" % repr(record)
record[0:0] = prefix # this looks odd, but is faster than 'prefix + record'
yield tuple(record), empty
finally:
if otab:
otab.close()
def Open(url, task=None):
if task:
disco_data = task.disco_data
ddfs_data = task.ddfs_data
else:
from disco.settings import DiscoSettings
settings = DiscoSettings()
disco_data = settings['DISCO_DATA']
ddfs_data = settings['DDFS_DATA']
scheme, netloc, rest = util.urlsplit(url)
path, rest = rest.split('!', 1) if '!' in rest else (rest, '')
discodb = DiscoDB.load(open(util.localize(path, disco_data=disco_data,
ddfs_data=ddfs_data)))
if rest:
method_name, arg = rest.split('/', 1) if '/' in rest else (rest, None)
method = getattr(discodb, method_name)
if method_name in ('metaquery', 'query'):
return method(Q.urlscan(arg))
return method(*filter(None, arg))
return discodb
def filename_input_stream(fd, size, url, params):
"""This input_stream simply returns the path of the local disk file for this map job"""
from disco import util
from disco.worker.classic import worker
try:
scheme, netloc, rest = util.urlsplit(url)
netloc = "%s:%s" % netloc if netloc[1] else netloc[0]
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
if scheme == 'file':
yield url, "/%s" % rest
else:
# print url, rest
fle = util.localize(rest,
disco_data=worker.Task.disco_data,
ddfs_data=worker.Task.ddfs_data)
yield url, fle
def filename_input_stream(fd, size, url, params):
"""This input_stream simply returns the path of the local disk file for this map job"""
from disco import util
from disco.worker.classic import worker
try:
scheme, netloc, rest = util.urlsplit(url)
netloc = "%s:%s" % netloc if netloc[1] else netloc[0]
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
if scheme == 'file':
yield url, "/%s" % rest
else:
# print url, rest
fle = util.localize(rest,
disco_data=worker.Task.disco_data,
ddfs_data=worker.Task.ddfs_data)
yield url, fle
def Open(url, task=None):
if task:
disco_data = task.disco_data
ddfs_data = task.ddfs_data
else:
from disco.settings import DiscoSettings
settings = DiscoSettings()
disco_data = settings['DISCO_DATA']
ddfs_data = settings['DDFS_DATA']
scheme, netloc, rest = util.urlsplit(url)
path, rest = rest.split('!', 1) if '!' in rest else (rest, '')
discodb = DiscoDB.load(
open(util.localize(path, disco_data=disco_data, ddfs_data=ddfs_data)))
if rest:
method_name, arg = rest.split('/', 1) if '/' in rest else (rest, None)
method = getattr(discodb, method_name)
if method_name in ('metaquery', 'query'):
return method(Q.urlscan(arg))
return method(*filter(None, arg))
return discodb
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index,
key_names, limit):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat, islice, imap
from sys import maxint
from pyebset import BitSet
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
fle = util.localize(rest,
disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
otab = None
try:
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
if limit != maxint:
bs = BitSet()
for i in islice(bm, 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
if limit != maxint:
bs = BitSet()
for i in islice(otab.iter_all(), 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
# row_iter = prefix_gen + \
# [otab.mget(col, bitmap) if col is not None else repeat(None, blen)
# for col in key_names[index]]
row_creators = []
for col, column_fn in key_names[index]:
if col is not None:
if column_fn is None:
row_creators.append(otab.mget(col, bitmap))
else:
row_creators.append(
imap(column_fn, otab.mget(col, bitmap)))
else:
row_creators.append(repeat(None, blen))
row_iter = prefix_gen + row_creators
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index, key_names, limit):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat, islice, imap
from sys import maxint
from pyebset import BitSet
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
fle = util.localize(rest, disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
otab = None
try:
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
if limit != maxint:
bs = BitSet()
for i in islice(bm, 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
if limit != maxint:
bs = BitSet()
for i in islice(otab.iter_all(), 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
# row_iter = prefix_gen + \
# [otab.mget(col, bitmap) if col is not None else repeat(None, blen)
# for col in key_names[index]]
row_creators = []
for col, column_fn in key_names[index]:
if col is not None:
if column_fn is None:
row_creators.append(otab.mget(col, bitmap))
else:
row_creators.append(imap(column_fn, otab.mget(col, bitmap)))
else:
row_creators.append(repeat(None, blen))
row_iter = prefix_gen + row_creators
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()