def hustle_input_stream(fd, size, url, params, wheres, gen_where_index,
key_names):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
fle = util.localize(rest,
disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s %s" % (url, fle)
otab = None
try:
# import sys
# sys.path.append('/Library/Python/2.7/site-packages/pycharm-debug.egg')
# import pydevd
# pydevd.settrace('localhost', port=12999, stdoutToServer=True, stderrToServer=True)
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
row_iter = prefix_gen + \
[otab.mget(col, bitmap) if col is not None else repeat(None, blen)
for col in key_names[index]]
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index, key_names):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
print "Error handling hustle_input_stream for %s. %s" % (url, e)
raise e
fle = util.localize(rest, disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
# print "FLOGLE: %s %s" % (url, fle)
otab = None
try:
# import sys
# sys.path.append('/Library/Python/2.7/site-packages/pycharm-debug.egg')
# import pydevd
# pydevd.settrace('localhost', port=12999, stdoutToServer=True, stderrToServer=True)
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
row_iter = prefix_gen + \
[otab.mget(col, bitmap) if col is not None else repeat(None, blen)
for col in key_names[index]]
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index,
key_names, limit):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat, islice, imap
from sys import maxint
from pyebset import BitSet
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
fle = util.localize(rest,
disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
otab = None
try:
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
if limit != maxint:
bs = BitSet()
for i in islice(bm, 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
if limit != maxint:
bs = BitSet()
for i in islice(otab.iter_all(), 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
# row_iter = prefix_gen + \
# [otab.mget(col, bitmap) if col is not None else repeat(None, blen)
# for col in key_names[index]]
row_creators = []
for col, column_fn in key_names[index]:
if col is not None:
if column_fn is None:
row_creators.append(otab.mget(col, bitmap))
else:
row_creators.append(
imap(column_fn, otab.mget(col, bitmap)))
else:
row_creators.append(repeat(None, blen))
row_iter = prefix_gen + row_creators
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()
def hustle_input_stream(fd, size, url, params, wheres, gen_where_index, key_names, limit):
from disco import util
from hustle.core.marble import Expr, MarbleStream
from itertools import izip, repeat, islice, imap
from sys import maxint
from pyebset import BitSet
empty = ()
try:
scheme, netloc, rest = util.urlsplit(url)
except Exception as e:
msg = "Error handling hustle_input_stream for %s. %s" % (url, e)
raise util.DataError(msg, url)
fle = util.localize(rest, disco_data=params._task.disco_data,
ddfs_data=params._task.ddfs_data)
otab = None
try:
otab = MarbleStream(fle)
bitmaps = {}
for index, where in enumerate(wheres):
# do not process where clauses that have nothing to do with this marble
if where._name == otab.marble._name:
if type(where) is Expr and not where.is_partition:
bm = where(otab)
if limit != maxint:
bs = BitSet()
for i in islice(bm, 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (bm, len(bm))
else:
# it is either the table itself, or a partition expression.
# Either way, returns the entire table
if limit != maxint:
bs = BitSet()
for i in islice(otab.iter_all(), 0, limit):
bs.set(i)
bitmaps[index] = (bs, len(bs))
else:
bitmaps[index] = (otab.iter_all(), otab.number_rows)
for index, (bitmap, blen) in bitmaps.iteritems():
prefix_gen = [repeat(index, blen)] if gen_where_index else []
# row_iter = prefix_gen + \
# [otab.mget(col, bitmap) if col is not None else repeat(None, blen)
# for col in key_names[index]]
row_creators = []
for col, column_fn in key_names[index]:
if col is not None:
if column_fn is None:
row_creators.append(otab.mget(col, bitmap))
else:
row_creators.append(imap(column_fn, otab.mget(col, bitmap)))
else:
row_creators.append(repeat(None, blen))
row_iter = prefix_gen + row_creators
for row in izip(*row_iter):
yield row, empty
finally:
if otab:
otab.close()