def iteraddfield(source, field, value, index):
it = iter(source)
flds = it.next()
# determine index of new field
if index is None:
index = len(flds)
# construct output fields
outflds = list(flds)
outflds.insert(index, field)
yield tuple(outflds)
# hybridise rows if using calculated value
if callable(value):
for row in hybridrows(flds, it):
outrow = list(row)
v = value(row)
outrow.insert(index, v)
yield tuple(outrow)
else:
for row in it:
outrow = list(row)
outrow.insert(index, value)
yield tuple(outrow)
def itersimplemultirangeaggregate(table, keys, widths, aggregation, value,
mins, maxs):
if aggregation == len:
aggregation = lambda grp: sum(1 for _ in grp) # count length of iterable
yield ('key', 'value')
# we want a recursive grouping algorithm so we could cope with any number of
# key fields
it = iter(table)
fields = it.next()
# wrap rows
it = hybridrows(fields, it)
# determine value function
if value is None:
getval = lambda v: v # identity function - i.e., whole row
else:
if callable(value):
getval = value
else:
vindices = asindices(fields, value)
getval = operator.itemgetter(*vindices)
for bindef, vals in _recursive_bin(it, 0, [], fields, keys, widths, getval,
mins, maxs):
yield bindef, aggregation(vals)
def iterrowselect(source, where, missing, complement):
it = iter(source)
flds = it.next()
yield tuple(flds)
for row in hybridrows(flds, it, missing): # convert to hybrid row/record
if where(row) != complement: # XOR
yield tuple(row) # need to convert back to tuple?
def iterrowselect(source, where, missing, complement):
it = iter(source)
flds = it.next()
yield tuple(flds)
for row in hybridrows(flds, it, missing): # convert to hybrid row/record
if where(row) != complement: # XOR
yield tuple(row) # need to convert back to tuple?
def iteraddfield(source, field, value, index):
it = iter(source)
flds = it.next()
# determine index of new field
if index is None:
index = len(flds)
# construct output fields
outflds = list(flds)
outflds.insert(index, field)
yield tuple(outflds)
# hybridise rows if using calculated value
if callable(value):
for row in hybridrows(flds, it):
outrow = list(row)
v = value(row)
outrow.insert(index, v)
yield tuple(outrow)
else:
for row in it:
outrow = list(row)
outrow.insert(index, value)
yield tuple(outrow)
def iterrowmapmany(source, rowgenerator, fields, failonerror, missing):
it = iter(source)
srcflds = it.next()
yield tuple(fields)
for row in hybridrows(srcflds, it, missing):
try:
for outrow in rowgenerator(row):
yield tuple(outrow)
except:
if failonerror:
raise
def iterrowmap(source, rowmapper, fields, failonerror, missing):
it = iter(source)
srcflds = it.next()
yield tuple(fields)
for row in hybridrows(srcflds, it, missing):
try:
outrow = rowmapper(row)
yield tuple(outrow)
except:
if failonerror:
raise
def iterrowmapmany(source, rowgenerator, fields, failonerror, missing):
it = iter(source)
srcflds = it.next()
yield tuple(fields)
for row in hybridrows(srcflds, it, missing):
try:
for outrow in rowgenerator(row):
yield tuple(outrow)
except:
if failonerror:
raise
def iterrowmap(source, rowmapper, fields, failonerror, missing):
it = iter(source)
srcflds = it.next()
yield tuple(fields)
for row in hybridrows(srcflds, it, missing):
try:
outrow = rowmapper(row)
yield tuple(outrow)
except:
if failonerror:
raise
def iterselectusingcontext(table, query):
it = iter(table)
fields = tuple(it.next())
yield fields
it = hybridrows(fields, it)
prv = None
cur = it.next()
for nxt in it:
if query(prv, cur, nxt):
yield cur
prv = cur
cur = nxt
# handle last row
if query(prv, cur, None):
yield cur
def iteraddfieldusingcontext(table, field, query):
it = iter(table)
fields = tuple(it.next())
yield fields + (field, )
it = hybridrows(fields, it)
prv = None
cur = it.next()
for nxt in it:
v = query(prv, cur, nxt)
yield tuple(cur) + (v, )
prv = cur
cur = nxt
# handle last row
v = query(prv, cur, None)
yield tuple(cur) + (v, )
def iterselectusingcontext(table, query):
it = iter(table)
fields = tuple(it.next())
yield fields
it = hybridrows(fields, it)
prv = None
cur = it.next()
for nxt in it:
if query(prv, cur, nxt):
yield cur
prv = cur
cur = nxt
# handle last row
if query(prv, cur, None):
yield cur
def iteraddfieldusingcontext(table, field, query):
it = iter(table)
fields = tuple(it.next())
yield fields + (field,)
it = hybridrows(fields, it)
prv = None
cur = it.next()
for nxt in it:
v = query(prv, cur, nxt)
yield tuple(cur) + (v,)
prv = cur
cur = nxt
# handle last row
v = query(prv, cur, None)
yield tuple(cur) + (v,)
def iterfieldmap(source, mappings, failonerror, errorvalue):
it = iter(source)
flds = it.next()
outflds = mappings.keys()
yield tuple(outflds)
mapfuns = dict()
for outfld, m in mappings.items():
if m in flds:
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, int) and m < len(flds):
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, basestring):
mapfuns[outfld] = expr(m)
elif callable(m):
mapfuns[outfld] = m
elif isinstance(m, (tuple, list)) and len(m) == 2:
srcfld = m[0]
fm = m[1]
if callable(fm):
mapfuns[outfld] = composefun(fm, srcfld)
elif isinstance(fm, dict):
mapfuns[outfld] = composedict(fm, srcfld)
else:
raise Exception(
'expected callable or dict') # TODO better error
else:
raise Exception('invalid mapping', outfld, m) # TODO better error
for row in hybridrows(flds, it):
try:
# use list comprehension if possible
outrow = [mapfuns[outfld](row) for outfld in outflds]
except:
# fall back to doing it one field at a time
outrow = list()
for outfld in outflds:
try:
val = mapfuns[outfld](row)
except:
if failonerror:
raise
else:
val = errorvalue
outrow.append(val)
yield tuple(outrow)
def iterfieldmap(source, mappings, failonerror, errorvalue):
it = iter(source)
flds = it.next()
outflds = mappings.keys()
yield tuple(outflds)
mapfuns = dict()
for outfld, m in mappings.items():
if m in flds:
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, int) and m < len(flds):
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, basestring):
mapfuns[outfld] = expr(m)
elif callable(m):
mapfuns[outfld] = m
elif isinstance(m, (tuple, list)) and len(m) == 2:
srcfld = m[0]
fm = m[1]
if callable(fm):
mapfuns[outfld] = composefun(fm, srcfld)
elif isinstance(fm, dict):
mapfuns[outfld] = composedict(fm, srcfld)
else:
raise Exception('expected callable or dict') # TODO better error
else:
raise Exception('invalid mapping', outfld, m) # TODO better error
for row in hybridrows(flds, it):
try:
# use list comprehension if possible
outrow = [mapfuns[outfld](row) for outfld in outflds]
except:
# fall back to doing it one field at a time
outrow = list()
for outfld in outflds:
try:
val = mapfuns[outfld](row)
except:
if failonerror:
raise
else:
val = errorvalue
outrow.append(val)
yield tuple(outrow)
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
pass_row):
# grab the fields in the source table
it = iter(source)
flds = it.next()
yield tuple(flds) # these are not modified
# build converter functions
converter_functions = dict()
with_row = set()
for k, c in converters.items():
# turn field names into row indices
if isinstance(k, basestring):
try:
k = flds.index(k)
except ValueError: # not in list
raise FieldSelectionError(k)
assert isinstance(k, int), 'expected integer, found %r' % k
# is converter a function?
if callable(c):
converter_functions[k] = c
if pass_row:
with_row.add(k)
# is converter a method name?
elif isinstance(c, basestring):
converter_functions[k] = methodcaller(c)
# is converter a method name with arguments?
elif isinstance(c, (tuple, list)) and isinstance(c[0], basestring):
methnm = c[0]
methargs = c[1:]
converter_functions[k] = methodcaller(methnm, *methargs)
# is converter a dictionary?
elif isinstance(c, dict):
converter_functions[k] = dictconverter(c)
# is it something else?
elif c is None:
pass # ignore
else:
raise Exception('unexpected converter specification on field %r: %r' % (k, c))
# define a function to transform a value
def transform_value(i, v, r):
if i not in converter_functions:
# no converter defined on this field, return value as-is
return v
else:
try:
if i in with_row:
return converter_functions[i](v, r)
else:
return converter_functions[i](v)
except:
if failonerror:
raise
else:
return errorvalue
# construct the data rows
if where is None:
for row in hybridrows(flds, it):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
if isinstance(where, basestring):
where = expr(where)
else:
assert callable(where), 'expected callable for "where" argument, found %r' % where
for row in hybridrows(flds, it):
if where(row):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
yield row
def iterfieldconvert(source, converters, failonerror, errorvalue, where):
# grab the fields in the source table
it = iter(source)
flds = it.next()
yield tuple(flds) # these are not modified
# build converter functions
converter_functions = dict()
with_row = set()
for k, c in converters.items():
# turn field names into row indices
if isinstance(k, basestring):
try:
k = flds.index(k)
except ValueError: # not in list
raise FieldSelectionError(k)
assert isinstance(k, int), 'expected integer, found %r' % k
# is converter a function?
if callable(c):
converter_functions[k] = c
# pass value only or also row?
if inspect.isfunction(c):
argspec = inspect.getargspec(c)
if len(argspec.args) > 1 or argspec.varargs is not None:
with_row.add(k)
# is converter a method name?
elif isinstance(c, basestring):
converter_functions[k] = methodcaller(c)
# is converter a method name with arguments?
elif isinstance(c, (tuple, list)) and isinstance(c[0], basestring):
methnm = c[0]
methargs = c[1:]
converter_functions[k] = methodcaller(methnm, *methargs)
# is converter a dictionary?
elif isinstance(c, dict):
converter_functions[k] = dictconverter(c)
# is it something else?
elif c is None:
pass # ignore
else:
raise Exception('unexpected converter specification on field %r: %r' % (k, c))
# define a function to transform a value
def transform_value(i, v, r):
if i not in converter_functions:
# no converter defined on this field, return value as-is
return v
else:
try:
if i in with_row:
return converter_functions[i](v, r)
else:
return converter_functions[i](v)
except:
if failonerror:
raise
else:
return errorvalue
# construct the data rows
if where is None:
for row in hybridrows(flds, it):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
if isinstance(where, basestring):
where = expr(where)
else:
assert callable(where), 'expected callable for "where" argument, found %r' % where
for row in hybridrows(flds, it):
if where(row):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
yield row