def test_addfield():
table = (('foo', 'bar'), ('M', 12), ('F', 34), ('-', 56))
result = addfield(table, 'baz', 42)
expectation = (('foo', 'bar', 'baz'), ('M', 12, 42), ('F', 34, 42),
('-', 56, 42))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', lambda row: '%s,%s' % (row.foo, row.bar))
expectation = (('foo', 'bar', 'baz'), ('M', 12, 'M,12'), ('F', 34, 'F,34'),
('-', 56, '-,56'))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', lambda rec: rec['bar'] * 2)
expectation = (('foo', 'bar', 'baz'), ('M', 12, 24), ('F', 34, 68),
('-', 56, 112))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', expr('{bar} * 2'))
expectation = (('foo', 'bar', 'baz'), ('M', 12, 24), ('F', 34, 68),
('-', 56, 112))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', 42, index=0)
expectation = (('baz', 'foo', 'bar'), (42, 'M', 12), (42, 'F', 34),
(42, '-', 56))
ieq(expectation, result)
ieq(expectation, result)
def iterfieldmap(source, mappings, failonerror, errorvalue):
it = iter(source)
flds = it.next()
outflds = mappings.keys()
yield tuple(outflds)
mapfuns = dict()
for outfld, m in mappings.items():
if m in flds:
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, int) and m < len(flds):
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, basestring):
mapfuns[outfld] = expr(m)
elif callable(m):
mapfuns[outfld] = m
elif isinstance(m, (tuple, list)) and len(m) == 2:
srcfld = m[0]
fm = m[1]
if callable(fm):
mapfuns[outfld] = composefun(fm, srcfld)
elif isinstance(fm, dict):
mapfuns[outfld] = composedict(fm, srcfld)
else:
raise Exception(
'expected callable or dict') # TODO better error
else:
raise Exception('invalid mapping', outfld, m) # TODO better error
for row in hybridrows(flds, it):
try:
# use list comprehension if possible
outrow = [mapfuns[outfld](row) for outfld in outflds]
except:
# fall back to doing it one field at a time
outrow = list()
for outfld in outflds:
try:
val = mapfuns[outfld](row)
except:
if failonerror:
raise
else:
val = errorvalue
outrow.append(val)
yield tuple(outrow)
def test_addfield():
table = (('foo', 'bar'),
('M', 12),
('F', 34),
('-', 56))
result = addfield(table, 'baz', 42)
expectation = (('foo', 'bar', 'baz'),
('M', 12, 42),
('F', 34, 42),
('-', 56, 42))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', lambda row: '%s,%s' % (row.foo, row.bar))
expectation = (('foo', 'bar', 'baz'),
('M', 12, 'M,12'),
('F', 34, 'F,34'),
('-', 56, '-,56'))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', lambda rec: rec['bar'] * 2)
expectation = (('foo', 'bar', 'baz'),
('M', 12, 24),
('F', 34, 68),
('-', 56, 112))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', expr('{bar} * 2'))
expectation = (('foo', 'bar', 'baz'),
('M', 12, 24),
('F', 34, 68),
('-', 56, 112))
ieq(expectation, result)
ieq(expectation, result)
result = addfield(table, 'baz', 42, index=0)
expectation = (('baz', 'foo', 'bar'),
(42, 'M', 12),
(42, 'F', 34),
(42, '-', 56))
ieq(expectation, result)
ieq(expectation, result)
def iterfieldmap(source, mappings, failonerror, errorvalue):
it = iter(source)
flds = it.next()
outflds = mappings.keys()
yield tuple(outflds)
mapfuns = dict()
for outfld, m in mappings.items():
if m in flds:
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, int) and m < len(flds):
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, basestring):
mapfuns[outfld] = expr(m)
elif callable(m):
mapfuns[outfld] = m
elif isinstance(m, (tuple, list)) and len(m) == 2:
srcfld = m[0]
fm = m[1]
if callable(fm):
mapfuns[outfld] = composefun(fm, srcfld)
elif isinstance(fm, dict):
mapfuns[outfld] = composedict(fm, srcfld)
else:
raise Exception('expected callable or dict') # TODO better error
else:
raise Exception('invalid mapping', outfld, m) # TODO better error
for row in hybridrows(flds, it):
try:
# use list comprehension if possible
outrow = [mapfuns[outfld](row) for outfld in outflds]
except:
# fall back to doing it one field at a time
outrow = list()
for outfld in outflds:
try:
val = mapfuns[outfld](row)
except:
if failonerror:
raise
else:
val = errorvalue
outrow.append(val)
yield tuple(outrow)
def iterfieldconvert(source, converters, failonerror, errorvalue, where):
# grab the fields in the source table
it = iter(source)
flds = it.next()
yield tuple(flds) # these are not modified
# build converter functions
converter_functions = dict()
with_row = set()
for k, c in converters.items():
# turn field names into row indices
if isinstance(k, basestring):
try:
k = flds.index(k)
except ValueError: # not in list
raise FieldSelectionError(k)
assert isinstance(k, int), 'expected integer, found %r' % k
# is converter a function?
if callable(c):
converter_functions[k] = c
# pass value only or also row?
if inspect.isfunction(c):
argspec = inspect.getargspec(c)
if len(argspec.args) > 1 or argspec.varargs is not None:
with_row.add(k)
# is converter a method name?
elif isinstance(c, basestring):
converter_functions[k] = methodcaller(c)
# is converter a method name with arguments?
elif isinstance(c, (tuple, list)) and isinstance(c[0], basestring):
methnm = c[0]
methargs = c[1:]
converter_functions[k] = methodcaller(methnm, *methargs)
# is converter a dictionary?
elif isinstance(c, dict):
converter_functions[k] = dictconverter(c)
# is it something else?
elif c is None:
pass # ignore
else:
raise Exception('unexpected converter specification on field %r: %r' % (k, c))
# define a function to transform a value
def transform_value(i, v, r):
if i not in converter_functions:
# no converter defined on this field, return value as-is
return v
else:
try:
if i in with_row:
return converter_functions[i](v, r)
else:
return converter_functions[i](v)
except:
if failonerror:
raise
else:
return errorvalue
# construct the data rows
if where is None:
for row in hybridrows(flds, it):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
if isinstance(where, basestring):
where = expr(where)
else:
assert callable(where), 'expected callable for "where" argument, found %r' % where
for row in hybridrows(flds, it):
if where(row):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
yield row
def select(table, *args, **kwargs):
"""
Select rows meeting a condition. E.g.::
>>> from petl import select, look
>>> look(table1)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 4 | 9.3 |
+-------+-------+-------+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
| 'b' | 1 | 23.3 |
+-------+-------+-------+
| 'c' | 8 | 42.0 |
+-------+-------+-------+
| 'd' | 7 | 100.9 |
+-------+-------+-------+
| 'c' | 2 | |
+-------+-------+-------+
>>> # the second positional argument can be a function accepting a record
... table2 = select(table1, lambda rec: rec[0] == 'a' and rec[1] > 88.1)
... # table2 = select(table1, lambda rec: rec['foo'] == 'a' and rec['baz'] > 88.1)
... # table2 = select(table1, lambda rec: rec.foo == 'a' and rec.baz > 88.1)
>>> look(table2)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
>>> # the second positional argument can also be an expression string, which
... # will be converted to a function using expr()
... table3 = select(table1, "{foo} == 'a' and {baz} > 88.1")
>>> look(table3)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
>>> # the condition can also be applied to a single field
... table4 = select(table1, 'foo', lambda v: v == 'a')
>>> look(table4)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 4 | 9.3 |
+-------+-------+-------+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
.. versionchanged:: 0.4
The complement of the selection can be returned (i.e., the query can be
inverted) by providing `complement=True` as a keyword argument.
"""
missing = kwargs.get('missing', None)
complement = kwargs.get('complement', False)
if len(args) == 0:
raise Exception('missing positional argument')
elif len(args) == 1:
where = args[0]
if isinstance(where, basestring):
where = expr(where)
else:
assert callable(
where), 'second argument must be string or callable'
return RowSelectView(table,
where,
missing=missing,
complement=complement)
else:
field = args[0]
where = args[1]
assert callable(where), 'third argument must be callable'
return FieldSelectView(table, field, where, complement=complement)
def select(table, *args, **kwargs):
"""
Select rows meeting a condition. E.g.::
>>> from petl import select, look
>>> look(table1)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 4 | 9.3 |
+-------+-------+-------+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
| 'b' | 1 | 23.3 |
+-------+-------+-------+
| 'c' | 8 | 42.0 |
+-------+-------+-------+
| 'd' | 7 | 100.9 |
+-------+-------+-------+
| 'c' | 2 | |
+-------+-------+-------+
>>> # the second positional argument can be a function accepting a record
... table2 = select(table1, lambda rec: rec[0] == 'a' and rec[1] > 88.1)
... # table2 = select(table1, lambda rec: rec['foo'] == 'a' and rec['baz'] > 88.1)
... # table2 = select(table1, lambda rec: rec.foo == 'a' and rec.baz > 88.1)
>>> look(table2)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
>>> # the second positional argument can also be an expression string, which
... # will be converted to a function using expr()
... table3 = select(table1, "{foo} == 'a' and {baz} > 88.1")
>>> look(table3)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
>>> # the condition can also be applied to a single field
... table4 = select(table1, 'foo', lambda v: v == 'a')
>>> look(table4)
+-------+-------+-------+
| 'foo' | 'bar' | 'baz' |
+=======+=======+=======+
| 'a' | 4 | 9.3 |
+-------+-------+-------+
| 'a' | 2 | 88.2 |
+-------+-------+-------+
.. versionchanged:: 0.4
The complement of the selection can be returned (i.e., the query can be
inverted) by providing `complement=True` as a keyword argument.
"""
missing = kwargs.get('missing', None)
complement = kwargs.get('complement', False)
if len(args) == 0:
raise Exception('missing positional argument')
elif len(args) == 1:
where = args[0]
if isinstance(where, basestring):
where = expr(where)
else:
assert callable(where), 'second argument must be string or callable'
return RowSelectView(table, where, missing=missing, complement=complement)
else:
field = args[0]
where = args[1]
assert callable(where), 'third argument must be callable'
return FieldSelectView(table, field, where, complement=complement)
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
pass_row):
# grab the fields in the source table
it = iter(source)
flds = it.next()
yield tuple(flds) # these are not modified
# build converter functions
converter_functions = dict()
with_row = set()
for k, c in converters.items():
# turn field names into row indices
if isinstance(k, basestring):
try:
k = flds.index(k)
except ValueError: # not in list
raise FieldSelectionError(k)
assert isinstance(k, int), 'expected integer, found %r' % k
# is converter a function?
if callable(c):
converter_functions[k] = c
if pass_row:
with_row.add(k)
# is converter a method name?
elif isinstance(c, basestring):
converter_functions[k] = methodcaller(c)
# is converter a method name with arguments?
elif isinstance(c, (tuple, list)) and isinstance(c[0], basestring):
methnm = c[0]
methargs = c[1:]
converter_functions[k] = methodcaller(methnm, *methargs)
# is converter a dictionary?
elif isinstance(c, dict):
converter_functions[k] = dictconverter(c)
# is it something else?
elif c is None:
pass # ignore
else:
raise Exception('unexpected converter specification on field %r: %r' % (k, c))
# define a function to transform a value
def transform_value(i, v, r):
if i not in converter_functions:
# no converter defined on this field, return value as-is
return v
else:
try:
if i in with_row:
return converter_functions[i](v, r)
else:
return converter_functions[i](v)
except:
if failonerror:
raise
else:
return errorvalue
# construct the data rows
if where is None:
for row in hybridrows(flds, it):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
if isinstance(where, basestring):
where = expr(where)
else:
assert callable(where), 'expected callable for "where" argument, found %r' % where
for row in hybridrows(flds, it):
if where(row):
yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
else:
yield row