def iterfieldmap(source, mappings, failonerror, errorvalue):
it = iter(source)
hdr = next(it)
flds = list(map(text_type, hdr))
outhdr = mappings.keys()
yield tuple(outhdr)
mapfuns = dict()
for outfld, m in mappings.items():
if m in hdr:
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, int) and m < len(hdr):
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, string_types):
mapfuns[outfld] = expr(m)
elif callable(m):
mapfuns[outfld] = m
elif isinstance(m, (tuple, list)) and len(m) == 2:
srcfld = m[0]
fm = m[1]
if callable(fm):
mapfuns[outfld] = composefun(fm, srcfld)
elif isinstance(fm, dict):
mapfuns[outfld] = composedict(fm, srcfld)
else:
raise ArgumentError('expected callable or dict')
else:
raise ArgumentError('invalid mapping %r: %r' % (outfld, m))
# wrap rows as records
it = (Record(row, flds) for row in it)
for row in it:
outrow = list()
for outfld in outhdr:
try:
val = mapfuns[outfld](row)
except Exception as e:
if failonerror == 'inline':
val = e
elif failonerror:
raise e
else:
val = errorvalue
outrow.append(val)
yield tuple(outrow)
def iterfieldmap(source, mappings, failonerror, errorvalue):
it = iter(source)
hdr = next(it)
flds = list(map(text_type, hdr))
outhdr = mappings.keys()
yield tuple(outhdr)
mapfuns = dict()
for outfld, m in mappings.items():
if m in hdr:
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, int) and m < len(hdr):
mapfuns[outfld] = operator.itemgetter(m)
elif isinstance(m, string_types):
mapfuns[outfld] = expr(m)
elif callable(m):
mapfuns[outfld] = m
elif isinstance(m, (tuple, list)) and len(m) == 2:
srcfld = m[0]
fm = m[1]
if callable(fm):
mapfuns[outfld] = composefun(fm, srcfld)
elif isinstance(fm, dict):
mapfuns[outfld] = composedict(fm, srcfld)
else:
raise ArgumentError('expected callable or dict')
else:
raise ArgumentError('invalid mapping %r: %r' % (outfld, m))
# wrap rows as records
it = (Record(row, flds) for row in it)
for row in it:
outrow = list()
for outfld in outhdr:
try:
val = mapfuns[outfld](row)
except Exception as e:
if failonerror:
raise e
else:
val = errorvalue
outrow.append(val)
yield tuple(outrow)
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
pass_row):
# grab the fields in the source table
it = iter(source)
hdr = next(it)
flds = list(map(text_type, hdr))
yield tuple(hdr) # these are not modified
# build converter functions
converter_functions = dict()
for k, c in converters.items():
# turn field names into row indices
if not isinstance(k, integer_types):
try:
k = flds.index(k)
except ValueError: # not in list
raise FieldSelectionError(k)
assert isinstance(k, int), 'expected integer, found %r' % k
# is converter a function?
if callable(c):
converter_functions[k] = c
# is converter a method name?
elif isinstance(c, string_types):
converter_functions[k] = methodcaller(c)
# is converter a method name with arguments?
elif isinstance(c, (tuple, list)) and isinstance(c[0], string_types):
methnm = c[0]
methargs = c[1:]
converter_functions[k] = methodcaller(methnm, *methargs)
# is converter a dictionary?
elif isinstance(c, dict):
converter_functions[k] = dictconverter(c)
# is it something else?
elif c is None:
pass # ignore
else:
raise ArgumentError(
'unexpected converter specification on field %r: %r' % (k, c))
# define a function to transform a value
def transform_value(i, v, *args):
if i not in converter_functions:
# no converter defined on this field, return value as-is
return v
else:
try:
return converter_functions[i](v, *args)
except Exception as e:
if failonerror:
raise e
else:
return errorvalue
# define a function to transform a row
if pass_row:
def transform_row(_row):
return tuple(
transform_value(i, v, _row) for i, v in enumerate(_row))
else:
def transform_row(_row):
return tuple(transform_value(i, v) for i, v in enumerate(_row))
# prepare where function
if isinstance(where, string_types):
where = expr(where)
elif where is not None:
assert callable(where), 'expected callable for "where" argument, ' \
'found %r' % where
# prepare iterator
if pass_row or where:
# wrap rows as records
it = (Record(row, flds) for row in it)
# construct the data rows
if where is None:
# simple case, transform all rows
for row in it:
yield transform_row(row)
else:
# conditionally transform rows
for row in it:
if where(row):
yield transform_row(row)
else:
yield row
def select(table, *args, **kwargs):
"""
Select rows meeting a condition. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar', 'baz'],
... ['a', 4, 9.3],
... ['a', 2, 88.2],
... ['b', 1, 23.3],
... ['c', 8, 42.0],
... ['d', 7, 100.9],
... ['c', 2]]
>>> # the second positional argument can be a function accepting
... # a row
... table2 = etl.select(table1,
... lambda rec: rec.foo == 'a' and rec.baz > 88.1)
>>> table2
+-----+-----+------+
| foo | bar | baz |
+=====+=====+======+
| 'a' | 2 | 88.2 |
+-----+-----+------+
>>> # the second positional argument can also be an expression
... # string, which will be converted to a function using petl.expr()
... table3 = etl.select(table1, "{foo} == 'a' and {baz} > 88.1")
>>> table3
+-----+-----+------+
| foo | bar | baz |
+=====+=====+======+
| 'a' | 2 | 88.2 |
+-----+-----+------+
>>> # the condition can also be applied to a single field
... table4 = etl.select(table1, 'foo', lambda v: v == 'a')
>>> table4
+-----+-----+------+
| foo | bar | baz |
+=====+=====+======+
| 'a' | 4 | 9.3 |
+-----+-----+------+
| 'a' | 2 | 88.2 |
+-----+-----+------+
The complement of the selection can be returned (i.e., the query can be
inverted) by providing `complement=True` as a keyword argument.
"""
missing = kwargs.get('missing', None)
complement = kwargs.get('complement', False)
if len(args) == 0:
raise ArgumentError('missing positional argument')
elif len(args) == 1:
where = args[0]
if isinstance(where, string_types):
where = expr(where)
else:
assert callable(where), 'second argument must be string or callable'
return RowSelectView(table, where, missing=missing,
complement=complement)
else:
field = args[0]
where = args[1]
assert callable(where), 'third argument must be callable'
return FieldSelectView(table, field, where, complement=complement,
missing=missing)
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
pass_row):
# grab the fields in the source table
it = iter(source)
hdr = next(it)
flds = list(map(text_type, hdr))
yield tuple(hdr) # these are not modified
# build converter functions
converter_functions = dict()
for k, c in converters.items():
# turn field names into row indices
if not isinstance(k, integer_types):
try:
k = flds.index(k)
except ValueError: # not in list
raise FieldSelectionError(k)
assert isinstance(k, int), 'expected integer, found %r' % k
# is converter a function?
if callable(c):
converter_functions[k] = c
# is converter a method name?
elif isinstance(c, string_types):
converter_functions[k] = methodcaller(c)
# is converter a method name with arguments?
elif isinstance(c, (tuple, list)) and isinstance(c[0], string_types):
methnm = c[0]
methargs = c[1:]
converter_functions[k] = methodcaller(methnm, *methargs)
# is converter a dictionary?
elif isinstance(c, dict):
converter_functions[k] = dictconverter(c)
# is it something else?
elif c is None:
pass # ignore
else:
raise ArgumentError(
'unexpected converter specification on field %r: %r' % (k, c)
)
# define a function to transform a value
def transform_value(i, v, *args):
if i not in converter_functions:
# no converter defined on this field, return value as-is
return v
else:
try:
return converter_functions[i](v, *args)
except Exception as e:
if failonerror:
raise e
else:
return errorvalue
# define a function to transform a row
if pass_row:
def transform_row(_row):
return tuple(transform_value(i, v, _row)
for i, v in enumerate(_row))
else:
def transform_row(_row):
return tuple(transform_value(i, v)
for i, v in enumerate(_row))
# prepare where function
if isinstance(where, string_types):
where = expr(where)
elif where is not None:
assert callable(where), 'expected callable for "where" argument, ' \
'found %r' % where
# prepare iterator
if pass_row or where:
# wrap rows as records
it = (Record(row, flds) for row in it)
# construct the data rows
if where is None:
# simple case, transform all rows
for row in it:
yield transform_row(row)
else:
# conditionally transform rows
for row in it:
if where(row):
yield transform_row(row)
else:
yield row
def select(table, *args, **kwargs):
"""
Select rows meeting a condition. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar', 'baz'],
... ['a', 4, 9.3],
... ['a', 2, 88.2],
... ['b', 1, 23.3],
... ['c', 8, 42.0],
... ['d', 7, 100.9],
... ['c', 2]]
>>> # the second positional argument can be a function accepting
... # a row
... table2 = etl.select(table1,
... lambda rec: rec.foo == 'a' and rec.baz > 88.1)
>>> table2
+-----+-----+------+
| foo | bar | baz |
+=====+=====+======+
| 'a' | 2 | 88.2 |
+-----+-----+------+
>>> # the second positional argument can also be an expression
... # string, which will be converted to a function using petl.expr()
... table3 = etl.select(table1, "{foo} == 'a' and {baz} > 88.1")
>>> table3
+-----+-----+------+
| foo | bar | baz |
+=====+=====+======+
| 'a' | 2 | 88.2 |
+-----+-----+------+
>>> # the condition can also be applied to a single field
... table4 = etl.select(table1, 'foo', lambda v: v == 'a')
>>> table4
+-----+-----+------+
| foo | bar | baz |
+=====+=====+======+
| 'a' | 4 | 9.3 |
+-----+-----+------+
| 'a' | 2 | 88.2 |
+-----+-----+------+
The complement of the selection can be returned (i.e., the query can be
inverted) by providing `complement=True` as a keyword argument.
"""
missing = kwargs.get('missing', None)
complement = kwargs.get('complement', False)
if len(args) == 0:
raise ArgumentError('missing positional argument')
elif len(args) == 1:
where = args[0]
if isinstance(where, string_types):
where = expr(where)
else:
assert callable(
where), 'second argument must be string or callable'
return RowSelectView(table,
where,
missing=missing,
complement=complement)
else:
field = args[0]
where = args[1]
assert callable(where), 'third argument must be callable'
return FieldSelectView(table,
field,
where,
complement=complement,
missing=missing)