コード例 #1
0
def test_addfield():
    table = (('foo', 'bar'), ('M', 12), ('F', 34), ('-', 56))

    result = addfield(table, 'baz', 42)
    expectation = (('foo', 'bar', 'baz'), ('M', 12, 42), ('F', 34, 42),
                   ('-', 56, 42))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', lambda row: '%s,%s' % (row.foo, row.bar))
    expectation = (('foo', 'bar', 'baz'), ('M', 12, 'M,12'), ('F', 34, 'F,34'),
                   ('-', 56, '-,56'))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', lambda rec: rec['bar'] * 2)
    expectation = (('foo', 'bar', 'baz'), ('M', 12, 24), ('F', 34, 68),
                   ('-', 56, 112))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', expr('{bar} * 2'))
    expectation = (('foo', 'bar', 'baz'), ('M', 12, 24), ('F', 34, 68),
                   ('-', 56, 112))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', 42, index=0)
    expectation = (('baz', 'foo', 'bar'), (42, 'M', 12), (42, 'F', 34),
                   (42, '-', 56))
    ieq(expectation, result)
    ieq(expectation, result)
コード例 #2
0
def iterfieldmap(source, mappings, failonerror, errorvalue):
    it = iter(source)
    flds = it.next()
    outflds = mappings.keys()
    yield tuple(outflds)

    mapfuns = dict()
    for outfld, m in mappings.items():
        if m in flds:
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, int) and m < len(flds):
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, basestring):
            mapfuns[outfld] = expr(m)
        elif callable(m):
            mapfuns[outfld] = m
        elif isinstance(m, (tuple, list)) and len(m) == 2:
            srcfld = m[0]
            fm = m[1]
            if callable(fm):
                mapfuns[outfld] = composefun(fm, srcfld)
            elif isinstance(fm, dict):
                mapfuns[outfld] = composedict(fm, srcfld)
            else:
                raise Exception(
                    'expected callable or dict')  # TODO better error
        else:
            raise Exception('invalid mapping', outfld, m)  # TODO better error

    for row in hybridrows(flds, it):
        try:
            # use list comprehension if possible
            outrow = [mapfuns[outfld](row) for outfld in outflds]
        except:
            # fall back to doing it one field at a time
            outrow = list()
            for outfld in outflds:
                try:
                    val = mapfuns[outfld](row)
                except:
                    if failonerror:
                        raise
                    else:
                        val = errorvalue
                outrow.append(val)
        yield tuple(outrow)
コード例 #3
0
ファイル: test_basics.py プロジェクト: rogerkwoodley/petl
def test_addfield():
    table = (('foo', 'bar'),
             ('M', 12),
             ('F', 34),
             ('-', 56))

    result = addfield(table, 'baz', 42)
    expectation = (('foo', 'bar', 'baz'),
                   ('M', 12, 42),
                   ('F', 34, 42),
                   ('-', 56, 42))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', lambda row: '%s,%s' % (row.foo, row.bar))
    expectation = (('foo', 'bar', 'baz'),
                   ('M', 12, 'M,12'),
                   ('F', 34, 'F,34'),
                   ('-', 56, '-,56'))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', lambda rec: rec['bar'] * 2)
    expectation = (('foo', 'bar', 'baz'),
                   ('M', 12, 24),
                   ('F', 34, 68),
                   ('-', 56, 112))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', expr('{bar} * 2'))
    expectation = (('foo', 'bar', 'baz'),
                   ('M', 12, 24),
                   ('F', 34, 68),
                   ('-', 56, 112))
    ieq(expectation, result)
    ieq(expectation, result)

    result = addfield(table, 'baz', 42, index=0)
    expectation = (('baz', 'foo', 'bar'),
                   (42, 'M', 12),
                   (42, 'F', 34),
                   (42, '-', 56))
    ieq(expectation, result)
    ieq(expectation, result)
コード例 #4
0
ファイル: maps.py プロジェクト: pombredanne/petl
def iterfieldmap(source, mappings, failonerror, errorvalue):
    it = iter(source)
    flds = it.next()
    outflds = mappings.keys()
    yield tuple(outflds)

    mapfuns = dict()
    for outfld, m in mappings.items():
        if m in flds:
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, int) and m < len(flds):
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, basestring):
            mapfuns[outfld] = expr(m)
        elif callable(m):
            mapfuns[outfld] = m
        elif isinstance(m, (tuple, list)) and len(m) == 2:
            srcfld = m[0]
            fm = m[1]
            if callable(fm):
                mapfuns[outfld] = composefun(fm, srcfld)
            elif isinstance(fm, dict):
                mapfuns[outfld] = composedict(fm, srcfld)
            else:
                raise Exception('expected callable or dict') # TODO better error
        else:
            raise Exception('invalid mapping', outfld, m) # TODO better error

    for row in hybridrows(flds, it):
        try:
            # use list comprehension if possible
            outrow = [mapfuns[outfld](row) for outfld in outflds]
        except:
            # fall back to doing it one field at a time
            outrow = list()
            for outfld in outflds:
                try:
                    val = mapfuns[outfld](row)
                except:
                    if failonerror:
                        raise
                    else:
                        val = errorvalue
                outrow.append(val)
        yield tuple(outrow)
コード例 #5
0
ファイル: conversions.py プロジェクト: pombredanne/petl
def iterfieldconvert(source, converters, failonerror, errorvalue, where):

    # grab the fields in the source table
    it = iter(source)
    flds = it.next()
    yield tuple(flds)  # these are not modified

    # build converter functions
    converter_functions = dict()
    with_row = set()
    for k, c in converters.items():

        # turn field names into row indices
        if isinstance(k, basestring):
            try:
                k = flds.index(k)
            except ValueError: # not in list
                raise FieldSelectionError(k)
        assert isinstance(k, int), 'expected integer, found %r' % k

        # is converter a function?
        if callable(c):
            converter_functions[k] = c
            # pass value only or also row?
            if inspect.isfunction(c):
                argspec = inspect.getargspec(c)
                if len(argspec.args) > 1 or argspec.varargs is not None:
                    with_row.add(k)

        # is converter a method name?
        elif isinstance(c, basestring):
            converter_functions[k] = methodcaller(c)

        # is converter a method name with arguments?
        elif isinstance(c, (tuple, list)) and isinstance(c[0], basestring):
            methnm = c[0]
            methargs = c[1:]
            converter_functions[k] = methodcaller(methnm, *methargs)

        # is converter a dictionary?
        elif isinstance(c, dict):
            converter_functions[k] = dictconverter(c)

        # is it something else?
        elif c is None:
            pass  # ignore
        else:
            raise Exception('unexpected converter specification on field %r: %r' % (k, c))

    # define a function to transform a value
    def transform_value(i, v, r):
        if i not in converter_functions:
            # no converter defined on this field, return value as-is
            return v
        else:
            try:
                if i in with_row:
                    return converter_functions[i](v, r)
                else:
                    return converter_functions[i](v)
            except:
                if failonerror:
                    raise
                else:
                    return errorvalue

    # construct the data rows
    if where is None:
        for row in hybridrows(flds, it):
            yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
    else:
        if isinstance(where, basestring):
            where = expr(where)
        else:
            assert callable(where), 'expected callable for "where" argument, found %r' % where
        for row in hybridrows(flds, it):
            if where(row):
                yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
            else:
                yield row
コード例 #6
0
ファイル: selects.py プロジェクト: pombredanne/petl
def select(table, *args, **kwargs):
    """
    Select rows meeting a condition. E.g.::

        >>> from petl import select, look
        >>> look(table1)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 4     | 9.3   |
        +-------+-------+-------+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+
        | 'b'   | 1     | 23.3  |
        +-------+-------+-------+
        | 'c'   | 8     | 42.0  |
        +-------+-------+-------+
        | 'd'   | 7     | 100.9 |
        +-------+-------+-------+
        | 'c'   | 2     |       |
        +-------+-------+-------+

        >>> # the second positional argument can be a function accepting a record
        ... table2 = select(table1, lambda rec: rec[0] == 'a' and rec[1] > 88.1)
        ... # table2 = select(table1, lambda rec: rec['foo'] == 'a' and rec['baz'] > 88.1)
        ... # table2 = select(table1, lambda rec: rec.foo == 'a' and rec.baz > 88.1)
        >>> look(table2)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+

        >>> # the second positional argument can also be an expression string, which
        ... # will be converted to a function using expr()
        ... table3 = select(table1, "{foo} == 'a' and {baz} > 88.1")
        >>> look(table3)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+

        >>> # the condition can also be applied to a single field
        ... table4 = select(table1, 'foo', lambda v: v == 'a')
        >>> look(table4)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 4     | 9.3   |
        +-------+-------+-------+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+

    .. versionchanged:: 0.4

    The complement of the selection can be returned (i.e., the query can be
    inverted) by providing `complement=True` as a keyword argument.

    """

    missing = kwargs.get('missing', None)
    complement = kwargs.get('complement', False)

    if len(args) == 0:
        raise Exception('missing positional argument')
    elif len(args) == 1:
        where = args[0]
        if isinstance(where, basestring):
            where = expr(where)
        else:
            assert callable(
                where), 'second argument must be string or callable'
        return RowSelectView(table,
                             where,
                             missing=missing,
                             complement=complement)
    else:
        field = args[0]
        where = args[1]
        assert callable(where), 'third argument must be callable'
        return FieldSelectView(table, field, where, complement=complement)
コード例 #7
0
ファイル: selects.py プロジェクト: pombredanne/petl
def select(table, *args, **kwargs):
    """
    Select rows meeting a condition. E.g.::

        >>> from petl import select, look
        >>> look(table1)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 4     | 9.3   |
        +-------+-------+-------+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+
        | 'b'   | 1     | 23.3  |
        +-------+-------+-------+
        | 'c'   | 8     | 42.0  |
        +-------+-------+-------+
        | 'd'   | 7     | 100.9 |
        +-------+-------+-------+
        | 'c'   | 2     |       |
        +-------+-------+-------+

        >>> # the second positional argument can be a function accepting a record
        ... table2 = select(table1, lambda rec: rec[0] == 'a' and rec[1] > 88.1)
        ... # table2 = select(table1, lambda rec: rec['foo'] == 'a' and rec['baz'] > 88.1)
        ... # table2 = select(table1, lambda rec: rec.foo == 'a' and rec.baz > 88.1)
        >>> look(table2)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+

        >>> # the second positional argument can also be an expression string, which
        ... # will be converted to a function using expr()
        ... table3 = select(table1, "{foo} == 'a' and {baz} > 88.1")
        >>> look(table3)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+

        >>> # the condition can also be applied to a single field
        ... table4 = select(table1, 'foo', lambda v: v == 'a')
        >>> look(table4)
        +-------+-------+-------+
        | 'foo' | 'bar' | 'baz' |
        +=======+=======+=======+
        | 'a'   | 4     | 9.3   |
        +-------+-------+-------+
        | 'a'   | 2     | 88.2  |
        +-------+-------+-------+

    .. versionchanged:: 0.4

    The complement of the selection can be returned (i.e., the query can be
    inverted) by providing `complement=True` as a keyword argument.

    """

    missing = kwargs.get('missing', None)
    complement = kwargs.get('complement', False)

    if len(args) == 0:
        raise Exception('missing positional argument')
    elif len(args) == 1:
        where = args[0]
        if isinstance(where, basestring):
            where = expr(where)
        else:
            assert callable(where), 'second argument must be string or callable'
        return RowSelectView(table, where, missing=missing, complement=complement)
    else:
        field = args[0]
        where = args[1]
        assert callable(where), 'third argument must be callable'
        return FieldSelectView(table, field, where, complement=complement)
コード例 #8
0
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
                     pass_row):

    # grab the fields in the source table
    it = iter(source)
    flds = it.next()
    yield tuple(flds)  # these are not modified

    # build converter functions
    converter_functions = dict()
    with_row = set()
    for k, c in converters.items():

        # turn field names into row indices
        if isinstance(k, basestring):
            try:
                k = flds.index(k)
            except ValueError: # not in list
                raise FieldSelectionError(k)
        assert isinstance(k, int), 'expected integer, found %r' % k

        # is converter a function?
        if callable(c):
            converter_functions[k] = c
            if pass_row:
                with_row.add(k)

        # is converter a method name?
        elif isinstance(c, basestring):
            converter_functions[k] = methodcaller(c)

        # is converter a method name with arguments?
        elif isinstance(c, (tuple, list)) and isinstance(c[0], basestring):
            methnm = c[0]
            methargs = c[1:]
            converter_functions[k] = methodcaller(methnm, *methargs)

        # is converter a dictionary?
        elif isinstance(c, dict):
            converter_functions[k] = dictconverter(c)

        # is it something else?
        elif c is None:
            pass  # ignore
        else:
            raise Exception('unexpected converter specification on field %r: %r' % (k, c))

    # define a function to transform a value
    def transform_value(i, v, r):
        if i not in converter_functions:
            # no converter defined on this field, return value as-is
            return v
        else:
            try:
                if i in with_row:
                    return converter_functions[i](v, r)
                else:
                    return converter_functions[i](v)
            except:
                if failonerror:
                    raise
                else:
                    return errorvalue

    # construct the data rows
    if where is None:
        for row in hybridrows(flds, it):
            yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
    else:
        if isinstance(where, basestring):
            where = expr(where)
        else:
            assert callable(where), 'expected callable for "where" argument, found %r' % where
        for row in hybridrows(flds, it):
            if where(row):
                yield tuple(transform_value(i, v, row) for i, v in enumerate(row))
            else:
                yield row