예제 #1
0
def iterfieldmap(source, mappings, failonerror, errorvalue):
    it = iter(source)
    hdr = next(it)
    flds = list(map(text_type, hdr))
    outhdr = mappings.keys()
    yield tuple(outhdr)

    mapfuns = dict()
    for outfld, m in mappings.items():
        if m in hdr:
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, int) and m < len(hdr):
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, string_types):
            mapfuns[outfld] = expr(m)
        elif callable(m):
            mapfuns[outfld] = m
        elif isinstance(m, (tuple, list)) and len(m) == 2:
            srcfld = m[0]
            fm = m[1]
            if callable(fm):
                mapfuns[outfld] = composefun(fm, srcfld)
            elif isinstance(fm, dict):
                mapfuns[outfld] = composedict(fm, srcfld)
            else:
                raise ArgumentError('expected callable or dict')
        else:
            raise ArgumentError('invalid mapping %r: %r' % (outfld, m))

    # wrap rows as records
    it = (Record(row, flds) for row in it)
    for row in it:
        outrow = list()
        for outfld in outhdr:
            try:
                val = mapfuns[outfld](row)
            except Exception as e:
                if failonerror == 'inline':
                    val = e
                elif failonerror:
                    raise e
                else:
                    val = errorvalue
            outrow.append(val)
        yield tuple(outrow)
예제 #2
0
파일: maps.py 프로젝트: DeanWay/petl
def iterfieldmap(source, mappings, failonerror, errorvalue):
    it = iter(source)
    hdr = next(it)
    flds = list(map(text_type, hdr))
    outhdr = mappings.keys()
    yield tuple(outhdr)

    mapfuns = dict()
    for outfld, m in mappings.items():
        if m in hdr:
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, int) and m < len(hdr):
            mapfuns[outfld] = operator.itemgetter(m)
        elif isinstance(m, string_types):
            mapfuns[outfld] = expr(m)
        elif callable(m):
            mapfuns[outfld] = m
        elif isinstance(m, (tuple, list)) and len(m) == 2:
            srcfld = m[0]
            fm = m[1]
            if callable(fm):
                mapfuns[outfld] = composefun(fm, srcfld)
            elif isinstance(fm, dict):
                mapfuns[outfld] = composedict(fm, srcfld)
            else:
                raise ArgumentError('expected callable or dict')
        else:
            raise ArgumentError('invalid mapping %r: %r' % (outfld, m))

    # wrap rows as records
    it = (Record(row, flds) for row in it)
    for row in it:
        outrow = list()
        for outfld in outhdr:
            try:
                val = mapfuns[outfld](row)
            except Exception as e:
                if failonerror:
                    raise e
                else:
                    val = errorvalue
            outrow.append(val)
        yield tuple(outrow)
예제 #3
0
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
                     pass_row):

    # grab the fields in the source table
    it = iter(source)
    hdr = next(it)
    flds = list(map(text_type, hdr))
    yield tuple(hdr)  # these are not modified

    # build converter functions
    converter_functions = dict()
    for k, c in converters.items():

        # turn field names into row indices
        if not isinstance(k, integer_types):
            try:
                k = flds.index(k)
            except ValueError:  # not in list
                raise FieldSelectionError(k)
        assert isinstance(k, int), 'expected integer, found %r' % k

        # is converter a function?
        if callable(c):
            converter_functions[k] = c

        # is converter a method name?
        elif isinstance(c, string_types):
            converter_functions[k] = methodcaller(c)

        # is converter a method name with arguments?
        elif isinstance(c, (tuple, list)) and isinstance(c[0], string_types):
            methnm = c[0]
            methargs = c[1:]
            converter_functions[k] = methodcaller(methnm, *methargs)

        # is converter a dictionary?
        elif isinstance(c, dict):
            converter_functions[k] = dictconverter(c)

        # is it something else?
        elif c is None:
            pass  # ignore
        else:
            raise ArgumentError(
                'unexpected converter specification on field %r: %r' % (k, c))

    # define a function to transform a value
    def transform_value(i, v, *args):
        if i not in converter_functions:
            # no converter defined on this field, return value as-is
            return v
        else:
            try:
                return converter_functions[i](v, *args)
            except Exception as e:
                if failonerror:
                    raise e
                else:
                    return errorvalue

    # define a function to transform a row
    if pass_row:

        def transform_row(_row):
            return tuple(
                transform_value(i, v, _row) for i, v in enumerate(_row))
    else:

        def transform_row(_row):
            return tuple(transform_value(i, v) for i, v in enumerate(_row))

    # prepare where function
    if isinstance(where, string_types):
        where = expr(where)
    elif where is not None:
        assert callable(where), 'expected callable for "where" argument, ' \
                                'found %r' % where

    # prepare iterator
    if pass_row or where:
        # wrap rows as records
        it = (Record(row, flds) for row in it)

    # construct the data rows
    if where is None:
        # simple case, transform all rows
        for row in it:
            yield transform_row(row)
    else:
        # conditionally transform rows
        for row in it:
            if where(row):
                yield transform_row(row)
            else:
                yield row
예제 #4
0
파일: selects.py 프로젝트: DeanWay/petl
def select(table, *args, **kwargs):
    """
    Select rows meeting a condition. E.g.::

        >>> import petl as etl
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           ['a', 4, 9.3],
        ...           ['a', 2, 88.2],
        ...           ['b', 1, 23.3],
        ...           ['c', 8, 42.0],
        ...           ['d', 7, 100.9],
        ...           ['c', 2]]
        >>> # the second positional argument can be a function accepting
        ... # a row
        ... table2 = etl.select(table1,
        ...                     lambda rec: rec.foo == 'a' and rec.baz > 88.1)
        >>> table2
        +-----+-----+------+
        | foo | bar | baz  |
        +=====+=====+======+
        | 'a' |   2 | 88.2 |
        +-----+-----+------+

        >>> # the second positional argument can also be an expression
        ... # string, which will be converted to a function using petl.expr()
        ... table3 = etl.select(table1, "{foo} == 'a' and {baz} > 88.1")
        >>> table3
        +-----+-----+------+
        | foo | bar | baz  |
        +=====+=====+======+
        | 'a' |   2 | 88.2 |
        +-----+-----+------+

        >>> # the condition can also be applied to a single field
        ... table4 = etl.select(table1, 'foo', lambda v: v == 'a')
        >>> table4
        +-----+-----+------+
        | foo | bar | baz  |
        +=====+=====+======+
        | 'a' |   4 |  9.3 |
        +-----+-----+------+
        | 'a' |   2 | 88.2 |
        +-----+-----+------+

    The complement of the selection can be returned (i.e., the query can be
    inverted) by providing `complement=True` as a keyword argument.

    """

    missing = kwargs.get('missing', None)
    complement = kwargs.get('complement', False)

    if len(args) == 0:
        raise ArgumentError('missing positional argument')
    elif len(args) == 1:
        where = args[0]
        if isinstance(where, string_types):
            where = expr(where)
        else:
            assert callable(where), 'second argument must be string or callable'
        return RowSelectView(table, where, missing=missing,
                             complement=complement)
    else:
        field = args[0]
        where = args[1]
        assert callable(where), 'third argument must be callable'
        return FieldSelectView(table, field, where, complement=complement,
                               missing=missing)
예제 #5
0
파일: conversions.py 프로젝트: DeanWay/petl
def iterfieldconvert(source, converters, failonerror, errorvalue, where,
                     pass_row):

    # grab the fields in the source table
    it = iter(source)
    hdr = next(it)
    flds = list(map(text_type, hdr))
    yield tuple(hdr)  # these are not modified

    # build converter functions
    converter_functions = dict()
    for k, c in converters.items():

        # turn field names into row indices
        if not isinstance(k, integer_types):
            try:
                k = flds.index(k)
            except ValueError:  # not in list
                raise FieldSelectionError(k)
        assert isinstance(k, int), 'expected integer, found %r' % k

        # is converter a function?
        if callable(c):
            converter_functions[k] = c

        # is converter a method name?
        elif isinstance(c, string_types):
            converter_functions[k] = methodcaller(c)

        # is converter a method name with arguments?
        elif isinstance(c, (tuple, list)) and isinstance(c[0], string_types):
            methnm = c[0]
            methargs = c[1:]
            converter_functions[k] = methodcaller(methnm, *methargs)

        # is converter a dictionary?
        elif isinstance(c, dict):
            converter_functions[k] = dictconverter(c)

        # is it something else?
        elif c is None:
            pass  # ignore
        else:
            raise ArgumentError(
                'unexpected converter specification on field %r: %r' % (k, c)
            )

    # define a function to transform a value
    def transform_value(i, v, *args):
        if i not in converter_functions:
            # no converter defined on this field, return value as-is
            return v
        else:
            try:
                return converter_functions[i](v, *args)
            except Exception as e:
                if failonerror:
                    raise e
                else:
                    return errorvalue

    # define a function to transform a row
    if pass_row:
        def transform_row(_row):
            return tuple(transform_value(i, v, _row)
                         for i, v in enumerate(_row))
    else:
        def transform_row(_row):
            return tuple(transform_value(i, v)
                         for i, v in enumerate(_row))

    # prepare where function
    if isinstance(where, string_types):
        where = expr(where)
    elif where is not None:
        assert callable(where), 'expected callable for "where" argument, ' \
                                'found %r' % where

    # prepare iterator
    if pass_row or where:
        # wrap rows as records
        it = (Record(row, flds) for row in it)

    # construct the data rows
    if where is None:
        # simple case, transform all rows
        for row in it:
            yield transform_row(row)
    else:
        # conditionally transform rows
        for row in it:
            if where(row):
                yield transform_row(row)
            else:
                yield row
예제 #6
0
def select(table, *args, **kwargs):
    """
    Select rows meeting a condition. E.g.::

        >>> import petl as etl
        >>> table1 = [['foo', 'bar', 'baz'],
        ...           ['a', 4, 9.3],
        ...           ['a', 2, 88.2],
        ...           ['b', 1, 23.3],
        ...           ['c', 8, 42.0],
        ...           ['d', 7, 100.9],
        ...           ['c', 2]]
        >>> # the second positional argument can be a function accepting
        ... # a row
        ... table2 = etl.select(table1,
        ...                     lambda rec: rec.foo == 'a' and rec.baz > 88.1)
        >>> table2
        +-----+-----+------+
        | foo | bar | baz  |
        +=====+=====+======+
        | 'a' |   2 | 88.2 |
        +-----+-----+------+

        >>> # the second positional argument can also be an expression
        ... # string, which will be converted to a function using petl.expr()
        ... table3 = etl.select(table1, "{foo} == 'a' and {baz} > 88.1")
        >>> table3
        +-----+-----+------+
        | foo | bar | baz  |
        +=====+=====+======+
        | 'a' |   2 | 88.2 |
        +-----+-----+------+

        >>> # the condition can also be applied to a single field
        ... table4 = etl.select(table1, 'foo', lambda v: v == 'a')
        >>> table4
        +-----+-----+------+
        | foo | bar | baz  |
        +=====+=====+======+
        | 'a' |   4 |  9.3 |
        +-----+-----+------+
        | 'a' |   2 | 88.2 |
        +-----+-----+------+

    The complement of the selection can be returned (i.e., the query can be
    inverted) by providing `complement=True` as a keyword argument.

    """

    missing = kwargs.get('missing', None)
    complement = kwargs.get('complement', False)

    if len(args) == 0:
        raise ArgumentError('missing positional argument')
    elif len(args) == 1:
        where = args[0]
        if isinstance(where, string_types):
            where = expr(where)
        else:
            assert callable(
                where), 'second argument must be string or callable'
        return RowSelectView(table,
                             where,
                             missing=missing,
                             complement=complement)
    else:
        field = args[0]
        where = args[1]
        assert callable(where), 'third argument must be callable'
        return FieldSelectView(table,
                               field,
                               where,
                               complement=complement,
                               missing=missing)