Python fix_fields示例

示例#1

文件： parse_maya_holdings_change.py 项目： wsheffel/budgetkey-data-pipelines

def flow(*_):
    return Flow(
        update_resource(
            -1,
            name='maya_holdings_change',
            path="data/maya_holdings_change.csv",
        ),
        filter_by_type,
        add_fields(
            FIELDS + OPTIONAL_FIELDS + TABLE_FIELDS + OPTIONAL_TABLE_FIELDS,
            'string'),
        rename_fields(RENAME_FIELDS),
        fix_fields(FIELDS + OPTIONAL_FIELDS + TABLE_FIELDS +
                   OPTIONAL_TABLE_FIELDS),
        validate,
        parse_document,
        delete_fields([
            'document',
            'other',
            'num_files',
            'parser_version',
            'source',
            's3_object_name',
        ]),
    )

示例#2

文件： parse_maya_vip_appointments.py 项目： wsheffel/budgetkey-data-pipelines

def flow(*_):
    return Flow(
        filter_by_type,
        rename_fields(RENAME_FIELDS),
        add_fields(FIELDS, 'string'),
        validate,
        parse_document,
        fix_fields(FIELDS),
        delete_fields([
            'document', 'pdf', 'other', 'num_files', 'parser_version',
            'source', 's3_object_name'
        ]),
    )

示例#3

def flow(*_):
    return Flow(
        update_resource(
            -1, name='maya_company_officer_list', path="data/maya_company_officer_list.csv",
        ),
        filter_by_type,
        rename_fields(RENAME_FIELDS),
        add_fields(FIELDS, 'string'),
        add_fields(OPTIONAL_FIELDS, 'string'),
        add_fields(TABLE_FIELDS,'string'),
        validate,
        parse_document,
        fix_fields(TABLE_FIELDS),
        delete_fields(['document', 'pdf', 'other', 'num_files', 'parser_version', 'source', 's3_object_name']),
    )

示例#4

文件： parse_reported_work_record.py 项目： BarVolunteering/budgetkey-data-pipelines

def flow(*_):
    return Flow(
        update_resource(
            -1,
            name='reported_work_record',
            path="data/reported_work_record.csv",
        ),
        filter_by_type,
        rename_fields(RENAME_FIELDS),
        add_fields(FIELDS, 'string'),
        add_fields(TABLE_FIELDS, 'string'),
        validate,
        parse_document,
        fix_fields(TABLE_FIELDS),
        delete_fields([
            'document', 'pdf', 'other', 'num_files', 'parser_version',
            'source', 's3_object_name', 'id', 'company', 'type', 'fix_for',
            'fixed_by', 'next_doc', 'prev_doc'
        ]),
    )

示例#5

文件： parse_maya_bonds_and_options_holdings_change.py 项目： wsheffel/budgetkey-data-pipelines

def flow(*_):
    return Flow(
        update_resource(
            -1,
            name='maya_bonds_and_options_holdings_change',
            path="data/maya_bonds_and_options_holdings_change.csv",
        ),
        filter_by_type,
        add_fields(FIELDS, 'string'),
        add_fields(TABLE_FIELDS, 'string'),
        add_fields(OPTIONAL_TABLE_FIELDS, 'string'),
        add_fields(ADDITIONAL_FIELDS, 'string'),
        rename_fields(RENAME_FIELDS),
        validate,
        parse_document,
        fix_fields(FIELDS + TABLE_FIELDS + OPTIONAL_TABLE_FIELDS +
                   ADDITIONAL_FIELDS),
        delete_fields([
            'document', 'pdf', 'other', 'num_files', 'parser_version',
            'source', 's3_object_name', 'Table776', 'Table774', 'Table775',
            'OtherChange'
        ]),
    )