コード例 #1
0
ファイル: merge.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph_locations_with_mapping():

    graph = bonobo.Graph()
    mapping_formatter = LineFormatter(file_locations_with_mapping)
    global location_provider_separator
    location_provider_separator = mapping_formatter.get_separator()
    graph.add_chain(extract_locations_with_mapping,
                    mapping_formatter.reformatMapping, load_mapping_provider)
    return graph
コード例 #2
0
ファイル: merge.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph_locations_with_mapping_other():

    graph = bonobo.Graph()
    mapping_formatter = LineFormatter(file_locations_with_mapping_other)
    graph.add_chain(extract_locations_with_mapping_other,
                    mapping_formatter.reformatMapping, load_mapping_other)
    return graph
コード例 #3
0
ファイル: merge.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph_old_file():
    graph = bonobo.Graph()
    old_formatter = LineFormatter(old_file_list_file)

    graph.add_chain(extract_old_file, old_formatter.reformat, load_old)

    return graph
コード例 #4
0
ファイル: merge.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph(owner):

    global location_mapper_separator, location_provider_separator

    new_formatter = LineFormatter(new_file_list_file)

    step1 = Step1Comparing(old_uics)
    step2 = Step2Mapping(mapping_provider, mapping_other_by_uic)
    step3 = Step3AmaFiles(ama_locations, ama_locations_out)
    step4 = Step4ProviderFiles(location_provider_out, location_mapper_out,
                               owner, location_provider_separator,
                               location_mapper_separator)

    graph = bonobo.Graph()

    graph.add_chain(extract_new_file, new_formatter.reformat, security_check,
                    step1.check_new_stations)

    graph.add_chain(step1.new_station,
                    step2.process_mapping,
                    _input=step1.check_new_stations)
    graph.add_chain(step1.old_station_updated,
                    update_counter,
                    _input=step1.check_new_stations)
    graph.add_chain(step1.old_station_unchanged,
                    update_counter,
                    _input=step1.check_new_stations)

    graph.add_chain(step2.no_mapping,
                    step3.generate_glc,
                    _input=step2.process_mapping)
    graph.add_chain(step2.mapped,
                    step4.reformat_provider,
                    step4.update_provider,
                    step4.reformat_mapper,
                    step4.update_mapper,
                    update_counter,
                    _input=step2.process_mapping)
    graph.add_chain(step2.replacing,
                    update_counter,
                    _input=step2.process_mapping)

    graph.add_chain(step3.create_ama_location,
                    step3.load_ama,
                    update_counter,
                    _input=step3.generate_glc)
    graph.add_chain(step4.reformat_provider,
                    step4.update_provider,
                    step4.reformat_mapper,
                    step4.update_mapper,
                    update_counter,
                    _input=step3.generate_glc)

    return graph
コード例 #5
0
ファイル: StepsTest.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph2():
    formatter = LineFormatter(file_new)
    step1 = Step1Comparing(old_uics)
    graph = bonobo.Graph()
    graph.add_chain(extract_new, formatter.reformat, step1.check_new_stations)

    graph.add_chain(step1.new_station, load, _input=step1.check_new_stations)
    graph.add_chain(step1.old_station_updated,
                    load,
                    _input=step1.check_new_stations)
    return graph
コード例 #6
0
ファイル: StepsTest.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph6():
    formatter = LineFormatter(file_new)
    step1 = Step1Comparing(old_uics)
    step2 = Step2Mapping(mapping_provider, mapping_other)
    graph = bonobo.Graph()

    graph.add_chain(extract_new, formatter.reformat, step1.check_new_stations,
                    step1.new_station, step2.process_mapping)

    graph.add_chain(step2.replacing,
                    load_test6_flag,
                    _input=step2.process_mapping)

    return graph
コード例 #7
0
ファイル: StepsTest.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph7():
    formatter = LineFormatter(file_new)
    step1 = Step1Comparing(old_uics)
    step2 = Step2Mapping(mapping_provider, mapping_other)
    step3 = Step3AmaFiles(ama_locations, out_ama)
    graph = bonobo.Graph()

    graph.add_chain(extract_new, formatter.reformat, step1.check_new_stations,
                    step1.new_station, step2.process_mapping)

    graph.add_chain(step2.no_mapping,
                    step3.generate_glc,
                    load,
                    _input=step2.process_mapping)

    return graph
コード例 #8
0
ファイル: merge.py プロジェクト: RenaudCosta/amadeus_etl
def set_rail_location_mapper_separator():
    global location_mapper_separator
    mapping_formatter = LineFormatter(file_rail_location_mapper)
    location_mapper_separator = mapping_formatter.get_separator()
コード例 #9
0
ファイル: StepsTest.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph_mapping_provider():
    graph = bonobo.Graph()
    mapping_formatter = LineFormatter(file_mapping_provider)
    graph.add_chain(extract_mapping_provider,
                    mapping_formatter.reformatMapping, load_mapping_provider)
    return graph
コード例 #10
0
ファイル: StepsTest.py プロジェクト: RenaudCosta/amadeus_etl
def get_graph9():
    formatter = LineFormatter(file_new)
    graph = bonobo.Graph()
    graph.add_chain(extract_new, formatter.reformat, load_ama)
    return graph