Exemplo n.º 1
0
def bc_reg_pipeline_single_thread():
    import bcreg

    pipeline1 = Pipeline(
        id='bc_reg_event_processor_single_thread',
        description=
        'A pipeline that processes BC Registries events and generates credentials.'
    )

    sub_pipeline1_2 = Pipeline(
        id='load_and_process_bc_reg_data_single_thread',
        description='Load BC Reg data and generate credentials')
    sub_pipeline1_2.add(
        Task(id='register_un_processed_events_single_thread',
             description='Register un-processed events',
             commands=[ExecutePython('./bcreg/find-unprocessed-events.py')]))
    sub_pipeline1_2.add(
        Task(id='load_bc_reg_data_single_thread',
             description='Load BC Registries data',
             commands=[ExecutePython('./bcreg/register_un_processed_events')]),
        ['register_un_processed_events_single_thread'])
    pipeline1.add(sub_pipeline1_2)

    sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials_single_thread',
                               description='Submit BC Reg credentials to P-X')
    sub_pipeline1_3.add(
        Task(id='submit_credentials_single_thread',
             description='Submit credentials',
             commands=[ExecutePython('./bcreg/submit-creds-single-thread.py')
                       ]))
    pipeline1.add(sub_pipeline1_3,
                  ['load_and_process_bc_reg_data_single_thread'])

    return pipeline1
Exemplo n.º 2
0
def bc_reg_pipeline_initial_load():
    import bcreg

    pipeline1 = Pipeline(
        id='bc_reg_corp_loader',
        description=
        'A pipeline that does the initial data load and credentials for all corporations.'
    )

    sub_pipeline1_2 = Pipeline(
        id='load_and_process_bc_reg_corps',
        description='Load Active BC Reg corps and generate credentials')
    sub_pipeline1_2.add(
        Task(id='register_un_processed_corps',
             description='Register un-processed active corps',
             commands=[
                 ExecutePython('./bcreg/find-unprocessed-corps_actve.py')
             ]))
    sub_pipeline1_2.add(
        Task(id='load_bc_reg_data_a',
             description='Load BC Registries data',
             commands=[
                 ExecutePython('./bcreg/process-corps-generate-creds.py')
             ]), ['register_un_processed_corps'])
    pipeline1.add(sub_pipeline1_2)

    return pipeline1
def bc_reg_pipeline_post_credentials():
    import bcreg

    pipeline1 = Pipeline(
        id='bc_reg_credential_poster',
        description='A pipeline that posts generated credentials to TOB.')

    sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials_a',
                               description='Submit BC Reg credentials to P-X')
    sub_pipeline1_3.add(
        Task(id='submit_credentials_a',
             description='Submit credentials',
             commands=[ExecutePython('./bcreg/submit-creds.py')]))
    pipeline1.add(sub_pipeline1_3)

    sub_pipeline1_4 = Pipeline(
        id='populate_evp_audit_table_a',
        description='Populate Event Processor Audit Table')
    sub_pipeline1_4.add(
        Task(id='populate_audit_table_a',
             description='Populate Audit Table',
             commands=[ExecutePython('./bcreg/populate_audit_table.py')]))
    pipeline1.add(sub_pipeline1_4, ['submit_bc_reg_credentials_a'])

    return pipeline1
Exemplo n.º 4
0
def von_data_pipeline():
    import von_pipeline

    pipeline1 = Pipeline(
        id='von_data_event_processor',
        description=
        'A pipeline that processes von_data events and generates credentials.')

    sub_pipeline1_2 = Pipeline(
        id='load_and_process_von_data_data',
        description='Load von_data data and generate credentials')
    sub_pipeline1_2.add(
        Task(id='create_von_data_credentials',
             description='Create credentials',
             commands=[ExecutePython('./von_pipeline/generate-creds.py')]))
    pipeline1.add(sub_pipeline1_2)

    sub_pipeline1_3 = Pipeline(
        id='submit_von_data_credentials',
        description='Submit von_data credentials to P-X')
    sub_pipeline1_3.add(
        Task(id='submit_credentials',
             description='Submit credentials',
             commands=[ExecutePython('./von_pipeline/submit-creds.py')]))
    pipeline1.add(sub_pipeline1_3, ['load_and_process_von_data_data'])

    return pipeline1
Exemplo n.º 5
0
def bc_reg_pipeline_bn_credential_load():
    import bcreg

    pipeline1 = Pipeline(
        id='bc_reg_bn_loader',
        description=
        'A pipeline that creates BN credentials for all existing corporations.'
    )

    sub_pipeline1_2 = Pipeline(
        id='load_existing_corps_no_bn',
        description='Load BC Reg corps with no BN credential')
    sub_pipeline1_2.add(
        Task(id='register_un_bned_corps',
             description='Register corps with no BN',
             commands=[ExecutePython('./bcreg/find-un-bned-corps.py')]))
    sub_pipeline1_2.add(
        Task(id='load_corp_bn_data',
             description='Load BN credentials from company data',
             commands=[
                 ExecutePython('./bcreg/process-corps-generate-bn-creds.py')
             ]), ['register_un_bned_corps'])
    pipeline1.add(sub_pipeline1_2)

    return pipeline1
def bc_reg_pipeline():
    import bcreg

    pipeline1 = Pipeline(
        id='bc_reg_event_processor',
        description='A pipeline that processes BC Registries events and generates credentials.')

    sub_pipeline1_2 = Pipeline(id='load_and_process_bc_reg_data', description='Load BC Reg data and generate credentials')
    sub_pipeline1_2.add(Task(id='register_un_processed_events', description='Register un-processed events',
                          commands=[ExecutePython('./bcreg/find-unprocessed-events.py')]))
    sub_pipeline1_2.add(Task(id='load_bc_reg_data', description='Load BC Registries data',
                          commands=[ExecutePython('./bcreg/process-corps-generate-creds.py')]), ['register_un_processed_events'])
    sub_pipeline1_2.add(Task(id='create_bc_reg_credentials', description='Create credentials',
                          commands=[ExecutePython('./bcreg/generate-creds.py')]), ['load_bc_reg_data'])
    pipeline1.add(sub_pipeline1_2)

    sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials', description='Submit BC Reg credentials to P-X')
    sub_pipeline1_3.add(Task(id='submit_credentials', description='Submit credentials',
                          commands=[ExecutePython('./bcreg/submit-creds.py')]))
    pipeline1.add(sub_pipeline1_3, ['load_and_process_bc_reg_data'])

    sub_pipeline1_4 = Pipeline(id='populate_evp_audit_table', description='Populate Event Processor Audit Table')
    sub_pipeline1_4.add(Task(id='populate_audit_table', description='Populate Audit Table',
                          commands=[ExecutePython('./bcreg/populate_audit_table.py')]))
    pipeline1.add(sub_pipeline1_4, ['submit_bc_reg_credentials'])

    return pipeline1
def db_init_pipeline():
    import bcreg

    pipeline = Pipeline(
      id = 'bc_reg_db_init',
      description = 'Initialize BC Registries Event Processor database')

    pipeline.add(Task(id='create_tables', description='Create event processing tables',
                        commands=[ExecutePython('./bcreg/create.py')]))
    pipeline.add(Task(id='initialize_tables', description='Insert configuration data',
                        commands=[ExecutePython('./bcreg/insert.py')]), ['create_tables'])

    return pipeline
def bc_reg_pipeline_status():
    import bcreg

    pipeline = Pipeline(
        id='bc_reg_pipeline_status',
        description='Display overall event processing status.')

    pipeline.add(Task(id='display_pipeline_status', description='Display status of the overall pipeline processing status',
                        commands=[ExecutePython('./bcreg/display_pipeline_status.py')]))
    # remove these from the pipeline due to issues connecting to DB's on openshift
    #pipeline.add(Task(id='display_pipeline_stats', description='Display stats of each stage in the pipeline processing',
    #                    commands=[ExecutePython('./bcreg/display_processed_corps_counts.py')]))
    pipeline.add(Task(id='display_event_processor_stats', description='Display stats of each event processor stage',
                        commands=[ExecutePython('./bcreg/display_event_processor_counts.py')]))

    return pipeline
Exemplo n.º 9
0
def db_init_pipeline():
    import von_pipeline

    pipeline = Pipeline(
        id='von_data_db_init',
        description='Initialize von_data Event Processor database')

    pipeline.add(
        Task(id='create_tables',
             description='Create event processing tables',
             commands=[ExecutePython('./von_pipeline/create.py')]))
    pipeline.add(
        Task(id='initialize_tables',
             description='Insert configuration data',
             commands=[ExecutePython('./von_pipeline/insert.py')]),
        ['create_tables'])

    return pipeline
def bc_reg_test_corps():
    import bcreg

    pipeline = Pipeline(
        id='bc_reg_test_corps',
        description='A pipeline that queues up a small set of test corporations.')

    pipeline.add(Task(id='register_test_corps', description='Register some test corps for processing',
                        commands=[ExecutePython('./bcreg/find-test-corps.py')]))

    return pipeline
def bc_init_test_data():
    import bcreg

    pipeline = Pipeline(
        id='bc_reg_test_data',
        description='A pipeline that initializes event processor database for testing.')

    pipeline.add(Task(id='register_test_corps', description='Insert some test data for processing',
                        commands=[ExecutePython('./bcreg/insert-test.py')]))

    return pipeline
def bc_reg_pipeline_status():
    import bcreg

    pipeline = Pipeline(
        id='bc_reg_pipeline_status',
        description='Display overall event processing status.')

    pipeline.add(Task(id='display_pipeline_status', description='Display status of the overall pipeline processing status',
                        commands=[ExecutePython('./bcreg/display_pipeline_status.py')]))

    return pipeline
def bc_reg_populate_audit_table():
    import bcreg

    pipeline = Pipeline(
        id='bc_reg_populate_audit_table',
        description='Populate Audit Table.')

    pipeline.add(Task(id='populate_audit_table', description='Populate audit table.',
                        commands=[ExecutePython('./bcreg/populate_audit_table.py')]))

    return pipeline
Exemplo n.º 14
0
def von_list_mongo_data():
    import von_pipeline

    pipeline = Pipeline(id='von_list_mongo_data',
                        description='A pipeline that lists data in mongodb.')

    pipeline.add(
        Task(id='list_mongo_data',
             description='List data queued for processing',
             commands=[ExecutePython('./von_pipeline/list_mongo_data.py')]))

    return pipeline
def bc_reg_pipeline_post_credentials():
    import bcreg

    pipeline1 = Pipeline(
        id='bc_reg_credential_poster',
        description='A pipeline that posts generated credentials to TOB.')

    sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials_a', description='Submit BC Reg credentials to P-X')
    sub_pipeline1_3.add(Task(id='submit_credentials_a', description='Submit credentials',
                          commands=[ExecutePython('./bcreg/submit-creds.py')]))
    pipeline1.add(sub_pipeline1_3)

    return pipeline1
def bc_reg_pipeline_jsonbender():
    import bcreg

    pipeline2 = Pipeline(
        id='bc_reg_event_processor_json_transform_demo',
        description='A demo pipeline that processes events and generates credentials using JSONBender.')

    sub_pipeline2_2 = Pipeline(id='load_and_process_bc_reg_data', description='Load BC Reg data and generate credentials')
    sub_pipeline2_2.add(Task(id='register_un_processed_events', description='Register un-processed events',
                          commands=[ExecutePython('./bcreg/find-unprocessed-events.py')]))
    sub_pipeline2_2.add(Task(id='load_bc_reg_data', description='Load BC Registries data',
                          commands=[ExecutePython('./bcreg/process-corps.py')]), ['register_un_processed_events'])
    sub_pipeline2_2.add(Task(id='create_credentials_jsonbender', description='Create credentials using JSONBender transform',
                          commands=[ExecutePython('./bcreg/generate-creds-bender.py')]), ['load_bc_reg_data'])
    pipeline2.add(sub_pipeline2_2)

    sub_pipeline2_3 = Pipeline(id='submit_bc_reg_credentials', description='Submit BC Reg credentials to P-X')
    sub_pipeline2_3.add(Task(id='submit_credentials', description='Submit credentials',
                          commands=[ExecutePython('./bcreg/submit-creds.py')]))
    pipeline2.add(sub_pipeline2_3, ['load_and_process_bc_reg_data'])

    return pipeline2
Exemplo n.º 17
0
def von_data_pipeline_single_thread():
    import von_pipeline

    pipeline1 = Pipeline(
        id='von_data_pipeline_single_thread',
        description=
        'A pipeline that processes von_data events and generates credentials.')

    sub_pipeline1_2 = Pipeline(
        id='load_and_process_von_data_data_single_thread',
        description='Load von_data data and generate credentials')
    sub_pipeline1_2.add(
        Task(id='register_un_processed_events_single_thread',
             description='Register un-processed events',
             commands=[
                 ExecutePython('./von_pipeline/find-unprocessed-events.py')
             ]))
    sub_pipeline1_2.add(
        Task(id='load_von_data_data_single_thread',
             description='Load von_data data',
             commands=[
                 ExecutePython('./von_pipeline/register_un_processed_events')
             ]), ['register_un_processed_events_single_thread'])
    pipeline1.add(sub_pipeline1_2)

    sub_pipeline1_3 = Pipeline(
        id='submit_von_data_credentials_single_thread',
        description='Submit von_data credentials to P-X')
    sub_pipeline1_3.add(
        Task(id='submit_credentials_single_thread',
             description='Submit credentials',
             commands=[
                 ExecutePython('./von_pipeline/submit-creds-single-thread.py')
             ]))
    pipeline1.add(sub_pipeline1_3,
                  ['load_and_process_von_data_data_single_thread'])

    return pipeline1
Exemplo n.º 18
0
def von_data_pipeline_status():
    import von_pipeline

    pipeline = Pipeline(id='von_data_pipeline_status',
                        description='Display overall event processing status.')

    pipeline.add(
        Task(id='display_pipeline_status',
             description=
             'Display status of the overall pipeline processing status',
             commands=[
                 ExecutePython('./von_pipeline/display_pipeline_status.py')
             ]))

    return pipeline
Exemplo n.º 19
0
def von_data_pipeline_post_credentials():
    import von_pipeline

    pipeline1 = Pipeline(
        id='von_data_credential_poster',
        description='A pipeline that posts generated credentials to TOB.')

    sub_pipeline1_3 = Pipeline(
        id='submit_von_data_credentials_a',
        description='Submit von_data credentials to P-X')
    sub_pipeline1_3.add(
        Task(id='submit_credentials_a',
             description='Submit credentials',
             commands=[ExecutePython('./von_pipeline/submit-creds.py')]))
    pipeline1.add(sub_pipeline1_3)

    return pipeline1