def bc_reg_pipeline_single_thread(): import bcreg pipeline1 = Pipeline( id='bc_reg_event_processor_single_thread', description= 'A pipeline that processes BC Registries events and generates credentials.' ) sub_pipeline1_2 = Pipeline( id='load_and_process_bc_reg_data_single_thread', description='Load BC Reg data and generate credentials') sub_pipeline1_2.add( Task(id='register_un_processed_events_single_thread', description='Register un-processed events', commands=[ExecutePython('./bcreg/find-unprocessed-events.py')])) sub_pipeline1_2.add( Task(id='load_bc_reg_data_single_thread', description='Load BC Registries data', commands=[ExecutePython('./bcreg/register_un_processed_events')]), ['register_un_processed_events_single_thread']) pipeline1.add(sub_pipeline1_2) sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials_single_thread', description='Submit BC Reg credentials to P-X') sub_pipeline1_3.add( Task(id='submit_credentials_single_thread', description='Submit credentials', commands=[ExecutePython('./bcreg/submit-creds-single-thread.py') ])) pipeline1.add(sub_pipeline1_3, ['load_and_process_bc_reg_data_single_thread']) return pipeline1
def bc_reg_pipeline_initial_load(): import bcreg pipeline1 = Pipeline( id='bc_reg_corp_loader', description= 'A pipeline that does the initial data load and credentials for all corporations.' ) sub_pipeline1_2 = Pipeline( id='load_and_process_bc_reg_corps', description='Load Active BC Reg corps and generate credentials') sub_pipeline1_2.add( Task(id='register_un_processed_corps', description='Register un-processed active corps', commands=[ ExecutePython('./bcreg/find-unprocessed-corps_actve.py') ])) sub_pipeline1_2.add( Task(id='load_bc_reg_data_a', description='Load BC Registries data', commands=[ ExecutePython('./bcreg/process-corps-generate-creds.py') ]), ['register_un_processed_corps']) pipeline1.add(sub_pipeline1_2) return pipeline1
def bc_reg_pipeline_post_credentials(): import bcreg pipeline1 = Pipeline( id='bc_reg_credential_poster', description='A pipeline that posts generated credentials to TOB.') sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials_a', description='Submit BC Reg credentials to P-X') sub_pipeline1_3.add( Task(id='submit_credentials_a', description='Submit credentials', commands=[ExecutePython('./bcreg/submit-creds.py')])) pipeline1.add(sub_pipeline1_3) sub_pipeline1_4 = Pipeline( id='populate_evp_audit_table_a', description='Populate Event Processor Audit Table') sub_pipeline1_4.add( Task(id='populate_audit_table_a', description='Populate Audit Table', commands=[ExecutePython('./bcreg/populate_audit_table.py')])) pipeline1.add(sub_pipeline1_4, ['submit_bc_reg_credentials_a']) return pipeline1
def von_data_pipeline(): import von_pipeline pipeline1 = Pipeline( id='von_data_event_processor', description= 'A pipeline that processes von_data events and generates credentials.') sub_pipeline1_2 = Pipeline( id='load_and_process_von_data_data', description='Load von_data data and generate credentials') sub_pipeline1_2.add( Task(id='create_von_data_credentials', description='Create credentials', commands=[ExecutePython('./von_pipeline/generate-creds.py')])) pipeline1.add(sub_pipeline1_2) sub_pipeline1_3 = Pipeline( id='submit_von_data_credentials', description='Submit von_data credentials to P-X') sub_pipeline1_3.add( Task(id='submit_credentials', description='Submit credentials', commands=[ExecutePython('./von_pipeline/submit-creds.py')])) pipeline1.add(sub_pipeline1_3, ['load_and_process_von_data_data']) return pipeline1
def bc_reg_pipeline_bn_credential_load(): import bcreg pipeline1 = Pipeline( id='bc_reg_bn_loader', description= 'A pipeline that creates BN credentials for all existing corporations.' ) sub_pipeline1_2 = Pipeline( id='load_existing_corps_no_bn', description='Load BC Reg corps with no BN credential') sub_pipeline1_2.add( Task(id='register_un_bned_corps', description='Register corps with no BN', commands=[ExecutePython('./bcreg/find-un-bned-corps.py')])) sub_pipeline1_2.add( Task(id='load_corp_bn_data', description='Load BN credentials from company data', commands=[ ExecutePython('./bcreg/process-corps-generate-bn-creds.py') ]), ['register_un_bned_corps']) pipeline1.add(sub_pipeline1_2) return pipeline1
def bc_reg_pipeline(): import bcreg pipeline1 = Pipeline( id='bc_reg_event_processor', description='A pipeline that processes BC Registries events and generates credentials.') sub_pipeline1_2 = Pipeline(id='load_and_process_bc_reg_data', description='Load BC Reg data and generate credentials') sub_pipeline1_2.add(Task(id='register_un_processed_events', description='Register un-processed events', commands=[ExecutePython('./bcreg/find-unprocessed-events.py')])) sub_pipeline1_2.add(Task(id='load_bc_reg_data', description='Load BC Registries data', commands=[ExecutePython('./bcreg/process-corps-generate-creds.py')]), ['register_un_processed_events']) sub_pipeline1_2.add(Task(id='create_bc_reg_credentials', description='Create credentials', commands=[ExecutePython('./bcreg/generate-creds.py')]), ['load_bc_reg_data']) pipeline1.add(sub_pipeline1_2) sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials', description='Submit BC Reg credentials to P-X') sub_pipeline1_3.add(Task(id='submit_credentials', description='Submit credentials', commands=[ExecutePython('./bcreg/submit-creds.py')])) pipeline1.add(sub_pipeline1_3, ['load_and_process_bc_reg_data']) sub_pipeline1_4 = Pipeline(id='populate_evp_audit_table', description='Populate Event Processor Audit Table') sub_pipeline1_4.add(Task(id='populate_audit_table', description='Populate Audit Table', commands=[ExecutePython('./bcreg/populate_audit_table.py')])) pipeline1.add(sub_pipeline1_4, ['submit_bc_reg_credentials']) return pipeline1
def db_init_pipeline(): import bcreg pipeline = Pipeline( id = 'bc_reg_db_init', description = 'Initialize BC Registries Event Processor database') pipeline.add(Task(id='create_tables', description='Create event processing tables', commands=[ExecutePython('./bcreg/create.py')])) pipeline.add(Task(id='initialize_tables', description='Insert configuration data', commands=[ExecutePython('./bcreg/insert.py')]), ['create_tables']) return pipeline
def bc_reg_pipeline_status(): import bcreg pipeline = Pipeline( id='bc_reg_pipeline_status', description='Display overall event processing status.') pipeline.add(Task(id='display_pipeline_status', description='Display status of the overall pipeline processing status', commands=[ExecutePython('./bcreg/display_pipeline_status.py')])) # remove these from the pipeline due to issues connecting to DB's on openshift #pipeline.add(Task(id='display_pipeline_stats', description='Display stats of each stage in the pipeline processing', # commands=[ExecutePython('./bcreg/display_processed_corps_counts.py')])) pipeline.add(Task(id='display_event_processor_stats', description='Display stats of each event processor stage', commands=[ExecutePython('./bcreg/display_event_processor_counts.py')])) return pipeline
def db_init_pipeline(): import von_pipeline pipeline = Pipeline( id='von_data_db_init', description='Initialize von_data Event Processor database') pipeline.add( Task(id='create_tables', description='Create event processing tables', commands=[ExecutePython('./von_pipeline/create.py')])) pipeline.add( Task(id='initialize_tables', description='Insert configuration data', commands=[ExecutePython('./von_pipeline/insert.py')]), ['create_tables']) return pipeline
def bc_reg_test_corps(): import bcreg pipeline = Pipeline( id='bc_reg_test_corps', description='A pipeline that queues up a small set of test corporations.') pipeline.add(Task(id='register_test_corps', description='Register some test corps for processing', commands=[ExecutePython('./bcreg/find-test-corps.py')])) return pipeline
def bc_init_test_data(): import bcreg pipeline = Pipeline( id='bc_reg_test_data', description='A pipeline that initializes event processor database for testing.') pipeline.add(Task(id='register_test_corps', description='Insert some test data for processing', commands=[ExecutePython('./bcreg/insert-test.py')])) return pipeline
def bc_reg_pipeline_status(): import bcreg pipeline = Pipeline( id='bc_reg_pipeline_status', description='Display overall event processing status.') pipeline.add(Task(id='display_pipeline_status', description='Display status of the overall pipeline processing status', commands=[ExecutePython('./bcreg/display_pipeline_status.py')])) return pipeline
def bc_reg_populate_audit_table(): import bcreg pipeline = Pipeline( id='bc_reg_populate_audit_table', description='Populate Audit Table.') pipeline.add(Task(id='populate_audit_table', description='Populate audit table.', commands=[ExecutePython('./bcreg/populate_audit_table.py')])) return pipeline
def von_list_mongo_data(): import von_pipeline pipeline = Pipeline(id='von_list_mongo_data', description='A pipeline that lists data in mongodb.') pipeline.add( Task(id='list_mongo_data', description='List data queued for processing', commands=[ExecutePython('./von_pipeline/list_mongo_data.py')])) return pipeline
def bc_reg_pipeline_post_credentials(): import bcreg pipeline1 = Pipeline( id='bc_reg_credential_poster', description='A pipeline that posts generated credentials to TOB.') sub_pipeline1_3 = Pipeline(id='submit_bc_reg_credentials_a', description='Submit BC Reg credentials to P-X') sub_pipeline1_3.add(Task(id='submit_credentials_a', description='Submit credentials', commands=[ExecutePython('./bcreg/submit-creds.py')])) pipeline1.add(sub_pipeline1_3) return pipeline1
def bc_reg_pipeline_jsonbender(): import bcreg pipeline2 = Pipeline( id='bc_reg_event_processor_json_transform_demo', description='A demo pipeline that processes events and generates credentials using JSONBender.') sub_pipeline2_2 = Pipeline(id='load_and_process_bc_reg_data', description='Load BC Reg data and generate credentials') sub_pipeline2_2.add(Task(id='register_un_processed_events', description='Register un-processed events', commands=[ExecutePython('./bcreg/find-unprocessed-events.py')])) sub_pipeline2_2.add(Task(id='load_bc_reg_data', description='Load BC Registries data', commands=[ExecutePython('./bcreg/process-corps.py')]), ['register_un_processed_events']) sub_pipeline2_2.add(Task(id='create_credentials_jsonbender', description='Create credentials using JSONBender transform', commands=[ExecutePython('./bcreg/generate-creds-bender.py')]), ['load_bc_reg_data']) pipeline2.add(sub_pipeline2_2) sub_pipeline2_3 = Pipeline(id='submit_bc_reg_credentials', description='Submit BC Reg credentials to P-X') sub_pipeline2_3.add(Task(id='submit_credentials', description='Submit credentials', commands=[ExecutePython('./bcreg/submit-creds.py')])) pipeline2.add(sub_pipeline2_3, ['load_and_process_bc_reg_data']) return pipeline2
def von_data_pipeline_single_thread(): import von_pipeline pipeline1 = Pipeline( id='von_data_pipeline_single_thread', description= 'A pipeline that processes von_data events and generates credentials.') sub_pipeline1_2 = Pipeline( id='load_and_process_von_data_data_single_thread', description='Load von_data data and generate credentials') sub_pipeline1_2.add( Task(id='register_un_processed_events_single_thread', description='Register un-processed events', commands=[ ExecutePython('./von_pipeline/find-unprocessed-events.py') ])) sub_pipeline1_2.add( Task(id='load_von_data_data_single_thread', description='Load von_data data', commands=[ ExecutePython('./von_pipeline/register_un_processed_events') ]), ['register_un_processed_events_single_thread']) pipeline1.add(sub_pipeline1_2) sub_pipeline1_3 = Pipeline( id='submit_von_data_credentials_single_thread', description='Submit von_data credentials to P-X') sub_pipeline1_3.add( Task(id='submit_credentials_single_thread', description='Submit credentials', commands=[ ExecutePython('./von_pipeline/submit-creds-single-thread.py') ])) pipeline1.add(sub_pipeline1_3, ['load_and_process_von_data_data_single_thread']) return pipeline1
def von_data_pipeline_status(): import von_pipeline pipeline = Pipeline(id='von_data_pipeline_status', description='Display overall event processing status.') pipeline.add( Task(id='display_pipeline_status', description= 'Display status of the overall pipeline processing status', commands=[ ExecutePython('./von_pipeline/display_pipeline_status.py') ])) return pipeline
def von_data_pipeline_post_credentials(): import von_pipeline pipeline1 = Pipeline( id='von_data_credential_poster', description='A pipeline that posts generated credentials to TOB.') sub_pipeline1_3 = Pipeline( id='submit_von_data_credentials_a', description='Submit von_data credentials to P-X') sub_pipeline1_3.add( Task(id='submit_credentials_a', description='Submit credentials', commands=[ExecutePython('./von_pipeline/submit-creds.py')])) pipeline1.add(sub_pipeline1_3) return pipeline1