def notify_error(message): # Use NOTIFY_OF_CREDENTIAL_POSTING_ERRORS to turn error notification on(true)/off(false); off by default. # It's recommended to have this off during bulk data loads as errors in these situations # can cause an unnecessary flood of notifications. # Turn this on during normal agent oppertion. if NOTIFY_OF_CREDENTIAL_POSTING_ERRORS and NOTIFY_OF_CREDENTIAL_POSTING_ERRORS.lower() == 'true': log_error(message)
# # Copyright 2017-2018 Government of Canada # Public Services and Procurement Canada - buyandsell.gc.ca # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import asyncio from bcreg.credssubmitter import CredsSubmitter from bcreg.rocketchat_hooks import log_error, log_warning, log_info try: loop = asyncio.get_event_loop() with CredsSubmitter() as creds_submitter: loop.run_until_complete( creds_submitter.process_credential_queue(False)) except Exception as e: print("Exception", e) log_error("submit_creds processing exception: " + str(e)) raise
#!/usr/bin/python import psycopg2 import datetime import os import logging from bcreg.config import config from bcreg.eventprocessor import EventProcessor from bcreg.bcregistries import BCRegistries, system_type from bcreg.eventprocessor import EventProcessor, bn_credential from bcreg.rocketchat_hooks import log_error, log_warning, log_info LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper() logging.basicConfig(level=LOG_LEVEL) try: with EventProcessor() as event_processor: event_processor.generate_credential_type(system_type, bn_credential) except Exception as e: print("Exception", e) log_error("process-corps-generate-bn-creds processing exception: " + str(e)) raise
#!/usr/bin/python import psycopg2 import datetime import os import logging from bcreg.config import config from bcreg.eventprocessor import EventProcessor from bcreg.bcregistries import BCRegistries, system_type from bcreg.eventprocessor import EventProcessor from bcreg.rocketchat_hooks import log_error, log_warning, log_info LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper() logging.basicConfig(level=LOG_LEVEL) try: with EventProcessor() as event_processor: event_processor.process_corp_generate_creds() except Exception as e: print("Exception", e) log_error("generate_creds processing exception: " + str(e)) raise
async def process_credential_queue(self, single_thread=False): sql1 = """SELECT RECORD_ID, SYSTEM_TYPE_CD, PREV_EVENT, LAST_EVENT, CORP_NUM, CORP_STATE, CREDENTIAL_TYPE_CD, CREDENTIAL_ID, CREDENTIAL_JSON, CREDENTIAL_REASON, SCHEMA_NAME, SCHEMA_VERSION, ENTRY_DATE FROM CREDENTIAL_LOG WHERE RECORD_ID IN ( SELECT RECORD_ID FROM CREDENTIAL_LOG WHERE PROCESS_DATE is null AND RECORD_ID > %s AND (CREDENTIAL_JSON->>'expiry_date' = '' or CREDENTIAL_JSON->>'expiry_date' is null or CREDENTIAL_JSON->>'expiry_date' <= %s) ORDER BY RECORD_ID LIMIT """ + str(CREDS_BATCH_SIZE) + """ ) ORDER BY RECORD_ID;""" sql1a = """SELECT count(*) cnt FROM CREDENTIAL_LOG WHERE PROCESS_DATE is null AND RECORD_ID > %s AND (CREDENTIAL_JSON->>'expiry_date' = '' or CREDENTIAL_JSON->>'expiry_date' is null or CREDENTIAL_JSON->>'expiry_date' <= %s) """ """ Connect to the PostgreSQL database server """ #conn = None cur = None # Track the current set of tasks. # When gathering tasks at the end we don't want to include these in the list. external_tasks = asyncio.Task.all_tasks() cutoff_time = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) cutoff_time_str = cutoff_time.strftime("%Y-%m-%dT%H:%M:%S") try: params = config(section='event_processor') pool = mpool.ThreadPool(MAX_CREDS_REQUESTS) loop = asyncio.get_event_loop() tasks = [] max_rec_id = 0 # ensure controller is available if not await check_controller_health(wait=True): raise Excecption("Error Issuer Controller is not available") # create a cursor cred_count = 0 cur = self.conn.cursor() cur.execute(sql1a, ( max_rec_id, cutoff_time_str, )) row = cur.fetchone() if row is not None: cred_count = row[0] cur.close() cur = None i = 0 cred_count_remaining = cred_count start_time = time.perf_counter() processing_time = 0 processed_count = 0 perf_proc_count = 0 max_processing_time = 60 * MAX_PROCESSING_MINS success_count = 0 failed_count = 0 progressive_error_count = 0 progressive_success_count = 0 progressive_error_factor = 0 success_seq = 0 current_max_creds_requests = MAX_CREDS_REQUESTS while 0 < cred_count_remaining and processing_time < max_processing_time and failed_count <= CONTROLLER_MAX_ERRORS: # create a cursor cur = self.conn.cursor() cur.execute(sql1, ( max_rec_id, cutoff_time_str, )) row = cur.fetchone() credentials = [] cred_owner_id = '' while row is not None: i = i + 1 processed_count = processed_count + 1 perf_proc_count = perf_proc_count + 1 if processed_count >= PROCESS_LOOP_REPORT_CT: print('>>> Processing {} of {} credentials.'.format( i, cred_count)) processing_time = time.perf_counter() - start_time print('Processing: ' + str(processing_time)) processed_count = 0 credential = { 'RECORD_ID': row[0], 'SYSTEM_TYP_CD': row[1], 'PREV_EVENT': row[2], 'LAST_EVENT': row[3], 'CORP_NUM': row[4], 'CORP_STATE': row[5], 'CREDENTIAL_TYPE_CD': row[6], 'CREDENTIAL_ID': row[7], 'CREDENTIAL_JSON': row[8], 'CREDENTIAL_REASON': row[9], 'SCHEMA_NAME': row[10], 'SCHEMA_VERSION': row[11], 'ENTRY_DATE': row[12] } if max_rec_id < row[0]: max_rec_id = row[0] # make sure to include all credentials for the same client id within the same batch # but also - limit batch size to avoid timeouts if (CREDS_REQUEST_SIZE <= len(credentials) and credential['CORP_NUM'] != cred_owner_id) or ( len(credentials) >= 2 * CREDS_REQUEST_SIZE): post_creds = credentials.copy() creds_task = loop.create_task( post_credentials(self.conn, post_creds)) tasks.append(creds_task) #await asyncio.sleep(1) if single_thread: # running single threaded - wait for each task to complete await creds_task else: # If we start getting too many errors, drop down our concurrent request count. # This will introduce a delay in posting any new credentials, because we need # to wait for many of the active threads to complete. if current_max_creds_requests > 1 and progressive_error_factor > PROGRESSIVE_FAIL_THRESHOLD: current_max_creds_requests = int( 0.5 + current_max_creds_requests * FAILURE_REDUCTION_FACTOR / 100) progressive_error_count = 0 progressive_success_count = 0 progressive_error_factor = 0 success_seq = 0 print( ">>> Too many errors, reducing concurrent threads to:", current_max_creds_requests) elif progressive_error_factor == 0: # if we have a long run of successful posts then bump our throughput back up success_seq = success_seq + 1 if success_seq > SUCCESS_INCREASE_FACTOR and current_max_creds_requests < MAX_CREDS_REQUESTS: current_max_creds_requests = min( MAX_CREDS_REQUESTS, int(0.5 + 1.6 * current_max_creds_requests)) success_seq = 0 print( ">>> Stable success, increasing concurrent threads to:", current_max_creds_requests) else: success_seq = 0 # multi-threaded, check if we are within MAX_CREDS_REQUESTS active requests active_tasks = len( [task for task in tasks if not task.done()]) while active_tasks >= current_max_creds_requests: # done is cumulative, includes the full set of "done" tasks done, pending = await asyncio.wait( tasks, return_when=asyncio.FIRST_COMPLETED) active_tasks = len(pending) # reset counters since we are counting *all* done tasks prev_failed_count = failed_count failed_count = 0 prev_success_count = success_count success_count = 0 prev_progressive_error_factor = progressive_error_factor for finished in done: done_result = finished.result() failed_count = failed_count + done_result[ 'failed'] success_count = success_count + done_result[ 'success'] progressive_error_count = failed_count - prev_failed_count progressive_success_count = success_count - prev_success_count progressive_error_factor = max( 0, prev_progressive_error_factor + (progressive_error_count * PROGRESSIVE_FAIL_FACTOR) - progressive_success_count) credentials = [] cred_owner_id = '' credentials.append(credential) cred_owner_id = credential['CORP_NUM'] row = cur.fetchone() cur.close() cur = None if 0 < len(credentials): post_creds = credentials.copy() tasks.append( loop.create_task( post_credentials(self.conn, post_creds))) credentials = [] cred_owner_id = '' # wait for the current batch of credential posts to complete print('>>> Processing {} of {} credentials.'.format( i, cred_count)) processing_time = time.perf_counter() - start_time print('*** Processing: ' + str(processing_time)) if perf_proc_count > 2 * (CREDS_REQUEST_SIZE * MAX_CREDS_REQUESTS): cpm = 60 * (perf_proc_count - (0.5 * CREDS_REQUEST_SIZE * MAX_CREDS_REQUESTS)) / processing_time print(cpm, "credentials per minute") # ensure controller is (still) available if 0 < failed_count and not await check_controller_health( wait=True): raise Excecption( "Error Issuer Controller is not available") cur = self.conn.cursor() cur.execute(sql1a, ( max_rec_id, cutoff_time_str, )) row = cur.fetchone() if row is not None: cred_count_remaining = row[0] cur.close() cur = None # wait for the current batch of credential posts to complete print(">>> Waiting for all outstanding tasks to complete ...") for response in await asyncio.gather(*tasks): pass tasks = [] print('>>> Completed.') processing_time = time.perf_counter() - start_time print('Processing: ' + str(processing_time)) print(60 * perf_proc_count / processing_time, "credentials per minute") except (Exception, psycopg2.DatabaseError) as error: print(error) print(traceback.format_exc()) log_error( 'An exception was encountered while processing the credential queue:\n{}' .format(str(error))) finally: # Gather all remaining tasks that were spawned during processing ... remaining_tasks = asyncio.Task.all_tasks() for task in external_tasks: # Remove any that were not created during processing ... remaining_tasks.discard(task) if len(remaining_tasks) > 0: await asyncio.gather(*remaining_tasks) if cur is not None: cur.close()
async def process_credential_queue(self, single_thread=False): sql1 = """SELECT RECORD_ID, SYSTEM_TYPE_CD, PREV_EVENT, LAST_EVENT, CORP_NUM, CORP_STATE, CREDENTIAL_TYPE_CD, CREDENTIAL_ID, CREDENTIAL_JSON, CREDENTIAL_REASON, SCHEMA_NAME, SCHEMA_VERSION, ENTRY_DATE FROM CREDENTIAL_LOG WHERE RECORD_ID IN ( SELECT RECORD_ID FROM CREDENTIAL_LOG WHERE PROCESS_DATE is null ORDER BY RECORD_ID LIMIT """ + str(CREDS_BATCH_SIZE) + """ ) ORDER BY RECORD_ID;""" sql1a = """SELECT count(*) cnt FROM CREDENTIAL_LOG WHERE PROCESS_DATE is null""" sql1_active = """SELECT RECORD_ID, SYSTEM_TYPE_CD, PREV_EVENT, LAST_EVENT, CORP_NUM, CORP_STATE, CREDENTIAL_TYPE_CD, CREDENTIAL_ID, CREDENTIAL_JSON, CREDENTIAL_REASON, SCHEMA_NAME, SCHEMA_VERSION, ENTRY_DATE FROM CREDENTIAL_LOG WHERE RECORD_ID IN ( SELECT RECORD_ID FROM CREDENTIAL_LOG WHERE CORP_STATE = 'ACT' and PROCESS_DATE is null ORDER BY RECORD_ID LIMIT """ + str(CREDS_BATCH_SIZE) + """ ) ORDER BY RECORD_ID;""" sql1a_active = """SELECT count(*) cnt FROM CREDENTIAL_LOG WHERE corp_state = 'ACT' and PROCESS_DATE is null;""" """ Connect to the PostgreSQL database server """ #conn = None cur = None # Track the current set of tasks. # When gathering tasks at the end we don't want to include these in the list. external_tasks = asyncio.Task.all_tasks() try: params = config(section='event_processor') pool = mpool.ThreadPool(MAX_CREDS_REQUESTS) loop = asyncio.get_event_loop() tasks = [] http_client = aiohttp.ClientSession() # create a cursor cred_count = 0 cur = self.conn.cursor() cur.execute(sql1a) row = cur.fetchone() if row is not None: cred_count = row[0] cur.close() cur = None i = 0 cred_count_remaining = cred_count start_time = time.perf_counter() processing_time = 0 processed_count = 0 max_processing_time = 10 * 60 while 0 < cred_count_remaining and processing_time < max_processing_time: active_cred_count = 0 cur = self.conn.cursor() cur.execute(sql1a_active) row = cur.fetchone() if row is not None: active_cred_count = row[0] cur.close() cur = None # create a cursor cur = self.conn.cursor() if 0 < active_cred_count: cur.execute(sql1_active) else: cur.execute(sql1) row = cur.fetchone() credentials = [] cred_owner_id = '' while row is not None: i = i + 1 processed_count = processed_count + 1 if processed_count >= 100: print('>>> Processing {} of {} credentials.'.format(i, cred_count)) processing_time = time.perf_counter() - start_time print('Processing: ' + str(processing_time)) processed_count = 0 credential = {'RECORD_ID':row[0], 'SYSTEM_TYP_CD':row[1], 'PREV_EVENT':row[2], 'LAST_EVENT':row[3], 'CORP_NUM':row[4], 'CORP_STATE':row[5], 'CREDENTIAL_TYPE_CD':row[6], 'CREDENTIAL_ID':row[7], 'CREDENTIAL_JSON':row[8], 'CREDENTIAL_REASON':row[9], 'SCHEMA_NAME':row[10], 'SCHEMA_VERSION':row[11], 'ENTRY_DATE':row[12]} # make sure to include all credentials for the same client id within the same batch if (CREDS_REQUEST_SIZE <= len(credentials) and credential['CORP_NUM'] != cred_owner_id) or (len(credentials) >= 2*CREDS_REQUEST_SIZE): post_creds = credentials.copy() creds_task = loop.create_task(post_credentials(http_client, self.conn, post_creds)) tasks.append(creds_task) #await asyncio.sleep(1) if single_thread: # running single threaded - wait for each task to complete await creds_task else: # multi-threaded, check if we are within MAX_CREDS_REQUESTS active requests active_tasks = len([task for task in tasks if not task.done()]) #print("Added task - active = ", active_tasks, ", posted creds = ", len(post_creds)) while active_tasks >= MAX_CREDS_REQUESTS: #await asyncio.gather(*tasks) done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) active_tasks = len(pending) # print("Waited task - active = ", active_tasks) credentials = [] cred_owner_id = '' credentials.append(credential) cred_owner_id = credential['CORP_NUM'] row = cur.fetchone() cur.close() cur = None if 0 < len(credentials): post_creds = credentials.copy() tasks.append(loop.create_task(post_credentials(http_client, self.conn, post_creds))) credentials = [] cred_owner_id = '' # wait for the current batch of credential posts to complete for response in await asyncio.gather(*tasks): pass # print('response:' + response) tasks = [] print('>>> Processing {} of {} credentials.'.format(i, cred_count)) processing_time = time.perf_counter() - start_time print('Processing: ' + str(processing_time)) cur = self.conn.cursor() cur.execute(sql1a) row = cur.fetchone() if row is not None: cred_count_remaining = row[0] cur.close() cur = None except (Exception, psycopg2.DatabaseError) as error: print(error) log_error('An exception was encountered while processing the credential queue:\n{}'.format(str(error))) print(traceback.print_exc()) finally: await http_client.close() # Gather all remaining tasks that were spawned during processing ... remaining_tasks = asyncio.Task.all_tasks() for task in external_tasks: # Remove any that were not created during processing ... remaining_tasks.discard(task) if len(remaining_tasks) > 0: await asyncio.gather(*remaining_tasks) if cur is not None: cur.close()
# do some reasonability checks before we update the queue cur_year = int(datetime.datetime.now().year) if prev_event_id == 0: raise Exception( 'no previous event id found: {}'.format(prev_event_id)) if prev_event_date.year < ( cur_year - 10) or prev_event_date.year > (cur_year + 10): raise Exception('previous event date unreasonable: {}'.format( prev_event_date)) if max_event_id == 0: raise Exception( 'no max event id found: {}'.format(max_event_id)) if max_event_dt.year < (cur_year - 10) or max_event_dt.year > (cur_year + 10): raise Exception( 'max event date unreasonable: {}'.format(max_event_dt)) # reasonability test on the number of outstanding records if MAX_CORPS < len(corps): log_warning("find-unpocessed-events More than max corps: " + str(len(corps))) print("Update our queue") event_processor.update_corp_event_queue(system_type, corps, max_event_id, max_event_date) except Exception as e: print("Exception", e) log_error("find-unpocessed-events processing exception: " + str(e)) raise
mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb') mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db') mara_port = os.environ.get('MARA_DB_PORT', '5432') mara_user = os.environ.get('MARA_DB_USER', 'mara_db') mara_password = os.environ.get('MARA_DB_PASSWORD') try: mara_db.config.databases \ = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)} (initial_load_pipeline, success) = data_integration.pipelines.find_node( ['initialization_and_load_tasks', 'bc_reg_corp_loader']) if success: corps_ct = 1 prev_corps_ct = 0 while 0 < corps_ct and corps_ct != prev_corps_ct: # run at least once to get an initial data load run_pipeline(initial_load_pipeline) with EventProcessor() as eventprocessor: prev_corps_ct = corps_ct corps_ct = eventprocessor.get_outstanding_corps_record_count() log_info("Ran bc_reg_corp_loader for " + str(corps_ct) + " corps") else: print("Pipeline not found") log_error("Pipeline not found for:" + "bc_reg_corp_loader") except Exception as e: print("Exception", e) log_error("bc_reg_corp_loader processing exception: " + str(e)) raise
#!/usr/bin/python import psycopg2 import datetime import json import decimal import os import logging from bcreg.config import config from bcreg.bcregistries import system_type from bcreg.eventprocessor import EventProcessor, bn_credential from bcreg.rocketchat_hooks import log_error, log_warning, log_info LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper() logging.basicConfig(level=LOG_LEVEL) try: with EventProcessor() as event_processor: event_processor.queue_reprocess_credential_type(system_type, bn_credential) except Exception as e: print("Exception", e) log_error("find-un-bned-corps processing exception: " + str(e)) raise
@patch(data_integration.config.root_pipeline) def root_pipeline(): return bc_reg_root_pipeline() mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb') mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db') mara_port = os.environ.get('MARA_DB_PORT', '5432') mara_user = os.environ.get('MARA_DB_USER', 'mara_db') mara_password = os.environ.get('MARA_DB_PASSWORD') try: log_info("Starting bc_reg_event_processor ...") mara_db.config.databases \ = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)} (child_pipeline, success) = data_integration.pipelines.find_node( ['bc_reg_event_processor']) if success: run_pipeline(child_pipeline) log_info("Ran bc_reg_event_processor - complete.") else: print("Pipeline not found") log_error("Pipeline not found for:" + "bc_reg_event_processor") except Exception as e: print("Exception", e) log_error("bc_reg_event_processor processing exception: " + str(e)) raise
# get future-dated corps that haven't yet been processed sql3 = """ select count(*) from event_by_corp_filing where process_success is null and corp_num not in (select corp_num from CORP_AUDIT_LOG); """ evp_future_corp_process_count = event_processor.get_sql_record_count(sql3) evp_bc_reg_match_count = evp_corp_history_count + evp_future_corp_process_count # check count of BC Reg corps vs event processor corps if evp_bc_reg_match_count != bc_reg_count: print("Error missing corps in Event Processor", bc_reg_count, evp_bc_reg_match_count) if ERROR_THRESHOLD_COUNT < abs(evp_bc_reg_match_count - bc_reg_count): log_error( "Error missing corps in Event Processor: BCReg={} EvP={}".format( bc_reg_count, evp_bc_reg_match_count)) else: log_warning( "Warning missing corps in Event Processor: BCReg={} EvP={}".format( bc_reg_count, evp_bc_reg_match_count)) # check counts of event processor input and output if evp_credential_count != evp_corp_history_count: print("Error missing credentials in Event Processor", evp_corp_history_count, evp_credential_count) if ERROR_THRESHOLD_COUNT < abs(evp_credential_count - evp_corp_history_count): log_error( "Error missing credentials in Event Processor: EvP corps={} EvP creds={}" .format(evp_corp_history_count, evp_credential_count))
if prev_event_date.year < ( cur_year - 10) or prev_event_date.year > (cur_year + 10): raise Exception('previous event date unreasonable: {}'.format( prev_event_date)) if max_event_id == 0: raise Exception( 'no max event id found: {}'.format(max_event_id)) if max_event_dt.year < (cur_year - 10) or max_event_dt.year > (cur_year + 10): raise Exception( 'max event date unreasonable: {}'.format(max_event_dt)) # reasonability test on the number of outstanding records if CRAZY_MAX_CORPS < len(corps): log_error( "find-unpocessed-events More than cRaZy MaX corps: " + str(len(corps))) elif MAX_CORPS < len(corps): log_warning("find-unpocessed-events More than max corps: " + str(len(corps))) LOGGER.info("Update our queue") event_processor.update_corp_event_queue(system_type, corps, max_event_id, max_event_date) # process any corps listed as "outstanding" in the audit table # create events in our processing queue LOGGER.info("Check audit report for outstanding/unprocessed corps") audit_corps = event_processor.get_outstanding_audit_corps() LOGGER.info("Unprocessed corps audit count is " +
mara_password = os.environ.get('MARA_DB_PASSWORD') try: log_info("Starting bc_reg_event_processor ...") mara_db.config.databases \ = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)} (child_pipeline, success) = data_integration.pipelines.find_node( ['bc_reg_event_processor']) if success: run_pipeline(child_pipeline) log_info("Ran bc_reg_event_processor - complete.") with EventProcessor() as eventprocessor: corps_ct = eventprocessor.get_outstanding_corps_record_count() if CRAZY_MAX_CORPS < corps_ct: log_error( "bc-reg-pipeline More than cRaZy MaX corps outstanding: " + str(corps_ct)) elif MAX_CORPS < corps_ct: log_warning( "bc-reg-pipeline More than max corps outstanding: " + str(corps_ct)) else: print("Pipeline not found") log_error("Pipeline not found for:" + "bc_reg_event_processor") except Exception as e: print("Exception", e) log_error("bc_reg_event_processor processing exception: " + str(e)) raise
mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb') mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db') mara_port = os.environ.get('MARA_DB_PORT', '5432') mara_user = os.environ.get('MARA_DB_USER', 'mara_db') mara_password = os.environ.get('MARA_DB_PASSWORD') try: mara_db.config.databases \ = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)} (post_credential_pipeline, success) = data_integration.pipelines.find_node( ['initialization_and_load_tasks', 'bc_reg_credential_poster']) if success: creds_ct = 0 with EventProcessor() as eventprocessor: creds_ct = eventprocessor.get_outstanding_creds_record_count() while 0 < creds_ct: run_pipeline(post_credential_pipeline) with EventProcessor() as eventprocessor: creds_ct = eventprocessor.get_outstanding_creds_record_count() log_info("Ran bc_reg_credential_poster for " + str(creds_ct) + " corps") else: print("Pipeline not found") log_error("Pipeline not found for:" + "bc_reg_credential_poster") except Exception as e: print("Exception", e) log_error("bc_reg_credential_poster processing exception: " + str(e)) raise
with EventProcessor() as event_processor: sql1 = """ SELECT count(*) FROM CORP_AUDIT_LOG; """ sql2 = """ SELECT count(*) FROM CORP_AUDIT_LOG WHERE last_credential_id is not null; """ evp_corp_history_count = event_processor.get_sql_record_count(sql1) evp_credential_count = event_processor.get_sql_record_count(sql2) if evp_corp_history_count != bc_reg_count: print("Error missing corps in Event Processor", bc_reg_count, evp_corp_history_count) if ERROR_THRESHOLD_COUNT < abs(evp_corp_history_count - bc_reg_count): log_error( "Error missing corps in Event Processor: reg={} evp={}".format( bc_reg_count, evp_corp_history_count)) else: log_warning( "Warning missing corps in Event Processor: reg={} evp={}".format( bc_reg_count, evp_corp_history_count)) if evp_credential_count != evp_corp_history_count: print("Error missing credentials in Event Processor", evp_corp_history_count, evp_credential_count) if ERROR_THRESHOLD_COUNT < abs(evp_credential_count - evp_corp_history_count): log_error( "Error missing credentials in Event Processor: evp corps={} evp creds={}" .format(evp_corp_history_count, evp_credential_count)) else: