def notify_error(message):
# Use NOTIFY_OF_CREDENTIAL_POSTING_ERRORS to turn error notification on(true)/off(false); off by default.
# It's recommended to have this off during bulk data loads as errors in these situations
# can cause an unnecessary flood of notifications.
# Turn this on during normal agent oppertion.
if NOTIFY_OF_CREDENTIAL_POSTING_ERRORS and NOTIFY_OF_CREDENTIAL_POSTING_ERRORS.lower() == 'true':
log_error(message)
#
# Copyright 2017-2018 Government of Canada
# Public Services and Procurement Canada - buyandsell.gc.ca
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import asyncio
from bcreg.credssubmitter import CredsSubmitter
from bcreg.rocketchat_hooks import log_error, log_warning, log_info
try:
loop = asyncio.get_event_loop()
with CredsSubmitter() as creds_submitter:
loop.run_until_complete(
creds_submitter.process_credential_queue(False))
except Exception as e:
print("Exception", e)
log_error("submit_creds processing exception: " + str(e))
raise
#!/usr/bin/python
import psycopg2
import datetime
import os
import logging
from bcreg.config import config
from bcreg.eventprocessor import EventProcessor
from bcreg.bcregistries import BCRegistries, system_type
from bcreg.eventprocessor import EventProcessor, bn_credential
from bcreg.rocketchat_hooks import log_error, log_warning, log_info
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper()
logging.basicConfig(level=LOG_LEVEL)
try:
with EventProcessor() as event_processor:
event_processor.generate_credential_type(system_type, bn_credential)
except Exception as e:
print("Exception", e)
log_error("process-corps-generate-bn-creds processing exception: " +
str(e))
raise
#!/usr/bin/python
import psycopg2
import datetime
import os
import logging
from bcreg.config import config
from bcreg.eventprocessor import EventProcessor
from bcreg.bcregistries import BCRegistries, system_type
from bcreg.eventprocessor import EventProcessor
from bcreg.rocketchat_hooks import log_error, log_warning, log_info
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper()
logging.basicConfig(level=LOG_LEVEL)
try:
with EventProcessor() as event_processor:
event_processor.process_corp_generate_creds()
except Exception as e:
print("Exception", e)
log_error("generate_creds processing exception: " + str(e))
raise
async def process_credential_queue(self, single_thread=False):
sql1 = """SELECT RECORD_ID,
SYSTEM_TYPE_CD,
PREV_EVENT,
LAST_EVENT,
CORP_NUM,
CORP_STATE,
CREDENTIAL_TYPE_CD,
CREDENTIAL_ID,
CREDENTIAL_JSON,
CREDENTIAL_REASON,
SCHEMA_NAME,
SCHEMA_VERSION,
ENTRY_DATE
FROM CREDENTIAL_LOG
WHERE RECORD_ID IN
(
SELECT RECORD_ID
FROM CREDENTIAL_LOG
WHERE PROCESS_DATE is null
AND RECORD_ID > %s
AND (CREDENTIAL_JSON->>'expiry_date' = '' or CREDENTIAL_JSON->>'expiry_date' is null or CREDENTIAL_JSON->>'expiry_date' <= %s)
ORDER BY RECORD_ID
LIMIT """ + str(CREDS_BATCH_SIZE) + """
)
ORDER BY RECORD_ID;"""
sql1a = """SELECT count(*) cnt
FROM CREDENTIAL_LOG
WHERE PROCESS_DATE is null
AND RECORD_ID > %s
AND (CREDENTIAL_JSON->>'expiry_date' = '' or CREDENTIAL_JSON->>'expiry_date' is null or CREDENTIAL_JSON->>'expiry_date' <= %s)
"""
""" Connect to the PostgreSQL database server """
#conn = None
cur = None
# Track the current set of tasks.
# When gathering tasks at the end we don't want to include these in the list.
external_tasks = asyncio.Task.all_tasks()
cutoff_time = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
cutoff_time_str = cutoff_time.strftime("%Y-%m-%dT%H:%M:%S")
try:
params = config(section='event_processor')
pool = mpool.ThreadPool(MAX_CREDS_REQUESTS)
loop = asyncio.get_event_loop()
tasks = []
max_rec_id = 0
# ensure controller is available
if not await check_controller_health(wait=True):
raise Excecption("Error Issuer Controller is not available")
# create a cursor
cred_count = 0
cur = self.conn.cursor()
cur.execute(sql1a, (
max_rec_id,
cutoff_time_str,
))
row = cur.fetchone()
if row is not None:
cred_count = row[0]
cur.close()
cur = None
i = 0
cred_count_remaining = cred_count
start_time = time.perf_counter()
processing_time = 0
processed_count = 0
perf_proc_count = 0
max_processing_time = 60 * MAX_PROCESSING_MINS
success_count = 0
failed_count = 0
progressive_error_count = 0
progressive_success_count = 0
progressive_error_factor = 0
success_seq = 0
current_max_creds_requests = MAX_CREDS_REQUESTS
while 0 < cred_count_remaining and processing_time < max_processing_time and failed_count <= CONTROLLER_MAX_ERRORS:
# create a cursor
cur = self.conn.cursor()
cur.execute(sql1, (
max_rec_id,
cutoff_time_str,
))
row = cur.fetchone()
credentials = []
cred_owner_id = ''
while row is not None:
i = i + 1
processed_count = processed_count + 1
perf_proc_count = perf_proc_count + 1
if processed_count >= PROCESS_LOOP_REPORT_CT:
print('>>> Processing {} of {} credentials.'.format(
i, cred_count))
processing_time = time.perf_counter() - start_time
print('Processing: ' + str(processing_time))
processed_count = 0
credential = {
'RECORD_ID': row[0],
'SYSTEM_TYP_CD': row[1],
'PREV_EVENT': row[2],
'LAST_EVENT': row[3],
'CORP_NUM': row[4],
'CORP_STATE': row[5],
'CREDENTIAL_TYPE_CD': row[6],
'CREDENTIAL_ID': row[7],
'CREDENTIAL_JSON': row[8],
'CREDENTIAL_REASON': row[9],
'SCHEMA_NAME': row[10],
'SCHEMA_VERSION': row[11],
'ENTRY_DATE': row[12]
}
if max_rec_id < row[0]:
max_rec_id = row[0]
# make sure to include all credentials for the same client id within the same batch
# but also - limit batch size to avoid timeouts
if (CREDS_REQUEST_SIZE <= len(credentials)
and credential['CORP_NUM'] != cred_owner_id) or (
len(credentials) >= 2 * CREDS_REQUEST_SIZE):
post_creds = credentials.copy()
creds_task = loop.create_task(
post_credentials(self.conn, post_creds))
tasks.append(creds_task)
#await asyncio.sleep(1)
if single_thread:
# running single threaded - wait for each task to complete
await creds_task
else:
# If we start getting too many errors, drop down our concurrent request count.
# This will introduce a delay in posting any new credentials, because we need
# to wait for many of the active threads to complete.
if current_max_creds_requests > 1 and progressive_error_factor > PROGRESSIVE_FAIL_THRESHOLD:
current_max_creds_requests = int(
0.5 + current_max_creds_requests *
FAILURE_REDUCTION_FACTOR / 100)
progressive_error_count = 0
progressive_success_count = 0
progressive_error_factor = 0
success_seq = 0
print(
">>> Too many errors, reducing concurrent threads to:",
current_max_creds_requests)
elif progressive_error_factor == 0:
# if we have a long run of successful posts then bump our throughput back up
success_seq = success_seq + 1
if success_seq > SUCCESS_INCREASE_FACTOR and current_max_creds_requests < MAX_CREDS_REQUESTS:
current_max_creds_requests = min(
MAX_CREDS_REQUESTS,
int(0.5 +
1.6 * current_max_creds_requests))
success_seq = 0
print(
">>> Stable success, increasing concurrent threads to:",
current_max_creds_requests)
else:
success_seq = 0
# multi-threaded, check if we are within MAX_CREDS_REQUESTS active requests
active_tasks = len(
[task for task in tasks if not task.done()])
while active_tasks >= current_max_creds_requests:
# done is cumulative, includes the full set of "done" tasks
done, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
active_tasks = len(pending)
# reset counters since we are counting *all* done tasks
prev_failed_count = failed_count
failed_count = 0
prev_success_count = success_count
success_count = 0
prev_progressive_error_factor = progressive_error_factor
for finished in done:
done_result = finished.result()
failed_count = failed_count + done_result[
'failed']
success_count = success_count + done_result[
'success']
progressive_error_count = failed_count - prev_failed_count
progressive_success_count = success_count - prev_success_count
progressive_error_factor = max(
0, prev_progressive_error_factor +
(progressive_error_count *
PROGRESSIVE_FAIL_FACTOR) -
progressive_success_count)
credentials = []
cred_owner_id = ''
credentials.append(credential)
cred_owner_id = credential['CORP_NUM']
row = cur.fetchone()
cur.close()
cur = None
if 0 < len(credentials):
post_creds = credentials.copy()
tasks.append(
loop.create_task(
post_credentials(self.conn, post_creds)))
credentials = []
cred_owner_id = ''
# wait for the current batch of credential posts to complete
print('>>> Processing {} of {} credentials.'.format(
i, cred_count))
processing_time = time.perf_counter() - start_time
print('*** Processing: ' + str(processing_time))
if perf_proc_count > 2 * (CREDS_REQUEST_SIZE *
MAX_CREDS_REQUESTS):
cpm = 60 * (perf_proc_count -
(0.5 * CREDS_REQUEST_SIZE *
MAX_CREDS_REQUESTS)) / processing_time
print(cpm, "credentials per minute")
# ensure controller is (still) available
if 0 < failed_count and not await check_controller_health(
wait=True):
raise Excecption(
"Error Issuer Controller is not available")
cur = self.conn.cursor()
cur.execute(sql1a, (
max_rec_id,
cutoff_time_str,
))
row = cur.fetchone()
if row is not None:
cred_count_remaining = row[0]
cur.close()
cur = None
# wait for the current batch of credential posts to complete
print(">>> Waiting for all outstanding tasks to complete ...")
for response in await asyncio.gather(*tasks):
pass
tasks = []
print('>>> Completed.')
processing_time = time.perf_counter() - start_time
print('Processing: ' + str(processing_time))
print(60 * perf_proc_count / processing_time,
"credentials per minute")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
print(traceback.format_exc())
log_error(
'An exception was encountered while processing the credential queue:\n{}'
.format(str(error)))
finally:
# Gather all remaining tasks that were spawned during processing ...
remaining_tasks = asyncio.Task.all_tasks()
for task in external_tasks:
# Remove any that were not created during processing ...
remaining_tasks.discard(task)
if len(remaining_tasks) > 0:
await asyncio.gather(*remaining_tasks)
if cur is not None:
cur.close()
async def process_credential_queue(self, single_thread=False):
sql1 = """SELECT RECORD_ID,
SYSTEM_TYPE_CD,
PREV_EVENT,
LAST_EVENT,
CORP_NUM,
CORP_STATE,
CREDENTIAL_TYPE_CD,
CREDENTIAL_ID,
CREDENTIAL_JSON,
CREDENTIAL_REASON,
SCHEMA_NAME,
SCHEMA_VERSION,
ENTRY_DATE
FROM CREDENTIAL_LOG
WHERE RECORD_ID IN
(
SELECT RECORD_ID
FROM CREDENTIAL_LOG
WHERE PROCESS_DATE is null
ORDER BY RECORD_ID
LIMIT """ + str(CREDS_BATCH_SIZE) + """
)
ORDER BY RECORD_ID;"""
sql1a = """SELECT count(*) cnt
FROM CREDENTIAL_LOG
WHERE PROCESS_DATE is null"""
sql1_active = """SELECT RECORD_ID,
SYSTEM_TYPE_CD,
PREV_EVENT,
LAST_EVENT,
CORP_NUM,
CORP_STATE,
CREDENTIAL_TYPE_CD,
CREDENTIAL_ID,
CREDENTIAL_JSON,
CREDENTIAL_REASON,
SCHEMA_NAME,
SCHEMA_VERSION,
ENTRY_DATE
FROM CREDENTIAL_LOG
WHERE RECORD_ID IN
(
SELECT RECORD_ID
FROM CREDENTIAL_LOG
WHERE CORP_STATE = 'ACT' and PROCESS_DATE is null
ORDER BY RECORD_ID
LIMIT """ + str(CREDS_BATCH_SIZE) + """
)
ORDER BY RECORD_ID;"""
sql1a_active = """SELECT count(*) cnt
FROM CREDENTIAL_LOG
WHERE corp_state = 'ACT' and PROCESS_DATE is null;"""
""" Connect to the PostgreSQL database server """
#conn = None
cur = None
# Track the current set of tasks.
# When gathering tasks at the end we don't want to include these in the list.
external_tasks = asyncio.Task.all_tasks()
try:
params = config(section='event_processor')
pool = mpool.ThreadPool(MAX_CREDS_REQUESTS)
loop = asyncio.get_event_loop()
tasks = []
http_client = aiohttp.ClientSession()
# create a cursor
cred_count = 0
cur = self.conn.cursor()
cur.execute(sql1a)
row = cur.fetchone()
if row is not None:
cred_count = row[0]
cur.close()
cur = None
i = 0
cred_count_remaining = cred_count
start_time = time.perf_counter()
processing_time = 0
processed_count = 0
max_processing_time = 10 * 60
while 0 < cred_count_remaining and processing_time < max_processing_time:
active_cred_count = 0
cur = self.conn.cursor()
cur.execute(sql1a_active)
row = cur.fetchone()
if row is not None:
active_cred_count = row[0]
cur.close()
cur = None
# create a cursor
cur = self.conn.cursor()
if 0 < active_cred_count:
cur.execute(sql1_active)
else:
cur.execute(sql1)
row = cur.fetchone()
credentials = []
cred_owner_id = ''
while row is not None:
i = i + 1
processed_count = processed_count + 1
if processed_count >= 100:
print('>>> Processing {} of {} credentials.'.format(i, cred_count))
processing_time = time.perf_counter() - start_time
print('Processing: ' + str(processing_time))
processed_count = 0
credential = {'RECORD_ID':row[0], 'SYSTEM_TYP_CD':row[1], 'PREV_EVENT':row[2], 'LAST_EVENT':row[3], 'CORP_NUM':row[4], 'CORP_STATE':row[5],
'CREDENTIAL_TYPE_CD':row[6], 'CREDENTIAL_ID':row[7], 'CREDENTIAL_JSON':row[8], 'CREDENTIAL_REASON':row[9],
'SCHEMA_NAME':row[10], 'SCHEMA_VERSION':row[11], 'ENTRY_DATE':row[12]}
# make sure to include all credentials for the same client id within the same batch
if (CREDS_REQUEST_SIZE <= len(credentials) and credential['CORP_NUM'] != cred_owner_id) or (len(credentials) >= 2*CREDS_REQUEST_SIZE):
post_creds = credentials.copy()
creds_task = loop.create_task(post_credentials(http_client, self.conn, post_creds))
tasks.append(creds_task)
#await asyncio.sleep(1)
if single_thread:
# running single threaded - wait for each task to complete
await creds_task
else:
# multi-threaded, check if we are within MAX_CREDS_REQUESTS active requests
active_tasks = len([task for task in tasks if not task.done()])
#print("Added task - active = ", active_tasks, ", posted creds = ", len(post_creds))
while active_tasks >= MAX_CREDS_REQUESTS:
#await asyncio.gather(*tasks)
done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
active_tasks = len(pending)
# print("Waited task - active = ", active_tasks)
credentials = []
cred_owner_id = ''
credentials.append(credential)
cred_owner_id = credential['CORP_NUM']
row = cur.fetchone()
cur.close()
cur = None
if 0 < len(credentials):
post_creds = credentials.copy()
tasks.append(loop.create_task(post_credentials(http_client, self.conn, post_creds)))
credentials = []
cred_owner_id = ''
# wait for the current batch of credential posts to complete
for response in await asyncio.gather(*tasks):
pass # print('response:' + response)
tasks = []
print('>>> Processing {} of {} credentials.'.format(i, cred_count))
processing_time = time.perf_counter() - start_time
print('Processing: ' + str(processing_time))
cur = self.conn.cursor()
cur.execute(sql1a)
row = cur.fetchone()
if row is not None:
cred_count_remaining = row[0]
cur.close()
cur = None
except (Exception, psycopg2.DatabaseError) as error:
print(error)
log_error('An exception was encountered while processing the credential queue:\n{}'.format(str(error)))
print(traceback.print_exc())
finally:
await http_client.close()
# Gather all remaining tasks that were spawned during processing ...
remaining_tasks = asyncio.Task.all_tasks()
for task in external_tasks:
# Remove any that were not created during processing ...
remaining_tasks.discard(task)
if len(remaining_tasks) > 0:
await asyncio.gather(*remaining_tasks)
if cur is not None:
cur.close()
# do some reasonability checks before we update the queue
cur_year = int(datetime.datetime.now().year)
if prev_event_id == 0:
raise Exception(
'no previous event id found: {}'.format(prev_event_id))
if prev_event_date.year < (
cur_year - 10) or prev_event_date.year > (cur_year + 10):
raise Exception('previous event date unreasonable: {}'.format(
prev_event_date))
if max_event_id == 0:
raise Exception(
'no max event id found: {}'.format(max_event_id))
if max_event_dt.year < (cur_year -
10) or max_event_dt.year > (cur_year + 10):
raise Exception(
'max event date unreasonable: {}'.format(max_event_dt))
# reasonability test on the number of outstanding records
if MAX_CORPS < len(corps):
log_warning("find-unpocessed-events More than max corps: " +
str(len(corps)))
print("Update our queue")
event_processor.update_corp_event_queue(system_type, corps,
max_event_id,
max_event_date)
except Exception as e:
print("Exception", e)
log_error("find-unpocessed-events processing exception: " + str(e))
raise
mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')
try:
mara_db.config.databases \
= lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}
(initial_load_pipeline, success) = data_integration.pipelines.find_node(
['initialization_and_load_tasks', 'bc_reg_corp_loader'])
if success:
corps_ct = 1
prev_corps_ct = 0
while 0 < corps_ct and corps_ct != prev_corps_ct:
# run at least once to get an initial data load
run_pipeline(initial_load_pipeline)
with EventProcessor() as eventprocessor:
prev_corps_ct = corps_ct
corps_ct = eventprocessor.get_outstanding_corps_record_count()
log_info("Ran bc_reg_corp_loader for " + str(corps_ct) + " corps")
else:
print("Pipeline not found")
log_error("Pipeline not found for:" + "bc_reg_corp_loader")
except Exception as e:
print("Exception", e)
log_error("bc_reg_corp_loader processing exception: " + str(e))
raise
#!/usr/bin/python
import psycopg2
import datetime
import json
import decimal
import os
import logging
from bcreg.config import config
from bcreg.bcregistries import system_type
from bcreg.eventprocessor import EventProcessor, bn_credential
from bcreg.rocketchat_hooks import log_error, log_warning, log_info
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper()
logging.basicConfig(level=LOG_LEVEL)
try:
with EventProcessor() as event_processor:
event_processor.queue_reprocess_credential_type(system_type, bn_credential)
except Exception as e:
print("Exception", e)
log_error("find-un-bned-corps processing exception: " + str(e))
raise
@patch(data_integration.config.root_pipeline)
def root_pipeline():
return bc_reg_root_pipeline()
mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')
try:
log_info("Starting bc_reg_event_processor ...")
mara_db.config.databases \
= lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}
(child_pipeline, success) = data_integration.pipelines.find_node(
['bc_reg_event_processor'])
if success:
run_pipeline(child_pipeline)
log_info("Ran bc_reg_event_processor - complete.")
else:
print("Pipeline not found")
log_error("Pipeline not found for:" + "bc_reg_event_processor")
except Exception as e:
print("Exception", e)
log_error("bc_reg_event_processor processing exception: " + str(e))
raise
# get future-dated corps that haven't yet been processed
sql3 = """
select count(*) from event_by_corp_filing
where process_success is null and corp_num not in
(select corp_num from CORP_AUDIT_LOG);
"""
evp_future_corp_process_count = event_processor.get_sql_record_count(sql3)
evp_bc_reg_match_count = evp_corp_history_count + evp_future_corp_process_count
# check count of BC Reg corps vs event processor corps
if evp_bc_reg_match_count != bc_reg_count:
print("Error missing corps in Event Processor", bc_reg_count,
evp_bc_reg_match_count)
if ERROR_THRESHOLD_COUNT < abs(evp_bc_reg_match_count - bc_reg_count):
log_error(
"Error missing corps in Event Processor: BCReg={} EvP={}".format(
bc_reg_count, evp_bc_reg_match_count))
else:
log_warning(
"Warning missing corps in Event Processor: BCReg={} EvP={}".format(
bc_reg_count, evp_bc_reg_match_count))
# check counts of event processor input and output
if evp_credential_count != evp_corp_history_count:
print("Error missing credentials in Event Processor",
evp_corp_history_count, evp_credential_count)
if ERROR_THRESHOLD_COUNT < abs(evp_credential_count -
evp_corp_history_count):
log_error(
"Error missing credentials in Event Processor: EvP corps={} EvP creds={}"
.format(evp_corp_history_count, evp_credential_count))
if prev_event_date.year < (
cur_year - 10) or prev_event_date.year > (cur_year + 10):
raise Exception('previous event date unreasonable: {}'.format(
prev_event_date))
if max_event_id == 0:
raise Exception(
'no max event id found: {}'.format(max_event_id))
if max_event_dt.year < (cur_year -
10) or max_event_dt.year > (cur_year + 10):
raise Exception(
'max event date unreasonable: {}'.format(max_event_dt))
# reasonability test on the number of outstanding records
if CRAZY_MAX_CORPS < len(corps):
log_error(
"find-unpocessed-events More than cRaZy MaX corps: " +
str(len(corps)))
elif MAX_CORPS < len(corps):
log_warning("find-unpocessed-events More than max corps: " +
str(len(corps)))
LOGGER.info("Update our queue")
event_processor.update_corp_event_queue(system_type, corps,
max_event_id,
max_event_date)
# process any corps listed as "outstanding" in the audit table
# create events in our processing queue
LOGGER.info("Check audit report for outstanding/unprocessed corps")
audit_corps = event_processor.get_outstanding_audit_corps()
LOGGER.info("Unprocessed corps audit count is " +
mara_password = os.environ.get('MARA_DB_PASSWORD')
try:
log_info("Starting bc_reg_event_processor ...")
mara_db.config.databases \
= lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}
(child_pipeline, success) = data_integration.pipelines.find_node(
['bc_reg_event_processor'])
if success:
run_pipeline(child_pipeline)
log_info("Ran bc_reg_event_processor - complete.")
with EventProcessor() as eventprocessor:
corps_ct = eventprocessor.get_outstanding_corps_record_count()
if CRAZY_MAX_CORPS < corps_ct:
log_error(
"bc-reg-pipeline More than cRaZy MaX corps outstanding: " +
str(corps_ct))
elif MAX_CORPS < corps_ct:
log_warning(
"bc-reg-pipeline More than max corps outstanding: " +
str(corps_ct))
else:
print("Pipeline not found")
log_error("Pipeline not found for:" + "bc_reg_event_processor")
except Exception as e:
print("Exception", e)
log_error("bc_reg_event_processor processing exception: " + str(e))
raise
mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')
try:
mara_db.config.databases \
= lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}
(post_credential_pipeline, success) = data_integration.pipelines.find_node(
['initialization_and_load_tasks', 'bc_reg_credential_poster'])
if success:
creds_ct = 0
with EventProcessor() as eventprocessor:
creds_ct = eventprocessor.get_outstanding_creds_record_count()
while 0 < creds_ct:
run_pipeline(post_credential_pipeline)
with EventProcessor() as eventprocessor:
creds_ct = eventprocessor.get_outstanding_creds_record_count()
log_info("Ran bc_reg_credential_poster for " + str(creds_ct) +
" corps")
else:
print("Pipeline not found")
log_error("Pipeline not found for:" + "bc_reg_credential_poster")
except Exception as e:
print("Exception", e)
log_error("bc_reg_credential_poster processing exception: " + str(e))
raise
with EventProcessor() as event_processor:
sql1 = """
SELECT count(*) FROM CORP_AUDIT_LOG;
"""
sql2 = """
SELECT count(*) FROM CORP_AUDIT_LOG WHERE last_credential_id is not null;
"""
evp_corp_history_count = event_processor.get_sql_record_count(sql1)
evp_credential_count = event_processor.get_sql_record_count(sql2)
if evp_corp_history_count != bc_reg_count:
print("Error missing corps in Event Processor", bc_reg_count,
evp_corp_history_count)
if ERROR_THRESHOLD_COUNT < abs(evp_corp_history_count - bc_reg_count):
log_error(
"Error missing corps in Event Processor: reg={} evp={}".format(
bc_reg_count, evp_corp_history_count))
else:
log_warning(
"Warning missing corps in Event Processor: reg={} evp={}".format(
bc_reg_count, evp_corp_history_count))
if evp_credential_count != evp_corp_history_count:
print("Error missing credentials in Event Processor",
evp_corp_history_count, evp_credential_count)
if ERROR_THRESHOLD_COUNT < abs(evp_credential_count -
evp_corp_history_count):
log_error(
"Error missing credentials in Event Processor: evp corps={} evp creds={}"
.format(evp_corp_history_count, evp_credential_count))
else:
