def get_valid_fi_ids(context):
conn = get_db_connection(context, database="map_config")
valid_fi_ids = data.consumer.get_valid_fi_ids(conn)
conn.close()
return valid_fi_ids
def producer_func(context, queue, fi_id, db_func):
connection = get_db_connection(context, database="fi_{}".format(fi_id))
if not connection.open:
logger.info("Connection to DB failed.")
return
gen = db_func(connection)
batch_consumers(fi_id, gen, queue)
gen = None
logger.info("Queued all batches for FI {} from thread {}".format(fi_id, threading.current_thread().getName()))
connection.close()
def send():
context = get_context()
client = MarketingCloudClient()
client.set_access_token(context)
valid_fi_ids = get_valid_fi_ids(context)
if not len(valid_fi_ids):
logger.info("No valid FIs to send to Marketing Cloud.")
return
cp = ProducerConsumer(context, client, valid_fi_ids)
cp.add_producer("customer_producer", customer_producer_func)
cp.add_producer("acxiom_prospect_producer", acxiom_prospect_producer_func)
cp.add_producer("fico_prospect_producer", fico_prospect_producer_func)
cp.set_consumer_func(consumer_func)
cp.set_num_consumers(5)
logger.info("Starting sending process with batch size of {}...".format(
data.consumer.BATCH_SIZE))
cp.run()
logger.info("Done sending data to Marketing Cloud API.")
# updating the task_last_run table for the FI to signal that there were no batches with errors
if client.successful_fis:
for fi_id in client.successful_fis:
connection = get_db_connection(context,
database="fi_{}".format(fi_id))
if not connection.open:
logger.info("Connection to DB failed.")
return
data.consumer.mark_run_as_successful(connection)
connection.close()
logger.info(
"Updated task_last_run for successful send for FI {}.".format(
fi_id))