def get_valid_fi_ids(context): conn = get_db_connection(context, database="map_config") valid_fi_ids = data.consumer.get_valid_fi_ids(conn) conn.close() return valid_fi_ids
def producer_func(context, queue, fi_id, db_func): connection = get_db_connection(context, database="fi_{}".format(fi_id)) if not connection.open: logger.info("Connection to DB failed.") return gen = db_func(connection) batch_consumers(fi_id, gen, queue) gen = None logger.info("Queued all batches for FI {} from thread {}".format(fi_id, threading.current_thread().getName())) connection.close()
def send(): context = get_context() client = MarketingCloudClient() client.set_access_token(context) valid_fi_ids = get_valid_fi_ids(context) if not len(valid_fi_ids): logger.info("No valid FIs to send to Marketing Cloud.") return cp = ProducerConsumer(context, client, valid_fi_ids) cp.add_producer("customer_producer", customer_producer_func) cp.add_producer("acxiom_prospect_producer", acxiom_prospect_producer_func) cp.add_producer("fico_prospect_producer", fico_prospect_producer_func) cp.set_consumer_func(consumer_func) cp.set_num_consumers(5) logger.info("Starting sending process with batch size of {}...".format( data.consumer.BATCH_SIZE)) cp.run() logger.info("Done sending data to Marketing Cloud API.") # updating the task_last_run table for the FI to signal that there were no batches with errors if client.successful_fis: for fi_id in client.successful_fis: connection = get_db_connection(context, database="fi_{}".format(fi_id)) if not connection.open: logger.info("Connection to DB failed.") return data.consumer.mark_run_as_successful(connection) connection.close() logger.info( "Updated task_last_run for successful send for FI {}.".format( fi_id))