def run(self):
"""
`run` spawns a thread per producer that is registered and threads matching the max_consumers config value.
The producer threads are spawned and passed the configuring args/kwargs during thread registration.
The consumer threads are spawned and passed no args/kwargs. These are picked up from the self.consumers
dictionary by the self._consumer method as it can't be predicted what work will need to be done.
`run` will wait until all threads have completed their work before finally closing out by logging the number
of unhandled errors that were raised.
:return:
"""
for name in self._producers.keys():
self.start_producer(name)
for num in range(0, self.max_consumers):
thread = threading.Thread(name=f"consumer_{num}",
target=self._consumer)
self.threads.append(thread)
thread.start()
for t in self.threads:
t.join()
if not self._work_queue.empty:
logger.error(
f"Work Queue was not empty when it should be for producer list {self._producers.keys}"
)
logger.info(
f"completed work, {len(self.thread_exception_list)} exceptions were encountered."
)
def _reset_work(self, key: str) -> bool:
path = Path(self.context.PATH, f'{key}.lock')
if not path.exists():
logger.error(
f"tried to reset a work item that does not exist: {key}")
return False
else:
path.rename(f'{self.context.PATH}/{key}')
return True
def _update_map(item, payload_map):
fi_id = item.get('AccountNumber', None)
if fi_id is None:
logger.error("one of the results had no AccountNumber value")
return payload_map
if payload_map.get(fi_id, None) is not None:
payload_map[fi_id]['product_data'].append(item)
else:
payload_map[fi_id] = dict()
payload_map[fi_id]['product_data'] = [item]
def register_producer_for_work(self, work: str):
"""Registers the producer that is relevant to the work being requested"""
producer = self.producers.get(work, None)
if producer is None:
logger.error(
f"Tried to register a producer that does not exist; {work}")
self.producer_consumer.register_producer(
name=work,
producer_func=producer['instance'].producer_func,
producer_args=producer.get('args', ()),
producer_kwargs=producer.get('kwargs', dict()))
def fetch_product_data(self, context):
"""
fetch_product_data fetches from Marketing Cloud, the previous days updates. In the event that more records exist
that were not returned in the first call, calling this method again will return the next results until there
are no more results to fetch.
:param context: MarketingCloudClientContext object
:return (list, bool): results from the call and the value of HasMoreRows if it exists
"""
retries = 0
success = False
content = dict()
while retries <= RETRY_LIMIT:
response = self._fetch_product_data(context)
if response.status_code == 200:
if hasattr(response, 'content'):
try:
content = response.json()
except json.decoder.JSONDecodeError:
# The main reason this will happen is an invalid access token and the endpoint
# returning nothing parsable in the content.
# This could also be because an error was returned as the content. But we cannot log the content
# because of potentially sensitive information being included.
# wait if another thread is setting the access token already
if self.access_token_lock.locked():
while self.access_token_lock.locked():
pass
else:
self.set_access_token(context)
retries += 1
continue
else:
success = True
break
else:
logger.error(response.text)
logger.info("Failed to send payload to {url}".format(url=context.GET_PRODUCT_DATA_URL))
break
if not success:
logger.error("Error fetching product data from marketing cloud")
self.request_id = content.get("RequestID", None)
has_more_rows = content.get('HasMoreRows', False)
if not has_more_rows:
self.request_id = None
return content.get('Results', []), has_more_rows
def _consumer(self):
"""
_consumer reads the work queue and spawns work based on the job_type argument received from the queue. The
self._consumers keys are matched against job type to determine which consumer function should be used
for the consumption of the attached payload.
If the consumer function raises an unhandled exception, _consumer will log it and move on to the next item
in the queue. This happens until the queue is drained and the event_manager has been notified by all producers
that there is no more work to be processed.
:return:
"""
while not self._work_queue.empty or not self.event_manager.is_set():
try:
job = self._work_queue.get()
func = self._consumers.get(job["job_type"])
func["func"](job["payload"], *func['args'], **func['kwargs'])
self._work_queue.task_done()
except queue.Empty:
# This handles a race condition between threads where the queue became empty after entering
# the current loop iteration.
if self.event_manager.is_set():
# if nothing in queue and producer signalled we are done,
# then exit out of loop to stop thread
break
sleep(1)
except Exception as e:
name = threading.current_thread().getName()
logger.error(
f"Thread {name} encountered an unhandled exception on consumer function: {e}",
exc_info=True)
with self._exception_list_lock:
self.thread_exception_list.append(dict(name=name, err=e))
self._work_queue.task_done()
continue # move on with processing the queue
logger.info(f"Finished {threading.current_thread().getName()}")
def _producer(self, producer_func, producer_args, producer_kwargs):
"""
_producer takes a function that receives a queue object as the first argument and any number of args or kwargs.
The provided function populates the queue with work that will be consumed by the consumer_func asynchronously.
_producer will wait until the producer_func returns and them close out the thread and notify the event manager
of completion of the work. If the producer function fails to handle an exception, the thread will be killed and
no further work will be processed by the producer.
"""
try:
# completes when the source the function is pulling from is empty
producer_func(self._work_queue, *producer_args, **producer_kwargs)
except Exception as e:
name = threading.current_thread().getName()
logger.error(
f"Thread {name} encountered an unhandled exception in producer function: {e}",
exc_info=True)
with self._exception_list_lock:
self.thread_exception_list.append(dict(name=name, err=e))
# notify consuming threads we are done
logger.info(f"Finished {threading.current_thread().getName()}")
self.event_manager.set(threading.current_thread().getName())