def test_exception_safe_process():
with pytest.raises(RuntimeError):
breaking_process()
process = ExceptionSafeProcess(target=breaking_process)
process.start()
process.join()
assert process.exception
assert str(process.exception[0]) == 'now that\'s annoying'
def worker_processing_with_timeout(self, worker_id, next_task):
manager = Manager()
result = manager.list()
process = ExceptionSafeProcess(target=self.process_next_object, args=(next_task, result))
process.start()
process.join(timeout=self.timeout)
if self.timeout_happened(process):
self._handle_failed_analysis(next_task, process, worker_id, 'Timeout')
elif process.exception:
self._handle_failed_analysis(next_task, process, worker_id, 'Exception')
else:
self.out_queue.put(result.pop())
logging.debug('Worker {}: Finished {} analysis on {}'.format(worker_id, self.NAME, next_task.uid))
def worker_processing_with_timeout(self, worker_id, next_task):
manager = Manager()
result = manager.list()
process = ExceptionSafeProcess(target=self.process_next_object, args=(next_task, result))
process.start()
process.join(timeout=self.timeout)
if self.timeout_happened(process):
terminate_process_and_childs(process)
self.out_queue.put(next_task)
logging.warning('Worker {}: Timeout {} analysis on {}'.format(worker_id, self.NAME, next_task.uid))
elif process.exception:
terminate_process_and_childs(process)
raise process.exception[0]
else:
self.out_queue.put(result.pop())
logging.debug('Worker {}: Finished {} analysis on {}'.format(worker_id, self.NAME, next_task.uid))
class CompareScheduler:
'''
This module handles all request regarding compares
'''
def __init__(self,
config=None,
db_interface=None,
testing=False,
callback=None):
self.config = config
self.db_interface = db_interface if db_interface else CompareDbInterface(
config=config)
self.stop_condition = Value('i', 1)
self.in_queue = Queue()
self.callback = callback
self.compare_module = Compare(config=self.config,
db_interface=self.db_interface)
self.worker = ExceptionSafeProcess(target=self._compare_scheduler_main)
if not testing:
self.start()
def start(self):
self.stop_condition.value = 0
self.worker.start()
logging.info('Compare Scheduler online...')
def shutdown(self):
'''
shutdown the scheduler
'''
logging.debug('Shutting down...')
if getattr(self.db_interface, 'shutdown', False):
self.db_interface.shutdown()
if self.stop_condition.value == 0:
self.stop_condition.value = 1
self.worker.join()
self.in_queue.close()
logging.info('Compare Scheduler offline')
def add_task(self, compare_task):
compare_id, redo = compare_task
try:
self.db_interface.check_objects_exist(compare_id)
except FactCompareException as exception:
return exception.get_message(
) # FIXME: return value gets ignored by backend intercom
logging.debug('Schedule for compare: {}'.format(compare_id))
self.in_queue.put((compare_id, redo))
return None
def _compare_scheduler_main(self):
compares_done = set()
while self.stop_condition.value == 0:
self._compare_single_run(compares_done)
logging.debug('Compare Thread terminated')
def _compare_single_run(self, compares_done):
try:
compare_id, redo = self.in_queue.get(
timeout=float(self.config['ExpertSettings']['block_delay']))
except Empty:
pass
else:
if self._decide_whether_to_process(compare_id, redo,
compares_done):
if redo:
self.db_interface.delete_old_compare_result(compare_id)
compares_done.add(compare_id)
self._process_compare(compare_id)
if self.callback:
self.callback()
def _process_compare(self, compare_id):
result = self.compare_module.compare(
convert_compare_id_to_list(compare_id))
if isinstance(result, dict):
self.db_interface.add_compare_result(result)
else:
logging.error(result)
@staticmethod
def _decide_whether_to_process(uid, redo, compares_done):
return redo or uid not in compares_done
def check_exceptions(self):
return_value = False
if self.worker.exception:
logging.error("{}Worker Exception Found!!{}".format(
bcolors.FAIL, bcolors.ENDC))
logging.error(self.worker.exception[1])
if self.config.getboolean('ExpertSettings', 'throw_exceptions'):
return_value = True
terminate_process_and_childs(self.worker)
return return_value
class CompareScheduler:
'''
This module handles all request regarding compares
'''
def __init__(self,
config=None,
db_interface=None,
testing=False,
callback=None):
self.config = config
self.db_interface = db_interface if db_interface else CompareDbInterface(
config=config)
self.stop_condition = Value('i', 1)
self.in_queue = Queue()
self.callback = callback
self.compare_module = Compare(config=self.config,
db_interface=self.db_interface)
self.worker = ExceptionSafeProcess(target=self._compare_scheduler_main)
if not testing:
self.start()
def start(self):
self.stop_condition.value = 0
self.worker.start()
logging.info('Compare Scheduler online...')
def shutdown(self):
'''
shutdown the scheduler
'''
logging.debug('Shutting down...')
if getattr(self.db_interface, 'shutdown', False):
self.db_interface.shutdown()
if self.stop_condition.value == 0:
self.stop_condition.value = 1
self.worker.join()
self.in_queue.close()
logging.info('Compare Scheduler offline')
def add_task(self, compare_task):
compare_id, redo = compare_task
try:
self.db_interface.check_objects_exist(compare_id)
except FactCompareException as exception:
return exception.get_message(
) # FIXME: return value gets ignored by backend intercom
logging.debug(f'Schedule for compare: {compare_id}')
self.in_queue.put((compare_id, redo))
return None
def _compare_scheduler_main(self):
compares_done = set()
while self.stop_condition.value == 0:
self._compare_single_run(compares_done)
logging.debug('Compare Thread terminated')
def _compare_single_run(self, compares_done):
try:
compare_id, redo = self.in_queue.get(
timeout=float(self.config['ExpertSettings']['block_delay']))
except Empty:
pass
else:
if self._decide_whether_to_process(compare_id, redo,
compares_done):
if redo:
self.db_interface.delete_old_compare_result(compare_id)
compares_done.add(compare_id)
self._process_compare(compare_id)
if self.callback:
self.callback()
def _process_compare(self, compare_id):
try:
self.db_interface.add_compare_result(
self.compare_module.compare(
convert_compare_id_to_list(compare_id)))
except Exception: # pylint: disable=broad-except
logging.error(f'Fatal error in compare process for {compare_id}',
exc_info=True)
@staticmethod
def _decide_whether_to_process(uid, redo, compares_done):
return redo or uid not in compares_done
def check_exceptions(self):
processes_to_check = [self.worker]
shutdown = check_worker_exceptions(processes_to_check, 'Compare',
self.config,
self._compare_scheduler_main)
if not shutdown and new_worker_was_started(
new_process=processes_to_check[0], old_process=self.worker):
self.worker = processes_to_check.pop()
return shutdown
