if debug:
msg = ('[mp_document_parser] PID %s finished parsing %s without any'
' exception')
args = (pid, http_resp.get_url())
om.out.debug(msg % args)
return document_parser
@atexit.register
def cleanup_pool():
if 'mp_doc_parser' in globals():
mp_doc_parser.stop_workers()
def init_worker(log_queue):
"""
This function is called right after each Process in the ProcessPool is
created, and it will initialized some variables/handlers which are required
for it to work as expected
:return: None
"""
signal.signal(signal.SIGINT, signal.SIG_IGN)
log_sink_factory(log_queue)
start_profiling_no_core()
if is_main_process():
mp_doc_parser = MultiProcessingDocumentParser()
self._log_return_empty(http_response, 'Reached memory usage limit')
return []
except ScanMustStopException, e:
msg = 'The document parser is in an invalid state! %s'
raise ScanMustStopException(msg % e)
except Exception, e:
# Act just like when there is no parser
msg = 'Unhandled exception running get_tags_by_filter("%s"): %s'
args = (http_response.get_url(), e)
raise BaseFrameworkException(msg % args)
else:
if cache:
self._cache[hash_string] = tags
else:
self._handle_no_cache(hash_string)
finally:
event.set()
self._parser_finished_events.pop(hash_string, None)
return tags
@atexit.register
def cleanup_pool():
if 'dpc' in globals():
dpc.clear()
if is_main_process():
dpc = ParserCache()
def test_is_main_process(self):
self.assertTrue(is_main_process())
