if debug: msg = ('[mp_document_parser] PID %s finished parsing %s without any' ' exception') args = (pid, http_resp.get_url()) om.out.debug(msg % args) return document_parser @atexit.register def cleanup_pool(): if 'mp_doc_parser' in globals(): mp_doc_parser.stop_workers() def init_worker(log_queue): """ This function is called right after each Process in the ProcessPool is created, and it will initialized some variables/handlers which are required for it to work as expected :return: None """ signal.signal(signal.SIGINT, signal.SIG_IGN) log_sink_factory(log_queue) start_profiling_no_core() if is_main_process(): mp_doc_parser = MultiProcessingDocumentParser()
self._log_return_empty(http_response, 'Reached memory usage limit') return [] except ScanMustStopException, e: msg = 'The document parser is in an invalid state! %s' raise ScanMustStopException(msg % e) except Exception, e: # Act just like when there is no parser msg = 'Unhandled exception running get_tags_by_filter("%s"): %s' args = (http_response.get_url(), e) raise BaseFrameworkException(msg % args) else: if cache: self._cache[hash_string] = tags else: self._handle_no_cache(hash_string) finally: event.set() self._parser_finished_events.pop(hash_string, None) return tags @atexit.register def cleanup_pool(): if 'dpc' in globals(): dpc.clear() if is_main_process(): dpc = ParserCache()
def test_is_main_process(self): self.assertTrue(is_main_process())