def pool_worker_main(item: WorkItemInput, output: multiprocessing.queues.Queue) -> None:
try:
# TODO figure out a more reliable way to suppress this. Redirect output?
# Ignore ctrl-c in workers to reduce noisy tracebacks (the parent will kill us):
signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(os, 'nice'): # analysis should run at a low priority
os.nice(10)
set_debug(False)
filename, options, deadline = item
stats: Counter[str] = Counter()
options.stats = stats
_, module_name = extract_module_from_file(filename)
try:
module = load_by_qualname(module_name)
except NotFound:
return
except ErrorDuringImport as e:
orig, frame = e.args
message = AnalysisMessage(MessageType.IMPORT_ERR, str(orig), frame.filename, frame.lineno, 0, '')
output.put((filename, stats, [message]))
debug(f'Not analyzing "{filename}" because import failed: {e}')
return
messages = analyze_any(module, options)
output.put((filename, stats, messages))
except BaseException as e:
raise CrosshairInternal(
'Worker failed while analyzing ' + filename) from e
def _worker(self, q: mp.queues.Queue, err: mp.queues.Queue):
""" The worker process target for handling requests. """
while not self._stop_event.is_set():
try:
df_key, task_key, result_queue, args, kwargs = q.get()
if df_key not in self.__df_proxies__.keys():
raise ValueError(f"DataFrameProxy with key of '{df_key}' does not exist.")
if task_key not in self.__tasks__.keys():
raise ValueError(f"Task function with qualname of '{task_key}' "
"does not exist in task map.")
result_queue.put(self.__tasks__[task_key](self.__df_proxies__[df_key], *args,
**kwargs))
except (KeyboardInterrupt, SystemExit):
# keybaord interput or system exit, just stop and return
self._stop_event.set()
break
except Exception as e:
# other exception, put the exception in the error queue and set system stop flag
# self.__error_queue__.put(e)
self._stop_event.set()
self.__error_queue__.put(e)
self._logger.error("Worker got error: %s", e)
def pool_worker_main(item: WorkItemInput,
output: multiprocessing.queues.Queue) -> None:
try:
# TODO figure out a more reliable way to suppress this. Redirect output?
# Ignore ctrl-c in workers to reduce noisy tracebacks (the parent will kill us):
signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(os, 'nice'): # analysis should run at a low priority
os.nice(10)
set_debug(False)
(stats, messages) = pool_worker_process_item(item)
filename = item[0]
output.put((filename, stats, messages))
except BaseException as e:
raise CrosshairInternal('Worker failed while analyzing ' +
filename) from e
def pool_worker_main(item: WorkItemInput,
output: multiprocessing.queues.Queue) -> None:
try:
# TODO figure out a more reliable way to suppress this. Redirect output?
# Ignore ctrl-c in workers to reduce noisy tracebacks (the parent will kill us):
signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(os,
'nice'): # <- is this the right way to detect availability?
os.nice(10) # analysis should run at a low priority
set_debug(False)
member, options, deadline = item
stats: Counter[str] = Counter()
options.stats = stats
try:
fn = member.get_member()
except NotFound:
return
messages = analyze_any(fn, options)
output.put((member, stats, messages))
except BaseException as e:
raise CrosshairInternal('Worker failed while analyzing ' +
member.qual_name) from e