def start_workers(cls, num=None):
if cls.workers:
return
try:
num = multiprocessing.cpu_count() if num is None else num
except NotImplementedError:
num = 4
cls.work_announce_address = get_announcer_address()
logger.info("Starting workers on address %s" % cls.work_announce_address)
if 'CP_DEBUG_WORKER' in os.environ:
if os.environ['CP_DEBUG_WORKER'] == 'NOT_INPROC':
return
from cellprofiler.analysis_worker import \
AnalysisWorker, NOTIFY_ADDR, NOTIFY_STOP, CancelledException
class WorkerRunner(threading.Thread):
def __init__(self, work_announce_address):
threading.Thread.__init__(self)
self.work_announce_address = work_announce_address
self.notify_socket = zmq.Context.instance().socket(zmq.PUB)
self.notify_socket.bind(NOTIFY_ADDR)
def run(self):
with AnalysisWorker(self.work_announce_address) as aw:
try:
aw.run()
except CancelledException:
logger.info("Exiting debug worker thread")
def wait(self):
self.notify_socket.send(NOTIFY_STOP)
self.join()
thread = WorkerRunner(cls.work_announce_address)
thread.setDaemon(True)
thread.start()
cls.workers.append(thread)
return
close_fds = False
# start workers
for idx in range(num):
if sys.platform == 'darwin':
close_all_on_exec()
# stdin for the subprocesses serves as a deadman's switch. When
# closed, the subprocess exits.
if hasattr(sys, 'frozen'):
if sys.platform == 'darwin':
# sys.argv[0] points at
# CellProfiler.app/Contents/Resources/CellProfiler.py
# We want
# CellProfiler.app/Contents/MacOS/CellProfiler
#
contents_resources_dir = os.path.split(sys.argv[0])[0]
contents_dir = os.path.split(contents_resources_dir)[0]
cp_executable = os.path.join(contents_dir, "MacOS", "CellProfiler")
assert os.path.isfile(cp_executable), \
"Did not find CellProfiler in its expected place: %s" % cp_executable
assert os.access(cp_executable, os.EX_OK), \
"%s is not executable" % cp_executable
args = ["arch", "-x86_64", cp_executable,
"--work-announce", cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()]
else:
aw_path = os.path.join(
os.path.split(
os.path.abspath(sys.argv[0]))[0],
"analysis_worker")
args = [aw_path,
'--work-announce',
cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()]
worker = subprocess.Popen(args,
env=find_worker_env(),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds = close_fds)
else:
worker = subprocess.Popen(
[find_python(),
'-u', # unbuffered
find_analysis_worker_source(),
'--work-announce',
cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()],
env=find_worker_env(),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds = close_fds)
def run_logger(workR, widx):
while(True):
try:
line = workR.stdout.readline()
if not line:
break
logger.info("Worker %d: %s", widx, line.rstrip())
except:
break
start_daemon_thread(target=run_logger, args=(worker, idx,), name='worker stdout logger')
cls.workers += [worker]
cls.deadman_switches += [worker.stdin] # closing stdin will kill subprocess
def start_workers(cls, num=None):
if cls.workers:
return
try:
num = multiprocessing.cpu_count() if num is None else num
except NotImplementedError:
num = 4
cls.work_announce_address = get_announcer_address()
logger.info("Starting workers on address %s" % cls.work_announce_address)
if 'CP_DEBUG_WORKER' in os.environ:
if os.environ['CP_DEBUG_WORKER'] == 'NOT_INPROC':
return
from cellprofiler.analysis_worker import \
AnalysisWorker, NOTIFY_ADDR, NOTIFY_STOP, CancelledException
class WorkerRunner(threading.Thread):
def __init__(self, work_announce_address):
threading.Thread.__init__(self)
self.work_announce_address = work_announce_address
self.notify_socket = zmq.Context.instance().socket(zmq.PUB)
self.notify_socket.bind(NOTIFY_ADDR)
def run(self):
with AnalysisWorker(self.work_announce_address) as aw:
try:
aw.run()
except CancelledException:
logger.info("Exiting debug worker thread")
def wait(self):
self.notify_socket.send(NOTIFY_STOP)
self.join()
thread = WorkerRunner(cls.work_announce_address)
thread.setDaemon(True)
thread.start()
cls.workers.append(thread)
return
close_fds = False
# start workers
for idx in range(num):
if sys.platform == 'darwin':
close_all_on_exec()
# stdin for the subprocesses serves as a deadman's switch. When
# closed, the subprocess exits.
if hasattr(sys, 'frozen'):
if sys.platform == 'darwin':
# sys.argv[0] points at
# CellProfiler.app/Contents/Resources/CellProfiler.py
# We want
# CellProfiler.app/Contents/MacOS/CellProfiler
#
contents_resources_dir = os.path.split(sys.argv[0])[0]
contents_dir = os.path.split(contents_resources_dir)[0]
cp_executable = os.path.join(contents_dir, "MacOS", "CellProfiler")
assert os.path.isfile(cp_executable), \
"Did not find CellProfiler in its expected place: %s" % cp_executable
assert os.access(cp_executable, os.EX_OK), \
"%s is not executable" % cp_executable
args = ["arch", "-x86_64", "-i386", cp_executable,
"--work-announce", cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()]
else:
aw_path = os.path.join(
os.path.split(
os.path.abspath(sys.argv[0]))[0],
"analysis_worker")
args = [aw_path,
'--work-announce',
cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()]
worker = subprocess.Popen(args,
env=find_worker_env(),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds = close_fds)
else:
worker = subprocess.Popen(
[find_python(),
'-u', # unbuffered
find_analysis_worker_source(),
'--work-announce',
cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()],
env=find_worker_env(),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds = close_fds)
def run_logger(workR, widx):
while(True):
try:
line = workR.stdout.readline()
if not line:
break
logger.info("Worker %d: %s", widx, line.rstrip())
except:
break
start_daemon_thread(target=run_logger, args=(worker, idx,), name='worker stdout logger')
cls.workers += [worker]
cls.deadman_switches += [worker.stdin] # closing stdin will kill subprocess
def start_workers(cls, num=None):
if cls.workers:
return
try:
num = multiprocessing.cpu_count() if num is None else num
except NotImplementedError:
num = 4
cls.work_announce_address = get_announcer_address()
logger.info("Starting workers on address %s" % cls.work_announce_address)
if 'CP_DEBUG_WORKER' in os.environ:
if os.environ['CP_DEBUG_WORKER'] == 'NOT_INPROC':
return
from cellprofiler.worker import \
AnalysisWorker, NOTIFY_ADDR, NOTIFY_STOP
from cellprofiler.pipeline import CancelledException
class WorkerRunner(threading.Thread):
def __init__(self, work_announce_address):
threading.Thread.__init__(self)
self.work_announce_address = work_announce_address
self.notify_socket = zmq.Context.instance().socket(zmq.PUB)
self.notify_socket.bind(NOTIFY_ADDR)
def run(self):
with AnalysisWorker(self.work_announce_address) as aw:
try:
aw.run()
except CancelledException:
logger.info("Exiting debug worker thread")
def wait(self):
self.notify_socket.send(NOTIFY_STOP)
self.join()
thread = WorkerRunner(cls.work_announce_address)
thread.setDaemon(True)
thread.start()
cls.workers.append(thread)
return
close_fds = False
# start workers
for idx in range(num):
if sys.platform == 'darwin':
close_all_on_exec()
aw_args = ["--work-announce", cls.work_announce_address,
"--plugins-directory", cpprefs.get_plugin_directory(),
"--ij-plugins-directory", cpprefs.get_ij_plugin_directory()]
jvm_arg = "%dm" % cpprefs.get_jvm_heap_mb()
aw_args.append("--jvm-heap-size=%s" % jvm_arg)
# stdin for the subprocesses serves as a deadman's switch. When
# closed, the subprocess exits.
if hasattr(sys, 'frozen'):
if sys.platform == 'darwin':
executable = os.path.join(
os.path.dirname(sys.executable), "CellProfiler")
args = ([executable] + aw_args)
elif sys.platform.startswith('linux'):
aw_path = os.path.join(os.path.dirname(__file__),
"worker.py")
args = [sys.executable, aw_path] + aw_args
else:
args = [sys.executable] + aw_args
worker = subprocess.Popen(args,
env=find_worker_env(idx),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=close_fds)
else:
worker = subprocess.Popen(
[find_python(),
'-u', # unbuffered
find_analysis_worker_source()] + aw_args,
env=find_worker_env(idx),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=close_fds)
def run_logger(workR, widx):
while True:
try:
line = workR.stdout.readline()
if not line:
break
logger.info("Worker %d: %s", widx, line.rstrip())
except:
break
start_daemon_thread(target=run_logger, args=(worker, idx,), name='worker stdout logger')
cls.workers += [worker]
cls.deadman_switches += [worker.stdin] # closing stdin will kill subprocess
