def _start(self, properties):
kwargs = dict(
args=' '.join([self.bin] + self.arguments),
stdout=subprocess.PIPE,
bufsize=1,
stderr=subprocess.STDOUT
)
self.cmd = kwargs['args']
with subprocess.Popen(**kwargs, shell=True) as proc:
try:
properties['running'] = True
properties['pid'] = proc.pid
while properties['running']:
if proc.poll() is None:
line = proc.stdout.readline().decode('utf-8')
if line:
self.parse(properties, line)
else:
properties['running'] = False
properties['exit'] = proc.returncode
except Exception:
error(traceback.format_exc())
def run(self):
info('starting worker')
self.running = True
self.client.name = f'worker-{self.work_id}'
self.client.push(RESULT_QUEUE, message={'worker': '1'}, mtype=WORKER_JOIN)
last_message = None
with self.client:
while self.running:
try:
workitem = self.client.pop(WORK_QUEUE)
if workitem is None:
time.sleep(0.01)
elif workitem.mtype == START_BROKER:
info('starting new message broker')
msg = workitem.message
# self.broker = make_message_broker(**msg.get('kwargs'))
# self.broker.start()
elif workitem.mtype == WORK_ITEM:
self.execute(workitem)
elif workitem.mtype == SHUTDOWN:
info(f'shutting down worker')
self.running = False
last_message = workitem
# Shutdown worker loop and start HPO that has it's own loop
elif workitem.mtype == START_HPO:
info('starting HPO service')
self.running = False
last_message = workitem
else:
error(f'Unrecognized (message: {workitem})')
except Exception:
error(traceback.format_exc())
# --
self.client.push(RESULT_QUEUE, message={'worker': '0'}, mtype=WORKER_LEFT)
if last_message:
self.client.mark_actioned(WORK_QUEUE, last_message)
if last_message.mtype == START_HPO:
info('HPO')
msg = last_message.message
hpo = WorkScheduler(**msg.get('kwargs'))
hpo.run()
if self.broker:
self.broker.stop()
def start(self, wait=True):
try:
self._process = Process(target=self._start, args=(self.properties,))
self._process.start()
# wait for all the properties to be populated
if wait:
while self.properties.get('nodeID') is None and self._process.is_alive():
time.sleep(0.01)
self.properties['db_pid'] = int(open(f'{self.location}/cockroach_pid', 'r').read())
self._setup()
except Exception as e:
error(traceback.format_exc(e))
def start(self, wait=True):
try:
self._process = Process(target=self._start,
args=(self.properties, ))
self._process.start()
# wait for all the properties to be populated
if wait:
while self.properties.get('ready') is None:
time.sleep(0.01)
self.properties['db_pid'] = int(open(self.pid_file, 'r').read())
self._setup()
except Exception as e:
error(traceback.format_exc(e))
def execute(self, message: Message):
msg = message.message
script = msg.get('script')
args = msg.get('args', list())
env = msg.get('env', dict())
env['MQ_CLIENT'] = self.uri
if script is None:
error(f'work item without script!')
else:
info(f'starting work process (cmd: {script} {" ".join(args)})')
command = [script] + args
process = subprocess.Popen(command, env=env)
return_code = process.wait()
info(f'finished work item (rc: {return_code})')
self.client.mark_actioned(WORK_QUEUE, uid=message.uid)
def _observe(self, results, r):
if r is None:
return results
elif r.mtype == RESULT_MESSAGE:
info('found result')
results.append(r)
elif r.mtype == WORKER_JOIN:
info('new worker')
self.worker_count += 1
elif r.mtype == WORKER_LEFT:
info('worker died')
self.worker_count -= 1
else:
error(f'Message not recognized (message: {r.uid})')
info(f'actioned (message: {r.uid})')
self.client.mark_actioned(RESULT_QUEUE, r)