def collect_result(self, session):
"""Collect result from processed workers."""
try:
workers, submissions = zip(*[
self._processing_worker_queue.get()
for _ in range(self._processing_worker_queue.qsize())
])
except ValueError:
logger.info('No workers are currently waiting or processed.')
if self.hunger_policy == 'sleep':
time.sleep(5)
elif self.hunger_policy == 'exit':
self._poison_pill = True
return
for worker, (submission_id,
submission_name) in zip(workers, submissions):
if worker.status == 'running':
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
logger.info('Worker {} is still running'.format(worker))
time.sleep(0)
else:
logger.info('Collecting results from worker {}'.format(worker))
returncode, stderr = worker.collect_results()
set_submission_state(
session, submission_id,
'tested' if not returncode else 'training_error')
set_submission_error_msg(session, submission_id, stderr)
self._processed_submission_queue.put_nowait(
(submission_id, submission_name))
worker.teardown()
def test_check_submission_error_msg(session_scope_module):
# check both get_submission_error_msg and set_submission_error_msg
submission_id = 1
expected_err_msg = 'tag submission as failed'
set_submission_error_msg(session_scope_module, submission_id,
expected_err_msg)
err_msg = get_submission_error_msg(session_scope_module, submission_id)
assert err_msg == expected_err_msg
def collect_result(self, session):
"""Collect result from processed workers."""
try:
workers, submissions = zip(*[
self._processing_worker_queue.get()
for _ in range(self._processing_worker_queue.qsize())
])
except ValueError:
if self.hunger_policy == 'sleep':
time.sleep(5)
elif self.hunger_policy == 'exit':
self._poison_pill = True
return
for worker, (submission_id,
submission_name) in zip(workers, submissions):
dt = worker.time_since_last_status_check()
if (dt is not None) and (dt < self.time_between_collection):
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
time.sleep(0)
continue
elif worker.status == 'running':
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
time.sleep(0)
elif worker.status == 'retry':
set_submission_state(session, submission_id, 'new')
self._logger.info(
f'Submission: {submission_id} has been interrupted. '
'It will be added to queue again and retried.')
worker.teardown()
else:
self._logger.info(f'Collecting results from worker {worker}')
returncode, stderr = worker.collect_results()
if returncode:
if returncode == 124:
self._logger.info(
f'Worker {worker} killed due to timeout.')
submission_status = 'training_error'
elif returncode == 2:
# Error occurred when downloading the logs
submission_status = 'checking_error'
else:
self._logger.info(
f'Worker {worker} killed due to an error '
f'during training: {stderr}')
submission_status = 'training_error'
else:
submission_status = 'tested'
set_submission_state(session, submission_id, submission_status)
set_submission_error_msg(session, submission_id, stderr)
self._processed_submission_queue.put_nowait(
(submission_id, submission_name))
worker.teardown()
def collect_result(self, session):
"""Collect result from processed workers."""
try:
workers, submissions = zip(*[
self._processing_worker_queue.get()
for _ in range(self._processing_worker_queue.qsize())
])
except ValueError:
if self.hunger_policy == 'sleep':
time.sleep(5)
elif self.hunger_policy == 'exit':
self._poison_pill = True
return
for worker, (submission_id,
submission_name) in zip(workers, submissions):
dt = worker.time_since_last_status_check()
if dt is not None and dt < self.time_between_collection:
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
time.sleep(0)
continue
elif worker.status == 'running':
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
time.sleep(0)
else:
logger.info(f'Collecting results from worker {worker}')
returncode, stderr = worker.collect_results()
if returncode:
if returncode == 124:
logger.info(
'Worker {} killed due to timeout.'.format(worker))
else:
logger.info(f'Worker {worker} killed due to an error '
'during training')
submission_status = 'training_error'
else:
submission_status = 'tested'
set_submission_state(session, submission_id, submission_status)
set_submission_error_msg(session, submission_id, stderr)
self._processed_submission_queue.put_nowait(
(submission_id, submission_name))
worker.teardown()
def launch_workers(self, session):
"""Launch the awaiting workers if possible."""
while (not self._processing_worker_queue.full()
and not self._awaiting_worker_queue.empty()):
worker, (submission_id, submission_name) = \
self._awaiting_worker_queue.get()
self._logger.info(f'Starting worker: {worker}')
try:
worker.setup()
if worker.status != "error":
worker.launch_submission()
except Exception as e:
self._logger.error(
f'Worker finished with unhandled exception:\n {e}')
worker.status = 'error'
if worker.status == 'error':
set_submission_state(session, submission_id, 'checking_error')
worker.teardown() # kill the worker
self._logger.info(f'Worker {worker} killed due to an error '
f'while connecting to AWS worker')
stderr = ("There was a problem with sending your submission"
" for training. This problem is on RAMP side"
" and most likely it is not related to your"
" code. If this happened for the first time"
" to this submission you might"
" consider submitting the same code once again."
" Else, please contact the event organizers.")
set_submission_error_msg(session, submission_id, stderr)
continue
set_submission_state(session, submission_id, 'training')
submission = get_submission_by_id(session, submission_id)
update_user_leaderboards(
session,
self._ramp_config['event_name'],
submission.team.name,
new_only=True,
)
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
self._logger.info(
f'Store the worker {worker} into the processing queue')