def test_execute_job_failure_releases_lock(self):
"""After a failure, the worker should be able to accept another job.
"""
n_jobs_a = 1
jobs_a, calls_a = TestWorker.new_jobs(n_jobs_a)
task_type_a = FakeTaskType([Exception()])
cms.service.Worker.get_task_type = Mock(return_value=task_type_a)
with self.assertRaises(JobException):
job_group = JobGroup([jobs_a[0]])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
cms.service.Worker.get_task_type.assert_has_calls(calls_a)
self.assertEquals(task_type_a.call_count, n_jobs_a)
n_jobs_b = 3
jobs_b, calls_b = TestWorker.new_jobs(n_jobs_b)
task_type_b = FakeTaskType([True] * n_jobs_b)
cms.service.Worker.get_task_type = Mock(return_value=task_type_b)
for job in jobs_b:
job_group = JobGroup([job])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
cms.service.Worker.get_task_type.assert_has_calls(calls_b)
self.assertEquals(task_type_b.call_count, n_jobs_b)
def test_execute_job_subsequent_success(self):
"""Executes three successful jobs, then four others.
"""
n_jobs_a = 3
jobs_a, calls_a = TestWorker.new_jobs(n_jobs_a, prefix="a")
task_type_a = FakeTaskType([True] * n_jobs_a)
cms.service.Worker.get_task_type = Mock(return_value=task_type_a)
for job in jobs_a:
job_group = JobGroup([job])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
cms.service.Worker.get_task_type.assert_has_calls(calls_a)
self.assertEquals(task_type_a.call_count, n_jobs_a)
n_jobs_b = 4
jobs_b, calls_b = TestWorker.new_jobs(n_jobs_b, prefix="b")
task_type_b = FakeTaskType([True] * n_jobs_b)
cms.service.Worker.get_task_type = Mock(return_value=task_type_b)
for job in jobs_b:
job_group = JobGroup([job])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
cms.service.Worker.get_task_type.assert_has_calls(calls_b)
self.assertEquals(task_type_b.call_count, n_jobs_b)
def test_execute_job_subsequent_locked(self):
"""Executes a long job, then another one that should fail
because of the lock.
"""
# Because of how gevent works, the interval here can be very small.
task_type = FakeTaskType([0.01])
cms.service.Worker.get_task_type = Mock(return_value=task_type)
jobs_a, calls_a = TestWorker.new_jobs(1, prefix="a")
jobs_b, calls_b = TestWorker.new_jobs(1, prefix="b")
def first_call():
job_group = JobGroup([jobs_a[0]])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
first_greenlet = gevent.spawn(first_call)
gevent.sleep(0) # To ensure we call jobgroup_a first.
with self.assertRaises(JobException):
job_group = JobGroup([jobs_b[0]])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
first_greenlet.get()
self.assertNotIn(calls_b[0],
cms.service.Worker.get_task_type.mock_calls)
cms.service.Worker.get_task_type.assert_has_calls(calls_a)
def acquire_worker(self, operations):
"""Tries to assign an operation to an available worker. If no workers
are available then this returns None, otherwise this returns
the chosen worker.
operations ([ESOperation]): the operations to assign to a worker.
return (int|None): None if no workers are available, the worker
assigned to the operation otherwise.
"""
# We look for an available worker.
try:
shard = self.find_worker(WorkerPool.WORKER_INACTIVE,
require_connection=True,
random_worker=True)
except LookupError:
self._workers_available_event.clear()
return None
# Then we fill the info for future memory.
self._add_operations(shard, operations)
logger.debug("Worker %s acquired.", shard)
self._start_time[shard] = make_datetime()
with SessionGen() as session:
jobs = []
datasets = {}
submissions = {}
user_tests = {}
for operation in operations:
if operation.dataset_id not in datasets:
datasets[operation.dataset_id] = Dataset.get_from_id(
operation.dataset_id, session)
object_ = None
if operation.for_submission():
if operation.object_id not in submissions:
submissions[operation.object_id] = \
Submission.get_from_id(
operation.object_id, session)
object_ = submissions[operation.object_id]
else:
if operation.object_id not in user_tests:
user_tests[operation.object_id] = \
UserTest.get_from_id(operation.object_id, session)
object_ = user_tests[operation.object_id]
logger.info("Asking worker %s to `%s'.", shard, operation)
jobs.append(
Job.from_operation(operation, object_,
datasets[operation.dataset_id]))
job_group_dict = JobGroup(jobs).export_to_dict()
self._worker[shard].execute_job_group(
job_group_dict=job_group_dict,
callback=self._service.action_finished,
plus=shard)
return shard
def new_jobgroup(number_of_jobs, prefix=None):
prefix = prefix if prefix is not None else ""
jobgroup_dict = {}
calls = []
for i in xrange(number_of_jobs):
job_params = ("fake_task_type", "fake_parameters_%s" % i)
job = EvaluationJob(*job_params, info="%s%d" % (prefix, i))
jobgroup_dict["%s" % i] = job
calls.append(call(*job_params))
return JobGroup(jobgroup_dict), calls
def new_job_groups(spec, prefix=None):
"""Return len(spec) job groups each with spec[i] jobs."""
prefix = prefix if prefix is not None else ""
job_groups = []
calls = []
for i, number_of_jobs in enumerate(spec):
jobs, this_calls = TestWorker.new_jobs(
number_of_jobs, str(i) + prefix)
job_groups.append(JobGroup(jobs))
calls += this_calls
return job_groups, calls
def test_execute_job_tasktype_raise(self):
"""Executes two jobs raising exceptions.
"""
n_jobs = 2
jobs, unused_calls = TestWorker.new_jobs(n_jobs)
task_type = FakeTaskType([Exception(), Exception()])
cms.service.Worker.get_task_type = Mock(return_value=task_type)
for job in jobs:
with self.assertRaises(JobException):
job_group = JobGroup([job])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
self.assertEquals(cms.service.Worker.get_task_type.call_count, n_jobs)
self.assertEquals(task_type.call_count, n_jobs)
def test_execute_job_success(self):
"""Executes three successful jobs.
"""
n_jobs = 3
jobs, calls = TestWorker.new_jobs(n_jobs)
task_type = FakeTaskType([True] * n_jobs)
cms.service.Worker.get_task_type = Mock(return_value=task_type)
for job in jobs:
job_group = JobGroup([job])
ret_job_group = JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
self.assertTrue(ret_job_group.jobs[0].success)
cms.service.Worker.get_task_type.assert_has_calls(calls)
self.assertEquals(task_type.call_count, n_jobs)
def test_execute_job_failure(self):
"""Executes two unsuccessful jobs.
"""
n_jobs = 2
jobs, unused_calls = TestWorker.new_jobs(n_jobs)
task_type = FakeTaskType([False] * n_jobs)
cms.service.Worker.get_task_type = Mock(return_value=task_type)
results = []
for job in jobs:
job_group = JobGroup([job])
results.append(JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict())))
for job_group in results:
for job in job_group.jobs:
self.assertFalse(job.success)
self.assertEquals(cms.service.Worker.get_task_type.call_count, n_jobs)
self.assertEquals(task_type.call_count, n_jobs)
def first_call():
job_group = JobGroup([jobs_a[0]])
JobGroup.import_from_dict(
self.service.execute_job_group(job_group.export_to_dict()))
