def execute_job_group(self, job_group_dict):
"""Receive a group of jobs in a dict format and executes them
one by one.
job_group_dict (dict): a dictionary suitable to be imported
from JobGroup.
"""
job_group = JobGroup.import_from_dict(job_group_dict)
if self.work_lock.acquire(False):
try:
self._ignore_job = False
for k, job in job_group.jobs.iteritems():
logger.info("Starting job.", extra={"operation": job.info})
job.shard = self.shard
# FIXME This is actually kind of a workaround...
# The only TaskType that needs it is OutputOnly.
job._key = k
# FIXME We're creating a new TaskType for each Job
# even if, at the moment, a JobGroup always uses
# the same TaskType and the same parameters. Yet,
# this could change in the future, so the best
# solution is to keep a cache of TaskTypes objects
# (like ScoringService does with ScoreTypes, except
# that we cannot index by Dataset ID here...).
task_type = get_task_type(job.task_type,
job.task_type_parameters)
task_type.execute_job(job, self.file_cacher)
logger.info("Finished job.", extra={"operation": job.info})
if not job.success or self._ignore_job:
job_group.success = False
break
else:
job_group.success = True
return job_group.export_to_dict()
except:
err_msg = "Worker failed."
logger.error(err_msg, exc_info=True)
raise JobException(err_msg)
finally:
self.work_lock.release()
else:
err_msg = "Request received, but declined because of acquired " \
"lock (Worker is busy executing another job group, this " \
"should not happen: check if there are more than one ES " \
"running, or for bugs in ES."
logger.warning(err_msg)
raise JobException(err_msg)
def execute_job(self, job_dict):
job = Job.import_from_dict_with_type(job_dict)
if self.work_lock.acquire(False):
try:
logger.operation = "job '%s'" % (job.info)
logger.info("Request received")
job.shard = self.shard
self.task_type = get_task_type(job, self.file_cacher)
self.task_type.execute_job()
logger.info("Request finished.")
return job.export_to_dict()
except:
err_msg = "Worker failed on operation `%s'" % logger.operation
logger.error("%s\n%s" % (err_msg, traceback.format_exc()))
raise JobException(err_msg)
finally:
self.task_type = None
self.session = None
logger.operation = ""
self.work_lock.release()
else:
err_msg = "Request '%s' received, " \
"but declined because of acquired lock" % \
(job.info)
logger.warning(err_msg)
raise JobException(err_msg)
def execute_job_group(self, job_group_dict):
"""Receive a group of jobs in a list format and executes them one by
one.
job_group_dict ({}): a JobGroup exported to dict.
return ({}): the same JobGroup in dict format, but containing
the results.
"""
start_time = time.time()
job_group = JobGroup.import_from_dict(job_group_dict)
if self.work_lock.acquire(False):
try:
logger.info("Starting job group.")
for job in job_group.jobs:
logger.info("Starting job.",
extra={"operation": job.info})
job.shard = self.shard
if self._fake_worker_time is None:
task_type = get_task_type(job.task_type,
job.task_type_parameters)
try:
task_type.execute_job(job, self.file_cacher)
except TombstoneError:
job.success = False
job.plus = {"tombstone": True}
else:
self._fake_work(job)
logger.info("Finished job.",
extra={"operation": job.info})
logger.info("Finished job group.")
return job_group.export_to_dict()
except Exception as e:
err_msg = "Worker failed: %s." % e
logger.error(err_msg, exc_info=True)
raise JobException(err_msg)
finally:
self._finalize(start_time)
self.work_lock.release()
else:
err_msg = "Request received, but declined because of acquired " \
"lock (Worker is busy executing another job, this should " \
"not happen: check if there are more than one ES running, " \
"or for bugs in ES."
logger.warning(err_msg)
self._finalize(start_time)
raise JobException(err_msg)
def execute_job(self, job_dict):
"""Receive a group of jobs in a dict format and executes them
one by one.
job_dict (dict): a dictionary suitable to be imported from Job.
"""
start_time = time.time()
job = Job.import_from_dict_with_type(job_dict)
if self.work_lock.acquire(False):
try:
logger.info("Starting job.", extra={"operation": job.info})
job.shard = self.shard
task_type = get_task_type(job.task_type,
job.task_type_parameters)
task_type.execute_job(job, self.file_cacher)
logger.info("Finished job.", extra={"operation": job.info})
return job.export_to_dict()
except:
err_msg = "Worker failed."
logger.error(err_msg, exc_info=True)
raise JobException(err_msg)
finally:
self._finalize(start_time)
self.work_lock.release()
else:
err_msg = "Request received, but declined because of acquired " \
"lock (Worker is busy executing another job, this should " \
"not happen: check if there are more than one ES running, " \
"or for bugs in ES."
logger.warning(err_msg)
self._finalize(start_time)
raise JobException(err_msg)
def create_sandbox(task_type):
"""Create a sandbox, and return it.
task_type (TaskType): a task type instance.
return (Sandbox): a sandbox.
raise: JobException
"""
try:
sandbox = Sandbox(task_type.file_cacher)
except (OSError, IOError):
err_msg = "Couldn't create sandbox."
logger.error("%s\n%s" % (err_msg, traceback.format_exc()))
raise JobException(err_msg)
return sandbox
def create_sandbox(file_cacher):
"""Create a sandbox, and return it.
file_cacher (FileCacher): a file cacher instance.
return (Sandbox): a sandbox.
raise (JobException): if the sandbox cannot be created.
"""
try:
sandbox = Sandbox(file_cacher)
except (OSError, IOError):
err_msg = "Couldn't create sandbox."
logger.error(err_msg, exc_info=True)
raise JobException(err_msg)
return sandbox
def create_sandbox(file_cacher):
"""Create a sandbox, and return it.
file_cacher (FileCacher): a file cacher instance.
return (Sandbox): a sandbox.
raise (JobException): if the sandbox cannot be created.
"""
try:
sandbox = Sandbox(file_cacher)
except (OSError, IOError):
err_msg = "Couldn't create sandbox."
logger.error("%s\n%s" % (err_msg, traceback.format_exc()))
raise JobException(err_msg)
return sandbox
def create_sandbox(file_cacher, multithreaded=False, name=None):
"""Create a sandbox, and return it.
file_cacher (FileCacher): a file cacher instance.
multithreaded (boolean): whether the sandbox should allow multithreading.
return (Sandbox): a sandbox.
raise (JobException): if the sandbox cannot be created.
"""
try:
sandbox = Sandbox(multithreaded, file_cacher, name=name)
except (OSError, IOError):
err_msg = "Couldn't create sandbox."
logger.error(err_msg, exc_info=True)
raise JobException(err_msg)
return sandbox
def create_sandbox(file_cacher, name=None):
"""Create a sandbox, and return it.
file_cacher (FileCacher): a file cacher instance.
name (str): name to include in the path of the sandbox.
return (Sandbox): a sandbox.
raise (JobException): if the sandbox cannot be created.
"""
try:
sandbox = Sandbox(file_cacher, name=name)
except OSError:
err_msg = "Couldn't create sandbox."
logger.error(err_msg, exc_info=True)
raise JobException(err_msg)
return sandbox
