def test_work_columns(self):
columns = work_columns(0, 0)
self.assertEqual(len(columns), 6)
self.assertTrue(
all(map(lambda column: isinstance(column, Column), columns)))
id, state, priority, time_submitted, time_started, time_finished = \
columns
self.assertIsInstance(id.type, IDTypeWork)
self.assertIsInstance(state.type, WorkStateEnum)
self.assertIsInstance(priority.type, Integer)
self.assertIsInstance(time_submitted.type, DateTime)
self.assertIsInstance(time_started.type, DateTime)
self.assertIsInstance(time_finished.type, DateTime)
def test_work_columns(self):
columns = work_columns(0, 0)
self.assertEqual(len(columns), 6)
self.assertTrue(
all(map(lambda column: isinstance(column, Column), columns)))
id, state, priority, time_submitted, time_started, time_finished = \
columns
self.assertIsInstance(id.type, IDTypeWork)
self.assertIsInstance(state.type, WorkStateEnum)
self.assertIsInstance(priority.type, Integer)
self.assertIsInstance(time_submitted.type, DateTime)
self.assertIsInstance(time_started.type, DateTime)
self.assertIsInstance(time_finished.type, DateTime)
class Job(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin,
WorkStateChangedMixin, ReprMixin, UtilityMixins):
"""
Defines the attributes and environment for a job. Individual commands
are kept track of by :class:`Task`
"""
__tablename__ = config.get("table_job")
REPR_COLUMNS = ("id", "state", "project")
REPR_CONVERT_COLUMN = {"state": repr}
STATE_ENUM = list(WorkState) + [None]
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(None, "job.priority")
jobtype_version_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_type_version")),
nullable=False,
doc="The foreign key which stores :class:`JobTypeVersion.id`")
job_queue_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_queue")),
nullable=True,
doc="The foreign key which stores :class:`JobQueue.id`")
job_group_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_group")),
nullable=True,
doc="The foreign key which stores:class:`JobGroup.id`")
user_id = db.Column(db.Integer,
db.ForeignKey("%s.id" % config.get("table_user")),
doc="The id of the user who owns this job")
minimum_agents = db.Column(
db.Integer,
nullable=True,
doc="The scheduler will try to assign at least this number "
"of agents to this job as long as it can use them, "
"before any other considerations.")
maximum_agents = db.Column(
db.Integer,
nullable=True,
doc="The scheduler will never assign more than this number"
"of agents to this job.")
weight = db.Column(
db.Integer,
nullable=False,
default=config.get("queue_default_weight"),
doc="The weight of this job. The scheduler will distribute "
"available agents between jobs and job queues in the "
"same queue in proportion to their weights.")
title = db.Column(db.String(config.get("jobtitle_max_length")),
nullable=False,
doc="The title of this job")
notes = db.Column(
db.Text,
default="",
doc="Notes that are provided on submission or added after "
"the fact. This column is only provided for human "
"consumption, is not scanned, indexed, or used when "
"searching")
output_link = db.Column(
db.Text,
nullable=True,
doc="An optional link to a URI where this job's output can "
"be viewed.")
# task data
by = db.Column(db.Numeric(10, 4),
default=1,
doc="The number of frames to count by between `start` and "
"`end`. This column may also sometimes be referred to "
"as 'step' by other software.")
num_tiles = db.Column(
db.Integer,
nullable=True,
doc="How many regions to split frames into for rendering.")
batch = db.Column(
db.Integer,
default=config.get("job_default_batch"),
doc="Number of tasks to run on a single agent at once. Depending "
"on the capabilities of the software being run this will "
"either cause a single process to execute on the agent "
"or multiple processes one after the other.")
requeue = db.Column(db.Integer,
default=config.get("job_requeue_default"),
doc="Number of times to requeue failed tasks "
""
".. csv-table:: **Special Values**"
" :header: Value, Result"
" :widths: 10, 50"
""
" 0, never requeue failed tasks"
" -1, requeue failed tasks indefinitely")
cpus = db.Column(
db.Integer,
default=config.get("job_default_cpus"),
doc="Number of cpus or threads each task should consume on"
"each agent. Depending on the job type being executed "
"this may result in additional cpu consumption, longer "
"wait times in the queue (2 cpus means 2 'fewer' cpus on "
"an agent), or all of the above."
""
".. csv-table:: **Special Values**"
" :header: Value, Result"
" :widths: 10, 50"
""
" 0, minimum number of cpu resources not required "
" -1, agent cpu is exclusive for a task from this job")
ram = db.Column(
db.Integer,
default=config.get("job_default_ram"),
doc="Amount of ram a task from this job will require to be "
"free in order to run. A task exceeding this value will "
"not result in any special behavior."
""
".. csv-table:: **Special Values**"
" :header: Value, Result"
" :widths: 10, 50"
""
"0, minimum amount of free ram not required"
"-1, agent ram is exclusive for a task from this job")
ram_warning = db.Column(
db.Integer,
nullable=True,
doc="Amount of ram used by a task before a warning raised. "
"A task exceeding this value will not cause any work "
"stopping behavior.")
ram_max = db.Column(
db.Integer,
nullable=True,
doc="Maximum amount of ram a task is allowed to consume on "
"an agent."
""
".. warning:: "
" If set, the task will be **terminated** if the ram in "
" use by the process exceeds this value.")
hidden = db.Column(
db.Boolean,
default=False,
nullable=False,
doc="If True, keep the job hidden from the queue and web "
"ui. This is typically set to True if you either want "
"to save a job for later viewing or if the jobs data "
"is being populated in a deferred manner.")
environ = db.Column(
JSONDict,
doc="Dictionary containing information about the environment "
"in which the job will execute. "
""
".. note::"
" Changes made directly to this object are **not** "
" applied to the session.")
data = db.Column(JSONDict,
doc="Json blob containing additional data for a job "
""
".. note:: "
" Changes made directly to this object are **not** "
" applied to the session.")
to_be_deleted = db.Column(
db.Boolean,
nullable=False,
default=False,
doc="If true, the master will stop all running tasks for "
"this job and then delete it.")
completion_notify_sent = db.Column(
db.Boolean,
nullable=False,
default=False,
doc="Whether or not the finish notification mail has already "
"been sent out.")
autodelete_time = db.Column(
db.Integer,
nullable=True,
default=None,
doc="If not None, this job will be automatically deleted this "
"number of seconds after it finishes.")
#
# Relationships
#
queue = db.relationship("JobQueue",
backref=db.backref("jobs", lazy="dynamic"),
doc="The queue for this job")
group = db.relationship("JobGroup",
backref=db.backref("jobs", lazy="dynamic"),
doc="The job group this job belongs to")
user = db.relationship("User",
backref=db.backref("jobs", lazy="dynamic"),
doc="The owner of this job")
# self-referential many-to-many relationship
parents = db.relationship("Job",
secondary=JobDependency,
primaryjoin=id == JobDependency.c.childid,
secondaryjoin=id == JobDependency.c.parentid,
backref="children")
notified_users = db.relationship("JobNotifiedUser",
lazy="dynamic",
backref=db.backref("job"),
cascade="all,delete")
tasks_queued = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == None) & "
"(Task.job_id == Job.id)",
doc="Relationship between this job and any :class:`Task` "
"objects which are queued.")
tasks_running = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.RUNNING,
doc="Relationship between this job and any :class:`Task` "
"objects which are running.")
tasks_done = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.DONE,
doc="Relationship between this job and any :class:`Task` objects "
"which are done.")
tasks_failed = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.FAILED,
doc="Relationship between this job and any :class:`Task` objects "
"which have failed.")
# resource relationships
tags = db.relationship(
"Tag",
backref="jobs",
lazy="dynamic",
secondary=JobTagAssociation,
doc="Relationship between this job and :class:`.Tag` objects")
def paused(self):
return self.state == WorkState.PAUSED
def update_state(self):
# Import here instead of at the top of the file to avoid a circular
# import
from pyfarm.scheduler.tasks import send_job_completion_mail
from pyfarm.models.agent import Agent
num_active_tasks = db.session.query(Task).\
filter(Task.job == self,
or_(Task.state == None, and_(
Task.state != WorkState.DONE,
Task.state != WorkState.FAILED))).count()
if num_active_tasks == 0:
num_failed_tasks = db.session.query(Task).filter(
Task.job == self, Task.state == WorkState.FAILED).count()
if num_failed_tasks == 0:
if self.state != _WorkState.DONE:
logger.info(
"Job %r (id %s): state transition %r -> 'done'",
self.title, self.id, self.state)
self.state = WorkState.DONE
send_job_completion_mail.apply_async(args=[self.id, True],
countdown=5)
else:
if self.state != _WorkState.FAILED:
logger.info(
"Job %r (id %s): state transition %r -> "
"'failed'", self.title, self.id, self.state)
self.state = WorkState.FAILED
send_job_completion_mail.apply_async(args=[self.id, False],
countdown=5)
db.session.add(self)
elif self.state != _WorkState.PAUSED:
num_running_tasks = db.session.query(Task).\
filter(Task.job == self,
Task.agent_id != None,
Task.agent.has(and_(Agent.state != AgentState.OFFLINE,
Agent.state != AgentState.DISABLED)),
or_(
Task.state == WorkState.RUNNING,
Task.state == None)).count()
if num_running_tasks == 0:
logger.debug(
"No running tasks in job %s (id %s), setting it "
"to queued", self.title, self.id)
self.state = None
db.session.add(self)
elif self.state != _WorkState.RUNNING:
self.state = WorkState.RUNNING
# Methods used by the scheduler
def num_assigned_agents(self):
# Import here instead of at the top of the file to avoid circular import
from pyfarm.models.agent import Agent
# Optimization: Blindly assume that we have no agents assigned if not
# running
if self.state != _WorkState.RUNNING:
return 0
try:
return self.assigned_agents_count
except AttributeError:
self.assigned_agents_count =\
db.session.query(distinct(Task.agent_id)).\
filter(Task.job == self,
Task.agent_id != None,
or_(Task.state == None,
Task.state == WorkState.RUNNING),
Task.agent.has(
and_(Agent.state != AgentState.OFFLINE,
Agent.state != AgentState.DISABLED)))\
.count()
return self.assigned_agents_count
def clear_assigned_counts(self):
try:
del self.assigned_agents_count
except AttributeError:
pass
if self.queue:
self.queue.clear_assigned_counts()
def can_use_more_agents(self):
# Import here instead of at the top of the file to avoid circular import
from pyfarm.models.agent import Agent
unassigned_tasks = Task.query.filter(
Task.job == self,
or_(Task.state == None,
~Task.state.in_([WorkState.DONE, WorkState.FAILED])),
or_(
Task.agent == None,
Task.agent.has(
Agent.state.in_([AgentState.OFFLINE,
AgentState.DISABLED])))).count()
return unassigned_tasks > 0
def get_batch(self, agent):
# Import here instead of at the top of the file to avoid circular import
from pyfarm.models.agent import Agent
tasks_query = Task.query.filter(
Task.job == self,
~Task.failed_in_agents.any(id=agent.id),
or_(Task.state == None,
~Task.state.in_([WorkState.DONE, WorkState.FAILED])),
or_(Task.agent == None,
Task.agent.has(Agent.state.in_(
[AgentState.OFFLINE, AgentState.DISABLED])))).\
order_by("frame asc, tile asc")
batch = []
for task in tasks_query:
if (len(batch) < self.batch and len(batch) <
(self.jobtype_version.max_batch or maxsize)
and (not self.jobtype_version.batch_contiguous or
(len(batch) == 0
or batch[-1].frame + self.by == task.frame))):
batch.append(task)
return batch
def alter_frame_range(self, start, end, by):
# We have to import this down here instead of at the top to break a
# circular dependency between the modules
from pyfarm.scheduler.tasks import delete_task
if end < start:
raise ValueError("`end` must be greater than or equal to `start`")
self.by = by
required_frames = []
current_frame = start
while current_frame <= end:
required_frames.append(current_frame)
current_frame += by
existing_tasks = Task.query.filter_by(job=self).all()
frames_to_create = required_frames
num_created = 0
for task in existing_tasks:
if task.frame not in required_frames:
delete_task.delay(task.id)
else:
frames_to_create.remove(task.frame)
for frame in frames_to_create:
if self.num_tiles:
for tile in range_(self.num_tiles - 1):
num_created += 1
task = Task()
task.job = self
task.frame = frame
task.tile = tile
task.priority = self.priority
db.session.add(task)
else:
num_created += 1
task = Task()
task.job = self
task.frame = frame
task.priority = self.priority
db.session.add(task)
if frames_to_create:
if self.state != WorkState.RUNNING:
self.state = None
if config.get("enable_statistics"):
task_event_count = TaskEventCount(num_new=num_created,
job_queue_id=self.job_queue_id)
task_event_count.time_start = datetime.utcnow()
task_event_count.time_end = datetime.utcnow()
db.session.add(task_event_count)
def rerun(self):
"""
Makes this job rerun all its task. Tasks that are currently running are
left untouched.
"""
num_restarted = 0
for task in self.tasks:
if task.state != _WorkState.RUNNING and task.state is not None:
task.state = None
task.agent = None
task.failures = 0
db.session.add(task)
num_restarted += 1
self.completion_notify_sent = False
self.update_state()
db.session.add(self)
if config.get("enable_statistics"):
task_event_count = TaskEventCount(job_queue_id=self.job_queue_id,
num_restarted=num_restarted)
task_event_count.time_start = datetime.utcnow()
task_event_count.time_end = datetime.utcnow()
db.session.add(task_event_count)
db.session.commit()
for child in self.children:
child.rerun()
def rerun_failed(self):
"""
Makes this job rerun all its failed tasks. Tasks that are done or are
currently running are left untouched
"""
num_restarted = 0
for task in self.tasks:
if task.state == _WorkState.FAILED:
task.state = None
task.agent = None
task.failures = 0
db.session.add(task)
num_restarted += 1
self.completion_notify_sent = False
self.update_state()
db.session.add(self)
if config.get("enable_statistics"):
task_event_count = TaskEventCount(job_queue_id=self.job_queue_id,
num_restarted=num_restarted)
task_event_count.time_start = datetime.utcnow()
task_event_count.time_end = datetime.utcnow()
db.session.commit()
for child in self.children:
child.rerun_failed()
@validates("ram", "cpus")
def validate_resource(self, key, value):
"""
Validation that ensures that the value provided for either
:attr:`.ram` or :attr:`.cpus` is a valid value with a given range
"""
assert isinstance(value, int), "%s must be an integer" % key
min_value = config.get("agent_min_%s" % key)
max_value = config.get("agent_max_%s" % key)
# check the provided input
if min_value > value or value > max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
@validates("progress")
def validate_progress(self, key, value):
if value < 0.0 or value > 1.0:
raise ValueError("Progress must be between 0.0 and 1.0")
class Task(db.Model, ValidatePriorityMixin, WorkStateChangedMixin,
UtilityMixins, ReprMixin):
"""
Defines a task which a child of a :class:`Job`. This table represents
rows which contain the individual work unit(s) for a job.
"""
__tablename__ = TABLE_TASK
STATE_ENUM = WorkState
STATE_DEFAULT = STATE_ENUM.QUEUED
REPR_COLUMNS = ("id", "state", "frame", "project")
REPR_CONVERT_COLUMN = {"state": partial(repr_enum, enum=STATE_ENUM)}
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(STATE_DEFAULT, "job.priority")
project_id = db.Column(db.Integer,
db.ForeignKey("%s.id" % TABLE_PROJECT),
doc="stores the project id")
agent_id = db.Column(IDTypeAgent,
db.ForeignKey("%s.id" % TABLE_AGENT),
doc="Foreign key which stores :attr:`Job.id`")
job_id = db.Column(IDTypeWork,
db.ForeignKey("%s.id" % TABLE_JOB),
doc="Foreign key which stores :attr:`Job.id`")
hidden = db.Column(db.Boolean,
default=False,
doc=dedent("""
hides the task from queue and web ui"""))
attempts = db.Column(db.Integer,
doc=dedent("""
The number attempts which have been made on this
task. This value is auto incremented when
:attr:`state` changes to a value synonyms with a
running state."""))
frame = db.Column(db.Float,
nullable=False,
doc=dedent("""
The frame the :class:`Task` will be executing."""))
# relationships
parents = db.relationship("Task",
secondary=TaskDependencies,
primaryjoin=id == TaskDependencies.c.parent_id,
secondaryjoin=id == TaskDependencies.c.child_id,
backref=db.backref("children", lazy="dynamic"))
project = db.relationship("Project",
backref=db.backref("tasks", lazy="dynamic"),
doc=dedent("""
relationship attribute which retrieves the
associated project for the task"""))
job = db.relationship("Job",
backref=db.backref("tasks", lazy="dynamic"),
doc=dedent("""
relationship attribute which retrieves the
associated job for this task"""))
@staticmethod
def agentChangedEvent(target, new_value, old_value, initiator):
"""set the state to ASSIGN whenever the agent is changed"""
if new_value is not None:
target.state = target.STATE_ENUM.ASSIGN
class Job(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, ReprMixin):
"""
Defines the attributes and environment for a job. Individual commands
are kept track of by |Task|
"""
__tablename__ = TABLE_JOB
REPR_COLUMNS = ("id", "state", "project")
REPR_CONVERT_COLUMN = {
"state": repr}
MIN_CPUS = read_env_int("PYFARM_QUEUE_MIN_CPUS", 1)
MAX_CPUS = read_env_int("PYFARM_QUEUE_MAX_CPUS", 256)
MIN_RAM = read_env_int("PYFARM_QUEUE_MIN_RAM", 16)
MAX_RAM = read_env_int("PYFARM_QUEUE_MAX_RAM", 262144)
SPECIAL_RAM = read_env("PYFARM_AGENT_SPECIAL_RAM", [0], eval_literal=True)
SPECIAL_CPUS = read_env("PYFARM_AGENT_SPECIAL_CPUS", [0], eval_literal=True)
# quick check of the configured data
assert MIN_CPUS >= 1, "$PYFARM_QUEUE_MIN_CPUS must be > 0"
assert MAX_CPUS >= 1, "$PYFARM_QUEUE_MAX_CPUS must be > 0"
assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS"
assert MIN_RAM >= 1, "$PYFARM_QUEUE_MIN_RAM must be > 0"
assert MAX_RAM >= 1, "$PYFARM_QUEUE_MAX_RAM must be > 0"
assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM"
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(WorkState.QUEUED, "job.priority")
project_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT),
doc="stores the project id")
job_type_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB_TYPE),
nullable=False,
doc=dedent("""
The foreign key which stores :class:`JobType.id`"""))
user = db.Column(db.String(MAX_USERNAME_LENGTH),
doc=dedent("""
The user this job should execute as. The agent
process will have to be running as root on platforms
that support setting the user id.
.. note::
The length of this field is limited by the
configuration value `job.max_username_length`
.. warning::
this may not behave as expected on all platforms
(windows in particular)"""))
notes = db.Column(db.Text, default="",
doc=dedent("""
Notes that are provided on submission or added after
the fact. This column is only provided for human
consumption is not scanned, index, or used when
searching"""))
# task data
cmd = db.Column(db.String(MAX_COMMAND_LENGTH),
doc=dedent("""
The platform independent command to run. Each agent will
resolve this value for itself when the task begins so a
command like `ping` will work on any platform it's
assigned to. The full command could be provided here,
but then the job must be tagged using
:class:`.JobSoftware` to limit which agent(s) it will
run on."""))
start = db.Column(db.Float,
doc=dedent("""
The first frame of the job to run. This value may
be a float so subframes can be processed."""))
end = db.Column(db.Float,
doc=dedent("""
The last frame of the job to run. This value may
be a float so subframes can be processed."""))
by = db.Column(db.Float, default=1,
doc=dedent("""
The number of frames to count by between `start` and
`end`. This column may also sometimes be referred to
as 'step' by other software."""))
batch = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_BATCH", 1),
doc=dedent("""
Number of tasks to run on a single agent at once.
Depending on the capabilities of the software being run
this will either cause a single process to execute on
the agent or multiple processes on after the other.
**configured by**: `job.batch`"""))
requeue = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_REQUEUE", 3),
doc=dedent("""
Number of times to requeue failed tasks
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, never requeue failed tasks
-1, requeue failed tasks indefinitely
**configured by**: `job.requeue`"""))
cpus = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_CPUS", 1),
doc=dedent("""
Number of cpus or threads each task should consume on
each agent. Depending on the job type being executed
this may result in additional cpu consumption, longer
wait times in the queue (2 cpus means 2 'fewer' cpus on
an agent), or all of the above.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, minimum number of cpu resources not required
-1, agent cpu is exclusive for a task from this job
**configured by**: `job.cpus`"""))
ram = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_RAM", 32),
doc=dedent("""
Amount of ram a task from this job will require to be
free in order to run. A task exceeding this value will
not result in any special behavior.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, minimum amount of free ram not required
-1, agent ram is exclusive for a task from this job
**configured by**: `job.ram`"""))
ram_warning = db.Column(db.Integer, default=-1,
doc=dedent("""
Amount of ram used by a task before a warning
raised. A task exceeding this value will not
cause any work stopping behavior.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
-1, not set"""))
ram_max = db.Column(db.Integer, default=-1,
doc=dedent("""
Maximum amount of ram a task is allowed to consume on
an agent.
.. warning::
The task will be **terminated** if the ram in use
by the process exceeds this value.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
-1, not set
"""))
attempts = db.Column(db.Integer,
doc=dedent("""
The number attempts which have been made on this
task. This value is auto incremented when
:attr:`state` changes to a value synonyms with a
running state."""))
hidden = db.Column(db.Boolean, default=False, nullable=False,
doc=dedent("""
If True, keep the job hidden from the queue and web
ui. This is typically set to True if you either want
to save a job for later viewing or if the jobs data
is being populated in a deferred manner."""))
environ = db.Column(JSONDict,
doc=dedent("""
Dictionary containing information about the environment
in which the job will execute.
.. note::
Changes made directly to this object are **not**
applied to the session."""))
args = db.Column(JSONList,
doc=dedent("""
List containing the command line arguments.
.. note::
Changes made directly to this object are **not**
applied to the session."""))
data = db.Column(JSONDict,
doc=dedent("""
Json blob containing additional data for a job
.. note::
Changes made directly to this object are **not**
applied to the session."""))
project = db.relationship("Project",
backref=db.backref("jobs", lazy="dynamic"),
doc=dedent("""
relationship attribute which retrieves the
associated project for the job"""))
# self-referential many-to-many relationship
parents = db.relationship("Job",
secondary=JobDependencies,
primaryjoin=id==JobDependencies.c.parentid,
secondaryjoin=id==JobDependencies.c.childid,
backref="children")
tasks_done = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.DONE,
doc=dedent("""
Relationship between this job and any |Task| objects which are
done."""))
tasks_failed = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.FAILED,
doc=dedent("""
Relationship between this job and any |Task| objects which have
failed."""))
tasks_queued = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.QUEUED,
doc=dedent("""
Relationship between this job and any |Task| objects which
are queued."""))
# resource relationships
tags = db.relationship("Tag", backref="jobs", lazy="dynamic",
secondary=JobTagAssociation,
doc=dedent("""
Relationship between this job and
:class:`.Tag` objects"""))
software = db.relationship("Software",
secondary=JobSoftwareDependency,
backref=db.backref("jobs", lazy="dynamic"),
lazy="dynamic",
doc="software needed by this job")
@validates("ram", "cpus")
def validate_resource(self, key, value):
"""
Validation that ensures that the value provided for either
:attr:`.ram` or :attr:`.cpus` is a valid value with a given range
"""
key_upper = key.upper()
special = getattr(self, "SPECIAL_%s" % key_upper)
if value is None or value in special:
return value
min_value = getattr(self, "MIN_%s" % key_upper)
max_value = getattr(self, "MAX_%s" % key_upper)
# quick sanity check of the incoming config
assert isinstance(min_value, int), "db.min_%s must be an integer" % key
assert isinstance(max_value, int), "db.max_%s must be an integer" % key
assert min_value >= 1, "db.min_%s must be > 0" % key
assert max_value >= 1, "db.max_%s must be > 0" % key
# check the provided input
if min_value > value or value > max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
class Task(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin,
UtilityMixins, ReprMixin):
"""
Defines a task which a child of a :class:`Job`. This table represents
rows which contain the individual work unit(s) for a job.
"""
__tablename__ = config.get("table_task")
STATE_ENUM = list(WorkState) + [None]
STATE_DEFAULT = None
REPR_COLUMNS = ("id", "state", "frame", "project")
REPR_CONVERT_COLUMN = {"state": partial(repr_enum, enum=STATE_ENUM)}
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(STATE_DEFAULT, "job.priority")
agent_id = db.Column(
IDTypeAgent,
db.ForeignKey("%s.id" % config.get("table_agent")),
doc="Foreign key which stores :attr:`Job.id`")
job_id = db.Column(
IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")),
nullable=False,
doc="Foreign key which stores :attr:`Job.id`")
hidden = db.Column(
db.Boolean, default=False,
doc="When True this hides the task from queue and web ui")
attempts = db.Column(
db.Integer,
nullable=False, default=0,
doc="The number of attempts which have been made on this "
"task. This value is auto incremented when "
"``state`` changes to a value synonymous with a "
"running state.")
failures = db.Column(
db.Integer,
nullable=False, default=0,
doc="The number of times this task has failed. This value "
"is auto incremented when :attr:`state` changes to a "
"value synonymous with a failed state.")
frame = db.Column(
db.Numeric(10, 4),
nullable=False,
doc="The frame this :class:`Task` will be executing.")
tile = db.Column(
db.Integer,
nullable=True,
doc="When using tiled rendering, the number of the tile this task "
"refers to. The jobtype will have to translate that into an "
"actual image region. This will be NULL if the job doesn't use "
"tiled rendering.")
last_error = db.Column(
db.UnicodeText,
nullable=True,
doc="This column may be set when an error is "
"present. The agent typically sets this "
"column when the job type either can't or "
"won't run a given task. This column will "
"be cleared whenever the task's state is "
"returned to a non-error state.")
sent_to_agent = db.Column(
db.Boolean,
default=False, nullable=False,
doc="Whether this task was already sent to the assigned agent")
progress = db.Column(
db.Float, default=0.0,
doc="The progress for this task, as a value between "
"0.0 and 1.0. Used purely for display purposes.")
#
# Relationships
#
job = db.relationship(
"Job",
backref=db.backref("tasks", lazy="dynamic"),
doc="relationship attribute which retrieves the "
"associated job for this task")
def running(self):
return self.state == WorkState.RUNNING
def failed(self):
return self.state == WorkState.FAILED
@staticmethod
def increment_attempts(target, new_value, old_value, initiator):
if new_value is not None and new_value != old_value:
target.attempts += 1
@staticmethod
def log_assign_change(target, new_value, old_value, initiator):
logger.debug("Agent change for task %s: old %s new: %s",
target.id, old_value, new_value)
@staticmethod
def update_failures(target, new_value, old_value, initiator):
if new_value == WorkState.FAILED and new_value != old_value:
target.failures += 1
if target not in target.agent.failed_tasks:
target.agent.failed_tasks.append(target)
@staticmethod
def set_progress_on_success(target, new_value, old_value, initiator):
if new_value == WorkState.DONE:
target.progress = 1.0
@staticmethod
def update_agent_on_success(target, new_value, old_value, initiator):
if new_value == WorkState.DONE:
agent = target.agent
if agent:
agent.last_success_on = datetime.utcnow()
db.session.add(agent)
@staticmethod
def reset_agent_if_failed_and_retry(
target, new_value, old_value, initiator):
# There's nothing else we should do here if
# we don't have a parent job. This can happen if you're
# testing or a job is disconnected from a task.
if target.job is None:
return new_value
if (new_value == WorkState.FAILED and
target.failures <= target.job.requeue):
logger.info("Failed task %s will be retried", target.id)
target.agent_id = None
return None
else:
return new_value
@staticmethod
def clear_error_state(target, new_value, old_value, initiator):
"""
Sets ``last_error`` column to ``None`` if the task's state is 'done'
"""
if new_value == WorkState.DONE and target.last_error is not None:
target.last_error = None
@staticmethod
def set_times(target, new_value, old_value, initiator):
"""update the datetime objects depending on the new value"""
if (new_value == _WorkState.RUNNING and
(old_value not in [_WorkState.RUNNING, _WorkState.PAUSED] or
target.time_started == None)):
if not target.job.jobtype_version.no_automatic_start_time:
target.time_started = datetime.utcnow()
target.time_finished = None
elif (new_value in (_WorkState.DONE, _WorkState.FAILED) and
not target.time_finished):
target.time_finished = datetime.utcnow()
@staticmethod
def reset_finished_time(target, new_value, old_value, initiator):
if (target.state not in (_WorkState.DONE, _WorkState.FAILED) or
new_value is None):
target.time_finished = None
elif new_value is not None:
if target.time_finished is not None:
target.time_finished = max(target.time_finished,
new_value)
else:
target.time_finished = max(new_value,
datetime.utcnow())