class Software(db.Model, UtilityMixins):
"""
Model to represent a versioned piece of software that can be present on an
agent and may be depended on by a job and/or jobtype
.. note::
This table enforces two forms of uniqueness. The :attr:`id` column
must be unique and the combination of these columns must also be
unique to limit the frequency of duplicate data:
* :attr:`software`
* :attr:`version`
.. autoattribute:: job_id
"""
__tablename__ = TABLE_SOFTWARE
__table_args__ = (
UniqueConstraint("software", "version"), )
id = id_column()
software = db.Column(db.String(MAX_TAG_LENGTH), nullable=False,
doc=dedent("""
The name of the software"""))
version = db.Column(db.String(MAX_TAG_LENGTH),
default="any", nullable=False,
doc=dedent("""
The version of the software. This value does not follow
any special formatting rules because the format depends
on the 3rd party."""))
class JobType(db.Model):
"""
Stores the unique information necessary to execute a task
"""
__tablename__ = TABLE_JOB_TYPE
id = id_column(db.Integer)
name = db.Column(db.String(MAX_JOBTYPE_LENGTH),
nullable=False,
doc=dedent("""
The name of the job type. This can be either a human
readable name or the name of the job type class
itself."""))
description = db.Column(db.Text,
nullable=True,
doc=dedent("""
Human readable description of the job type. This
field is not required and is not directly relied
upon anywhere."""))
classname = db.Column(db.String(MAX_JOBTYPE_LENGTH),
nullable=True,
doc=dedent("""
The name of the job class contained within the file
being loaded. This field may be null but when it's
not provided :attr:`name` will be used instead."""))
code = db.Column(db.UnicodeText,
nullable=False,
doc=dedent("""
General field containing the 'code' to retrieve the job
type. See below for information on what this field will
contain depending on how the job will be loaded."""))
mode = db.Column(JobTypeLoadModeEnum,
default=JobTypeLoadMode.IMPORT,
nullable=False,
doc=dedent("""
Indicates how the job type should be retrieved.
.. csv-table:: **JobTypeLoadMode Enums**
:header: Value, Result
:widths: 10, 50
DOWNLOAD, job type will be downloaded remotely
IMPORT, the remote agent will import the job type
OPEN, code is loaded directly from a file on disk"""))
jobs = db.relationship("Job",
backref="job_type",
lazy="dynamic",
doc=dedent("""
Relationship between this jobtype and
:class:`.Job` objects."""))
@validates("mode")
def validates_mode(self, key, value):
"""ensures the value provided to :attr:`mode` is valid"""
if value not in JobTypeLoadMode:
raise ValueError("invalid value for mode")
return value
class SoftwareVersion(db.Model, UtilityMixins):
"""
Model to represent a version for a given software
"""
__tablename__ = config.get("table_software_version")
__table_args__ = (UniqueConstraint("software_id", "version"),
UniqueConstraint("software_id", "rank"))
id = id_column()
software_id = db.Column(db.Integer,
db.ForeignKey("%s.id" %
config.get("table_software")),
nullable=False,
doc="The software this version belongs to")
version = db.Column(
db.String(config.get("max_tag_length")),
default="any",
nullable=False,
doc="The version of the software. This value does not "
"follow any special formatting rules because the "
"format depends on the 3rd party.")
rank = db.Column(
db.Integer,
nullable=False,
doc="The rank of this version relative to other versions of "
"the same software. Used to determine whether a version "
"is higher or lower than another.")
default = db.Column(db.Boolean,
default=False,
nullable=False,
doc="If true, this software version will be registered"
"on new nodes by default.")
discovery_code = db.Column(
db.UnicodeText,
nullable=True,
doc="Python code to discover if this software version is installed "
"on a node")
discovery_function_name = db.Column(
db.String(config.get("max_discovery_function_name_length")),
nullable=True,
doc="The name of a function in `discovery_code` to call when "
"checking for the presence of this software version on an agent.\n"
"The function should return either a boolean (true if present, "
"false if not) or a tuple of a boolean and a dict of named "
"parameters describing this installation.")
class MixinModel(db.Model, UtilityMixins):
__tablename__ = "%s_mixin_test" % config.get("table_prefix")
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
a = db.Column(db.Integer)
b = db.Column(db.String(512))
c = db.Column(IPv4Address)
d = db.Column(db.Integer, nullable=False)
e = db.relationship("MixinModel", secondary=MixinModelRelation1)
f = db.relationship("MixinModel", secondary=MixinModelRelation2)
class MixinModel(db.Model, UtilityMixins):
__tablename__ = "%s_mixin_test" % TABLE_PREFIX
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
a = db.Column(db.Integer)
b = db.Column(db.String(512))
c = db.Column(IPv4Address)
def serialize_column(self, column):
return column
class GPU(db.Model, UtilityMixins, ReprMixin):
__tablename__ = config.get("table_gpu")
__table_args__ = (UniqueConstraint("fullname"), )
id = id_column(db.Integer)
fullname = db.Column(db.String(config.get("max_gpu_name_length")),
nullable=False,
doc="The full name of this graphics card model")
class PathMap(db.Model, ReprMixin, UtilityMixins):
"""
Defines a table which is used for cross-platform
file path mappings.
"""
__tablename__ = config.get("table_path_map")
id = id_column(db.Integer)
path_linux = db.Column(
db.String(config.get("max_path_length")),
nullable=False,
doc="The path on linux platforms")
path_windows = db.Column(
db.String(config.get("max_path_length")),
nullable=False,
doc="The path on Windows platforms")
path_osx = db.Column(
db.String(config.get("max_path_length")),
nullable=False,
doc="The path on Mac OS X platforms")
tag_id = db.Column(
db.Integer,
db.ForeignKey("%s.id" % config.get("table_tag")),
nullable=True,
doc="The tag an agent needs to have for this path map "
"to apply to it. "
"If this is NULL, this path map applies to all "
"agents, but is overridden by applying path maps "
"that do specify a tag.")
#
# Relationships
#
tag = db.relationship(
"Tag",
backref=db.backref("path_maps", lazy="dynamic"),
doc="Relationship attribute for the tag this path map "
"applies to.")
class Tag(db.Model, UtilityMixins):
"""
Model which provides tagging for :class:`.Job` and class:`.Agent` objects
"""
__tablename__ = TABLE_TAG
__table_args__ = (UniqueConstraint("tag"), )
id = id_column()
tag = db.Column(db.String(MAX_TAG_LENGTH), nullable=False,
doc=dedent("""The actual value of the tag"""))
class JobType(db.Model, UtilityMixins, ReprMixin):
"""
Stores the unique information necessary to execute a task
"""
__tablename__ = config.get("table_job_type")
__table_args__ = (UniqueConstraint("name"), )
REPR_COLUMNS = ("id", "name")
id = id_column(IDTypeWork)
name = db.Column(
db.String(config.get("job_type_max_name_length")),
nullable=False,
doc="The name of the job type. This can be either a human "
"readable name or the name of the job type class itself.")
description = db.Column(
db.Text,
nullable=True,
doc="Human readable description of the job type. This field is not "
"required and is not directly relied upon anywhere.")
success_subject = db.Column(
db.Text,
nullable=True,
doc="The subject line to use for notifications in case of "
"success. Some substitutions, for example for the job title, "
"are available.")
success_body = db.Column(db.Text,
nullable=True,
doc="The email body to use for notifications in "
"in case of success. Some substitutions, for "
"example for the job title, are available.")
fail_subject = db.Column(db.Text,
nullable=True,
doc="The subject line to use for notifications "
"in case of failure. Some substitutions, for "
"example for the job title, are available.")
fail_body = db.Column(db.Text,
nullable=True,
doc="The email body to use for notifications in "
"in case of success. Some substitutions, for "
"example for the job title, are available.")
@validates("name")
def validate_name(self, key, value):
if value == "":
raise ValueError("Name cannot be empty")
return value
class Tag(db.Model, UtilityMixins):
"""
Model which provides tagging for :class:`.Job` and class:`.Agent` objects
"""
__tablename__ = config.get("table_tag")
__table_args__ = (UniqueConstraint("tag"), )
id = id_column()
tag = db.Column(db.String(config.get("max_tag_length")),
nullable=False,
doc="The actual value of the tag")
class TaskLog(db.Model, UtilityMixins, ReprMixin):
"""Table which represents a single task log entry"""
__tablename__ = config.get("table_task_log")
__table_args__ = (UniqueConstraint("identifier"),)
id = id_column(db.Integer)
identifier = db.Column(
db.String(255),
nullable=False,
doc="The identifier for this log")
agent_id = db.Column(
IDTypeAgent,
db.ForeignKey("%s.id" % config.get("table_agent")),
nullable=True,
doc="The agent this log was created on")
created_on = db.Column(
db.DateTime,
default=datetime.utcnow,
doc="The time when this log was created")
#
# Relationships
#
agent = db.relationship(
"Agent",
backref=db.backref("task_logs", lazy="dynamic"),
doc="Relationship between an :class:`TaskLog`"
"and the :class:`pyfarm.models.Agent` it was "
"created on")
task_associations = db.relationship(
TaskTaskLogAssociation, backref="log",
doc="Relationship between tasks and their logs."
)
def num_queued_tasks(self):
return TaskTaskLogAssociation.query.filter_by(
log=self, state=None).count()
def num_running_tasks(self):
return TaskTaskLogAssociation.query.filter_by(
log=self, state=WorkState.RUNNING).count()
def num_failed_tasks(self):
return TaskTaskLogAssociation.query.filter_by(
log=self, state=WorkState.FAILED).count()
def num_done_tasks(self):
return TaskTaskLogAssociation.query.filter_by(
log=self, state=WorkState.DONE).count()
class Role(db.Model):
"""
Stores role information that can be used to give a user access
to individual resources.
"""
__tablename__ = config.get("table_role")
id = db.Column(db.Integer, primary_key=True, nullable=False)
active = db.Column(db.Boolean,
default=True,
doc="Enables or disables a role. Disabling a role "
"will prevent any users of this role from accessing "
"PyFarm")
name = db.Column(db.String(config.get("max_role_length")),
unique=True,
nullable=False,
doc="The name of the role")
expiration = db.Column(
db.DateTime,
doc="Role expiration. If this value is set then the role, and "
"anyone assigned to it, will no longer be able to access "
"PyFarm past the expiration.")
description = db.Column(db.Text, doc="Human description of the role.")
@classmethod
def create(cls, name, description=None):
"""
Creates a role by the given name or returns an existing
role if it already exists.
"""
if isinstance(name, Role):
return name
role = Role.query.filter_by(name=name).first()
if role is None:
role = cls(name=name, description=description)
db.session.add(role)
db.session.commit()
return role
def is_active(self):
if self.expiration is None:
return self.active
return self.active and datetime.utcnow() < self.expiration
class JobGroup(db.Model, UtilityMixins):
"""
Used to group jobs together for better presentation in the UI
"""
__tablename__ = config.get("table_job_group")
id = id_column(IDTypeWork)
title = db.Column(
db.String(config.get("max_jobgroup_name_length")),
nullable=False,
doc="The title of the job group's name")
main_jobtype_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_type")),
nullable=False,
doc="ID of the jobtype of the main job in this "
"group. Purely for display and filtering.")
user_id = db.Column(
db.Integer,
db.ForeignKey("%s.id" % config.get("table_user")),
doc="The id of the user who owns these jobs")
#
# Relationships
#
main_jobtype = db.relationship(
"JobType",
backref=db.backref("jobgroups", lazy="dynamic"),
doc="The jobtype of the main job in this group")
user = db.relationship(
"User",
backref=db.backref("jobgroups", lazy="dynamic"),
doc="The user who owns these jobs")
class Software(db.Model, UtilityMixins):
"""
Model to represent a versioned piece of software that can be present on an
agent and may be depended on by a job and/or jobtype through the appropriate
SoftwareRequirement table
"""
__tablename__ = config.get("table_software")
__table_args__ = (UniqueConstraint("software"), )
id = id_column()
software = db.Column(db.String(config.get("max_tag_length")),
nullable=False,
doc="The name of the software")
#
# Relationships
#
versions = db.relationship("SoftwareVersion",
backref=db.backref("software"),
lazy="dynamic",
order_by="asc(SoftwareVersion.rank)",
cascade="all, delete-orphan",
doc="All known versions of this software")
class Job(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin,
WorkStateChangedMixin, ReprMixin, UtilityMixins):
"""
Defines the attributes and environment for a job. Individual commands
are kept track of by :class:`Task`
"""
__tablename__ = config.get("table_job")
REPR_COLUMNS = ("id", "state", "project")
REPR_CONVERT_COLUMN = {"state": repr}
STATE_ENUM = list(WorkState) + [None]
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(None, "job.priority")
jobtype_version_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_type_version")),
nullable=False,
doc="The foreign key which stores :class:`JobTypeVersion.id`")
job_queue_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_queue")),
nullable=True,
doc="The foreign key which stores :class:`JobQueue.id`")
job_group_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_group")),
nullable=True,
doc="The foreign key which stores:class:`JobGroup.id`")
user_id = db.Column(db.Integer,
db.ForeignKey("%s.id" % config.get("table_user")),
doc="The id of the user who owns this job")
minimum_agents = db.Column(
db.Integer,
nullable=True,
doc="The scheduler will try to assign at least this number "
"of agents to this job as long as it can use them, "
"before any other considerations.")
maximum_agents = db.Column(
db.Integer,
nullable=True,
doc="The scheduler will never assign more than this number"
"of agents to this job.")
weight = db.Column(
db.Integer,
nullable=False,
default=config.get("queue_default_weight"),
doc="The weight of this job. The scheduler will distribute "
"available agents between jobs and job queues in the "
"same queue in proportion to their weights.")
title = db.Column(db.String(config.get("jobtitle_max_length")),
nullable=False,
doc="The title of this job")
notes = db.Column(
db.Text,
default="",
doc="Notes that are provided on submission or added after "
"the fact. This column is only provided for human "
"consumption, is not scanned, indexed, or used when "
"searching")
output_link = db.Column(
db.Text,
nullable=True,
doc="An optional link to a URI where this job's output can "
"be viewed.")
# task data
by = db.Column(db.Numeric(10, 4),
default=1,
doc="The number of frames to count by between `start` and "
"`end`. This column may also sometimes be referred to "
"as 'step' by other software.")
num_tiles = db.Column(
db.Integer,
nullable=True,
doc="How many regions to split frames into for rendering.")
batch = db.Column(
db.Integer,
default=config.get("job_default_batch"),
doc="Number of tasks to run on a single agent at once. Depending "
"on the capabilities of the software being run this will "
"either cause a single process to execute on the agent "
"or multiple processes one after the other.")
requeue = db.Column(db.Integer,
default=config.get("job_requeue_default"),
doc="Number of times to requeue failed tasks "
""
".. csv-table:: **Special Values**"
" :header: Value, Result"
" :widths: 10, 50"
""
" 0, never requeue failed tasks"
" -1, requeue failed tasks indefinitely")
cpus = db.Column(
db.Integer,
default=config.get("job_default_cpus"),
doc="Number of cpus or threads each task should consume on"
"each agent. Depending on the job type being executed "
"this may result in additional cpu consumption, longer "
"wait times in the queue (2 cpus means 2 'fewer' cpus on "
"an agent), or all of the above."
""
".. csv-table:: **Special Values**"
" :header: Value, Result"
" :widths: 10, 50"
""
" 0, minimum number of cpu resources not required "
" -1, agent cpu is exclusive for a task from this job")
ram = db.Column(
db.Integer,
default=config.get("job_default_ram"),
doc="Amount of ram a task from this job will require to be "
"free in order to run. A task exceeding this value will "
"not result in any special behavior."
""
".. csv-table:: **Special Values**"
" :header: Value, Result"
" :widths: 10, 50"
""
"0, minimum amount of free ram not required"
"-1, agent ram is exclusive for a task from this job")
ram_warning = db.Column(
db.Integer,
nullable=True,
doc="Amount of ram used by a task before a warning raised. "
"A task exceeding this value will not cause any work "
"stopping behavior.")
ram_max = db.Column(
db.Integer,
nullable=True,
doc="Maximum amount of ram a task is allowed to consume on "
"an agent."
""
".. warning:: "
" If set, the task will be **terminated** if the ram in "
" use by the process exceeds this value.")
hidden = db.Column(
db.Boolean,
default=False,
nullable=False,
doc="If True, keep the job hidden from the queue and web "
"ui. This is typically set to True if you either want "
"to save a job for later viewing or if the jobs data "
"is being populated in a deferred manner.")
environ = db.Column(
JSONDict,
doc="Dictionary containing information about the environment "
"in which the job will execute. "
""
".. note::"
" Changes made directly to this object are **not** "
" applied to the session.")
data = db.Column(JSONDict,
doc="Json blob containing additional data for a job "
""
".. note:: "
" Changes made directly to this object are **not** "
" applied to the session.")
to_be_deleted = db.Column(
db.Boolean,
nullable=False,
default=False,
doc="If true, the master will stop all running tasks for "
"this job and then delete it.")
completion_notify_sent = db.Column(
db.Boolean,
nullable=False,
default=False,
doc="Whether or not the finish notification mail has already "
"been sent out.")
autodelete_time = db.Column(
db.Integer,
nullable=True,
default=None,
doc="If not None, this job will be automatically deleted this "
"number of seconds after it finishes.")
#
# Relationships
#
queue = db.relationship("JobQueue",
backref=db.backref("jobs", lazy="dynamic"),
doc="The queue for this job")
group = db.relationship("JobGroup",
backref=db.backref("jobs", lazy="dynamic"),
doc="The job group this job belongs to")
user = db.relationship("User",
backref=db.backref("jobs", lazy="dynamic"),
doc="The owner of this job")
# self-referential many-to-many relationship
parents = db.relationship("Job",
secondary=JobDependency,
primaryjoin=id == JobDependency.c.childid,
secondaryjoin=id == JobDependency.c.parentid,
backref="children")
notified_users = db.relationship("JobNotifiedUser",
lazy="dynamic",
backref=db.backref("job"),
cascade="all,delete")
tasks_queued = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == None) & "
"(Task.job_id == Job.id)",
doc="Relationship between this job and any :class:`Task` "
"objects which are queued.")
tasks_running = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.RUNNING,
doc="Relationship between this job and any :class:`Task` "
"objects which are running.")
tasks_done = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.DONE,
doc="Relationship between this job and any :class:`Task` objects "
"which are done.")
tasks_failed = db.relationship(
"Task",
lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.FAILED,
doc="Relationship between this job and any :class:`Task` objects "
"which have failed.")
# resource relationships
tags = db.relationship(
"Tag",
backref="jobs",
lazy="dynamic",
secondary=JobTagAssociation,
doc="Relationship between this job and :class:`.Tag` objects")
def paused(self):
return self.state == WorkState.PAUSED
def update_state(self):
# Import here instead of at the top of the file to avoid a circular
# import
from pyfarm.scheduler.tasks import send_job_completion_mail
from pyfarm.models.agent import Agent
num_active_tasks = db.session.query(Task).\
filter(Task.job == self,
or_(Task.state == None, and_(
Task.state != WorkState.DONE,
Task.state != WorkState.FAILED))).count()
if num_active_tasks == 0:
num_failed_tasks = db.session.query(Task).filter(
Task.job == self, Task.state == WorkState.FAILED).count()
if num_failed_tasks == 0:
if self.state != _WorkState.DONE:
logger.info(
"Job %r (id %s): state transition %r -> 'done'",
self.title, self.id, self.state)
self.state = WorkState.DONE
send_job_completion_mail.apply_async(args=[self.id, True],
countdown=5)
else:
if self.state != _WorkState.FAILED:
logger.info(
"Job %r (id %s): state transition %r -> "
"'failed'", self.title, self.id, self.state)
self.state = WorkState.FAILED
send_job_completion_mail.apply_async(args=[self.id, False],
countdown=5)
db.session.add(self)
elif self.state != _WorkState.PAUSED:
num_running_tasks = db.session.query(Task).\
filter(Task.job == self,
Task.agent_id != None,
Task.agent.has(and_(Agent.state != AgentState.OFFLINE,
Agent.state != AgentState.DISABLED)),
or_(
Task.state == WorkState.RUNNING,
Task.state == None)).count()
if num_running_tasks == 0:
logger.debug(
"No running tasks in job %s (id %s), setting it "
"to queued", self.title, self.id)
self.state = None
db.session.add(self)
elif self.state != _WorkState.RUNNING:
self.state = WorkState.RUNNING
# Methods used by the scheduler
def num_assigned_agents(self):
# Import here instead of at the top of the file to avoid circular import
from pyfarm.models.agent import Agent
# Optimization: Blindly assume that we have no agents assigned if not
# running
if self.state != _WorkState.RUNNING:
return 0
try:
return self.assigned_agents_count
except AttributeError:
self.assigned_agents_count =\
db.session.query(distinct(Task.agent_id)).\
filter(Task.job == self,
Task.agent_id != None,
or_(Task.state == None,
Task.state == WorkState.RUNNING),
Task.agent.has(
and_(Agent.state != AgentState.OFFLINE,
Agent.state != AgentState.DISABLED)))\
.count()
return self.assigned_agents_count
def clear_assigned_counts(self):
try:
del self.assigned_agents_count
except AttributeError:
pass
if self.queue:
self.queue.clear_assigned_counts()
def can_use_more_agents(self):
# Import here instead of at the top of the file to avoid circular import
from pyfarm.models.agent import Agent
unassigned_tasks = Task.query.filter(
Task.job == self,
or_(Task.state == None,
~Task.state.in_([WorkState.DONE, WorkState.FAILED])),
or_(
Task.agent == None,
Task.agent.has(
Agent.state.in_([AgentState.OFFLINE,
AgentState.DISABLED])))).count()
return unassigned_tasks > 0
def get_batch(self, agent):
# Import here instead of at the top of the file to avoid circular import
from pyfarm.models.agent import Agent
tasks_query = Task.query.filter(
Task.job == self,
~Task.failed_in_agents.any(id=agent.id),
or_(Task.state == None,
~Task.state.in_([WorkState.DONE, WorkState.FAILED])),
or_(Task.agent == None,
Task.agent.has(Agent.state.in_(
[AgentState.OFFLINE, AgentState.DISABLED])))).\
order_by("frame asc, tile asc")
batch = []
for task in tasks_query:
if (len(batch) < self.batch and len(batch) <
(self.jobtype_version.max_batch or maxsize)
and (not self.jobtype_version.batch_contiguous or
(len(batch) == 0
or batch[-1].frame + self.by == task.frame))):
batch.append(task)
return batch
def alter_frame_range(self, start, end, by):
# We have to import this down here instead of at the top to break a
# circular dependency between the modules
from pyfarm.scheduler.tasks import delete_task
if end < start:
raise ValueError("`end` must be greater than or equal to `start`")
self.by = by
required_frames = []
current_frame = start
while current_frame <= end:
required_frames.append(current_frame)
current_frame += by
existing_tasks = Task.query.filter_by(job=self).all()
frames_to_create = required_frames
num_created = 0
for task in existing_tasks:
if task.frame not in required_frames:
delete_task.delay(task.id)
else:
frames_to_create.remove(task.frame)
for frame in frames_to_create:
if self.num_tiles:
for tile in range_(self.num_tiles - 1):
num_created += 1
task = Task()
task.job = self
task.frame = frame
task.tile = tile
task.priority = self.priority
db.session.add(task)
else:
num_created += 1
task = Task()
task.job = self
task.frame = frame
task.priority = self.priority
db.session.add(task)
if frames_to_create:
if self.state != WorkState.RUNNING:
self.state = None
if config.get("enable_statistics"):
task_event_count = TaskEventCount(num_new=num_created,
job_queue_id=self.job_queue_id)
task_event_count.time_start = datetime.utcnow()
task_event_count.time_end = datetime.utcnow()
db.session.add(task_event_count)
def rerun(self):
"""
Makes this job rerun all its task. Tasks that are currently running are
left untouched.
"""
num_restarted = 0
for task in self.tasks:
if task.state != _WorkState.RUNNING and task.state is not None:
task.state = None
task.agent = None
task.failures = 0
db.session.add(task)
num_restarted += 1
self.completion_notify_sent = False
self.update_state()
db.session.add(self)
if config.get("enable_statistics"):
task_event_count = TaskEventCount(job_queue_id=self.job_queue_id,
num_restarted=num_restarted)
task_event_count.time_start = datetime.utcnow()
task_event_count.time_end = datetime.utcnow()
db.session.add(task_event_count)
db.session.commit()
for child in self.children:
child.rerun()
def rerun_failed(self):
"""
Makes this job rerun all its failed tasks. Tasks that are done or are
currently running are left untouched
"""
num_restarted = 0
for task in self.tasks:
if task.state == _WorkState.FAILED:
task.state = None
task.agent = None
task.failures = 0
db.session.add(task)
num_restarted += 1
self.completion_notify_sent = False
self.update_state()
db.session.add(self)
if config.get("enable_statistics"):
task_event_count = TaskEventCount(job_queue_id=self.job_queue_id,
num_restarted=num_restarted)
task_event_count.time_start = datetime.utcnow()
task_event_count.time_end = datetime.utcnow()
db.session.commit()
for child in self.children:
child.rerun_failed()
@validates("ram", "cpus")
def validate_resource(self, key, value):
"""
Validation that ensures that the value provided for either
:attr:`.ram` or :attr:`.cpus` is a valid value with a given range
"""
assert isinstance(value, int), "%s must be an integer" % key
min_value = config.get("agent_min_%s" % key)
max_value = config.get("agent_max_%s" % key)
# check the provided input
if min_value > value or value > max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
@validates("progress")
def validate_progress(self, key, value):
if value < 0.0 or value > 1.0:
raise ValueError("Progress must be between 0.0 and 1.0")
class Agent(db.Model, ValidatePriorityMixin, UtilityMixins, ReprMixin):
"""
Stores information about an agent include its network address,
state, allocation configuration, etc.
.. note::
This table enforces two forms of uniqueness. The :attr:`id` column
must be unique and the combination of these columns must also be
unique to limit the frequency of duplicate data:
* :attr:`hostname`
* :attr:`ip`
* :attr:`port`
"""
__tablename__ = TABLE_AGENT
__table_args__ = (UniqueConstraint("hostname", "ip", "port"), )
STATE_DEFAULT = "online"
REPR_COLUMNS = (
"id", "hostname", "state", "ip", "remote_ip", "port", "cpus",
"ram", "free_ram")
REPR_CONVERT_COLUMN = {
"ip": repr_ip,
"remote_ip": repr_ip,
"state": repr}
MIN_PORT = read_env_int("PYFARM_AGENT_MIN_PORT", 1024)
MAX_PORT = read_env_int("PYFARM_AGENT_MAX_PORT", 65535)
MIN_CPUS = read_env_int("PYFARM_AGENT_MIN_CPUS", 1)
MAX_CPUS = read_env_int("PYFARM_AGENT_MAX_CPUS", 256)
MIN_RAM = read_env_int("PYFARM_AGENT_MIN_RAM", 16)
MAX_RAM = read_env_int("PYFARM_AGENT_MAX_RAM", 262144)
# quick check of the configured data
assert MIN_PORT >= 1, "$PYFARM_AGENT_MIN_PORT must be > 0"
assert MAX_PORT >= 1, "$PYFARM_AGENT_MAX_PORT must be > 0"
assert MAX_PORT >= MIN_PORT, "MIN_PORT must be <= MAX_PORT"
assert MIN_CPUS >= 1, "$PYFARM_AGENT_MIN_CPUS must be > 0"
assert MAX_CPUS >= 1, "$PYFARM_AGENT_MAX_CPUS must be > 0"
assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS"
assert MIN_RAM >= 1, "$PYFARM_AGENT_MIN_RAM must be > 0"
assert MAX_RAM >= 1, "$PYFARM_AGENT_MAX_RAM must be > 0"
assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM"
id = id_column(IDTypeAgent)
# basic host attribute information
hostname = db.Column(db.String(MAX_HOSTNAME_LENGTH), nullable=False,
doc=dedent("""
The hostname we should use to talk to this host.
Preferably this value will be the fully qualified
name instead of the base hostname alone."""))
ip = db.Column(IPv4Address, nullable=True,
doc="The IPv4 network address this host resides on")
remote_ip = db.Column(IPv4Address, nullable=True,
doc="the remote address which came in with the "
"request")
use_address = db.Column(UseAgentAddressEnum, nullable=False,
default="remote",
doc="The address we should use when communicating "
"with the agent")
ram = db.Column(db.Integer, nullable=False,
doc="The amount of ram installed on the agent in megabytes")
free_ram = db.Column(db.Integer, nullable=False,
doc="The amount of ram which was last considered free")
cpus = db.Column(db.Integer, nullable=False,
doc="The number of cpus installed on the agent")
port = db.Column(db.Integer, nullable=False,
doc="The port the agent is currently running on")
time_offset = db.Column(db.Integer, nullable=False, default=0,
doc="the offset in seconds the agent is from "
"an official time server")
# host state
state = db.Column(AgentStateEnum, default=AgentState.ONLINE,
nullable=False,
doc=dedent("""
Stores the current state of the host. This value can be
changed either by a master telling the host to do
something with a task or from the host via REST api."""))
# Max allocation of the two primary resources which `1.0` is 100%
# allocation. For `cpu_allocation` 100% allocation typically means
# one task per cpu.
ram_allocation = db.Column(db.Float,
default=read_env_number(
"PYFARM_AGENT_RAM_ALLOCATION", .8),
doc=dedent("""
The amount of ram the agent is allowed to
allocate towards work. A value of 1.0 would
mean to let the agent use all of the memory
installed on the system when assigning work."""))
cpu_allocation = db.Column(db.Float,
default=read_env_number(
"PYFARM_AGENT_CPU_ALLOCATION", 1.0),
doc=dedent("""
The total amount of cpu space an agent is
allowed to process work in. A value of 1.0
would mean an agent can handle as much work
as the system could handle given the
requirements of a task. For example if an agent
has 8 cpus, cpu_allocation is .5, and a task
requires 4 cpus then only that task will run
on the system."""))
# relationships
tasks = db.relationship("Task", backref="agent", lazy="dynamic",
doc=dedent("""
Relationship between an :class:`Agent`
and any :class:`pyfarm.models.Task`
objects"""))
tags = db.relationship("Tag", secondary=AgentTagAssociation,
backref=db.backref("agents", lazy="dynamic"),
lazy="dynamic",
doc="Tags associated with this agent")
software = db.relationship("Software",
secondary=AgentSoftwareAssociation,
backref=db.backref("agents", lazy="dynamic"),
lazy="dynamic",
doc="software this agent has installed or is "
"configured for")
projects = db.relationship("Project",
secondary=AgentProjects,
backref=db.backref("agents", lazy="dynamic"),
lazy="dynamic",
doc="The project or projects this agent is "
"associated with. By default an agent "
"which is not associated with any projects "
"will be a member of all projects.")
@classmethod
def validate_hostname(cls, key, value):
"""
Ensures that the hostname provided by `value` matches a regular
expression that expresses what a valid hostname is.
"""
# ensure hostname does not contain characters we can't use
if not REGEX_HOSTNAME.match(value):
raise ValueError("%s is not valid for %s" % (value, key))
return value
@classmethod
def validate_resource(cls, key, value):
"""
Ensure the `value` provided for `key` is within an expected range as
specified in `agent.yml`
"""
min_value = getattr(cls, "MIN_%s" % key.upper())
max_value = getattr(cls, "MAX_%s" % key.upper())
# check the provided input
if min_value > value or value > max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
@classmethod
def validate_ip_address(cls, key, value):
"""
Ensures the :attr:`ip` address is valid. This checks to ensure
that the value provided is:
* not a hostmask
* not link local (:rfc:`3927`)
* not used for multicast (:rfc:`1112`)
* not a netmask (:rfc:`4632`)
* not reserved (:rfc:`6052`)
* a private address (:rfc:`1918`)
"""
if not value:
return
try:
ip = netaddr.IPAddress(value)
except (AddrFormatError, ValueError) as e:
raise ValueError(
"%s is not a valid address format: %s" % (value, e))
if not app.config.get("DEV_ALLOW_ANY_AGENT_ADDRESS", False):
if PYFARM_REQUIRE_PRIVATE_IP and not ip.is_private():
raise ValueError("%s is not a private ip address" % value)
if not app.config.get("DEV_ALLOW_ANY_AGENT_ADDRESS", False) and \
not all([
not ip.is_hostmask(), not ip.is_link_local(),
not ip.is_loopback(), not ip.is_multicast(),
not ip.is_netmask(), not ip.is_reserved()
]):
raise ValueError("%s is not a usable ip address" % value)
return value
@validates("ip")
def validate_address_column(self, key, value):
"""validates the ip column"""
return self.validate_ip_address(key, value)
@validates("hostname")
def validate_hostname_column(self, key, value):
"""validates the hostname column"""
return self.validate_hostname(key, value)
@validates("ram", "cpus", "port")
def validate_resource_column(self, key, value):
"""validates the ram, cpus, and port columns"""
return self.validate_resource(key, value)
def serialize_column(self, column):
"""serializes a single column, typically used by a dictionary mixin"""
if isinstance(column, IPAddress):
return str(column)
return column
class JobTypeVersion(db.Model, UtilityMixins, ReprMixin):
"""
Defines a specific jobtype version.
"""
__tablename__ = config.get("table_job_type_version")
__table_args__ = (UniqueConstraint("jobtype_id", "version"), )
REPR_COLUMNS = ("id", "jobtype_id", "version")
id = id_column(IDTypeWork)
jobtype_id = db.Column(IDTypeWork,
db.ForeignKey("%s.id" %
config.get("table_job_type")),
nullable=False,
doc="The jobtype this version belongs to")
version = db.Column(db.Integer, nullable=False, doc="The version number")
max_batch = db.Column(
db.Integer,
default=config.get("job_type_max_batch"),
doc="When the queue runs, this is the maximum number of tasks "
"that the queue can select to assign to a single"
"agent. If left empty, no maximum applies")
batch_contiguous = db.Column(
db.Boolean,
default=config.get("job_type_batch_contiguous"),
doc="If True then the queue will be forced to batch"
"numerically contiguous tasks only for this job type. "
"For example if True it would batch frames 1, 2, 3, 4 "
"together but not 2, 4, 6, 8. If this column is False "
"however the queue will batch non-contiguous tasks too.")
no_automatic_start_time = db.Column(
db.Boolean,
nullable=False,
default=False,
doc="If set, we will not automatically set `time_started_on` "
"for the tasks in jobs of this type when they are set "
"to `running`.")
supports_tiling = db.Column(
db.Boolean,
default=False,
doc="Whether or not the jobtype supports tiling, i.e. splitting single "
"frames into regions and then rendering those independently from "
"each other.")
classname = db.Column(
db.String(config.get("job_type_max_class_name_length")),
nullable=True,
doc="The name of the job class contained within the file being "
"loaded. This field may be null but when it's not provided "
"job type name will be used instead.")
code = db.Column(db.UnicodeText,
nullable=False,
doc="The source code of the job type")
#
# Relationships
#
jobtype = db.relationship("JobType",
backref=db.backref("versions",
lazy="dynamic",
cascade="all, delete-orphan"),
doc="Relationship between this version and the "
":class:`JobType` it belongs to"
"")
jobs = db.relationship("Job",
backref="jobtype_version",
lazy="dynamic",
doc="Relationship between this jobtype version and "
":class:`.Job` objects.")
@validates("max_batch")
def validate_max_batch(self, key, value):
if isinstance(value, int) and value < 1:
raise ValueError("max_batch must be greater than or equal to 1")
return value
@validates("version")
def validate_version(self, key, value):
if isinstance(value, int) and value < 1:
raise ValueError("version must be greater than or equal to 1")
return value
class JobQueue(db.Model, UtilityMixins, ReprMixin):
"""
Stores information about a job queue. Used for flexible, configurable
distribution of computing capacity to jobs.
"""
__tablename__ = config.get("table_job_queue")
__table_args__ = (UniqueConstraint("parent_jobqueue_id", "name"), )
REPR_COLUMNS = ("id", "name")
id = id_column(IDTypeWork)
parent_jobqueue_id = db.Column(
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job_queue")),
nullable=True,
doc="The parent queue of this queue. If NULL, this is a top "
"level queue.")
name = db.Column(db.String(config.get("max_queue_name_length")),
nullable=False,
doc="The name of the job queue")
minimum_agents = db.Column(
db.Integer,
nullable=True,
doc="The scheduler will try to assign at least this number of "
"agents to jobs in or below this queue as long as it "
"can use them, before any other considerations.")
maximum_agents = db.Column(
db.Integer,
nullable=True,
doc="The scheduler will never assign more than this number of "
"agents to jobs in or below this queue.")
priority = db.Column(
db.Integer,
nullable=False,
default=config.get("queue_default_priority"),
doc="The priority of this job queue. The scheduler will not "
"assign any nodes to other job queues or jobs with the "
"same parent and a lower priority as long as this one "
"can still use nodes. The minimum_agents column takes "
"precedence over this.")
weight = db.Column(db.Integer,
nullable=False,
default=config.get("queue_default_weight"),
doc="The weight of this job queue. The scheduler will "
"distribute available agents between jobs and job "
"queues in the same queue in proportion to their "
"weights.")
fullpath = db.Column(db.String(config.get("max_queue_path_length")),
doc="The path of this jobqueue. This column is a "
"database denormalization. It is technically "
"redundant, but faster to access than recursively "
"querying all parent queues. If set to NULL, the "
"path must be computed by recursively querying "
"the parent queues.")
#
# Relationship
#
parent = db.relationship("JobQueue",
remote_side=[id],
backref=db.backref("children", lazy="dynamic"),
doc="Relationship between this queue its parent")
def path(self):
# Import here instead of at the top to break circular dependency
from pyfarm.scheduler.tasks import cache_jobqueue_path
if self.fullpath:
return self.fullpath
else:
cache_jobqueue_path.delay(self.id)
path = "/%s" % (self.name or "")
if self.parent:
return self.parent.path() + path
else:
return path
def child_queues_sorted(self):
"""
Return child queues sorted by number of currently assigned agents with
priority as a secondary sort key.
"""
queues = [x for x in self.children]
return sorted(queues,
key=lambda x: x.num_assigned_agents(),
reverse=True)
def child_jobs(self, filters):
# Import down here instead of at the top to avoid circular import
from pyfarm.models.job import Job
jobs_query = Job.query
if self.id:
jobs_query = jobs_query.filter_by(queue=self)
wanted_states = []
if filters["state_paused"]:
wanted_states.append(WorkState.PAUSED)
if filters["state_running"]:
wanted_states.append(WorkState.RUNNING)
if filters["state_done"]:
wanted_states.append(WorkState.DONE)
if filters["state_failed"]:
wanted_states.append(WorkState.FAILED)
if filters["state_queued"]:
jobs_query = jobs_query.filter(
or_(Job.state == None, Job.state.in_(wanted_states)))
else:
jobs_query = jobs_query.filter(Job.state.in_(wanted_states))
return sorted(jobs_query.all(),
key=lambda x: x.num_assigned_agents(),
reverse=True)
def num_assigned_agents(self):
try:
return self.assigned_agents_count
except AttributeError:
# Import down here instead of at the top to avoid circular import
from pyfarm.models.task import Task
from pyfarm.models.job import Job
self.assigned_agents_count = 0
for queue in self.children:
self.assigned_agents_count += queue.num_assigned_agents()
self.assigned_agents_count +=\
db.session.query(distinct(Task.agent_id)).\
filter(Task.job.has(Job.queue == self),
Task.agent_id != None,
Task.agent.has(
and_(Agent.state != AgentState.OFFLINE,
Agent.state != AgentState.DISABLED)),
or_(Task.state == None,
Task.state == WorkState.RUNNING)).count()
return self.assigned_agents_count
def clear_assigned_counts(self):
try:
del self.assigned_agents_count
except AttributeError:
pass
if self.parent:
self.parent.clear_assigned_counts()
def get_job_for_agent(self, agent, unwanted_job_ids=None):
# Import down here instead of at the top to avoid circular import
from pyfarm.models.job import Job
supported_types = agent.get_supported_types()
if not supported_types:
return None
available_ram = agent.ram if USE_TOTAL_RAM else agent.free_ram
child_jobs = Job.query.filter(
or_(Job.state == WorkState.RUNNING, Job.state == None),
Job.job_queue_id == self.id, ~Job.parents.any(
or_(Job.state == None, Job.state != WorkState.DONE)),
Job.jobtype_version_id.in_(supported_types),
Job.ram <= available_ram).all()
child_jobs = [
x for x in child_jobs if (agent.satisfies_job_requirements(x)
and x.id not in unwanted_job_ids)
]
if unwanted_job_ids:
child_jobs = [
x for x in child_jobs if x.id not in unwanted_job_ids
]
child_queues = JobQueue.query.filter(
JobQueue.parent_jobqueue_id == self.id).all()
# Before anything else, enforce minimums
for job in child_jobs:
if job.state == _WorkState.RUNNING:
if (job.num_assigned_agents() < (job.minimum_agents or 0)
and job.num_assigned_agents() <
(job.maximum_agents or maxsize)
and job.can_use_more_agents()):
return job
elif job.minimum_agents and job.minimum_agents > 0:
return job
for queue in child_queues:
if (queue.num_assigned_agents() < (queue.minimum_agents or 0)
and queue.num_assigned_agents() <
(queue.maximum_agents or maxsize)):
job = queue.get_job_for_agent(agent, unwanted_job_ids)
if job:
return job
objects_by_priority = {}
for queue in child_queues:
if queue.priority in objects_by_priority:
objects_by_priority[queue.priority] += [queue]
else:
objects_by_priority[queue.priority] = [queue]
for job in child_jobs:
if job.priority in objects_by_priority:
objects_by_priority[job.priority] += [job]
else:
objects_by_priority[job.priority] = [job]
available_priorities = sorted(objects_by_priority.keys(), reverse=True)
# Work through the priorities in descending order
for priority in available_priorities:
objects = objects_by_priority[priority]
active_objects = [
x for x in objects
if (type(x) != Job or x.state == _WorkState.RUNNING)
]
weight_sum = reduce(lambda a, b: a + b.weight, active_objects, 0)
total_assigned = reduce(lambda a, b: a + b.num_assigned_agents(),
objects, 0)
objects.sort(key=(lambda x: ((float(x.num_assigned_agents(
)) / total_assigned) if total_assigned else 0) / ((float(
x.weight) / weight_sum) if weight_sum and x.weight else 1)))
selected_job = None
for item in objects:
if isinstance(item, Job):
if item.state == _WorkState.RUNNING:
if (item.can_use_more_agents()
and item.num_assigned_agents() <
(item.maximum_agents or maxsize)):
if PREFER_RUNNING_JOBS:
return item
elif (selected_job is None
or selected_job.time_submitted >
item.time_submitted):
selected_job = item
elif (selected_job is None or
selected_job.time_submitted > item.time_submitted):
# If this job is not running yet, remember it, but keep
# looking for already running or queued but older jobs
selected_job = item
if isinstance(item, JobQueue):
if (item.num_assigned_agents() <
(item.maximum_agents or maxsize)):
job = item.get_job_for_agent(agent, unwanted_job_ids)
if job:
return job
if selected_job:
return selected_job
return None
@staticmethod
def top_level_unique_check(mapper, connection, target):
if target.parent_jobqueue_id is None:
count = JobQueue.query.filter_by(parent_jobqueue_id=None,
name=target.name).count()
if count > 0:
raise ValueError("Cannot have two jobqueues named %r at the "
"top level" % target.name)
class User(db.Model, UserMixin, ReprMixin):
"""
Stores information about a user including the roles they belong to
"""
__tablename__ = TABLE_USERS_USER
REPR_COLUMNS = ("id", "username")
id = db.Column(db.Integer, primary_key=True, nullable=False)
active = db.Column(db.Boolean, default=True,
doc=dedent("""
Enables or disables a particular user across the
entire system"""))
username = db.Column(
db.String(MAX_USERNAME_LENGTH), unique=True, nullable=False,
doc="The username used to login.")
password = db.Column(db.String(SHA256_ASCII_LENGTH),
doc="The password used to login")
email = db.Column(db.String(MAX_EMAILADDR_LENGTH), unique=True,
doc=dedent("""
Contact email for registration and possible
notifications"""))
expiration = db.Column(db.DateTime,
doc=dedent("""
User expiration. If this value is set then the user
will no longer be able to access PyFarm past the
expiration."""))
onetime_code = db.Column(db.String(SHA256_ASCII_LENGTH),
doc=dedent("""
SHA256 one time use code which can be used for
unique urls such as for password resets."""))
last_login = db.Column(db.DateTime,
doc=dedent("""
The last date that this user was logged in."""))
roles = db.relationship("Role", secondary=UserRoles,
backref=db.backref("users", lazy="dynamic"))
projects = db.relationship("Project",
secondary=UserProjects,
backref=db.backref("users", lazy="dynamic"),
lazy="dynamic",
doc="The project or projects this user is "
"associated with. By default a user "
"which is not associated with any projects "
"will be a member of all projects.")
@classmethod
def create(cls, username, password, email=None, roles=None):
# create the list or roles to add
if roles is None:
roles = []
elif isinstance(roles, STRING_TYPES):
roles = [roles]
# create the user with the proper initial values
user = cls(
username=username,
password=cls.hash_password(password),
email=email)
user.roles.extend(map(Role.create, roles))
# commit and return
db.session.add(user)
db.session.commit()
return user
@classmethod
def get(cls, id_or_username):
"""Get a user model either by id or by the user's username"""
try:
id_or_username = int(id_or_username)
except ValueError:
pass
if isinstance(id_or_username, int):
return cls.query.filter_by(id=id_or_username).first()
elif isinstance(id_or_username, STRING_TYPES):
return cls.query.filter_by(username=id_or_username).first()
else:
raise TypeError("string or integer required for User.get()")
@classmethod
def hash_password(cls, value):
value = app.secret_key + value
if PY3:
value = value.encode("utf-8")
return sha256(value).hexdigest()
def get_auth_token(self):
return login_serializer.dumps([str(self.id), self.password])
def get_id(self):
return self.id
def check_password(self, password):
"""checks the password provided against the stored password"""
assert isinstance(password, STRING_TYPES)
return self.hash_password(password) == self.password
def is_active(self):
"""returns true if the user and the roles it belongs to are active"""
logger.debug("%(self)s.is_active()" % locals())
now = datetime.now()
# user is not active
if not self.active:
return False
# user has expired
if self.expiration is not None and now > self.expiration:
return False
# TODO: there's probably some way to cache this information
return all(role.is_active() for role in self.roles)
def has_roles(self, allowed=None, required=None):
"""checks the provided arguments against the roles assigned"""
if not allowed and not required:
return True
allowed = split_and_extend(allowed)
required = split_and_extend(required)
logger.debug(
"%(self)s.has_roles(allowed=%(allowed)s, required=%(required)s)"
% locals())
if allowed:
# Ask the database if the user has any of the allowed roles. For
# smaller numbers of roles this is very slightly slower with
# SQLite in :memory: but is a good amount faster over the network
# or with large role sets.
return bool(
User.query.filter(
User.roles.any(
Role.name.in_(allowed))
).filter(User.id == self.id).count())
if required:
# Ask the database for all roles matching ``required``. In order
# for this to return True is the number of entries found must
# be equal to len(required).
count = Role.query.filter(
Role.name.in_(required)).filter(User.id == self.id).count()
return count == len(required)
class Job(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, ReprMixin):
"""
Defines the attributes and environment for a job. Individual commands
are kept track of by |Task|
"""
__tablename__ = TABLE_JOB
REPR_COLUMNS = ("id", "state", "project")
REPR_CONVERT_COLUMN = {
"state": repr}
MIN_CPUS = read_env_int("PYFARM_QUEUE_MIN_CPUS", 1)
MAX_CPUS = read_env_int("PYFARM_QUEUE_MAX_CPUS", 256)
MIN_RAM = read_env_int("PYFARM_QUEUE_MIN_RAM", 16)
MAX_RAM = read_env_int("PYFARM_QUEUE_MAX_RAM", 262144)
SPECIAL_RAM = read_env("PYFARM_AGENT_SPECIAL_RAM", [0], eval_literal=True)
SPECIAL_CPUS = read_env("PYFARM_AGENT_SPECIAL_CPUS", [0], eval_literal=True)
# quick check of the configured data
assert MIN_CPUS >= 1, "$PYFARM_QUEUE_MIN_CPUS must be > 0"
assert MAX_CPUS >= 1, "$PYFARM_QUEUE_MAX_CPUS must be > 0"
assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS"
assert MIN_RAM >= 1, "$PYFARM_QUEUE_MIN_RAM must be > 0"
assert MAX_RAM >= 1, "$PYFARM_QUEUE_MAX_RAM must be > 0"
assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM"
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(WorkState.QUEUED, "job.priority")
project_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT),
doc="stores the project id")
job_type_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB_TYPE),
nullable=False,
doc=dedent("""
The foreign key which stores :class:`JobType.id`"""))
user = db.Column(db.String(MAX_USERNAME_LENGTH),
doc=dedent("""
The user this job should execute as. The agent
process will have to be running as root on platforms
that support setting the user id.
.. note::
The length of this field is limited by the
configuration value `job.max_username_length`
.. warning::
this may not behave as expected on all platforms
(windows in particular)"""))
notes = db.Column(db.Text, default="",
doc=dedent("""
Notes that are provided on submission or added after
the fact. This column is only provided for human
consumption is not scanned, index, or used when
searching"""))
# task data
cmd = db.Column(db.String(MAX_COMMAND_LENGTH),
doc=dedent("""
The platform independent command to run. Each agent will
resolve this value for itself when the task begins so a
command like `ping` will work on any platform it's
assigned to. The full command could be provided here,
but then the job must be tagged using
:class:`.JobSoftware` to limit which agent(s) it will
run on."""))
start = db.Column(db.Float,
doc=dedent("""
The first frame of the job to run. This value may
be a float so subframes can be processed."""))
end = db.Column(db.Float,
doc=dedent("""
The last frame of the job to run. This value may
be a float so subframes can be processed."""))
by = db.Column(db.Float, default=1,
doc=dedent("""
The number of frames to count by between `start` and
`end`. This column may also sometimes be referred to
as 'step' by other software."""))
batch = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_BATCH", 1),
doc=dedent("""
Number of tasks to run on a single agent at once.
Depending on the capabilities of the software being run
this will either cause a single process to execute on
the agent or multiple processes on after the other.
**configured by**: `job.batch`"""))
requeue = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_REQUEUE", 3),
doc=dedent("""
Number of times to requeue failed tasks
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, never requeue failed tasks
-1, requeue failed tasks indefinitely
**configured by**: `job.requeue`"""))
cpus = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_CPUS", 1),
doc=dedent("""
Number of cpus or threads each task should consume on
each agent. Depending on the job type being executed
this may result in additional cpu consumption, longer
wait times in the queue (2 cpus means 2 'fewer' cpus on
an agent), or all of the above.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, minimum number of cpu resources not required
-1, agent cpu is exclusive for a task from this job
**configured by**: `job.cpus`"""))
ram = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_RAM", 32),
doc=dedent("""
Amount of ram a task from this job will require to be
free in order to run. A task exceeding this value will
not result in any special behavior.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, minimum amount of free ram not required
-1, agent ram is exclusive for a task from this job
**configured by**: `job.ram`"""))
ram_warning = db.Column(db.Integer, default=-1,
doc=dedent("""
Amount of ram used by a task before a warning
raised. A task exceeding this value will not
cause any work stopping behavior.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
-1, not set"""))
ram_max = db.Column(db.Integer, default=-1,
doc=dedent("""
Maximum amount of ram a task is allowed to consume on
an agent.
.. warning::
The task will be **terminated** if the ram in use
by the process exceeds this value.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
-1, not set
"""))
attempts = db.Column(db.Integer,
doc=dedent("""
The number attempts which have been made on this
task. This value is auto incremented when
:attr:`state` changes to a value synonyms with a
running state."""))
hidden = db.Column(db.Boolean, default=False, nullable=False,
doc=dedent("""
If True, keep the job hidden from the queue and web
ui. This is typically set to True if you either want
to save a job for later viewing or if the jobs data
is being populated in a deferred manner."""))
environ = db.Column(JSONDict,
doc=dedent("""
Dictionary containing information about the environment
in which the job will execute.
.. note::
Changes made directly to this object are **not**
applied to the session."""))
args = db.Column(JSONList,
doc=dedent("""
List containing the command line arguments.
.. note::
Changes made directly to this object are **not**
applied to the session."""))
data = db.Column(JSONDict,
doc=dedent("""
Json blob containing additional data for a job
.. note::
Changes made directly to this object are **not**
applied to the session."""))
project = db.relationship("Project",
backref=db.backref("jobs", lazy="dynamic"),
doc=dedent("""
relationship attribute which retrieves the
associated project for the job"""))
# self-referential many-to-many relationship
parents = db.relationship("Job",
secondary=JobDependencies,
primaryjoin=id==JobDependencies.c.parentid,
secondaryjoin=id==JobDependencies.c.childid,
backref="children")
tasks_done = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.DONE,
doc=dedent("""
Relationship between this job and any |Task| objects which are
done."""))
tasks_failed = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.FAILED,
doc=dedent("""
Relationship between this job and any |Task| objects which have
failed."""))
tasks_queued = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.QUEUED,
doc=dedent("""
Relationship between this job and any |Task| objects which
are queued."""))
# resource relationships
tags = db.relationship("Tag", backref="jobs", lazy="dynamic",
secondary=JobTagAssociation,
doc=dedent("""
Relationship between this job and
:class:`.Tag` objects"""))
software = db.relationship("Software",
secondary=JobSoftwareDependency,
backref=db.backref("jobs", lazy="dynamic"),
lazy="dynamic",
doc="software needed by this job")
@validates("ram", "cpus")
def validate_resource(self, key, value):
"""
Validation that ensures that the value provided for either
:attr:`.ram` or :attr:`.cpus` is a valid value with a given range
"""
key_upper = key.upper()
special = getattr(self, "SPECIAL_%s" % key_upper)
if value is None or value in special:
return value
min_value = getattr(self, "MIN_%s" % key_upper)
max_value = getattr(self, "MAX_%s" % key_upper)
# quick sanity check of the incoming config
assert isinstance(min_value, int), "db.min_%s must be an integer" % key
assert isinstance(max_value, int), "db.max_%s must be an integer" % key
assert min_value >= 1, "db.min_%s must be > 0" % key
assert max_value >= 1, "db.max_%s must be > 0" % key
# check the provided input
if min_value > value or value > max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
class Agent(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin,
UtilityMixins, ReprMixin):
"""
Stores information about an agent include its network address,
state, allocation configuration, etc.
.. note::
This table enforces two forms of uniqueness. The :attr:`id` column
must be unique and the combination of these columns must also be
unique to limit the frequency of duplicate data:
* :attr:`hostname`
* :attr:`port`
* :attr:`id`
"""
__tablename__ = config.get("table_agent")
__table_args__ = (UniqueConstraint("hostname", "port", "id"), )
STATE_ENUM = AgentState
STATE_DEFAULT = "online"
REPR_COLUMNS = (
"id", "hostname", "port", "state", "remote_ip",
"cpus", "ram", "free_ram")
REPR_CONVERT_COLUMN = {"remote_ip": repr_ip}
URL_TEMPLATE = config.get("agent_api_url_template")
MIN_PORT = config.get("agent_min_port")
MAX_PORT = config.get("agent_max_port")
MIN_CPUS = config.get("agent_min_cpus")
MAX_CPUS = config.get("agent_max_cpus")
MIN_RAM = config.get("agent_min_ram")
MAX_RAM = config.get("agent_max_ram")
# quick check of the configured data
assert MIN_PORT >= 1, "`agent_min_port` must be > 0"
assert MAX_PORT >= 1, "`agent_max_port` must be > 0"
assert MAX_PORT >= MIN_PORT, "MIN_PORT must be <= MAX_PORT"
assert MIN_CPUS >= 1, "`agent_min_cpus` must be > 0"
assert MAX_CPUS >= 1, "`agent_max_cpus` must be > 0"
assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS"
assert MIN_RAM >= 1, "`agent_min_ram` must be > 0"
assert MAX_RAM >= 1, "`agent_max_ram` must be > 0"
assert MAX_RAM >= MIN_RAM, "`agent_min_ram` must be <= `agent_max_ram`"
id = id_column(IDTypeAgent, default=uuid.uuid4, autoincrement=False)
# basic host attribute information
hostname = db.Column(
db.String(config.get("max_hostname_length")),
nullable=False,
doc="The hostname we should use to talk to this host. "
"Preferably this value will be the fully qualified "
"name instead of the base hostname alone.")
notes = db.Column(
db.Text,
default="",
doc="Free form notes about this agent")
remote_ip = db.Column(
IPv4Address, nullable=True,
doc="the remote address which came in with the request")
use_address = db.Column(
UseAgentAddressEnum,
nullable=False, default=UseAgentAddress.REMOTE,
doc="The address we should use when communicating with the agent")
# TODO Make non-nullable later
os_class = db.Column(
OperatingSystemEnum,
doc="The type of operating system running on the "
"agent; 'linux', 'windows', or 'mac'.")
os_fullname = db.Column(
db.String(config.get("max_osname_length")),
doc="The full human-readable name of the agent's OS, as returned "
"by platform.platform()")
ram = db.Column(
db.Integer,
nullable=False,
doc="The amount of ram installed on the agent in megabytes")
free_ram = db.Column(
db.Integer,
nullable=False,
doc="The amount of ram which was last considered free")
cpus = db.Column(
db.Integer,
nullable=False,
doc="The number of logical CPU cores installed on the agent")
cpu_name = db.Column(
db.String(config.get("max_cpuname_length")),
doc="The make and model of CPUs in this agents")
port = db.Column(
db.Integer,
nullable=False,
doc="The port the agent is currently running on")
time_offset = db.Column(
db.Integer,
nullable=False, default=0,
doc="The offset in seconds the agent is from an official time server")
version = db.Column(
db.String(16),
nullable=True,
doc="The pyfarm version number this agent is running.")
upgrade_to = db.Column(
db.String(16),
nullable=True,
doc="The version this agent should upgrade to.")
restart_requested = db.Column(
db.Boolean,
default=False, nullable=False,
doc="If True, the agent will be restarted")
# host state
state = db.Column(
AgentStateEnum,
default=AgentState.ONLINE, nullable=False,
doc="Stores the current state of the host. This value can be "
"changed either by a master telling the host to do "
"something with a task or from the host via REST api.")
last_heard_from = db.Column(
db.DateTime,
default=datetime.utcnow,
doc="Time we last had contact with this agent")
last_success_on = db.Column(
db.DateTime,
nullable=True,
doc="The last time this agent has set a task to `done`")
last_polled = db.Column(
db.DateTime,
doc="Time we last tried to contact the agent")
# Max allocation of the two primary resources which `1.0` is 100%
# allocation. For `cpu_allocation` 100% allocation typically means
# one task per cpu.
ram_allocation = db.Column(
db.Float,
default=config.get("agent_ram_allocation"),
doc="The amount of ram the agent is allowed to allocate "
"towards work. A value of 1.0 would mean to let the "
"agent use all of the memory installed on the system "
"when assigning work.")
cpu_allocation = db.Column(
db.Float,
default=config.get("agent_cpu_allocation"),
doc="The total amount of cpu space an agent is allowed to "
"process work in. A value of 1.0 would mean an agent "
"can handle as much work as the system could handle "
"given the requirements of a task. For example if "
"an agent has 8 cpus, cpu_allocation is .5, and a "
"task requires 4 cpus then only that task will "
"run on the system.")
#
# Relationships
#
tasks = db.relationship(
"Task",
backref="agent", lazy="dynamic",
doc="Relationship between an :class:`Agent` and any "
":class:`pyfarm.models.Task` objects")
tags = db.relationship(
"Tag",
secondary=AgentTagAssociation,
backref=db.backref("agents", lazy="dynamic"),
lazy="dynamic",
doc="Tags associated with this agent")
software_versions = db.relationship(
"SoftwareVersion",
secondary=AgentSoftwareVersionAssociation,
backref=db.backref("agents", lazy="dynamic"),
lazy="dynamic",
doc="software this agent has installed or is configured for")
mac_addresses = db.relationship(
"AgentMacAddress", backref="agent",
lazy="dynamic",
doc="The MAC addresses this agent has",
cascade="save-update, merge, delete, delete-orphan")
gpus = db.relationship(
"GPU",
secondary=GPUInAgent,
backref=db.backref("agents", lazy="dynamic"),
lazy="dynamic",
doc="The graphics cards that are installed in this agent")
disks = db.relationship(
"AgentDisk",
backref=db.backref("agent"),
lazy="dynamic",
doc="The known disks available to this agent",
cascade="save-update, merge, delete, delete-orphan")
failed_tasks = db.relationship(
"Task",
secondary=FailedTaskInAgent,
backref=db.backref("failed_in_agents", lazy="dynamic"),
lazy="dynamic",
doc="The tasks this agents failed to execute")
def is_offline(self):
return self.state == AgentState.OFFLINE
def is_disabled(self):
return self.state == AgentState.DISABLED
def get_supported_types(self):
try:
return self.support_jobtype_versions
except AttributeError:
jobtype_versions_query = JobTypeVersion.query.filter(
JobTypeVersion.jobs.any(
or_(Job.state == None, Job.state == WorkState.RUNNING)))
self.support_jobtype_versions = []
for jobtype_version in jobtype_versions_query:
if self.satisfies_jobtype_requirements(jobtype_version):
self.support_jobtype_versions.append(jobtype_version.id)
return self.support_jobtype_versions
def satisfies_jobtype_requirements(self, jobtype_version):
requirements_to_satisfy = list(jobtype_version.software_requirements)
for software_version in self.software_versions:
for requirement in list(requirements_to_satisfy):
if (software_version.software == requirement.software and
(requirement.min_version == None or
requirement.min_version.rank <= software_version.rank) and
(requirement.max_version == None or
requirement.max_version.rank >= software_version.rank)):
requirements_to_satisfy.remove(requirement)
return len(requirements_to_satisfy) == 0
def satisfies_job_requirements(self, job):
if not self.satisfies_jobtype_requirements(job.jobtype_version):
return False
if self.cpus < job.cpus:
return False
if self.free_ram < job.ram:
return False
for tag_requirement in job.tag_requirements:
if (not tag_requirement.negate and
tag_requirement.tag not in self.tags):
return False
if (tag_requirement.negate and
tag_requirement.tag in self.tags):
return False
return True
@classmethod
def validate_hostname(cls, key, value):
"""
Ensures that the hostname provided by `value` matches a regular
expression that expresses what a valid hostname is.
"""
# ensure hostname does not contain characters we can't use
if not REGEX_HOSTNAME.match(value):
raise ValueError("%s is not valid for %s" % (value, key))
return value
@classmethod
def validate_resource(cls, key, value):
"""
Ensure the ``value`` provided for ``key`` is within an expected
range. This classmethod retrieves the min and max values from
the :class:`Agent` class directory using:
>>> min_value = getattr(Agent, "MIN_%s" % key.upper())
>>> max_value = getattr(Agent, "MAX_%s" % key.upper())
"""
min_value = getattr(cls, "MIN_%s" % key.upper())
max_value = getattr(cls, "MAX_%s" % key.upper())
# check the provided input
if not min_value <= value <= max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
@classmethod
def validate_ipv4_address(cls, _, value):
"""
Ensures the :attr:`ip` address is valid. This checks to ensure
that the value provided is:
* not a hostmask
* not link local (:rfc:`3927`)
* not used for multicast (:rfc:`1112`)
* not a netmask (:rfc:`4632`)
* not reserved (:rfc:`6052`)
* a private address (:rfc:`1918`)
"""
if value is None:
return value
try:
address = IPAddress(value)
except (AddrFormatError, ValueError) as e:
raise ValueError(
"%s is not a valid address format: %s" % (value, e))
if ALLOW_AGENT_LOOPBACK:
loopback = lambda: False
else:
loopback = address.is_loopback
if any([address.is_hostmask(), address.is_link_local(),
loopback(), address.is_multicast(),
address.is_netmask(), address.is_reserved()]):
raise ValueError("%s is not a valid address type" % value)
return value
def api_url(self):
"""
Returns the base url which should be used to access the api
of this specific agent.
:except ValueError:
Raised if this function is called while the agent's
:attr:`use_address` column is set to ``PASSIVE``
"""
if self.use_address == UseAgentAddress.REMOTE:
return self.URL_TEMPLATE.format(
host=self.remote_ip,
port=self.port
)
elif self.use_address == UseAgentAddress.HOSTNAME:
return self.URL_TEMPLATE.format(
host=self.hostname,
port=self.port
)
else:
raise ValueError(
"Cannot construct an agent API url using mode %r "
"`use_address`" % self.use_address)
@validates("hostname")
def validate_hostname_column(self, key, value):
"""Validates the hostname column"""
return self.validate_hostname(key, value)
@validates("ram", "cpus", "port")
def validate_numeric_column(self, key, value):
"""
Validates several numerical columns. Columns such as ram, cpus
and port a are validated with this method.
"""
return self.validate_resource(key, value)
@validates("remote_ip")
def validate_remote_ip(self, key, value):
"""Validates the remote_ip column"""
return self.validate_ipv4_address(key, value)
