class JobTagRequirement(db.Model, UtilityMixins): """ Model representing a dependency of a job on a tag If a job has a tag requirement, it will only run on agents that have that tag. """ __tablename__ = config.get("table_job_tag_req") __table_args__ = (UniqueConstraint("tag_id", "job_id"), ) id = id_column() tag_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_tag")), nullable=False, doc="Reference to the required tag") job_id = db.Column(IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), nullable=False, doc="Foreign key to :class:`Job.id`") negate = db.Column( db.Boolean, nullable=False, default=False, doc="If true, an agent that has this tag can not work on this job") job = db.relationship("Job", backref=db.backref("tag_requirements", lazy="dynamic", cascade="all, delete-orphan")) tag = db.relationship("Tag")
class JobTypeSoftwareRequirement(db.Model, UtilityMixins): """ Model representing a dependency of a job on a software tag, with optional version constraints """ __tablename__ = config.get("table_job_type_software_req") __table_args__ = (UniqueConstraint("software_id", "jobtype_version_id"), ) software_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_software")), primary_key=True, doc="Reference to the required software") jobtype_version_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type_version")), primary_key=True, doc="Foreign key to :class:`JobTypeVersion.id`") min_version_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_software_version")), nullable=True, doc="Reference to the minimum required version") max_version_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_software_version")), nullable=True, doc="Reference to the maximum required version") # # Relationships # jobtype_version = db.relationship("JobTypeVersion", backref=db.backref( "software_requirements", lazy="dynamic", cascade="all, delete-orphan")) software = db.relationship("Software") min_version = db.relationship("SoftwareVersion", foreign_keys=[min_version_id]) max_version = db.relationship("SoftwareVersion", foreign_keys=[max_version_id])
class AgentMacAddress(db.Model): __tablename__ = config.get("table_agent_mac_address") __table_args__ = (UniqueConstraint("agent_id", "mac_address"), ) agent_id = db.Column( IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), primary_key=True, nullable=False) mac_address = db.Column( MACAddress, primary_key=True, nullable=False, autoincrement=False)
class TaskLog(db.Model, UtilityMixins, ReprMixin): """Table which represents a single task log entry""" __tablename__ = config.get("table_task_log") __table_args__ = (UniqueConstraint("identifier"),) id = id_column(db.Integer) identifier = db.Column( db.String(255), nullable=False, doc="The identifier for this log") agent_id = db.Column( IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), nullable=True, doc="The agent this log was created on") created_on = db.Column( db.DateTime, default=datetime.utcnow, doc="The time when this log was created") # # Relationships # agent = db.relationship( "Agent", backref=db.backref("task_logs", lazy="dynamic"), doc="Relationship between an :class:`TaskLog`" "and the :class:`pyfarm.models.Agent` it was " "created on") task_associations = db.relationship( TaskTaskLogAssociation, backref="log", doc="Relationship between tasks and their logs." ) def num_queued_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=None).count() def num_running_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=WorkState.RUNNING).count() def num_failed_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=WorkState.FAILED).count() def num_done_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=WorkState.DONE).count()
class JobNotifiedUser(db.Model): """ Defines the table containing users to be notified of certain events pertaining to jobs. """ __tablename__ = config.get("table_job_notified_users") user_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), primary_key=True, doc="The id of the user to be notified") job_id = db.Column(IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), primary_key=True, doc="The id of the associated job") on_success = db.Column( db.Boolean, nullable=False, default=True, doc="True if a user should be notified on successful " "completion of a job") on_failure = db.Column( db.Boolean, nullable=False, default=True, doc="True if a user should be notified of a job's failure") on_deletion = db.Column( db.Boolean, nullable=False, default=False, doc="True if a user should be notified on deletion of " "a job") user = db.relationship("User", backref=db.backref("subscribed_jobs", lazy="dynamic"))
class SoftwareVersion(db.Model, UtilityMixins): """ Model to represent a version for a given software """ __tablename__ = config.get("table_software_version") __table_args__ = (UniqueConstraint("software_id", "version"), UniqueConstraint("software_id", "rank")) id = id_column() software_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_software")), nullable=False, doc="The software this version belongs to") version = db.Column( db.String(config.get("max_tag_length")), default="any", nullable=False, doc="The version of the software. This value does not " "follow any special formatting rules because the " "format depends on the 3rd party.") rank = db.Column( db.Integer, nullable=False, doc="The rank of this version relative to other versions of " "the same software. Used to determine whether a version " "is higher or lower than another.") default = db.Column(db.Boolean, default=False, nullable=False, doc="If true, this software version will be registered" "on new nodes by default.") discovery_code = db.Column( db.UnicodeText, nullable=True, doc="Python code to discover if this software version is installed " "on a node") discovery_function_name = db.Column( db.String(config.get("max_discovery_function_name_length")), nullable=True, doc="The name of a function in `discovery_code` to call when " "checking for the presence of this software version on an agent.\n" "The function should return either a boolean (true if present, " "false if not) or a tuple of a boolean and a dict of named " "parameters describing this installation.")
class JobGroup(db.Model, UtilityMixins): """ Used to group jobs together for better presentation in the UI """ __tablename__ = config.get("table_job_group") id = id_column(IDTypeWork) title = db.Column( db.String(config.get("max_jobgroup_name_length")), nullable=False, doc="The title of the job group's name") main_jobtype_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type")), nullable=False, doc="ID of the jobtype of the main job in this " "group. Purely for display and filtering.") user_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), doc="The id of the user who owns these jobs") # # Relationships # main_jobtype = db.relationship( "JobType", backref=db.backref("jobgroups", lazy="dynamic"), doc="The jobtype of the main job in this group") user = db.relationship( "User", backref=db.backref("jobgroups", lazy="dynamic"), doc="The user who owns these jobs")
class TaskTaskLogAssociation(db.Model): """Stores an association between the task table and a task log""" __tablename__ = config.get("table_task_log_assoc") __table_args__ = ( PrimaryKeyConstraint("task_log_id", "task_id", "attempt"),) task_log_id = db.Column( db.Integer, db.ForeignKey( "%s.id" % config.get("table_task_log"), ondelete="CASCADE"), doc="The ID of the task log") task_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_task"), ondelete="CASCADE"), doc="The ID of the job a task log is associated with") attempt = db.Column( db.Integer, autoincrement=False, doc="The attempt number for the given task log") state = db.Column( WorkStateEnum, nullable=True, doc="The state of the work being performed") # # Relationships # task = db.relationship( "Task", backref=db.backref( "log_associations", lazy="dynamic", passive_deletes=True))
class PathMap(db.Model, ReprMixin, UtilityMixins): """ Defines a table which is used for cross-platform file path mappings. """ __tablename__ = config.get("table_path_map") id = id_column(db.Integer) path_linux = db.Column( db.String(config.get("max_path_length")), nullable=False, doc="The path on linux platforms") path_windows = db.Column( db.String(config.get("max_path_length")), nullable=False, doc="The path on Windows platforms") path_osx = db.Column( db.String(config.get("max_path_length")), nullable=False, doc="The path on Mac OS X platforms") tag_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_tag")), nullable=True, doc="The tag an agent needs to have for this path map " "to apply to it. " "If this is NULL, this path map applies to all " "agents, but is overridden by applying path maps " "that do specify a tag.") # # Relationships # tag = db.relationship( "Tag", backref=db.backref("path_maps", lazy="dynamic"), doc="Relationship attribute for the tag this path map " "applies to.")
try: # pylint: disable=undefined-variable range_ = xrange except NameError: range_ = range __all__ = ("Job", ) logger = getLogger("models.job") JobTagAssociation = db.Table( config.get("table_job_tag_assoc"), db.metadata, db.Column("job_id", IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), primary_key=True, doc="The id of the job associated with this task"), db.Column("tag_id", db.Integer, db.ForeignKey("%s.id" % config.get("table_tag")), primary_key=True, doc="The id of the tag being associated with the job")) JobDependency = db.Table( config.get("table_job_dependency"), db.metadata, db.Column("parentid", IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), primary_key=True, doc="The parent job id of the job dependency"),
class Task(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin, UtilityMixins, ReprMixin): """ Defines a task which a child of a :class:`Job`. This table represents rows which contain the individual work unit(s) for a job. """ __tablename__ = config.get("table_task") STATE_ENUM = list(WorkState) + [None] STATE_DEFAULT = None REPR_COLUMNS = ("id", "state", "frame", "project") REPR_CONVERT_COLUMN = {"state": partial(repr_enum, enum=STATE_ENUM)} # shared work columns id, state, priority, time_submitted, time_started, time_finished = \ work_columns(STATE_DEFAULT, "job.priority") agent_id = db.Column( IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), doc="Foreign key which stores :attr:`Job.id`") job_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), nullable=False, doc="Foreign key which stores :attr:`Job.id`") hidden = db.Column( db.Boolean, default=False, doc="When True this hides the task from queue and web ui") attempts = db.Column( db.Integer, nullable=False, default=0, doc="The number of attempts which have been made on this " "task. This value is auto incremented when " "``state`` changes to a value synonymous with a " "running state.") failures = db.Column( db.Integer, nullable=False, default=0, doc="The number of times this task has failed. This value " "is auto incremented when :attr:`state` changes to a " "value synonymous with a failed state.") frame = db.Column( db.Numeric(10, 4), nullable=False, doc="The frame this :class:`Task` will be executing.") tile = db.Column( db.Integer, nullable=True, doc="When using tiled rendering, the number of the tile this task " "refers to. The jobtype will have to translate that into an " "actual image region. This will be NULL if the job doesn't use " "tiled rendering.") last_error = db.Column( db.UnicodeText, nullable=True, doc="This column may be set when an error is " "present. The agent typically sets this " "column when the job type either can't or " "won't run a given task. This column will " "be cleared whenever the task's state is " "returned to a non-error state.") sent_to_agent = db.Column( db.Boolean, default=False, nullable=False, doc="Whether this task was already sent to the assigned agent") progress = db.Column( db.Float, default=0.0, doc="The progress for this task, as a value between " "0.0 and 1.0. Used purely for display purposes.") # # Relationships # job = db.relationship( "Job", backref=db.backref("tasks", lazy="dynamic"), doc="relationship attribute which retrieves the " "associated job for this task") def running(self): return self.state == WorkState.RUNNING def failed(self): return self.state == WorkState.FAILED @staticmethod def increment_attempts(target, new_value, old_value, initiator): if new_value is not None and new_value != old_value: target.attempts += 1 @staticmethod def log_assign_change(target, new_value, old_value, initiator): logger.debug("Agent change for task %s: old %s new: %s", target.id, old_value, new_value) @staticmethod def update_failures(target, new_value, old_value, initiator): if new_value == WorkState.FAILED and new_value != old_value: target.failures += 1 if target not in target.agent.failed_tasks: target.agent.failed_tasks.append(target) @staticmethod def set_progress_on_success(target, new_value, old_value, initiator): if new_value == WorkState.DONE: target.progress = 1.0 @staticmethod def update_agent_on_success(target, new_value, old_value, initiator): if new_value == WorkState.DONE: agent = target.agent if agent: agent.last_success_on = datetime.utcnow() db.session.add(agent) @staticmethod def reset_agent_if_failed_and_retry( target, new_value, old_value, initiator): # There's nothing else we should do here if # we don't have a parent job. This can happen if you're # testing or a job is disconnected from a task. if target.job is None: return new_value if (new_value == WorkState.FAILED and target.failures <= target.job.requeue): logger.info("Failed task %s will be retried", target.id) target.agent_id = None return None else: return new_value @staticmethod def clear_error_state(target, new_value, old_value, initiator): """ Sets ``last_error`` column to ``None`` if the task's state is 'done' """ if new_value == WorkState.DONE and target.last_error is not None: target.last_error = None @staticmethod def set_times(target, new_value, old_value, initiator): """update the datetime objects depending on the new value""" if (new_value == _WorkState.RUNNING and (old_value not in [_WorkState.RUNNING, _WorkState.PAUSED] or target.time_started == None)): if not target.job.jobtype_version.no_automatic_start_time: target.time_started = datetime.utcnow() target.time_finished = None elif (new_value in (_WorkState.DONE, _WorkState.FAILED) and not target.time_finished): target.time_finished = datetime.utcnow() @staticmethod def reset_finished_time(target, new_value, old_value, initiator): if (target.state not in (_WorkState.DONE, _WorkState.FAILED) or new_value is None): target.time_finished = None elif new_value is not None: if target.time_finished is not None: target.time_finished = max(target.time_finished, new_value) else: target.time_finished = max(new_value, datetime.utcnow())
class Task(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, UtilityMixins, ReprMixin): """ Defines a task which a child of a :class:`Job`. This table represents rows which contain the individual work unit(s) for a job. """ __tablename__ = TABLE_TASK STATE_ENUM = WorkState STATE_DEFAULT = STATE_ENUM.QUEUED REPR_COLUMNS = ("id", "state", "frame", "project") REPR_CONVERT_COLUMN = {"state": partial(repr_enum, enum=STATE_ENUM)} # shared work columns id, state, priority, time_submitted, time_started, time_finished = \ work_columns(STATE_DEFAULT, "job.priority") project_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT), doc="stores the project id") agent_id = db.Column(IDTypeAgent, db.ForeignKey("%s.id" % TABLE_AGENT), doc="Foreign key which stores :attr:`Job.id`") job_id = db.Column(IDTypeWork, db.ForeignKey("%s.id" % TABLE_JOB), doc="Foreign key which stores :attr:`Job.id`") hidden = db.Column(db.Boolean, default=False, doc=dedent(""" hides the task from queue and web ui""")) attempts = db.Column(db.Integer, doc=dedent(""" The number attempts which have been made on this task. This value is auto incremented when :attr:`state` changes to a value synonyms with a running state.""")) frame = db.Column(db.Float, nullable=False, doc=dedent(""" The frame the :class:`Task` will be executing.""")) # relationships parents = db.relationship("Task", secondary=TaskDependencies, primaryjoin=id == TaskDependencies.c.parent_id, secondaryjoin=id == TaskDependencies.c.child_id, backref=db.backref("children", lazy="dynamic")) project = db.relationship("Project", backref=db.backref("tasks", lazy="dynamic"), doc=dedent(""" relationship attribute which retrieves the associated project for the task""")) job = db.relationship("Job", backref=db.backref("tasks", lazy="dynamic"), doc=dedent(""" relationship attribute which retrieves the associated job for this task""")) @staticmethod def agentChangedEvent(target, new_value, old_value, initiator): """set the state to ASSIGN whenever the agent is changed""" if new_value is not None: target.state = target.STATE_ENUM.ASSIGN
from pyfarm.core.enums import WorkState from pyfarm.master.application import db from pyfarm.models.core.types import IDTypeAgent, IDTypeWork from pyfarm.models.core.functions import work_columns, repr_enum from pyfarm.models.core.cfg import (TABLE_JOB, TABLE_TASK, TABLE_AGENT, TABLE_TASK_DEPENDENCIES, TABLE_PROJECT) from pyfarm.models.core.mixins import (ValidatePriorityMixin, WorkStateChangedMixin, UtilityMixins, ReprMixin) TaskDependencies = db.Table( TABLE_TASK_DEPENDENCIES, db.metadata, db.Column("parent_id", IDTypeWork, db.ForeignKey("%s.id" % TABLE_TASK), primary_key=True), db.Column("child_id", IDTypeWork, db.ForeignKey("%s.id" % TABLE_TASK), primary_key=True)) class Task(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, UtilityMixins, ReprMixin): """ Defines a task which a child of a :class:`Job`. This table represents rows which contain the individual work unit(s) for a job. """ __tablename__ = TABLE_TASK STATE_ENUM = WorkState
from pyfarm.core.enums import STRING_TYPES, PY3 from pyfarm.master.application import app, db, login_serializer from pyfarm.models.core.mixins import ReprMixin from pyfarm.models.core.functions import split_and_extend from pyfarm.models.core.cfg import ( TABLE_USERS_USER, TABLE_USERS_ROLE, TABLE_USERS_USER_ROLES, MAX_USERNAME_LENGTH, SHA256_ASCII_LENGTH, MAX_EMAILADDR_LENGTH, MAX_ROLE_LENGTH, TABLE_USERS_PROJECTS, TABLE_PROJECT) logger = getLogger("models.users") # roles the user is a member of UserRoles = db.Table( TABLE_USERS_USER_ROLES, db.Column("user_id", db.Integer, db.ForeignKey("%s.id" % TABLE_USERS_USER)), db.Column("role_id", db.Integer, db.ForeignKey("%s.id" % TABLE_USERS_ROLE))) # projects the user is a member of UserProjects = db.Table( TABLE_USERS_PROJECTS, db.Column("user_id", db.Integer, db.ForeignKey("%s.id" % TABLE_USERS_USER), primary_key=True), db.Column("project_id", db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT), primary_key=True)) class User(db.Model, UserMixin, ReprMixin): """ Stores information about a user including the roles they belong to
class Job(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, ReprMixin): """ Defines the attributes and environment for a job. Individual commands are kept track of by |Task| """ __tablename__ = TABLE_JOB REPR_COLUMNS = ("id", "state", "project") REPR_CONVERT_COLUMN = { "state": repr} MIN_CPUS = read_env_int("PYFARM_QUEUE_MIN_CPUS", 1) MAX_CPUS = read_env_int("PYFARM_QUEUE_MAX_CPUS", 256) MIN_RAM = read_env_int("PYFARM_QUEUE_MIN_RAM", 16) MAX_RAM = read_env_int("PYFARM_QUEUE_MAX_RAM", 262144) SPECIAL_RAM = read_env("PYFARM_AGENT_SPECIAL_RAM", [0], eval_literal=True) SPECIAL_CPUS = read_env("PYFARM_AGENT_SPECIAL_CPUS", [0], eval_literal=True) # quick check of the configured data assert MIN_CPUS >= 1, "$PYFARM_QUEUE_MIN_CPUS must be > 0" assert MAX_CPUS >= 1, "$PYFARM_QUEUE_MAX_CPUS must be > 0" assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS" assert MIN_RAM >= 1, "$PYFARM_QUEUE_MIN_RAM must be > 0" assert MAX_RAM >= 1, "$PYFARM_QUEUE_MAX_RAM must be > 0" assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM" # shared work columns id, state, priority, time_submitted, time_started, time_finished = \ work_columns(WorkState.QUEUED, "job.priority") project_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT), doc="stores the project id") job_type_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB_TYPE), nullable=False, doc=dedent(""" The foreign key which stores :class:`JobType.id`""")) user = db.Column(db.String(MAX_USERNAME_LENGTH), doc=dedent(""" The user this job should execute as. The agent process will have to be running as root on platforms that support setting the user id. .. note:: The length of this field is limited by the configuration value `job.max_username_length` .. warning:: this may not behave as expected on all platforms (windows in particular)""")) notes = db.Column(db.Text, default="", doc=dedent(""" Notes that are provided on submission or added after the fact. This column is only provided for human consumption is not scanned, index, or used when searching""")) # task data cmd = db.Column(db.String(MAX_COMMAND_LENGTH), doc=dedent(""" The platform independent command to run. Each agent will resolve this value for itself when the task begins so a command like `ping` will work on any platform it's assigned to. The full command could be provided here, but then the job must be tagged using :class:`.JobSoftware` to limit which agent(s) it will run on.""")) start = db.Column(db.Float, doc=dedent(""" The first frame of the job to run. This value may be a float so subframes can be processed.""")) end = db.Column(db.Float, doc=dedent(""" The last frame of the job to run. This value may be a float so subframes can be processed.""")) by = db.Column(db.Float, default=1, doc=dedent(""" The number of frames to count by between `start` and `end`. This column may also sometimes be referred to as 'step' by other software.""")) batch = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_BATCH", 1), doc=dedent(""" Number of tasks to run on a single agent at once. Depending on the capabilities of the software being run this will either cause a single process to execute on the agent or multiple processes on after the other. **configured by**: `job.batch`""")) requeue = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_REQUEUE", 3), doc=dedent(""" Number of times to requeue failed tasks .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 0, never requeue failed tasks -1, requeue failed tasks indefinitely **configured by**: `job.requeue`""")) cpus = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_CPUS", 1), doc=dedent(""" Number of cpus or threads each task should consume on each agent. Depending on the job type being executed this may result in additional cpu consumption, longer wait times in the queue (2 cpus means 2 'fewer' cpus on an agent), or all of the above. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 0, minimum number of cpu resources not required -1, agent cpu is exclusive for a task from this job **configured by**: `job.cpus`""")) ram = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_RAM", 32), doc=dedent(""" Amount of ram a task from this job will require to be free in order to run. A task exceeding this value will not result in any special behavior. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 0, minimum amount of free ram not required -1, agent ram is exclusive for a task from this job **configured by**: `job.ram`""")) ram_warning = db.Column(db.Integer, default=-1, doc=dedent(""" Amount of ram used by a task before a warning raised. A task exceeding this value will not cause any work stopping behavior. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 -1, not set""")) ram_max = db.Column(db.Integer, default=-1, doc=dedent(""" Maximum amount of ram a task is allowed to consume on an agent. .. warning:: The task will be **terminated** if the ram in use by the process exceeds this value. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 -1, not set """)) attempts = db.Column(db.Integer, doc=dedent(""" The number attempts which have been made on this task. This value is auto incremented when :attr:`state` changes to a value synonyms with a running state.""")) hidden = db.Column(db.Boolean, default=False, nullable=False, doc=dedent(""" If True, keep the job hidden from the queue and web ui. This is typically set to True if you either want to save a job for later viewing or if the jobs data is being populated in a deferred manner.""")) environ = db.Column(JSONDict, doc=dedent(""" Dictionary containing information about the environment in which the job will execute. .. note:: Changes made directly to this object are **not** applied to the session.""")) args = db.Column(JSONList, doc=dedent(""" List containing the command line arguments. .. note:: Changes made directly to this object are **not** applied to the session.""")) data = db.Column(JSONDict, doc=dedent(""" Json blob containing additional data for a job .. note:: Changes made directly to this object are **not** applied to the session.""")) project = db.relationship("Project", backref=db.backref("jobs", lazy="dynamic"), doc=dedent(""" relationship attribute which retrieves the associated project for the job""")) # self-referential many-to-many relationship parents = db.relationship("Job", secondary=JobDependencies, primaryjoin=id==JobDependencies.c.parentid, secondaryjoin=id==JobDependencies.c.childid, backref="children") tasks_done = db.relationship("Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.DONE, doc=dedent(""" Relationship between this job and any |Task| objects which are done.""")) tasks_failed = db.relationship("Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.FAILED, doc=dedent(""" Relationship between this job and any |Task| objects which have failed.""")) tasks_queued = db.relationship("Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.QUEUED, doc=dedent(""" Relationship between this job and any |Task| objects which are queued.""")) # resource relationships tags = db.relationship("Tag", backref="jobs", lazy="dynamic", secondary=JobTagAssociation, doc=dedent(""" Relationship between this job and :class:`.Tag` objects""")) software = db.relationship("Software", secondary=JobSoftwareDependency, backref=db.backref("jobs", lazy="dynamic"), lazy="dynamic", doc="software needed by this job") @validates("ram", "cpus") def validate_resource(self, key, value): """ Validation that ensures that the value provided for either :attr:`.ram` or :attr:`.cpus` is a valid value with a given range """ key_upper = key.upper() special = getattr(self, "SPECIAL_%s" % key_upper) if value is None or value in special: return value min_value = getattr(self, "MIN_%s" % key_upper) max_value = getattr(self, "MAX_%s" % key_upper) # quick sanity check of the incoming config assert isinstance(min_value, int), "db.min_%s must be an integer" % key assert isinstance(max_value, int), "db.max_%s must be an integer" % key assert min_value >= 1, "db.min_%s must be > 0" % key assert max_value >= 1, "db.max_%s must be > 0" % key # check the provided input if min_value > value or value > max_value: msg = "value for `%s` must be between " % key msg += "%s and %s" % (min_value, max_value) raise ValueError(msg) return value
AgentStateEnum) from pyfarm.models.core.cfg import ( TABLE_AGENT, TABLE_SOFTWARE, TABLE_TAG, TABLE_AGENT_TAG_ASSOC, MAX_HOSTNAME_LENGTH, MAX_TAG_LENGTH, TABLE_AGENT_SOFTWARE_ASSOC, TABLE_PROJECT_AGENTS, TABLE_PROJECT) PYFARM_REQUIRE_PRIVATE_IP = read_env_bool("PYFARM_REQUIRE_PRIVATE_IP", False) REGEX_HOSTNAME = re.compile("^(?!-)[A-Z\d-]{1,63}(?<!-)" "(\.(?!-)[A-Z\d-]{1,63}(?<!-))*\.?$" , re.IGNORECASE) AgentSoftwareAssociation = db.Table( TABLE_AGENT_SOFTWARE_ASSOC, db.metadata, db.Column("agent_id", IDTypeAgent, db.ForeignKey("%s.id" % TABLE_AGENT), primary_key=True), db.Column("software_id", db.Integer, db.ForeignKey("%s.id" % TABLE_SOFTWARE), primary_key=True)) AgentTagAssociation = db.Table( TABLE_AGENT_TAG_ASSOC, db.metadata, db.Column("agent_id", IDTypeAgent, db.ForeignKey("%s.id" % TABLE_AGENT), primary_key=True), db.Column("tag_id", db.Integer, db.ForeignKey("%s.id" % TABLE_TAG), primary_key=True)) AgentProjects = db.Table( TABLE_PROJECT_AGENTS, db.metadata, db.Column("agent_id", IDTypeAgent,
class Job(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin, WorkStateChangedMixin, ReprMixin, UtilityMixins): """ Defines the attributes and environment for a job. Individual commands are kept track of by :class:`Task` """ __tablename__ = config.get("table_job") REPR_COLUMNS = ("id", "state", "project") REPR_CONVERT_COLUMN = {"state": repr} STATE_ENUM = list(WorkState) + [None] # shared work columns id, state, priority, time_submitted, time_started, time_finished = \ work_columns(None, "job.priority") jobtype_version_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type_version")), nullable=False, doc="The foreign key which stores :class:`JobTypeVersion.id`") job_queue_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_queue")), nullable=True, doc="The foreign key which stores :class:`JobQueue.id`") job_group_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_group")), nullable=True, doc="The foreign key which stores:class:`JobGroup.id`") user_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), doc="The id of the user who owns this job") minimum_agents = db.Column( db.Integer, nullable=True, doc="The scheduler will try to assign at least this number " "of agents to this job as long as it can use them, " "before any other considerations.") maximum_agents = db.Column( db.Integer, nullable=True, doc="The scheduler will never assign more than this number" "of agents to this job.") weight = db.Column( db.Integer, nullable=False, default=config.get("queue_default_weight"), doc="The weight of this job. The scheduler will distribute " "available agents between jobs and job queues in the " "same queue in proportion to their weights.") title = db.Column(db.String(config.get("jobtitle_max_length")), nullable=False, doc="The title of this job") notes = db.Column( db.Text, default="", doc="Notes that are provided on submission or added after " "the fact. This column is only provided for human " "consumption, is not scanned, indexed, or used when " "searching") output_link = db.Column( db.Text, nullable=True, doc="An optional link to a URI where this job's output can " "be viewed.") # task data by = db.Column(db.Numeric(10, 4), default=1, doc="The number of frames to count by between `start` and " "`end`. This column may also sometimes be referred to " "as 'step' by other software.") num_tiles = db.Column( db.Integer, nullable=True, doc="How many regions to split frames into for rendering.") batch = db.Column( db.Integer, default=config.get("job_default_batch"), doc="Number of tasks to run on a single agent at once. Depending " "on the capabilities of the software being run this will " "either cause a single process to execute on the agent " "or multiple processes one after the other.") requeue = db.Column(db.Integer, default=config.get("job_requeue_default"), doc="Number of times to requeue failed tasks " "" ".. csv-table:: **Special Values**" " :header: Value, Result" " :widths: 10, 50" "" " 0, never requeue failed tasks" " -1, requeue failed tasks indefinitely") cpus = db.Column( db.Integer, default=config.get("job_default_cpus"), doc="Number of cpus or threads each task should consume on" "each agent. Depending on the job type being executed " "this may result in additional cpu consumption, longer " "wait times in the queue (2 cpus means 2 'fewer' cpus on " "an agent), or all of the above." "" ".. csv-table:: **Special Values**" " :header: Value, Result" " :widths: 10, 50" "" " 0, minimum number of cpu resources not required " " -1, agent cpu is exclusive for a task from this job") ram = db.Column( db.Integer, default=config.get("job_default_ram"), doc="Amount of ram a task from this job will require to be " "free in order to run. A task exceeding this value will " "not result in any special behavior." "" ".. csv-table:: **Special Values**" " :header: Value, Result" " :widths: 10, 50" "" "0, minimum amount of free ram not required" "-1, agent ram is exclusive for a task from this job") ram_warning = db.Column( db.Integer, nullable=True, doc="Amount of ram used by a task before a warning raised. " "A task exceeding this value will not cause any work " "stopping behavior.") ram_max = db.Column( db.Integer, nullable=True, doc="Maximum amount of ram a task is allowed to consume on " "an agent." "" ".. warning:: " " If set, the task will be **terminated** if the ram in " " use by the process exceeds this value.") hidden = db.Column( db.Boolean, default=False, nullable=False, doc="If True, keep the job hidden from the queue and web " "ui. This is typically set to True if you either want " "to save a job for later viewing or if the jobs data " "is being populated in a deferred manner.") environ = db.Column( JSONDict, doc="Dictionary containing information about the environment " "in which the job will execute. " "" ".. note::" " Changes made directly to this object are **not** " " applied to the session.") data = db.Column(JSONDict, doc="Json blob containing additional data for a job " "" ".. note:: " " Changes made directly to this object are **not** " " applied to the session.") to_be_deleted = db.Column( db.Boolean, nullable=False, default=False, doc="If true, the master will stop all running tasks for " "this job and then delete it.") completion_notify_sent = db.Column( db.Boolean, nullable=False, default=False, doc="Whether or not the finish notification mail has already " "been sent out.") autodelete_time = db.Column( db.Integer, nullable=True, default=None, doc="If not None, this job will be automatically deleted this " "number of seconds after it finishes.") # # Relationships # queue = db.relationship("JobQueue", backref=db.backref("jobs", lazy="dynamic"), doc="The queue for this job") group = db.relationship("JobGroup", backref=db.backref("jobs", lazy="dynamic"), doc="The job group this job belongs to") user = db.relationship("User", backref=db.backref("jobs", lazy="dynamic"), doc="The owner of this job") # self-referential many-to-many relationship parents = db.relationship("Job", secondary=JobDependency, primaryjoin=id == JobDependency.c.childid, secondaryjoin=id == JobDependency.c.parentid, backref="children") notified_users = db.relationship("JobNotifiedUser", lazy="dynamic", backref=db.backref("job"), cascade="all,delete") tasks_queued = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == None) & " "(Task.job_id == Job.id)", doc="Relationship between this job and any :class:`Task` " "objects which are queued.") tasks_running = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.RUNNING, doc="Relationship between this job and any :class:`Task` " "objects which are running.") tasks_done = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.DONE, doc="Relationship between this job and any :class:`Task` objects " "which are done.") tasks_failed = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.FAILED, doc="Relationship between this job and any :class:`Task` objects " "which have failed.") # resource relationships tags = db.relationship( "Tag", backref="jobs", lazy="dynamic", secondary=JobTagAssociation, doc="Relationship between this job and :class:`.Tag` objects") def paused(self): return self.state == WorkState.PAUSED def update_state(self): # Import here instead of at the top of the file to avoid a circular # import from pyfarm.scheduler.tasks import send_job_completion_mail from pyfarm.models.agent import Agent num_active_tasks = db.session.query(Task).\ filter(Task.job == self, or_(Task.state == None, and_( Task.state != WorkState.DONE, Task.state != WorkState.FAILED))).count() if num_active_tasks == 0: num_failed_tasks = db.session.query(Task).filter( Task.job == self, Task.state == WorkState.FAILED).count() if num_failed_tasks == 0: if self.state != _WorkState.DONE: logger.info( "Job %r (id %s): state transition %r -> 'done'", self.title, self.id, self.state) self.state = WorkState.DONE send_job_completion_mail.apply_async(args=[self.id, True], countdown=5) else: if self.state != _WorkState.FAILED: logger.info( "Job %r (id %s): state transition %r -> " "'failed'", self.title, self.id, self.state) self.state = WorkState.FAILED send_job_completion_mail.apply_async(args=[self.id, False], countdown=5) db.session.add(self) elif self.state != _WorkState.PAUSED: num_running_tasks = db.session.query(Task).\ filter(Task.job == self, Task.agent_id != None, Task.agent.has(and_(Agent.state != AgentState.OFFLINE, Agent.state != AgentState.DISABLED)), or_( Task.state == WorkState.RUNNING, Task.state == None)).count() if num_running_tasks == 0: logger.debug( "No running tasks in job %s (id %s), setting it " "to queued", self.title, self.id) self.state = None db.session.add(self) elif self.state != _WorkState.RUNNING: self.state = WorkState.RUNNING # Methods used by the scheduler def num_assigned_agents(self): # Import here instead of at the top of the file to avoid circular import from pyfarm.models.agent import Agent # Optimization: Blindly assume that we have no agents assigned if not # running if self.state != _WorkState.RUNNING: return 0 try: return self.assigned_agents_count except AttributeError: self.assigned_agents_count =\ db.session.query(distinct(Task.agent_id)).\ filter(Task.job == self, Task.agent_id != None, or_(Task.state == None, Task.state == WorkState.RUNNING), Task.agent.has( and_(Agent.state != AgentState.OFFLINE, Agent.state != AgentState.DISABLED)))\ .count() return self.assigned_agents_count def clear_assigned_counts(self): try: del self.assigned_agents_count except AttributeError: pass if self.queue: self.queue.clear_assigned_counts() def can_use_more_agents(self): # Import here instead of at the top of the file to avoid circular import from pyfarm.models.agent import Agent unassigned_tasks = Task.query.filter( Task.job == self, or_(Task.state == None, ~Task.state.in_([WorkState.DONE, WorkState.FAILED])), or_( Task.agent == None, Task.agent.has( Agent.state.in_([AgentState.OFFLINE, AgentState.DISABLED])))).count() return unassigned_tasks > 0 def get_batch(self, agent): # Import here instead of at the top of the file to avoid circular import from pyfarm.models.agent import Agent tasks_query = Task.query.filter( Task.job == self, ~Task.failed_in_agents.any(id=agent.id), or_(Task.state == None, ~Task.state.in_([WorkState.DONE, WorkState.FAILED])), or_(Task.agent == None, Task.agent.has(Agent.state.in_( [AgentState.OFFLINE, AgentState.DISABLED])))).\ order_by("frame asc, tile asc") batch = [] for task in tasks_query: if (len(batch) < self.batch and len(batch) < (self.jobtype_version.max_batch or maxsize) and (not self.jobtype_version.batch_contiguous or (len(batch) == 0 or batch[-1].frame + self.by == task.frame))): batch.append(task) return batch def alter_frame_range(self, start, end, by): # We have to import this down here instead of at the top to break a # circular dependency between the modules from pyfarm.scheduler.tasks import delete_task if end < start: raise ValueError("`end` must be greater than or equal to `start`") self.by = by required_frames = [] current_frame = start while current_frame <= end: required_frames.append(current_frame) current_frame += by existing_tasks = Task.query.filter_by(job=self).all() frames_to_create = required_frames num_created = 0 for task in existing_tasks: if task.frame not in required_frames: delete_task.delay(task.id) else: frames_to_create.remove(task.frame) for frame in frames_to_create: if self.num_tiles: for tile in range_(self.num_tiles - 1): num_created += 1 task = Task() task.job = self task.frame = frame task.tile = tile task.priority = self.priority db.session.add(task) else: num_created += 1 task = Task() task.job = self task.frame = frame task.priority = self.priority db.session.add(task) if frames_to_create: if self.state != WorkState.RUNNING: self.state = None if config.get("enable_statistics"): task_event_count = TaskEventCount(num_new=num_created, job_queue_id=self.job_queue_id) task_event_count.time_start = datetime.utcnow() task_event_count.time_end = datetime.utcnow() db.session.add(task_event_count) def rerun(self): """ Makes this job rerun all its task. Tasks that are currently running are left untouched. """ num_restarted = 0 for task in self.tasks: if task.state != _WorkState.RUNNING and task.state is not None: task.state = None task.agent = None task.failures = 0 db.session.add(task) num_restarted += 1 self.completion_notify_sent = False self.update_state() db.session.add(self) if config.get("enable_statistics"): task_event_count = TaskEventCount(job_queue_id=self.job_queue_id, num_restarted=num_restarted) task_event_count.time_start = datetime.utcnow() task_event_count.time_end = datetime.utcnow() db.session.add(task_event_count) db.session.commit() for child in self.children: child.rerun() def rerun_failed(self): """ Makes this job rerun all its failed tasks. Tasks that are done or are currently running are left untouched """ num_restarted = 0 for task in self.tasks: if task.state == _WorkState.FAILED: task.state = None task.agent = None task.failures = 0 db.session.add(task) num_restarted += 1 self.completion_notify_sent = False self.update_state() db.session.add(self) if config.get("enable_statistics"): task_event_count = TaskEventCount(job_queue_id=self.job_queue_id, num_restarted=num_restarted) task_event_count.time_start = datetime.utcnow() task_event_count.time_end = datetime.utcnow() db.session.commit() for child in self.children: child.rerun_failed() @validates("ram", "cpus") def validate_resource(self, key, value): """ Validation that ensures that the value provided for either :attr:`.ram` or :attr:`.cpus` is a valid value with a given range """ assert isinstance(value, int), "%s must be an integer" % key min_value = config.get("agent_min_%s" % key) max_value = config.get("agent_max_%s" % key) # check the provided input if min_value > value or value > max_value: msg = "value for `%s` must be between " % key msg += "%s and %s" % (min_value, max_value) raise ValueError(msg) return value @validates("progress") def validate_progress(self, key, value): if value < 0.0 or value > 1.0: raise ValueError("Progress must be between 0.0 and 1.0")
from pyfarm.models.job import Job __all__ = ("Agent", ) ALLOW_AGENT_LOOPBACK = config.get("allow_agents_from_loopback") REGEX_HOSTNAME = re.compile("^(?!-)[A-Z\d-]{1,63}(?<!-)" "(\.(?!-)[A-Z\d-]{1,63}(?<!-))*\.?$", re.IGNORECASE) AgentSoftwareVersionAssociation = db.Table( config.get("table_agent_software_version_assoc"), db.metadata, db.Column( "agent_id", IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), primary_key=True), db.Column( "software_version_id", db.Integer, db.ForeignKey("%s.id" % config.get("table_software_version")), primary_key=True)) AgentTagAssociation = db.Table( config.get("table_agent_tag_assoc"), db.metadata, db.Column( "agent_id", IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), primary_key=True), db.Column( "tag_id", db.Integer,
class JobTypeVersion(db.Model, UtilityMixins, ReprMixin): """ Defines a specific jobtype version. """ __tablename__ = config.get("table_job_type_version") __table_args__ = (UniqueConstraint("jobtype_id", "version"), ) REPR_COLUMNS = ("id", "jobtype_id", "version") id = id_column(IDTypeWork) jobtype_id = db.Column(IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type")), nullable=False, doc="The jobtype this version belongs to") version = db.Column(db.Integer, nullable=False, doc="The version number") max_batch = db.Column( db.Integer, default=config.get("job_type_max_batch"), doc="When the queue runs, this is the maximum number of tasks " "that the queue can select to assign to a single" "agent. If left empty, no maximum applies") batch_contiguous = db.Column( db.Boolean, default=config.get("job_type_batch_contiguous"), doc="If True then the queue will be forced to batch" "numerically contiguous tasks only for this job type. " "For example if True it would batch frames 1, 2, 3, 4 " "together but not 2, 4, 6, 8. If this column is False " "however the queue will batch non-contiguous tasks too.") no_automatic_start_time = db.Column( db.Boolean, nullable=False, default=False, doc="If set, we will not automatically set `time_started_on` " "for the tasks in jobs of this type when they are set " "to `running`.") supports_tiling = db.Column( db.Boolean, default=False, doc="Whether or not the jobtype supports tiling, i.e. splitting single " "frames into regions and then rendering those independently from " "each other.") classname = db.Column( db.String(config.get("job_type_max_class_name_length")), nullable=True, doc="The name of the job class contained within the file being " "loaded. This field may be null but when it's not provided " "job type name will be used instead.") code = db.Column(db.UnicodeText, nullable=False, doc="The source code of the job type") # # Relationships # jobtype = db.relationship("JobType", backref=db.backref("versions", lazy="dynamic", cascade="all, delete-orphan"), doc="Relationship between this version and the " ":class:`JobType` it belongs to" "") jobs = db.relationship("Job", backref="jobtype_version", lazy="dynamic", doc="Relationship between this jobtype version and " ":class:`.Job` objects.") @validates("max_batch") def validate_max_batch(self, key, value): if isinstance(value, int) and value < 1: raise ValueError("max_batch must be greater than or equal to 1") return value @validates("version") def validate_version(self, key, value): if isinstance(value, int) and value < 1: raise ValueError("version must be greater than or equal to 1") return value
from pyfarm.core.enums import STRING_TYPES, PY3 from pyfarm.master.application import app, db, login_serializer from pyfarm.master.config import config from pyfarm.models.core.mixins import ReprMixin from pyfarm.models.core.functions import split_and_extend __all__ = ("User", "Role") SHA256_ASCII_LENGTH = 64 # static length of a sha256 string # roles the user is a member of UserRole = db.Table( config.get("table_user_role"), db.Column("user_id", db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), doc="The id of the associated user"), db.Column("role_id", db.Integer, db.ForeignKey("%s.id" % config.get("table_role")), doc="The id of the associated role")) class User(db.Model, UserMixin, ReprMixin): """ Stores information about a user including the roles they belong to """ __tablename__ = config.get("table_user") REPR_COLUMNS = ("id", "username") id = db.Column(db.Integer, primary_key=True, nullable=False)
__tablename__ = "%s_state_change_test" % config.get("table_prefix") id = db.Column(Integer, primary_key=True, autoincrement=True) state = db.Column(WorkStateEnum) attempts = db.Column(Integer, nullable=False, default=0) time_started = db.Column(DateTime) time_finished = db.Column(DateTime) event.listen(WorkStateChangedModel.state, "set", WorkStateChangedModel.state_changed) MixinModelRelation1 = db.Table( "%s_mixin_rel_test1" % config.get("table_prefix"), db.metadata, db.Column("mixin_id", db.Integer, db.ForeignKey("%s.id" % "%s_mixin_test" % config.get("table_prefix")), primary_key=True)) MixinModelRelation2 = db.Table( "%s_mixin_rel_test2" % config.get("table_prefix"), db.metadata, db.Column("mixin_id", db.Integer, db.ForeignKey("%s.id" % "%s_mixin_test" % config.get("table_prefix")), primary_key=True)) class MixinModel(db.Model, UtilityMixins): __tablename__ = "%s_mixin_test" % config.get("table_prefix") id = db.Column(db.Integer, primary_key=True, autoincrement=True) a = db.Column(db.Integer)
from pyfarm.master.application import db from pyfarm.models.core.functions import work_columns from pyfarm.models.core.types import id_column, JSONDict, JSONList, IDTypeWork from pyfarm.models.core.cfg import ( TABLE_JOB, TABLE_JOB_SOFTWARE_DEP, TABLE_JOB_TYPE, TABLE_TAG, TABLE_JOB_TAG_ASSOC, MAX_COMMAND_LENGTH, MAX_TAG_LENGTH, MAX_USERNAME_LENGTH, TABLE_SOFTWARE, TABLE_JOB_DEPENDENCIES, TABLE_PROJECT) from pyfarm.models.core.mixins import ( ValidatePriorityMixin, WorkStateChangedMixin, ReprMixin) from pyfarm.models.jobtype import JobType # required for a relationship JobSoftwareDependency = db.Table( TABLE_JOB_SOFTWARE_DEP, db.metadata, db.Column("job_id", IDTypeWork, db.ForeignKey("%s.id" % TABLE_JOB), primary_key=True), db.Column("software_id", db.Integer, db.ForeignKey("%s.id" % TABLE_SOFTWARE), primary_key=True)) JobTagAssociation = db.Table( TABLE_JOB_TAG_ASSOC, db.metadata, db.Column("job_id", IDTypeWork, db.ForeignKey("%s.id" % TABLE_JOB), primary_key=True), db.Column("tag_id", db.Integer, db.ForeignKey("%s.id" % TABLE_TAG), primary_key=True)) JobDependencies = db.Table( TABLE_JOB_DEPENDENCIES, db.metadata, db.Column("parentid", IDTypeWork,
class JobQueue(db.Model, UtilityMixins, ReprMixin): """ Stores information about a job queue. Used for flexible, configurable distribution of computing capacity to jobs. """ __tablename__ = config.get("table_job_queue") __table_args__ = (UniqueConstraint("parent_jobqueue_id", "name"), ) REPR_COLUMNS = ("id", "name") id = id_column(IDTypeWork) parent_jobqueue_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_queue")), nullable=True, doc="The parent queue of this queue. If NULL, this is a top " "level queue.") name = db.Column(db.String(config.get("max_queue_name_length")), nullable=False, doc="The name of the job queue") minimum_agents = db.Column( db.Integer, nullable=True, doc="The scheduler will try to assign at least this number of " "agents to jobs in or below this queue as long as it " "can use them, before any other considerations.") maximum_agents = db.Column( db.Integer, nullable=True, doc="The scheduler will never assign more than this number of " "agents to jobs in or below this queue.") priority = db.Column( db.Integer, nullable=False, default=config.get("queue_default_priority"), doc="The priority of this job queue. The scheduler will not " "assign any nodes to other job queues or jobs with the " "same parent and a lower priority as long as this one " "can still use nodes. The minimum_agents column takes " "precedence over this.") weight = db.Column(db.Integer, nullable=False, default=config.get("queue_default_weight"), doc="The weight of this job queue. The scheduler will " "distribute available agents between jobs and job " "queues in the same queue in proportion to their " "weights.") fullpath = db.Column(db.String(config.get("max_queue_path_length")), doc="The path of this jobqueue. This column is a " "database denormalization. It is technically " "redundant, but faster to access than recursively " "querying all parent queues. If set to NULL, the " "path must be computed by recursively querying " "the parent queues.") # # Relationship # parent = db.relationship("JobQueue", remote_side=[id], backref=db.backref("children", lazy="dynamic"), doc="Relationship between this queue its parent") def path(self): # Import here instead of at the top to break circular dependency from pyfarm.scheduler.tasks import cache_jobqueue_path if self.fullpath: return self.fullpath else: cache_jobqueue_path.delay(self.id) path = "/%s" % (self.name or "") if self.parent: return self.parent.path() + path else: return path def child_queues_sorted(self): """ Return child queues sorted by number of currently assigned agents with priority as a secondary sort key. """ queues = [x for x in self.children] return sorted(queues, key=lambda x: x.num_assigned_agents(), reverse=True) def child_jobs(self, filters): # Import down here instead of at the top to avoid circular import from pyfarm.models.job import Job jobs_query = Job.query if self.id: jobs_query = jobs_query.filter_by(queue=self) wanted_states = [] if filters["state_paused"]: wanted_states.append(WorkState.PAUSED) if filters["state_running"]: wanted_states.append(WorkState.RUNNING) if filters["state_done"]: wanted_states.append(WorkState.DONE) if filters["state_failed"]: wanted_states.append(WorkState.FAILED) if filters["state_queued"]: jobs_query = jobs_query.filter( or_(Job.state == None, Job.state.in_(wanted_states))) else: jobs_query = jobs_query.filter(Job.state.in_(wanted_states)) return sorted(jobs_query.all(), key=lambda x: x.num_assigned_agents(), reverse=True) def num_assigned_agents(self): try: return self.assigned_agents_count except AttributeError: # Import down here instead of at the top to avoid circular import from pyfarm.models.task import Task from pyfarm.models.job import Job self.assigned_agents_count = 0 for queue in self.children: self.assigned_agents_count += queue.num_assigned_agents() self.assigned_agents_count +=\ db.session.query(distinct(Task.agent_id)).\ filter(Task.job.has(Job.queue == self), Task.agent_id != None, Task.agent.has( and_(Agent.state != AgentState.OFFLINE, Agent.state != AgentState.DISABLED)), or_(Task.state == None, Task.state == WorkState.RUNNING)).count() return self.assigned_agents_count def clear_assigned_counts(self): try: del self.assigned_agents_count except AttributeError: pass if self.parent: self.parent.clear_assigned_counts() def get_job_for_agent(self, agent, unwanted_job_ids=None): # Import down here instead of at the top to avoid circular import from pyfarm.models.job import Job supported_types = agent.get_supported_types() if not supported_types: return None available_ram = agent.ram if USE_TOTAL_RAM else agent.free_ram child_jobs = Job.query.filter( or_(Job.state == WorkState.RUNNING, Job.state == None), Job.job_queue_id == self.id, ~Job.parents.any( or_(Job.state == None, Job.state != WorkState.DONE)), Job.jobtype_version_id.in_(supported_types), Job.ram <= available_ram).all() child_jobs = [ x for x in child_jobs if (agent.satisfies_job_requirements(x) and x.id not in unwanted_job_ids) ] if unwanted_job_ids: child_jobs = [ x for x in child_jobs if x.id not in unwanted_job_ids ] child_queues = JobQueue.query.filter( JobQueue.parent_jobqueue_id == self.id).all() # Before anything else, enforce minimums for job in child_jobs: if job.state == _WorkState.RUNNING: if (job.num_assigned_agents() < (job.minimum_agents or 0) and job.num_assigned_agents() < (job.maximum_agents or maxsize) and job.can_use_more_agents()): return job elif job.minimum_agents and job.minimum_agents > 0: return job for queue in child_queues: if (queue.num_assigned_agents() < (queue.minimum_agents or 0) and queue.num_assigned_agents() < (queue.maximum_agents or maxsize)): job = queue.get_job_for_agent(agent, unwanted_job_ids) if job: return job objects_by_priority = {} for queue in child_queues: if queue.priority in objects_by_priority: objects_by_priority[queue.priority] += [queue] else: objects_by_priority[queue.priority] = [queue] for job in child_jobs: if job.priority in objects_by_priority: objects_by_priority[job.priority] += [job] else: objects_by_priority[job.priority] = [job] available_priorities = sorted(objects_by_priority.keys(), reverse=True) # Work through the priorities in descending order for priority in available_priorities: objects = objects_by_priority[priority] active_objects = [ x for x in objects if (type(x) != Job or x.state == _WorkState.RUNNING) ] weight_sum = reduce(lambda a, b: a + b.weight, active_objects, 0) total_assigned = reduce(lambda a, b: a + b.num_assigned_agents(), objects, 0) objects.sort(key=(lambda x: ((float(x.num_assigned_agents( )) / total_assigned) if total_assigned else 0) / ((float( x.weight) / weight_sum) if weight_sum and x.weight else 1))) selected_job = None for item in objects: if isinstance(item, Job): if item.state == _WorkState.RUNNING: if (item.can_use_more_agents() and item.num_assigned_agents() < (item.maximum_agents or maxsize)): if PREFER_RUNNING_JOBS: return item elif (selected_job is None or selected_job.time_submitted > item.time_submitted): selected_job = item elif (selected_job is None or selected_job.time_submitted > item.time_submitted): # If this job is not running yet, remember it, but keep # looking for already running or queued but older jobs selected_job = item if isinstance(item, JobQueue): if (item.num_assigned_agents() < (item.maximum_agents or maxsize)): job = item.get_job_for_agent(agent, unwanted_job_ids) if job: return job if selected_job: return selected_job return None @staticmethod def top_level_unique_check(mapper, connection, target): if target.parent_jobqueue_id is None: count = JobQueue.query.filter_by(parent_jobqueue_id=None, name=target.name).count() if count > 0: raise ValueError("Cannot have two jobqueues named %r at the " "top level" % target.name)