class ValidationModel(db.Model, ValidateWorkStateMixin, ValidatePriorityMixin): __tablename__ = "%s_validation_mixin_test" % config.get("table_prefix") STATE_ENUM = WorkState id = db.Column(Integer, primary_key=True, autoincrement=True) state = db.Column(WorkStateEnum) attempts = db.Column(Integer) priority = db.Column(Integer)
class WorkStateChangedModel(db.Model, WorkStateChangedMixin): __tablename__ = "%s_state_change_test" % config.get("table_prefix") id = db.Column(Integer, primary_key=True, autoincrement=True) state = db.Column(WorkStateEnum) attempts = db.Column(Integer, nullable=False, default=0) time_started = db.Column(DateTime) time_finished = db.Column(DateTime)
class Software(db.Model, UtilityMixins): """ Model to represent a versioned piece of software that can be present on an agent and may be depended on by a job and/or jobtype .. note:: This table enforces two forms of uniqueness. The :attr:`id` column must be unique and the combination of these columns must also be unique to limit the frequency of duplicate data: * :attr:`software` * :attr:`version` .. autoattribute:: job_id """ __tablename__ = TABLE_SOFTWARE __table_args__ = ( UniqueConstraint("software", "version"), ) id = id_column() software = db.Column(db.String(MAX_TAG_LENGTH), nullable=False, doc=dedent(""" The name of the software""")) version = db.Column(db.String(MAX_TAG_LENGTH), default="any", nullable=False, doc=dedent(""" The version of the software. This value does not follow any special formatting rules because the format depends on the 3rd party."""))
class AgentCount(db.Model): __bind_key__ = 'statistics' __tablename__ = config.get("table_statistics_agent_count") counted_time = db.Column( db.DateTime, primary_key=True, nullable=False, autoincrement=False, doc="The point in time at which these counts were done") num_online = db.Column( db.Integer, nullable=False, doc="The number of agents that were in state `online` at counted_time") num_running = db.Column( db.Integer, nullable=False, doc="The number of agents that were in state `running` at counted_time" ) num_offline = db.Column( db.Integer, nullable=False, doc="The number of agents that were in state `offline` at counted_time" ) num_disabled = db.Column( db.Integer, nullable=False, doc="The number of agents that were in state `disabled` at " "counted_time")
class WorkStateChangedModel(db.Model, WorkStateChangedMixin): __tablename__ = "%s_state_change_test" % TABLE_PREFIX id = db.Column(Integer, primary_key=True, autoincrement=True) state = db.Column(WorkStateEnum) attempts = db.Column(Integer, default=0) time_started = db.Column(DateTime) time_finished = db.Column(DateTime)
class JobTagRequirement(db.Model, UtilityMixins): """ Model representing a dependency of a job on a tag If a job has a tag requirement, it will only run on agents that have that tag. """ __tablename__ = config.get("table_job_tag_req") __table_args__ = (UniqueConstraint("tag_id", "job_id"), ) id = id_column() tag_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_tag")), nullable=False, doc="Reference to the required tag") job_id = db.Column(IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), nullable=False, doc="Foreign key to :class:`Job.id`") negate = db.Column( db.Boolean, nullable=False, default=False, doc="If true, an agent that has this tag can not work on this job") job = db.relationship("Job", backref=db.backref("tag_requirements", lazy="dynamic", cascade="all, delete-orphan")) tag = db.relationship("Tag")
class JobType(db.Model): """ Stores the unique information necessary to execute a task """ __tablename__ = TABLE_JOB_TYPE id = id_column(db.Integer) name = db.Column(db.String(MAX_JOBTYPE_LENGTH), nullable=False, doc=dedent(""" The name of the job type. This can be either a human readable name or the name of the job type class itself.""")) description = db.Column(db.Text, nullable=True, doc=dedent(""" Human readable description of the job type. This field is not required and is not directly relied upon anywhere.""")) classname = db.Column(db.String(MAX_JOBTYPE_LENGTH), nullable=True, doc=dedent(""" The name of the job class contained within the file being loaded. This field may be null but when it's not provided :attr:`name` will be used instead.""")) code = db.Column(db.UnicodeText, nullable=False, doc=dedent(""" General field containing the 'code' to retrieve the job type. See below for information on what this field will contain depending on how the job will be loaded.""")) mode = db.Column(JobTypeLoadModeEnum, default=JobTypeLoadMode.IMPORT, nullable=False, doc=dedent(""" Indicates how the job type should be retrieved. .. csv-table:: **JobTypeLoadMode Enums** :header: Value, Result :widths: 10, 50 DOWNLOAD, job type will be downloaded remotely IMPORT, the remote agent will import the job type OPEN, code is loaded directly from a file on disk""")) jobs = db.relationship("Job", backref="job_type", lazy="dynamic", doc=dedent(""" Relationship between this jobtype and :class:`.Job` objects.""")) @validates("mode") def validates_mode(self, key, value): """ensures the value provided to :attr:`mode` is valid""" if value not in JobTypeLoadMode: raise ValueError("invalid value for mode") return value
class MixinModel(db.Model, UtilityMixins): __tablename__ = "%s_mixin_test" % config.get("table_prefix") id = db.Column(db.Integer, primary_key=True, autoincrement=True) a = db.Column(db.Integer) b = db.Column(db.String(512)) c = db.Column(IPv4Address) d = db.Column(db.Integer, nullable=False) e = db.relationship("MixinModel", secondary=MixinModelRelation1) f = db.relationship("MixinModel", secondary=MixinModelRelation2)
class MixinModel(db.Model, UtilityMixins): __tablename__ = "%s_mixin_test" % TABLE_PREFIX id = db.Column(db.Integer, primary_key=True, autoincrement=True) a = db.Column(db.Integer) b = db.Column(db.String(512)) c = db.Column(IPv4Address) def serialize_column(self, column): return column
class AgentMacAddress(db.Model): __tablename__ = config.get("table_agent_mac_address") __table_args__ = (UniqueConstraint("agent_id", "mac_address"), ) agent_id = db.Column( IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), primary_key=True, nullable=False) mac_address = db.Column( MACAddress, primary_key=True, nullable=False, autoincrement=False)
class TaskLog(db.Model, UtilityMixins, ReprMixin): """Table which represents a single task log entry""" __tablename__ = config.get("table_task_log") __table_args__ = (UniqueConstraint("identifier"),) id = id_column(db.Integer) identifier = db.Column( db.String(255), nullable=False, doc="The identifier for this log") agent_id = db.Column( IDTypeAgent, db.ForeignKey("%s.id" % config.get("table_agent")), nullable=True, doc="The agent this log was created on") created_on = db.Column( db.DateTime, default=datetime.utcnow, doc="The time when this log was created") # # Relationships # agent = db.relationship( "Agent", backref=db.backref("task_logs", lazy="dynamic"), doc="Relationship between an :class:`TaskLog`" "and the :class:`pyfarm.models.Agent` it was " "created on") task_associations = db.relationship( TaskTaskLogAssociation, backref="log", doc="Relationship between tasks and their logs." ) def num_queued_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=None).count() def num_running_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=WorkState.RUNNING).count() def num_failed_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=WorkState.FAILED).count() def num_done_tasks(self): return TaskTaskLogAssociation.query.filter_by( log=self, state=WorkState.DONE).count()
class Role(db.Model): """ Stores role information that can be used to give a user access to individual resources. """ __tablename__ = config.get("table_role") id = db.Column(db.Integer, primary_key=True, nullable=False) active = db.Column(db.Boolean, default=True, doc="Enables or disables a role. Disabling a role " "will prevent any users of this role from accessing " "PyFarm") name = db.Column(db.String(config.get("max_role_length")), unique=True, nullable=False, doc="The name of the role") expiration = db.Column( db.DateTime, doc="Role expiration. If this value is set then the role, and " "anyone assigned to it, will no longer be able to access " "PyFarm past the expiration.") description = db.Column(db.Text, doc="Human description of the role.") @classmethod def create(cls, name, description=None): """ Creates a role by the given name or returns an existing role if it already exists. """ if isinstance(name, Role): return name role = Role.query.filter_by(name=name).first() if role is None: role = cls(name=name, description=description) db.session.add(role) db.session.commit() return role def is_active(self): if self.expiration is None: return self.active return self.active and datetime.utcnow() < self.expiration
class TaskEventCount(db.Model): __bind_key__ = 'statistics' __tablename__ = config.get("table_statistics_task_event_count") id = id_column(db.Integer) time_start = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) time_end = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) # No foreign key reference, because this table is stored in a separate db # Code reading it will have to check for referential integrity manually. job_queue_id = db.Column(db.Integer, nullable=True, doc="ID of the jobqueue these stats refer to") num_new = db.Column( db.Integer, nullable=False, default=0, doc="Number of tasks that were newly created during the time period") num_deleted = db.Column( db.Integer, nullable=False, default=0, doc="Number of tasks that were deleted during the time period") num_restarted = db.Column( db.Integer, nullable=False, default=0, doc="Number of tasks that were restarted during the time period") num_started = db.Column( db.Integer, nullable=False, default=0, doc="Number of tasks that work was started on during the time period") num_failed = db.Column( db.Integer, nullable=False, default=0, doc="Number of tasks that failed during the time period") num_done = db.Column( db.Integer, nullable=False, default=0, doc="Number of tasks that were finished successfully during the time " "period")
class TypeModel(db.Model): __tablename__ = "%s_test_types" % config.get("table_prefix") id = db.Column(db.Integer, primary_key=True, autoincrement=True) ipv4 = db.Column(IPv4Address) mac = db.Column(MACAddress) json_dict = db.Column(JSONDict) json_list = db.Column(JSONList) agent_addr = db.Column(UseAgentAddressEnum) agent_state = db.Column(AgentStateEnum) work_state = db.Column(WorkStateEnum) uuid = db.Column(UUIDType)
class JobTypeSoftwareRequirement(db.Model, UtilityMixins): """ Model representing a dependency of a job on a software tag, with optional version constraints """ __tablename__ = config.get("table_job_type_software_req") __table_args__ = (UniqueConstraint("software_id", "jobtype_version_id"), ) software_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_software")), primary_key=True, doc="Reference to the required software") jobtype_version_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type_version")), primary_key=True, doc="Foreign key to :class:`JobTypeVersion.id`") min_version_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_software_version")), nullable=True, doc="Reference to the minimum required version") max_version_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_software_version")), nullable=True, doc="Reference to the maximum required version") # # Relationships # jobtype_version = db.relationship("JobTypeVersion", backref=db.backref( "software_requirements", lazy="dynamic", cascade="all, delete-orphan")) software = db.relationship("Software") min_version = db.relationship("SoftwareVersion", foreign_keys=[min_version_id]) max_version = db.relationship("SoftwareVersion", foreign_keys=[max_version_id])
def work_columns(state_default, priority_default): """ Produces some default columns which are used by models which produce work. Currently this includes |Job| and |Task| """ return ( # id id_column(IDTypeWork), # state db.Column(WorkStateEnum, default=state_default, doc=dedent(""" The state of the job with a value provided by :class:`.WorkState`""")), # priority db.Column(db.Integer, default=DEFAULT_PRIORITY, doc=dedent(""" The priority of the job relative to others in the queue. This is not the same as task priority. **configured by**: `%s`""" % priority_default)), # time_submitted db.Column(db.DateTime, default=datetime.now, doc=dedent(""" The time the job was submitted. By default this defaults to using :meth:`datetime.datetime.now` as the source of submission time. This value will not be set more than once and will not change even after a job is requeued.""")), # time_started db.Column(db.DateTime, doc=dedent(""" The time this job was started. By default this value is set when :attr:`state` is changed to an appropriate value or when a job is requeued.""")), # time_finished db.Column(db.DateTime, doc=dedent(""" Time the job was finished. This will be set when the last task finishes and reset if a job is requeued.""")))
class GPU(db.Model, UtilityMixins, ReprMixin): __tablename__ = config.get("table_gpu") __table_args__ = (UniqueConstraint("fullname"), ) id = id_column(db.Integer) fullname = db.Column(db.String(config.get("max_gpu_name_length")), nullable=False, doc="The full name of this graphics card model")
class PathMap(db.Model, ReprMixin, UtilityMixins): """ Defines a table which is used for cross-platform file path mappings. """ __tablename__ = config.get("table_path_map") id = id_column(db.Integer) path_linux = db.Column( db.String(config.get("max_path_length")), nullable=False, doc="The path on linux platforms") path_windows = db.Column( db.String(config.get("max_path_length")), nullable=False, doc="The path on Windows platforms") path_osx = db.Column( db.String(config.get("max_path_length")), nullable=False, doc="The path on Mac OS X platforms") tag_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_tag")), nullable=True, doc="The tag an agent needs to have for this path map " "to apply to it. " "If this is NULL, this path map applies to all " "agents, but is overridden by applying path maps " "that do specify a tag.") # # Relationships # tag = db.relationship( "Tag", backref=db.backref("path_maps", lazy="dynamic"), doc="Relationship attribute for the tag this path map " "applies to.")
class TypeModel(db.Model): __tablename__ = "%s_test_types" % TABLE_PREFIX id = db.Column(db.Integer, primary_key=True, autoincrement=True) ipv4 = db.Column(IPv4Address) json_dict = db.Column(JSONDict) json_list = db.Column(JSONList) agent_addr = db.Column(UseAgentAddressEnum) agent_state = db.Column(AgentStateEnum) work_state = db.Column(WorkStateEnum)
class Tag(db.Model, UtilityMixins): """ Model which provides tagging for :class:`.Job` and class:`.Agent` objects """ __tablename__ = TABLE_TAG __table_args__ = (UniqueConstraint("tag"), ) id = id_column() tag = db.Column(db.String(MAX_TAG_LENGTH), nullable=False, doc=dedent("""The actual value of the tag"""))
class Tag(db.Model, UtilityMixins): """ Model which provides tagging for :class:`.Job` and class:`.Agent` objects """ __tablename__ = config.get("table_tag") __table_args__ = (UniqueConstraint("tag"), ) id = id_column() tag = db.Column(db.String(config.get("max_tag_length")), nullable=False, doc="The actual value of the tag")
def id_column(column_type=None, **kwargs): """ Produces a column used for `id` on each table. Typically this is done using a class in :mod:`pyfarm.models.mixins` however because of the ORM and the table relationships it's cleaner to have a function produce the column. """ kwargs.setdefault("primary_key", True) kwargs.setdefault("autoincrement", True) kwargs.setdefault("doc", ID_DOCSTRING) kwargs.setdefault("nullable", False) return db.Column(column_type or Integer, **kwargs)
class JobNotifiedUser(db.Model): """ Defines the table containing users to be notified of certain events pertaining to jobs. """ __tablename__ = config.get("table_job_notified_users") user_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), primary_key=True, doc="The id of the user to be notified") job_id = db.Column(IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job")), primary_key=True, doc="The id of the associated job") on_success = db.Column( db.Boolean, nullable=False, default=True, doc="True if a user should be notified on successful " "completion of a job") on_failure = db.Column( db.Boolean, nullable=False, default=True, doc="True if a user should be notified of a job's failure") on_deletion = db.Column( db.Boolean, nullable=False, default=False, doc="True if a user should be notified on deletion of " "a job") user = db.relationship("User", backref=db.backref("subscribed_jobs", lazy="dynamic"))
class TaskCount(db.Model): __bind_key__ = 'statistics' __tablename__ = config.get("table_statistics_task_count") id = id_column(db.Integer) counted_time = db.Column( db.DateTime, nullable=False, default=datetime.utcnow, doc="The point in time at which these counts were done") # No foreign key reference, because this table is stored in a separate db # Code reading it will have to check for referential integrity manually. job_queue_id = db.Column(db.Integer, nullable=True, doc="ID of the jobqueue these stats refer to") total_queued = db.Column(db.Integer, nullable=False, doc="Number of queued tasks at `counted_time`") total_running = db.Column(db.Integer, nullable=False, doc="Number of running tasks at `counted_time`") total_done = db.Column(db.Integer, nullable=False, doc="Number of done tasks at `counted_time`") total_failed = db.Column(db.Integer, nullable=False, doc="Number of failed tasks at `counted_time`")
class JobGroup(db.Model, UtilityMixins): """ Used to group jobs together for better presentation in the UI """ __tablename__ = config.get("table_job_group") id = id_column(IDTypeWork) title = db.Column( db.String(config.get("max_jobgroup_name_length")), nullable=False, doc="The title of the job group's name") main_jobtype_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type")), nullable=False, doc="ID of the jobtype of the main job in this " "group. Purely for display and filtering.") user_id = db.Column( db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), doc="The id of the user who owns these jobs") # # Relationships # main_jobtype = db.relationship( "JobType", backref=db.backref("jobgroups", lazy="dynamic"), doc="The jobtype of the main job in this group") user = db.relationship( "User", backref=db.backref("jobgroups", lazy="dynamic"), doc="The user who owns these jobs")
class TaskTaskLogAssociation(db.Model): """Stores an association between the task table and a task log""" __tablename__ = config.get("table_task_log_assoc") __table_args__ = ( PrimaryKeyConstraint("task_log_id", "task_id", "attempt"),) task_log_id = db.Column( db.Integer, db.ForeignKey( "%s.id" % config.get("table_task_log"), ondelete="CASCADE"), doc="The ID of the task log") task_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_task"), ondelete="CASCADE"), doc="The ID of the job a task log is associated with") attempt = db.Column( db.Integer, autoincrement=False, doc="The attempt number for the given task log") state = db.Column( WorkStateEnum, nullable=True, doc="The state of the work being performed") # # Relationships # task = db.relationship( "Task", backref=db.backref( "log_associations", lazy="dynamic", passive_deletes=True))
class JobType(db.Model, UtilityMixins, ReprMixin): """ Stores the unique information necessary to execute a task """ __tablename__ = config.get("table_job_type") __table_args__ = (UniqueConstraint("name"), ) REPR_COLUMNS = ("id", "name") id = id_column(IDTypeWork) name = db.Column( db.String(config.get("job_type_max_name_length")), nullable=False, doc="The name of the job type. This can be either a human " "readable name or the name of the job type class itself.") description = db.Column( db.Text, nullable=True, doc="Human readable description of the job type. This field is not " "required and is not directly relied upon anywhere.") success_subject = db.Column( db.Text, nullable=True, doc="The subject line to use for notifications in case of " "success. Some substitutions, for example for the job title, " "are available.") success_body = db.Column(db.Text, nullable=True, doc="The email body to use for notifications in " "in case of success. Some substitutions, for " "example for the job title, are available.") fail_subject = db.Column(db.Text, nullable=True, doc="The subject line to use for notifications " "in case of failure. Some substitutions, for " "example for the job title, are available.") fail_body = db.Column(db.Text, nullable=True, doc="The email body to use for notifications in " "in case of success. Some substitutions, for " "example for the job title, are available.") @validates("name") def validate_name(self, key, value): if value == "": raise ValueError("Name cannot be empty") return value
class SoftwareVersion(db.Model, UtilityMixins): """ Model to represent a version for a given software """ __tablename__ = config.get("table_software_version") __table_args__ = (UniqueConstraint("software_id", "version"), UniqueConstraint("software_id", "rank")) id = id_column() software_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_software")), nullable=False, doc="The software this version belongs to") version = db.Column( db.String(config.get("max_tag_length")), default="any", nullable=False, doc="The version of the software. This value does not " "follow any special formatting rules because the " "format depends on the 3rd party.") rank = db.Column( db.Integer, nullable=False, doc="The rank of this version relative to other versions of " "the same software. Used to determine whether a version " "is higher or lower than another.") default = db.Column(db.Boolean, default=False, nullable=False, doc="If true, this software version will be registered" "on new nodes by default.") discovery_code = db.Column( db.UnicodeText, nullable=True, doc="Python code to discover if this software version is installed " "on a node") discovery_function_name = db.Column( db.String(config.get("max_discovery_function_name_length")), nullable=True, doc="The name of a function in `discovery_code` to call when " "checking for the presence of this software version on an agent.\n" "The function should return either a boolean (true if present, " "false if not) or a tuple of a boolean and a dict of named " "parameters describing this installation.")
class Software(db.Model, UtilityMixins): """ Model to represent a versioned piece of software that can be present on an agent and may be depended on by a job and/or jobtype through the appropriate SoftwareRequirement table """ __tablename__ = config.get("table_software") __table_args__ = (UniqueConstraint("software"), ) id = id_column() software = db.Column(db.String(config.get("max_tag_length")), nullable=False, doc="The name of the software") # # Relationships # versions = db.relationship("SoftwareVersion", backref=db.backref("software"), lazy="dynamic", order_by="asc(SoftwareVersion.rank)", cascade="all, delete-orphan", doc="All known versions of this software")
class Job(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin, WorkStateChangedMixin, ReprMixin, UtilityMixins): """ Defines the attributes and environment for a job. Individual commands are kept track of by :class:`Task` """ __tablename__ = config.get("table_job") REPR_COLUMNS = ("id", "state", "project") REPR_CONVERT_COLUMN = {"state": repr} STATE_ENUM = list(WorkState) + [None] # shared work columns id, state, priority, time_submitted, time_started, time_finished = \ work_columns(None, "job.priority") jobtype_version_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_type_version")), nullable=False, doc="The foreign key which stores :class:`JobTypeVersion.id`") job_queue_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_queue")), nullable=True, doc="The foreign key which stores :class:`JobQueue.id`") job_group_id = db.Column( IDTypeWork, db.ForeignKey("%s.id" % config.get("table_job_group")), nullable=True, doc="The foreign key which stores:class:`JobGroup.id`") user_id = db.Column(db.Integer, db.ForeignKey("%s.id" % config.get("table_user")), doc="The id of the user who owns this job") minimum_agents = db.Column( db.Integer, nullable=True, doc="The scheduler will try to assign at least this number " "of agents to this job as long as it can use them, " "before any other considerations.") maximum_agents = db.Column( db.Integer, nullable=True, doc="The scheduler will never assign more than this number" "of agents to this job.") weight = db.Column( db.Integer, nullable=False, default=config.get("queue_default_weight"), doc="The weight of this job. The scheduler will distribute " "available agents between jobs and job queues in the " "same queue in proportion to their weights.") title = db.Column(db.String(config.get("jobtitle_max_length")), nullable=False, doc="The title of this job") notes = db.Column( db.Text, default="", doc="Notes that are provided on submission or added after " "the fact. This column is only provided for human " "consumption, is not scanned, indexed, or used when " "searching") output_link = db.Column( db.Text, nullable=True, doc="An optional link to a URI where this job's output can " "be viewed.") # task data by = db.Column(db.Numeric(10, 4), default=1, doc="The number of frames to count by between `start` and " "`end`. This column may also sometimes be referred to " "as 'step' by other software.") num_tiles = db.Column( db.Integer, nullable=True, doc="How many regions to split frames into for rendering.") batch = db.Column( db.Integer, default=config.get("job_default_batch"), doc="Number of tasks to run on a single agent at once. Depending " "on the capabilities of the software being run this will " "either cause a single process to execute on the agent " "or multiple processes one after the other.") requeue = db.Column(db.Integer, default=config.get("job_requeue_default"), doc="Number of times to requeue failed tasks " "" ".. csv-table:: **Special Values**" " :header: Value, Result" " :widths: 10, 50" "" " 0, never requeue failed tasks" " -1, requeue failed tasks indefinitely") cpus = db.Column( db.Integer, default=config.get("job_default_cpus"), doc="Number of cpus or threads each task should consume on" "each agent. Depending on the job type being executed " "this may result in additional cpu consumption, longer " "wait times in the queue (2 cpus means 2 'fewer' cpus on " "an agent), or all of the above." "" ".. csv-table:: **Special Values**" " :header: Value, Result" " :widths: 10, 50" "" " 0, minimum number of cpu resources not required " " -1, agent cpu is exclusive for a task from this job") ram = db.Column( db.Integer, default=config.get("job_default_ram"), doc="Amount of ram a task from this job will require to be " "free in order to run. A task exceeding this value will " "not result in any special behavior." "" ".. csv-table:: **Special Values**" " :header: Value, Result" " :widths: 10, 50" "" "0, minimum amount of free ram not required" "-1, agent ram is exclusive for a task from this job") ram_warning = db.Column( db.Integer, nullable=True, doc="Amount of ram used by a task before a warning raised. " "A task exceeding this value will not cause any work " "stopping behavior.") ram_max = db.Column( db.Integer, nullable=True, doc="Maximum amount of ram a task is allowed to consume on " "an agent." "" ".. warning:: " " If set, the task will be **terminated** if the ram in " " use by the process exceeds this value.") hidden = db.Column( db.Boolean, default=False, nullable=False, doc="If True, keep the job hidden from the queue and web " "ui. This is typically set to True if you either want " "to save a job for later viewing or if the jobs data " "is being populated in a deferred manner.") environ = db.Column( JSONDict, doc="Dictionary containing information about the environment " "in which the job will execute. " "" ".. note::" " Changes made directly to this object are **not** " " applied to the session.") data = db.Column(JSONDict, doc="Json blob containing additional data for a job " "" ".. note:: " " Changes made directly to this object are **not** " " applied to the session.") to_be_deleted = db.Column( db.Boolean, nullable=False, default=False, doc="If true, the master will stop all running tasks for " "this job and then delete it.") completion_notify_sent = db.Column( db.Boolean, nullable=False, default=False, doc="Whether or not the finish notification mail has already " "been sent out.") autodelete_time = db.Column( db.Integer, nullable=True, default=None, doc="If not None, this job will be automatically deleted this " "number of seconds after it finishes.") # # Relationships # queue = db.relationship("JobQueue", backref=db.backref("jobs", lazy="dynamic"), doc="The queue for this job") group = db.relationship("JobGroup", backref=db.backref("jobs", lazy="dynamic"), doc="The job group this job belongs to") user = db.relationship("User", backref=db.backref("jobs", lazy="dynamic"), doc="The owner of this job") # self-referential many-to-many relationship parents = db.relationship("Job", secondary=JobDependency, primaryjoin=id == JobDependency.c.childid, secondaryjoin=id == JobDependency.c.parentid, backref="children") notified_users = db.relationship("JobNotifiedUser", lazy="dynamic", backref=db.backref("job"), cascade="all,delete") tasks_queued = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == None) & " "(Task.job_id == Job.id)", doc="Relationship between this job and any :class:`Task` " "objects which are queued.") tasks_running = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.RUNNING, doc="Relationship between this job and any :class:`Task` " "objects which are running.") tasks_done = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.DONE, doc="Relationship between this job and any :class:`Task` objects " "which are done.") tasks_failed = db.relationship( "Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.FAILED, doc="Relationship between this job and any :class:`Task` objects " "which have failed.") # resource relationships tags = db.relationship( "Tag", backref="jobs", lazy="dynamic", secondary=JobTagAssociation, doc="Relationship between this job and :class:`.Tag` objects") def paused(self): return self.state == WorkState.PAUSED def update_state(self): # Import here instead of at the top of the file to avoid a circular # import from pyfarm.scheduler.tasks import send_job_completion_mail from pyfarm.models.agent import Agent num_active_tasks = db.session.query(Task).\ filter(Task.job == self, or_(Task.state == None, and_( Task.state != WorkState.DONE, Task.state != WorkState.FAILED))).count() if num_active_tasks == 0: num_failed_tasks = db.session.query(Task).filter( Task.job == self, Task.state == WorkState.FAILED).count() if num_failed_tasks == 0: if self.state != _WorkState.DONE: logger.info( "Job %r (id %s): state transition %r -> 'done'", self.title, self.id, self.state) self.state = WorkState.DONE send_job_completion_mail.apply_async(args=[self.id, True], countdown=5) else: if self.state != _WorkState.FAILED: logger.info( "Job %r (id %s): state transition %r -> " "'failed'", self.title, self.id, self.state) self.state = WorkState.FAILED send_job_completion_mail.apply_async(args=[self.id, False], countdown=5) db.session.add(self) elif self.state != _WorkState.PAUSED: num_running_tasks = db.session.query(Task).\ filter(Task.job == self, Task.agent_id != None, Task.agent.has(and_(Agent.state != AgentState.OFFLINE, Agent.state != AgentState.DISABLED)), or_( Task.state == WorkState.RUNNING, Task.state == None)).count() if num_running_tasks == 0: logger.debug( "No running tasks in job %s (id %s), setting it " "to queued", self.title, self.id) self.state = None db.session.add(self) elif self.state != _WorkState.RUNNING: self.state = WorkState.RUNNING # Methods used by the scheduler def num_assigned_agents(self): # Import here instead of at the top of the file to avoid circular import from pyfarm.models.agent import Agent # Optimization: Blindly assume that we have no agents assigned if not # running if self.state != _WorkState.RUNNING: return 0 try: return self.assigned_agents_count except AttributeError: self.assigned_agents_count =\ db.session.query(distinct(Task.agent_id)).\ filter(Task.job == self, Task.agent_id != None, or_(Task.state == None, Task.state == WorkState.RUNNING), Task.agent.has( and_(Agent.state != AgentState.OFFLINE, Agent.state != AgentState.DISABLED)))\ .count() return self.assigned_agents_count def clear_assigned_counts(self): try: del self.assigned_agents_count except AttributeError: pass if self.queue: self.queue.clear_assigned_counts() def can_use_more_agents(self): # Import here instead of at the top of the file to avoid circular import from pyfarm.models.agent import Agent unassigned_tasks = Task.query.filter( Task.job == self, or_(Task.state == None, ~Task.state.in_([WorkState.DONE, WorkState.FAILED])), or_( Task.agent == None, Task.agent.has( Agent.state.in_([AgentState.OFFLINE, AgentState.DISABLED])))).count() return unassigned_tasks > 0 def get_batch(self, agent): # Import here instead of at the top of the file to avoid circular import from pyfarm.models.agent import Agent tasks_query = Task.query.filter( Task.job == self, ~Task.failed_in_agents.any(id=agent.id), or_(Task.state == None, ~Task.state.in_([WorkState.DONE, WorkState.FAILED])), or_(Task.agent == None, Task.agent.has(Agent.state.in_( [AgentState.OFFLINE, AgentState.DISABLED])))).\ order_by("frame asc, tile asc") batch = [] for task in tasks_query: if (len(batch) < self.batch and len(batch) < (self.jobtype_version.max_batch or maxsize) and (not self.jobtype_version.batch_contiguous or (len(batch) == 0 or batch[-1].frame + self.by == task.frame))): batch.append(task) return batch def alter_frame_range(self, start, end, by): # We have to import this down here instead of at the top to break a # circular dependency between the modules from pyfarm.scheduler.tasks import delete_task if end < start: raise ValueError("`end` must be greater than or equal to `start`") self.by = by required_frames = [] current_frame = start while current_frame <= end: required_frames.append(current_frame) current_frame += by existing_tasks = Task.query.filter_by(job=self).all() frames_to_create = required_frames num_created = 0 for task in existing_tasks: if task.frame not in required_frames: delete_task.delay(task.id) else: frames_to_create.remove(task.frame) for frame in frames_to_create: if self.num_tiles: for tile in range_(self.num_tiles - 1): num_created += 1 task = Task() task.job = self task.frame = frame task.tile = tile task.priority = self.priority db.session.add(task) else: num_created += 1 task = Task() task.job = self task.frame = frame task.priority = self.priority db.session.add(task) if frames_to_create: if self.state != WorkState.RUNNING: self.state = None if config.get("enable_statistics"): task_event_count = TaskEventCount(num_new=num_created, job_queue_id=self.job_queue_id) task_event_count.time_start = datetime.utcnow() task_event_count.time_end = datetime.utcnow() db.session.add(task_event_count) def rerun(self): """ Makes this job rerun all its task. Tasks that are currently running are left untouched. """ num_restarted = 0 for task in self.tasks: if task.state != _WorkState.RUNNING and task.state is not None: task.state = None task.agent = None task.failures = 0 db.session.add(task) num_restarted += 1 self.completion_notify_sent = False self.update_state() db.session.add(self) if config.get("enable_statistics"): task_event_count = TaskEventCount(job_queue_id=self.job_queue_id, num_restarted=num_restarted) task_event_count.time_start = datetime.utcnow() task_event_count.time_end = datetime.utcnow() db.session.add(task_event_count) db.session.commit() for child in self.children: child.rerun() def rerun_failed(self): """ Makes this job rerun all its failed tasks. Tasks that are done or are currently running are left untouched """ num_restarted = 0 for task in self.tasks: if task.state == _WorkState.FAILED: task.state = None task.agent = None task.failures = 0 db.session.add(task) num_restarted += 1 self.completion_notify_sent = False self.update_state() db.session.add(self) if config.get("enable_statistics"): task_event_count = TaskEventCount(job_queue_id=self.job_queue_id, num_restarted=num_restarted) task_event_count.time_start = datetime.utcnow() task_event_count.time_end = datetime.utcnow() db.session.commit() for child in self.children: child.rerun_failed() @validates("ram", "cpus") def validate_resource(self, key, value): """ Validation that ensures that the value provided for either :attr:`.ram` or :attr:`.cpus` is a valid value with a given range """ assert isinstance(value, int), "%s must be an integer" % key min_value = config.get("agent_min_%s" % key) max_value = config.get("agent_max_%s" % key) # check the provided input if min_value > value or value > max_value: msg = "value for `%s` must be between " % key msg += "%s and %s" % (min_value, max_value) raise ValueError(msg) return value @validates("progress") def validate_progress(self, key, value): if value < 0.0 or value > 1.0: raise ValueError("Progress must be between 0.0 and 1.0")