def test_readenv_eval(self): key = uuid.uuid4().hex for value in (True, False, 42, 3.141, None, [1, 2, 3]): os.environ[key] = str(value) self.assertEqual(read_env(key, eval_literal=True), value) os.environ[key] = "f" with self.assertRaises(ValueError): read_env(key, eval_literal=True) self.assertEqual(read_env(key, 42, eval_literal=True, raise_eval_exception=False), 42) del os.environ[key]
def test_readenv_exists(self): key = uuid.uuid4().hex value = uuid.uuid4().hex os.environ[key] = value self.assertEqual(read_env(key), value) del os.environ[key]
def test_readenv_missing(self): key = uuid.uuid4().hex with self.assertRaises(EnvironmentError): read_env(key) self.assertEqual(read_env(key, 42), 42)
return True expected_regexp = self.expected_regexp if isinstance(expected_regexp, STRING_TYPES): expected_regexp = re.compile(expected_regexp) if not expected_regexp.search(str(exc_value)): raise self.failureException('"%s" does not match "%s"' % (expected_regexp.pattern, str(exc_value))) return True from voluptuous import Schema from twisted.internet.defer import Deferred, succeed from pyfarm.agent.entrypoints.parser import AgentArgumentParser from pyfarm.agent.http.api.base import APIResource PYFARM_AGENT_MASTER = read_env("PYFARM_AGENT_TEST_MASTER", "127.0.0.1:80") DEFAULT_SOCKET_TIMEOUT = socket.getdefaulttimeout() if ":" not in PYFARM_AGENT_MASTER: raise ValueError("$PYFARM_AGENT_TEST_MASTER's format should be `ip:port`") os.environ["PYFARM_AGENT_TEST_RUNNING"] = str(os.getpid()) class skipIf(object): """ Wrapping a test with this class will allow the test to be skipped if ``should_skip`` evals as True. """ def __init__(self, should_skip, reason):
class Job(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, ReprMixin): """ Defines the attributes and environment for a job. Individual commands are kept track of by |Task| """ __tablename__ = TABLE_JOB REPR_COLUMNS = ("id", "state", "project") REPR_CONVERT_COLUMN = { "state": repr} MIN_CPUS = read_env_int("PYFARM_QUEUE_MIN_CPUS", 1) MAX_CPUS = read_env_int("PYFARM_QUEUE_MAX_CPUS", 256) MIN_RAM = read_env_int("PYFARM_QUEUE_MIN_RAM", 16) MAX_RAM = read_env_int("PYFARM_QUEUE_MAX_RAM", 262144) SPECIAL_RAM = read_env("PYFARM_AGENT_SPECIAL_RAM", [0], eval_literal=True) SPECIAL_CPUS = read_env("PYFARM_AGENT_SPECIAL_CPUS", [0], eval_literal=True) # quick check of the configured data assert MIN_CPUS >= 1, "$PYFARM_QUEUE_MIN_CPUS must be > 0" assert MAX_CPUS >= 1, "$PYFARM_QUEUE_MAX_CPUS must be > 0" assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS" assert MIN_RAM >= 1, "$PYFARM_QUEUE_MIN_RAM must be > 0" assert MAX_RAM >= 1, "$PYFARM_QUEUE_MAX_RAM must be > 0" assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM" # shared work columns id, state, priority, time_submitted, time_started, time_finished = \ work_columns(WorkState.QUEUED, "job.priority") project_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT), doc="stores the project id") job_type_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB_TYPE), nullable=False, doc=dedent(""" The foreign key which stores :class:`JobType.id`""")) user = db.Column(db.String(MAX_USERNAME_LENGTH), doc=dedent(""" The user this job should execute as. The agent process will have to be running as root on platforms that support setting the user id. .. note:: The length of this field is limited by the configuration value `job.max_username_length` .. warning:: this may not behave as expected on all platforms (windows in particular)""")) notes = db.Column(db.Text, default="", doc=dedent(""" Notes that are provided on submission or added after the fact. This column is only provided for human consumption is not scanned, index, or used when searching""")) # task data cmd = db.Column(db.String(MAX_COMMAND_LENGTH), doc=dedent(""" The platform independent command to run. Each agent will resolve this value for itself when the task begins so a command like `ping` will work on any platform it's assigned to. The full command could be provided here, but then the job must be tagged using :class:`.JobSoftware` to limit which agent(s) it will run on.""")) start = db.Column(db.Float, doc=dedent(""" The first frame of the job to run. This value may be a float so subframes can be processed.""")) end = db.Column(db.Float, doc=dedent(""" The last frame of the job to run. This value may be a float so subframes can be processed.""")) by = db.Column(db.Float, default=1, doc=dedent(""" The number of frames to count by between `start` and `end`. This column may also sometimes be referred to as 'step' by other software.""")) batch = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_BATCH", 1), doc=dedent(""" Number of tasks to run on a single agent at once. Depending on the capabilities of the software being run this will either cause a single process to execute on the agent or multiple processes on after the other. **configured by**: `job.batch`""")) requeue = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_REQUEUE", 3), doc=dedent(""" Number of times to requeue failed tasks .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 0, never requeue failed tasks -1, requeue failed tasks indefinitely **configured by**: `job.requeue`""")) cpus = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_CPUS", 1), doc=dedent(""" Number of cpus or threads each task should consume on each agent. Depending on the job type being executed this may result in additional cpu consumption, longer wait times in the queue (2 cpus means 2 'fewer' cpus on an agent), or all of the above. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 0, minimum number of cpu resources not required -1, agent cpu is exclusive for a task from this job **configured by**: `job.cpus`""")) ram = db.Column(db.Integer, default=read_env_int("PYFARM_QUEUE_DEFAULT_RAM", 32), doc=dedent(""" Amount of ram a task from this job will require to be free in order to run. A task exceeding this value will not result in any special behavior. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 0, minimum amount of free ram not required -1, agent ram is exclusive for a task from this job **configured by**: `job.ram`""")) ram_warning = db.Column(db.Integer, default=-1, doc=dedent(""" Amount of ram used by a task before a warning raised. A task exceeding this value will not cause any work stopping behavior. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 -1, not set""")) ram_max = db.Column(db.Integer, default=-1, doc=dedent(""" Maximum amount of ram a task is allowed to consume on an agent. .. warning:: The task will be **terminated** if the ram in use by the process exceeds this value. .. csv-table:: **Special Values** :header: Value, Result :widths: 10, 50 -1, not set """)) attempts = db.Column(db.Integer, doc=dedent(""" The number attempts which have been made on this task. This value is auto incremented when :attr:`state` changes to a value synonyms with a running state.""")) hidden = db.Column(db.Boolean, default=False, nullable=False, doc=dedent(""" If True, keep the job hidden from the queue and web ui. This is typically set to True if you either want to save a job for later viewing or if the jobs data is being populated in a deferred manner.""")) environ = db.Column(JSONDict, doc=dedent(""" Dictionary containing information about the environment in which the job will execute. .. note:: Changes made directly to this object are **not** applied to the session.""")) args = db.Column(JSONList, doc=dedent(""" List containing the command line arguments. .. note:: Changes made directly to this object are **not** applied to the session.""")) data = db.Column(JSONDict, doc=dedent(""" Json blob containing additional data for a job .. note:: Changes made directly to this object are **not** applied to the session.""")) project = db.relationship("Project", backref=db.backref("jobs", lazy="dynamic"), doc=dedent(""" relationship attribute which retrieves the associated project for the job""")) # self-referential many-to-many relationship parents = db.relationship("Job", secondary=JobDependencies, primaryjoin=id==JobDependencies.c.parentid, secondaryjoin=id==JobDependencies.c.childid, backref="children") tasks_done = db.relationship("Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.DONE, doc=dedent(""" Relationship between this job and any |Task| objects which are done.""")) tasks_failed = db.relationship("Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.FAILED, doc=dedent(""" Relationship between this job and any |Task| objects which have failed.""")) tasks_queued = db.relationship("Task", lazy="dynamic", primaryjoin="(Task.state == %s) & " "(Task.job_id == Job.id)" % DBWorkState.QUEUED, doc=dedent(""" Relationship between this job and any |Task| objects which are queued.""")) # resource relationships tags = db.relationship("Tag", backref="jobs", lazy="dynamic", secondary=JobTagAssociation, doc=dedent(""" Relationship between this job and :class:`.Tag` objects""")) software = db.relationship("Software", secondary=JobSoftwareDependency, backref=db.backref("jobs", lazy="dynamic"), lazy="dynamic", doc="software needed by this job") @validates("ram", "cpus") def validate_resource(self, key, value): """ Validation that ensures that the value provided for either :attr:`.ram` or :attr:`.cpus` is a valid value with a given range """ key_upper = key.upper() special = getattr(self, "SPECIAL_%s" % key_upper) if value is None or value in special: return value min_value = getattr(self, "MIN_%s" % key_upper) max_value = getattr(self, "MAX_%s" % key_upper) # quick sanity check of the incoming config assert isinstance(min_value, int), "db.min_%s must be an integer" % key assert isinstance(max_value, int), "db.max_%s must be an integer" % key assert min_value >= 1, "db.min_%s must be > 0" % key assert max_value >= 1, "db.max_%s must be > 0" % key # check the provided input if min_value > value or value > max_value: msg = "value for `%s` must be between " % key msg += "%s and %s" % (min_value, max_value) raise ValueError(msg) return value
:const integer MAX_USERNAME_LENGTH: the max length of a username :const integer MAX_TAG_LENGTH: the max length of a tag .. note:: this value is shared amongst all tag columns and may be split into multiple values at a later time """ from pyfarm.core.config import read_env, read_env_int # table names TABLE_PREFIX = read_env("PYFARM_DB_PREFIX", "pyfarm_") TABLE_SOFTWARE = "%ssoftware" % TABLE_PREFIX TABLE_TAG = "%stag" % TABLE_PREFIX TABLE_AGENT = "%sagents" % TABLE_PREFIX TABLE_AGENT_SOFTWARE_ASSOC = "%s_software_assoc" % TABLE_AGENT TABLE_AGENT_TAG_ASSOC = "%s_tag_assoc" % TABLE_AGENT TABLE_JOB = "%sjobs" % TABLE_PREFIX TABLE_JOB_TYPE = "%s_jobtypes" % TABLE_JOB TABLE_JOB_TAG_ASSOC = "%s_tag_assoc" % TABLE_JOB TABLE_JOB_DEPENDENCIES = "%s_dependencies" % TABLE_JOB TABLE_JOB_SOFTWARE_DEP = "%s_software_dep" % TABLE_JOB TABLE_TASK = "%stask" % TABLE_PREFIX TABLE_TASK_DEPENDENCIES = "%s_dependencies" % TABLE_TASK TABLE_USERS = "%susers" % TABLE_PREFIX TABLE_USERS_PROJECTS = "%s_projects" % TABLE_USERS TABLE_USERS_USER = "******" % TABLE_USERS