def test_readenv_eval(self):
key = uuid.uuid4().hex
for value in (True, False, 42, 3.141, None, [1, 2, 3]):
os.environ[key] = str(value)
self.assertEqual(read_env(key, eval_literal=True), value)
os.environ[key] = "f"
with self.assertRaises(ValueError):
read_env(key, eval_literal=True)
self.assertEqual(read_env(key, 42, eval_literal=True, raise_eval_exception=False), 42)
del os.environ[key]
def test_readenv_exists(self):
key = uuid.uuid4().hex
value = uuid.uuid4().hex
os.environ[key] = value
self.assertEqual(read_env(key), value)
del os.environ[key]
def test_readenv_missing(self):
key = uuid.uuid4().hex
with self.assertRaises(EnvironmentError):
read_env(key)
self.assertEqual(read_env(key, 42), 42)
return True
expected_regexp = self.expected_regexp
if isinstance(expected_regexp, STRING_TYPES):
expected_regexp = re.compile(expected_regexp)
if not expected_regexp.search(str(exc_value)):
raise self.failureException('"%s" does not match "%s"' % (expected_regexp.pattern, str(exc_value)))
return True
from voluptuous import Schema
from twisted.internet.defer import Deferred, succeed
from pyfarm.agent.entrypoints.parser import AgentArgumentParser
from pyfarm.agent.http.api.base import APIResource
PYFARM_AGENT_MASTER = read_env("PYFARM_AGENT_TEST_MASTER", "127.0.0.1:80")
DEFAULT_SOCKET_TIMEOUT = socket.getdefaulttimeout()
if ":" not in PYFARM_AGENT_MASTER:
raise ValueError("$PYFARM_AGENT_TEST_MASTER's format should be `ip:port`")
os.environ["PYFARM_AGENT_TEST_RUNNING"] = str(os.getpid())
class skipIf(object):
"""
Wrapping a test with this class will allow the test to
be skipped if ``should_skip`` evals as True.
"""
def __init__(self, should_skip, reason):
class Job(db.Model, ValidatePriorityMixin, WorkStateChangedMixin, ReprMixin):
"""
Defines the attributes and environment for a job. Individual commands
are kept track of by |Task|
"""
__tablename__ = TABLE_JOB
REPR_COLUMNS = ("id", "state", "project")
REPR_CONVERT_COLUMN = {
"state": repr}
MIN_CPUS = read_env_int("PYFARM_QUEUE_MIN_CPUS", 1)
MAX_CPUS = read_env_int("PYFARM_QUEUE_MAX_CPUS", 256)
MIN_RAM = read_env_int("PYFARM_QUEUE_MIN_RAM", 16)
MAX_RAM = read_env_int("PYFARM_QUEUE_MAX_RAM", 262144)
SPECIAL_RAM = read_env("PYFARM_AGENT_SPECIAL_RAM", [0], eval_literal=True)
SPECIAL_CPUS = read_env("PYFARM_AGENT_SPECIAL_CPUS", [0], eval_literal=True)
# quick check of the configured data
assert MIN_CPUS >= 1, "$PYFARM_QUEUE_MIN_CPUS must be > 0"
assert MAX_CPUS >= 1, "$PYFARM_QUEUE_MAX_CPUS must be > 0"
assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS"
assert MIN_RAM >= 1, "$PYFARM_QUEUE_MIN_RAM must be > 0"
assert MAX_RAM >= 1, "$PYFARM_QUEUE_MAX_RAM must be > 0"
assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM"
# shared work columns
id, state, priority, time_submitted, time_started, time_finished = \
work_columns(WorkState.QUEUED, "job.priority")
project_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_PROJECT),
doc="stores the project id")
job_type_id = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB_TYPE),
nullable=False,
doc=dedent("""
The foreign key which stores :class:`JobType.id`"""))
user = db.Column(db.String(MAX_USERNAME_LENGTH),
doc=dedent("""
The user this job should execute as. The agent
process will have to be running as root on platforms
that support setting the user id.
.. note::
The length of this field is limited by the
configuration value `job.max_username_length`
.. warning::
this may not behave as expected on all platforms
(windows in particular)"""))
notes = db.Column(db.Text, default="",
doc=dedent("""
Notes that are provided on submission or added after
the fact. This column is only provided for human
consumption is not scanned, index, or used when
searching"""))
# task data
cmd = db.Column(db.String(MAX_COMMAND_LENGTH),
doc=dedent("""
The platform independent command to run. Each agent will
resolve this value for itself when the task begins so a
command like `ping` will work on any platform it's
assigned to. The full command could be provided here,
but then the job must be tagged using
:class:`.JobSoftware` to limit which agent(s) it will
run on."""))
start = db.Column(db.Float,
doc=dedent("""
The first frame of the job to run. This value may
be a float so subframes can be processed."""))
end = db.Column(db.Float,
doc=dedent("""
The last frame of the job to run. This value may
be a float so subframes can be processed."""))
by = db.Column(db.Float, default=1,
doc=dedent("""
The number of frames to count by between `start` and
`end`. This column may also sometimes be referred to
as 'step' by other software."""))
batch = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_BATCH", 1),
doc=dedent("""
Number of tasks to run on a single agent at once.
Depending on the capabilities of the software being run
this will either cause a single process to execute on
the agent or multiple processes on after the other.
**configured by**: `job.batch`"""))
requeue = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_REQUEUE", 3),
doc=dedent("""
Number of times to requeue failed tasks
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, never requeue failed tasks
-1, requeue failed tasks indefinitely
**configured by**: `job.requeue`"""))
cpus = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_CPUS", 1),
doc=dedent("""
Number of cpus or threads each task should consume on
each agent. Depending on the job type being executed
this may result in additional cpu consumption, longer
wait times in the queue (2 cpus means 2 'fewer' cpus on
an agent), or all of the above.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, minimum number of cpu resources not required
-1, agent cpu is exclusive for a task from this job
**configured by**: `job.cpus`"""))
ram = db.Column(db.Integer,
default=read_env_int("PYFARM_QUEUE_DEFAULT_RAM", 32),
doc=dedent("""
Amount of ram a task from this job will require to be
free in order to run. A task exceeding this value will
not result in any special behavior.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
0, minimum amount of free ram not required
-1, agent ram is exclusive for a task from this job
**configured by**: `job.ram`"""))
ram_warning = db.Column(db.Integer, default=-1,
doc=dedent("""
Amount of ram used by a task before a warning
raised. A task exceeding this value will not
cause any work stopping behavior.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
-1, not set"""))
ram_max = db.Column(db.Integer, default=-1,
doc=dedent("""
Maximum amount of ram a task is allowed to consume on
an agent.
.. warning::
The task will be **terminated** if the ram in use
by the process exceeds this value.
.. csv-table:: **Special Values**
:header: Value, Result
:widths: 10, 50
-1, not set
"""))
attempts = db.Column(db.Integer,
doc=dedent("""
The number attempts which have been made on this
task. This value is auto incremented when
:attr:`state` changes to a value synonyms with a
running state."""))
hidden = db.Column(db.Boolean, default=False, nullable=False,
doc=dedent("""
If True, keep the job hidden from the queue and web
ui. This is typically set to True if you either want
to save a job for later viewing or if the jobs data
is being populated in a deferred manner."""))
environ = db.Column(JSONDict,
doc=dedent("""
Dictionary containing information about the environment
in which the job will execute.
.. note::
Changes made directly to this object are **not**
applied to the session."""))
args = db.Column(JSONList,
doc=dedent("""
List containing the command line arguments.
.. note::
Changes made directly to this object are **not**
applied to the session."""))
data = db.Column(JSONDict,
doc=dedent("""
Json blob containing additional data for a job
.. note::
Changes made directly to this object are **not**
applied to the session."""))
project = db.relationship("Project",
backref=db.backref("jobs", lazy="dynamic"),
doc=dedent("""
relationship attribute which retrieves the
associated project for the job"""))
# self-referential many-to-many relationship
parents = db.relationship("Job",
secondary=JobDependencies,
primaryjoin=id==JobDependencies.c.parentid,
secondaryjoin=id==JobDependencies.c.childid,
backref="children")
tasks_done = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.DONE,
doc=dedent("""
Relationship between this job and any |Task| objects which are
done."""))
tasks_failed = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.FAILED,
doc=dedent("""
Relationship between this job and any |Task| objects which have
failed."""))
tasks_queued = db.relationship("Task", lazy="dynamic",
primaryjoin="(Task.state == %s) & "
"(Task.job_id == Job.id)" % DBWorkState.QUEUED,
doc=dedent("""
Relationship between this job and any |Task| objects which
are queued."""))
# resource relationships
tags = db.relationship("Tag", backref="jobs", lazy="dynamic",
secondary=JobTagAssociation,
doc=dedent("""
Relationship between this job and
:class:`.Tag` objects"""))
software = db.relationship("Software",
secondary=JobSoftwareDependency,
backref=db.backref("jobs", lazy="dynamic"),
lazy="dynamic",
doc="software needed by this job")
@validates("ram", "cpus")
def validate_resource(self, key, value):
"""
Validation that ensures that the value provided for either
:attr:`.ram` or :attr:`.cpus` is a valid value with a given range
"""
key_upper = key.upper()
special = getattr(self, "SPECIAL_%s" % key_upper)
if value is None or value in special:
return value
min_value = getattr(self, "MIN_%s" % key_upper)
max_value = getattr(self, "MAX_%s" % key_upper)
# quick sanity check of the incoming config
assert isinstance(min_value, int), "db.min_%s must be an integer" % key
assert isinstance(max_value, int), "db.max_%s must be an integer" % key
assert min_value >= 1, "db.min_%s must be > 0" % key
assert max_value >= 1, "db.max_%s must be > 0" % key
# check the provided input
if min_value > value or value > max_value:
msg = "value for `%s` must be between " % key
msg += "%s and %s" % (min_value, max_value)
raise ValueError(msg)
return value
:const integer MAX_USERNAME_LENGTH:
the max length of a username
:const integer MAX_TAG_LENGTH:
the max length of a tag
.. note::
this value is shared amongst all tag columns and may be split into
multiple values at a later time
"""
from pyfarm.core.config import read_env, read_env_int
# table names
TABLE_PREFIX = read_env("PYFARM_DB_PREFIX", "pyfarm_")
TABLE_SOFTWARE = "%ssoftware" % TABLE_PREFIX
TABLE_TAG = "%stag" % TABLE_PREFIX
TABLE_AGENT = "%sagents" % TABLE_PREFIX
TABLE_AGENT_SOFTWARE_ASSOC = "%s_software_assoc" % TABLE_AGENT
TABLE_AGENT_TAG_ASSOC = "%s_tag_assoc" % TABLE_AGENT
TABLE_JOB = "%sjobs" % TABLE_PREFIX
TABLE_JOB_TYPE = "%s_jobtypes" % TABLE_JOB
TABLE_JOB_TAG_ASSOC = "%s_tag_assoc" % TABLE_JOB
TABLE_JOB_DEPENDENCIES = "%s_dependencies" % TABLE_JOB
TABLE_JOB_SOFTWARE_DEP = "%s_software_dep" % TABLE_JOB
TABLE_TASK = "%stask" % TABLE_PREFIX
TABLE_TASK_DEPENDENCIES = "%s_dependencies" % TABLE_TASK
TABLE_USERS = "%susers" % TABLE_PREFIX
TABLE_USERS_PROJECTS = "%s_projects" % TABLE_USERS
TABLE_USERS_USER = "******" % TABLE_USERS
:const integer MAX_USERNAME_LENGTH:
the max length of a username
:const integer MAX_TAG_LENGTH:
the max length of a tag
.. note::
this value is shared amongst all tag columns and may be split into
multiple values at a later time
"""
from pyfarm.core.config import read_env, read_env_int
# table names
TABLE_PREFIX = read_env("PYFARM_DB_PREFIX", "pyfarm_")
TABLE_SOFTWARE = "%ssoftware" % TABLE_PREFIX
TABLE_TAG = "%stag" % TABLE_PREFIX
TABLE_AGENT = "%sagents" % TABLE_PREFIX
TABLE_AGENT_SOFTWARE_ASSOC = "%s_software_assoc" % TABLE_AGENT
TABLE_AGENT_TAG_ASSOC = "%s_tag_assoc" % TABLE_AGENT
TABLE_JOB = "%sjobs" % TABLE_PREFIX
TABLE_JOB_TYPE = "%s_jobtypes" % TABLE_JOB
TABLE_JOB_TAG_ASSOC = "%s_tag_assoc" % TABLE_JOB
TABLE_JOB_DEPENDENCIES = "%s_dependencies" % TABLE_JOB
TABLE_JOB_SOFTWARE_DEP = "%s_software_dep" % TABLE_JOB
TABLE_TASK = "%stask" % TABLE_PREFIX
TABLE_TASK_DEPENDENCIES = "%s_dependencies" % TABLE_TASK
TABLE_USERS = "%susers" % TABLE_PREFIX
TABLE_USERS_PROJECTS = "%s_projects" % TABLE_USERS
TABLE_USERS_USER = "******" % TABLE_USERS
