general implementation.
"""
from sqlalchemy.orm import validates
from sqlalchemy.schema import UniqueConstraint
from pyfarm.core.logger import getLogger
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork
__all__ = ("JobType", )
logger = getLogger("models.jobtype")
class JobType(db.Model, UtilityMixins, ReprMixin):
"""
Stores the unique information necessary to execute a task
"""
__tablename__ = config.get("table_job_type")
__table_args__ = (UniqueConstraint("name"),)
REPR_COLUMNS = ("id", "name")
id = id_column(IDTypeWork)
name = db.Column(
db.String(config.get("job_type_max_name_length")),
nullable=False,
from http.client import (
OK, CREATED, CONFLICT, NOT_FOUND, NO_CONTENT, BAD_REQUEST)
from flask import g
from flask.views import MethodView
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.job import Job
from pyfarm.models.jobqueue import JobQueue
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model
logger = getLogger("api.jobqueues")
# Load model mappings once per process
JOBQUEUE_MODEL_MAPPINGS = JobQueue.types().mappings
def schema():
"""
Returns the basic schema of :class:`.JobQueue`
.. http:get:: /api/v1/jobqueues/schema HTTP/1.1
**Request**
.. sourcecode:: http
from flask.views import MethodView
from flask import g
from sqlalchemy import func, asc
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.models.user import User
from pyfarm.models.jobtype import JobType
from pyfarm.models.job import Job
from pyfarm.models.task import Task
from pyfarm.models.jobgroup import JobGroup
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify, validate_with_model
from pyfarm.master.application import db
logger = getLogger("api.jobgroups")
AUTOCREATE_USERS = config.get("autocreate_users")
AUTO_USER_EMAIL = config.get("autocreate_user_email")
def schema():
"""
Returns the basic schema of :class:`.JobGroup`
.. http:get:: /api/v1/jobgroups/schema HTTP/1.1
**Request**
.. sourcecode:: http
def test_get_logger_name(self):
logger = getLogger("foo")
self.assertEqual(logger.name, "pf.foo")
self.assertFalse(logger.handlers)
import json
from calendar import timegm
from datetime import timedelta, datetime
from flask import render_template, request
from sqlalchemy import or_
from pyfarm.core.logger import getLogger
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.config import config
from pyfarm.master.application import db
logger = getLogger("ui.task_events")
class TotalsAverage(object):
def __init__(self, first_sample, last_sum=None):
self.time_start = first_sample.counted_time
self.num_samples_by_queue = {first_sample.job_queue_id: 1}
self.avg_queued_by_queue = (
last_sum.avg_queued_by_queue if last_sum else {})
self.avg_queued_by_queue[first_sample.job_queue_id] =\
first_sample.total_queued
self.avg_running_by_queue = (
last_sum.avg_running_by_queue if last_sum else {})
self.avg_running_by_queue[first_sample.job_queue_id] =\
from flask import g
from flask.views import MethodView
from sqlalchemy import or_
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.pathmap import PathMap
from pyfarm.models.tag import Tag
from pyfarm.models.agent import Agent
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify, validate_with_model, get_uuid_argument
logger = getLogger("api.pathmaps")
def schema():
"""
Returns the basic schema of :class:`.Agent`
.. http:get:: /api/v1/pathmaps/schema HTTP/1.1
**Request**
.. sourcecode:: http
GET /api/v1/pathmaps/schema HTTP/1.1
Accept: application/json
from logging import DEBUG
from sqlalchemy import event, distinct, or_, and_
from sqlalchemy.schema import UniqueConstraint
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork
from pyfarm.models.agent import Agent
PREFER_RUNNING_JOBS = config.get("queue_prefer_running_jobs")
USE_TOTAL_RAM = config.get("use_total_ram_for_scheduling")
logger = getLogger("pf.models.jobqueue")
if config.get("debug_queue"):
logger.setLevel(DEBUG)
class JobQueue(db.Model, UtilityMixins, ReprMixin):
"""
Stores information about a job queue. Used for flexible, configurable
distribution of computing capacity to jobs.
"""
__tablename__ = config.get("table_job_queue")
__table_args__ = (UniqueConstraint("parent_jobqueue_id", "name"), )
REPR_COLUMNS = ("id", "name")
from pyfarm.models.job import Job, JobDependency
from pyfarm.models.jobtype import JobType
from pyfarm.models.disk import AgentDisk
from pyfarm.models.agent import Agent, AgentTagAssociation, GPUInAgent
from pyfarm.models.user import User, Role
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.pathmap import PathMap
from pyfarm.models.tasklog import TaskLog
from pyfarm.models.gpu import GPU
from pyfarm.models.jobgroup import JobGroup
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.utility import timedelta_format
logger = getLogger("master.entrypoints")
def load_before_first(app_instance, database_instance):
if app_instance.debug:
app_instance.before_first_request_funcs.append(
database_instance.create_all)
def load_error_handlers(app_instance):
"""loads the error handlers onto application instance"""
# create the handlers
bad_request = partial(
error_handler, code=BAD_REQUEST,
default=lambda: "bad request to %s" % request.url)
unauthorized = partial(
from pyfarm.core.enums import AgentState, WorkState
from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.task import Task
from pyfarm.models.job import Job
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.config import config
from pyfarm.master.application import db
from pyfarm.scheduler.celery_app import celery_app
logger = getLogger("pf.scheduler.statistics_tasks")
# TODO Get logger configuration from pyfarm config
logger.setLevel(DEBUG)
@celery_app.task(ignore_result=True)
def count_agents():
logger.debug("Counting known agents now")
num_online = Agent.query.filter_by(state=AgentState.ONLINE).count()
num_offline = Agent.query.filter_by(state=AgentState.OFFLINE).count()
num_running = Agent.query.filter_by(state=AgentState.RUNNING).count()
num_disabled = Agent.query.filter_by(state=AgentState.DISABLED).count()
agent_count = AgentCount(counted_time=datetime.utcnow(),
num_online=num_online,
num_offline=num_offline,
from logging import DEBUG
from sqlalchemy import event, distinct, or_, and_
from sqlalchemy.schema import UniqueConstraint
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork
from pyfarm.models.agent import Agent
PREFER_RUNNING_JOBS = config.get("queue_prefer_running_jobs")
USE_TOTAL_RAM = config.get("use_total_ram_for_scheduling")
logger = getLogger("pf.models.jobqueue")
if config.get("debug_queue"):
logger.setLevel(DEBUG)
class JobQueue(db.Model, UtilityMixins, ReprMixin):
"""
Stores information about a job queue. Used for flexible, configurable
distribution of computing capacity to jobs.
"""
__tablename__ = config.get("table_job_queue")
__table_args__ = (UniqueConstraint("parent_jobqueue_id", "name"),)
REPR_COLUMNS = ("id", "name")
ValidateWorkStateMixin,
UtilityMixins,
)
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.task import Task
try:
# pylint: disable=undefined-variable
range_ = xrange
except NameError:
range_ = range
__all__ = ("Job",)
logger = getLogger("models.job")
JobTagAssociation = db.Table(
config.get("table_job_tag_assoc"),
db.metadata,
db.Column(
"job_id",
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job")),
primary_key=True,
doc="The id of the job associated with this task",
),
db.Column(
"tag_id",
db.Integer,
from itsdangerous import URLSafeTimedSerializer
from sqlalchemy.engine import Engine
from sqlalchemy import event
from werkzeug.exceptions import BadRequest
from werkzeug.routing import BaseConverter, ValidationError
from pyfarm.core.enums import NOTSET, STRING_TYPES, PY3
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
POST_METHODS = set(("POST", "PUT"))
IGNORED_MIMETYPES = set((
"application/x-www-form-urlencoded", "multipart/form-data",
"application/zip", "text/csv"))
logger = getLogger("app")
class UUIDConverter(BaseConverter):
"""
A URL converter for UUIDs. This class is loaded as part of
the Flask application setup and may be used in url routing:
.. code-block:: python
@app.route('/foo/<uuid:value>')
def foobar(value):
pass
When a request such as ``GET /foo/F9A63B47-66BF-4E2B-A545-879986BB7CA9``
is made :class:`UUIDConverter` will receive ``value`` to :meth:`to_python`
from datetime import datetime
from collections import namedtuple
try:
from httplib import INTERNAL_SERVER_ERROR
except ImportError:
from http.client import INTERNAL_SERVER_ERROR
from sqlalchemy.orm import validates, class_mapper
from pyfarm.core.enums import _WorkState, Values, PY2
from pyfarm.core.logger import getLogger
from pyfarm.models.core.types import IPAddress
from pyfarm.master.config import config
logger = getLogger("models.mixin")
# stores information about a model's columns
# and relationships
ModelTypes = namedtuple("ModelTypes",
("primary_keys", "autoincrementing", "columns",
"required", "relationships", "mappings"))
class ValidatePriorityMixin(object):
"""
Mixin that adds a `state` column and uses a class
level `STATE_ENUM` attribute to assist in validation.
"""
MIN_PRIORITY = config.get("queue_min_priority")
MAX_PRIORITY = config.get("queue_max_priority")
from io import StringIO
from pkg_resources import (DistributionNotFound, get_distribution,
resource_filename)
import yaml
try:
from yaml.loader import CLoader as Loader
except ImportError: # pragma: no cover
from yaml.loader import Loader
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import (STRING_TYPES, NUMERIC_TYPES, NOTSET, LINUX, MAC,
WINDOWS, range_)
logger = getLogger("core.config")
# Boolean values as strings that can match a value we
# pulled from the environment after calling .lower().
BOOLEAN_TRUE = set(["1", "t", "y", "true", "yes"])
BOOLEAN_FALSE = set(["0", "f", "n", "false", "no"])
def read_env(envvar,
default=NOTSET,
warn_if_unset=False,
eval_literal=False,
raise_eval_exception=True,
log_result=True,
desc=None,
log_defaults=False):
NOT_FOUND, NO_CONTENT, OK, CREATED, BAD_REQUEST, INTERNAL_SERVER_ERROR,
CONFLICT)
from flask import g, Response
from flask.views import MethodView
from sqlalchemy import func
from sqlalchemy.exc import DatabaseError
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.software import Software, SoftwareVersion
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model
logger = getLogger("api.software")
class VersionParseError(Exception):
"""
Raised by :func:`extract_version_dicts` when the
function is unable to parse a version.
"""
def extract_version_dicts(json_in):
"""Extracts and returns a list of versions from ``json_in``."""
out = []
version_objects = json_in.pop("versions", [])
if not isinstance(version_objects, list):
raise VersionParseError("Column versions must be a list.")
from os.path import join, realpath
from errno import EEXIST
from flask.views import MethodView
from flask import g, redirect, send_file, request, Response
from sqlalchemy.exc import IntegrityError
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
from pyfarm.models.tasklog import TaskLog, TaskTaskLogAssociation
from pyfarm.models.task import Task
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model, isuuid
logger = getLogger("api.tasklogs")
LOGFILES_DIR = config.get("tasklogs_dir")
try:
makedirs(LOGFILES_DIR)
except OSError as e: # pragma: no cover
if e.errno != EEXIST:
raise
class LogsInTaskAttemptsIndexAPI(MethodView):
def get(self, job_id, task_id, attempt):
"""
A ``GET`` to this endpoint will return a list of all known logs that are
associated with this attempt at running this task
from httplib import NOT_FOUND, NO_CONTENT, OK, CREATED, BAD_REQUEST
except ImportError: # pragma: no cover
from http.client import NOT_FOUND, NO_CONTENT, OK, CREATED, BAD_REQUEST
from flask import url_for, g
from flask.views import MethodView
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.agent import Agent
from pyfarm.models.job import Job
from pyfarm.models.tag import Tag
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model
logger = getLogger("api.tags")
def schema():
"""
Returns the basic schema of :class:`.Tag`
.. http:get:: /api/v1/tags/schema/ HTTP/1.1
**Request**
.. sourcecode:: http
GET /api/v1/tags/schema/ HTTP/1.1
Accept: application/json
from sqlalchemy import func, desc, asc, or_, distinct
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.scheduler.tasks import delete_job, stop_task, assign_tasks
from pyfarm.models.job import (
Job, JobDependency, JobTagAssociation, JobNotifiedUser)
from pyfarm.models.tag import Tag, JobTagRequirement
from pyfarm.models.task import Task
from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.user import User
from pyfarm.master.application import db
logger = getLogger("ui.jobs")
def jobs():
queued_count_query = db.session.query(
Task.job_id, func.count('*').label('t_queued')).\
filter(Task.state == None).group_by(Task.job_id).subquery()
running_count_query = db.session.query(
Task.job_id, func.count('*').label('t_running')).\
filter(Task.state == WorkState.RUNNING).\
group_by(Task.job_id).subquery()
done_count_query = db.session.query(
Task.job_id, func.count('*').label('t_done')).\
filter(Task.state == WorkState.DONE).\
group_by(Task.job_id).subquery()
failed_count_query = db.session.query(
Task.job_id, func.count('*').label('t_failed')).\
from os.path import join, isfile
from errno import EEXIST
try:
from httplib import BAD_REQUEST, CREATED, NOT_FOUND
except ImportError: # pragma: no cover
from http.client import BAD_REQUEST, CREATED, NOT_FOUND
from flask.views import MethodView
from flask import request, redirect, send_file
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify
logger = getLogger("api.agents")
VERSION_REGEX = re.compile("\d+(\.\d+(\.\d+)?)?((-pre\d?)|(-dev\d?)|(-rc?\d?)|"
"(-alpha\d?)|(-beta\d?))?$")
UPDATES_DIR = config.get("agent_updates_dir")
UPDATES_WEBDIR = config.get("agent_updates_webdir")
try:
makedirs(UPDATES_DIR)
except OSError as e: # pragma: no cover
if e.errno != EEXIST:
raise
class AgentUpdatesAPI(MethodView):
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.scheduler.tasks import delete_job, stop_task, assign_tasks
from pyfarm.models.job import (Job, JobDependency, JobTagAssociation,
JobNotifiedUser)
from pyfarm.models.tag import Tag, JobTagRequirement
from pyfarm.models.task import Task
from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.user import User
from pyfarm.master.application import db
from pyfarm.master.config import config
logger = getLogger("ui.jobs")
def jobs():
queued_count_query = db.session.query(
Task.job_id, func.count('*').label('t_queued')).\
filter(Task.state == None).group_by(Task.job_id).subquery()
running_count_query = db.session.query(
Task.job_id, func.count('*').label('t_running')).\
filter(Task.state == WorkState.RUNNING).\
group_by(Task.job_id).subquery()
done_count_query = db.session.query(
Task.job_id, func.count('*').label('t_done')).\
filter(Task.state == WorkState.DONE).\
group_by(Task.job_id).subquery()
failed_count_query = db.session.query(
from datetime import datetime
from sqlalchemy import event
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.types import IDTypeAgent, IDTypeWork
from pyfarm.models.core.functions import work_columns, repr_enum
from pyfarm.models.core.mixins import (
ValidatePriorityMixin, UtilityMixins, ReprMixin, ValidateWorkStateMixin)
__all__ = ("Task", )
logger = getLogger("models.task")
class Task(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin,
UtilityMixins, ReprMixin):
"""
Defines a task which a child of a :class:`Job`. This table represents
rows which contain the individual work unit(s) for a job.
"""
__tablename__ = config.get("table_task")
STATE_ENUM = list(WorkState) + [None]
STATE_DEFAULT = None
REPR_COLUMNS = ("id", "state", "frame", "project")
REPR_CONVERT_COLUMN = {"state": partial(repr_enum, enum=STATE_ENUM)}
# shared work columns
def test_get_logger_name(self):
logger = getLogger("foo")
self.assertEqual(logger.name, "pf.foo")
self.assertFalse(logger.handlers)
to an individual job. See :mod:`pyfarm.models.job` for more the more
general implementation.
"""
from sqlalchemy.orm import validates
from sqlalchemy.schema import UniqueConstraint
from pyfarm.core.logger import getLogger
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork
__all__ = ("JobType", )
logger = getLogger("models.jobtype")
class JobType(db.Model, UtilityMixins, ReprMixin):
"""
Stores the unique information necessary to execute a task
"""
__tablename__ = config.get("table_job_type")
__table_args__ = (UniqueConstraint("name"), )
REPR_COLUMNS = ("id", "name")
id = id_column(IDTypeWork)
name = db.Column(
db.String(config.get("job_type_max_name_length")),
nullable=False,
from datetime import datetime
from textwrap import dedent
from flask.ext.login import UserMixin
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES, PY3
from pyfarm.master.application import app, db, login_serializer
from pyfarm.models.core.mixins import ReprMixin
from pyfarm.models.core.functions import split_and_extend
from pyfarm.models.core.cfg import (
TABLE_USERS_USER, TABLE_USERS_ROLE, TABLE_USERS_USER_ROLES,
MAX_USERNAME_LENGTH, SHA256_ASCII_LENGTH, MAX_EMAILADDR_LENGTH,
MAX_ROLE_LENGTH, TABLE_USERS_PROJECTS, TABLE_PROJECT)
logger = getLogger("models.users")
# roles the user is a member of
UserRoles = db.Table(
TABLE_USERS_USER_ROLES,
db.Column("user_id", db.Integer,
db.ForeignKey("%s.id" % TABLE_USERS_USER)),
db.Column("role_id", db.Integer,
db.ForeignKey("%s.id" % TABLE_USERS_ROLE)))
# projects the user is a member of
UserProjects = db.Table(
TABLE_USERS_PROJECTS,
db.Column("user_id", db.Integer,
db.ForeignKey("%s.id" % TABLE_USERS_USER), primary_key=True),
db.Column("project_id", db.Integer,
except ImportError: # pragma: no cover
from http.client import OK, CREATED, CONFLICT, NOT_FOUND, BAD_REQUEST, NO_CONTENT, METHOD_NOT_ALLOWED
from flask import g, Response
from flask.views import MethodView
from sqlalchemy import or_, func, sql
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES, PY3
from pyfarm.models.software import Software, SoftwareVersion, JobTypeSoftwareRequirement
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify
logger = getLogger("api.jobtypes")
class ObjectNotFound(Exception):
pass
def parse_requirements(requirements):
"""
Takes a list dicts specifying a software and optional min- and max-versions
and returns a list of :class:`JobRequirement` objects.
Raises TypeError if the input was not as expected or ObjectNotFound if a
referenced software of or version was not found.
:param list requirements:
from io import StringIO
from pkg_resources import (
DistributionNotFound, get_distribution, resource_filename)
import yaml
try:
from yaml.loader import CLoader as Loader
except ImportError: # pragma: no cover
from yaml.loader import Loader
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import (
STRING_TYPES, NUMERIC_TYPES, NOTSET, LINUX, MAC, WINDOWS, range_)
logger = getLogger("core.config")
# Boolean values as strings that can match a value we
# pulled from the environment after calling .lower().
BOOLEAN_TRUE = set(["1", "t", "y", "true", "yes"])
BOOLEAN_FALSE = set(["0", "f", "n", "false", "no"])
def read_env(envvar, default=NOTSET, warn_if_unset=False, eval_literal=False,
raise_eval_exception=True, log_result=True, desc=None,
log_defaults=False):
"""
Lookup and evaluate an environment variable.
:param string envvar:
The environment variable to lookup in :class:`os.environ`
from flask.ext.sqlalchemy import SQLAlchemy
from itsdangerous import URLSafeTimedSerializer
from sqlalchemy.engine import Engine
from sqlalchemy import event
from werkzeug.exceptions import BadRequest
from werkzeug.routing import BaseConverter, ValidationError
from pyfarm.core.enums import NOTSET, STRING_TYPES, PY3
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
POST_METHODS = set(("POST", "PUT"))
IGNORED_MIMETYPES = set(("application/x-www-form-urlencoded",
"multipart/form-data", "application/zip", "text/csv"))
logger = getLogger("app")
class UUIDConverter(BaseConverter):
"""
A URL converter for UUIDs. This class is loaded as part of
the Flask application setup and may be used in url routing:
.. code-block:: python
@app.route('/foo/<uuid:value>')
def foobar(value):
pass
When a request such as ``GET /foo/F9A63B47-66BF-4E2B-A545-879986BB7CA9``
is made :class:`UUIDConverter` will receive ``value`` to :meth:`to_python`
from flask.views import MethodView
from flask import g
from sqlalchemy import func, asc
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.models.user import User
from pyfarm.models.jobtype import JobType
from pyfarm.models.job import Job
from pyfarm.models.task import Task
from pyfarm.models.jobgroup import JobGroup
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify, validate_with_model
from pyfarm.master.application import db
logger = getLogger("api.jobgroups")
AUTOCREATE_USERS = config.get("autocreate_users")
AUTO_USER_EMAIL = config.get("autocreate_user_email")
def schema():
"""
Returns the basic schema of :class:`.JobGroup`
.. http:get:: /api/v1/jobgroups/schema HTTP/1.1
**Request**
.. sourcecode:: http
from pyfarm.core.enums import AgentState, WorkState
from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.task import Task
from pyfarm.models.job import Job
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.config import config
from pyfarm.master.application import db
from pyfarm.scheduler.celery_app import celery_app
logger = getLogger("pf.scheduler.statistics_tasks")
# TODO Get logger configuration from pyfarm config
logger.setLevel(DEBUG)
@celery_app.task(ignore_result=True)
def count_agents():
logger.debug("Counting known agents now")
num_online = Agent.query.filter_by(state=AgentState.ONLINE).count()
num_offline = Agent.query.filter_by(state=AgentState.OFFLINE).count()
num_running = Agent.query.filter_by(state=AgentState.RUNNING).count()
num_disabled = Agent.query.filter_by(state=AgentState.DISABLED).count()
agent_count = AgentCount(counted_time=datetime.utcnow(),
num_online=num_online,
num_offline=num_offline,
from os.path import join, realpath
from errno import EEXIST
from flask.views import MethodView
from flask import g, redirect, send_file, request, Response
from sqlalchemy.exc import IntegrityError
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
from pyfarm.models.tasklog import TaskLog, TaskTaskLogAssociation
from pyfarm.models.task import Task
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model, isuuid
logger = getLogger("api.tasklogs")
LOGFILES_DIR = config.get("tasklogs_dir")
try:
makedirs(LOGFILES_DIR)
except OSError as e: # pragma: no cover
if e.errno != EEXIST:
raise
class LogsInTaskAttemptsIndexAPI(MethodView):
def get(self, job_id, task_id, attempt):
"""
A ``GET`` to this endpoint will return a list of all known logs that are
associated with this attempt at running this task
except ImportError: # pragma: no cover
from http.client import (OK, CREATED, CONFLICT, NOT_FOUND, NO_CONTENT,
BAD_REQUEST)
from flask import g
from flask.views import MethodView
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.job import Job
from pyfarm.models.jobqueue import JobQueue
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model
logger = getLogger("api.jobqueues")
# Load model mappings once per process
JOBQUEUE_MODEL_MAPPINGS = JobQueue.types().mappings
def schema():
"""
Returns the basic schema of :class:`.JobQueue`
.. http:get:: /api/v1/jobqueues/schema HTTP/1.1
**Request**
.. sourcecode:: http
"""
Mixin Classes
=============
Module containing mixins which can be used by multiple models.
"""
from datetime import datetime
from sqlalchemy.orm import validates
from pyfarm.core.enums import DBWorkState, _WorkState, Values
from pyfarm.core.logger import getLogger
from pyfarm.core.config import read_env_int
logger = getLogger("models.mixin")
class ValidatePriorityMixin(object):
"""
Mixin that adds a `state` column and uses a class
level `STATE_ENUM` attribute to assist in validation.
"""
MIN_PRIORITY = read_env_int("PYFARM_QUEUE_MIN_PRIORITY", -1000)
MAX_PRIORITY = read_env_int("PYFARM_QUEUE_MAX_PRIORITY", 1000)
# quick check of the configured data
assert MAX_PRIORITY >= MIN_PRIORITY, "MIN_PRIORITY must be <= MAX_PRIORITY"
@validates("priority")
def validate_priority(self, key, value):
NO_CONTENT, METHOD_NOT_ALLOWED)
from flask import g, Response
from flask.views import MethodView
from sqlalchemy import or_, func, sql
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES, PY3
from pyfarm.models.software import (Software, SoftwareVersion,
JobTypeSoftwareRequirement)
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify
logger = getLogger("api.jobtypes")
class ObjectNotFound(Exception):
pass
def parse_requirements(requirements):
"""
Takes a list dicts specifying a software and optional min- and max-versions
and returns a list of :class:`JobRequirement` objects.
Raises TypeError if the input was not as expected or ObjectNotFound if a
referenced software of or version was not found.
:param list requirements:
from pyfarm.models.job import Job, JobDependency
from pyfarm.models.jobtype import JobType
from pyfarm.models.disk import AgentDisk
from pyfarm.models.agent import Agent, AgentTagAssociation, GPUInAgent
from pyfarm.models.user import User, Role
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.pathmap import PathMap
from pyfarm.models.tasklog import TaskLog
from pyfarm.models.gpu import GPU
from pyfarm.models.jobgroup import JobGroup
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.utility import timedelta_format
logger = getLogger("master.entrypoints")
def load_before_first(app_instance, database_instance):
if app_instance.debug:
app_instance.before_first_request_funcs.append(
database_instance.create_all)
def load_error_handlers(app_instance):
"""loads the error handlers onto application instance"""
# create the handlers
bad_request = partial(error_handler,
code=BAD_REQUEST,
default=lambda: "bad request to %s" % request.url)
unauthorized = partial(
import json
from calendar import timegm
from datetime import timedelta, datetime
from flask import render_template, request
from sqlalchemy import or_
from pyfarm.core.logger import getLogger
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.config import config
from pyfarm.master.application import db
logger = getLogger("ui.task_events")
class TotalsAverage(object):
def __init__(self, first_sample, last_sum=None):
self.time_start = first_sample.counted_time
self.num_samples_by_queue = {first_sample.job_queue_id: 1}
self.avg_queued_by_queue = (last_sum.avg_queued_by_queue
if last_sum else {})
self.avg_queued_by_queue[first_sample.job_queue_id] =\
first_sample.total_queued
self.avg_running_by_queue = (last_sum.avg_running_by_queue
if last_sum else {})
self.avg_running_by_queue[first_sample.job_queue_id] =\
try:
from httplib import SEE_OTHER, NOT_FOUND
except ImportError: # pragma: no cover
from http.client import SEE_OTHER, NOT_FOUND
from flask import render_template, request, redirect, url_for, flash
from sqlalchemy import or_
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.master.application import db
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.job import Job
logger = getLogger("ui.jobqueues")
def jobqueues():
jobqueues = JobQueue.query.filter_by(parent_jobqueue_id=None).all()
jobqueues = sorted(jobqueues,
key=lambda x: x.num_assigned_agents(),
reverse=True)
top_level_jobs_query = Job.query.filter_by(queue=None)
filters = {}
if ("state_paused" not in request.args
and "state_queued" not in request.args
and "state_running" not in request.args
and "state_done" not in request.args
from pyfarm.scheduler.tasks import (
assign_tasks, update_agent, assign_tasks_to_agent, send_tasks_to_agent)
from pyfarm.models.agent import (
Agent, AgentMacAddress, AgentSoftwareVersionAssociation)
from pyfarm.models.gpu import GPU
from pyfarm.models.task import Task
from pyfarm.models.software import Software, SoftwareVersion
from pyfarm.master.config import config
from pyfarm.models.tag import Tag
from pyfarm.models.disk import AgentDisk
from pyfarm.master.application import db
from pyfarm.master.utility import (
jsonify, validate_with_model, get_ipaddr_argument, get_integer_argument,
get_hostname_argument, get_port_argument, isuuid)
logger = getLogger("api.agents")
MAC_RE = re.compile("^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$")
OUR_FARM_NAME = config.get("farm_name")
def fail_missing_assignments(agent, current_assignments):
known_task_ids = []
for assignment in current_assignments.values():
for task in assignment["tasks"]:
known_task_ids.append(task["id"])
tasks_query = Task.query.filter(Task.agent == agent,
or_(Task.state == None,
~Task.state.in_(
[WorkState.FAILED, WorkState.DONE])))
if known_task_ids:
from pyfarm.models.core.mixins import (ValidatePriorityMixin,
WorkStateChangedMixin, ReprMixin,
ValidateWorkStateMixin, UtilityMixins)
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.task import Task
try:
# pylint: disable=undefined-variable
range_ = xrange
except NameError:
range_ = range
__all__ = ("Job", )
logger = getLogger("models.job")
JobTagAssociation = db.Table(
config.get("table_job_tag_assoc"), db.metadata,
db.Column("job_id",
IDTypeWork,
db.ForeignKey("%s.id" % config.get("table_job")),
primary_key=True,
doc="The id of the job associated with this task"),
db.Column("tag_id",
db.Integer,
db.ForeignKey("%s.id" % config.get("table_tag")),
primary_key=True,
doc="The id of the tag being associated with the job"))
JobDependency = db.Table(
try:
from httplib import SEE_OTHER, NOT_FOUND
except ImportError: # pragma: no cover
from http.client import SEE_OTHER, NOT_FOUND
from flask import render_template, request, redirect, url_for, flash
from sqlalchemy import or_
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.master.application import db
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.job import Job
logger = getLogger("ui.jobqueues")
def jobqueues():
jobqueues = JobQueue.query.filter_by(parent_jobqueue_id=None).all()
jobqueues = sorted(jobqueues, key=lambda x: x.num_assigned_agents(),
reverse=True)
top_level_jobs_query = Job.query.filter_by(queue=None)
filters = {}
if ("state_paused" not in request.args and
"state_queued" not in request.args and
"state_running" not in request.args and
"state_done" not in request.args and
"state_failed" not in request.args):
filters["state_paused"] = False
