Esempio n. 1
0
general implementation.
"""

from sqlalchemy.orm import validates
from sqlalchemy.schema import UniqueConstraint

from pyfarm.core.logger import getLogger
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork


__all__ = ("JobType", )

logger = getLogger("models.jobtype")


class JobType(db.Model, UtilityMixins, ReprMixin):
    """
    Stores the unique information necessary to execute a task
    """
    __tablename__ = config.get("table_job_type")
    __table_args__ = (UniqueConstraint("name"),)
    REPR_COLUMNS = ("id", "name")

    id = id_column(IDTypeWork)

    name = db.Column(
        db.String(config.get("job_type_max_name_length")),
        nullable=False,
Esempio n. 2
0
    from http.client import (
        OK, CREATED, CONFLICT, NOT_FOUND, NO_CONTENT, BAD_REQUEST)

from flask import g
from flask.views import MethodView

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES

from pyfarm.models.job import Job
from pyfarm.models.jobqueue import JobQueue
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model


logger = getLogger("api.jobqueues")


# Load model mappings once per process
JOBQUEUE_MODEL_MAPPINGS = JobQueue.types().mappings


def schema():
    """
    Returns the basic schema of :class:`.JobQueue`

    .. http:get:: /api/v1/jobqueues/schema HTTP/1.1

        **Request**

        .. sourcecode:: http
Esempio n. 3
0
from flask.views import MethodView
from flask import g

from sqlalchemy import func, asc
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.models.user import User
from pyfarm.models.jobtype import JobType
from pyfarm.models.job import Job
from pyfarm.models.task import Task
from pyfarm.models.jobgroup import JobGroup
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify, validate_with_model
from pyfarm.master.application import db

logger = getLogger("api.jobgroups")

AUTOCREATE_USERS = config.get("autocreate_users")
AUTO_USER_EMAIL = config.get("autocreate_user_email")


def schema():
    """
    Returns the basic schema of :class:`.JobGroup`

    .. http:get:: /api/v1/jobgroups/schema HTTP/1.1

        **Request**

        .. sourcecode:: http
Esempio n. 4
0
 def test_get_logger_name(self):
     logger = getLogger("foo")
     self.assertEqual(logger.name, "pf.foo")
     self.assertFalse(logger.handlers)
Esempio n. 5
0
import json
from calendar import timegm
from datetime import timedelta, datetime

from flask import render_template, request

from sqlalchemy import or_

from pyfarm.core.logger import getLogger
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.config import config
from pyfarm.master.application import db

logger = getLogger("ui.task_events")


class TotalsAverage(object):
    def __init__(self, first_sample, last_sum=None):
        self.time_start = first_sample.counted_time
        self.num_samples_by_queue = {first_sample.job_queue_id: 1}

        self.avg_queued_by_queue = (
            last_sum.avg_queued_by_queue if last_sum else {})
        self.avg_queued_by_queue[first_sample.job_queue_id] =\
            first_sample.total_queued

        self.avg_running_by_queue = (
            last_sum.avg_running_by_queue if last_sum else {})
        self.avg_running_by_queue[first_sample.job_queue_id] =\
Esempio n. 6
0
from flask import g
from flask.views import MethodView

from sqlalchemy import or_

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.pathmap import PathMap
from pyfarm.models.tag import Tag
from pyfarm.models.agent import Agent
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify, validate_with_model, get_uuid_argument


logger = getLogger("api.pathmaps")


def schema():
    """
    Returns the basic schema of :class:`.Agent`

    .. http:get:: /api/v1/pathmaps/schema HTTP/1.1

        **Request**

        .. sourcecode:: http

            GET /api/v1/pathmaps/schema HTTP/1.1
            Accept: application/json
Esempio n. 7
0
from logging import DEBUG

from sqlalchemy import event, distinct, or_, and_
from sqlalchemy.schema import UniqueConstraint

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork
from pyfarm.models.agent import Agent

PREFER_RUNNING_JOBS = config.get("queue_prefer_running_jobs")
USE_TOTAL_RAM = config.get("use_total_ram_for_scheduling")
logger = getLogger("pf.models.jobqueue")

if config.get("debug_queue"):
    logger.setLevel(DEBUG)


class JobQueue(db.Model, UtilityMixins, ReprMixin):
    """
    Stores information about a job queue. Used for flexible, configurable
    distribution of computing capacity to jobs.
    """
    __tablename__ = config.get("table_job_queue")
    __table_args__ = (UniqueConstraint("parent_jobqueue_id", "name"), )

    REPR_COLUMNS = ("id", "name")
Esempio n. 8
0
from pyfarm.models.job import Job, JobDependency
from pyfarm.models.jobtype import JobType
from pyfarm.models.disk import AgentDisk
from pyfarm.models.agent import Agent, AgentTagAssociation, GPUInAgent
from pyfarm.models.user import User, Role
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.pathmap import PathMap
from pyfarm.models.tasklog import TaskLog
from pyfarm.models.gpu import GPU
from pyfarm.models.jobgroup import JobGroup
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.utility import timedelta_format

logger = getLogger("master.entrypoints")


def load_before_first(app_instance, database_instance):
    if app_instance.debug:
        app_instance.before_first_request_funcs.append(
            database_instance.create_all)


def load_error_handlers(app_instance):
    """loads the error handlers onto application instance"""
    # create the handlers
    bad_request = partial(
        error_handler, code=BAD_REQUEST,
        default=lambda: "bad request to %s" % request.url)
    unauthorized = partial(
from pyfarm.core.enums import AgentState, WorkState

from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.task import Task
from pyfarm.models.job import Job
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount

from pyfarm.master.config import config
from pyfarm.master.application import db

from pyfarm.scheduler.celery_app import celery_app

logger = getLogger("pf.scheduler.statistics_tasks")
# TODO Get logger configuration from pyfarm config
logger.setLevel(DEBUG)


@celery_app.task(ignore_result=True)
def count_agents():
    logger.debug("Counting known agents now")
    num_online = Agent.query.filter_by(state=AgentState.ONLINE).count()
    num_offline = Agent.query.filter_by(state=AgentState.OFFLINE).count()
    num_running = Agent.query.filter_by(state=AgentState.RUNNING).count()
    num_disabled = Agent.query.filter_by(state=AgentState.DISABLED).count()

    agent_count = AgentCount(counted_time=datetime.utcnow(),
                             num_online=num_online,
                             num_offline=num_offline,
Esempio n. 10
0
from logging import DEBUG

from sqlalchemy import event, distinct, or_, and_
from sqlalchemy.schema import UniqueConstraint

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork
from pyfarm.models.agent import Agent

PREFER_RUNNING_JOBS = config.get("queue_prefer_running_jobs")
USE_TOTAL_RAM = config.get("use_total_ram_for_scheduling")
logger = getLogger("pf.models.jobqueue")

if config.get("debug_queue"):
    logger.setLevel(DEBUG)


class JobQueue(db.Model, UtilityMixins, ReprMixin):
    """
    Stores information about a job queue. Used for flexible, configurable
    distribution of computing capacity to jobs.
    """
    __tablename__ = config.get("table_job_queue")
    __table_args__ = (UniqueConstraint("parent_jobqueue_id", "name"),)

    REPR_COLUMNS = ("id", "name")
Esempio n. 11
0
    ValidateWorkStateMixin,
    UtilityMixins,
)
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.task import Task

try:
    # pylint: disable=undefined-variable
    range_ = xrange
except NameError:
    range_ = range

__all__ = ("Job",)

logger = getLogger("models.job")


JobTagAssociation = db.Table(
    config.get("table_job_tag_assoc"),
    db.metadata,
    db.Column(
        "job_id",
        IDTypeWork,
        db.ForeignKey("%s.id" % config.get("table_job")),
        primary_key=True,
        doc="The id of the job associated with this task",
    ),
    db.Column(
        "tag_id",
        db.Integer,
Esempio n. 12
0
from itsdangerous import URLSafeTimedSerializer
from sqlalchemy.engine import Engine
from sqlalchemy import event
from werkzeug.exceptions import BadRequest
from werkzeug.routing import BaseConverter, ValidationError

from pyfarm.core.enums import NOTSET, STRING_TYPES, PY3
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config

POST_METHODS = set(("POST", "PUT"))
IGNORED_MIMETYPES = set((
    "application/x-www-form-urlencoded", "multipart/form-data",
    "application/zip", "text/csv"))

logger = getLogger("app")


class UUIDConverter(BaseConverter):
    """
    A URL converter for UUIDs.  This class is loaded as part of
    the Flask application setup and may be used in url routing:

    .. code-block:: python

        @app.route('/foo/<uuid:value>')
        def foobar(value):
            pass

    When a request such as ``GET /foo/F9A63B47-66BF-4E2B-A545-879986BB7CA9``
    is made :class:`UUIDConverter` will receive ``value`` to :meth:`to_python`
Esempio n. 13
0
from datetime import datetime
from collections import namedtuple

try:
    from httplib import INTERNAL_SERVER_ERROR
except ImportError:
    from http.client import INTERNAL_SERVER_ERROR

from sqlalchemy.orm import validates, class_mapper

from pyfarm.core.enums import _WorkState, Values, PY2
from pyfarm.core.logger import getLogger
from pyfarm.models.core.types import IPAddress
from pyfarm.master.config import config

logger = getLogger("models.mixin")

# stores information about a model's columns
# and relationships
ModelTypes = namedtuple("ModelTypes",
                        ("primary_keys", "autoincrementing", "columns",
                         "required", "relationships", "mappings"))


class ValidatePriorityMixin(object):
    """
    Mixin that adds a `state` column and uses a class
    level `STATE_ENUM` attribute to assist in validation.
    """
    MIN_PRIORITY = config.get("queue_min_priority")
    MAX_PRIORITY = config.get("queue_max_priority")
Esempio n. 14
0
    from io import StringIO

from pkg_resources import (DistributionNotFound, get_distribution,
                           resource_filename)

import yaml
try:
    from yaml.loader import CLoader as Loader
except ImportError:  # pragma: no cover
    from yaml.loader import Loader

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import (STRING_TYPES, NUMERIC_TYPES, NOTSET, LINUX, MAC,
                               WINDOWS, range_)

logger = getLogger("core.config")

# Boolean values as strings that can match a value we
# pulled from the environment after calling .lower().
BOOLEAN_TRUE = set(["1", "t", "y", "true", "yes"])
BOOLEAN_FALSE = set(["0", "f", "n", "false", "no"])


def read_env(envvar,
             default=NOTSET,
             warn_if_unset=False,
             eval_literal=False,
             raise_eval_exception=True,
             log_result=True,
             desc=None,
             log_defaults=False):
Esempio n. 15
0
        NOT_FOUND, NO_CONTENT, OK, CREATED, BAD_REQUEST, INTERNAL_SERVER_ERROR,
        CONFLICT)

from flask import g, Response
from flask.views import MethodView

from sqlalchemy import func
from sqlalchemy.exc import DatabaseError

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.software import Software, SoftwareVersion
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model

logger = getLogger("api.software")


class VersionParseError(Exception):
    """
    Raised by :func:`extract_version_dicts` when the
    function is unable to parse a version.
    """


def extract_version_dicts(json_in):
    """Extracts and returns a list of versions from ``json_in``."""
    out = []
    version_objects = json_in.pop("versions", [])
    if not isinstance(version_objects, list):
        raise VersionParseError("Column versions must be a list.")
Esempio n. 16
0
from os.path import join, realpath
from errno import EEXIST

from flask.views import MethodView
from flask import g, redirect, send_file, request, Response

from sqlalchemy.exc import IntegrityError

from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
from pyfarm.models.tasklog import TaskLog, TaskTaskLogAssociation
from pyfarm.models.task import Task
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model, isuuid

logger = getLogger("api.tasklogs")

LOGFILES_DIR = config.get("tasklogs_dir")

try:
    makedirs(LOGFILES_DIR)
except OSError as e:  # pragma: no cover
    if e.errno != EEXIST:
        raise


class LogsInTaskAttemptsIndexAPI(MethodView):
    def get(self, job_id, task_id, attempt):
        """
        A ``GET`` to this endpoint will return a list of all known logs that are
        associated with this attempt at running this task
Esempio n. 17
0
    from httplib import NOT_FOUND, NO_CONTENT, OK, CREATED, BAD_REQUEST
except ImportError:  # pragma: no cover
    from http.client import NOT_FOUND, NO_CONTENT, OK, CREATED, BAD_REQUEST

from flask import url_for, g
from flask.views import MethodView

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES
from pyfarm.models.agent import Agent
from pyfarm.models.job import Job
from pyfarm.models.tag import Tag
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model

logger = getLogger("api.tags")


def schema():
    """
    Returns the basic schema of :class:`.Tag`

    .. http:get:: /api/v1/tags/schema/ HTTP/1.1

        **Request**

        .. sourcecode:: http

            GET /api/v1/tags/schema/ HTTP/1.1
            Accept: application/json
Esempio n. 18
0
from sqlalchemy import func, desc, asc, or_, distinct

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.scheduler.tasks import delete_job, stop_task, assign_tasks
from pyfarm.models.job import (
    Job, JobDependency, JobTagAssociation, JobNotifiedUser)
from pyfarm.models.tag import Tag, JobTagRequirement
from pyfarm.models.task import Task
from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.user import User
from pyfarm.master.application import db

logger = getLogger("ui.jobs")

def jobs():
    queued_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_queued')).\
            filter(Task.state == None).group_by(Task.job_id).subquery()
    running_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_running')).\
            filter(Task.state == WorkState.RUNNING).\
                group_by(Task.job_id).subquery()
    done_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_done')).\
            filter(Task.state == WorkState.DONE).\
                group_by(Task.job_id).subquery()
    failed_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_failed')).\
Esempio n. 19
0
from os.path import join, isfile
from errno import EEXIST

try:
    from httplib import BAD_REQUEST, CREATED, NOT_FOUND
except ImportError:  # pragma: no cover
    from http.client import BAD_REQUEST, CREATED, NOT_FOUND

from flask.views import MethodView
from flask import request, redirect, send_file

from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify

logger = getLogger("api.agents")

VERSION_REGEX = re.compile("\d+(\.\d+(\.\d+)?)?((-pre\d?)|(-dev\d?)|(-rc?\d?)|"
                           "(-alpha\d?)|(-beta\d?))?$")

UPDATES_DIR = config.get("agent_updates_dir")
UPDATES_WEBDIR = config.get("agent_updates_webdir")

try:
    makedirs(UPDATES_DIR)
except OSError as e:  # pragma: no cover
    if e.errno != EEXIST:
        raise


class AgentUpdatesAPI(MethodView):
Esempio n. 20
0
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState, AgentState
from pyfarm.scheduler.tasks import delete_job, stop_task, assign_tasks
from pyfarm.models.job import (Job, JobDependency, JobTagAssociation,
                               JobNotifiedUser)
from pyfarm.models.tag import Tag, JobTagRequirement
from pyfarm.models.task import Task
from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.user import User
from pyfarm.master.application import db
from pyfarm.master.config import config

logger = getLogger("ui.jobs")


def jobs():
    queued_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_queued')).\
            filter(Task.state == None).group_by(Task.job_id).subquery()
    running_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_running')).\
            filter(Task.state == WorkState.RUNNING).\
                group_by(Task.job_id).subquery()
    done_count_query = db.session.query(
        Task.job_id, func.count('*').label('t_done')).\
            filter(Task.state == WorkState.DONE).\
                group_by(Task.job_id).subquery()
    failed_count_query = db.session.query(
Esempio n. 21
0
from datetime import datetime

from sqlalchemy import event

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState, _WorkState
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.types import IDTypeAgent, IDTypeWork
from pyfarm.models.core.functions import work_columns, repr_enum
from pyfarm.models.core.mixins import (
    ValidatePriorityMixin, UtilityMixins, ReprMixin, ValidateWorkStateMixin)

__all__ = ("Task", )

logger = getLogger("models.task")


class Task(db.Model, ValidatePriorityMixin, ValidateWorkStateMixin,
           UtilityMixins, ReprMixin):
    """
    Defines a task which a child of a :class:`Job`.  This table represents
    rows which contain the individual work unit(s) for a job.
    """
    __tablename__ = config.get("table_task")
    STATE_ENUM = list(WorkState) + [None]
    STATE_DEFAULT = None
    REPR_COLUMNS = ("id", "state", "frame", "project")
    REPR_CONVERT_COLUMN = {"state": partial(repr_enum, enum=STATE_ENUM)}

    # shared work columns
Esempio n. 22
0
 def test_get_logger_name(self):
     logger = getLogger("foo")
     self.assertEqual(logger.name, "pf.foo")
     self.assertFalse(logger.handlers)
Esempio n. 23
0
to an individual job.  See :mod:`pyfarm.models.job` for more the more
general implementation.
"""

from sqlalchemy.orm import validates
from sqlalchemy.schema import UniqueConstraint

from pyfarm.core.logger import getLogger
from pyfarm.master.application import db
from pyfarm.master.config import config
from pyfarm.models.core.mixins import UtilityMixins, ReprMixin
from pyfarm.models.core.types import id_column, IDTypeWork

__all__ = ("JobType", )

logger = getLogger("models.jobtype")


class JobType(db.Model, UtilityMixins, ReprMixin):
    """
    Stores the unique information necessary to execute a task
    """
    __tablename__ = config.get("table_job_type")
    __table_args__ = (UniqueConstraint("name"), )
    REPR_COLUMNS = ("id", "name")

    id = id_column(IDTypeWork)

    name = db.Column(
        db.String(config.get("job_type_max_name_length")),
        nullable=False,
Esempio n. 24
0
from datetime import datetime
from textwrap import dedent

from flask.ext.login import UserMixin

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES, PY3
from pyfarm.master.application import app, db, login_serializer
from pyfarm.models.core.mixins import ReprMixin
from pyfarm.models.core.functions import split_and_extend
from pyfarm.models.core.cfg import (
    TABLE_USERS_USER, TABLE_USERS_ROLE, TABLE_USERS_USER_ROLES,
    MAX_USERNAME_LENGTH, SHA256_ASCII_LENGTH, MAX_EMAILADDR_LENGTH,
    MAX_ROLE_LENGTH, TABLE_USERS_PROJECTS, TABLE_PROJECT)

logger = getLogger("models.users")

# roles the user is a member of
UserRoles = db.Table(
    TABLE_USERS_USER_ROLES,
    db.Column("user_id", db.Integer,
              db.ForeignKey("%s.id" % TABLE_USERS_USER)),
    db.Column("role_id", db.Integer,
              db.ForeignKey("%s.id" % TABLE_USERS_ROLE)))

# projects the user is a member of
UserProjects = db.Table(
    TABLE_USERS_PROJECTS,
    db.Column("user_id", db.Integer,
              db.ForeignKey("%s.id" % TABLE_USERS_USER), primary_key=True),
    db.Column("project_id", db.Integer,
Esempio n. 25
0
except ImportError:  # pragma: no cover
    from http.client import OK, CREATED, CONFLICT, NOT_FOUND, BAD_REQUEST, NO_CONTENT, METHOD_NOT_ALLOWED

from flask import g, Response
from flask.views import MethodView

from sqlalchemy import or_, func, sql

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES, PY3
from pyfarm.models.software import Software, SoftwareVersion, JobTypeSoftwareRequirement
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify

logger = getLogger("api.jobtypes")


class ObjectNotFound(Exception):
    pass


def parse_requirements(requirements):
    """
    Takes a list dicts specifying a software and optional min- and max-versions
    and returns a list of :class:`JobRequirement` objects.

    Raises TypeError if the input was not as expected or ObjectNotFound if a
    referenced software of or version was not found.

    :param list requirements:
Esempio n. 26
0
    from io import StringIO

from pkg_resources import (
    DistributionNotFound, get_distribution, resource_filename)

import yaml
try:
    from yaml.loader import CLoader as Loader
except ImportError:  # pragma: no cover
    from yaml.loader import Loader

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import (
    STRING_TYPES, NUMERIC_TYPES, NOTSET, LINUX, MAC, WINDOWS, range_)

logger = getLogger("core.config")

# Boolean values as strings that can match a value we
# pulled from the environment after calling .lower().
BOOLEAN_TRUE = set(["1", "t", "y", "true", "yes"])
BOOLEAN_FALSE = set(["0", "f", "n", "false", "no"])


def read_env(envvar, default=NOTSET, warn_if_unset=False, eval_literal=False,
             raise_eval_exception=True, log_result=True, desc=None,
             log_defaults=False):
    """
    Lookup and evaluate an environment variable.

    :param string envvar:
        The environment variable to lookup in :class:`os.environ`
Esempio n. 27
0
from flask.ext.sqlalchemy import SQLAlchemy
from itsdangerous import URLSafeTimedSerializer
from sqlalchemy.engine import Engine
from sqlalchemy import event
from werkzeug.exceptions import BadRequest
from werkzeug.routing import BaseConverter, ValidationError

from pyfarm.core.enums import NOTSET, STRING_TYPES, PY3
from pyfarm.core.logger import getLogger
from pyfarm.master.config import config

POST_METHODS = set(("POST", "PUT"))
IGNORED_MIMETYPES = set(("application/x-www-form-urlencoded",
                         "multipart/form-data", "application/zip", "text/csv"))

logger = getLogger("app")


class UUIDConverter(BaseConverter):
    """
    A URL converter for UUIDs.  This class is loaded as part of
    the Flask application setup and may be used in url routing:

    .. code-block:: python

        @app.route('/foo/<uuid:value>')
        def foobar(value):
            pass

    When a request such as ``GET /foo/F9A63B47-66BF-4E2B-A545-879986BB7CA9``
    is made :class:`UUIDConverter` will receive ``value`` to :meth:`to_python`
from flask.views import MethodView
from flask import g

from sqlalchemy import func, asc
from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.models.user import User
from pyfarm.models.jobtype import JobType
from pyfarm.models.job import Job
from pyfarm.models.task import Task
from pyfarm.models.jobgroup import JobGroup
from pyfarm.master.config import config
from pyfarm.master.utility import jsonify, validate_with_model
from pyfarm.master.application import db

logger = getLogger("api.jobgroups")

AUTOCREATE_USERS = config.get("autocreate_users")
AUTO_USER_EMAIL = config.get("autocreate_user_email")


def schema():
    """
    Returns the basic schema of :class:`.JobGroup`

    .. http:get:: /api/v1/jobgroups/schema HTTP/1.1

        **Request**

        .. sourcecode:: http
Esempio n. 29
0
from pyfarm.core.enums import AgentState, WorkState

from pyfarm.models.agent import Agent
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.task import Task
from pyfarm.models.job import Job
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount

from pyfarm.master.config import config
from pyfarm.master.application import db

from pyfarm.scheduler.celery_app import celery_app

logger = getLogger("pf.scheduler.statistics_tasks")
# TODO Get logger configuration from pyfarm config
logger.setLevel(DEBUG)


@celery_app.task(ignore_result=True)
def count_agents():
    logger.debug("Counting known agents now")
    num_online = Agent.query.filter_by(state=AgentState.ONLINE).count()
    num_offline = Agent.query.filter_by(state=AgentState.OFFLINE).count()
    num_running = Agent.query.filter_by(state=AgentState.RUNNING).count()
    num_disabled = Agent.query.filter_by(state=AgentState.DISABLED).count()

    agent_count = AgentCount(counted_time=datetime.utcnow(),
                             num_online=num_online,
                             num_offline=num_offline,
Esempio n. 30
0
from os.path import join, realpath
from errno import EEXIST

from flask.views import MethodView
from flask import g, redirect, send_file, request, Response

from sqlalchemy.exc import IntegrityError

from pyfarm.core.logger import getLogger
from pyfarm.master.config import config
from pyfarm.models.tasklog import TaskLog, TaskTaskLogAssociation
from pyfarm.models.task import Task
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model, isuuid

logger = getLogger("api.tasklogs")

LOGFILES_DIR = config.get("tasklogs_dir")

try:
    makedirs(LOGFILES_DIR)
except OSError as e:  # pragma: no cover
    if e.errno != EEXIST:
        raise


class LogsInTaskAttemptsIndexAPI(MethodView):
    def get(self, job_id, task_id, attempt):
        """
        A ``GET`` to this endpoint will return a list of all known logs that are
        associated with this attempt at running this task
Esempio n. 31
0
except ImportError:  # pragma: no cover
    from http.client import (OK, CREATED, CONFLICT, NOT_FOUND, NO_CONTENT,
                             BAD_REQUEST)

from flask import g
from flask.views import MethodView

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES

from pyfarm.models.job import Job
from pyfarm.models.jobqueue import JobQueue
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify, validate_with_model

logger = getLogger("api.jobqueues")

# Load model mappings once per process
JOBQUEUE_MODEL_MAPPINGS = JobQueue.types().mappings


def schema():
    """
    Returns the basic schema of :class:`.JobQueue`

    .. http:get:: /api/v1/jobqueues/schema HTTP/1.1

        **Request**

        .. sourcecode:: http
Esempio n. 32
0
"""
Mixin Classes
=============

Module containing mixins which can be used by multiple models.
"""

from datetime import datetime

from sqlalchemy.orm import validates

from pyfarm.core.enums import DBWorkState, _WorkState, Values
from pyfarm.core.logger import getLogger
from pyfarm.core.config import read_env_int

logger = getLogger("models.mixin")


class ValidatePriorityMixin(object):
    """
    Mixin that adds a `state` column and uses a class
    level `STATE_ENUM` attribute to assist in validation.
    """
    MIN_PRIORITY = read_env_int("PYFARM_QUEUE_MIN_PRIORITY", -1000)
    MAX_PRIORITY = read_env_int("PYFARM_QUEUE_MAX_PRIORITY", 1000)

    # quick check of the configured data
    assert MAX_PRIORITY >= MIN_PRIORITY, "MIN_PRIORITY must be <= MAX_PRIORITY"

    @validates("priority")
    def validate_priority(self, key, value):
Esempio n. 33
0
                             NO_CONTENT, METHOD_NOT_ALLOWED)

from flask import g, Response
from flask.views import MethodView

from sqlalchemy import or_, func, sql

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import STRING_TYPES, PY3
from pyfarm.models.software import (Software, SoftwareVersion,
                                    JobTypeSoftwareRequirement)
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.master.application import db
from pyfarm.master.utility import jsonify

logger = getLogger("api.jobtypes")


class ObjectNotFound(Exception):
    pass


def parse_requirements(requirements):
    """
    Takes a list dicts specifying a software and optional min- and max-versions
    and returns a list of :class:`JobRequirement` objects.

    Raises TypeError if the input was not as expected or ObjectNotFound if a
    referenced software of or version was not found.

    :param list requirements:
from pyfarm.models.job import Job, JobDependency
from pyfarm.models.jobtype import JobType
from pyfarm.models.disk import AgentDisk
from pyfarm.models.agent import Agent, AgentTagAssociation, GPUInAgent
from pyfarm.models.user import User, Role
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.pathmap import PathMap
from pyfarm.models.tasklog import TaskLog
from pyfarm.models.gpu import GPU
from pyfarm.models.jobgroup import JobGroup
from pyfarm.models.statistics.agent_count import AgentCount
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.utility import timedelta_format

logger = getLogger("master.entrypoints")


def load_before_first(app_instance, database_instance):
    if app_instance.debug:
        app_instance.before_first_request_funcs.append(
            database_instance.create_all)


def load_error_handlers(app_instance):
    """loads the error handlers onto application instance"""
    # create the handlers
    bad_request = partial(error_handler,
                          code=BAD_REQUEST,
                          default=lambda: "bad request to %s" % request.url)
    unauthorized = partial(
import json
from calendar import timegm
from datetime import timedelta, datetime

from flask import render_template, request

from sqlalchemy import or_

from pyfarm.core.logger import getLogger
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.statistics.task_count import TaskCount
from pyfarm.master.config import config
from pyfarm.master.application import db

logger = getLogger("ui.task_events")


class TotalsAverage(object):
    def __init__(self, first_sample, last_sum=None):
        self.time_start = first_sample.counted_time
        self.num_samples_by_queue = {first_sample.job_queue_id: 1}

        self.avg_queued_by_queue = (last_sum.avg_queued_by_queue
                                    if last_sum else {})
        self.avg_queued_by_queue[first_sample.job_queue_id] =\
            first_sample.total_queued

        self.avg_running_by_queue = (last_sum.avg_running_by_queue
                                     if last_sum else {})
        self.avg_running_by_queue[first_sample.job_queue_id] =\
Esempio n. 36
0
try:
    from httplib import SEE_OTHER, NOT_FOUND
except ImportError:  # pragma: no cover
    from http.client import SEE_OTHER, NOT_FOUND

from flask import render_template, request, redirect, url_for, flash
from sqlalchemy import or_

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.master.application import db
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.job import Job

logger = getLogger("ui.jobqueues")


def jobqueues():
    jobqueues = JobQueue.query.filter_by(parent_jobqueue_id=None).all()
    jobqueues = sorted(jobqueues,
                       key=lambda x: x.num_assigned_agents(),
                       reverse=True)

    top_level_jobs_query = Job.query.filter_by(queue=None)

    filters = {}
    if ("state_paused" not in request.args
            and "state_queued" not in request.args
            and "state_running" not in request.args
            and "state_done" not in request.args
Esempio n. 37
0
from pyfarm.scheduler.tasks import (
    assign_tasks, update_agent, assign_tasks_to_agent, send_tasks_to_agent)
from pyfarm.models.agent import (
    Agent, AgentMacAddress, AgentSoftwareVersionAssociation)
from pyfarm.models.gpu import GPU
from pyfarm.models.task import Task
from pyfarm.models.software import Software, SoftwareVersion
from pyfarm.master.config import config
from pyfarm.models.tag import Tag
from pyfarm.models.disk import AgentDisk
from pyfarm.master.application import db
from pyfarm.master.utility import (
    jsonify, validate_with_model, get_ipaddr_argument, get_integer_argument,
    get_hostname_argument, get_port_argument, isuuid)

logger = getLogger("api.agents")

MAC_RE = re.compile("^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$")
OUR_FARM_NAME = config.get("farm_name")


def fail_missing_assignments(agent, current_assignments):
    known_task_ids = []
    for assignment in current_assignments.values():
        for task in assignment["tasks"]:
            known_task_ids.append(task["id"])
    tasks_query = Task.query.filter(Task.agent == agent,
                                    or_(Task.state == None,
                                        ~Task.state.in_(
                                            [WorkState.FAILED, WorkState.DONE])))
    if known_task_ids:
Esempio n. 38
0
from pyfarm.models.core.mixins import (ValidatePriorityMixin,
                                       WorkStateChangedMixin, ReprMixin,
                                       ValidateWorkStateMixin, UtilityMixins)
from pyfarm.models.statistics.task_event_count import TaskEventCount
from pyfarm.models.jobtype import JobType, JobTypeVersion
from pyfarm.models.task import Task

try:
    # pylint: disable=undefined-variable
    range_ = xrange
except NameError:
    range_ = range

__all__ = ("Job", )

logger = getLogger("models.job")

JobTagAssociation = db.Table(
    config.get("table_job_tag_assoc"), db.metadata,
    db.Column("job_id",
              IDTypeWork,
              db.ForeignKey("%s.id" % config.get("table_job")),
              primary_key=True,
              doc="The id of the job associated with this task"),
    db.Column("tag_id",
              db.Integer,
              db.ForeignKey("%s.id" % config.get("table_tag")),
              primary_key=True,
              doc="The id of the tag being associated with the job"))

JobDependency = db.Table(
Esempio n. 39
0
try:
    from httplib import SEE_OTHER, NOT_FOUND
except ImportError:  # pragma: no cover
    from http.client import SEE_OTHER, NOT_FOUND

from flask import render_template, request, redirect, url_for, flash
from sqlalchemy import or_

from pyfarm.core.logger import getLogger
from pyfarm.core.enums import WorkState
from pyfarm.master.application import db
from pyfarm.models.jobqueue import JobQueue
from pyfarm.models.job import Job

logger = getLogger("ui.jobqueues")

def jobqueues():
    jobqueues = JobQueue.query.filter_by(parent_jobqueue_id=None).all()
    jobqueues = sorted(jobqueues, key=lambda x: x.num_assigned_agents(),
                       reverse=True)

    top_level_jobs_query = Job.query.filter_by(queue=None)

    filters = {}
    if ("state_paused" not in request.args and
        "state_queued" not in request.args and
        "state_running" not in request.args and
        "state_done" not in request.args and
        "state_failed" not in request.args):
        filters["state_paused"] = False