示例#1
0
try:
    from xml.etree.cElementTree import ElementTree
except ImportError:  # pragma: no cover
    from xml.etree.ElementTree import ElementTree

try:
    from wmi import WMI
except ImportError:  # pragma: no cover
    WMI = NotImplemented

from pyfarm.core.enums import WINDOWS, LINUX, MAC
from pyfarm.agent.logger import getLogger
from pyfarm.agent.config import config

logger = getLogger("agent.sysinfo.gpu")


class GPULookupError(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)


def graphics_cards():
    """
    Returns a list of the full names of GPUs installed in this system
    """
    if WINDOWS:
示例#2
0
except ImportError:  # pragma: no cover
    getgrgid = NotImplemented
    getpwuid = NotImplemented
    setuid = NotImplemented
    getuid = NotImplemented
    setgid = NotImplemented
    getgid = NotImplemented

from pyfarm.core.enums import INTEGER_TYPES, OS, operating_system
from pyfarm.agent.config import config
from pyfarm.agent.entrypoints.parser import AgentArgumentParser
from pyfarm.agent.entrypoints.utility import start_daemon_posix
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import remove_file, remove_directory

logger = getLogger("agent.supervisor")


def supervisor():
    logger.debug("Supervisor called with: %r", sys.argv)
    supervisor_args = []
    agent_args = []
    in_agent_args = False
    tmp_argv = list(sys.argv)
    del tmp_argv[0]
    for arg in tmp_argv:
        if not in_agent_args and arg != "--":
            supervisor_args.append(arg)
        elif not in_agent_args and arg == "--":
            in_agent_args = True
        else:
示例#3
0
from os.path import join, isdir

from treq import get, collect
from twisted.internet import reactor
from twisted.web.server import NOT_DONE_YET
import twisted.python.failure

from voluptuous import Schema, Required

from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.http.core.client import http_retry_delay
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import STRINGS

logger = getLogger("agent.http.update")


class Update(APIResource):
    """
    Requests the agent to download and apply the specified version of itself.
    Will make the agent restart at the next opportunity.

    .. http:post:: /api/v1/update HTTP/1.1

        **Request**

        .. sourcecode:: http

            POST /api/v1/update HTTP/1.1
            Accept: application/json
示例#4
0
from errno import EEXIST
from functools import partial, wraps
from os.path import isdir, isfile, abspath
from uuid import UUID

from netaddr import AddrFormatError, IPAddress

from pyfarm.core.enums import OS, NUMERIC_TYPES
from pyfarm.core.utility import convert

from pyfarm.agent.logger import getLogger
from pyfarm.agent.config import config

INFINITE = set(["inf", "infinite", "unlimited"])

logger = getLogger("agent.parser")


def assert_parser(func):
    """
    ensures that the instance argument passed along to the validation
    function contains data we expect
    """
    @wraps(func)
    def run(*args, **kwargs):
        parser = kwargs.get("parser")
        assert parser is not None and isinstance(parser, ArgumentParser)
        return func(*args, **kwargs)
    return run

示例#5
0
"""
Disks
-----

Contains information about the local disks.
"""

from collections import namedtuple

import psutil

from pyfarm.agent.logger import getLogger

DiskInfo = namedtuple("DiskInfo", ("mountpoint", "free", "size"))

logger = getLogger("agent.disks")


def disks(as_dict=False):
    """
    Returns a list of disks in the system, in the form of :class:`DiskInfo`
    objects.

    :param bool as_dict:
        If True then return a dictionary value instead of :class:`DiskInfo`
        instances.  This is mainly used by the agent to eliminate an extra
        loop for translation.
    """
    out = []
    for partition in psutil.disk_partitions():
        try:
示例#6
0
    WindowsError = OSError

from twisted.internet import reactor
from twisted.internet.threads import deferToThreadPool
from twisted.python.threadpool import ThreadPool

from pyfarm.agent.config import config
from pyfarm.agent.sysinfo import cpu
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import UnicodeCSVWriter

STDOUT = 0
STDERR = 1
STREAMS = set([STDOUT, STDERR])

logger = getLogger("jobtypes.log")


class CSVLog(object):
    """
    Internal class which represents a log object and handle
    data that's internal to log handling.
    """
    def __init__(self, fileobj):
        # Check the type explicit here because UnicodeCSVWriter will
        # happy accept a non-file object the fail later.
        if not isinstance(fileobj, file):
            raise TypeError("`fileobj` should be a file")

        self.lock = RLock()
        self.file = fileobj
示例#7
0
from pyfarm.agent.http.api.base import APIRoot, Versions
from pyfarm.agent.http.api.config import Config
from pyfarm.agent.http.api.state import Status, Stop, Restart
from pyfarm.agent.http.api.tasks import Tasks
from pyfarm.agent.http.api.tasklogs import TaskLogs
from pyfarm.agent.http.api.update import Update
from pyfarm.agent.http.api.software import CheckSoftware
from pyfarm.agent.http.core.client import http_retry_delay, post_direct
from pyfarm.agent.http.core.resource import Resource
from pyfarm.agent.http.core.server import Site, StaticPath
from pyfarm.agent.http.system import Index, Configuration
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo import memory, network, system, cpu, graphics, disks
from pyfarm.agent import utility

svclog = getLogger("agent.service")
ntplog = getLogger("agent.service.ntp")


class Agent(object):
    """
    Main class associated with getting getting the internals of the
    agent's operations up and running including adding or updating
    itself with the master, starting the periodic task manager,
    and handling shutdown conditions.
    """
    def __init__(self):
        # so parts of this instance are accessible elsewhere
        assert "agent" not in config
        config["agent"] = self
        self.services = {}
示例#8
0
from twisted.internet.defer import DeferredList
from voluptuous import Schema, Required

from pyfarm.core.enums import WorkState, AgentState
from pyfarm.agent.config import config
from pyfarm.agent.http.core.client import post, http_retry_delay
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import request_from_master
from pyfarm.agent.sysinfo.memory import free_ram
from pyfarm.agent.utility import JOBTYPE_SCHEMA, TASKS_SCHEMA, JOB_SCHEMA
from pyfarm.jobtypes.core.internals import InsufficientSpaceError
from pyfarm.jobtypes.core.jobtype import JobType
from pyfarm.agent.utility import dumps

logger = getLogger("agent.http.assign")


class Assign(APIResource):
    isLeaf = False  # this is not really a collection of things

    # Schemas used for validating the request before
    # the target function will handle it.  These make
    # assertions about what kind of input data is required
    # or not based on the agent's internal code.
    SCHEMAS = {
        "POST": Schema({
            Required("job"): JOB_SCHEMA,
            Required("jobtype"): JOBTYPE_SCHEMA,
            Required("tasks"): TASKS_SCHEMA})}
示例#9
0
try:
    from httplib import BAD_REQUEST, OK, NOT_FOUND, INTERNAL_SERVER_ERROR
except ImportError:  # pragma: no cover
    from http.client import BAD_REQUEST, OK, NOT_FOUND, INTERNAL_SERVER_ERROR

from os.path import join

from twisted.web.server import NOT_DONE_YET
from twisted.protocols.basic import FileSender

from pyfarm.agent.config import config
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import request_from_master

logger = getLogger("agent.http.tasklogs")


class TaskLogs(APIResource):
    def _open_file(self, path, mode="rb"):
        """
        Small wrapper around the built-in :func:`open`.  We mainly
        use this for patching behavior in tests but could also use
        this for handling additional modes/exception/etc
        """
        return open(path, mode=mode)

    def get(self, **kwargs):
        """
        Get the contents of the specified task log.  The log will be
        returned in CSV format with the following fields:
示例#10
0
module to be used:

    >>> from pyfarm.agent.config import config
"""

import os
from datetime import datetime
from os.path import join, abspath, dirname
from logging import _levelNames, getLogger as _getLogger

from pyfarm.core.enums import NOTSET, STRING_TYPES
from pyfarm.core.config import Configuration
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo import memory, cpu, network

logger = getLogger("agent.config")


class LoggingConfiguration(Configuration):
    """
    Special configuration object which logs when a key is changed in
    a dictionary.  If the reactor is not running then log messages will
    be queued until they can be emitted so they are not lost.

    .. automethod:: _expandvars
    """
    MODIFIED = "modified"
    CREATED = "created"
    DELETED = "deleted"

    def __init__(self, data=None, environment=None, load=True):
示例#11
0
        responses, NOT_FOUND, BAD_REQUEST, UNSUPPORTED_MEDIA_TYPE,
        METHOD_NOT_ALLOWED, INTERNAL_SERVER_ERROR, OK, NOT_ACCEPTABLE)

from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.web.server import NOT_DONE_YET
from twisted.web.resource import Resource as _Resource
from twisted.web.static import File
from twisted.web.http import Request
from voluptuous import Invalid, Schema

from pyfarm.core.enums import STRING_TYPES
from pyfarm.agent.http.core import template
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import dumps

logger = getLogger("agent.http.resource")


class Resource(_Resource):
    """
    Basic subclass of :class:`._Resource` for passing requests to
    specific methods.  Unlike :class:`._Resource` however this will
    will also handle:

        * Templates
        * Content type discovery and validation
        * Handling of deferred responses
        * Validation of POST/PUT data against a schema

    :cvar string TEMPLATE:
        The name of the template this class will use when rendering
示例#12
0
from inspect import ismethod

from twisted.conch.insults.insults import ServerProtocol
from twisted.conch.manhole import ColoredManhole
from twisted.conch.telnet import ITelnetProtocol, TelnetBootstrapProtocol, AuthenticatingTelnetProtocol, TelnetTransport
from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
from twisted.cred.portal import IRealm, Portal
from twisted.internet.protocol import ServerFactory
from zope.interface import implements

from pyfarm.core.enums import STRING_TYPES, INTEGER_TYPES, NOTSET
from pyfarm.agent.logger import getLogger

ITERABLE_TYPES = (list, tuple, dict)

logger = getLogger("agent.manhole")


class LoggingManhole(ColoredManhole):
    """
    A slightly modified implementation of :class:`ColoredManhole`
    which logs information to the logger so we can track activity in
    the agent's log.
    """

    def connectionMade(self):  # pragma: no cover
        peer = self.terminal.transport.getPeer()
        logger.info("Connection made from %s@%s", peer.host, peer.port)
        super(LoggingManhole, self).connectionMade()

    def connectionLost(self, reason):  # pragma: no cover
示例#13
0
try:
    WindowsError
except NameError:  # pragma: no cover
    WindowsError = OSError

from voluptuous import Schema, Any, Required, Optional, Invalid
from twisted.internet import reactor
from twisted.internet.defer import AlreadyCalledError, DeferredLock, Deferred
from twisted.internet.error import AlreadyCalled

from pyfarm.core.enums import STRING_TYPES
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger

logger = getLogger("agent.util")
STRINGS = Any(*STRING_TYPES)
try:
    WHOLE_NUMBERS = Any(*(int, long))
    NUMBERS = Any(*(int, long, float, Decimal))
except NameError:  # pragma: no cover
    WHOLE_NUMBERS = int
    NUMBERS = Any(*(int, float, Decimal))


def validate_environment(values):
    """
    Ensures that ``values`` is a dictionary and that it only
    contains string keys and values.
    """
    if not isinstance(values, dict):
示例#14
0
from twisted.web._newclient import ResponseNeverReceived, RequestTransmissionFailed

import treq

from pyfarm.core.enums import WINDOWS, INTEGER_TYPES, STRING_TYPES, WorkState
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.http.core.client import get, post, http_retry_delay
from pyfarm.agent.utility import remove_file, remove_directory
from pyfarm.jobtypes.core.log import STDOUT, STDERR, logpool
from pyfarm.jobtypes.core.process import ReplaceEnvironment, ProcessProtocol

USER_GROUP_TYPES = tuple(list(STRING_TYPES) + list(INTEGER_TYPES) + [type(None)])
ITERABLE_CONTAINERS = (list, tuple, set)

logcache = getLogger("jobtypes.cache")
logger = getLogger("jobtypes.core")
logfile = getLogger("jobtypes.log")
ProcessData = namedtuple("ProcessData", ("protocol", "started", "stopped"))


class InsufficientSpaceError(Exception):
    pass


class Cache(object):
    """Internal methods for caching job types"""

    cache = {}
    JOBTYPE_VERSION_URL = "%(master_api)s/jobtypes/%(name)s/versions/%(version)s"
    CACHE_DIRECTORY = config.get("jobtype_cache_directory", "")
示例#15
0
"""

import os
import platform
import sys
import time
import tempfile
import uuid
from os.path import isfile

import psutil

from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import remove_file

logger = getLogger("agent.sysinfo")

# Determine if file system is case sensitive
try:
    _filesystem_is_case_sensitive
except NameError:  # pragma: no cover
    fd, path = tempfile.mkstemp()
    _filesystem_is_case_sensitive = \
        not all(map(isfile, [path, path.lower(), path.upper()]))

    try:
        os.close(fd)
    except OSError:  # pragma: no cover
        pass

    remove_file(path, retry_on_exit=True, raise_=False)
示例#16
0
    .. seealso::
        :rfc:`1918`

:const IP_NONNETWORK:
    set of non-network address ranges including all of the
    above constants except the :const:`IP_PRIVATE`
"""

import socket

import netifaces
from netaddr import IPSet, IPNetwork, IPAddress

from pyfarm.agent.logger import getLogger

logger = getLogger("agent.dns")

IP_PRIVATE = IPSet([
    IPNetwork("10.0.0.0/8"),
    IPNetwork("172.16.0.0/12"),
    IPNetwork("192.168.0.0/16")])
IP_NONNETWORK = IPSet([
    IPNetwork("0.0.0.0/8"),         # special use
    IPNetwork("169.254.0.0/16"),    # link local
    IPNetwork("127.0.0.0/8"),       # loopback
    IPNetwork("224.0.0.0/4"),       # multicast
    IPNetwork("255.255.255.255")])  # broadcast


def mac_addresses(long_addresses=False, as_integers=False):
    """
示例#17
0
    from httplib import OK, INTERNAL_SERVER_ERROR, NOT_FOUND
except ImportError:  # pragma: no cover
    from http.client import OK, INTERNAL_SERVER_ERROR, NOT_FOUND

import imp, sys

import treq

from twisted.internet import reactor, threads
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred

from pyfarm.agent.logger import getLogger
from pyfarm.agent.config import config
from pyfarm.agent.http.core.client import get_direct, http_retry_delay

logger = getLogger("agent.sysinfo.software")


class VersionNotFound(Exception):
    pass


@inlineCallbacks
def get_software_version_data(software, version):
    """
    Asynchronously fetches the known data about the given software version from
    the master.

    :param str software:
        The name of the software to get data for
示例#18
0
import psutil

from pyfarm.core.enums import WINDOWS, LINUX, MAC
from pyfarm.agent.logger import getLogger

try:
    from wmi import WMI
except ImportError:  # pragma: no cover
    WMI = NotImplemented

try:
    LIBC = cdll.LoadLibrary(find_library("c"))
except OSError:  # pragma: no cover
    LIBC = NotImplemented

logger = getLogger("agent.cpu")


def cpu_name():
    """
    Returns the full name of the CPU installed in the system.
    """
    if WINDOWS:
        wmi = WMI()
        processor = wmi.Win32_Processor()[0]
        return processor.name

    elif LINUX:
        with open("/proc/cpuinfo", "r") as cpuinfo:
            for line in cpuinfo:
                if line.startswith("model name"):
示例#19
0
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol, connectionDone
from twisted.python import log
from twisted.python.failure import Failure
from twisted.web.client import (
    Response as TWResponse, GzipDecoder as TWGzipDecoder, ResponseDone)
from twisted.web._newclient import (
    ResponseNeverReceived, RequestTransmissionFailed)

from pyfarm.core.enums import STRING_TYPES, NOTSET, INTEGER_TYPES
from pyfarm.core.utility import ImmutableDict
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import quote_url, json_safe

logger = getLogger("agent.http.client")

# response classes which are allowed to the `response` argument
# to Response.__init__
if TQResponse is not NotImplemented:
    RESPONSE_CLASSES = (TWResponse, TWGzipDecoder, TQResponse)
else:  # pragma: no cover
    RESPONSE_CLASSES = (TWResponse, TWGzipDecoder)

USERAGENT = "PyFarm/1.0 (agent)"
DELAY_NUMBER_TYPES = tuple(list(INTEGER_TYPES) + [float])
HTTP_METHODS = frozenset(("HEAD", "GET", "POST", "PUT", "PATCH", "DELETE"))
HTTP_SCHEMES = frozenset(("http", "https"))


class HTTPLog(object):
示例#20
0
from twisted.web.server import NOT_DONE_YET
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks

from voluptuous import Schema, Required

from pyfarm.agent.config import config
from pyfarm.agent.utility import STRINGS
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo.software import check_software_availability
from pyfarm.agent.http.core.client import (
    post_direct, delete_direct, http_retry_delay)
from pyfarm.agent.http.api.base import APIResource

logger = getLogger("agent.http.software")


class CheckSoftware(APIResource):
    """
    Requests the agent to check whether a given software version is installed
    locally.

    The agent will asynchronously update its software information on the master.

    .. http:post:: /api/v1/check_software HTTP/1.1

        **Request**

        .. sourcecode:: http
示例#21
0
from collections import namedtuple
from functools import partial
from random import choice, randint, random
from textwrap import dedent

import psutil
import requests

from pyfarm.core.enums import NUMERIC_TYPES
from pyfarm.core.utility import convert
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import dumps


logger = getLogger("agent.cmd")

config["start"] = time.time()


def fake_render():
    process = psutil.Process()
    memory_usage = lambda: convert.bytetomb(process.get_memory_info().rss)
    memory_used_at_start = memory_usage()

    logger.info("sys.argv: %r", sys.argv)

    # build parser
    parser = argparse.ArgumentParser(
        description="Very basic command line tool which vaguely simulates a "
                    "render.")
示例#22
0
except ImportError:  # pragma: no cover
    from http.client import ACCEPTED, OK, BAD_REQUEST

import psutil
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.web.server import NOT_DONE_YET
from voluptuous import Schema, Optional

from pyfarm.agent.config import config
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo import memory
from pyfarm.agent.utility import dumps, total_seconds, remove_file

logger = getLogger("agent.http.state")


class Stop(APIResource):
    isLeaf = False  # this is not really a collection of things
    SCHEMAS = {
        "POST": Schema({
            Optional("wait"): bool})}

    def post(self, **kwargs):
        request = kwargs["request"]
        data = kwargs["data"]
        agent = config["agent"]

        remove_file(
            config["run_control_file"],