try:
from xml.etree.cElementTree import ElementTree
except ImportError: # pragma: no cover
from xml.etree.ElementTree import ElementTree
try:
from wmi import WMI
except ImportError: # pragma: no cover
WMI = NotImplemented
from pyfarm.core.enums import WINDOWS, LINUX, MAC
from pyfarm.agent.logger import getLogger
from pyfarm.agent.config import config
logger = getLogger("agent.sysinfo.gpu")
class GPULookupError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def graphics_cards():
"""
Returns a list of the full names of GPUs installed in this system
"""
if WINDOWS:
except ImportError: # pragma: no cover
getgrgid = NotImplemented
getpwuid = NotImplemented
setuid = NotImplemented
getuid = NotImplemented
setgid = NotImplemented
getgid = NotImplemented
from pyfarm.core.enums import INTEGER_TYPES, OS, operating_system
from pyfarm.agent.config import config
from pyfarm.agent.entrypoints.parser import AgentArgumentParser
from pyfarm.agent.entrypoints.utility import start_daemon_posix
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import remove_file, remove_directory
logger = getLogger("agent.supervisor")
def supervisor():
logger.debug("Supervisor called with: %r", sys.argv)
supervisor_args = []
agent_args = []
in_agent_args = False
tmp_argv = list(sys.argv)
del tmp_argv[0]
for arg in tmp_argv:
if not in_agent_args and arg != "--":
supervisor_args.append(arg)
elif not in_agent_args and arg == "--":
in_agent_args = True
else:
from os.path import join, isdir
from treq import get, collect
from twisted.internet import reactor
from twisted.web.server import NOT_DONE_YET
import twisted.python.failure
from voluptuous import Schema, Required
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.http.core.client import http_retry_delay
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import STRINGS
logger = getLogger("agent.http.update")
class Update(APIResource):
"""
Requests the agent to download and apply the specified version of itself.
Will make the agent restart at the next opportunity.
.. http:post:: /api/v1/update HTTP/1.1
**Request**
.. sourcecode:: http
POST /api/v1/update HTTP/1.1
Accept: application/json
from errno import EEXIST
from functools import partial, wraps
from os.path import isdir, isfile, abspath
from uuid import UUID
from netaddr import AddrFormatError, IPAddress
from pyfarm.core.enums import OS, NUMERIC_TYPES
from pyfarm.core.utility import convert
from pyfarm.agent.logger import getLogger
from pyfarm.agent.config import config
INFINITE = set(["inf", "infinite", "unlimited"])
logger = getLogger("agent.parser")
def assert_parser(func):
"""
ensures that the instance argument passed along to the validation
function contains data we expect
"""
@wraps(func)
def run(*args, **kwargs):
parser = kwargs.get("parser")
assert parser is not None and isinstance(parser, ArgumentParser)
return func(*args, **kwargs)
return run
"""
Disks
-----
Contains information about the local disks.
"""
from collections import namedtuple
import psutil
from pyfarm.agent.logger import getLogger
DiskInfo = namedtuple("DiskInfo", ("mountpoint", "free", "size"))
logger = getLogger("agent.disks")
def disks(as_dict=False):
"""
Returns a list of disks in the system, in the form of :class:`DiskInfo`
objects.
:param bool as_dict:
If True then return a dictionary value instead of :class:`DiskInfo`
instances. This is mainly used by the agent to eliminate an extra
loop for translation.
"""
out = []
for partition in psutil.disk_partitions():
try:
WindowsError = OSError
from twisted.internet import reactor
from twisted.internet.threads import deferToThreadPool
from twisted.python.threadpool import ThreadPool
from pyfarm.agent.config import config
from pyfarm.agent.sysinfo import cpu
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import UnicodeCSVWriter
STDOUT = 0
STDERR = 1
STREAMS = set([STDOUT, STDERR])
logger = getLogger("jobtypes.log")
class CSVLog(object):
"""
Internal class which represents a log object and handle
data that's internal to log handling.
"""
def __init__(self, fileobj):
# Check the type explicit here because UnicodeCSVWriter will
# happy accept a non-file object the fail later.
if not isinstance(fileobj, file):
raise TypeError("`fileobj` should be a file")
self.lock = RLock()
self.file = fileobj
from pyfarm.agent.http.api.base import APIRoot, Versions
from pyfarm.agent.http.api.config import Config
from pyfarm.agent.http.api.state import Status, Stop, Restart
from pyfarm.agent.http.api.tasks import Tasks
from pyfarm.agent.http.api.tasklogs import TaskLogs
from pyfarm.agent.http.api.update import Update
from pyfarm.agent.http.api.software import CheckSoftware
from pyfarm.agent.http.core.client import http_retry_delay, post_direct
from pyfarm.agent.http.core.resource import Resource
from pyfarm.agent.http.core.server import Site, StaticPath
from pyfarm.agent.http.system import Index, Configuration
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo import memory, network, system, cpu, graphics, disks
from pyfarm.agent import utility
svclog = getLogger("agent.service")
ntplog = getLogger("agent.service.ntp")
class Agent(object):
"""
Main class associated with getting getting the internals of the
agent's operations up and running including adding or updating
itself with the master, starting the periodic task manager,
and handling shutdown conditions.
"""
def __init__(self):
# so parts of this instance are accessible elsewhere
assert "agent" not in config
config["agent"] = self
self.services = {}
from twisted.internet.defer import DeferredList
from voluptuous import Schema, Required
from pyfarm.core.enums import WorkState, AgentState
from pyfarm.agent.config import config
from pyfarm.agent.http.core.client import post, http_retry_delay
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import request_from_master
from pyfarm.agent.sysinfo.memory import free_ram
from pyfarm.agent.utility import JOBTYPE_SCHEMA, TASKS_SCHEMA, JOB_SCHEMA
from pyfarm.jobtypes.core.internals import InsufficientSpaceError
from pyfarm.jobtypes.core.jobtype import JobType
from pyfarm.agent.utility import dumps
logger = getLogger("agent.http.assign")
class Assign(APIResource):
isLeaf = False # this is not really a collection of things
# Schemas used for validating the request before
# the target function will handle it. These make
# assertions about what kind of input data is required
# or not based on the agent's internal code.
SCHEMAS = {
"POST": Schema({
Required("job"): JOB_SCHEMA,
Required("jobtype"): JOBTYPE_SCHEMA,
Required("tasks"): TASKS_SCHEMA})}
try:
from httplib import BAD_REQUEST, OK, NOT_FOUND, INTERNAL_SERVER_ERROR
except ImportError: # pragma: no cover
from http.client import BAD_REQUEST, OK, NOT_FOUND, INTERNAL_SERVER_ERROR
from os.path import join
from twisted.web.server import NOT_DONE_YET
from twisted.protocols.basic import FileSender
from pyfarm.agent.config import config
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import request_from_master
logger = getLogger("agent.http.tasklogs")
class TaskLogs(APIResource):
def _open_file(self, path, mode="rb"):
"""
Small wrapper around the built-in :func:`open`. We mainly
use this for patching behavior in tests but could also use
this for handling additional modes/exception/etc
"""
return open(path, mode=mode)
def get(self, **kwargs):
"""
Get the contents of the specified task log. The log will be
returned in CSV format with the following fields:
module to be used:
>>> from pyfarm.agent.config import config
"""
import os
from datetime import datetime
from os.path import join, abspath, dirname
from logging import _levelNames, getLogger as _getLogger
from pyfarm.core.enums import NOTSET, STRING_TYPES
from pyfarm.core.config import Configuration
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo import memory, cpu, network
logger = getLogger("agent.config")
class LoggingConfiguration(Configuration):
"""
Special configuration object which logs when a key is changed in
a dictionary. If the reactor is not running then log messages will
be queued until they can be emitted so they are not lost.
.. automethod:: _expandvars
"""
MODIFIED = "modified"
CREATED = "created"
DELETED = "deleted"
def __init__(self, data=None, environment=None, load=True):
responses, NOT_FOUND, BAD_REQUEST, UNSUPPORTED_MEDIA_TYPE,
METHOD_NOT_ALLOWED, INTERNAL_SERVER_ERROR, OK, NOT_ACCEPTABLE)
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.web.server import NOT_DONE_YET
from twisted.web.resource import Resource as _Resource
from twisted.web.static import File
from twisted.web.http import Request
from voluptuous import Invalid, Schema
from pyfarm.core.enums import STRING_TYPES
from pyfarm.agent.http.core import template
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import dumps
logger = getLogger("agent.http.resource")
class Resource(_Resource):
"""
Basic subclass of :class:`._Resource` for passing requests to
specific methods. Unlike :class:`._Resource` however this will
will also handle:
* Templates
* Content type discovery and validation
* Handling of deferred responses
* Validation of POST/PUT data against a schema
:cvar string TEMPLATE:
The name of the template this class will use when rendering
from inspect import ismethod
from twisted.conch.insults.insults import ServerProtocol
from twisted.conch.manhole import ColoredManhole
from twisted.conch.telnet import ITelnetProtocol, TelnetBootstrapProtocol, AuthenticatingTelnetProtocol, TelnetTransport
from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
from twisted.cred.portal import IRealm, Portal
from twisted.internet.protocol import ServerFactory
from zope.interface import implements
from pyfarm.core.enums import STRING_TYPES, INTEGER_TYPES, NOTSET
from pyfarm.agent.logger import getLogger
ITERABLE_TYPES = (list, tuple, dict)
logger = getLogger("agent.manhole")
class LoggingManhole(ColoredManhole):
"""
A slightly modified implementation of :class:`ColoredManhole`
which logs information to the logger so we can track activity in
the agent's log.
"""
def connectionMade(self): # pragma: no cover
peer = self.terminal.transport.getPeer()
logger.info("Connection made from %s@%s", peer.host, peer.port)
super(LoggingManhole, self).connectionMade()
def connectionLost(self, reason): # pragma: no cover
try:
WindowsError
except NameError: # pragma: no cover
WindowsError = OSError
from voluptuous import Schema, Any, Required, Optional, Invalid
from twisted.internet import reactor
from twisted.internet.defer import AlreadyCalledError, DeferredLock, Deferred
from twisted.internet.error import AlreadyCalled
from pyfarm.core.enums import STRING_TYPES
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
logger = getLogger("agent.util")
STRINGS = Any(*STRING_TYPES)
try:
WHOLE_NUMBERS = Any(*(int, long))
NUMBERS = Any(*(int, long, float, Decimal))
except NameError: # pragma: no cover
WHOLE_NUMBERS = int
NUMBERS = Any(*(int, float, Decimal))
def validate_environment(values):
"""
Ensures that ``values`` is a dictionary and that it only
contains string keys and values.
"""
if not isinstance(values, dict):
from twisted.web._newclient import ResponseNeverReceived, RequestTransmissionFailed
import treq
from pyfarm.core.enums import WINDOWS, INTEGER_TYPES, STRING_TYPES, WorkState
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.http.core.client import get, post, http_retry_delay
from pyfarm.agent.utility import remove_file, remove_directory
from pyfarm.jobtypes.core.log import STDOUT, STDERR, logpool
from pyfarm.jobtypes.core.process import ReplaceEnvironment, ProcessProtocol
USER_GROUP_TYPES = tuple(list(STRING_TYPES) + list(INTEGER_TYPES) + [type(None)])
ITERABLE_CONTAINERS = (list, tuple, set)
logcache = getLogger("jobtypes.cache")
logger = getLogger("jobtypes.core")
logfile = getLogger("jobtypes.log")
ProcessData = namedtuple("ProcessData", ("protocol", "started", "stopped"))
class InsufficientSpaceError(Exception):
pass
class Cache(object):
"""Internal methods for caching job types"""
cache = {}
JOBTYPE_VERSION_URL = "%(master_api)s/jobtypes/%(name)s/versions/%(version)s"
CACHE_DIRECTORY = config.get("jobtype_cache_directory", "")
"""
import os
import platform
import sys
import time
import tempfile
import uuid
from os.path import isfile
import psutil
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import remove_file
logger = getLogger("agent.sysinfo")
# Determine if file system is case sensitive
try:
_filesystem_is_case_sensitive
except NameError: # pragma: no cover
fd, path = tempfile.mkstemp()
_filesystem_is_case_sensitive = \
not all(map(isfile, [path, path.lower(), path.upper()]))
try:
os.close(fd)
except OSError: # pragma: no cover
pass
remove_file(path, retry_on_exit=True, raise_=False)
.. seealso::
:rfc:`1918`
:const IP_NONNETWORK:
set of non-network address ranges including all of the
above constants except the :const:`IP_PRIVATE`
"""
import socket
import netifaces
from netaddr import IPSet, IPNetwork, IPAddress
from pyfarm.agent.logger import getLogger
logger = getLogger("agent.dns")
IP_PRIVATE = IPSet([
IPNetwork("10.0.0.0/8"),
IPNetwork("172.16.0.0/12"),
IPNetwork("192.168.0.0/16")])
IP_NONNETWORK = IPSet([
IPNetwork("0.0.0.0/8"), # special use
IPNetwork("169.254.0.0/16"), # link local
IPNetwork("127.0.0.0/8"), # loopback
IPNetwork("224.0.0.0/4"), # multicast
IPNetwork("255.255.255.255")]) # broadcast
def mac_addresses(long_addresses=False, as_integers=False):
"""
from httplib import OK, INTERNAL_SERVER_ERROR, NOT_FOUND
except ImportError: # pragma: no cover
from http.client import OK, INTERNAL_SERVER_ERROR, NOT_FOUND
import imp, sys
import treq
from twisted.internet import reactor, threads
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
from pyfarm.agent.logger import getLogger
from pyfarm.agent.config import config
from pyfarm.agent.http.core.client import get_direct, http_retry_delay
logger = getLogger("agent.sysinfo.software")
class VersionNotFound(Exception):
pass
@inlineCallbacks
def get_software_version_data(software, version):
"""
Asynchronously fetches the known data about the given software version from
the master.
:param str software:
The name of the software to get data for
import psutil
from pyfarm.core.enums import WINDOWS, LINUX, MAC
from pyfarm.agent.logger import getLogger
try:
from wmi import WMI
except ImportError: # pragma: no cover
WMI = NotImplemented
try:
LIBC = cdll.LoadLibrary(find_library("c"))
except OSError: # pragma: no cover
LIBC = NotImplemented
logger = getLogger("agent.cpu")
def cpu_name():
"""
Returns the full name of the CPU installed in the system.
"""
if WINDOWS:
wmi = WMI()
processor = wmi.Win32_Processor()[0]
return processor.name
elif LINUX:
with open("/proc/cpuinfo", "r") as cpuinfo:
for line in cpuinfo:
if line.startswith("model name"):
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol, connectionDone
from twisted.python import log
from twisted.python.failure import Failure
from twisted.web.client import (
Response as TWResponse, GzipDecoder as TWGzipDecoder, ResponseDone)
from twisted.web._newclient import (
ResponseNeverReceived, RequestTransmissionFailed)
from pyfarm.core.enums import STRING_TYPES, NOTSET, INTEGER_TYPES
from pyfarm.core.utility import ImmutableDict
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import quote_url, json_safe
logger = getLogger("agent.http.client")
# response classes which are allowed to the `response` argument
# to Response.__init__
if TQResponse is not NotImplemented:
RESPONSE_CLASSES = (TWResponse, TWGzipDecoder, TQResponse)
else: # pragma: no cover
RESPONSE_CLASSES = (TWResponse, TWGzipDecoder)
USERAGENT = "PyFarm/1.0 (agent)"
DELAY_NUMBER_TYPES = tuple(list(INTEGER_TYPES) + [float])
HTTP_METHODS = frozenset(("HEAD", "GET", "POST", "PUT", "PATCH", "DELETE"))
HTTP_SCHEMES = frozenset(("http", "https"))
class HTTPLog(object):
from twisted.web.server import NOT_DONE_YET
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks
from voluptuous import Schema, Required
from pyfarm.agent.config import config
from pyfarm.agent.utility import STRINGS
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo.software import check_software_availability
from pyfarm.agent.http.core.client import (
post_direct, delete_direct, http_retry_delay)
from pyfarm.agent.http.api.base import APIResource
logger = getLogger("agent.http.software")
class CheckSoftware(APIResource):
"""
Requests the agent to check whether a given software version is installed
locally.
The agent will asynchronously update its software information on the master.
.. http:post:: /api/v1/check_software HTTP/1.1
**Request**
.. sourcecode:: http
from collections import namedtuple
from functools import partial
from random import choice, randint, random
from textwrap import dedent
import psutil
import requests
from pyfarm.core.enums import NUMERIC_TYPES
from pyfarm.core.utility import convert
from pyfarm.agent.config import config
from pyfarm.agent.logger import getLogger
from pyfarm.agent.utility import dumps
logger = getLogger("agent.cmd")
config["start"] = time.time()
def fake_render():
process = psutil.Process()
memory_usage = lambda: convert.bytetomb(process.get_memory_info().rss)
memory_used_at_start = memory_usage()
logger.info("sys.argv: %r", sys.argv)
# build parser
parser = argparse.ArgumentParser(
description="Very basic command line tool which vaguely simulates a "
"render.")
except ImportError: # pragma: no cover
from http.client import ACCEPTED, OK, BAD_REQUEST
import psutil
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.web.server import NOT_DONE_YET
from voluptuous import Schema, Optional
from pyfarm.agent.config import config
from pyfarm.agent.http.api.base import APIResource
from pyfarm.agent.logger import getLogger
from pyfarm.agent.sysinfo import memory
from pyfarm.agent.utility import dumps, total_seconds, remove_file
logger = getLogger("agent.http.state")
class Stop(APIResource):
isLeaf = False # this is not really a collection of things
SCHEMAS = {
"POST": Schema({
Optional("wait"): bool})}
def post(self, **kwargs):
request = kwargs["request"]
data = kwargs["data"]
agent = config["agent"]
remove_file(
config["run_control_file"],
