def _parse_args(args):
parser = argparse.ArgumentParser(
description='Start up a simple heartbeat server on localhost.')
parser.add_argument(
'--port', default=8080,
help='The port to start the heartbeat server.')
parser.add_argument(
'--db_host',
default=config.get_config_value('AUTOTEST_WEB', 'host'),
help='Db server ip address.')
parser.add_argument(
'--db_name',
default=config.get_config_value('AUTOTEST_WEB', 'database'),
help='Name of the db table.')
parser.add_argument(
'--db_user',
default=config.get_config_value('AUTOTEST_WEB', 'user'),
help='User for the db server.')
parser.add_argument(
'--db_password',
default=config.get_config_value('AUTOTEST_WEB', 'password'),
help='Password for the db server.')
parser.add_argument(
'--db_port',
default=config.get_config_value('AUTOTEST_WEB', 'port', default=''),
help='Port of the db server.')
return parser.parse_args(args)
def use_server_db():
"""Check if use_server_db is enabled in configuration.
@return: True if use_server_db is set to True in global config.
"""
return global_config.get_config_value('SERVER',
'use_server_db',
default=False,
type=bool)
def get_logging_handler():
"""Return a logging handler.
Configure a RPC logging handler based on global_config and return
the handler.
"""
max_log_size = config.get_config_value('SERVER',
'rpc_max_log_size_mb',
type=int)
number_of_old_logs = config.get_config_value('SERVER',
'rpc_num_old_logs',
type=int)
log_path = config.get_config_value('SERVER', 'rpc_log_path')
formatter = logging.Formatter(fmt=LOGGING_FORMAT, datefmt='%m/%d %H:%M:%S')
handler = logging.handlers.RotatingFileHandler(
log_path,
maxBytes=max_log_size * MEGABYTE,
backupCount=number_of_old_logs)
handler.setFormatter(formatter)
return handler
def initialize(self,
test_env=TestEnv.sandbox,
account='*****@*****.**'):
"""
Initialize the test.
@param test_env: The test environment to use
"""
super(webstore_test, self).initialize()
self.username = account
self.password = global_config.get_config_value(
'CLIENT', 'webstore_test_password', type=str)
self.test_env = test_env
self._chrome_flags = _get_chrome_flags(test_env)
self.webstore_url = {
TestEnv.staging: 'https://webstore-staging.corp.google.com',
TestEnv.sandbox:
'https://webstore-staging.sandbox.google.com/webstore',
TestEnv.pnl:
'https://webstore-prod-not-live.corp.google.com/webstore',
TestEnv.prod: 'https://chrome.google.com/webstore'
}[test_env]
class RDBHostCacheManager(object):
"""RDB Cache manager."""
key = collections.namedtuple('key', ['deps', 'acls'])
use_cache = global_config.get_config_value('RDB',
'use_cache',
type=bool,
default=True)
def __init__(self):
self._cache_backend = (InMemoryCacheBackend()
if self.use_cache else DummyCacheBackend())
self.hits = 0
self.misses = 0
self.stale_entries = []
def mean_staleness(self):
"""Compute the average stale entries per line.
@return: A floating point representing the mean staleness.
"""
return (reduce(lambda x, y: float(x + y), self.stale_entries) /
len(self.stale_entries)) if self.stale_entries else 0
def hit_ratio(self):
"""Compute the hit ratio of this cache.
@return: A floating point percentage of the hit ratio.
"""
if not self.hits and not self.misses:
return 0
requests = float(self.hits + self.misses)
return (self.hits / requests) * 100
def record_stats(self):
"""Record stats about the cache managed by this instance."""
hit_ratio = self.hit_ratio()
staleness = self.mean_staleness()
logging.debug(
'Cache stats: hit ratio: %.2f%%, '
'avg staleness per line: %.2f%%.', hit_ratio, staleness)
metrics.Float('chromeos/autotest/scheduler/rdb/cache/hit_ratio').set(
hit_ratio)
metrics.Float('chromeos/autotest/scheduler/rdb/cache/mean_staleness'
).set(staleness)
@classmethod
def get_key(cls, deps, acls):
"""Return a key for the given deps, acls.
@param deps: A list of deps, as taken by the AcquireHostRequest.
@param acls: A list of acls, as taken by the AcquireHostRequest.
@return: A cache key for the given deps/acls.
"""
# All requests with the same deps, acls should hit the same cache line.
# TODO: Do something smarter with acls, only one needs to match.
return cls.key(deps=frozenset(deps), acls=frozenset(acls))
def get_line(self, key):
"""Clear and return the cache line matching the key.
@param key: The key the desired cache_line is stored under.
@return: A list of rdb hosts matching the key, or None.
@raises rdb_utils.CacheMiss: If the key isn't in the cache.
"""
try:
cache_line = self._cache_backend.get(key)
except KeyError:
self.misses += 1
raise rdb_utils.CacheMiss('Key %s not in cache' % (key, ))
self.hits += 1
self._cache_backend.delete(key)
return list(cache_line)
def _check_line(self, line, key):
"""Sanity check a cache line.
This method assumes that a cache line is made up of RDBHost objects,
and checks to see if they all match each other/the key passed in.
Checking is done in terms of host labels and acls, note that the hosts
in the line can have different deps/acls, as long as they all have the
deps required by the key, and at least one matching acl of the key.
@param line: The cache line value.
@param key: The key the line will be stored under.
@raises rdb_utils.RDBException:
If one of the hosts in the cache line is already leased.
The cache already has a different line under the given key.
The given key doesn't match the hosts in the line.
"""
# Note that this doesn't mean that all hosts in the cache are unleased.
if any(host.leased for host in line):
raise rdb_utils.RDBException('Cannot cache leased hosts %s' % line)
# Confirm that the given line can be used to service the key by checking
# that all hosts have the deps mentioned in the key, and at least one
# matching acl.
h_keys = set([self.get_key(host.labels, host.acls) for host in line])
for h_key in h_keys:
if (not h_key.deps.issuperset(key.deps)
or not key.acls.intersection(h_key.acls)):
raise rdb_utils.RDBException(
'Given key: %s does not match key '
'computed from hosts in line: %s' % (key, h_keys))
if self._cache_backend.has_key(key):
raise rdb_utils.RDBException(
'Cannot override a cache line. It '
'must be cleared before setting. Key: %s, hosts %s' %
(key, line))
def set_line(self, key, hosts):
"""Cache a list of similar hosts.
set_line will no-op if:
The hosts aren't all unleased.
The hosts don't have deps/acls matching the key.
A cache line under the same key already exists.
The first 2 cases will lead to a cache miss in the corresponding get.
@param hosts: A list of unleased hosts with the same deps/acls.
@raises RDBException: If hosts is None, since None is reserved for
key expiration.
"""
if hosts is None:
raise rdb_utils.RDBException('Cannot set None in the cache.')
# An empty list means no hosts matching the request are available.
# This can happen if a previous request leased all matching hosts.
if not hosts or not self.use_cache:
self._cache_backend.set(key, [])
return
try:
self._check_line(hosts, key)
except rdb_utils.RDBException as e:
logging.error(e)
else:
self._cache_backend.set(key, set(hosts))
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import common
from autotest_lib.client.bin import utils as common_utils
from autotest_lib.client.common_lib.global_config import global_config
# Name of the base container.
BASE = global_config.get_config_value('AUTOSERV', 'container_base_name')
# Path to folder that contains autotest code inside container.
CONTAINER_AUTOTEST_DIR = '/usr/local/autotest'
# Naming convention of the result directory in test container.
RESULT_DIR_FMT = os.path.join(CONTAINER_AUTOTEST_DIR, 'results', '%s')
# Attributes to retrieve about containers.
ATTRIBUTES = ['name', 'state']
SSP_ENABLED = global_config.get_config_value('AUTOSERV',
'enable_ssp_container',
type=bool,
default=True)
# url to the folder stores base container.
CONTAINER_BASE_FOLDER_URL = global_config.get_config_value(
'AUTOSERV', 'container_base_folder_url')
CONTAINER_BASE_URL_FMT = '%s/%%s.tar.xz' % CONTAINER_BASE_FOLDER_URL
CONTAINER_BASE_URL = CONTAINER_BASE_URL_FMT % BASE
# Default directory used to store LXC containers.
"""
Library for autotest-remote usage.
"""
import sys, os, re, traceback, signal, time, logging, getpass
try:
import autotest.common as common
except ImportError:
import common
from autotest_lib.client.common_lib.global_config import global_config
require_atfork = global_config.get_config_value('AUTOSERV',
'require_atfork_module',
type=bool,
default=True)
try:
import atfork
atfork.monkeypatch_os_fork_functions()
import atfork.stdlib_fixer
# Fix the Python standard library for threading+fork safety with its
# internal locks. http://code.google.com/p/python-atfork/
import warnings
warnings.filterwarnings('ignore', 'logging module already imported')
atfork.stdlib_fixer.fix_logging_module()
except ImportError, e:
from autotest_lib.client.common_lib import global_config
if global_config.global_config.get_config_value('AUTOSERV',
'require_atfork_module',
type=bool,
from autotest_lib.site_utils.lxc.container import Container
from autotest_lib.site_utils.lxc.container_factory import ContainerFactory
try:
from chromite.lib import metrics
from infra_libs import ts_mon
except ImportError:
import mock
metrics = utils.metrics_mock
ts_mon = mock.Mock()
# Timeout (in seconds) for container pool operations.
_CONTAINER_POOL_TIMEOUT = 3
_USE_LXC_POOL = global_config.get_config_value('LXC_POOL', 'use_lxc_pool',
type=bool)
class ContainerBucket(object):
"""A wrapper class to interact with containers in a specific container path.
"""
def __init__(self, container_path=constants.DEFAULT_CONTAINER_PATH,
container_factory=None):
"""Initialize a ContainerBucket.
@param container_path: Path to the directory used to store containers.
Default is set to AUTOSERV/container_path in
global config.
@param container_factory: A factory for creating Containers.
"""
self.container_path = os.path.realpath(container_path)
"""
Library for autotest-remote usage.
"""
import sys, os, re, traceback, signal, time, logging, getpass
import common
from autotest_lib.client.common_lib.global_config import global_config
require_atfork = global_config.get_config_value(
'AUTOSERV', 'require_atfork_module', type=bool, default=True)
try:
import atfork
atfork.monkeypatch_os_fork_functions()
import atfork.stdlib_fixer
# Fix the Python standard library for threading+fork safety with its
# internal locks. http://code.google.com/p/python-atfork/
import warnings
warnings.filterwarnings('ignore', 'logging module already imported')
atfork.stdlib_fixer.fix_logging_module()
except ImportError, e:
from autotest_lib.client.common_lib import global_config
if global_config.global_config.get_config_value(
'AUTOSERV', 'require_atfork_module', type=bool, default=False):
print >>sys.stderr, 'Please run utils/build_externals.py'
print e
sys.exit(1)
from autotest_lib.server import server_logging_config
from autotest_lib.server import server_job, utils, autoserv_parser, autotest