def get_mappings_post_2x():
KERNEL_BASE_URL = settings.get_value('CLIENT', 'kernel_mirror', default='')
GITWEB_BASE_URL = settings.get_value('CLIENT', 'kernel_gitweb', default='')
STABLE_GITWEB_BASE_URL = settings.get_value('CLIENT', 'stable_kernel_gitweb', default='')
MAPPINGS_POST_2X = [
[r'^\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', GITWEB_BASE_URL.split())
],
[r'^\d+\.\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', STABLE_GITWEB_BASE_URL.split())
],
[r'-rc\d+$', '', True,
map(lambda x: x + 'v%(major)s/testing/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', GITWEB_BASE_URL.split())
],
[r'[a-f0-9]{7,40}$', '', True,
map(lambda x: x + ';a=snapshot;h=%(full)s;sf=tgz', GITWEB_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=%(full)s;sf=tgz', STABLE_GITWEB_BASE_URL.split())
]
]
return MAPPINGS_POST_2X
def load_profiler(self, profiler, args, dargs):
prof_dir = os.path.join(self.job.autodir, "profilers", profiler)
try:
self.job.install_pkg(profiler, "profiler", prof_dir)
except error.PackageInstallError:
pass
if not os.path.exists(prof_dir):
raise profiler_manager.ProfilerNotPresentError(profiler)
profiler_module = common.setup_modules.import_module(
profiler, "autotest.client.profilers.%s" % profiler)
newprofiler = getattr(profiler_module, profiler)(self.job)
newprofiler.name = profiler
newprofiler.bindir = os.path.join(prof_dir)
try:
autodir = os.path.abspath(os.environ['AUTODIR'])
except KeyError:
autodir = settings.get_value('COMMON', 'autotest_top_path')
tmpdir = os.path.join(autodir, 'tmp')
output_config = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
newprofiler.srcdir = os.path.join(output_config,
os.path.basename(newprofiler.bindir),
'src')
newprofiler.tmpdir = os.path.join(self.tmpdir, profiler)
newprofiler.initialize(*args, **dargs)
utils.update_version(newprofiler.srcdir, newprofiler.preserve_srcdir,
newprofiler.version, newprofiler.setup,
*args, **dargs)
return newprofiler
def _initialize_dir_properties(self):
"""
Initializes all the secondary self.*dir properties. Requires autodir,
clientdir and serverdir to already be initialized.
"""
# create some stubs for use as shortcuts
def readonly_dir(*args):
return self._job_directory(os.path.join(*args))
def readwrite_dir(*args):
return self._job_directory(os.path.join(*args), True)
# various client-specific directories
self._bindir = readonly_dir(self.clientdir)
self._configdir = readonly_dir(self.clientdir, 'config')
self._profdir = readonly_dir(self.clientdir, 'profilers')
self._toolsdir = readonly_dir(self.clientdir, 'tools')
try:
autodir = os.path.abspath(os.environ['AUTODIR'])
except KeyError:
autodir = settings.get_value('COMMON', 'autotest_top_path')
tmpdir = os.path.join(autodir, 'tmp')
test_out_dir = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
if self.serverdir:
root = self.serverdir
else:
root = self.clientdir
test_dir = settings.get_value('COMMON', 'test_dir',
default=os.path.join(root, 'tests'))
self._tmpdir = readwrite_dir(test_out_dir, 'tmp')
# special case packages for backwards compatibility
pkg_dir = test_out_dir
if pkg_dir == self.serverdir:
pkg_dir = self.clientdir
self._pkgdir = readwrite_dir(pkg_dir, 'packages')
# Now tests are read-only modules
self._testdir = readonly_dir(test_dir)
self._site_testdir = readwrite_dir(test_out_dir, 'site_tests')
# various server-specific directories
if self.serverdir:
# This is a NOT a flexible solution, but I feel that this does
# not deserve a configuration option by itself. So, let's just
# try to find conmux in the system and let the user know about it
if os.path.isdir('/usr/share/conmux'):
self._conmuxdir = readonly_dir('/usr/share/conmux')
else:
self._conmuxdir = readonly_dir(self.autodir, 'conmux')
logging.info('conmux directory set to: %s' % self._conmuxdir)
else:
self._conmuxdir = None
def __init__(self, job, bindir, outputdir):
self.job = job
self.pkgmgr = job.pkgmgr
self.autodir = job.autodir
self.outputdir = outputdir
self.tagged_testname = os.path.basename(self.outputdir)
self.resultsdir = os.path.join(self.outputdir, 'results')
os.mkdir(self.resultsdir)
self.profdir = os.path.join(self.outputdir, 'profiling')
os.mkdir(self.profdir)
self.debugdir = os.path.join(self.outputdir, 'debug')
os.mkdir(self.debugdir)
if getpass.getuser() == 'root':
self.configure_crash_handler()
else:
self.crash_handling_enabled = False
self.bindir = bindir
try:
autodir = os.path.abspath(os.environ['AUTODIR'])
except KeyError:
autodir = settings.get_value('COMMON', 'autotest_top_path')
tmpdir = os.path.join(autodir, 'tmp')
output_config = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
self.srcdir = os.path.join(output_config, os.path.basename(self.bindir),
'src')
self.tmpdir = tempfile.mkdtemp("_" + self.tagged_testname,
dir=job.tmpdir)
self._keyvals = []
self._new_keyval = False
self.failed_constraints = []
self.iteration = 0
self.before_iteration_hooks = []
self.after_iteration_hooks = []
def result_notify(self, job, email_from, email_to):
"""
Notify about the result of a job. Will always print, if email data
is provided, will send email for it as well.
job: job object to notify about
email_from: send notification email upon completion from here
email_from: send notification email upon completion to here
"""
if job.result == True:
subject = 'Testing PASSED: '
else:
subject = 'Testing FAILED: '
subject += '%s : %s\n' % (job.name, job.id)
text = []
for platform in job.results_platform_map:
for status in job.results_platform_map[platform]:
if status == 'Total':
continue
for host in job.results_platform_map[platform][status]:
text.append('%20s %10s %10s' % (platform, status, host))
if status == 'Failed':
for test_status in job.test_status[host].fail:
text.append('(%s, %s) : %s' % \
(host, test_status.test_name,
test_status.reason))
text.append('')
base_url = 'http://' + self.server
params = ('columns=test',
'rows=machine_group',
"condition=tag~'%s-%%25'" % job.id,
'title=Report')
query_string = '&'.join(params)
url = '%s/tko/compose_query.cgi?%s' % (base_url, query_string)
text.append(url + '\n')
url = '%s/afe/#tab_id=view_job&object_id=%s' % (base_url, job.id)
text.append(url + '\n')
smtp_info = {}
smtp_info['server'] = settings.get_value('SERVER', 'smtp_server',
default='localhost')
smtp_info['port'] = settings.get_value('SERVER', 'smtp_port',
default='')
smtp_info['user'] = settings.get_value('SERVER', 'smtp_user',
default='')
smtp_info['password'] = settings.get_value('SERVER', 'smtp_password',
default='')
body = '\n'.join(text)
print '---------------------------------------------------'
print 'Subject: ', subject
print body
print '---------------------------------------------------'
if email_from and email_to:
print 'Sending email ...'
utils.send_email(email_from, email_to, subject, body, smtp_info)
print
def _initialize_dir_properties(self):
"""
Initializes all the secondary self.*dir properties. Requires autodir,
clientdir and serverdir to already be initialized.
"""
# create some stubs for use as shortcuts
def readonly_dir(*args):
return self._job_directory(os.path.join(*args))
def readwrite_dir(*args):
return self._job_directory(os.path.join(*args), True)
# various client-specific directories
self._bindir = readonly_dir(self.clientdir)
self._configdir = readonly_dir(self.clientdir, 'config')
self._profdir = readonly_dir(self.clientdir, 'profilers')
self._toolsdir = readonly_dir(self.clientdir, 'tools')
try:
autodir = os.path.abspath(os.environ['AUTODIR'])
except KeyError:
autodir = settings.get_value('COMMON', 'autotest_top_path')
tmpdir = os.path.join(autodir, 'tmp')
test_out_dir = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
if self.serverdir:
root = self.serverdir
else:
root = self.clientdir
test_dir = settings.get_value('COMMON', 'test_dir',
default=os.path.join(root, 'tests'))
self._tmpdir = readwrite_dir(test_out_dir, 'tmp')
# special case packages for backwards compatibility
pkg_dir = test_out_dir
if pkg_dir == self.serverdir:
pkg_dir = self.clientdir
self._pkgdir = readwrite_dir(pkg_dir, 'packages')
# Now tests are read-only modules
self._testdir = readonly_dir(test_dir)
self._site_testdir = readwrite_dir(test_out_dir, 'site_tests')
# various server-specific directories
if self.serverdir:
self._conmuxdir = readonly_dir(self.autodir, 'conmux')
else:
self._conmuxdir = None
def check_diskspace(repo, min_free=None):
'''
Check if the remote directory over at the pkg repo has available diskspace
If the amount of free space is not supplied, it is taken from the global
configuration file, section [PACKAGES], key 'mininum_free_space'. The unit
used are in SI, that is, 1 GB = 10**9 bytes.
@type repo: string
@param repo: a remote package repo URL
@type min_free: int
@param: min_free mininum amount of free space, in GB (10**9 bytes)
:raise error.RepoUnknownError: general repository error condition
:raise error.RepoDiskFullError: repository does not have at least the
requested amount of free disk space.
'''
if min_free is None:
min_free = settings.get_value('PACKAGES', 'minimum_free_space',
type=int, default=1)
try:
df = repo_run_command(repo,
'df -PB %d . | tail -1' % 10 ** 9).stdout.split()
free_space_gb = int(df[3])
except Exception, e:
raise error.RepoUnknownError('Unknown Repo Error: %s' % e)
def _disable_host_installation(self, host):
server_info = remote.get_install_server_info()
if remote.install_server_is_configured():
timeout = settings.get_value('INSTALL_SERVER',
'default_install_timeout',
type=int,
default=3600)
end_time = time.time() + (timeout / 10)
step = int(timeout / 100)
ServerInterface = remote.RemoteHost.INSTALL_SERVER_MAPPING[server_info['type']]
server_interface = None
while time.time() < end_time:
try:
server_interface = ServerInterface(**server_info)
break
except socket.error:
logging.error('Install server unavailable. Trying '
'again in %s s...', step)
time.sleep(step)
if server_interface is None:
raise InstallServerUnavailable("%s install server at (%s) "
"unavailable. Tried to "
"communicate for %s s" %
(server_info['type'],
server_info['xmlrpc_url'],
timeout / 10))
server_interface._disable_host_installation(host)
def _load_state(self):
autodir = os.path.abspath(os.environ['AUTODIR'])
tmpdir = os.path.join(autodir, 'tmp')
state_config = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
if not os.path.isdir(state_config):
os.makedirs(state_config)
init_state_file = os.path.join(state_config,
("%s.init.state" %
os.path.basename(self.control)))
self._state_file = os.path.join(state_config,
("%s.state" %
os.path.basename(self.control)))
if os.path.exists(init_state_file):
shutil.move(init_state_file, self._state_file)
self._state.set_backing_file(self._state_file)
# initialize the state engine, if necessary
has_steps = self._state.has('client', 'steps')
if not self._is_continuation and has_steps:
raise RuntimeError('Loaded state can only contain client.steps if '
'this is a continuation')
if not has_steps:
logging.debug('Initializing the state engine')
self._state.set('client', 'steps', [])
def get_wait_up_processes(self):
""" Gets the list of local processes to wait for in wait_up. """
proc_list = settings.get_value("HOSTS", "wait_up_processes",
default="").strip()
processes = set(p.strip() for p in proc_list.split(","))
processes.discard("")
return processes
def reboot(self, tag=LAST_BOOT_TAG):
if tag == LAST_BOOT_TAG:
tag = self.last_boot_tag
else:
self.last_boot_tag = tag
self.reboot_setup()
self.harness.run_reboot()
default = self.config_get("boot.set_default")
if default:
self.bootloader.set_default(tag)
else:
self.bootloader.boot_once(tag)
# HACK: using this as a module sometimes hangs shutdown, so if it's
# installed unload it first
utils.system("modprobe -r netconsole", ignore_status=True)
# sync first, so that a sync during shutdown doesn't time out
utils.system("sync; sync", ignore_status=True)
sleep_before_reboot = settings.get_value("CLIENT", "sleep_before_reboot", default="5")
sleep_cmd = "(sleep %s; reboot) </dev/null >/dev/null 2>&1 &" % sleep_before_reboot
utils.system(sleep_cmd)
self.quit()
def __init__(self, path, user, server, print_log, debug, reply_debug):
"""
Create a cached instance of a connection to the frontend
user: username to connect as
server: frontend server to connect to
print_log: pring a logging message to stdout on every operation
debug: print out all RPC traffic
"""
if not user:
user = getpass.getuser()
if not server:
if 'AUTOTEST_WEB' in os.environ:
server = os.environ['AUTOTEST_WEB']
else:
server = settings.get_value('SERVER', 'hostname',
default=DEFAULT_SERVER)
self.server = server
self.user = user
self.print_log = print_log
self.debug = debug
self.reply_debug = reply_debug
http_server = 'http://' + server
headers = rpc_client_lib.authorization_headers(user, http_server)
rpc_server = http_server + path
if debug:
print 'SERVER: %s' % rpc_server
print 'HEADERS: %s' % headers
self.proxy = rpc_client_lib.get_proxy(rpc_server, headers=headers)
def collect_stalled_info(self):
INFO_TO_COLLECT = ["uptime", "ps auxwww", "iostat -k -x 2 4"]
db_cmd = "/usr/bin/mysqladmin --verbose processlist -u%s -p%s"
try:
user = settings.get_value("BACKUP", "user")
password = settings.get_value("BACKUP", "password")
db_cmd %= (user, password)
INFO_TO_COLLECT.append(db_cmd)
except SettingsError:
pass
stall_log_path = self.log_path + ".stall_info"
log = open(stall_log_path, "w")
for cmd in INFO_TO_COLLECT:
log.write(run_banner_output(cmd))
log.close()
def _get_option(self, name, provided_value):
if provided_value is not None:
return provided_value
if self.settings_section:
settings_name = _GLOBAL_CONFIG_NAMES.get(name, name)
return settings.get_value(self.settings_section, settings_name)
return getattr(self, name, None)
def get_server_status():
"""
Get autotest server system information.
:return: Dict with keys:
* 'disk_space_percentage' Autotest log directory disk usage
* 'scheduler_running' Whether the autotest scheduler is running
* 'sheduler_watcher_running' Whether the scheduler watcher is
running
* 'concerns' Global evaluation of whether there are problems to
be addressed
"""
server_status = {}
concerns = False
disk_treshold = int(settings.get_value('SERVER', 'logs_disk_usage_treshold',
default="80"))
used_space_logs = _get_logs_used_space()
if used_space_logs > disk_treshold:
concerns = True
server_status['used_space_logs'] = used_space_logs
scheduler_running = _process_running('autotest-scheduler')
if not scheduler_running:
concerns = True
server_status['scheduler_running'] = scheduler_running
watcher_running = _process_running('autotest-scheduler-watcher')
if not watcher_running:
concerns = True
server_status['scheduler_watcher_running'] = watcher_running
install_server_running = get_install_server_profiles() is not None
if not install_server_running:
concerns = True
server_status['install_server_running'] = install_server_running
server_status['concerns'] = concerns
return server_status
def _init_packages(self):
"""
Perform the packages support initialization.
"""
tmpdir = settings.get_value('COMMON', 'test_output_dir',
default=self.autodir)
self.pkgmgr = packages.PackageManager(
tmpdir, run_function_dargs={'timeout':3600})
def __init__(self, module="scheduler"):
"""
Initialize an email notification manager.
:param subsystem: String describing the module this manager is
handling. Example: 'scheduler'.
"""
self.module = module
self.email_queue = []
self.html_email = settings.get_value("NOTIFICATION",
"html_notify_email",
type=bool,
default=False)
self.from_email = settings.get_value("NOTIFICATION",
"notify_email_from",
default=DEFAULT_FROM_EMAIL)
self.grid_admin_email = settings.get_value("NOTIFICATION",
"grid_admin_email",
default='')
server = settings.get_value("EMAIL", "smtp_server", default='localhost')
port = settings.get_value("EMAIL", "smtp_port", default=None)
user = settings.get_value("EMAIL", "smtp_user", default=None)
password = settings.get_value("EMAIL", "smtp_password", default=None)
self.smtp_info = {'server': server,
'port': port,
'user': user,
'password': password}
def _load_config(self, host, database, user, password):
# grab the host, database
if host:
self.host = host
else:
self.host = settings.get_value("AUTOTEST_WEB", "host")
if database:
self.database = database
else:
self.database = settings.get_value("AUTOTEST_WEB", "database")
# grab the user and password
if user:
self.user = user
else:
self.user = settings.get_value("AUTOTEST_WEB", "user")
if password is not None:
self.password = password
else:
self.password = settings.get_value("AUTOTEST_WEB", "password")
# grab the timeout configuration
self.query_timeout = settings.get_value("AUTOTEST_WEB", "query_timeout",
type=int, default=3600)
self.min_delay = settings.get_value("AUTOTEST_WEB", "min_retry_delay", type=int,
default=20)
self.max_delay = settings.get_value("AUTOTEST_WEB", "max_retry_delay", type=int,
default=60)
def main(args):
if len(args) <= 1:
usage()
entries = args[1:]
for entry in entries:
try:
section, var = entry.split('.')
except ValueError:
print "Invalid SECTION.varable supplied: " + entry
usage()
try:
print settings.get_value(section, var)
except SettingsError:
print "Error reading %s.%s" % (section, var)
def configure_logging():
MAX_LOG_SIZE = settings.get_value('SERVER', 'rpc_max_log_size_mb', type=int)
NUMBER_OF_OLD_LOGS = settings.get_value('SERVER', 'rpc_num_old_logs',
type=int)
log_path = settings.get_value('SERVER', 'rpc_log_path')
formatter = logging.Formatter(
fmt='[%(asctime)s %(levelname)-5.5s] %(message)s',
datefmt='%m/%d %H:%M:%S')
handler = logging.handlers.RotatingFileHandler(log_path,
maxBytes=MAX_LOG_SIZE * MEGABYTE,
backupCount=NUMBER_OF_OLD_LOGS)
handler.setFormatter(formatter)
global rpc_logger
rpc_logger = logging.getLogger('rpc_logger')
rpc_logger.addHandler(handler)
rpc_logger.propagate = False
rpc_logger.setLevel(logging.DEBUG)
def _init_drop_caches(self, drop_caches):
"""
Perform the drop caches initialization.
"""
self.drop_caches_between_iterations = (settings.get_value('CLIENT',
'drop_caches_between_iterations',
type=bool, default=True))
self.drop_caches = drop_caches
if self.drop_caches:
utils.drop_caches()
def _get_max_pidfile_refreshes(self):
"""
Normally refresh() is called on every monitor_db.Dispatcher.tick().
@returns: The number of refresh() calls before we forget a pidfile.
"""
pidfile_timeout = settings.get_value(
scheduler_config.CONFIG_SECTION, 'max_pidfile_refreshes',
type=int, default=2000)
return pidfile_timeout
def _init_drop_caches(self, drop_caches):
"""
Perform the drop caches initialization.
"""
self.drop_caches_between_iterations = settings.get_value(
"CLIENT", "drop_caches_between_iterations", type=bool, default=True
)
self.drop_caches = drop_caches
if self.drop_caches:
utils_memory.drop_caches()
def _get_logs_used_space():
"""
Return disk usage (percentage) for the results directory.
"""
logs_dir = settings.get_value("COMMON", "test_output_dir", default=None)
autodir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
if logs_dir is None:
logs_dir = os.path.join(autodir, "results")
usage = psutil.disk_usage(logs_dir)
return int(usage.percent)
def archive_results(self, path):
results_host = settings.get_value('SCHEDULER', 'results_host',
default=None)
if not results_host or results_host == 'localhost':
return
if not path.endswith('/'):
path += '/'
logging.info('Archiving %s to %s', path, results_host)
utility = drone_utility.DroneUtility()
utility.sync_send_file_to(results_host, path, path, can_fail=True)
def get_mappings_2x():
KERNEL_BASE_URL = settings.get_value('CLIENT', 'kernel_mirror', default='')
GITWEB_BASE_URL = settings.get_value('CLIENT', 'kernel_gitweb', default='')
STABLE_GITWEB_BASE_URL = settings.get_value('CLIENT', 'stable_kernel_gitweb', default='')
MAPPINGS_2X = [
[r'^\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', GITWEB_BASE_URL.split())
],
[r'^\d+\.\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', GITWEB_BASE_URL.split())
],
[r'^\d+\.\d+\.\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', STABLE_GITWEB_BASE_URL.split())
],
[r'-rc\d+$', '%(minor-prev)s', True,
map(lambda x: x + 'v%(major)s/testing/v%(minor)s/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + 'v%(major)s/testing/linux-%(full)s.tar.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz', GITWEB_BASE_URL.split())
],
[r'-(git|bk)\d+$', '%(base)s', False,
map(lambda x: x + 'v%(major)s/snapshots/old/patch-%(full)s.bz2', KERNEL_BASE_URL.split()) +
map(lambda x: x + 'v%(major)s/snapshots/patch-%(full)s.bz2', KERNEL_BASE_URL.split())
],
[r'-mm\d+$', '%(base)s', False,
map(lambda x: x + 'people/akpm/patches/' + '%(major)s/%(base)s/%(full)s/%(full)s.bz2', KERNEL_BASE_URL.split())
],
[r'-mjb\d+$', '%(base)s', False,
map(lambda x: x + 'people/mbligh/%(base)s/patch-%(full)s.bz2', KERNEL_BASE_URL.split())
],
[r'[a-f0-9]{7,40}$', '', True,
map(lambda x: x + ';a=snapshot;h=%(full)s;sf=tgz', GITWEB_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=%(full)s;sf=tgz', STABLE_GITWEB_BASE_URL.split())
]
]
return MAPPINGS_2X
def refresh_drone_configs(self):
"""
Reread global config options for all drones.
"""
section = scheduler_config.CONFIG_SECTION
settings.parse_config_file()
for hostname, drone in self._drones.iteritems():
disabled = settings.get_value(section, '%s_disabled' % hostname,
default='')
drone.enabled = not bool(disabled)
drone.max_processes = settings.get_value(
section, '%s_max_processes' % hostname, type=int,
default=scheduler_config.config.max_processes_per_drone)
allowed_users = settings.get_value(section, '%s_users' % hostname,
default=None)
if allowed_users is not None:
allowed_users = set(allowed_users.split())
drone.allowed_users = allowed_users
self._reorder_drone_queue() # max_processes may have changed
def get_autotest_server(web_server=None):
if not web_server:
if "AUTOTEST_WEB" in os.environ:
web_server = os.environ["AUTOTEST_WEB"]
else:
web_server = "http://" + settings.get_value("SERVER", "hostname", default=DEFAULT_SERVER)
# if the name doesn't start with http://,
# nonexistant hosts get an obscure error
if not web_server.startswith("http://"):
web_server = "http://" + web_server
return web_server
def _get_logs_used_space():
"""
(Internal) Return disk usage (percentage) for the results directory.
:return: Usage in percents (integer value).
"""
logs_dir = settings.get_value('COMMON', 'test_output_dir', default=None)
autodir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..'))
if logs_dir is None:
logs_dir = os.path.join(autodir, 'results')
usage = psutil.disk_usage(logs_dir)
return int(usage.percent)
def _send_tarball(self, pkg_name, remote_dest):
name, pkg_type = self.job.pkgmgr.parse_tarball_name(pkg_name)
src_dirs = []
if pkg_type == 'test':
test_dirs = ['site_tests', 'tests']
# if test_dir is defined in global config
# package the tests from there (if exists)
settings_test_dirs = settings.get_value('COMMON', 'test_dir',
default="")
if settings_test_dirs:
test_dirs = settings_test_dirs.strip().split(',') + test_dirs
for test_dir in test_dirs:
src_dir = os.path.join(self.job.clientdir, test_dir, name)
if os.path.exists(src_dir):
src_dirs += [src_dir]
if autoserv_prebuild:
prebuild.setup(self.job.clientdir, src_dir)
break
elif pkg_type == 'profiler':
src_dirs += [os.path.join(self.job.clientdir, 'profilers', name)]
if autoserv_prebuild:
prebuild.setup(self.job.clientdir, src_dir)
elif pkg_type == 'dep':
src_dirs += [os.path.join(self.job.clientdir, 'deps', name)]
elif pkg_type == 'client':
return # you must already have a client to hit this anyway
else:
return # no other types are supported
# iterate over src_dirs until we find one that exists, then tar it
for src_dir in src_dirs:
if os.path.exists(src_dir):
try:
logging.info('Bundling %s into %s', src_dir, pkg_name)
temp_dir = autotemp.tempdir(unique_id='autoserv-packager',
dir=self.job.tmpdir)
exclude_paths = None
exclude_file_path = os.path.join(src_dir, ".pack_exclude")
if os.path.exists(exclude_file_path):
exclude_file = open(exclude_file_path)
exclude_paths = exclude_file.read().splitlines()
exclude_file.close()
tarball_path = self.job.pkgmgr.tar_package(
pkg_name, src_dir, temp_dir.name,
" .", exclude_paths)
self.host.send_file(tarball_path, remote_dest)
finally:
temp_dir.clean()
return
class BaseAutotest(installable_object.InstallableObject):
"""
This class represents the Autotest program.
Autotest is used to run tests automatically and collect the results.
It also supports profilers.
Implementation details:
This is a leaf class in an abstract class hierarchy, it must
implement the unimplemented methods in parent classes.
"""
def __init__(self, host=None):
self.host = host
self.got = False
self.installed = False
self.serverdir = utils.get_server_dir()
self.os_vendor = client_utils.get_os_vendor()
self.server_system_wide_install = _server_system_wide_install()
super(BaseAutotest, self).__init__()
install_in_tmpdir = False
@classmethod
def set_install_in_tmpdir(cls, flag):
"""
Sets a flag that controls whether or not Autotest should by
default be installed in a "standard" directory (e.g. /home/autotest) or
a temporary directory.
"""
cls.install_in_tmpdir = flag
@classmethod
def get_client_autodir_paths(cls, host):
return settings.get_value('AUTOSERV', 'client_autodir_paths', type=list)
@classmethod
def get_installed_autodir(cls, host):
"""
Find where the Autotest client is installed on the host.
:return: an absolute path to an installed Autotest client root.
:raise AutodirNotFoundError if no Autotest installation can be found.
"""
autodir = host.get_autodir()
if autodir:
logging.debug('Using existing host autodir: %s', autodir)
return autodir
if not _server_system_wide_install():
for path in Autotest.get_client_autodir_paths(host):
try:
autotest_binary = os.path.join(path, CLIENT_BINARY)
host.run('test -x %s' % utils.sh_escape(autotest_binary))
host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
return path
except error.AutoservRunError:
logging.debug('%s does not exist on %s', autotest_binary,
host.hostname)
else:
for path in Autotest.get_client_autodir_paths(host):
host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
host.autodir = path
return path
raise AutodirNotFoundError
@classmethod
def get_install_dir(cls, host):
"""
Determines the location where autotest should be installed on
host. If self.install_in_tmpdir is set, it will return a unique
temporary directory that autotest can be installed in. Otherwise, looks
for an existing installation to use; if none is found, looks for a
usable directory in the global config client_autodir_paths.
"""
try:
install_dir = cls.get_installed_autodir(host)
except AutodirNotFoundError:
install_dir = cls._find_installable_dir(host)
if cls.install_in_tmpdir:
return host.get_tmp_dir(parent=install_dir)
return install_dir
@classmethod
def _find_installable_dir(cls, host):
client_autodir_paths = cls.get_client_autodir_paths(host)
for path in client_autodir_paths:
try:
host.run('mkdir -p %s' % utils.sh_escape(path))
host.run('test -w %s' % utils.sh_escape(path))
return path
except error.AutoservRunError:
logging.debug('Failed to create %s', path)
raise error.AutoservInstallError(
'Unable to find a place to install Autotest; tried %s' %
', '.join(client_autodir_paths))
def _create_test_output_dir(self, host, autodir):
tmpdir = os.path.join(autodir, 'tmp')
state_autodir = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
host.run('mkdir -p %s' % utils.sh_escape(state_autodir))
def get_fetch_location(self):
repos = settings.get_value("PACKAGES", 'fetch_location', type=list,
default=[])
repos.reverse()
return repos
def install(self, host=None, autodir=None):
self._install(host=host, autodir=autodir)
def install_full_client(self, host=None, autodir=None):
self._install(host=host, autodir=autodir, use_autoserv=False,
use_packaging=False)
def install_no_autoserv(self, host=None, autodir=None):
self._install(host=host, autodir=autodir, use_autoserv=False)
def _install_using_packaging(self, host, autodir):
repos = self.get_fetch_location()
if not repos:
raise error.PackageInstallError("No repos to install an "
"autotest client from")
pkgmgr = packages.PackageManager(autodir, hostname=host.hostname,
repo_urls=repos,
do_locking=False,
run_function=host.run,
run_function_dargs=dict(timeout=600))
# The packages dir is used to store all the packages that
# are fetched on that client. (for the tests,deps etc.
# too apart from the client)
pkg_dir = os.path.join(autodir, 'packages')
# clean up the autodir except for the packages directory
host.run('cd %s && ls | grep -v "^packages$"'
' | xargs rm -rf && rm -rf .[^.]*' % autodir)
pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir,
preserve_install_dir=True)
self._create_test_output_dir(host, autodir)
logging.info("Installation of autotest completed")
self.installed = True
def _install_using_send_file(self, host, autodir):
dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"])
light_files = [os.path.join(self.source_material, f)
for f in os.listdir(self.source_material)
if f not in dirs_to_exclude]
# there should be one and only one grubby tarball
grubby_glob = os.path.join(self.source_material,
"deps/grubby/grubby-*.tar.bz2")
grubby_tarball_paths = glob.glob(grubby_glob)
if grubby_tarball_paths:
grubby_tarball_path = grubby_tarball_paths[0]
if os.path.exists(grubby_tarball_path):
light_files.append(grubby_tarball_path)
host.send_file(light_files, autodir, delete_dest=True)
profilers_autodir = os.path.join(autodir, 'profilers')
profilers_init = os.path.join(self.source_material, 'profilers',
'__init__.py')
host.run("mkdir -p %s" % profilers_autodir)
host.send_file(profilers_init, profilers_autodir, delete_dest=True)
dirs_to_exclude.discard("profilers")
# create empty dirs for all the stuff we excluded
commands = []
for path in dirs_to_exclude:
abs_path = os.path.join(autodir, path)
abs_path = utils.sh_escape(abs_path)
commands.append("mkdir -p '%s'" % abs_path)
commands.append("touch '%s'/__init__.py" % abs_path)
host.run(';'.join(commands))
def _install(self, host=None, autodir=None, use_autoserv=True,
use_packaging=True):
"""
Install autotest.
:param host A Host instance on which autotest will be installed
:param autodir Location on the remote host to install to
:param use_autoserv Enable install modes that depend on the client
running with the autoserv harness
:param use_packaging Enable install modes that use the packaging system
@exception AutoservError If it wasn't possible to install the client
after trying all available methods
"""
if not host:
host = self.host
if not self.got:
self.get()
host.wait_up(timeout=30)
host.setup()
logging.info("Installing autotest on %s", host.hostname)
if self.server_system_wide_install:
msg_install = ("Autotest seems to be installed in the "
"client on a system wide location, proceeding...")
logging.info("Verifying client package install")
if _client_system_wide_install(host):
logging.info(msg_install)
self.installed = True
return
install_cmd = INSTALL_CLIENT_CMD_MAPPING.get(self.os_vendor, None)
if install_cmd is not None:
logging.info(msg_install)
host.run(install_cmd)
if _client_system_wide_install(host):
logging.info("Autotest seems to be installed in the "
"client on a system wide location, proceeding...")
self.installed = True
return
raise error.AutoservError("The autotest client package "
"does not seem to be installed "
"on %s" % host.hostname)
# set up the autotest directory on the remote machine
if not autodir:
autodir = self.get_install_dir(host)
logging.info('Using installation dir %s', autodir)
host.set_autodir(autodir)
host.run('mkdir -p %s' % utils.sh_escape(autodir))
# make sure there are no files in $AUTODIR/results
results_path = os.path.join(autodir, 'results')
host.run('rm -rf %s/*' % utils.sh_escape(results_path),
ignore_status=True)
# Fetch the autotest client from the nearest repository
if use_packaging:
try:
self._install_using_packaging(host, autodir)
self._create_test_output_dir(host, autodir)
logging.info("Installation of autotest completed")
self.installed = True
return
except (error.PackageInstallError, error.AutoservRunError,
SettingsError), e:
logging.info("Could not install autotest using the packaging "
"system: %s. Trying other methods", e)
# try to install from file or directory
if self.source_material:
supports_autoserv_packaging = settings.get_value("PACKAGES",
"serve_packages_from_autoserv",
type=bool)
# Copy autotest recursively
if supports_autoserv_packaging and use_autoserv:
self._install_using_send_file(host, autodir)
else:
host.send_file(self.source_material, autodir, delete_dest=True)
self._create_test_output_dir(host, autodir)
logging.info("Installation of autotest completed")
self.installed = True
return
raise error.AutoservError('Could not install autotest on '
'target machine: %s' % host.name)
def __init__(self, job, harness_args):
"""
job
The job object for this job
"""
self.autodir = os.path.abspath(os.environ['AUTODIR'])
self.setup(job)
tmpdir = os.path.join(self.autodir, 'tmp')
tests_dir = settings.get_value('COMMON',
'test_output_dir',
default=tmpdir)
src = job.control_get()
dest = os.path.join(tests_dir, 'control')
if os.path.abspath(src) != os.path.abspath(dest):
shutil.copyfile(src, dest)
job.control_set(dest)
def yield_default_initlevel():
"""
If we really can't figure out something better, default to '2',
which is the case with some debian systems
"""
init_default = '2'
logging.error('Could not determine initlevel, assuming %s' %
init_default)
return init_default
rc = os.path.join(self.autodir, 'tools/autotest')
# see if system supports event.d versus systemd versus inittab
supports_eventd = os.path.exists('/etc/event.d')
supports_systemd = os.path.exists('/etc/systemd')
supports_inittab = os.path.exists('/etc/inittab')
# This is the best heuristics I can think of for identifying
# an embedded system running busybox
busybox_system = (os.readlink('/bin/sh') == 'busybox')
# Small busybox systems usually use /etc/rc.d/ straight
if busybox_system:
initdefault = ''
elif supports_eventd or supports_systemd:
try:
# NB: assuming current runlevel is default
cmd_result = utils.run('/sbin/runlevel', verbose=False)
initdefault = cmd_result.stdout.split()[1]
except (error.CmdError, IndexError):
initdefault = yield_default_initlevel()
elif supports_inittab:
try:
cmd_result = utils.run('grep :initdefault: /etc/inittab',
verbose=False)
initdefault = cmd_result.stdout.split(':')[1]
except (error.CmdError, IndexError):
initdefault = yield_default_initlevel()
else:
initdefault = yield_default_initlevel()
vendor = distro.detect().name
service = '/etc/init.d/autotest'
if vendor == 'SUSE':
service_link = '/etc/init.d/rc%s.d/S99autotest' % initdefault
else:
service_link = '/etc/rc%s.d/S99autotest' % initdefault
try:
if os.path.islink(service):
os.remove(service)
if os.path.islink(service_link):
os.remove(service_link)
os.symlink(rc, service)
os.symlink(rc, service_link)
except (OSError, IOError):
logging.info(
"Could not symlink init scripts (lack of permissions)")
import glob
import logging
import os
import shutil
import socket
import tempfile
import time
import traceback
from autotest.client.shared import autotemp, error
from autotest.client.shared.settings import settings
from autotest.server import utils, autotest_remote
from autotest.server.hosts import remote
enable_master_ssh = settings.get_value('AUTOSERV',
'enable_master_ssh',
type=bool,
default=False)
def _make_ssh_cmd_default(user="******",
port=22,
opts='',
hosts_file='/dev/null',
connect_timeout=30,
alive_interval=300):
base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=%s -o BatchMode=yes "
"-o ConnectTimeout=%d -o ServerAliveInterval=%d "
"-l %s -p %d")
assert isinstance(connect_timeout, (int, long))
assert connect_timeout > 0 # can't disable the timeout
class AbstractSSHHost(SiteHost):
"""
This class represents a generic implementation of most of the
framework necessary for controlling a host via ssh. It implements
almost all of the abstract Host methods, except for the core
Host.run method.
"""
def _initialize(self,
hostname,
user="******",
port=22,
password="",
*args,
**dargs):
super(AbstractSSHHost, self)._initialize(hostname=hostname,
*args,
**dargs)
self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0]
self.user = user
self.port = port
self.password = password
self._use_rsync = None
self.known_hosts_file = tempfile.mkstemp()[1]
"""
Master SSH connection background job, socket temp directory and socket
control path option. If master-SSH is enabled, these fields will be
initialized by start_master_ssh when a new SSH connection is initiated.
"""
self.master_ssh_job = None
self.master_ssh_tempdir = None
self.master_ssh_option = ''
def use_rsync(self):
if self._use_rsync is not None:
return self._use_rsync
# Check if rsync is available on the remote host. If it's not,
# don't try to use it for any future file transfers.
self._use_rsync = self._check_rsync()
if not self._use_rsync:
logging.warn("rsync not available on remote host %s -- disabled",
self.hostname)
return self._use_rsync
def _check_rsync(self):
"""
Check if rsync is available on the remote host.
"""
try:
self.run("rsync --version", stdout_tee=None, stderr_tee=None)
except error.AutoservRunError:
return False
return True
def _encode_remote_paths(self, paths, escape=True):
"""
Given a list of file paths, encodes it as a single remote path, in
the style used by rsync and scp.
"""
if escape:
paths = [utils.scp_remote_escape(path) for path in paths]
return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths))
def _make_rsync_cmd(self, sources, dest, delete_dest, preserve_symlinks):
"""
Given a list of source paths and a destination path, produces the
appropriate rsync command for copying them. Remote paths must be
pre-encoded.
"""
ssh_cmd = make_ssh_command(user=self.user,
port=self.port,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file)
if delete_dest:
delete_flag = "--delete"
else:
delete_flag = ""
if preserve_symlinks:
symlink_flag = ""
else:
symlink_flag = "-L"
command = "rsync %s %s --timeout=1800 --rsh='%s' -az %s %s"
return command % (symlink_flag, delete_flag, ssh_cmd,
" ".join(sources), dest)
def _make_ssh_cmd(self, cmd):
"""
Create a base ssh command string for the host which can be used
to run commands directly on the machine
"""
base_cmd = make_ssh_command(user=self.user,
port=self.port,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file)
return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd))
def _make_scp_cmd(self, sources, dest):
"""
Given a list of source paths and a destination path, produces the
appropriate scp command for encoding it. Remote paths must be
pre-encoded.
"""
command = ("scp -rq %s -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=%s -P %d %s '%s'")
return command % (self.master_ssh_option, self.known_hosts_file,
self.port, " ".join(sources), dest)
def _make_rsync_compatible_globs(self, path, is_local):
"""
Given an rsync-style path, returns a list of globbed paths
that will hopefully provide equivalent behaviour for scp. Does not
support the full range of rsync pattern matching behaviour, only that
exposed in the get/send_file interface (trailing slashes).
The is_local param is flag indicating if the paths should be
interpreted as local or remote paths.
"""
# non-trailing slash paths should just work
if len(path) == 0 or path[-1] != "/":
return [path]
# make a function to test if a pattern matches any files
if is_local:
def glob_matches_files(path, pattern):
return len(glob.glob(path + pattern)) > 0
else:
def glob_matches_files(path, pattern):
result = self.run("ls \"%s\"%s" %
(utils.sh_escape(path), pattern),
stdout_tee=None,
ignore_status=True)
return result.exit_status == 0
# take a set of globs that cover all files, and see which are needed
patterns = ["*", ".[!.]*"]
patterns = [p for p in patterns if glob_matches_files(path, p)]
# convert them into a set of paths suitable for the commandline
if is_local:
return [
"\"%s\"%s" % (utils.sh_escape(path), pattern)
for pattern in patterns
]
else:
return [
utils.scp_remote_escape(path) + pattern for pattern in patterns
]
def _make_rsync_compatible_source(self, source, is_local):
"""
Applies the same logic as _make_rsync_compatible_globs, but
applies it to an entire list of sources, producing a new list of
sources, properly quoted.
"""
return sum((self._make_rsync_compatible_globs(path, is_local)
for path in source), [])
def _set_umask_perms(self, dest):
"""
Given a destination file/dir (recursively) set the permissions on
all the files and directories to the max allowed by running umask.
"""
# now this looks strange but I haven't found a way in Python to _just_
# get the umask, apparently the only option is to try to set it
umask = os.umask(0)
os.umask(umask)
max_privs = 0777 & ~umask
def set_file_privs(filename):
file_stat = os.stat(filename)
file_privs = max_privs
# if the original file permissions do not have at least one
# executable bit then do not set it anywhere
if not file_stat.st_mode & 0111:
file_privs &= ~0111
os.chmod(filename, file_privs)
# try a bottom-up walk so changes on directory permissions won't cut
# our access to the files/directories inside it
for root, dirs, files in os.walk(dest, topdown=False):
# when setting the privileges we emulate the chmod "X" behaviour
# that sets to execute only if it is a directory or any of the
# owner/group/other already has execute right
for dirname in dirs:
os.chmod(os.path.join(root, dirname), max_privs)
for filename in files:
set_file_privs(os.path.join(root, filename))
# now set privs for the dest itself
if os.path.isdir(dest):
os.chmod(dest, max_privs)
else:
set_file_privs(dest)
def get_file(self,
source,
dest,
delete_dest=False,
preserve_perm=True,
preserve_symlinks=False):
"""
Copy files from the remote host to a local path.
Directories will be copied recursively.
If a source component is a directory with a trailing slash,
the content of the directory will be copied, otherwise, the
directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
Args:
source: either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed)
files or directories
dest: a file or a directory (if source contains a
directory or more than one element, you must
supply a directory dest)
delete_dest: if this is true, the command will also clear
out any old files at dest that are not in the
source
preserve_perm: tells get_file() to try to preserve the sources
permissions on files and dirs
preserve_symlinks: try to preserve symlinks instead of
transforming them into files/dirs on copy
Raises:
AutoservRunError: the scp command failed
"""
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(source, basestring):
source = [source]
dest = os.path.abspath(dest)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(source)
local_dest = utils.sh_escape(dest)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dest, preserve_symlinks)
utils.run(rsync)
try_scp = False
except error.CmdError as e:
logging.warn("trying scp, rsync failed: %s" % e)
if try_scp:
# scp has no equivalent to --delete, just drop the entire dest dir
if delete_dest and os.path.isdir(dest):
shutil.rmtree(dest)
os.mkdir(dest)
remote_source = self._make_rsync_compatible_source(source, False)
if remote_source:
# _make_rsync_compatible_source() already did the escaping
remote_source = self._encode_remote_paths(remote_source,
escape=False)
local_dest = utils.sh_escape(dest)
scp = self._make_scp_cmd([remote_source], local_dest)
try:
utils.run(scp)
except error.CmdError as e:
raise error.AutoservRunError(e.args[0], e.args[1])
if not preserve_perm:
# we have no way to tell scp to not try to preserve the
# permissions so set them after copy instead.
# for rsync we could use "--no-p --chmod=ugo=rwX" but those
# options are only in very recent rsync versions
self._set_umask_perms(dest)
def send_file(self,
source,
dest,
delete_dest=False,
preserve_symlinks=False):
"""
Copy files from a local path to the remote host.
Directories will be copied recursively.
If a source component is a directory with a trailing slash,
the content of the directory will be copied, otherwise, the
directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
Args:
source: either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed)
files or directories
dest: a file or a directory (if source contains a
directory or more than one element, you must
supply a directory dest)
delete_dest: if this is true, the command will also clear
out any old files at dest that are not in the
source
preserve_symlinks: controls if symlinks on the source will be
copied as such on the destination or transformed into the
referenced file/directory
Raises:
AutoservRunError: the scp command failed
"""
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(source, basestring):
source_is_dir = os.path.isdir(source)
source = [source]
remote_dest = self._encode_remote_paths([dest])
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
local_sources = [utils.sh_escape(path) for path in source]
rsync = self._make_rsync_cmd(local_sources, remote_dest,
delete_dest, preserve_symlinks)
utils.run(rsync)
try_scp = False
except error.CmdError as e:
logging.warn("trying scp, rsync failed: %s" % e)
if try_scp:
# scp has no equivalent to --delete, just drop the entire dest dir
if delete_dest:
dest_exists = False
try:
self.run("test -x %s" % dest)
dest_exists = True
except error.AutoservRunError:
pass
dest_is_dir = False
if dest_exists:
try:
self.run("test -d %s" % dest)
dest_is_dir = True
except error.AutoservRunError:
pass
# If there is a list of more than one path, destination *has*
# to be a dir. If there's a single path being transferred and
# it is a dir, the destination also has to be a dir. Therefore
# it has to be created on the remote machine in case it doesn't
# exist, otherwise we will have an scp failure.
if len(source) > 1 or source_is_dir:
dest_is_dir = True
if dest_exists and dest_is_dir:
cmd = "rm -rf %s && mkdir %s" % (dest, dest)
self.run(cmd)
elif not dest_exists and dest_is_dir:
cmd = "mkdir %s" % dest
self.run(cmd)
local_sources = self._make_rsync_compatible_source(source, True)
if local_sources:
scp = self._make_scp_cmd(local_sources, remote_dest)
try:
utils.run(scp)
except error.CmdError as e:
raise error.AutoservRunError(e.args[0], e.args[1])
def ssh_ping(self, timeout=60):
try:
# Complex inheritance confuses pylint here
# pylint: disable=E1123
self.run("true", timeout=timeout, connect_timeout=timeout)
except error.AutoservSSHTimeout:
msg = "Host (ssh) verify timed out (timeout = %d)" % timeout
raise error.AutoservSSHTimeout(msg)
except error.AutoservSshPermissionDeniedError:
# let AutoservSshPermissionDeniedError be visible to the callers
raise
except error.AutoservRunError as e:
# convert the generic AutoservRunError into something more
# specific for this context
raise error.AutoservSshPingHostError(e.description + '\n' +
repr(e.result_obj))
def is_up(self):
"""
Check if the remote host is up.
:return: True if the remote host is up, False otherwise
"""
try:
self.ssh_ping()
except error.AutoservError:
return False
else:
return True
def wait_up(self, timeout=None):
"""
Wait until the remote host is up or the timeout expires.
In fact, it will wait until an ssh connection to the remote
host can be established, and getty is running.
:param timeout time limit in seconds before returning even
if the host is not up.
:return: True if the host was found to be up, False otherwise
"""
if timeout:
end_time = time.time() + timeout
while not timeout or time.time() < end_time:
if self.is_up():
try:
if self.are_wait_up_processes_up():
logging.debug('Host %s is now up', self.hostname)
return True
except error.AutoservError:
pass
time.sleep(1)
logging.debug('Host %s is still down after waiting %d seconds',
self.hostname, int(timeout + time.time() - end_time))
return False
def wait_down(self, timeout=None, warning_timer=None, old_boot_id=None):
"""
Wait until the remote host is down or the timeout expires.
If old_boot_id is provided, this will wait until either the machine
is unpingable or self.get_boot_id() returns a value different from
old_boot_id. If the boot_id value has changed then the function
returns true under the assumption that the machine has shut down
and has now already come back up.
If old_boot_id is None then until the machine becomes unreachable the
method assumes the machine has not yet shut down.
:param timeout Time limit in seconds before returning even
if the host is still up.
:param warning_timer Time limit in seconds that will generate
a warning if the host is not down yet.
:param old_boot_id A string containing the result of self.get_boot_id()
prior to the host being told to shut down. Can be None if this is
not available.
:return: True if the host was found to be down, False otherwise
"""
# TODO: there is currently no way to distinguish between knowing
# TODO: boot_id was unsupported and not knowing the boot_id.
current_time = time.time()
if timeout:
end_time = current_time + timeout
if warning_timer:
warn_time = current_time + warning_timer
if old_boot_id is not None:
logging.debug('Host %s pre-shutdown boot_id is %s', self.hostname,
old_boot_id)
while not timeout or current_time < end_time:
try:
new_boot_id = self.get_boot_id()
except error.AutoservError:
logging.debug('Host %s is now unreachable over ssh, is down',
self.hostname)
return True
else:
# if the machine is up but the boot_id value has changed from
# old boot id, then we can assume the machine has gone down
# and then already come back up
if old_boot_id is not None and old_boot_id != new_boot_id:
logging.debug(
'Host %s now has boot_id %s and so must '
'have rebooted', self.hostname, new_boot_id)
return True
if warning_timer and current_time > warn_time:
self.record("WARN", None, "shutdown",
"Shutdown took longer than %ds" % warning_timer)
# Print the warning only once.
warning_timer = None
# If a machine is stuck switching runlevels
# This may cause the machine to reboot.
self.run('kill -HUP 1', ignore_status=True)
time.sleep(1)
current_time = time.time()
return False
# tunable constants for the verify & repair code
AUTOTEST_GB_DISKSPACE_REQUIRED = settings.get_value(
"SERVER", "gb_diskspace_required", type=int, default=20)
def verify_connectivity(self):
super(AbstractSSHHost, self).verify_connectivity()
logging.info('Pinging host ' + self.hostname)
self.ssh_ping()
logging.info("Host (ssh) %s is alive", self.hostname)
if self.is_shutting_down():
raise error.AutoservHostIsShuttingDownError(
"Host is shutting down")
def verify_software(self):
super(AbstractSSHHost, self).verify_software()
try:
self.check_diskspace(
autotest_remote.Autotest.get_install_dir(self),
self.AUTOTEST_GB_DISKSPACE_REQUIRED)
except error.AutoservHostError:
raise # only want to raise if it's a space issue
except autotest_remote.AutodirNotFoundError:
# autotest_remote dir may not exist, etc. ignore
logging.debug('autodir space check exception, this is probably '
'safe to ignore\n' + traceback.format_exc())
def close(self):
super(AbstractSSHHost, self).close()
self._cleanup_master_ssh()
os.remove(self.known_hosts_file)
def _cleanup_master_ssh(self):
"""
Release all resources (process, temporary directory) used by an active
master SSH connection.
"""
# If a master SSH connection is running, kill it.
if self.master_ssh_job is not None:
utils.nuke_subprocess(self.master_ssh_job.sp)
self.master_ssh_job = None
# Remove the temporary directory for the master SSH socket.
if self.master_ssh_tempdir is not None:
self.master_ssh_tempdir.clean()
self.master_ssh_tempdir = None
self.master_ssh_option = ''
def start_master_ssh(self):
"""
Called whenever a slave SSH connection needs to be initiated (e.g., by
run, rsync, scp). If master SSH support is enabled and a master SSH
connection is not active already, start a new one in the background.
Also, cleanup any zombie master SSH connections (e.g., dead due to
reboot).
"""
if not enable_master_ssh:
return
# If a previously started master SSH connection is not running
# anymore, it needs to be cleaned up and then restarted.
if self.master_ssh_job is not None:
if self.master_ssh_job.sp.poll() is not None:
logging.info("Master ssh connection to %s is down.",
self.hostname)
self._cleanup_master_ssh()
# Start a new master SSH connection.
if self.master_ssh_job is None:
# Create a shared socket in a temp location.
self.master_ssh_tempdir = autotemp.tempdir(unique_id='ssh-master')
self.master_ssh_option = ("-o ControlPath=%s/socket" %
self.master_ssh_tempdir.name)
# Start the master SSH connection in the background.
master_cmd = self.ssh_command(options="-N -o ControlMaster=yes")
logging.info("Starting master ssh connection '%s'" % master_cmd)
self.master_ssh_job = utils.BgJob(master_cmd)
def clear_known_hosts(self):
"""Clears out the temporary ssh known_hosts file.
This is useful if the test SSHes to the machine, then reinstalls it,
then SSHes to it again. It can be called after the reinstall to
reduce the spam in the logs.
"""
logging.info("Clearing known hosts for host '%s', file '%s'.",
self.hostname, self.known_hosts_file)
# Clear out the file by opening it for writing and then closing.
fh = open(self.known_hosts_file, "w")
fh.close()
def _create_test_output_dir(self, host, autodir):
tmpdir = os.path.join(autodir, 'tmp')
state_autodir = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
host.run('mkdir -p %s' % utils.sh_escape(state_autodir))
import logging
import os
import re
import sys
import tempfile
import time
import traceback
from autotest.client import os_dep
from autotest.client import utils as client_utils
from autotest.client.shared import base_job, error, autotemp
from autotest.client.shared import packages
from autotest.client.shared.settings import settings, SettingsError
from autotest.server import installable_object, prebuild, utils
autoserv_prebuild = settings.get_value('AUTOSERV', 'enable_server_prebuild',
type=bool, default=False)
CLIENT_BINARY = 'autotest-local-streamhandler'
class AutodirNotFoundError(Exception):
"""No Autotest installation could be found."""
# Paths you'll find when autotest is installed via distro package
SYSTEM_WIDE_PATHS = ['/usr/bin/autotest-local',
'/usr/bin/autotest-local-streamhandler',
'/usr/bin/autotest-daemon',
'/usr/bin/autotest-daemon-monitor']
def runjob(control, drop_caches, options):
"""
Run a job using the given control file.
This is the main interface to this module.
:see: base_job.__init__ for parameter info.
"""
control = os.path.abspath(control)
try:
autodir = os.path.abspath(os.environ['AUTODIR'])
except KeyError:
autodir = settings.get_value('COMMON', 'autotest_top_path')
tmpdir = os.path.join(autodir, 'tmp')
tests_out_dir = settings.get_value('COMMON', 'test_output_dir',
default=tmpdir)
state = os.path.join(tests_out_dir, os.path.basename(control) + '.state')
# Ensure state file is cleaned up before the job starts to run if autotest
# is not running with the --continue flag
if not options.cont and os.path.isfile(state):
os.remove(state)
# instantiate the job object ready for the control file.
myjob = None
try:
# Check that the control file is valid
if not os.path.exists(control):
raise error.JobError(control + ": control file not found")
# When continuing, the job is complete when there is no
# state file, ensure we don't try and continue.
if options.cont and not os.path.exists(state):
raise error.JobComplete("all done")
myjob = job(control=control, drop_caches=drop_caches, options=options)
# Load in the users control file, may do any one of:
# 1) execute in toto
# 2) define steps, and select the first via next_step()
myjob.step_engine()
except error.JobContinue:
sys.exit(5)
except error.JobComplete:
sys.exit(1)
except error.JobError, instance:
logging.error("JOB ERROR: " + str(instance))
if myjob:
command = None
if len(instance.args) > 1:
command = instance.args[1]
myjob.record('ABORT', None, command, str(instance))
myjob.record('END ABORT', None, None, str(instance))
assert myjob._record_indent == 0
myjob.complete(1)
else:
sys.exit(1)
def get_hosts(multiple_labels=(), exclude_only_if_needed_labels=False,
exclude_atomic_group_hosts=False, valid_only=True, **filter_data):
"""
@param multiple_labels: match hosts in all of the labels given. Should
be a list of label names.
@param exclude_only_if_needed_labels: Exclude hosts with at least one
"only_if_needed" label applied.
@param exclude_atomic_group_hosts: Exclude hosts that have one or more
atomic group labels associated with them.
"""
hosts = rpc_utils.get_host_query(multiple_labels,
exclude_only_if_needed_labels,
exclude_atomic_group_hosts,
valid_only, filter_data)
hosts = list(hosts)
models.Host.objects.populate_relationships(hosts, models.Label,
'label_list')
models.Host.objects.populate_relationships(hosts, models.AclGroup,
'acl_list')
models.Host.objects.populate_relationships(hosts, models.HostAttribute,
'attribute_list')
install_server = None
install_server_info = get_install_server_info()
install_server_type = install_server_info.get('type', None)
install_server_url = install_server_info.get('xmlrpc_url', None)
if install_server_type == 'cobbler' and install_server_url:
install_server = xmlrpclib.ServerProxy(install_server_url)
host_dicts = []
for host_obj in hosts:
host_dict = host_obj.get_object_dict()
host_dict['labels'] = [label.name for label in host_obj.label_list]
host_dict['platform'], host_dict['atomic_group'] = (rpc_utils.
find_platform_and_atomic_group(host_obj))
host_dict['acls'] = [acl.name for acl in host_obj.acl_list]
host_dict['attributes'] = dict((attribute.attribute, attribute.value)
for attribute in host_obj.attribute_list)
error_encountered = True
if install_server is not None:
system_params = {"name": host_dict['hostname']}
system_list = install_server.find_system(system_params, True)
if len(system_list) < 1:
msg = 'System "%s" not found on install server'
rpc_logger = logging.getLogger('rpc_logger')
rpc_logger.info(msg, host_dict['hostname'])
elif len(system_list) > 1:
msg = 'Found multiple systems on install server named %s'
if install_server_type == 'cobbler':
msg = '%s. This should never happen on cobbler' % msg
rpc_logger = logging.getLogger('rpc_logger')
rpc_logger.error(msg, host_dict['hostname'])
else:
system = system_list[0]
if host_dict['platform']:
error_encountered = False
profiles = sorted(install_server.get_item_names('profile'))
host_dict['profiles'] = profiles
host_dict['profiles'].insert(0, 'Do_not_install')
use_current_profile = settings.get_value('INSTALL_SERVER',
'use_current_profile', type=bool, default=True)
if use_current_profile:
host_dict['current_profile'] = system['profile']
else:
host_dict['current_profile'] = 'Do_not_install'
if error_encountered:
host_dict['profiles'] = ['N/A']
host_dict['current_profile'] = 'N/A'
host_dicts.append(host_dict)
return rpc_utils.prepare_for_serialization(host_dicts)
import cPickle
import logging
import os
try:
import autotest.common as common # pylint: disable=W0611
except ImportError:
import common # pylint: disable=W0611
from autotest.scheduler import drone_utility
from autotest.client.shared.settings import settings
from autotest.client.shared import mail
AUTOTEST_INSTALL_DIR = settings.get_value('SCHEDULER',
'drone_installation_directory')
class DroneUnreachable(Exception):
"""The drone is non-sshable."""
pass
class _AbstractDrone(object):
"""
Attributes:
* allowed_users: set of usernames allowed to use this drone. if None,
any user can use this drone.
"""
def __init__(self):
self._calls = []
self.hostname = None
self.enabled = True
def result_notify(self, job, email_from, email_to):
"""
Notify about the result of a job. Will always print, if email data
is provided, will send email for it as well.
job: job object to notify about
email_from: send notification email upon completion from here
email_from: send notification email upon completion to here
"""
if job.result is True:
subject = 'Testing PASSED: '
else:
subject = 'Testing FAILED: '
subject += '%s : %s\n' % (job.name, job.id)
text = []
for platform in job.results_platform_map:
for status in job.results_platform_map[platform]:
if status == 'Total':
continue
for host in job.results_platform_map[platform][status]:
text.append('%20s %10s %10s' % (platform, status, host))
if status == 'Failed':
for test_status in job.test_status[host].fail:
text.append('(%s, %s) : %s' %
(host, test_status.test_name,
test_status.reason))
text.append('')
base_url = 'http://' + self.server
params = ('columns=test',
'rows=machine_group',
"condition=tag~'%s-%%25'" % job.id,
'title=Report')
query_string = '&'.join(params)
url = '%s/tko/compose_query.cgi?%s' % (base_url, query_string)
text.append(url + '\n')
url = '%s/afe/#tab_id=view_job&object_id=%s' % (base_url, job.id)
text.append(url + '\n')
smtp_info = {}
smtp_info['server'] = settings.get_value('SERVER', 'smtp_server',
default='localhost')
smtp_info['port'] = settings.get_value('SERVER', 'smtp_port',
default='')
smtp_info['user'] = settings.get_value('SERVER', 'smtp_user',
default='')
smtp_info['password'] = settings.get_value('SERVER', 'smtp_password',
default='')
body = '\n'.join(text)
print '---------------------------------------------------'
print 'Subject: ', subject
print body
print '---------------------------------------------------'
if email_from and email_to:
print 'Sending email ...'
mail.send(from_address=email_from,
to_addresses=email_to,
subject=subject,
body=body,
smtp_info=smtp_info)
print
def runtest(job,
url,
tag,
args,
dargs,
local_namespace={},
global_namespace={},
before_test_hook=None,
after_test_hook=None,
before_iteration_hook=None,
after_iteration_hook=None):
local_namespace = local_namespace.copy()
global_namespace = global_namespace.copy()
# if this is not a plain test name then download and install the
# specified test
if url.endswith('.tar.bz2'):
(testgroup, testname) = _installtest(job, url)
bindir = os.path.join(job.testdir, 'download', testgroup, testname)
importdir = os.path.join(job.testdir, 'download')
modulename = '%s.%s' % (re.sub('/', '.', testgroup), testname)
classname = '%s.%s' % (modulename, testname)
path = testname
else:
# If the test is local, it may be under either testdir or site_testdir.
# Tests in site_testdir override tests defined in testdir
testname = path = url
testgroup = ''
path = re.sub(':', '/', testname)
modulename = os.path.basename(path)
classname = '%s.%s' % (modulename, modulename)
# Try installing the test package
# The job object may be either a server side job or a client side job.
# 'install_pkg' method will be present only if it's a client side job.
if hasattr(job, 'install_pkg'):
try:
bindir = os.path.join(job.site_testdir, testname)
job.install_pkg(testname, 'test', bindir)
except error.PackageInstallError:
# continue as a fall back mechanism and see if the test code
# already exists on the machine
pass
testdir_list = [job.testdir, getattr(job, 'site_testdir', None)]
bindir_config = settings.get_value('COMMON', 'test_dir', default="")
if bindir_config:
testdir_list.extend(bindir_config.strip().split(','))
bindir = None
for t_dir in testdir_list:
if t_dir is not None and os.path.exists(os.path.join(t_dir, path)):
importdir = bindir = os.path.join(t_dir, path)
if not bindir:
raise error.TestError(testname + ': test does not exist')
subdir = os.path.join(dargs.pop('master_testpath', ""), testname)
outputdir = os.path.join(job.resultdir, subdir)
if tag:
outputdir += '.' + tag
local_namespace['job'] = job
local_namespace['outputdir'] = outputdir
local_namespace['bindir'] = bindir
sys.path.insert(0, importdir)
try:
exec('import %s' % modulename, local_namespace, global_namespace)
exec("mytest = %s(job, bindir, outputdir)" % classname,
local_namespace, global_namespace)
finally:
sys.path.pop(0)
pwd = os.getcwd()
os.chdir(outputdir)
try:
mytest = global_namespace['mytest']
if before_test_hook:
before_test_hook(mytest)
# we use the register iteration hooks methods to register the passed
# in hooks
if before_iteration_hook:
mytest.register_before_iteration_hook(before_iteration_hook)
if after_iteration_hook:
mytest.register_after_iteration_hook(after_iteration_hook)
mytest._exec(args, dargs)
finally:
os.chdir(pwd)
if after_test_hook:
after_test_hook(mytest)
shutil.rmtree(mytest.tmpdir, ignore_errors=True)
def main():
# grab the parser
parser = autoserv_parser.autoserv_parser
parser.parse_args()
if len(sys.argv) == 1:
parser.parser.print_help()
sys.exit(1)
if parser.options.no_logging:
results = None
else:
output_dir = settings.get_value('COMMON',
'test_output_dir',
default="")
results = parser.options.results
if not results:
results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
if output_dir:
results = os.path.join(output_dir, results)
results = os.path.abspath(results)
resultdir_exists = False
for filename in ('control.srv', 'status.log', '.autoserv_execute'):
if os.path.exists(os.path.join(results, filename)):
resultdir_exists = True
if not parser.options.use_existing_results and resultdir_exists:
error = "Error: results directory already exists: %s\n" % results
sys.stderr.write(error)
sys.exit(1)
# Now that we certified that there's no leftover results dir from
# previous jobs, lets create the result dir since the logging system
# needs to create the log file in there.
if not os.path.isdir(results):
os.makedirs(results)
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
results_dir=results,
use_console=not parser.options.no_tee,
verbose=parser.options.verbose,
no_console_prefix=parser.options.no_console_prefix)
if results:
logging.info("Results placed in %s" % results)
# wait until now to perform this check, so it get properly logged
if parser.options.use_existing_results and not resultdir_exists:
logging.error("No existing results directory found: %s", results)
sys.exit(1)
if parser.options.write_pidfile:
pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
results)
pid_file_manager.open_file()
else:
pid_file_manager = None
autotest_remote.BaseAutotest.set_install_in_tmpdir(
parser.options.install_in_tmpdir)
exit_code = 0
try:
try:
run_autoserv(pid_file_manager, results, parser)
except SystemExit as e:
exit_code = e.code
except Exception:
traceback.print_exc()
# If we don't know what happened, we'll classify it as
# an 'abort' and return 1.
exit_code = 1
finally:
if pid_file_manager:
pid_file_manager.close_file(exit_code)
sys.exit(exit_code)
import logging
import os
import re
import signal
import sys
import time
import traceback
try:
import autotest.common as common # pylint: disable=W0611
except ImportError:
import common # pylint: disable=W0611
from autotest.client.shared.settings import settings
require_atfork = settings.get_value('AUTOSERV',
'require_atfork_module',
type=bool,
default=True)
try:
import atfork
atfork.monkeypatch_os_fork_functions()
import atfork.stdlib_fixer
# Fix the Python standard library for threading+fork safety with its
# internal locks. http://code.google.com/p/python-atfork/
import warnings
warnings.filterwarnings('ignore', 'logging module already imported')
try:
atfork.stdlib_fixer.fix_logging_module()
except Exception:
pass
except ImportError as e:
# -*- coding:utf-8 -*-
import os, sys
import re
import shutil
from autotest.client import test, utils
from autotest.client.shared import error
# lpt env set
from autotest.client.shared.settings import settings
try:
autodir = os.path.abspath(os.environ['AUTODIR'])
except KeyError:
autodir = settings.get_value('COMMON', 'autotest_top_path')
autodir = os.path.abspath(os.environ['AUTODIR'])
#lptdir = os.path.join(os.path.dirname(autodir), "lpt")
lptdir = os.path.join(autodir, "lpts")
os.environ['LPTROOT'] = lptdir
from autotest.client import setup_modules
setup_modules.setup(base_path=lptdir, root_module_name="lpt")
import lpt.api.test as lpt_test
from lpt.lib import lptlog
from lpt.lib.share import utils as lutils
unixbench_keys = [
'Dhrystone2-using-register-variables', 'Double-Precision-Whetstone',
'Execl-Throughput', 'FileCopy1024-bufsize2000-maxblocks',
'FileCopy256-bufsize500-maxblocks', 'FileCopy4096-bufsize8000-maxblocks',
def get_fetch_location(self):
repos = settings.get_value("PACKAGES", 'fetch_location', type=list,
default=[])
repos.reverse()
return repos
def get_mappings_2x():
KERNEL_BASE_URL = settings.get_value('CLIENT', 'kernel_mirror', default='')
GITWEB_BASE_URL = settings.get_value('CLIENT', 'kernel_gitweb', default='')
STABLE_GITWEB_BASE_URL = settings.get_value('CLIENT',
'stable_kernel_gitweb',
default='')
MAPPINGS_2X = [
[
r'^\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2',
KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz',
GITWEB_BASE_URL.split())
],
[
r'^\d+\.\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2',
KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz',
GITWEB_BASE_URL.split())
],
[
r'^\d+\.\d+\.\d+\.\d+$', '', True,
map(lambda x: x + 'v%(major)s/linux-%(full)s.tar.bz2',
KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz',
STABLE_GITWEB_BASE_URL.split())
],
[
r'-rc\d+$', '%(minor-prev)s', True,
map(
lambda x: x +
'v%(major)s/testing/v%(minor)s/linux-%(full)s.tar.bz2',
KERNEL_BASE_URL.split()) +
map(lambda x: x + 'v%(major)s/testing/linux-%(full)s.tar.bz2',
KERNEL_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=refs/tags/v%(full)s;sf=tgz',
GITWEB_BASE_URL.split())
],
[
r'-(git|bk)\d+$', '%(base)s', False,
map(lambda x: x + 'v%(major)s/snapshots/old/patch-%(full)s.bz2',
KERNEL_BASE_URL.split()) +
map(lambda x: x + 'v%(major)s/snapshots/patch-%(full)s.bz2',
KERNEL_BASE_URL.split())
],
[
r'-mm\d+$', '%(base)s', False,
map(
lambda x: x + 'people/akpm/patches/' +
'%(major)s/%(base)s/%(full)s/%(full)s.bz2',
KERNEL_BASE_URL.split())
],
[
r'-mjb\d+$', '%(base)s', False,
map(lambda x: x + 'people/mbligh/%(base)s/patch-%(full)s.bz2',
KERNEL_BASE_URL.split())
],
[
r'[a-f0-9]{7,40}$', '', True,
map(lambda x: x + ';a=snapshot;h=%(full)s;sf=tgz',
GITWEB_BASE_URL.split()) +
map(lambda x: x + ';a=snapshot;h=%(full)s;sf=tgz',
STABLE_GITWEB_BASE_URL.split())
]
]
return MAPPINGS_2X
def get_client_autodir_paths(cls, host):
return settings.get_value('AUTOSERV', 'client_autodir_paths', type=list)
