def main():
parser = OptionParser()
(options, args) = parser.parse_args()
logging_manager.configure_logging(MigrateLoggingConfig(),
verbose=True)
if len(args) > 0:
manager = get_migration_manager(db_name=options.database,
debug=options.debug,
force=options.force)
if len(args) > 1:
version = int(args[1])
else:
version = None
if args[0] == 'sync':
manager.do_sync_db(version)
elif args[0] == 'test':
manager.simulate = True
manager.test_sync_db(version)
elif args[0] == 'simulate':
manager.simulate = True
manager.simulate_sync_db(version)
elif args[0] == 'safesync':
logging.info('Simulating migration')
manager.simulate = True
manager.simulate_sync_db(version)
logging.info('Performing real migration')
manager.simulate = False
manager.do_sync_db(version)
else:
parser.print_help()
return
parser.print_help()
def parse_args(self, args, options):
"""
Process a client side command.
:param args: Command line args.
"""
logging_manager.configure_logging(CmdParserLoggingConfig(),
verbose=options.verbose)
if len(args) and args[0] in self.COMMAND_LIST:
cmd = args.pop(0)
else:
# Do things the traditional way
return args
# List is a python reserved word
if cmd == 'list':
cmd = 'list_tests'
try:
try:
args = getattr(self, cmd)(args, options)
except TypeError:
args = getattr(self, cmd)()
except SystemExit, return_code:
sys.exit(return_code.code)
def parse_args(self, args, options):
"""
Process a client side command.
:param args: Command line args.
"""
logging_manager.configure_logging(CmdParserLoggingConfig(),
verbose=options.verbose)
if len(args) and args[0] in self.COMMAND_LIST:
cmd = args.pop(0)
else:
# Do things the traditional way
return args
# List is a python reserved word
if cmd == 'list':
cmd = 'list_tests'
try:
try:
args = getattr(self, cmd)(args, options)
except TypeError:
args = getattr(self, cmd)()
except SystemExit as return_code:
sys.exit(return_code.code)
except Exception as error_detail:
if DEBUG:
raise
sys.stderr.write("Command failed: %s -> %s\n" %
(cmd, error_detail))
self.help()
sys.exit(1)
# Args are cleaned up, return to process the traditional way
return args
def main():
parser = OptionParser()
(options, args) = parser.parse_args()
logging_manager.configure_logging(MigrateLoggingConfig(),
verbose=True)
if len(args) > 0:
manager = get_migration_manager(db_name=options.database,
debug=options.debug,
force=options.force)
if len(args) > 1:
version = int(args[1])
else:
version = None
if args[0] == 'sync':
manager.do_sync_db(version)
elif args[0] == 'test':
manager.simulate = True
manager.test_sync_db(version)
elif args[0] == 'simulate':
manager.simulate = True
manager.simulate_sync_db(version)
elif args[0] == 'safesync':
logging.info('Simulating migration')
manager.simulate = True
manager.simulate_sync_db(version)
logging.info('Performing real migration')
manager.simulate = False
manager.do_sync_db(version)
else:
parser.print_help()
return
parser.print_help()
def bootstrap(options, interactive=False):
"""
Common virt test assistant module.
:param options: Command line options.
:param interactive: Whether to ask for confirmation.
:raise error.CmdError: If JeOS image failed to uncompress
:raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=options.vt_verbose)
logging.info("%s test config helper", options.vt_type)
step = 0
logging.info("")
step += 1
logging.info("%d - Updating all test providers", step)
asset.download_all_test_providers(options.vt_update_providers)
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
guest_os = options.vt_guest_os or defaults.DEFAULT_GUEST_OS
try:
verify_mandatory_programs(options.vt_type, guest_os)
except Exception, details:
logging.info(details)
logging.info('Install the missing programs and/or headers and '
're-run boostrap')
sys.exit(1)
def main():
parser = OptionParser()
parser.add_option("-r",
action="store_true",
dest="recover",
help=("run recovery mode (implicit after any crash)"))
parser.add_option("--background",
dest="background",
action="store_true",
default=False,
help=("runs the scheduler monitor on "
"background"))
(options, args) = parser.parse_args()
recover = (options.recover == True)
if len(args) != 0:
parser.print_help()
sys.exit(1)
if os.getuid() == 0:
logging.critical("Running as root, aborting!")
sys.exit(1)
if utils.program_is_alive(monitor_db.WATCHER_PID_FILE_PREFIX):
logging.critical("autotest-monitor-watcher already running, aborting!")
sys.exit(1)
utils.write_pid(monitor_db.WATCHER_PID_FILE_PREFIX)
if options.background:
logging_manager.configure_logging(
watcher_logging_config.WatcherLoggingConfig(use_console=False))
# Double fork - see http://code.activestate.com/recipes/66012/
try:
pid = os.fork()
if (pid > 0):
sys.exit(0) # exit from first parent
except OSError, e:
sys.stderr.write("fork #1 failed: (%d) %s\n" %
(e.errno, e.strerror))
sys.exit(1)
# Decouple from parent environment
os.chdir("/")
os.umask(0)
os.setsid()
# Second fork
try:
pid = os.fork()
if (pid > 0):
sys.exit(0) # exit from second parent
except OSError, e:
sys.stderr.write("fork #2 failed: (%d) %s\n" %
(e.errno, e.strerror))
sys.exit(1)
def setUp(self):
# make god
self.god = mock.mock_god(ut=self)
# need to set some environ variables
self.autodir = "autodir"
os.environ['AUTODIR'] = self.autodir
# set up some variables
self.control = "control"
self.jobtag = "jobtag"
# get rid of stdout and logging
sys.stdout = io.StringIO()
logging_manager.configure_logging(logging_config.TestingConfig())
logging.disable(logging.CRITICAL)
def dummy_configure_logging(*args, **kwargs):
pass
self.god.stub_with(logging_manager, 'configure_logging',
dummy_configure_logging)
real_get_logging_manager = logging_manager.get_logging_manager
def get_logging_manager_no_fds(manage_stdout_and_stderr=False,
redirect_fds=False):
return real_get_logging_manager(manage_stdout_and_stderr, False)
self.god.stub_with(logging_manager, 'get_logging_manager',
get_logging_manager_no_fds)
# stub out some stuff
self.god.stub_function(os.path, 'exists')
self.god.stub_function(os.path, 'isdir')
self.god.stub_function(os, 'makedirs')
self.god.stub_function(os, 'mkdir')
self.god.stub_function(os, 'remove')
self.god.stub_function(shutil, 'rmtree')
self.god.stub_function(shutil, 'copyfile')
self.god.stub_function(job, 'open')
self.god.stub_function(utils, 'system')
self.god.stub_function(utils_memory, 'drop_caches')
self.god.stub_function(harness, 'select')
self.god.stub_function(sysinfo, 'log_per_reboot_data')
self.god.stub_class(config, 'config')
self.god.stub_class(job.local_host, 'LocalHost')
self.god.stub_class(boottool, 'boottool')
self.god.stub_class(sysinfo, 'sysinfo')
self.god.stub_class_method(job.base_client_job,
'_cleanup_debugdir_files')
self.god.stub_class_method(job.base_client_job, '_cleanup_results_dir')
self.god.stub_with(job.base_job.job_directory, '_ensure_valid',
lambda *_: None)
def setUp(self):
# make god
self.god = mock.mock_god(ut=self)
# need to set some environ variables
self.autodir = "autodir"
os.environ['AUTODIR'] = self.autodir
# set up some variables
self.control = "control"
self.jobtag = "jobtag"
# get rid of stdout and logging
sys.stdout = StringIO.StringIO()
logging_manager.configure_logging(logging_config.TestingConfig())
logging.disable(logging.CRITICAL)
def dummy_configure_logging(*args, **kwargs):
pass
self.god.stub_with(logging_manager, 'configure_logging',
dummy_configure_logging)
real_get_logging_manager = logging_manager.get_logging_manager
def get_logging_manager_no_fds(manage_stdout_and_stderr=False,
redirect_fds=False):
return real_get_logging_manager(manage_stdout_and_stderr, False)
self.god.stub_with(logging_manager, 'get_logging_manager',
get_logging_manager_no_fds)
# stub out some stuff
self.god.stub_function(os.path, 'exists')
self.god.stub_function(os.path, 'isdir')
self.god.stub_function(os, 'makedirs')
self.god.stub_function(os, 'mkdir')
self.god.stub_function(os, 'remove')
self.god.stub_function(shutil, 'rmtree')
self.god.stub_function(shutil, 'copyfile')
self.god.stub_function(job, 'open')
self.god.stub_function(utils, 'system')
self.god.stub_function(utils_memory, 'drop_caches')
self.god.stub_function(harness, 'select')
self.god.stub_function(sysinfo, 'log_per_reboot_data')
self.god.stub_class(config, 'config')
self.god.stub_class(job.local_host, 'LocalHost')
self.god.stub_class(boottool, 'boottool')
self.god.stub_class(sysinfo, 'sysinfo')
self.god.stub_class_method(job.base_client_job,
'_cleanup_debugdir_files')
self.god.stub_class_method(job.base_client_job, '_cleanup_results_dir')
self.god.stub_with(job.base_job.job_directory, '_ensure_valid',
lambda *_: None)
def main():
parser = OptionParser()
parser.add_option("-r", action="store_true", dest="recover", help=("run recovery mode (implicit after any crash)"))
parser.add_option(
"--background",
dest="background",
action="store_true",
default=False,
help=("runs the scheduler monitor on " "background"),
)
(options, args) = parser.parse_args()
recover = options.recover is True
if len(args) != 0:
parser.print_help()
sys.exit(1)
if os.getuid() == 0:
logging.critical("Running as root, aborting!")
sys.exit(1)
if utils.program_is_alive(monitor_db.WATCHER_PID_FILE_PREFIX):
logging.critical("autotest-monitor-watcher already running, aborting!")
sys.exit(1)
utils.write_pid(monitor_db.WATCHER_PID_FILE_PREFIX)
if options.background:
logging_manager.configure_logging(watcher_logging_config.WatcherLoggingConfig(use_console=False))
# Double fork - see http://code.activestate.com/recipes/66012/
try:
pid = os.fork()
if pid > 0:
sys.exit(0) # exit from first parent
except OSError, e:
sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
# Decouple from parent environment
os.chdir("/")
os.umask(0)
os.setsid()
# Second fork
try:
pid = os.fork()
if pid > 0:
sys.exit(0) # exit from second parent
except OSError, e:
sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
def main():
logging_manager.configure_logging(CompileClientsLoggingConfig(),
verbose=True)
parser = optparse.OptionParser()
parser.add_option('-l', '--list-projects',
action='store_true', dest='list_projects',
default=False,
help='List all projects and clients that can be compiled')
parser.add_option('-a', '--compile-all',
action='store_true', dest='compile_all',
default=False,
help='Compile all available projects and clients')
parser.add_option('-c', '--compile',
dest='compile_list', action='store',
help='List of clients to compiled (e.g. -c "x.X c.C")')
parser.add_option('-e', '--extra-args',
dest='extra_args', action='store',
default='',
help='Extra arguments to pass to java')
parser.add_option('-d', '--no-install', dest='install_client',
action='store_false', default=True,
help='Do not install the clients just compile them')
options, args = parser.parse_args()
if len(sys.argv) < 2:
parser.print_help()
sys.exit(0)
elif options.list_projects:
print_projects()
sys.exit(0)
elif options.compile_all and options.compile_list:
logging.error('Options -c and -a are mutually exclusive')
parser.print_help()
sys.exit(1)
failed_clients = []
if options.compile_all:
failed_clients = compile_all_projects(options.extra_args)
elif options.compile_list:
for client in options.compile_list.split():
if not compile_and_install_client(client, options.extra_args,
options.install_client):
failed_clients.append(client)
if os.path.exists(_TMP_COMPILE_DIR):
shutil.rmtree(_TMP_COMPILE_DIR)
if failed_clients:
logging.error('The following clients failed: %s',
'\n'.join(failed_clients))
sys.exit(1)
def setUp(self):
# make god
self.god = mock.mock_god(ut=self)
# need to set some environ variables
self.autodir = "autodir"
os.environ["AUTODIR"] = self.autodir
# set up some variables
self.control = "control"
self.jobtag = "jobtag"
# get rid of stdout and logging
sys.stdout = StringIO.StringIO()
logging_manager.configure_logging(logging_config.TestingConfig())
logging.disable(logging.CRITICAL)
def dummy_configure_logging(*args, **kwargs):
pass
self.god.stub_with(logging_manager, "configure_logging", dummy_configure_logging)
real_get_logging_manager = logging_manager.get_logging_manager
def get_logging_manager_no_fds(manage_stdout_and_stderr=False, redirect_fds=False):
return real_get_logging_manager(manage_stdout_and_stderr, False)
self.god.stub_with(logging_manager, "get_logging_manager", get_logging_manager_no_fds)
# stub out some stuff
self.god.stub_function(os.path, "exists")
self.god.stub_function(os.path, "isdir")
self.god.stub_function(os, "makedirs")
self.god.stub_function(os, "mkdir")
self.god.stub_function(os, "remove")
self.god.stub_function(shutil, "rmtree")
self.god.stub_function(shutil, "copyfile")
self.god.stub_function(job, "open")
self.god.stub_function(utils, "system")
self.god.stub_function(utils, "drop_caches")
self.god.stub_function(harness, "select")
self.god.stub_function(sysinfo, "log_per_reboot_data")
self.god.stub_class(config, "config")
self.god.stub_class(job.local_host, "LocalHost")
self.god.stub_class(boottool, "boottool")
self.god.stub_class(sysinfo, "sysinfo")
self.god.stub_class_method(job.base_client_job, "_cleanup_debugdir_files")
self.god.stub_class_method(job.base_client_job, "_cleanup_results_dir")
self.god.stub_with(job.base_job.job_directory, "_ensure_valid", lambda *_: None)
def _pre_record_init(self, control, options):
"""
Initialization function that should peform ONLY the required
setup so that the self.record() method works.
As of now self.record() needs self.resultdir, self._group_level,
self.harness and of course self._logger.
"""
if not options.cont:
self._cleanup_debugdir_files()
self._cleanup_results_dir()
logging_manager.configure_logging(
client_logging_config.ClientLoggingConfig(), results_dir=self.resultdir, verbose=options.verbose
)
logging.info("Writing results to %s", self.resultdir)
# init_group_level needs the state
self.control = os.path.realpath(control)
self._is_continuation = options.cont
self._current_step_ancestry = []
self._next_step_index = 0
self._load_state()
_harness = self.handle_persistent_option(options, "harness")
_harness_args = self.handle_persistent_option(options, "harness_args")
self.harness = harness.select(_harness, self, _harness_args)
# set up the status logger
def client_job_record_hook(entry):
msg_tag = ""
if "." in self._logger.global_filename:
msg_tag = self._logger.global_filename.split(".", 1)[1]
# send the entry to the job harness
message = "\n".join([entry.message] + entry.extra_message_lines)
rendered_entry = self._logger.render_entry(entry)
self.harness.test_status_detail(
entry.status_code, entry.subdir, entry.operation, message, msg_tag, entry.fields
)
self.harness.test_status(rendered_entry, msg_tag)
# send the entry to stdout, if it's enabled
logging.info(rendered_entry)
self._logger = base_job.status_logger(
self, status_indenter(self), record_hook=client_job_record_hook, tap_writer=self._tap
)
def _pre_record_init(self, control, options):
"""
Initialization function that should peform ONLY the required
setup so that the self.record() method works.
As of now self.record() needs self.resultdir, self._group_level,
self.harness and of course self._logger.
"""
if not options.cont:
self._cleanup_debugdir_files()
self._cleanup_results_dir()
logging_manager.configure_logging(
client_logging_config.ClientLoggingConfig(),
results_dir=self.resultdir,
verbose=options.verbose)
logging.info('Writing results to %s', self.resultdir)
# init_group_level needs the state
self.control = os.path.realpath(control)
self._is_continuation = options.cont
self._current_step_ancestry = []
self._next_step_index = 0
self._load_state()
_harness = self.handle_persistent_option(options, 'harness')
_harness_args = self.handle_persistent_option(options, 'harness_args')
self.harness = harness.select(_harness, self, _harness_args)
# set up the status logger
def client_job_record_hook(entry):
msg_tag = ''
if '.' in self._logger.global_filename:
msg_tag = self._logger.global_filename.split('.', 1)[1]
# send the entry to the job harness
message = '\n'.join([entry.message] + entry.extra_message_lines)
rendered_entry = self._logger.render_entry(entry)
self.harness.test_status_detail(entry.status_code, entry.subdir,
entry.operation, message, msg_tag,
entry.fields)
self.harness.test_status(rendered_entry, msg_tag)
# send the entry to stdout, if it's enabled
logging.info(rendered_entry)
self._logger = base_job.status_logger(
self, status_indenter(self), record_hook=client_job_record_hook,
tap_writer=self._tap)
def main():
"""
Find all ExternalPackage classes defined in this file and ask them to
fetch, build and install themselves.
"""
logging_manager.configure_logging(BuildExternalsLoggingConfig(),
verbose=True)
os.umask(022)
top_of_tree = external_packages.find_top_of_autotest_tree()
package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
install_dir = os.path.join(top_of_tree, INSTALL_DIR)
# Make sure the install_dir is in our python module search path
# as well as the PYTHONPATH being used by all our setup.py
# install subprocesses.
if install_dir not in sys.path:
sys.path.insert(0, install_dir)
env_python_path_varname = 'PYTHONPATH'
env_python_path = os.environ.get(env_python_path_varname, '')
if install_dir + ':' not in env_python_path:
os.environ[env_python_path_varname] = ':'.join([
install_dir, env_python_path])
fetched_packages, fetch_errors = fetch_necessary_packages(package_dir,
install_dir)
install_errors = build_and_install_packages(fetched_packages, install_dir)
# Byte compile the code after it has been installed in its final
# location as .pyc files contain the path passed to compile_dir().
# When printing exception tracebacks, python uses that path first to look
# for the source code before checking the directory of the .pyc file.
# Don't leave references to our temporary build dir in the files.
logging.info('compiling .py files in %s to .pyc', install_dir)
compileall.compile_dir(install_dir, quiet=True)
# Some things install with whacky permissions, fix that.
external_packages.system("chmod -R a+rX '%s'" % install_dir)
errors = fetch_errors + install_errors
for error_msg in errors:
logging.error(error_msg)
return len(errors)
def main():
"""
Find all ExternalPackage classes defined in this file and ask them to
fetch, build and install themselves.
"""
logging_manager.configure_logging(BuildExternalsLoggingConfig(),
verbose=True)
os.umask(022)
top_of_tree = external_packages.find_top_of_autotest_tree()
package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
install_dir = os.path.join(top_of_tree, INSTALL_DIR)
# Make sure the install_dir is in our python module search path
# as well as the PYTHONPATH being used by all our setup.py
# install subprocesses.
if install_dir not in sys.path:
sys.path.insert(0, install_dir)
env_python_path_varname = 'PYTHONPATH'
env_python_path = os.environ.get(env_python_path_varname, '')
if install_dir + ':' not in env_python_path:
os.environ[env_python_path_varname] = ':'.join(
[install_dir, env_python_path])
fetched_packages, fetch_errors = fetch_necessary_packages(
package_dir, install_dir)
install_errors = build_and_install_packages(fetched_packages, install_dir)
# Byte compile the code after it has been installed in its final
# location as .pyc files contain the path passed to compile_dir().
# When printing exception tracebacks, python uses that path first to look
# for the source code before checking the directory of the .pyc file.
# Don't leave references to our temporary build dir in the files.
logging.info('compiling .py files in %s to .pyc', install_dir)
compileall.compile_dir(install_dir, quiet=True)
# Some things install with whacky permissions, fix that.
external_packages.system("chmod -R a+rX '%s'" % install_dir)
errors = fetch_errors + install_errors
for error_msg in errors:
logging.error(error_msg)
return len(errors)
def parse_args(self, args, options):
"""
Process a client side command.
:param args: Command line args.
"""
logging_manager.configure_logging(CmdParserLoggingConfig(), verbose=options.verbose)
if len(args) and args[0] in self.COMMAND_LIST:
cmd = args.pop(0)
else:
# Do things the traditional way
return args
# List is a python reserved word
if cmd == 'list':
cmd = 'list_tests'
try:
try:
args = getattr(self, cmd)(args, options)
except TypeError:
args = getattr(self, cmd)()
except SystemExit, return_code:
sys.exit(return_code.code)
if not os.path.isdir(FETCHDIRTEST):
os.makedirs(FETCHDIRTEST)
DEBUG = False
class CmdParserLoggingConfig(logging_config.LoggingConfig):
"""
Used with the sole purpose of providing convenient logging setup
for the KVM test auxiliary programs.
"""
def configure_logging(self, results_dir=None, verbose=False):
super(CmdParserLoggingConfig, self).configure_logging(use_console=True,
verbose=False)
logging_manager.configure_logging(CmdParserLoggingConfig())
class CommandParser(object):
"""
A client-side command wrapper for the autotest client.
"""
COMMAND_LIST = ['help', 'list', 'run', 'fetch']
@classmethod
def _print_control_list(cls, pipe, path):
"""
Print the list of control files available.
@param pipe: Pipe opened to an output stream (may be a pager)
@param path: Path we'll walk through
def main():
# grab the parser
parser = autoserv_parser.autoserv_parser
parser.parse_args()
if len(sys.argv) == 1:
parser.parser.print_help()
sys.exit(1)
if parser.options.no_logging:
results = None
else:
output_dir = settings.get_value('COMMON',
'test_output_dir',
default="")
results = parser.options.results
if not results:
results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
if output_dir:
results = os.path.join(output_dir, results)
results = os.path.abspath(results)
resultdir_exists = False
for filename in ('control.srv', 'status.log', '.autoserv_execute'):
if os.path.exists(os.path.join(results, filename)):
resultdir_exists = True
if not parser.options.use_existing_results and resultdir_exists:
error = "Error: results directory already exists: %s\n" % results
sys.stderr.write(error)
sys.exit(1)
# Now that we certified that there's no leftover results dir from
# previous jobs, lets create the result dir since the logging system
# needs to create the log file in there.
if not os.path.isdir(results):
os.makedirs(results)
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
results_dir=results,
use_console=not parser.options.no_tee,
verbose=parser.options.verbose,
no_console_prefix=parser.options.no_console_prefix)
if results:
logging.info("Results placed in %s" % results)
# wait until now to perform this check, so it get properly logged
if parser.options.use_existing_results and not resultdir_exists:
logging.error("No existing results directory found: %s", results)
sys.exit(1)
if parser.options.write_pidfile:
pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
results)
pid_file_manager.open_file()
else:
pid_file_manager = None
autotest_remote.BaseAutotest.set_install_in_tmpdir(
parser.options.install_in_tmpdir)
exit_code = 0
try:
try:
run_autoserv(pid_file_manager, results, parser)
except SystemExit, e:
exit_code = e.code
except:
traceback.print_exc()
# If we don't know what happened, we'll classify it as
# an 'abort' and return 1.
exit_code = 1
"""
import os, sys, optparse, logging
try:
import autotest.common as common
except ImportError:
import common
from autotest.client.shared import logging_manager
from autotest.client import utils
from virttest import utils_misc
if __name__ == "__main__":
parser = optparse.OptionParser("usage: %prog [options] [filenames]")
options, args = parser.parse_args()
logging_manager.configure_logging(utils_misc.VirtLoggingConfig())
if args:
filenames = args
else:
parser.print_help()
sys.exit(1)
for filename in filenames:
filename = os.path.abspath(filename)
file_exists = os.path.isfile(filename)
can_read_file = os.access(filename, os.R_OK)
if not file_exists:
logging.critical("File %s does not exist!", filename)
continue
"""
Populate/update config files for virt-test
@copyright: Red Hat 2013
"""
import os, sys
import common
from autotest.client.shared import logging_manager
from virttest import data_dir, bootstrap, utils_misc
test_dir = os.path.dirname(sys.modules[__name__].__file__)
test_dir = os.path.abspath(test_dir)
t_type = os.path.basename(test_dir)
shared_dir = os.path.join(data_dir.get_root_dir(), "shared")
if __name__ == "__main__":
import optparse
option_parser = optparse.OptionParser()
option_parser.add_option("-v", "--verbose",
action="store_true", dest="verbose",
help="Exhibit debug messages")
options, args = option_parser.parse_args()
if options.verbose:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=options.verbose)
bootstrap.create_config_files(test_dir, shared_dir, interactive=False,
force_update=True)
bootstrap.create_subtests_cfg(t_type)
bootstrap.create_guest_os_cfg(t_type)
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths,
check_modules, online_docs_url, restore_image=False,
download_image=True, interactive=True, verbose=False):
"""
Common virt test assistant module.
:param test_name: Test name, such as "qemu".
:param test_dir: Path with the test directory.
:param base_dir: Base directory used to hold images and isos.
:param default_userspace_paths: Important programs for a successful test
execution.
:param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
:param online_docs_url: URL to an online documentation system, such as a
wiki page.
:param restore_image: Whether to restore the image from the pristine.
:param interactive: Whether to ask for confirmation.
:raise error.CmdError: If JeOS image failed to uncompress
:raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data", "gpg"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating",
sub_dir_path)
# lvsb test doesn't use any shared configs
if test_name == 'lvsb':
create_subtests_cfg(test_name)
else:
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if download_image or restore_image:
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
asset.download_asset('jeos-19-64', interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning("Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info("%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
help='Answer yes to all questions')
options, args = parser.parse_args()
local_patch = options.local_patch
pw_id = options.pw_id
gh_id = options.gh_id
debug = options.debug
run_pylint.set_verbosity(debug)
full_check = options.full_check
confirm = options.confirm
pwhost = options.patchwork_host
vcs = VCS()
if vcs.backend is None:
vcs = None
logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
extension_blacklist = [
"common.py", ".java", ".html", ".png", ".css", ".xml", ".pyc", ".orig",
".rej", ".bak", ".so"
]
dir_blacklist = [
".svn", ".git", "logs", "virt", "site-packages", "ExternalSource"
]
if full_check:
failed_paths = []
run_pylint.set_verbosity(False)
logging.info("Autotest full tree check")
logging.info("")
if os.path.isfile("tko/Makefile"):
proto_cmd = "make -C tko"
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths,
check_modules, online_docs_url, restore_image=False,
download_image=True, interactive=True, selinux=False,
verbose=False, update_providers=False,
guest_os=defaults.DEFAULT_GUEST_OS):
"""
Common virt test assistant module.
:param test_name: Test name, such as "qemu".
:param test_dir: Path with the test directory.
:param base_dir: Base directory used to hold images and isos.
:param default_userspace_paths: Important programs for a successful test
execution.
:param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
:param online_docs_url: URL to an online documentation system, such as a
wiki page.
:param restore_image: Whether to restore the image from the pristine.
:param interactive: Whether to ask for confirmation.
:param verbose: Verbose output.
:param selinux: Whether setup SELinux contexts for shared/data.
:param update_providers: Whether to update test providers if they are already
downloaded.
:param guest_os: Specify the guest image used for bootstrapping. By default
the JeOS image is used.
:raise error.CmdError: If JeOS image failed to uncompress
:raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Updating all test providers", step)
asset.download_all_test_providers(update_providers)
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data", "gpg"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating",
sub_dir_path)
datadir = data_dir.get_data_dir()
if test_name == 'libvirt':
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
# Don't bother checking if changes can't be made
if os.getuid() == 0:
verify_selinux(datadir,
os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(),
interactive, selinux)
# lvsb test doesn't use any shared configs
elif test_name == 'lvsb':
create_subtests_cfg(test_name)
if os.getuid() == 0:
# Don't bother checking if changes can't be made
verify_selinux(datadir,
os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(),
interactive, selinux)
else: # Some other test
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if download_image or restore_image:
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
for os_info in get_guest_os_info_list(test_name, guest_os):
os_asset = os_info['asset']
asset.download_asset(os_asset, interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning("Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info("%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
import common
from autotest.client.shared import logging_manager
from autotest.client.virt import virt_utils
def destroy_tap(tapfd_list):
for tapfd in tapfd_list:
try:
os.close(tapfd)
# File descriptor is already closed
except OSError:
pass
if __name__ == "__main__":
logging_manager.configure_logging(virt_utils.VirtLoggingConfig(),
verbose=True)
if len(sys.argv) <= 2:
logging.info("Usage: %s bridge_name qemu_command_line", sys.argv[0])
sys.exit(255)
brname = sys.argv[1]
cmd_line = ' '.join(sys.argv[2:])
if re.findall("-netdev\s", cmd_line):
# so we get the new qemu cli with netdev parameter.
tap_list_re = r"tap,id=(.*?),"
tap_replace_re = r"(tap,id=%s.*?,fd=)\d+"
else:
# the old cli contain "-net" parameter.
tap_list_re = r"tap,vlan=(\d+),"
tap_replace_re = r"(tap,vlan=%s,fd=)\d+"
import os
import sys
import common
from autotest.client.shared import logging_manager
from virttest import data_dir, bootstrap, utils_misc
test_dir = os.path.dirname(sys.modules[__name__].__file__)
test_dir = os.path.abspath(test_dir)
t_type = os.path.basename(test_dir)
shared_dir = os.path.join(data_dir.get_root_dir(), "shared")
if __name__ == "__main__":
import optparse
option_parser = optparse.OptionParser()
option_parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="Exhibit debug messages")
options, args = option_parser.parse_args()
if options.verbose:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=options.verbose)
bootstrap.create_config_files(test_dir,
shared_dir,
interactive=False,
force_update=True)
bootstrap.create_subtests_cfg(t_type)
bootstrap.create_guest_os_cfg(t_type)
else:
prefix = ''
path = log_path
return prefix + path
def retrieve_logs(path):
host = find_repository_host(path)
if host is None:
logging.info('No special host was found holding the results')
# It's not clear what was intended here. Maybe some custom action to
# fetch/unpack the logs?
site_retrieve_logs(path)
results_url = get_full_url(host, path)
logging.info('Results url: %s', results_url)
return results_url
if __name__ == '__main__':
logging_manager.configure_logging(LoggingConfig(), verbose=True)
if len(sys.argv) <= 1:
logging.error('Usage: %s [log_path]', sys.argv[0])
raise SystemExit
path = sys.argv[1]
retrieve_logs(path)
result = software_manager.install(text)
return result
if __name__ == '__main__':
parser = optparse.OptionParser(
"usage: %prog [install|remove|check-installed|list-all|list-files|add-repo|"
"remove-repo| upgrade|what-provides|install-what-provides] arguments")
parser.add_option('--verbose',
dest="debug",
action='store_true',
help='include debug messages in console output')
options, args = parser.parse_args()
debug = options.debug
logging_manager.configure_logging(SoftwareManagerLoggingConfig(),
verbose=debug)
software_manager = SoftwareManager()
if args:
action = args[0]
args = " ".join(args[1:])
else:
action = 'show-help'
if action == 'install':
if software_manager.install(args):
logging.info("Packages %s installed successfuly", args)
else:
logging.error("Failed to install %s", args)
elif action == 'remove':
if software_manager.remove(args):
text = " ".join(needed_pkgs)
logging.info('Installing packages "%s"', text)
result = software_manager.install(text)
return result
if __name__ == "__main__":
parser = optparse.OptionParser(
"usage: %prog [install|remove|check-installed|list-all|list-files|add-repo|"
"remove-repo| upgrade|what-provides|install-what-provides] arguments"
)
parser.add_option("--verbose", dest="debug", action="store_true", help="include debug messages in console output")
options, args = parser.parse_args()
debug = options.debug
logging_manager.configure_logging(SoftwareManagerLoggingConfig(), verbose=debug)
software_manager = SoftwareManager()
if args:
action = args[0]
args = " ".join(args[1:])
else:
action = "show-help"
if action == "install":
if software_manager.install(args):
logging.info("Packages %s installed successfuly", args)
else:
logging.error("Failed to install %s", args)
elif action == "remove":
if software_manager.remove(args):
def main(argv):
"""Main function"""
global DRY_RUN
parser = optparse.OptionParser()
parser.add_option(
"-c",
"--db-clean-tests",
dest="clean_tests",
action="store_true",
default=False,
help="Clean client and server tests with invalid control files",
)
parser.add_option(
"-C",
"--db-clear-all-tests",
dest="clear_all_tests",
action="store_true",
default=False,
help="Clear ALL client and server tests",
)
parser.add_option(
"-d", "--dry-run", dest="dry_run", action="store_true", default=False, help="Dry run for operation"
)
parser.add_option(
"-A",
"--add-all",
dest="add_all",
action="store_true",
default=False,
help="Add site_tests, tests, and test_suites",
)
parser.add_option(
"-S", "--add-samples", dest="add_samples", action="store_true", default=False, help="Add samples."
)
parser.add_option(
"-E",
"--add-experimental",
dest="add_experimental",
action="store_true",
default=True,
help="Add experimental tests to frontend",
)
parser.add_option(
"-N",
"--add-noncompliant",
dest="add_noncompliant",
action="store_true",
default=False,
help="Add non-compliant tests (i.e. tests that do not " "define all required control variables)",
)
parser.add_option("-p", "--profile-dir", dest="profile_dir", help="Directory to recursively check for profiles")
parser.add_option("-t", "--tests-dir", dest="tests_dir", help="Directory to recursively check for control.*")
parser.add_option(
"-r",
"--control-pattern",
dest="control_pattern",
default="^control.*",
help="The pattern to look for in directories for control files",
)
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Run in verbose mode")
parser.add_option(
"-w", "--whitelist-file", dest="whitelist_file", help="Filename for list of test names that must match"
)
parser.add_option(
"-z",
"--autotest-dir",
dest="autotest_dir",
default=os.path.join(os.path.dirname(__file__), ".."),
help="Autotest directory root, or base test directory",
)
options, args = parser.parse_args()
logging_manager.configure_logging(TestImporterLoggingConfig(), verbose=options.verbose)
DRY_RUN = options.dry_run
if DRY_RUN:
logging.getLogger().setLevel(logging.WARN)
# Make sure autotest_dir is the absolute path
options.autotest_dir = os.path.abspath(options.autotest_dir)
if len(args) > 0:
logging.error("Invalid option(s) provided: %s", args)
parser.print_help()
return 1
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
if len(argv) == 1 or (len(argv) == 2 and options.verbose):
update_all(options.autotest_dir, options.add_noncompliant, options.add_experimental)
db_clean_broken(options.autotest_dir)
return 0
if options.clear_all_tests:
if options.clean_tests or options.add_all or options.add_samples or options.add_noncompliant:
logging.error("Can only pass --autotest-dir, --dry-run and --verbose with " "--db-clear-all-tests")
return 1
db_clean_all(options.autotest_dir)
whitelist_set = None
if options.whitelist_file:
if options.add_all:
logging.error("Cannot pass both --add-all and --whitelist-file")
return 1
whitelist_path = os.path.abspath(options.whitelist_file)
if not os.path.isfile(whitelist_path):
logging.error("--whitelist-file (%s) not found", whitelist_path)
return 1
logging.info("Using whitelist file %s", whitelist_path)
whitelist_set = _create_whitelist_set(whitelist_path)
update_from_whitelist(
whitelist_set,
add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir,
)
if options.add_all:
update_all(options.autotest_dir, options.add_noncompliant, options.add_experimental)
if options.add_samples:
update_samples(options.autotest_dir, options.add_noncompliant, options.add_experimental)
if options.tests_dir:
options.tests_dir = os.path.abspath(options.tests_dir)
tests = get_tests_from_fs(options.tests_dir, options.control_pattern, add_noncompliant=options.add_noncompliant)
update_tests_in_db(
tests,
add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir,
)
if options.profile_dir:
profilers = get_tests_from_fs(options.profile_dir, ".*py$")
update_profilers_in_db(profilers, add_noncompliant=options.add_noncompliant, description="NA")
if options.clean_tests:
db_clean_broken(options.autotest_dir)
def bootstrap(test_name,
test_dir,
base_dir,
default_userspace_paths,
check_modules,
online_docs_url,
restore_image=False,
interactive=True,
selinux=False,
verbose=False,
update_providers=False,
guest_os=defaults.DEFAULT_GUEST_OS,
force_update=False):
"""
Common virt test assistant module.
:param test_name: Test name, such as "qemu".
:param test_dir: Path with the test directory.
:param base_dir: Base directory used to hold images and isos.
:param default_userspace_paths: Important programs for a successful test
execution.
:param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
:param online_docs_url: URL to an online documentation system, such as a
wiki page.
:param restore_image: Whether to restore the image from the pristine.
:param interactive: Whether to ask for confirmation.
:param verbose: Verbose output.
:param selinux: Whether setup SELinux contexts for shared/data.
:param update_providers: Whether to update test providers if they are already
downloaded.
:param guest_os: Specify the guest image used for bootstrapping. By default
the JeOS image is used.
:raise error.CmdError: If JeOS image failed to uncompress
:raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Updating all test providers", step)
asset.download_all_test_providers(update_providers)
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name, guest_os)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data", "gpg"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating", sub_dir_path)
datadir = data_dir.get_data_dir()
if test_name == 'libvirt':
create_config_files(test_dir, shared_dir, interactive, step,
force_update)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
# Don't bother checking if changes can't be made
if os.getuid() == 0:
verify_selinux(datadir, os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(), interactive, selinux)
# lvsb test doesn't use any shared configs
elif test_name == 'lvsb':
create_subtests_cfg(test_name)
if os.getuid() == 0:
# Don't bother checking if changes can't be made
verify_selinux(datadir, os.path.join(datadir, 'images'),
os.path.join(datadir, 'isos'),
data_dir.get_tmp_dir(), interactive, selinux)
else: # Some other test
create_config_files(test_dir, shared_dir, interactive, step,
force_update)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if restore_image:
logging.info("")
step += 1
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
for os_info in get_guest_os_info_list(test_name, guest_os):
os_asset = os_info['asset']
asset.download_asset(os_asset,
interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning(
"Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info(
"%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
def bootstrap(test_name,
test_dir,
base_dir,
default_userspace_paths,
check_modules,
online_docs_url,
restore_image=False,
interactive=True,
verbose=False):
"""
Common virt test assistant module.
@param test_name: Test name, such as "kvm".
@param test_dir: Path with the test directory.
@param base_dir: Base directory used to hold images and isos.
@param default_userspace_paths: Important programs for a successful test
execution.
@param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
@param online_docs_url: URL to an online documentation system, such as a
wiki page.
@param restore_image: Whether to restore the image from the pristine.
@param interactive: Whether to ask for confirmation.
@raise error.CmdError: If JeOS image failed to uncompress
@raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
shared_dir = os.path.join(shared_dir, "cfg")
sub_dir_list = ["images", "isos", "steps_data"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating" % sub_dir_path)
create_config_files(test_dir, shared_dir, interactive, step)
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image", step)
sha1_file = "SHA1SUM"
guest_tarball = "jeos-17-64.qcow2.7z"
base_location = "http://lmr.fedorapeople.org/jeos/"
url = os.path.join(base_location, guest_tarball)
tarball_sha1_url = os.path.join(base_location, sha1_file)
destination = os.path.join(base_dir, 'images')
uncompressed_file_path = os.path.join(base_dir, 'images',
'jeos-17-64.qcow2')
uncompressed_file_exists = os.path.isfile(uncompressed_file_path)
if (interactive
and not os.path.isfile(os.path.join(destination, guest_tarball))):
answer = utils.ask("Minimal basic guest image (JeOS) not present. "
"Do you want to download it (~ 180MB)?")
else:
answer = "y"
if answer == "y":
had_to_download = download_file(url,
destination,
tarball_sha1_url,
title="Downloading JeOS x86_64",
interactive=interactive)
restore_image = (restore_image or had_to_download
or not uncompressed_file_exists)
tarball_path = os.path.join(destination, guest_tarball)
if os.path.isfile(tarball_path) and restore_image:
os.chdir(destination)
utils.run("7za -y e %s" % tarball_path)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning(
"Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info(
"%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
"""
import os, sys, optparse, logging
try:
import autotest.common as common
except ImportError:
import common
from autotest.client.shared import logging_manager
from autotest.client import utils
from autotest.client.virt import virt_utils
if __name__ == "__main__":
parser = optparse.OptionParser("usage: %prog [options] [filenames]")
options, args = parser.parse_args()
logging_manager.configure_logging(virt_utils.VirtLoggingConfig())
if args:
filenames = args
else:
parser.print_help()
sys.exit(1)
for filename in filenames:
filename = os.path.abspath(filename)
file_exists = os.path.isfile(filename)
can_read_file = os.access(filename, os.R_OK)
if not file_exists:
logging.critical("File %s does not exist!", filename)
continue
if self.active:
self.plot_2d_graphs()
self.plot_3d_graphs()
class AnalyzerLoggingConfig(logging_config.LoggingConfig):
def configure_logging(self, results_dir=None, verbose=False):
super(AnalyzerLoggingConfig, self).configure_logging(use_console=True,
verbose=verbose)
if __name__ == "__main__":
parser = optparse.OptionParser("usage: %prog [options] [filenames]")
options, args = parser.parse_args()
logging_manager.configure_logging(AnalyzerLoggingConfig())
if args:
filenames = args
else:
parser.print_help()
sys.exit(1)
if len(args) > 2:
parser.print_help()
sys.exit(1)
o = os.path.join(os.getcwd(),
"iozone-graphs-%s" % time.strftime('%Y-%m-%d-%H.%M.%S'))
if not os.path.isdir(o):
os.makedirs(o)
for k in asset_keys:
logging.info(" %s = %s" % (k, asset_info[k]))
logging.info("")
indexes = raw_input("%s INFO | Type the index for the assets you want to "
"download (comma separated, Ctrl+C to abort): " %
time.strftime("%H:%M:%S", time.localtime()))
index_list = []
for idx in indexes.split(","):
try:
index = int(idx) - 1
index_list.append(index)
all_assets[index]
except (ValueError, IndexError):
logging.error("Invalid index(es), aborting...")
sys.exit(1)
for idx in index_list:
asset_info = all_assets[idx]
asset.download_file(asset_info, interactive=True)
if __name__ == "__main__":
logging_manager.configure_logging(utils_misc.VirtLoggingConfig())
try:
download_assets()
except KeyboardInterrupt:
print
logging.info("Aborting...")
sys.exit(0)
cmd += " --recursive --in-place"
else:
cmd += " --in-place"
cmd += " %s" % path
return utils.system(cmd, ignore_status=True) == 0
def fix(path):
if DEPS_SATISFIED:
_fix(path)
else:
raise SetupError(E_MSG)
if __name__ == "__main__":
logging_manager.configure_logging(Pep8LoggingConfig(),
verbose=True)
import optparse
usage = "usage: %prog [options] [list of files]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("-f", "--fix",
action="store_true", dest="fix",
help="Auto fix files in place")
options, args = parser.parse_args()
if options.fix:
func = fix
else:
func = check
if len(args) > 0:
for path_to_check in args:
if self.active:
self.plot_2d_graphs()
self.plot_3d_graphs()
class AnalyzerLoggingConfig(logging_config.LoggingConfig):
def configure_logging(self, results_dir=None, verbose=False):
super(AnalyzerLoggingConfig, self).configure_logging(use_console=True,
verbose=verbose)
if __name__ == "__main__":
parser = optparse.OptionParser("usage: %prog [options] [filenames]")
options, args = parser.parse_args()
logging_manager.configure_logging(AnalyzerLoggingConfig())
if args:
filenames = args
else:
parser.print_help()
sys.exit(1)
if len(args) > 2:
parser.print_help()
sys.exit(1)
o = os.path.join(os.getcwd(),
"iozone-graphs-%s" % time.strftime('%Y-%m-%d-%H.%M.%S'))
if not os.path.isdir(o):
os.makedirs(o)
def bootstrap(test_name, test_dir, base_dir, default_userspace_paths,
check_modules, online_docs_url, restore_image=False,
interactive=True, verbose=False):
"""
Common virt test assistant module.
@param test_name: Test name, such as "kvm".
@param test_dir: Path with the test directory.
@param base_dir: Base directory used to hold images and isos.
@param default_userspace_paths: Important programs for a successful test
execution.
@param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
@param online_docs_url: URL to an online documentation system, such as a
wiki page.
@param restore_image: Whether to restore the image from the pristine.
@param interactive: Whether to ask for confirmation.
@raise error.CmdError: If JeOS image failed to uncompress
@raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
shared_dir = os.path.join(shared_dir, "cfg")
sub_dir_list = ["images", "isos", "steps_data"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating" %
sub_dir_path)
create_config_files(test_dir, shared_dir, interactive, step)
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image", step)
sha1_file = "SHA1SUM"
guest_tarball = "jeos-17-64.qcow2.7z"
base_location = "http://lmr.fedorapeople.org/jeos/"
url = os.path.join(base_location, guest_tarball)
tarball_sha1_url = os.path.join(base_location, sha1_file)
destination = os.path.join(base_dir, 'images')
uncompressed_file_path = os.path.join(base_dir, 'images',
'jeos-17-64.qcow2')
uncompressed_file_exists = os.path.isfile(uncompressed_file_path)
if (interactive and not
os.path.isfile(os.path.join(destination, guest_tarball))):
answer = utils.ask("Minimal basic guest image (JeOS) not present. "
"Do you want to download it (~ 180MB)?")
else:
answer = "y"
if answer == "y":
had_to_download = download_file(url, destination, tarball_sha1_url,
title="Downloading JeOS x86_64",
interactive=interactive)
restore_image = (restore_image or had_to_download or not
uncompressed_file_exists)
tarball_path = os.path.join(destination, guest_tarball)
if os.path.isfile(tarball_path) and restore_image:
os.chdir(destination)
utils.run("7za -y e %s" % tarball_path)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning("Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info("%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
def main():
# grab the parser
parser = autoserv_parser.autoserv_parser
parser.parse_args()
if len(sys.argv) == 1:
parser.parser.print_help()
sys.exit(1)
if parser.options.no_logging:
results = None
else:
output_dir = settings.get_value('COMMON', 'test_output_dir', default="")
results = parser.options.results
if not results:
results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
if output_dir:
results = os.path.join(output_dir, results)
results = os.path.abspath(results)
resultdir_exists = False
for filename in ('control.srv', 'status.log', '.autoserv_execute'):
if os.path.exists(os.path.join(results, filename)):
resultdir_exists = True
if not parser.options.use_existing_results and resultdir_exists:
error = "Error: results directory already exists: %s\n" % results
sys.stderr.write(error)
sys.exit(1)
# Now that we certified that there's no leftover results dir from
# previous jobs, lets create the result dir since the logging system
# needs to create the log file in there.
if not os.path.isdir(results):
os.makedirs(results)
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(), results_dir=results,
use_console=not parser.options.no_tee,
verbose=parser.options.verbose,
no_console_prefix=parser.options.no_console_prefix)
if results:
logging.info("Results placed in %s" % results)
# wait until now to perform this check, so it get properly logged
if parser.options.use_existing_results and not resultdir_exists:
logging.error("No existing results directory found: %s", results)
sys.exit(1)
if parser.options.write_pidfile:
pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
results)
pid_file_manager.open_file()
else:
pid_file_manager = None
autotest_remote.BaseAutotest.set_install_in_tmpdir(
parser.options.install_in_tmpdir)
exit_code = 0
try:
try:
run_autoserv(pid_file_manager, results, parser)
except SystemExit, e:
exit_code = e.code
except:
traceback.print_exc()
# If we don't know what happened, we'll classify it as
# an 'abort' and return 1.
exit_code = 1
help='Answer yes to all questions')
options, args = parser.parse_args()
local_patch = options.local_patch
pw_id = options.pw_id
gh_id = options.gh_id
debug = options.debug
run_pylint.set_verbosity(debug)
full_check = options.full_check
confirm = options.confirm
pwhost = options.patchwork_host
vcs = VCS()
if vcs.backend is None:
vcs = None
logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
logging.info("Detected project name: %s", PROJECT_NAME)
logging.info("Log saved to file: %s", LOG_FILE_PATH)
extension_blacklist = EXTENSION_BLACKLIST.get(PROJECT_NAME, [])
dir_blacklist = DIR_BLACKLIST.get(PROJECT_NAME, [])
if full_check:
failed_paths = []
run_pylint.set_verbosity(False)
logging.info("%s full tree check", PROJECT_NAME)
logging.info("")
if PROJECT_NAME == 'autotest' and os.path.isfile("tko/Makefile"):
proto_cmd = "make -C tko"
try:
if pid > 0:
sys.exit(0) # exit from first parent
except OSError, e:
sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
# Decouple from parent environment
os.chdir("/")
os.umask(0)
os.setsid()
# Second fork
try:
pid = os.fork()
if pid > 0:
sys.exit(0) # exit from second parent
except OSError, e:
sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
else:
logging_manager.configure_logging(watcher_logging_config.WatcherLoggingConfig())
while True:
proc = MonitorProc(do_recovery=recover)
proc.start()
time.sleep(PAUSE_LENGTH)
while proc.is_running():
logging.info("Tick")
time.sleep(PAUSE_LENGTH)
recover = False
sys.stderr.write("fork #1 failed: (%d) %s\n" %
(e.errno, e.strerror))
sys.exit(1)
# Decouple from parent environment
os.chdir("/")
os.umask(0)
os.setsid()
# Second fork
try:
pid = os.fork()
if (pid > 0):
sys.exit(0) # exit from second parent
except OSError, e:
sys.stderr.write("fork #2 failed: (%d) %s\n" %
(e.errno, e.strerror))
sys.exit(1)
else:
logging_manager.configure_logging(
watcher_logging_config.WatcherLoggingConfig())
while True:
proc = MonitorProc(do_recovery=recover)
proc.start()
time.sleep(PAUSE_LENGTH)
while proc.is_running():
logging.info("Tick")
time.sleep(PAUSE_LENGTH)
recover = False
import common
from autotest.client.shared import logging_manager
from virttest import utils_net, utils_misc
def destroy_tap(tapfd_list):
for tapfd in tapfd_list:
try:
os.close(tapfd)
# File descriptor is already closed
except OSError:
pass
if __name__ == "__main__":
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=True)
if len(sys.argv) <= 2:
logging.info("Usage: %s bridge_name qemu_command_line", sys.argv[0])
sys.exit(255)
brname = sys.argv[1]
cmd_line = ' '.join(sys.argv[2:])
if re.findall("-netdev\s", cmd_line):
# so we get the new qemu cli with netdev parameter.
tap_list_re = r"tap,id=(.*?),"
tap_replace_re = r"(tap,id=%s.*?,fd=)\d+"
else:
# the old cli contain "-net" parameter.
tap_list_re = r"tap,vlan=(\d+),"
tap_replace_re = r"(tap,vlan=%s,fd=)\d+"
def main(argv):
"""Main function"""
global DRY_RUN
parser = optparse.OptionParser()
parser.add_option(
'-c',
'--db-clean-tests',
dest='clean_tests',
action='store_true',
default=False,
help='Clean client and server tests with invalid control files')
parser.add_option('-C',
'--db-clear-all-tests',
dest='clear_all_tests',
action='store_true',
default=False,
help='Clear ALL client and server tests')
parser.add_option('-d',
'--dry-run',
dest='dry_run',
action='store_true',
default=False,
help='Dry run for operation')
parser.add_option('-A',
'--add-all',
dest='add_all',
action='store_true',
default=False,
help='Add site_tests, tests, and test_suites')
parser.add_option('-S',
'--add-samples',
dest='add_samples',
action='store_true',
default=False,
help='Add samples.')
parser.add_option('-E',
'--add-experimental',
dest='add_experimental',
action='store_true',
default=True,
help='Add experimental tests to frontend')
parser.add_option('-N',
'--add-noncompliant',
dest='add_noncompliant',
action='store_true',
default=False,
help='Add non-compliant tests (i.e. tests that do not '
'define all required control variables)')
parser.add_option('-p',
'--profile-dir',
dest='profile_dir',
help='Directory to recursively check for profiles')
parser.add_option('-t',
'--tests-dir',
dest='tests_dir',
help='Directory to recursively check for control.*')
parser.add_option(
'-r',
'--control-pattern',
dest='control_pattern',
default='^control.*',
help='The pattern to look for in directories for control files')
parser.add_option('-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help='Run in verbose mode')
parser.add_option('-w',
'--whitelist-file',
dest='whitelist_file',
help='Filename for list of test names that must match')
parser.add_option('-z',
'--autotest-dir',
dest='autotest_dir',
default=os.path.join(os.path.dirname(__file__), '..'),
help='Autotest directory root, or base test directory')
options, args = parser.parse_args()
logging_manager.configure_logging(TestImporterLoggingConfig(),
verbose=options.verbose)
DRY_RUN = options.dry_run
if DRY_RUN:
logging.getLogger().setLevel(logging.WARN)
# Make sure autotest_dir is the absolute path
options.autotest_dir = os.path.abspath(options.autotest_dir)
if len(args) > 0:
logging.error("Invalid option(s) provided: %s", args)
parser.print_help()
return 1
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
if len(argv) == 1 or (len(argv) == 2 and options.verbose):
update_all(options.autotest_dir, options.add_noncompliant,
options.add_experimental)
db_clean_broken(options.autotest_dir)
return 0
if options.clear_all_tests:
if (options.clean_tests or options.add_all or options.add_samples
or options.add_noncompliant):
logging.error(
"Can only pass --autotest-dir, --dry-run and --verbose with "
"--db-clear-all-tests")
return 1
db_clean_all(options.autotest_dir)
whitelist_set = None
if options.whitelist_file:
if options.add_all:
logging.error("Cannot pass both --add-all and --whitelist-file")
return 1
whitelist_path = os.path.abspath(options.whitelist_file)
if not os.path.isfile(whitelist_path):
logging.error("--whitelist-file (%s) not found", whitelist_path)
return 1
logging.info("Using whitelist file %s", whitelist_path)
whitelist_set = _create_whitelist_set(whitelist_path)
update_from_whitelist(whitelist_set,
add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir)
if options.add_all:
update_all(options.autotest_dir, options.add_noncompliant,
options.add_experimental)
if options.add_samples:
update_samples(options.autotest_dir, options.add_noncompliant,
options.add_experimental)
if options.tests_dir:
options.tests_dir = os.path.abspath(options.tests_dir)
tests = get_tests_from_fs(options.tests_dir,
options.control_pattern,
add_noncompliant=options.add_noncompliant)
update_tests_in_db(tests,
add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir)
if options.profile_dir:
profilers = get_tests_from_fs(options.profile_dir, '.*py$')
update_profilers_in_db(profilers,
add_noncompliant=options.add_noncompliant,
description='NA')
if options.clean_tests:
db_clean_broken(options.autotest_dir)
def main(argv):
"""Main function"""
global DRY_RUN
parser = optparse.OptionParser()
parser.add_option('-c', '--db-clean-tests',
dest='clean_tests', action='store_true',
default=False,
help='Clean client and server tests with invalid control files')
parser.add_option('-C', '--db-clear-all-tests',
dest='clear_all_tests', action='store_true',
default=False,
help='Clear ALL client and server tests')
parser.add_option('-d', '--dry-run',
dest='dry_run', action='store_true', default=False,
help='Dry run for operation')
parser.add_option('-A', '--add-all',
dest='add_all', action='store_true',
default=False,
help='Add site_tests, tests, and test_suites')
parser.add_option('-S', '--add-samples',
dest='add_samples', action='store_true',
default=False,
help='Add samples.')
parser.add_option('-E', '--add-experimental',
dest='add_experimental', action='store_true',
default=True,
help='Add experimental tests to frontend')
parser.add_option('-N', '--add-noncompliant',
dest='add_noncompliant', action='store_true',
default=False,
help='Add non-compliant tests (i.e. tests that do not '
'define all required control variables)')
parser.add_option('-p', '--profile-dir', dest='profile_dir',
help='Directory to recursively check for profiles')
parser.add_option('-t', '--tests-dir', dest='tests_dir',
help='Directory to recursively check for control.*')
parser.add_option('-r', '--control-pattern', dest='control_pattern',
default='^control.*',
help='The pattern to look for in directories for control files')
parser.add_option('-v', '--verbose',
dest='verbose', action='store_true', default=False,
help='Run in verbose mode')
parser.add_option('-w', '--whitelist-file', dest='whitelist_file',
help='Filename for list of test names that must match')
parser.add_option('-z', '--autotest-dir', dest='autotest_dir',
default=os.path.join(os.path.dirname(__file__), '..'),
help='Autotest directory root, or base test directory')
options, args = parser.parse_args()
logging_manager.configure_logging(TestImporterLoggingConfig(),
verbose=options.verbose)
DRY_RUN = options.dry_run
if DRY_RUN:
logging.getLogger().setLevel(logging.WARN)
# Make sure autotest_dir is the absolute path
options.autotest_dir = os.path.abspath(options.autotest_dir)
if len(args) > 0:
logging.error("Invalid option(s) provided: %s", args)
parser.print_help()
return 1
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
if len(argv) == 1 or (len(argv) == 2 and options.verbose):
update_all(options.autotest_dir, options.add_noncompliant,
options.add_experimental)
db_clean_broken(options.autotest_dir)
return 0
if options.clear_all_tests:
if (options.clean_tests or options.add_all or options.add_samples or
options.add_noncompliant):
logging.error(
"Can only pass --autotest-dir, --dry-run and --verbose with "
"--db-clear-all-tests")
return 1
db_clean_all(options.autotest_dir)
whitelist_set = None
if options.whitelist_file:
if options.add_all:
logging.error("Cannot pass both --add-all and --whitelist-file")
return 1
whitelist_path = os.path.abspath(options.whitelist_file)
if not os.path.isfile(whitelist_path):
logging.error("--whitelist-file (%s) not found", whitelist_path)
return 1
logging.info("Using whitelist file %s", whitelist_path)
whitelist_set = _create_whitelist_set(whitelist_path)
update_from_whitelist(whitelist_set,
add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir)
if options.add_all:
update_all(options.autotest_dir, options.add_noncompliant,
options.add_experimental)
if options.add_samples:
update_samples(options.autotest_dir, options.add_noncompliant,
options.add_experimental)
if options.tests_dir:
options.tests_dir = os.path.abspath(options.tests_dir)
tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
add_noncompliant=options.add_noncompliant)
update_tests_in_db(tests, add_experimental=options.add_experimental,
add_noncompliant=options.add_noncompliant,
autotest_dir=options.autotest_dir)
if options.profile_dir:
profilers = get_tests_from_fs(options.profile_dir, '.*py$')
update_profilers_in_db(profilers,
add_noncompliant=options.add_noncompliant,
description='NA')
if options.clean_tests:
db_clean_broken(options.autotest_dir)
def bootstrap(test_name,
test_dir,
base_dir,
default_userspace_paths,
check_modules,
online_docs_url,
restore_image=False,
download_image=True,
interactive=True,
verbose=False):
"""
Common virt test assistant module.
@param test_name: Test name, such as "qemu".
@param test_dir: Path with the test directory.
@param base_dir: Base directory used to hold images and isos.
@param default_userspace_paths: Important programs for a successful test
execution.
@param check_modules: Whether we want to verify if a given list of modules
is loaded in the system.
@param online_docs_url: URL to an online documentation system, such as a
wiki page.
@param restore_image: Whether to restore the image from the pristine.
@param interactive: Whether to ask for confirmation.
@raise error.CmdError: If JeOS image failed to uncompress
@raise ValueError: If 7za was not found
"""
if interactive:
logging_manager.configure_logging(utils_misc.VirtLoggingConfig(),
verbose=verbose)
logging.info("%s test config helper", test_name)
step = 0
logging.info("")
step += 1
logging.info("%d - Checking the mandatory programs and headers", step)
verify_mandatory_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Checking the recommended programs", step)
verify_recommended_programs(test_name)
logging.info("")
step += 1
logging.info("%d - Verifying directories", step)
shared_dir = os.path.dirname(data_dir.get_data_dir())
sub_dir_list = ["images", "isos", "steps_data"]
for sub_dir in sub_dir_list:
sub_dir_path = os.path.join(base_dir, sub_dir)
if not os.path.isdir(sub_dir_path):
logging.debug("Creating %s", sub_dir_path)
os.makedirs(sub_dir_path)
else:
logging.debug("Dir %s exists, not creating", sub_dir_path)
create_config_files(test_dir, shared_dir, interactive, step)
create_subtests_cfg(test_name)
create_guest_os_cfg(test_name)
if download_image or restore_image:
logging.info("")
step += 2
logging.info("%s - Verifying (and possibly downloading) guest image",
step)
asset.download_asset('jeos-17-64',
interactive=interactive,
restore_image=restore_image)
if check_modules:
logging.info("")
step += 1
logging.info("%d - Checking for modules %s", step,
", ".join(check_modules))
for module in check_modules:
if not utils.module_is_loaded(module):
logging.warning(
"Module %s is not loaded. You might want to "
"load it", module)
else:
logging.debug("Module %s loaded", module)
if online_docs_url:
logging.info("")
step += 1
logging.info(
"%d - If you wish, take a look at the online docs for "
"more info", step)
logging.info("")
logging.info(online_docs_url)
Used with the sole purpose of providing convenient logging setup
for this program.
"""
def configure_logging(self, results_dir=None, verbose=False):
super(ReportLoggingConfig, self).configure_logging(use_console=True,
verbose=verbose)
ERROR_INVALID_RESULT_DIR = 1
ERROR_INVALID_REPORT_PATH = 2
ERROR_WRONG_INPUT = 3
if __name__ == "__main__":
logging_manager.configure_logging(ReportLoggingConfig(), verbose=True)
parser = ReportOptionParser()
options, args = parser.parse_args()
if args:
parser.print_help()
sys.exit(ERROR_WRONG_INPUT)
try:
write_html_report(results_dir=options.results_dir,
report_path=options.report_path,
encoding=options.encoding)
except InvalidAutotestResultDirError, e:
logging.error(e)
sys.exit(ERROR_INVALID_RESULT_DIR)
except InvalidOutputDirError, e:
import os, sys, optparse, logging
try:
import autotest.common as common
except ImportError:
import common
from autotest.client.shared import logging_manager
from autotest.client import utils
from autotest.client.virt import virt_utils
if __name__ == "__main__":
parser = optparse.OptionParser("usage: %prog [options] [filenames]")
options, args = parser.parse_args()
logging_manager.configure_logging(virt_utils.VirtLoggingConfig())
if args:
filenames = args
else:
parser.print_help()
sys.exit(1)
for filename in filenames:
filename = os.path.abspath(filename)
file_exists = os.path.isfile(filename)
can_read_file = os.access(filename, os.R_OK)
if not file_exists:
logging.critical("File %s does not exist!", filename)
continue
