def setup_tests(options):
"""
Load and instantiate all client tests.
This function is inspired from runtest() on client/common_lib/test.py.
@param options: an object passed in from command line OptionParser.
See all options defined on client/bin/autotest.
"""
assert options.client_test_setup, 'Specify prebuild client tests on the ' \
'command line.'
requested_tests = options.client_test_setup.split(',')
candidates, broken_tests = load_all_client_tests(options)
failed_tests = []
if 'all' in requested_tests:
need_to_setup = candidates
failed_tests += broken_tests
else:
need_to_setup = []
for candidate in candidates:
if candidate.__class__.__name__ in requested_tests:
need_to_setup.append(candidate)
for broken_test in broken_tests:
if broken_test in requested_tests:
failed_tests.append(broken_test)
if need_to_setup:
cwd = os.getcwd()
os.chdir(need_to_setup[0].job.clientdir)
os.system('tools/make_clean')
os.chdir(cwd)
elif not failed_tests:
logging.error('### No test setup candidates ###')
raise error.AutoservError('No test setup candidates.')
for client_test in need_to_setup:
good_setup = setup_test(client_test)
if not good_setup:
failed_tests.append(client_test.__class__.__name__)
logging.info('############################# SUMMARY '
'#############################')
# Print out tests that failed
if failed_tests:
logging.info('Finished setup -- The following tests failed')
for failed_test in failed_tests:
logging.info(failed_test)
else:
logging.info('Finished setup -- All tests built successfully')
logging.info('######################### END SUMMARY '
'##############################')
if failed_tests:
raise error.AutoservError('Finished setup with errors.')
def _verify_moblab_services(self):
"""Verify the required Moblab services are up and running.
@raises AutoservError if any moblab service is not running.
"""
for service in MOBLAB_SERVICES:
if not self.upstart_status(service):
raise error.AutoservError(
'Moblab service: %s is not running.' % service)
for process in MOBLAB_PROCESSES:
try:
self.run('pgrep %s' % process)
except error.AutoservRunError:
raise error.AutoservError(
'Moblab process: %s is not running.' % process)
def _verify_duts(self):
"""Verify the Moblab DUTs are up and running.
@raises AutoservError if no DUTs are in the Ready State.
"""
# Check whether afe is well connected, if not, restart it.
try:
self._check_afe()
except:
self.wait_afe_up()
# Add the DUTs if they have not yet been added.
self.find_and_add_duts()
# Ensure a boto file is installed in case this Moblab was wiped in
# repair.
self.install_boto_file()
hosts = self.afe.reverify_hosts()
logging.debug('DUTs scheduled for reverification: %s', hosts)
# Wait till all pending special tasks are completed.
total_time = 0
while (self.afe.get_special_tasks(is_complete=False)
and total_time < DUT_VERIFY_TIMEOUT):
total_time = total_time + DUT_VERIFY_SLEEP_SECS
time.sleep(DUT_VERIFY_SLEEP_SECS)
if not self.afe.get_hosts(status='Ready'):
for host in self.afe.get_hosts():
logging.error('DUT: %s Status: %s', host, host.status)
raise error.AutoservError('Moblab has 0 Ready DUTs')
def install(self, host, **kwargs):
"""
Install a kernel on the remote host.
This will also invoke the guest's bootloader to set this
kernel as the default kernel.
Args:
host: the host on which to install the kernel
[kwargs]: remaining keyword arguments will be passed
to Bootloader.add_kernel()
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
remote_tmpdir = host.get_tmp_dir()
basename = os.path.basename(self.source_material)
remote_filename = os.path.join(remote_tmpdir, basename)
host.send_file(self.source_material, remote_filename)
host.run('dpkg -i "%s"' % (utils.sh_escape(remote_filename), ))
host.run('mkinitramfs -o "%s" "%s"' % (
utils.sh_escape(self.get_initrd_name()),
utils.sh_escape(self.get_version()),
))
host.bootloader.add_kernel(self.get_image_name(),
initrd=self.get_initrd_name(),
**kwargs)
def verify_moblab_services(self, timeout_m):
"""Verify the required Moblab services are up and running.
@param timeout_m: Timeout (in minutes) for how long to wait for services
to start. Actual time taken may be slightly more than this.
@raises AutoservError if any moblab service is not running.
"""
if not MOBLAB_SERVICES:
return
service = MOBLAB_SERVICES[0]
try:
# First service can take a long time to start, especially on first
# boot where container setup can take 5-10 minutes, depending on the
# device.
self._verify_upstart_service(service, timeout_m)
except error.TimeoutException:
raise UpstartServiceNotRunning(service)
for service in MOBLAB_SERVICES[1:]:
try:
# Follow up services should come up quickly.
self._verify_upstart_service(service, 0.5)
except error.TimeoutException:
raise UpstartServiceNotRunning(service)
for process in MOBLAB_PROCESSES:
try:
self.run('pgrep %s' % process)
except error.AutoservRunError:
raise error.AutoservError(
'Moblab process: %s is not running.' % process)
def extract(self, host):
"""Extract the kernel package.
This function is only useful to access the content of the
package (for example the kernel image) without
installing it. It is not necessary to run this function to
install the kernel.
Args:
host: the host on which to extract the kernel package.
Returns:
The full path to the temporary directory on host where
the package was extracted.
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
remote_tmpdir = host.get_tmp_dir()
basename = os.path.basename(self.source_material)
remote_filename = os.path.join(remote_tmpdir, basename)
host.send_file(self.source_material, remote_filename)
content_dir = os.path.join(remote_tmpdir, "contents")
host.run('dpkg -x "%s" "%s"' % (
utils.sh_escape(remote_filename),
utils.sh_escape(content_dir),
))
return content_dir
def install(self, host, label='autotest',
default=False, kernel_args = '', install_vmlinux=True):
"""
Install a kernel on the remote host.
This will also invoke the guest's bootloader to set this
kernel as the default kernel if default=True.
Args:
host: the host on which to install the kernel
[kwargs]: remaining keyword arguments will be passed
to Bootloader.add_kernel()
Raises:
AutoservError: no package has yet been obtained. Call
RPMKernel.get() with a .rpm package.
"""
if len(label) > 15:
raise error.AutoservError("label for kernel is too long \
(> 15 chars): %s" % label)
if self.source_material is None:
raise error.AutoservError("A kernel must first be \
specified via get()")
rpm = self.source_material
remote_tmpdir = host.get_tmp_dir()
remote_rpm = os.path.join(remote_tmpdir, os.path.basename(rpm))
rpm_package = utils.run('/usr/bin/rpm -q -p %s' % rpm).stdout
vmlinuz = self.get_image_name()
host.send_file(rpm, remote_rpm)
host.run('rpm -e ' + rpm_package, ignore_status = True)
host.run('rpm --force -i ' + remote_rpm)
# Copy over the uncompressed image if there is one
if install_vmlinux:
vmlinux = self.get_vmlinux_name()
host.run('cd /;rpm2cpio %s | cpio -imuv .%s'
% (remote_rpm, vmlinux))
host.run('ls ' + vmlinux) # Verify
host.bootloader.remove_kernel(label)
host.bootloader.add_kernel(vmlinuz, label,
args=kernel_args, default=default)
if kernel_args:
host.bootloader.add_args(label, kernel_args)
if not default:
host.bootloader.boot_once(label)
def _install_boottool(self):
if self._host() is None:
raise error.AutoservError(
"Host does not exist anymore")
tmpdir = self._host().get_tmp_dir()
self._host().send_file(os.path.abspath(os.path.join(
utils.get_server_dir(), BOOTTOOL_SRC)), tmpdir)
self._boottool_path= os.path.join(tmpdir,
os.path.basename(BOOTTOOL_SRC))
def _control_path_on_disk(control_name):
"""Find the control file corresponding to the given control name, on disk.
@param control_name: NAME attribute of the control file to fetch.
@return: Path to the control file.
"""
cf_getter = suite.create_fs_getter(_AUTOTEST_ROOT)
control_name_predicate = suite.test_name_matches_pattern_predicate(
'^%s$' % control_name)
tests = suite.find_and_parse_tests(cf_getter, control_name_predicate)
if not tests:
raise error.AutoservError(
'Failed to find any control files with NAME %s' % control_name)
if len(tests) > 1:
logging.error('Found more than one control file with NAME %s: %s',
control_name, [t.path for t in tests])
raise error.AutoservError(
'Found more than one control file with NAME %s' % control_name)
return tests[0].path
def verify_special_tasks_complete(self):
"""Wait till the special tasks on the moblab host are complete."""
total_time = 0
while (self.afe.get_special_tasks(is_complete=False)
and total_time < DUT_VERIFY_TIMEOUT):
total_time = total_time + DUT_VERIFY_SLEEP_SECS
time.sleep(DUT_VERIFY_SLEEP_SECS)
if not self.afe.get_hosts(status='Ready'):
for host in self.afe.get_hosts():
logging.error('DUT: %s Status: %s', host, host.status)
raise error.AutoservError('Moblab has 0 Ready DUTs')
def repair(self, host_protection):
if not self.machines:
raise error.AutoservError('No machines specified to repair')
if self.resultdir:
os.chdir(self.resultdir)
namespace = {'machines': self.machines, 'job': self,
'ssh_user': self._ssh_user, 'ssh_port': self._ssh_port,
'ssh_pass': self._ssh_pass,
'protection_level': host_protection}
self._execute_code(REPAIR_CONTROL_FILE, namespace, protect=False)
def stage_server_side_package(self, image=None):
"""Stage autotest server-side package on devserver.
@param image: Full path of an OS image to install or a build name.
@return: A url to the autotest server-side package.
@raise: error.AutoservError if fail to locate the build to test with, or
fail to stage server-side package.
"""
if image:
image_name = tools.get_build_from_image(image)
if not image_name:
raise error.AutoservError(
'Failed to parse build name from %s' % image)
ds = dev_server.ImageServer.resolve(image_name)
else:
job_repo_url = afe_utils.get_host_attribute(
self, ds_constants.JOB_REPO_URL)
if job_repo_url:
devserver_url, image_name = (
tools.get_devserver_build_from_package_url(job_repo_url))
ds = dev_server.ImageServer.resolve(image_name)
else:
labels = afe_utils.get_labels(self, self.VERSION_PREFIX)
if not labels:
raise error.AutoservError(
'Failed to stage server-side package. The host has '
'no job_report_url attribute or version label.')
image_name = labels[0][len(self.VERSION_PREFIX + ':'):]
ds = dev_server.ImageServer.resolve(image_name)
ds.stage_artifacts(image_name, ['autotest_server_package'],
archive_url=ARCHIVE_URL + image_name)
return '%s/static/%s/%s' % (ds.url(), image_name,
'autotest_server_package.tar.bz2')
def verify(self):
if not self.machines:
raise error.AutoservError('No machines specified to verify')
if self.resultdir:
os.chdir(self.resultdir)
try:
namespace = {'machines' : self.machines, 'job' : self,
'ssh_user' : self._ssh_user,
'ssh_port' : self._ssh_port,
'ssh_pass' : self._ssh_pass}
self._execute_code(VERIFY_CONTROL_FILE, namespace, protect=False)
except Exception, e:
msg = ('Verify failed\n' + str(e) + '\n' + traceback.format_exc())
self.record('ABORT', None, None, msg)
raise
def get_initrd_name(self):
"""Get the name of the initrd file to be installed.
Returns:
The full path to the initrd file as it will be
installed on the host. If the package includes no
initrd file, None is returned
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
return "/boot/initrd.img-%s" % (self.get_version(), )
def _import_names(module_name, names=()):
"""
Import a module and assign named attributes into namespace.
Args:
module_name: The string module name.
names: A limiting list of names to import from module_name. If
empty (the default), all names are imported from the module
similar to a "from foo.bar import *" statement.
Raises:
error.AutoservError: When a name being imported would clobber
a name already in namespace.
"""
module = __import__(module_name, {}, {}, names)
# No names supplied? Import * from the lowest level module.
# (Ugh, why do I have to implement this part myself?)
if not names:
for submodule_name in module_name.split('.')[1:]:
module = getattr(module, submodule_name)
if hasattr(module, '__all__'):
names = getattr(module, '__all__')
else:
names = dir(module)
# Install each name into namespace, checking to make sure it
# doesn't override anything that already exists.
for name in names:
# Check for conflicts to help prevent future problems.
if name in namespace and protect:
if namespace[name] is not getattr(module, name):
raise error.AutoservError(
'importing name '
'%s from %s %r would override %r' %
(name, module_name, getattr(
module, name), namespace[name]))
else:
# Encourage cleanliness and the use of __all__ for a
# more concrete API with less surprises on '*' imports.
warnings.warn('%s (%r) being imported from %s for use '
'in server control files is not the '
'first occurrance of that import.' %
(name, namespace[name], module_name))
namespace[name] = getattr(module, name)
def _choose_connectivity_class(hostname, ssh_port):
"""Choose a connectivity class for this hostname.
@param hostname: hostname that we need a connectivity class for.
@param ssh_port: SSH port to connect to the host.
@returns a connectivity host class.
"""
if (hostname == 'localhost' and ssh_port == DEFAULT_SSH_PORT):
return local_host.LocalHost
# by default assume we're using SSH support
elif SSH_ENGINE == 'raw_ssh':
return ssh_host.SSHHost
else:
raise error.AutoservError(
"Unknown SSH engine %s. Please verify the "
"value of the configuration key 'ssh_engine' "
"on autotest's global_config.ini file." % SSH_ENGINE)
def get_version(self):
"""Get the version of the kernel to be installed.
Returns:
The version string, as would be returned
by 'make kernelrelease'.
Raises:
AutoservError: no package has yet been obtained. Call
RPMKernel.get() with a .rpm package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be \
specified via get()")
retval = utils.run('rpm -qpi %s | grep Version | awk \'{print($3);}\''
% utils.sh_escape(self.source_material))
return retval.stdout.strip()
def update_host_attribute(host, attribute, value):
"""Updates the host attribute with given value.
@param host: A Host object to update attribute value.
@param attribute: Name of the host attribute.
@param value: Value for the host attribute.
@raises AutoservError: If we failed to update the attribute.
"""
host._afe_host.attributes[attribute] = value
if not _host_in_lab(host):
return
AFE.set_host_attribute(attribute, value, hostname=host.hostname)
if get_host_attribute(host, attribute, use_local_value=False) != value:
raise error.AutoservError(
'Failed to update host attribute `%s` with %s, host %s' %
(attribute, value, host.hostname))
def get_image_name(self):
"""Get the name of the kernel image to be installed.
Returns:
The full path to the kernel image file as it will be
installed on the host.
Raises:
AutoservError: no package has yet been obtained. Call
RPMKernel.get() with a .rpm package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be \
specified via get()")
vmlinuz = utils.run('rpm -q -l -p %s | grep /boot/vmlinuz'
% self.source_material).stdout.strip()
return vmlinuz
def get_version(self):
"""Get the version of the kernel to be installed.
Returns:
The version string, as would be returned
by 'make kernelrelease'.
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
retval = utils.run(
'dpkg-deb -f "%s" version' %
utils.sh_escape(self.source_material), )
return retval.stdout.strip()
def get_initrd_name(self):
"""Get the name of the initrd file to be installed.
Returns:
The full path to the initrd file as it will be
installed on the host. If the package includes no
initrd file, None is returned
Raises:
AutoservError: no package has yet been obtained. Call
RPMKernel.get() with a .rpm package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be \
specified via get()")
res = utils.run('rpm -q -l -p %s | grep /boot/initrd'
% self.source_material, ignore_status=True)
if res.exit_status:
return None
return res.stdout.strip()
def _check_afe(self):
"""Verify whether afe of moblab works before verifying its DUTs.
Verifying moblab sometimes happens after a successful provision, in
which case moblab is restarted but tunnel of afe is not re-connected.
This func is used to check whether afe is working now.
@return True if afe works.
@raises error.AutoservError if AFE is down; other exceptions are passed
through.
"""
try:
self.afe.get_hosts()
except (error.TimeoutException, timeout_util.TimeoutError) as e:
raise error.AutoservError('Moblab AFE is not responding: %s' %
str(e))
except Exception as e:
logging.error('Unknown exception when checking moblab AFE: %s', e)
raise
return True
def parallel(tasklist, timeout=None, return_results=False):
"""
Run a set of predefined subcommands in parallel.
@param tasklist: A list of subcommand instances to execute.
@param timeout: Number of seconds after which the commands should timeout.
@param return_results: If True instead of an AutoServError being raised
on any error a list of the results|exceptions from the tasks is
returned. [default: False]
"""
run_error = False
for task in tasklist:
task.fork_start()
remaining_timeout = None
if timeout:
endtime = time.time() + timeout
results = []
for task in tasklist:
if timeout:
remaining_timeout = max(endtime - time.time(), 1)
try:
status = task.fork_waitfor(timeout=remaining_timeout)
except error.AutoservSubcommandError:
run_error = True
else:
if status != 0:
run_error = True
results.append(cPickle.load(task.result_pickle))
task.result_pickle.close()
if return_results:
return results
elif run_error:
message = 'One or more subcommands failed:\n'
for task, result in zip(tasklist, results):
message += 'task: %s returned/raised: %r\n' % (task, result)
raise error.AutoservError(message)
def _initialize(self, hostname, gce_args=None,
*args, **dargs):
"""Initializes this instance of GceHost.
@param hostname: the hostnname to be passed down to AbstractSSHHost.
@param gce_args: GCE-specific arguments extracted using
extract_arguments().
"""
super(GceHost, self)._initialize(hostname=hostname,
*args, **dargs)
if gce_args:
self._gce_project = gce_args['gce_project']
self._gce_zone = gce_args['gce_zone']
self._gce_instance = gce_args['gce_instance']
self._gce_key_file = gce_args['gce_key_file']
else:
# TODO(andreyu): determine project, zone and instance names by
# querying metadata from the DUT instance
raise error.AutoservError('No GCE flags provided.')
self.gce = gce.GceContext.ForServiceAccountThreadSafe(
self._gce_project, self._gce_zone, self._gce_key_file)
def setup_test(client_test):
"""
Direct invoke test.setup() method.
@returns A boolean to represent success or not.
"""
# TODO: check if its already build. .version? hash?
test_name = client_test.__class__.__name__
cwd = os.getcwd()
good_setup = False
try:
try:
outputdir = os.path.join(client_test.job.resultdir, test_name)
try:
os.makedirs(outputdir)
os.chdir(outputdir)
except OSError:
pass
logging.info('setup %s.' % test_name)
client_test.setup()
# Touch .version file under src to prevent further setup on client
# host. See client/common_lib/utils.py update_version()
if os.path.exists(client_test.srcdir):
versionfile = os.path.join(client_test.srcdir, '.version')
pickle.dump(client_test.version, open(versionfile, 'w'))
good_setup = True
except Exception, err:
logging.error(err)
raise error.AutoservError('Failed to build client test %s on '
'server.' % test_name)
finally:
# back to original working dir
os.chdir(cwd)
return good_setup
c = global_config.global_config
supports_autoserv_packaging = c.get_config_value(
"PACKAGES", "serve_packages_from_autoserv", type=bool)
# Copy autotest recursively
if supports_autoserv_packaging and use_autoserv:
self._install_using_send_file(host, autodir)
else:
host.send_file(self.source_material, autodir, delete_dest=True)
logging.info("Installation of autotest completed from %s",
self.source_material)
self.installed = True
return
# if that fails try to install using svn
if utils.run('which svn').exit_status:
raise error.AutoservError('svn not found on target machine: %s' %
host.hostname)
try:
host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir))
except error.AutoservRunError, e:
host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir))
logging.info("Installation of autotest completed using SVN.")
self.installed = True
def uninstall(self, host=None):
"""
Uninstall (i.e. delete) autotest. Removes the autotest client install
from the specified host.
@params host a Host instance from which the client will be removed
"""
if not self.installed:
def _run_with_ssp(job, container_id, job_id, results, parser, ssp_url,
machines):
"""Run the server job with server-side packaging.
@param job: The server job object.
@param container_id: ID of the container to run the test.
@param job_id: ID of the test job.
@param results: Folder to store results. This could be different from
parser.options.results:
parser.options.results can be set to None for results to be
stored in a temp folder.
results can be None for autoserv run requires no logging.
@param parser: Command line parser that contains the options.
@param ssp_url: url of the staged server-side package.
@param machines: A list of machines to run the test.
"""
if not ssp_url:
job.record('FAIL', None, None, 'Failed to stage server-side package')
raise error.AutoservError('Failed to stage server-side package')
bucket = lxc.ContainerBucket()
control = (parser.args[0]
if len(parser.args) > 0 and parser.args[0] != '' else None)
try:
dut_name = machines[0] if len(machines) >= 1 else None
test_container = bucket.setup_test(container_id,
job_id,
ssp_url,
results,
control=control,
job_folder=_LXC_JOB_FOLDER,
dut_name=dut_name,
isolate_hash=parser.options.isolate)
except Exception as e:
job.record(
'FAIL', None, None,
'Failed to setup container for test: %s. Check logs in '
'ssp_logs folder for more details.' % e)
raise
args = sys.argv[:]
args.remove('--require-ssp')
# --parent_job_id is only useful in autoserv running in host, not in
# container. Include this argument will cause test to fail for builds before
# CL 286265 was merged.
if '--parent_job_id' in args:
index = args.index('--parent_job_id')
args.remove('--parent_job_id')
# Remove the actual parent job id in command line arg.
del args[index]
# A dictionary of paths to replace in the command line. Key is the path to
# be replaced with the one in value.
paths_to_replace = {}
# Replace the control file path with the one in container.
if control:
container_control_filename = os.path.join(lxc.CONTROL_TEMP_PATH,
os.path.basename(control))
paths_to_replace[control] = container_control_filename
# Update result directory with the one in container.
container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % _LXC_JOB_FOLDER)
if parser.options.results:
paths_to_replace[parser.options.results] = container_result_dir
args = [paths_to_replace.get(arg, arg) for arg in args]
# Apply --use-existing-results, results directory is aready created and
# mounted in container. Apply this arg to avoid exception being raised.
if not '--use-existing-results' in args:
args.append('--use-existing-results')
# Make sure autoserv running in container using a different pid file.
if not '--pidfile-label' in args:
args.extend(['--pidfile-label', 'container_autoserv'])
cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args])
logging.info('Run command in container: %s', cmd_line)
success = False
try:
test_container.attach_run(cmd_line)
success = True
except Exception as e:
# If the test run inside container fails without generating any log,
# write a message to status.log to help troubleshooting.
debug_files = os.listdir(os.path.join(results, 'debug'))
if not debug_files:
job.record(
'FAIL', None, None,
'Failed to run test inside the container: %s. Check '
'logs in ssp_logs folder for more details.' % e)
raise
finally:
metrics.Counter('chromeos/autotest/experimental/execute_job_in_ssp'
).increment(fields={'success': success})
test_container.destroy()
class BaseAutotest(installable_object.InstallableObject):
"""
This class represents the Autotest program.
Autotest is used to run tests automatically and collect the results.
It also supports profilers.
Implementation details:
This is a leaf class in an abstract class hierarchy, it must
implement the unimplemented methods in parent classes.
"""
def __init__(self, host=None):
self.host = host
self.got = False
self.installed = False
self.serverdir = utils.get_server_dir()
super(BaseAutotest, self).__init__()
install_in_tmpdir = False
@classmethod
def set_install_in_tmpdir(cls, flag):
""" Sets a flag that controls whether or not Autotest should by
default be installed in a "standard" directory (e.g.
/home/autotest, /usr/local/autotest) or a temporary directory. """
cls.install_in_tmpdir = flag
@classmethod
def get_client_autodir_paths(cls, host):
return global_config.global_config.get_config_value(
'AUTOSERV', 'client_autodir_paths', type=list)
@classmethod
def get_installed_autodir(cls, host):
"""
Find where the Autotest client is installed on the host.
@returns an absolute path to an installed Autotest client root.
@raises AutodirNotFoundError if no Autotest installation can be found.
"""
autodir = host.get_autodir()
if autodir:
logging.debug('Using existing host autodir: %s', autodir)
return autodir
for path in Autotest.get_client_autodir_paths(host):
try:
autotest_binary = os.path.join(path, 'bin', 'autotest')
host.run('test -x %s' % utils.sh_escape(autotest_binary))
host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
return path
except error.AutoservRunError:
logging.debug('%s does not exist on %s', autotest_binary,
host.hostname)
raise AutodirNotFoundError
@classmethod
def get_install_dir(cls, host):
"""
Determines the location where autotest should be installed on
host. If self.install_in_tmpdir is set, it will return a unique
temporary directory that autotest can be installed in. Otherwise, looks
for an existing installation to use; if none is found, looks for a
usable directory in the global config client_autodir_paths.
"""
try:
install_dir = cls.get_installed_autodir(host)
except AutodirNotFoundError:
install_dir = cls._find_installable_dir(host)
if cls.install_in_tmpdir:
return host.get_tmp_dir(parent=install_dir)
return install_dir
@classmethod
def _find_installable_dir(cls, host):
client_autodir_paths = cls.get_client_autodir_paths(host)
for path in client_autodir_paths:
try:
host.run('mkdir -p %s' % utils.sh_escape(path))
host.run('test -w %s' % utils.sh_escape(path))
return path
except error.AutoservRunError:
logging.debug('Failed to create %s', path)
raise error.AutoservInstallError(
'Unable to find a place to install Autotest; tried %s' %
', '.join(client_autodir_paths))
def get_fetch_location(self):
c = global_config.global_config
repos = c.get_config_value("PACKAGES",
'fetch_location',
type=list,
default=[])
repos.reverse()
return repos
def install(self, host=None, autodir=None):
self._install(host=host, autodir=autodir)
def install_full_client(self, host=None, autodir=None):
self._install(host=host,
autodir=autodir,
use_autoserv=False,
use_packaging=False)
def install_no_autoserv(self, host=None, autodir=None):
self._install(host=host, autodir=autodir, use_autoserv=False)
def _install_using_packaging(self, host, autodir):
repos = self.get_fetch_location()
if not repos:
raise error.PackageInstallError("No repos to install an "
"autotest client from")
pkgmgr = packages.PackageManager(autodir,
hostname=host.hostname,
repo_urls=repos,
do_locking=False,
run_function=host.run,
run_function_dargs=dict(timeout=600))
# The packages dir is used to store all the packages that
# are fetched on that client. (for the tests,deps etc.
# too apart from the client)
pkg_dir = os.path.join(autodir, 'packages')
# clean up the autodir except for the packages directory
host.run('cd %s && ls | grep -v "^packages$"'
' | xargs rm -rf && rm -rf .[^.]*' % autodir)
pkgmgr.install_pkg('autotest',
'client',
pkg_dir,
autodir,
preserve_install_dir=True)
self.installed = True
def _install_using_send_file(self, host, autodir):
dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"])
light_files = [
os.path.join(self.source_material, f)
for f in os.listdir(self.source_material)
if f not in dirs_to_exclude
]
host.send_file(light_files, autodir, delete_dest=True)
# create empty dirs for all the stuff we excluded
commands = []
for path in dirs_to_exclude:
abs_path = os.path.join(autodir, path)
abs_path = utils.sh_escape(abs_path)
commands.append("mkdir -p '%s'" % abs_path)
commands.append("touch '%s'/__init__.py" % abs_path)
host.run(';'.join(commands))
def _install(self,
host=None,
autodir=None,
use_autoserv=True,
use_packaging=True):
"""
Install autotest. If get() was not called previously, an
attempt will be made to install from the autotest svn
repository.
@param host A Host instance on which autotest will be installed
@param autodir Location on the remote host to install to
@param use_autoserv Enable install modes that depend on the client
running with the autoserv harness
@param use_packaging Enable install modes that use the packaging system
@exception AutoservError if a tarball was not specified and
the target host does not have svn installed in its path
"""
if not host:
host = self.host
if not self.got:
self.get()
host.wait_up(timeout=30)
host.setup()
logging.info("Installing autotest on %s", host.hostname)
# set up the autotest directory on the remote machine
if not autodir:
autodir = self.get_install_dir(host)
logging.info('Using installation dir %s', autodir)
host.set_autodir(autodir)
host.run('mkdir -p %s' % utils.sh_escape(autodir))
# make sure there are no files in $AUTODIR/results
results_path = os.path.join(autodir, 'results')
host.run('rm -rf %s/*' % utils.sh_escape(results_path),
ignore_status=True)
# Fetch the autotest client from the nearest repository
if use_packaging:
try:
self._install_using_packaging(host, autodir)
return
except (error.PackageInstallError, error.AutoservRunError,
global_config.ConfigError), e:
logging.info(
"Could not install autotest using the packaging "
"system: %s. Trying other methods", e)
# try to install from file or directory
if self.source_material:
c = global_config.global_config
supports_autoserv_packaging = c.get_config_value(
"PACKAGES", "serve_packages_from_autoserv", type=bool)
# Copy autotest recursively
if supports_autoserv_packaging and use_autoserv:
self._install_using_send_file(host, autodir)
else:
host.send_file(self.source_material, autodir, delete_dest=True)
logging.info("Installation of autotest completed")
self.installed = True
return
# if that fails try to install using svn
if utils.run('which svn').exit_status:
raise error.AutoservError('svn not found on target machine: %s' %
host.name)
try:
host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir))
except error.AutoservRunError, e:
host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir))
def install_apk_from_build(host,
apk,
build_artifact,
package_name=None,
force_reinstall=False,
build_name=None):
"""Install the specific apk from given build artifact.
@param host: An ADBHost object to install apk.
@param apk: Name of the apk to install, e.g., sl4a.apk
@param build_artifact: Name of the build artifact, e.g., test_zip. Note
that it's not the name of the artifact file. Refer to
artifact_info.py in devserver code for the mapping between
artifact name to a build artifact.
@param package_name: Name of the package, e.g., com.android.phone.
If package_name is given, it checks if the package exists before
trying to install it, unless force_reinstall is set to True.
@param force_reinstall: True to reinstall the apk even if it's already
installed. Default is set to False.
@param build_name: None unless DUT is CrOS with ARC++ container. build_name
points to ARC++ build artifacts.
"""
# Check if apk is already installed.
if package_name and not force_reinstall:
if host.is_apk_installed(package_name):
logging.info('Package %s is already installed.', package_name)
return
if build_name:
# Pull devserver_url given ARC++ enabled host
host_devserver_url = dev_server.AndroidBuildServer.resolve(
build_name, host.hostname).url()
# Return devserver_url given Android build path
job_repo_url = os.path.join(host_devserver_url, build_name)
else:
info = host.host_info_store.get()
job_repo_url = info.attributes.get(host.job_repo_url_attribute, '')
if not job_repo_url:
raise error.AutoservError(
'The host %s has no attribute %s. `install_apk_from_build` '
'only works for test with image specified.' %
(host.hostname, host.job_repo_url_attribute))
devserver_url = dev_server.AndroidBuildServer.get_server_url(job_repo_url)
devserver = dev_server.AndroidBuildServer(devserver_url)
build_info = host.get_build_info_from_build_url(job_repo_url)
devserver.trigger_download(build_info['target'],
build_info['build_id'],
build_info['branch'],
build_artifact,
synchronous=True)
build_info['os_type'] = 'android'
apk_url = devserver.locate_file(apk, build_artifact, None, build_info)
logging.debug('Found apk at: %s', apk_url)
tmp_dir = host.teststation.get_tmp_dir()
try:
host.download_file(apk_url, apk, tmp_dir)
result = host.install_apk(os.path.join(tmp_dir, apk),
force_reinstall=force_reinstall)
logging.debug(result.stdout)
if package_name and not host.is_apk_installed(package_name):
raise error.AutoservError('No package found with name of %s',
package_name)
logging.info('%s is installed successfully.', apk)
finally:
host.teststation.run('rm -rf %s' % tmp_dir)
