def test_timeout(self): # we expect a logging.warn() message, don't care about the contents utils.logging.warn.expect_any_call() try: utils.run('echo -n output && sleep 10', timeout=1, verbose=False) except utils.error.CmdError, err: self.assertEquals(err.result_obj.stdout, 'output')
def _check_unittest(self): """ Verifies if the file in question has a unittest suite, if so, run the unittest and report on any failures. This is important to keep our unit tests up to date. """ if "unittest" not in self.basename: stripped_name = self.basename.strip(".py") unittest_name = stripped_name + "_unittest.py" unittest_path = self.path.replace(self.basename, unittest_name) if os.path.isfile(unittest_path): unittest_cmd = 'python %s' % unittest_path try: utils.run(unittest_cmd, verbose=False) except error.CmdError, e: e_msg = ("Found unittest issues during '%s'" % unittest_cmd) logging.error(e_msg) for stdout_line in e.result_obj.stdout.split("\n"): if stdout_line: logging.error(" [stdout]: %s", stdout_line) for stderr_line in e.result_obj.stderr.split("\n"): if stderr_line: logging.error(" [stdout]: %s", stderr_line) logging.error("")
def _check_indent(self): """ Verifies the file with reindent.py. This tool performs the following checks on python files: * Trailing whitespaces * Tabs * End of line * Incorrect indentation For the purposes of checking, the dry run mode is used and no changes are made. It is up to the user to decide if he wants to run reindent to correct the issues. """ indent_exception = 'cli/job_unittest.py' if re.search (indent_exception, self.path): return if not self.path.endswith(".py"): path = "%s-cp.py" % self.path utils.run("cp %s %s" % (self.path, path), verbose=False) else: path = self.path try: cmdstatus = utils.run(('reindent.py -v -d %s' % path), verbose=False) except error.CmdError, e: logging.error("Error executing reindent.py: %s" % e)
def upload_pkg_file(self, file_path, upload_path): ''' Upload a single file. Depending on the upload path, the appropriate method for that protocol is called. Currently this simply copies the file to the target directory (but can be extended for other protocols) This assumes that the web server is running on the same machine where the method is being called from. The upload_path's files are basically served by that web server. ''' try: if upload_path.startswith('ssh://'): # parse ssh://user@host/usr/local/autotest/packages hostline, remote_path = parse_ssh_path(upload_path) try: utils.run('scp %s %s:%s' % (file_path, hostline, remote_path)) r_path = os.path.join(remote_path, os.path.basename(file_path)) utils.run("ssh %s 'chmod 644 %s'" % (hostline, r_path)) except error.CmdError: logging.error("Error uploading to repository %s", upload_path) else: shutil.copy(file_path, upload_path) os.chmod(os.path.join(upload_path, os.path.basename(file_path)), 0644) except (IOError, os.error), why: logging.error("Upload of %s to %s failed: %s", file_path, upload_path, why)
def compile_and_install_client(project_client, extra_args='', install_client=True): """Compile the client into a temporary directory, if successful call install_completed_client to install the new client. @param project_client: project.client pair e.g. autotest.AfeClient @param install_client: Boolean, if True install the clients @return True if install and compile was successful False if it failed """ java_args = {} java_args['compile_dir'] = _TMP_COMPILE_DIR java_args['app_dir'] = _DEFAULT_APP_DIR java_args['gwt_dir'] = find_gwt_dir() java_args['extra_args'] = extra_args java_args['project_client'] = project_client cmd = _COMPILE_LINE % java_args logging.info('Compiling client %s', project_client) try: utils.run(cmd, verbose=True) if install_client: return install_completed_client(java_args['compile_dir'], project_client) return True except error.CmdError: logging.info('Error compiling %s, leaving old client', project_client) return False
def process_all_packages(pkgmgr, client_dir, upload_paths, remove=False): """Process a full upload of packages as a directory upload.""" test_dir = os.path.join(client_dir, "tests") site_test_dir = os.path.join(client_dir, "site_tests") dep_dir = os.path.join(client_dir, "deps") prof_dir = os.path.join(client_dir, "profilers") # Directory where all are kept temp_dir = tempfile.mkdtemp() try: packages.check_diskspace(temp_dir) except packages.RepoDiskFull: print ("Temp destination for packages is full %s, aborting upload" % temp_dir) os.rmdir(temp_dir) sys.exit(1) # process tests tests_list = get_subdir_list('tests', client_dir) tests = ','.join(tests_list) # process site_tests site_tests_list = get_subdir_list('site_tests', client_dir) site_tests = ','.join(site_tests_list) # process deps deps_list = get_subdir_list('deps', client_dir) deps = ','.join(deps_list) # process profilers profilers_list = get_subdir_list('profilers', client_dir) profilers = ','.join(profilers_list) # Update md5sum if not remove: tar_packages(pkgmgr, 'profiler', profilers, prof_dir, temp_dir) tar_packages(pkgmgr, 'dep', deps, dep_dir, temp_dir) tar_packages(pkgmgr, 'test', site_tests, client_dir, temp_dir) tar_packages(pkgmgr, 'test', tests, client_dir, temp_dir) tar_packages(pkgmgr, 'client', 'autotest', client_dir, temp_dir) cwd = os.getcwd() os.chdir(temp_dir) client_utils.system('md5sum * > packages.checksum') os.chdir(cwd) for path in upload_paths: print "Uploading to: " + path pkgmgr.upload_pkg(temp_dir, path) client_utils.run('rm -rf ' + temp_dir) else: for repo_url in upload_paths: process_packages(pkgmgr, 'test', tests, client_dir, repo_url, remove=remove) process_packages(pkgmgr, 'test', site_tests, client_dir, repo_url, remove=remove) process_packages(pkgmgr, 'client', 'autotest', client_dir, repo_url, remove=remove) process_packages(pkgmgr, 'dep', deps, dep_dir, repo_url, remove=remove) process_packages(pkgmgr, 'profiler', profilers, prof_dir, repo_url, remove=remove)
def initialize(self, results_dir): temporary_directory = os.path.join(results_dir, _TEMPORARY_DIRECTORY) if os.path.exists(temporary_directory): shutil.rmtree(temporary_directory) self._ensure_directory_exists(temporary_directory) build_extern_cmd = os.path.join(results_dir, '../utils/build_externals.py') utils.run(build_extern_cmd)
def close(self): ''' Perform umount operation on the temporary dir ''' if os.path.ismount(self.mnt_dir): utils.run('fuser -k %s' % self.mnt_dir, ignore_status=True) utils.run('umount %s' % self.mnt_dir) shutil.rmtree(self.mnt_dir)
def revert_file(self, file): """ Revert file against last revision. @param file: Path to file to be reverted. """ try: utils.run('svn revert %s' % file) except error.CmdError, e: logging.error("Problem reverting file %s: %s", file, e) sys.exit(1)
def add_untracked_file(self, file): """ Add an untracked file under revision control. @param file: Path to untracked file. """ try: utils.run('svn add %s' % file) except error.CmdError, e: logging.error("Problem adding file %s to svn: %s", file, e) sys.exit(1)
def run_migration_with_file_transfer(test, params, env): """ KVM migration test: 1) Get a live VM and clone it. 2) Verify that the source VM supports migration. If it does, proceed with the test. 3) Reboot the VM 4) Send a migration command to the source VM and wait until it's finished. 5) Kill off the source VM. 6) Log into the destination VM after the migration is finished. @param test: kvm test object. @param params: Dictionary with test parameters. @param env: Dictionary with the test environment. """ def transfer_test(vm, host_path, guest_path, timeout=120): """ vm.copy_files_to does not raise exception, so we need a wrapper in order to make it to be used by BackgroundTest. """ if not vm.copy_files_to(host_path, guest_path, timeout=timeout): raise error.TestError("Fail to do the file transfer!") vm = kvm_test_utils.get_living_vm(env, params.get("main_vm")) timeout = int(params.get("login_timeout", 360)) session = kvm_test_utils.wait_for_login(vm, timeout=timeout) mig_timeout = float(params.get("mig_timeout", "3600")) mig_protocol = params.get("migration_protocol", "tcp") guest_path = params.get("guest_path", "/tmp") file_size = params.get("file_size", "1000") transfer_timeout = int(params.get("transfer_timeout", "240")) bg = None try: utils.run("dd if=/dev/zero of=/tmp/file bs=1M count=%s" % file_size) # Transfer file from host to guest bg = kvm_test_utils.BackgroundTest(transfer_test, (vm, "/tmp/file", guest_path, transfer_timeout)) bg.start() while bg.is_alive(): logging.info("File transfer is not ended, start a round of migration ...") # Migrate the VM dest_vm = kvm_test_utils.migrate(vm, env, mig_timeout, mig_protocol, False) vm = dest_vm finally: if bg: bg.join() session.close() utils.run("rm -rf /tmp/zero")
def _fetch_from_github(self, id): """ Gets a patch file from patchwork and puts it under the cwd so it can be applied. @param id: Patchwork patch id. """ patch_url = "https://github.com/autotest/autotest/pull/%s.patch" % id patch_dest = os.path.join(self.base_dir, 'github-%s.patch' % id) utils.run("curl %s > %s" % (patch_url, patch_dest)) return patch_dest
def start(self, test): self.pid = os.fork() if self.pid: # parent return else: # child while True: for cmd, outputfile in zip(self.cmds, self.outputfiles): logfile = open(os.path.join(test.profdir, outputfile), 'a') utils.run(cmd, stdout_tee=logfile, stderr_tee=logfile) logfile.write('\n') logfile.close() time.sleep(self.interval)
def compare_capabilities_xml(source): dom = parseString(source) host = dom.getElementsByTagName('host')[0] # check that host has a non-empty UUID tag. uuid = host.getElementsByTagName('uuid')[0] host_uuid_output = uuid.firstChild.data logging.info("The host uuid in capabilities_xml:%s", host_uuid_output) if host_uuid_output == "": raise error.TestFail("The host uuid in capabilities_xml is none!") # check the host arch. arch = host.getElementsByTagName('arch')[0] host_arch_output = arch.firstChild.data logging.info("The host arch in capabilities_xml:%s", host_arch_output) cmd_result = utils.run("arch", ignore_status=True) if cmp(host_arch_output, cmd_result.stdout.strip()) != 0: raise error.TestFail("The host arch in capabilities_xml is wrong!") # check the host cpus num. cpus = dom.getElementsByTagName('cpus')[0] host_cpus_output = cpus.getAttribute('num') logging.info("The host cpus num in capabilities_xml:%s",\ host_cpus_output) cmd = "less /proc/cpuinfo | grep processor | wc -l" cmd_result = utils.run(cmd, ignore_status=True) if cmp(host_cpus_output, cmd_result.stdout.strip()) != 0: raise error.TestFail("The host cpus num in capabilities_xml is\ wrong") # check the arch of guest supported. cmd = "/usr/libexec/qemu-kvm --cpu ? | grep qemu" cmd_result = utils.run(cmd, ignore_status=True) guest_wordsize_array = dom.getElementsByTagName('wordsize') length = len(guest_wordsize_array) for i in range(length): element = guest_wordsize_array[i] guest_wordsize = element.firstChild.data logging.info("The arch of guest supported in capabilities_xml:%s",\ guest_wordsize) if not re.search(guest_wordsize, cmd_result.stdout.strip()): raise error.TestFail("The capabilities_xml gives an extra arch \ of guest to support! ") # check the type of hyperviosr. guest_domain_type = dom.getElementsByTagName('domain')[0] guest_domain_type_output = guest_domain_type.getAttribute('type') logging.info("The hyperviosr in capabilities_xml:%s", \ guest_domain_type_output) cmd_result = utils.run("virsh uri", ignore_status=True) if not re.search(guest_domain_type_output, cmd_result.stdout.strip()): raise error.TestFail("The capabilities_xml gives an different\ hyperviosr")
def upload_pkg_dir(self, dir_path, upload_path): ''' Upload a full directory. Depending on the upload path, the appropriate method for that protocol is called. Currently this copies the whole tmp package directory to the target directory. This assumes that the web server is running on the same machine where the method is being called from. The upload_path's files are basically served by that web server. ''' local_path = os.path.join(dir_path, "*") try: if upload_path.startswith('ssh://'): hostline, remote_path = parse_ssh_path(upload_path) try: utils.run('scp %s %s:%s' % (local_path, hostline, remote_path)) ssh_path = os.path.join(remote_path, "*") utils.run("ssh %s 'chmod 644 %s'" % (hostline, ssh_path)) except error.CmdError: logging.error("Error uploading to repository: %s", upload_path) else: utils.run("cp %s %s " % (local_path, upload_path)) up_path = os.path.join(upload_path, "*") utils.run("chmod 644 %s" % up_path) except (IOError, os.error), why: raise error.PackageUploadError("Upload of %s to %s failed: %s" % (dir_path, upload_path, why))
def remove_pkg_file(self, filename, pkg_dir): """ Remove the file named filename from pkg_dir """ try: # Remove the file if pkg_dir.startswith("ssh://"): hostline, remote_path = parse_ssh_path(pkg_dir) path = os.path.join(remote_path, filename) utils.run("ssh %s 'rm -rf %s/%s'" % (hostline, remote_path, path)) else: os.remove(os.path.join(pkg_dir, filename)) except (IOError, os.error), why: raise error.PackageRemoveError("Could not remove %s from %s: %s " % (filename, pkg_dir, why))
def testHostDirMissing(self): """Verifies that a missing host dir does not cause cleanup to crash. """ host_dir = lxc.SharedHostDir(self.shared_host_path) # Manually destroy the host path utils.run('sudo umount %(path)s && sudo rmdir %(path)s' % {'path': self.shared_host_path}) # Verify that the host path does not exist. self.assertFalse(os.path.exists(self.shared_host_path)) try: host_dir.cleanup() except: self.fail('SharedHostDir.cleanup crashed.\n%s' % error.format_error())
def cleanup(self): shutil.copy('/var/log/update_engine.log', self.resultsdir) # Turn adapters back on utils.run('ifconfig eth0 up', ignore_status=True) utils.run('ifconfig eth1 up', ignore_status=True) utils.start_service('recover_duts', ignore_status=True) # We can't return right after reconnecting the network or the server # test may not receive the message. So we wait a bit longer for the # DUT to be reconnected. utils.poll_for_condition(lambda: utils.ping( self._update_server, deadline=5, timeout=5) == 0, timeout=60, sleep_interval=1) logging.info('Online ready to return to server test')
def _run_p2p_client(self, args, timeout=10., ignore_status=False): """Run p2p-client with the provided arguments. @param args: list of strings, each one representing an argument. @param timeout: Timeout for p2p-client in seconds before it's killed. @return: the return value of the process and the stdout content. """ fd, tempfn = tempfile.mkstemp(prefix='p2p-output') ret = utils.run(P2P_CLIENT, args=['-v=1'] + list(args), timeout=timeout, ignore_timeout=True, ignore_status=True, stdout_tee=open(tempfn, 'w'), stderr_tee=sys.stdout) url = os.fdopen(fd).read() os.unlink(tempfn) if not ignore_status and ret is None: self._join_simulator() raise error.TestFail('p2p-client %s timeout.' % ' '.join(args)) if not ignore_status and ret.exit_status != 0: self._join_simulator() raise error.TestFail('p2p-client %s finished with value: %d' % (' '.join(args), ret.exit_status)) return None if ret is None else ret.exit_status, url
def test(self): cmd = qemu_binary + " -cpu ?cpuid" result = utils.run(cmd) if result.stdout is "": raise error.TestFail("There aren't any cpu Flag in output" " '%s' of command \n%s" % (cmd, result.stdout))
def _run_cmd(self, cmd): """Execute an command in a external shell and capture the output. @param cmd: String of is a valid shell command. @returns The standard out, standard error and the integer exit code of the executed command. """ logging.debug('Running: %s', cmd) output = StringIO.StringIO() error_output = StringIO.StringIO() exit_code = 0 try: result = utils.run(cmd, stdout_tee=output, stderr_tee=error_output, timeout=TELEMETRY_TIMEOUT_MINS * 60) exit_code = result.exit_status except error.CmdError as e: logging.debug('Error occurred executing.') exit_code = e.result_obj.exit_status stdout = output.getvalue() stderr = error_output.getvalue() logging.debug('Completed with exit code: %d.\nstdout:%s\n' 'stderr:%s', exit_code, stdout, stderr) return stdout, stderr, exit_code
def test(self): #This is virtual cpu flags which are supported by #qemu but no with host cpu. cpu_model, extra_flags = parse_cpu_model() flags = HgFlags(cpu_model, extra_flags) logging.debug("Unsupported flags %s.", str(flags.host_all_unsupported_flags)) cpuf_model = cpu_model + ",enforce" # Add unsupported flags. for fadd in flags.host_all_unsupported_flags: cpuf_model += ",+" + fadd vnc_port = virt_utils.find_free_port(5900, 6100) - 5900 cmd = "%s -cpu %s -vnc :%d" % (qemu_binary, cpuf_model, vnc_port) out = None try: try: out = utils.run(cmd, timeout=5, ignore_status=True).stderr except error.CmdError: logging.error("Host boot with unsupported flag") finally: uns_re = re.compile("^warning:.*flag '(.+)'", re.MULTILINE) warn_flags = set(map(virt_utils.Flag, uns_re.findall(out))) fwarn_flags = flags.host_all_unsupported_flags - warn_flags if fwarn_flags: raise error.TestFail("Qemu did not warn the use of " "flags %s" % str(fwarn_flags))
def test(self): #This is virtual cpu flags which are supported by #qemu but no with host cpu. cpu_model, extra_flags = parse_cpu_model() flags = HgFlags(cpu_model, extra_flags) logging.debug("Unsupported flags %s.", str(flags.host_all_unsupported_flags)) cpuf_model = cpu_model + ",check" # Add unsupported flags. for fadd in flags.host_all_unsupported_flags: cpuf_model += ",+" + str(fadd) vnc_port = virt_utils.find_free_port(5900, 6100) - 5900 cmd = "%s -cpu %s -vnc :%d" % (qemu_binary, cpuf_model, vnc_port) out = None try: try: out = utils.run(cmd, timeout=5, ignore_status=True).stderr raise error.TestFail("Guest not boot with unsupported " "flags.") except error.CmdError, e: out = e.result_obj.stderr finally: uns_re = re.compile("^warning:.*flag '(.+)'", re.MULTILINE) warn_flags = set(map(virt_utils.Flag, uns_re.findall(out))) fwarn_flags = flags.host_all_unsupported_flags - warn_flags if fwarn_flags: raise error.TestFail("Qemu did not warn the use of " "flags %s" % str(fwarn_flags))
def virsh_capabilities(option): cmd = "virsh capabilities %s" % option cmd_result = utils.run(cmd, ignore_status=True) logging.info("Output: %s", cmd_result.stdout.strip()) logging.error("Error: %s", cmd_result.stderr.strip()) logging.info("Status: %d", cmd_result.exit_status) return cmd_result.exit_status, cmd_result.stdout.strip()
def virsh_version(option): cmd = "virsh version %s" % option cmd_result = utils.run(cmd, ignore_status=True) logging.debug("Output: %s", cmd_result.stdout.strip()) logging.debug("Error: %s", cmd_result.stderr.strip()) logging.debug("Status: %d", cmd_result.exit_status) return cmd_result.exit_status
def _call_shim_with_args(self, args, protobuffer): """Calls the crypto_util shim with args and stdin as specified. Calls crypto shim with args for paramters, and the serialized form of protobuffer on stdin. Asserts that the command completes successfully and returns the raw output bytestring. @param args: tuple of string arguments to the shim. @param protobuffer: python protocol buffer object. @return stdout of the shim call as string. """ raw_input_bytes = protobuffer.SerializeToString() result = utils.run(self.shim_path, stderr_tee=utils.TEE_TO_LOGS, stdin=raw_input_bytes, verbose=True, args=args, timeout=10, ignore_status=True) if result.exit_status: return None raw_output = result.stdout logging.debug('Got raw output: %s.', ''.join([hex(ord(c)) for c in raw_output])) return raw_output
def virsh_hostname(option): cmd = "virsh hostname %s" % option cmd_result = utils.run(cmd, ignore_status=True) logging.debug("Output: %s", cmd_result.stdout.strip()) logging.debug("Error: %s", cmd_result.stderr.strip()) logging.debug("Status: %d", cmd_result.exit_status) return cmd_result.exit_status, cmd_result.stdout.strip()
def run_once(self): # Get the start line of message belonging to this current boot. start_line = utils.run( 'grep -ni "%s" "%s" | tail -n 1 | cut -d ":" -f 1' % (self.BOOT_START_LINE_MSG, self.MESSAGE_PATH), ignore_status=True).stdout.strip() logging.info('Start line: %s', start_line) if not start_line: raise error.TestFail('Start line of boot is not found.') def _grep_messages(pattern): return utils.run('tail -n +%s %s | grep "%s"' % (start_line, self.MESSAGE_PATH, pattern), ignore_status=True).stdout # Check hammerd is triggered on boot. if not _grep_messages(self.TRIGGER_ON_BOOT_MSG): raise error.TestFail('hammerd is not triggered on boot.') # Check hammerd is terminated normally. if _grep_messages(self.PROCESS_FAILED_MSG): hammerd_log = _grep_messages(self.PROCESS_NAME) logging.error('Hammerd log: %s', hammerd_log) raise error.TestFail('hammerd terminated with non-zero value') # Check that hammerd wrote to WRITE_SYSFS_PATH sysfspath = utils.read_one_line(self.WRITE_SYSFS_PATH) if not sysfspath: raise error.TestFail('%s is empty' % (self.WRITE_SYSFS_PATH)) # Check that autosuspend is enabled power = utils.read_one_line(sysfspath + '/' + self.SYSFS_POWER_CONTROL_PATH) if power != 'auto': raise error.TestFail('Autosuspend not enabled on hammer port')
def remove_pkg_file(self, filename, pkg_dir): ''' Remove the file named filename from pkg_dir ''' try: # Remove the file if pkg_dir.startswith('ssh://'): hostline, remote_path = parse_ssh_path(pkg_dir) path = os.path.join(remote_path, filename) utils.run("ssh %s 'rm -rf %s/%s'" % (hostline, remote_path, path)) else: os.remove(os.path.join(pkg_dir, filename)) except (IOError, os.error), why: raise error.PackageRemoveError("Could not remove %s from %s: %s " % (filename, pkg_dir, why))
def test(self, cpu_model, extra_flags): #This is virtual cpu flags which are supported by #qemu but no with host cpu. flags = HgFlags(cpu_model, extra_flags) logging.debug("Unsupported flags %s.", str(flags.host_all_unsupported_flags)) cpuf_model = cpu_model + ",enforce" # Add unsupported flags. for fadd in flags.host_all_unsupported_flags: cpuf_model += ",+" + fadd cmd = qemu_binary + " -cpu " + cpuf_model out = None try: try: out = utils.run(cmd, timeout=5, ignore_status=True).stderr except error.CmdError: logging.error("Host boot with unsupported flag") finally: uns_re = re.compile("^warning:.*flag '(.+)'", re.MULTILINE) warn_flags = set(map(virt_utils.Flag, uns_re.findall(out))) fwarn_flags = flags.host_all_unsupported_flags - warn_flags if fwarn_flags: raise error.TestFail("Qemu did not warn the use of " "flags %s" % str(fwarn_flags))
def _get_cpufreq_paths(filename): """ Returns a list of paths to the governors. """ cmd = 'ls /sys/devices/system/cpu/cpu*/cpufreq/' + filename paths = utils.run(cmd, verbose=False).stdout.splitlines() return paths
def _upload_files(host, path, result_pattern, multiprocessing): keyval = models.test.parse_job_keyval(host) build = keyval.get('build') suite = keyval.get('suite') if not _is_valid_result(build, result_pattern, suite): # No need to upload current folder, return. return parent_job_id = str(keyval['parent_job_id']) folders = path.split(os.sep) job_id = folders[-6] package = folders[-4] timestamp = folders[-1] # Path: bucket/build/parent_job_id/cheets_CTS.*/job_id_timestamp/ # or bucket/build/parent_job_id/cheets_GTS.*/job_id_timestamp/ cts_apfe_gs_path = os.path.join(DEFAULT_CTS_APFE_GSURI, build, parent_job_id, package, job_id + '_' + timestamp) + '/' # Path: bucket/cheets_CTS.*/job_id_timestamp/ # or bucket/cheets_GTS.*/job_id_timestamp/ test_result_gs_path = os.path.join(DEFAULT_CTS_RESULTS_GSURI, package, job_id + '_' + timestamp) + '/' for zip_file in glob.glob(os.path.join('%s.zip' % path)): utils.run(' '.join( _get_cmd_list(multiprocessing, zip_file, cts_apfe_gs_path))) logging.debug('Upload %s to %s ', zip_file, cts_apfe_gs_path) for test_result_file in glob.glob(os.path.join(path, result_pattern)): # gzip test_result_file(testResult.xml/test_result.xml) test_result_file_gz = '%s.gz' % test_result_file with open(test_result_file, 'r') as f_in, (gzip.open(test_result_file_gz, 'w')) as f_out: shutil.copyfileobj(f_in, f_out) utils.run(' '.join( _get_cmd_list(multiprocessing, test_result_file_gz, test_result_gs_path))) logging.debug('Zip and upload %s to %s', test_result_file_gz, test_result_gs_path) # Remove test_result_file_gz(testResult.xml.gz/test_result.xml.gz) os.remove(test_result_file_gz)
def verify_user(require_sudo=True): """Checks that the current user is not root, but has sudo. Running unit tests as root can mask permissions problems, as not all the code runs as root in production. """ # Ensure this process is not running as root. if getpass.getuser() == 'root': raise EnvironmentError('Unittests should not be run as root.') # However, most of the unit tests do require sudo. # TODO(dshi): crbug.com/459344 Set remove this enforcement when test # container can be unprivileged container. if require_sudo and utils.sudo_require_password(): logging.warn('SSP requires root privilege to run commands, please ' 'grant root access to this process.') utils.run('sudo true')
def testEnableNotificationUsingCredentialsInBucketFail(self): config_mock = self.mox.CreateMockAnything() moblab_rpc_interface._CONFIG = config_mock config_mock.get_config_value( 'CROS', 'image_storage_server').AndReturn('gs://bucket1/') self.mox.StubOutWithMock(moblab_rpc_interface.GsUtil, 'get_gsutil_cmd') moblab_rpc_interface.GsUtil.get_gsutil_cmd().AndReturn( '/path/to/gsutil') self.mox.StubOutWithMock(common_lib_utils, 'run') common_lib_utils.run('/path/to/gsutil', args=('cp', 'gs://bucket1/pubsub-key-do-not-delete.json', '/tmp')).AndRaise( error.CmdError("fakecommand", common_lib_utils.CmdResult(), "")) self.mox.ReplayAll() moblab_rpc_interface._enable_notification_using_credentials_in_bucket()
def main(argv): common_util.setup_logging() # We parse even though we don't use args here to ensure our args are valid # before we ssh anywhere. common_util.parse_args(argv) master = autotest_master() # Take the first user. main_user = autotest_user() remote_path = os.path.join(_AUTOTEST_PATH, _HELPER_PATH) command = ('ssh %(master)s -- become %(user)s -- ' '%(remote_path)s/lab_deploy_helper.py %(argv)s' % dict(remote_path=remote_path, master=master, user=main_user, argv=' '.join(argv))) logging.info('Running %s' % command) utils.run(command, stderr_tee=sys.stderr, stdout_tee=sys.stdout)
def submit_wizard_config_info(cloud_storage_info, wifi_info): """RPC handler to submit the cloud storage info. @param cloud_storage_info: The JSON RPC object for cloud storage info. @param wifi_info: The JSON RPC object for DUT wifi info. """ config_update = {} config_update['CROS'] = [ (_IMAGE_STORAGE_SERVER, cloud_storage_info[_IMAGE_STORAGE_SERVER]), (_RESULT_STORAGE_SERVER, cloud_storage_info[_RESULT_STORAGE_SERVER]) ] config_update['MOBLAB'] = [(_WIFI_AP_NAME, wifi_info.get(_WIFI_AP_NAME) or ''), (_WIFI_AP_PASS, wifi_info.get(_WIFI_AP_PASS) or '')] _update_partial_config(config_update) if not cloud_storage_info[_USE_EXISTING_BOTO_FILE]: boto_config = ConfigParser.RawConfigParser() boto_config.add_section('Credentials') boto_config.set('Credentials', _GS_ACCESS_KEY_ID, cloud_storage_info[_GS_ACCESS_KEY_ID]) boto_config.set('Credentials', _GS_SECRET_ACCESS_KEY, cloud_storage_info[_GS_SECRET_ACCESS_KEY]) _write_config_file(MOBLAB_BOTO_LOCATION, boto_config, True) _CONFIG.parse_config_file() _enable_notification_using_credentials_in_bucket() services = [ 'moblab-devserver-init', 'moblab-devserver-cleanup-init', 'moblab-gsoffloader_s-init', 'moblab-scheduler-init', 'moblab-gsoffloader-init' ] cmd = 'export ATEST_RESULTS_DIR=/usr/local/autotest/results;' cmd += 'sudo stop ' + ';sudo stop '.join(services) cmd += ';sudo start ' + ';sudo start '.join(services) cmd += ';sudo apache2 -k graceful' logging.info(cmd) try: utils.run(cmd) except error.CmdError as e: logging.error(e) # if all else fails reboot the device. utils.run('sudo reboot') return _create_operation_status_response(True, None)
def reset_connector_if_applicable(self, connector_type): """Resets Type-C video connector from host end if applicable. It's the workaround sequence since sometimes Type-C dongle becomes corrupted and needs to be re-plugged. @param connector_type: A string, like "VGA", "DVI", "HDMI", or "DP". """ if connector_type != 'HDMI' and connector_type != 'DP': return # Decide if we need to add --name=cros_pd usbpd_command = 'ectool --name=cros_pd usbpd' try: common_utils.run('%s 0' % usbpd_command) except error.CmdError: usbpd_command = 'ectool usbpd' port = 0 while port < self.MAX_TYPEC_PORT: # We use usbpd to get Role information and then power cycle the # SRC one. command = '%s %d' % (usbpd_command, port) try: output = common_utils.run(command).stdout if re.compile('Role.*SRC').search(output): logging.info('power-cycle Type-C port %d', port) common_utils.run('%s sink' % command) common_utils.run('%s auto' % command) port += 1 except error.CmdError: break
def _ensure_deps(self, dut, test_name): """ Ensure the dependencies are locally available on DUT. @param dut: The autotest host object representing DUT. @param test_name: Name of the telemetry test. """ # Get DEPs using host's telemetry. format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') command = format_string % (self._telemetry_path, test_name) stdout = StringIO.StringIO() stderr = StringIO.StringIO() if self._devserver: devserver_hostname = self._devserver.url().split( 'http://')[1].split(':')[0] command = 'ssh %s %s' % (devserver_hostname, command) logging.info('Getting DEPs: %s', command) try: result = utils.run(command, stdout_tee=stdout, stderr_tee=stderr) except error.CmdError as e: logging.debug('Error occurred getting DEPs: %s\n %s\n', stdout.getvalue(), stderr.getvalue()) raise error.TestFail('Error occurred while getting DEPs.') # Download DEPs to DUT. # send_file() relies on rsync over ssh. Couldn't be better. stdout_str = stdout.getvalue() stdout.close() stderr.close() for dep in stdout_str.split(): src = os.path.join(self._telemetry_path, dep) dst = os.path.join(DUT_CHROME_ROOT, dep) if self._devserver: logging.info('Copying: %s -> %s', src, dst) rsync_cmd = utils.sh_escape('rsync %s %s %s:%s' % (self._host.rsync_options(), src, self._host.hostname, dst)) utils.run('ssh %s "%s"' % (devserver_hostname, rsync_cmd)) else: if not os.path.isfile(src): raise error.TestFail('Error occurred while saving DEPs.') logging.info('Copying: %s -> %s', src, dst) dut.send_file(src, dst)
def main(argv): """ Entry point for test_droid script. @param argv: arguments list """ arguments = _parse_arguments_internal(argv) serials = arguments.serials if serials is None: result = utils.run(['adb', 'devices']) devices = adb_host.ADBHost.parse_device_serials(result.stdout) if len(devices) != 1: logging.error('could not detect exactly one device; please select ' 'one with -s: %s', devices) return 1 serials = devices[0] results_directory = test_runner_utils.create_results_directory( arguments.results_dir) arguments.results_dir = results_directory autotest_path = os.path.dirname(os.path.dirname( os.path.realpath(__file__))) site_utils_path = os.path.join(autotest_path, 'site_utils') realpath = os.path.realpath(__file__) site_utils_path = os.path.realpath(site_utils_path) host_attributes = {'serials' : serials, 'os_type' : 'android'} if arguments.fastboot_serial: host_attributes['fastboot_serial'] = arguments.fastboot_serial fb_service = None try: # Start the feedback service if needed. if arguments.interactive: fb_service = tester_feedback.FeedbackService() fb_service.start() if arguments.args: arguments.args += ' ' else: arguments.args = '' arguments.args += ( 'feedback=interactive feedback_args=localhost:%d' % fb_service.server_port) return test_runner_utils.perform_run_from_autotest_root( autotest_path, argv, arguments.tests, arguments.remote, args=arguments.args, ignore_deps=not arguments.enforce_deps, results_directory=results_directory, iterations=arguments.iterations, fast_mode=arguments.fast_mode, debug=arguments.debug, host_attributes=host_attributes, pretend=arguments.pretend) finally: if fb_service is not None: fb_service.stop()
def is_shill_running(): """ Check if shill is running. @return True or False. """ cmd_result = utils.run("status shill", ignore_status=True) return cmd_result.stdout.find("start/running") != -1
def run_canned_update(self, allow_failure): """ Performs the update. @param allow_failure: True if we dont raise an error on failure. """ utils.run('restart update-engine') self._omaha.start() try: utils.run('update_engine_client -update -omaha_url=' + 'http://127.0.0.1:%d/update ' % self._omaha.get_port()) except error.CmdError as e: self._omaha.stop() if not allow_failure: raise error.TestFail('Update attempt failed: %s' % e) else: logging.info('Ignoring failed update. Failure reason: %s', e)
def limit_file_count(dir_entry): """Limit the number of files in given directory. The method checks the total number of files in the given directory. If the number is greater than MAX_FILE_COUNT, the method will compress each folder in the given directory, except folders in FOLDERS_NEVER_ZIP. @param dir_entry: Directory entry to be checked. """ count = utils.run('find "%s" | wc -l' % dir_entry, ignore_status=True).stdout.strip() try: count = int(count) except (ValueError, TypeError): logging.warning('Fail to get the file count in folder %s.', dir_entry) return if count < MAX_FILE_COUNT: return # For test job, zip folders in a second level, e.g. 123-debug/host1. # This is to allow autoserv debug folder still be accessible. # For special task, it does not need to dig one level deeper. is_special_task = re.match(job_directories.SPECIAL_TASK_PATTERN, dir_entry) folders = _get_zippable_folders(dir_entry) if not is_special_task: subfolders = [] for folder in folders: subfolders.extend(_get_zippable_folders(folder)) folders = subfolders for folder in folders: try: zip_name = '%s.tgz' % folder utils.run('tar -cz -C "%s" -f "%s" "%s"' % (os.path.dirname(folder), zip_name, os.path.basename(folder))) except error.CmdError as e: logging.error('Fail to compress folder %s. Error: %s', folder, e) continue shutil.rmtree(folder)
def _get_builds_for_in_directory(directory_name, milestone_limit=3, build_limit=20): """ Fetch the most recent builds for the last three milestones from gcs. @param directory_name: The sub-directory under the configured GCS image storage bucket to search. @return: A string list no longer than <milestone_limit> x <build_limit> items, containing the most recent <build_limit> builds from the last milestone_limit milestones. """ output = StringIO.StringIO() gs_image_location = _CONFIG.get_config_value('CROS', _IMAGE_STORAGE_SERVER) utils.run(GsUtil.get_gsutil_cmd(), args=('ls', gs_image_location + directory_name), stdout_tee=output) lines = output.getvalue().split('\n') output.close() builds = [ line.replace(gs_image_location, '').strip('/ ') for line in lines if line != '' ] build_matcher = re.compile(r'^.*\/R([0-9]*)-.*') build_map = {} for build in builds: match = build_matcher.match(build) if match: milestone = match.group(1) if milestone not in build_map: build_map[milestone] = [] build_map[milestone].append(build) milestones = build_map.keys() milestones.sort() milestones.reverse() build_list = [] for milestone in milestones[:milestone_limit]: builds = build_map[milestone] builds.sort(key=_get_sortable_build_number) builds.reverse() build_list.extend(builds[:build_limit]) return build_list
def get_default_netdev(): """ Get the name of the network device with the default route. @return A string such as "eth0" or "wlan0". """ cmd_result = utils.run( "ip route show default match 0/0 | awk '{print $5}'") return cmd_result.stdout
def test_safe_args(self): # NOTE: The string in expected_quoted_cmd depends on the internal # implementation of shell quoting which is used by utils.run(), # in this case, sh_quote_word(). expected_quoted_cmd = "echo 'hello \"world' again" self.__check_result(utils.run('echo', verbose=False, args=('hello "world', 'again')), expected_quoted_cmd, stdout='hello "world again\n')
def test_stdout_stderr_tee(self): cmd = 'echo output && echo error >&2' stdout_tee = StringIO.StringIO() stderr_tee = StringIO.StringIO() self.__check_result(utils.run( cmd, stdout_tee=stdout_tee, stderr_tee=stderr_tee, verbose=False), cmd, stdout='output\n', stderr='error\n') self.assertEqual(stdout_tee.getvalue(), 'output\n') self.assertEqual(stderr_tee.getvalue(), 'error\n')
def run_once(self, dir=None, nprocs=None, args=''): (lower, upper) = utils.get_ip_local_port_range() utils.set_ip_local_port_range(4096, 65535) try: result = utils.run(os.path.join(self.srcdir, 'ipv6connect'), None, False, stdout_tee=sys.stdout, stderr_tee=sys.stderr) finally: utils.set_ip_local_port_range(lower, upper) self.results.append(result.stderr)
def run_banner_output(cmd): """Returns ------ CMD ------\nCMD_OUTPUT in a string""" banner_output = '%s\n%%s\n\n' % cmd.center(60, '-') command_output = '' try: cmd_out = utils.run(cmd, ignore_status=True, timeout=30) command_output = cmd_out.stdout + cmd_out.stderr except error.CmdError: command_output = 'Timed out' return banner_output % command_output
def get_cpu_models(): """ Get all cpu models from qemu. @return: cpu models. """ cmd = qemu_binary + " -cpu ?" output = utils.run(cmd).stdout cpu_re = re.compile("\w+\s+\[?(\w+)\]?") return cpu_re.findall(output)
def run_once(self): cmd = os.path.join(self.srcdir, 'gbmtest') result = utils.run(cmd, stderr_is_expected=False, stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, ignore_status=True) report = re.findall(r'\[ PASSED \]', result.stdout) if not report: raise error.TestFail('Failed: Gbm test failed (' + result.stdout + ')')
def initialize(self, results_dir): temporary_directory = os.path.join(results_dir, _TEMPORARY_DIRECTORY) if os.path.exists(temporary_directory): # TODO crbug.com/391111: before we have a better solution to # periodically cleanup tmp files, we have to use rm -rf to delete # the whole folder. shutil.rmtree has performance issue when a # folder has large amount of files, e.g., drone_tmp. os.system('rm -rf %s' % temporary_directory) self._ensure_directory_exists(temporary_directory) # TODO (sbasi) crbug.com/345011 - Remove this configuration variable # and clean up build_externals so it NO-OP's. build_externals = global_config.global_config.get_config_value( scheduler_config.CONFIG_SECTION, 'drone_build_externals', default=True, type=bool) if build_externals: build_extern_cmd = os.path.join(common.autotest_dir, 'utils', 'build_externals.py') utils.run(build_extern_cmd)
def get_all_qemu_flags(): cmd = qemu_binary + " -cpu ?cpuid" output = utils.run(cmd).stdout flags_re = re.compile(r".*\n.*f_edx:(.*)\n.*f_ecx:(.*)\n.*extf_edx:" "(.*)\n.*extf_ecx:(.*)") m = flags_re.search(output) flags = [] for a in m.groups(): flags += a.split() return set(map(virt_utils.Flag, flags))
def _check_code(self): """ Verifies the file with run_pylint.py. This tool will call the static code checker pylint using the special autotest conventions and warn only on problems. If problems are found, a report will be generated. Some of the problems reported might be bogus, but it's allways good to look at them. """ non_py_ended = None if not self.path.endswith(".py"): path = "%s-cp.py" % self.path utils.run("cp %s %s" % (self.path, path), verbose=False) non_py_ended = " (filename has no .py)" else: path = self.path c_cmd = 'run_pylint.py %s' % path try: utils.run(c_cmd, verbose=False) except error.CmdError, e: e_msg = "Found syntax issues during '%s'" % c_cmd if non_py_ended is not None: e_msg += non_py_ended utils.run("rm %s" % path, verbose=False) logging.error(e_msg) for stdout_line in e.result_obj.stdout.split("\n"): if stdout_line: logging.error(" [stdout]: %s", stdout_line) for stderr_line in e.result_obj.stderr.split("\n"): if stderr_line: logging.error(" [stdout]: %s", stderr_line) logging.error("")
def host_is_supported(cls, hostname, conmux_server=None, conmux_attach=None): """ Returns a boolean indicating if the remote host with "hostname" supports use as a SerialHost """ conmux_attach = cls._get_conmux_attach(conmux_attach) conmux_hostname = cls._get_conmux_hostname(hostname, conmux_server) cmd = "%s %s echo 2> /dev/null" % (conmux_attach, conmux_hostname) try: result = utils.run(cmd, ignore_status=True, timeout=10) return result.exit_status == 0 except error.CmdError: logging.warning("Timed out while trying to attach to conmux") return False
def execute(self): os.chdir(self.tmpdir) tasks_bin = os.path.join(self.srcdir, "tasks") p = subprocess.Popen([tasks_bin, " 25000"]) time.sleep(5) try: result = utils.run("cat /dev/cpuset/autotest_container/tasks", ignore_status=True) except IOError: utils.nuke_subprocess(p) raise error.TestFail("cat cpuset/tasks failed with IOError") utils.nuke_subprocess(p) if result and result.exit_status: raise error.TestFail("cat cpuset/tasks failed")
def test(self): cpu_models = params.get("cpu_models", "core2duo").split() cmd = qemu_binary + " -cpu ?dump" result = utils.run(cmd) cpu_models = map(separe_cpu_model, cpu_models) missing = [] for cpu_model in cpu_models: if not cpu_model in result.stdout: missing.append(cpu_model) if missing: raise error.TestFail("CPU models %s are not in output " "'%s' of command \n%s" % (missing, cmd, result.stdout))
def configure_crash_handler(self): """ Configure the crash handler by: * Setting up core size to unlimited * Putting an appropriate crash handler on /proc/sys/kernel/core_pattern * Create files that the crash handler will use to figure which tests are active at a given moment The crash handler will pick up the core file and write it to self.debugdir, and perform analysis on it to generate a report. The program also outputs some results to syslog. If multiple tests are running, an attempt to verify if we still have the old PID on the system process table to determine whether it is a parent of the current test execution. If we can't determine it, the core file and the report file will be copied to all test debug dirs. """ self.crash_handling_enabled = False # make sure this script will run with a new enough python to work cmd = ("python -c 'import sys; " "print sys.version_info[0], sys.version_info[1]'") result = utils.run(cmd, ignore_status=True, verbose=False) if result.exit_status != 0: logging.warning('System python is too old, crash handling disabled') return major, minor = [int(x) for x in result.stdout.strip().split()] if (major, minor) < (2, 4): logging.warning('System python is too old, crash handling disabled') return self.pattern_file = '/proc/sys/kernel/core_pattern' try: # Enable core dumps resource.setrlimit(resource.RLIMIT_CORE, (-1, -1)) # Trying to backup core pattern and register our script self.core_pattern_backup = open(self.pattern_file, 'r').read() pattern_file = open(self.pattern_file, 'w') tools_dir = os.path.join(self.autodir, 'tools') crash_handler_path = os.path.join(tools_dir, 'crash_handler.py') pattern_file.write('|' + crash_handler_path + ' %p %t %u %s %h %e') # Writing the files that the crash handler is going to use self.debugdir_tmp_file = ('/tmp/autotest_results_dir.%s' % os.getpid()) utils.open_write_close(self.debugdir_tmp_file, self.debugdir + "\n") except Exception, e: logging.warning('Crash handling disabled: %s', e)