def CreateAndOwnDir(self, dir_name): retv = 0 if not os.path.exists(dir_name): command = 'mkdir -p ' + dir_name command += ' || sudo mkdir -p ' + dir_name retv = command_executer.GetCommandExecuter().RunCommand(command) if retv != 0: return retv pw = pwd.getpwnam(self.owner) if os.stat(dir_name).st_uid != pw.pw_uid: command = 'sudo chown -f ' + self.owner + ' ' + dir_name retv = command_executer.GetCommandExecuter().RunCommand(command) return retv
def __init__(self, board, remotes, configs, clean, public, force_mismatch, noschedv2=False): self._board = board self._remotes = remotes self._chromeos_root = 'chromeos' self._configs = configs self._clean = clean self._public = public self._force_mismatch = force_mismatch self._ce = command_executer.GetCommandExecuter() self._l = logger.GetLogger() timestamp = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%H:%M:%S') self._reports_dir = os.path.join( NIGHTLY_TESTS_DIR, '%s.%s' % (timestamp, board), ) self._noschedv2 = noschedv2 ChromeOSCheckout.__init__(self, board, self._chromeos_root)
def __init__(self, name, benchmark, label, iteration, cache_conditions, machine_manager, logger_to_use, log_level, share_cache): threading.Thread.__init__(self) self.name = name self._logger = logger_to_use self.log_level = log_level self.benchmark = benchmark self.iteration = iteration self.label = label self.result = None self.terminated = False self.retval = None self.run_completed = False self.machine_manager = machine_manager self.suite_runner = SuiteRunner(self._logger, self.log_level) self.machine = None self.cache_conditions = cache_conditions self.runs_complete = 0 self.cache_hit = False self.failure_reason = '' self.test_args = benchmark.test_args self.cache = None self.profiler_args = self.GetExtraAutotestArgs() self._ce = command_executer.GetCommandExecuter( self._logger, log_level=self.log_level) self.timeline = timeline.Timeline() self.timeline.Record(STATUS_PENDING) self.share_cache = share_cache self.cache_has_been_read = False # This is used by schedv2. self.owner_thread = None
def __init__(self, name, source_path, chromeos_root, board, incremental, build_env, gcc_enable_ccache=False): self._name = name self._source_path = misc.CanonicalizePath(source_path) self._chromeos_root = chromeos_root self._board = board self._ctarget = misc.GetCtargetFromBoard(self._board, self._chromeos_root) self._gcc_libs_dest = misc.GetGccLibsDestForBoard( self._board, self._chromeos_root) self.tag = '%s-%s' % (name, self._ctarget) self._ce = command_executer.GetCommandExecuter() self._mask_file = os.path.join( self._chromeos_root, 'chroot', 'etc/portage/package.mask/cross-%s' % self._ctarget) self._new_mask_file = None self._chroot_source_path = os.path.join( constants.MOUNTED_TOOLCHAIN_ROOT, self._name).lstrip('/') self._incremental = incremental self._build_env = build_env self._gcc_enable_ccache = gcc_enable_ccache
def EmailReport(report_file, report_type, date): subject = '%s Waterfall Summary report, %s' % (report_type, date) email_to = getpass.getuser() sendgmr_path = '/google/data/ro/projects/gws-sre/sendgmr' command = ('%s --to=%[email protected] --subject="%s" --body_file=%s' % (sendgmr_path, email_to, subject, report_file)) command_executer.GetCommandExecuter().RunCommand(command)
def __init__(self, chromeos_root, ndk_dir, gcc_branch=None, gcc_dir=None, binutils_branch=None, binutils_dir=None, board=None, disable_2nd_bootstrap=False, setup_tool_ebuild_file_only=False): self._chromeos_root = chromeos_root self._ndk_dir = ndk_dir self._gcc_branch = gcc_branch self._gcc_branch_tree = None self._gcc_dir = gcc_dir self._gcc_ebuild_file = None self._gcc_ebuild_file_name = None self._binutils_branch = binutils_branch self._binutils_branch_tree = None self._binutils_dir = binutils_dir self._binutils_ebuild_file = None self._binutils_ebuild_file_name = None self._setup_tool_ebuild_file_only = setup_tool_ebuild_file_only self._ce = command_executer.GetCommandExecuter() self._logger = logger.GetLogger() self._board = board self._disable_2nd_bootstrap = disable_2nd_bootstrap
def ResetToolEbuildFile(chromeos_root, tool_name): """Reset tool ebuild file to clean state. Args: chromeos_root: chromeos source tree tool_name: either "gcc" or "binutils" Returns: True if operation succeds. """ rv = misc.GetGitChangesAsList( os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH), path=('sys-devel/{0}/{0}-*.ebuild'.format(tool_name)), staged=False) if rv: cmd = 'cd {0} && git checkout --'.format(os.path.join( chromeos_root, CHROMIUMOS_OVERLAY_PATH)) for g in rv: cmd += ' ' + g rv = command_executer.GetCommandExecuter().RunCommand(cmd) if rv: logger.GetLogger().LogWarning( 'Failed to reset the ebuild file. Please refer to log above.') return False else: logger.GetLogger().LogWarning( 'Note - did not find any modified {0} ebuild file.'.format(tool_name)) # Fall through return True
def Main(): ce = command_executer.GetCommandExecuter() _, l, _ = ce.RunCommandWOutput('cat {0} | wc -l'.format( common.OBJECTS_FILE), print_to_console=False) for i in range(0, int(l)): print(i)
def __init__(self, board, remote, gdb_dir, chromeos_root, cleanup): self._board = board self._remote = remote self._gdb_dir = gdb_dir self._chromeos_root = chromeos_root self._cleanup = cleanup self._cmd_exec = command_executer.GetCommandExecuter()
def __init__(self, base_dir, mount, chromeos_root, remote, board, flags, keep_intermediate_files, tools, cleanup): self._l = logger.GetLogger() self._chromeos_root = chromeos_root self._chromeos_chroot = path.join(chromeos_root, 'chroot') if mount: self._gcc_source_dir_to_mount = mount self._gcc_source_dir = path.join(constants.MOUNTED_TOOLCHAIN_ROOT, 'gcc') else: self._gcc_source_dir = None self._remote = remote self._board = board ## Compute target from board self._target = misc.GetCtargetFromBoard(board, chromeos_root) if not self._target: raise ValueError('Unsupported board "%s"' % board) self._executer = command_executer.GetCommandExecuter() self._flags = flags or '' self._base_dir = base_dir self._tmp_abs = None self._keep_intermediate_files = keep_intermediate_files self._tools = tools.split(',') self._cleanup = cleanup
def __init__(self, get_initial_items, switch_to_good, switch_to_bad, test_setup_script, test_script, incremental, prune, iterations, prune_iterations, verify, file_args, verbose): """BinarySearchState constructor, see Run for full args documentation.""" self.get_initial_items = get_initial_items self.switch_to_good = switch_to_good self.switch_to_bad = switch_to_bad self.test_setup_script = test_setup_script self.test_script = test_script self.incremental = incremental self.prune = prune self.iterations = iterations self.prune_iterations = prune_iterations self.verify = verify self.file_args = file_args self.verbose = verbose self.l = logger.GetLogger() self.ce = command_executer.GetCommandExecuter() self.resumed = False self.prune_cycles = 0 self.search_cycles = 0 self.binary_search = None self.all_items = None self.PopulateItemsUsingCommand(self.get_initial_items) self.currently_good_items = set([]) self.currently_bad_items = set([]) self.found_items = set([]) self.known_good = set([]) self.start_time = time.time()
def __init__(self, base_dir, source_dir, chromeos_root, remote, board, cleanup): self._l = logger.GetLogger() self._chromeos_root = chromeos_root self._chromeos_chroot = path.join(chromeos_root, 'chroot') self._remote = remote self._board = board ## Compute target from board self._target = misc.GetCtargetFromBoard(board, chromeos_root) if not self._target: raise RuntimeError('Unsupported board "%s"' % board) self._executer = command_executer.GetCommandExecuter() self._base_dir = base_dir self._tmp_abs = None self._cleanup = cleanup self._sshflag = ('-o StrictHostKeyChecking=no ' + '-o CheckHostIP=no ' + '-o UserKnownHostsFile=$(mktemp) ') if source_dir: self._source_dir = source_dir self._mount_flag = 'USE="mounted_sources"' self.MountSource() else: self._source_dir = None self._mount_flag = ''
def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir, cmd_exec=None, lgr=None): self._lock = threading.RLock() self._all_machines = [] self._machines = [] self.image_lock = threading.Lock() self.num_reimages = 0 self.chromeos_root = None self.machine_checksum = {} self.machine_checksum_string = {} self.acquire_timeout = acquire_timeout self.log_level = log_level self.locks_dir = locks_dir self.ce = cmd_exec or command_executer.GetCommandExecuter( log_level=self.log_level) self.logger = lgr or logger.GetLogger() if self.locks_dir and not os.path.isdir(self.locks_dir): raise MissingLocksDirectory('Cannot access locks directory: %s' % self.locks_dir) self._initialized_machines = [] self.chromeos_root = chromeos_root
def DownloadLogFile(builder, buildnum, test, test_family): ce = command_executer.GetCommandExecuter() os.system('mkdir -p %s/%s/%s' % (DOWNLOAD_DIR, builder, test)) if builder in ROTATING_BUILDERS: source = ('https://uberchromegw.corp.google.com/i/chromiumos.tryserver' '/builders/%s/builds/%d/steps/%s%%20%%5B%s%%5D/logs/stdio' % (builder, buildnum, test_family, test)) build_link = ( 'https://uberchromegw.corp.google.com/i/chromiumos.tryserver' '/builders/%s/builds/%d' % (builder, buildnum)) else: source = ( 'https://uberchromegw.corp.google.com/i/chromeos/builders/%s/' 'builds/%d/steps/%s%%20%%5B%s%%5D/logs/stdio' % (builder, buildnum, test_family, test)) build_link = ( 'https://uberchromegw.corp.google.com/i/chromeos/builders/%s' '/builds/%d' % (builder, buildnum)) target = '%s/%s/%s/%d' % (DOWNLOAD_DIR, builder, test, buildnum) if not os.path.isfile(target) or os.path.getsize(target) == 0: cmd = 'sso_client %s > %s' % (source, target) status = ce.RunCommand(cmd) if status != 0: return '', '' return target, build_link
def _setup_dut_label(self): """Try to match dut image with a certain experiment label. If such match is found, we just skip doing reimage and jump to execute some benchmark_runs. """ checksum_file = '/usr/local/osimage_checksum_file' try: rv, checksum, _ = command_executer.GetCommandExecuter().\ CrosRunCommandWOutput( 'cat ' + checksum_file, chromeos_root=self._sched.get_labels(0).chromeos_root, machine=self._dut.name, print_to_console=False) if rv == 0: checksum = checksum.strip() for l in self._sched.get_labels(): if l.checksum == checksum: self._logger.LogOutput("Dut '{}' is pre-installed with '{}'".format( self._dut.name, l)) self._dut.label = l return except RuntimeError: traceback.print_exc(file=sys.stdout) self._dut.label = None
def GetVersionSpecFile(version, versions_git): temp = tempfile.mkdtemp() commands = ['cd {0}'.format(temp), \ 'git clone {0} versions'.format(versions_git)] cmd_executer = command_executer.GetCommandExecuter() ret = cmd_executer.RunCommands(commands) err_msg = None if ret: err_msg = 'Failed to checkout versions_git - {0}'.format(versions_git) ret = None else: v, m = version.split('.', 1) paladin_spec = 'paladin/buildspecs/{0}/{1}.xml'.format(v, m) generic_spec = 'buildspecs/{0}/{1}.xml'.format(v, m) paladin_path = '{0}/versions/{1}'.format(temp, paladin_spec) generic_path = '{0}/versions/{1}'.format(temp, generic_spec) if os.path.exists(paladin_path): ret = paladin_spec elif os.path.exists(generic_path): ret = generic_spec else: err_msg = 'No spec found for version {0}'.format(version) ret = None # Fall through to clean up. commands = ['rm -rf {0}'.format(temp)] cmd_executer.RunCommands(commands) if err_msg: logger.GetLogger().LogFatal(err_msg) return ret
def __init__(self, board, remotes, chromeos_root, weekday, patches, noschedv2=False): self._board = board self._remotes = remotes self._chromeos_root = chromeos_root self._base_dir = os.getcwd() self._ce = command_executer.GetCommandExecuter() self._l = logger.GetLogger() self._build = '%s-release' % board self._patches = patches.split(',') if patches else [] self._patches_string = '_'.join(str(p) for p in self._patches) self._noschedv2 = noschedv2 if not weekday: self._weekday = time.strftime('%a') else: self._weekday = weekday timestamp = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%H:%M:%S') self._reports_dir = os.path.join( NIGHTLY_TESTS_DIR, '%s.%s' % (timestamp, board), )
def LoadState(cls): """Create BinarySearchState object from STATE_FILE.""" if not os.path.isfile(STATE_FILE): return None try: bss = pickle.load(file(STATE_FILE)) bss.l = logger.GetLogger() bss.ce = command_executer.GetCommandExecuter() bss.binary_search.logger = bss.l bss.start_time = time.time() # Set resumed to be True so we can enter DoBinarySearch without the method # resetting our current search_cycles to 0. bss.resumed = True # Set currently_good_items and currently_bad_items to empty so that the # first iteration after resuming will always be non-incremental. This is # just in case the environment changes, the user makes manual changes, or # a previous switch_script corrupted the environment. bss.currently_good_items = set([]) bss.currently_bad_items = set([]) binary_search_perforce.verbose = bss.verbose return bss except StandardError: return None
def RunRemote(chromeos_root, branch, patches, is_local, target, chrome_version, dest_dir): """The actual running commands.""" ce = command_executer.GetCommandExecuter() if is_local: local_flag = '--local -r {0}'.format(dest_dir) else: local_flag = '--remote' patch = '' for p in patches: patch += ' -g {0}'.format(p) cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot') os.chdir(cbuildbot_path) branch_flag = '' if branch != 'master': branch_flag = ' -b {0}'.format(branch) chrome_version_flag = '' if chrome_version: chrome_version_flag = ' --chrome_version={0}'.format(chrome_version) description = '{0}_{1}_{2}'.format(branch, GetPatchString(patches), target) command = ('yes | ./cbuildbot {0} {1} {2} {3} {4} {5}' ' --remote-description={6}' ' --chrome_rev=tot'.format(patch, branch_flag, chrome_version, local_flag, chrome_version_flag, target, description)) ce.RunCommand(command) return description
def DownloadImage(target, index, dest, version): """Download artifacts from cloud.""" if not os.path.exists(dest): os.makedirs(dest) rversion = manifest_versions.RFormatCrosVersion(version) print(str(rversion)) # ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}" # .format(target, rversion, index)) ls_cmd = ('gsutil ls gs://chromeos-image-archive/trybot-{0}/*-b{2}'.format( target, index)) download_cmd = ('$(which gsutil) cp {0} {1}'.format('{0}', dest)) ce = command_executer.GetCommandExecuter() _, out, _ = ce.RunCommandWOutput(ls_cmd, print_to_console=True) lines = out.splitlines() download_files = [ 'autotest.tar', 'chromeos-chrome', 'chromiumos_test_image', 'debug.tgz', 'sysroot_chromeos-base_chromeos-chrome.tar.xz' ] for line in lines: if any([e in line for e in download_files]): cmd = download_cmd.format(line) if ce.RunCommand(cmd): logger.GetLogger().LogFatal( 'Command {0} failed, existing...'.format(cmd))
def RestartUI(remote, chromeos_root, login=True): chromeos_root = os.path.expanduser(chromeos_root) ce = command_executer.GetCommandExecuter() # First, restart ui. command = 'rm -rf %s && restart ui' % LOGIN_PROMPT_VISIBLE_MAGIC_FILE ce.CrosRunCommand(command, machine=remote, chromeos_root=chromeos_root) host_login_script = tempfile.mktemp() device_login_script = '/tmp/login.py' login_script_list = [script_header, wait_for_login_screen] if login: login_script_list.append(do_login) full_login_script_contents = '\n'.join(login_script_list) with open(host_login_script, 'w') as f: f.write(full_login_script_contents) ce.CopyFiles(host_login_script, device_login_script, dest_machine=remote, chromeos_root=chromeos_root, recursive=False, dest_cros=True) ret = ce.CrosRunCommand('python %s' % device_login_script, chromeos_root=chromeos_root, machine=remote) if os.path.exists(host_login_script): os.remove(host_login_script) return ret
def DoesImageExist(chromeos_root, build): """Check if the image for the given build exists.""" ce = command_executer.GetCommandExecuter() command = ('gsutil ls gs://chromeos-image-archive/%s' '/chromiumos_test_image.tar.xz' % (build)) ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) return not ret
def __init__(self, inputs, output, chunk_size, merge_program, multipliers): self._inputs = inputs self._output = output self._chunk_size = chunk_size self._merge_program = merge_program self._multipliers = multipliers self._ce = command_executer.GetCommandExecuter() self._l = logger.GetLogger()
def Main(argv): parser = argparse.ArgumentParser() _ = parser.parse_args(argv) command = 'sleep 1000' ce = command_executer.GetCommandExecuter() ce.RunCommand(command, command_timeout=1) return 0
def GetGSContent(chromeos_root, path): """gsutil cat path""" ce = command_executer.GetCommandExecuter() command = ('gsutil cat gs://chromeos-image-archive/%s' % path) _, out, _ = ce.ChrootRunCommandWOutput( chromeos_root, command, print_to_console=False) return out
def CreateSymlink(target, link_name): logger.GetLogger().LogFatalIf(target.startswith('/'), "Can't create symlink to absolute path!") real_from_file = misc.GetRoot(link_name)[0] + '/' + target if os.path.realpath(real_from_file) != os.path.realpath(link_name): if os.path.exists(link_name): command = 'rm -rf ' + link_name command_executer.GetCommandExecuter().RunCommand(command) os.symlink(target, link_name)
def CheckProdAccess(): status, output, _ = command_executer.GetCommandExecuter( ).RunCommandWOutput('prodcertstatus') if status != 0: return False # Verify that status is not expired if 'expires' in output: return True return False
def __init__(self, logger_to_use=None, log_level='verbose', cmd_exec=None, cmd_term=None): self.logger = logger_to_use self.log_level = log_level self._ce = cmd_exec or command_executer.GetCommandExecuter( self.logger, log_level=self.log_level) self._ct = cmd_term or command_executer.CommandTerminator()
def __init__(self, p4_port, p4_paths, test_command): VCSBinarySearcher.__init__(self) self.p4_port = p4_port self.p4_paths = p4_paths self.test_command = test_command self.checkout_dir = tempfile.mkdtemp() self.ce = command_executer.GetCommandExecuter() self.client_name = 'binary-searcher-$HOSTNAME-$USER' self.job_log_root = '/home/asharif/www/coreboot_triage/' self.changes = None
def __init__(self, image, chromeos_root, no_unmount): self.image = image self.chromeos_root = chromeos_root self.mounted = False self._ce = command_executer.GetCommandExecuter() self.logger = logger.GetLogger() self.elf_files = [] self.no_unmount = no_unmount self.unmount_script = '' self.stateful = '' self.rootfs = ''