def Main(argv):
"""The main function."""
# Common initializations
parser = argparse.ArgumentParser()
common.BuildArgParser(parser)
logger.GetLogger().LogOutput(' '.join(argv))
options = parser.parse_args(argv)
if not (options.get_initial_items and options.switch_to_good
and options.switch_to_bad
and options.test_script) and not options.resume:
parser.print_help()
return 1
if options.resume:
logger.GetLogger().LogOutput('Resuming from %s' % STATE_FILE)
if len(argv) > 1:
logger.GetLogger().LogOutput(
('Note: resuming from previous state, '
'ignoring given options and loading saved '
'options instead.'))
# Get dictionary of all options
args = vars(options)
return Run(**args)
def ResetToolEbuildFile(chromeos_root, tool_name):
"""Reset tool ebuild file to clean state.
Args:
chromeos_root: chromeos source tree
tool_name: either "gcc" or "binutils"
Returns:
True if operation succeds.
"""
rv = misc.GetGitChangesAsList(
os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH),
path=('sys-devel/{0}/{0}-*.ebuild'.format(tool_name)),
staged=False)
if rv:
cmd = 'cd {0} && git checkout --'.format(os.path.join(
chromeos_root, CHROMIUMOS_OVERLAY_PATH))
for g in rv:
cmd += ' ' + g
rv = command_executer.GetCommandExecuter().RunCommand(cmd)
if rv:
logger.GetLogger().LogWarning(
'Failed to reset the ebuild file. Please refer to log above.')
return False
else:
logger.GetLogger().LogWarning(
'Note - did not find any modified {0} ebuild file.'.format(tool_name))
# Fall through
return True
def FindBuildId(description):
"""Find the build id of the build at trybot server."""
running_time = 0
while True:
(result, number) = FindBuildIdFromLog(description)
if result >= 0:
return (result, number)
logger.GetLogger().LogOutput('{0} minutes passed.'
.format(running_time / 60))
logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
time.sleep(SLEEP_TIME)
running_time += SLEEP_TIME
def CheckForCrosFlash(chromeos_root, remote, log_level):
cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
# Check to see if remote machine has cherrypy, ctypes
command = "python -c 'import cherrypy, ctypes'"
ret = cmd_executer.CrosRunCommand(command,
chromeos_root=chromeos_root,
machine=remote)
logger.GetLogger().LogFatalIf(
ret == 255, 'Failed ssh to %s (for checking cherrypy)' % remote)
logger.GetLogger().LogFatalIf(
ret != 0, "Failed to find cherrypy or ctypes on remote '{}', "
'cros flash cannot work.'.format(remote))
def DisableCrosBeeps(chromeos_root, remote, log_level):
"""Disable annoying chromebooks beeps after reboots."""
cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
command = '/usr/share/vboot/bin/set_gbb_flags.sh 0x1'
logger.GetLogger().LogOutput('Trying to disable beeping.')
ret, o, _ = cmd_executer.CrosRunCommandWOutput(command,
chromeos_root=chromeos_root,
machine=remote)
if ret != 0:
logger.GetLogger().LogOutput(o)
logger.GetLogger().LogOutput('Failed to disable beeps.')
def DoMount(self):
ce = command_executer.GetCommandExecuter()
mount_signature = '%s on %s' % (self.external_dir, self.mount_dir)
command = 'mount'
retv, out, _ = ce.RunCommandWOutput(command)
if mount_signature not in out:
retv = self.CreateAndOwnDir(self.mount_dir)
logger.GetLogger().LogFatalIf(retv, 'Cannot create mount_dir!')
retv = self.CreateAndOwnDir(self.external_dir)
logger.GetLogger().LogFatalIf(retv, 'Cannot create external_dir!')
retv = self.MountDir()
logger.GetLogger().LogFatalIf(retv, 'Cannot mount!')
return retv
else:
return 0
def WaitForImage(chromeos_root, build):
"""Wait for an image to be ready."""
elapsed_time = 0
while elapsed_time < TIME_OUT:
if DoesImageExist(chromeos_root, build):
return
logger.GetLogger().LogOutput(
'Image %s not ready, waiting for 10 minutes' % build)
time.sleep(SLEEP_TIME)
elapsed_time += SLEEP_TIME
logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
(build, (TIME_OUT / 3600)))
raise BuildbotTimeout('Timeout while waiting for image %s' % build)
def __init__(self,
board,
remotes,
chromeos_root,
weekday,
patches,
noschedv2=False):
self._board = board
self._remotes = remotes
self._chromeos_root = chromeos_root
self._base_dir = os.getcwd()
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
self._build = '%s-release' % board
self._patches = patches.split(',') if patches else []
self._patches_string = '_'.join(str(p) for p in self._patches)
self._noschedv2 = noschedv2
if not weekday:
self._weekday = time.strftime('%a')
else:
self._weekday = weekday
timestamp = datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d_%H:%M:%S')
self._reports_dir = os.path.join(
NIGHTLY_TESTS_DIR,
'%s.%s' % (timestamp, board),
)
def DownloadImage(target, index, dest, version):
"""Download artifacts from cloud."""
if not os.path.exists(dest):
os.makedirs(dest)
rversion = manifest_versions.RFormatCrosVersion(version)
print(str(rversion))
# ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}"
# .format(target, rversion, index))
ls_cmd = ('gsutil ls gs://chromeos-image-archive/trybot-{0}/*-b{2}'.format(
target, index))
download_cmd = ('$(which gsutil) cp {0} {1}'.format('{0}', dest))
ce = command_executer.GetCommandExecuter()
_, out, _ = ce.RunCommandWOutput(ls_cmd, print_to_console=True)
lines = out.splitlines()
download_files = [
'autotest.tar', 'chromeos-chrome', 'chromiumos_test_image',
'debug.tgz', 'sysroot_chromeos-base_chromeos-chrome.tar.xz'
]
for line in lines:
if any([e in line for e in download_files]):
cmd = download_cmd.format(line)
if ce.RunCommand(cmd):
logger.GetLogger().LogFatal(
'Command {0} failed, existing...'.format(cmd))
def __init__(self,
board,
remotes,
configs,
clean,
public,
force_mismatch,
noschedv2=False):
self._board = board
self._remotes = remotes
self._chromeos_root = 'chromeos'
self._configs = configs
self._clean = clean
self._public = public
self._force_mismatch = force_mismatch
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
timestamp = datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d_%H:%M:%S')
self._reports_dir = os.path.join(
NIGHTLY_TESTS_DIR,
'%s.%s' % (timestamp, board),
)
self._noschedv2 = noschedv2
ChromeOSCheckout.__init__(self, board, self._chromeos_root)
def __init__(self,
autotest,
chromeos_root='',
chromeos_image='',
board='',
remote='',
iteration=0,
image_checksum='',
exact_remote=False,
rerun=False,
rerun_if_failed=False):
self.autotest = autotest
self.chromeos_root = chromeos_root
self.chromeos_image = chromeos_image
self.board = board
self.remote = remote
self.iteration = iteration
l = logger.GetLogger()
l.LogFatalIf(not image_checksum, "Checksum shouldn't be None")
self.image_checksum = image_checksum
self.results = {}
threading.Thread.__init__(self)
self.terminate = False
self.retval = None
self.status = 'PENDING'
self.run_completed = False
self.exact_remote = exact_remote
self.rerun = rerun
self.rerun_if_failed = rerun_if_failed
self.results_dir = None
self.full_name = None
def GetVersionSpecFile(version, versions_git):
temp = tempfile.mkdtemp()
commands = ['cd {0}'.format(temp), \
'git clone {0} versions'.format(versions_git)]
cmd_executer = command_executer.GetCommandExecuter()
ret = cmd_executer.RunCommands(commands)
err_msg = None
if ret:
err_msg = 'Failed to checkout versions_git - {0}'.format(versions_git)
ret = None
else:
v, m = version.split('.', 1)
paladin_spec = 'paladin/buildspecs/{0}/{1}.xml'.format(v, m)
generic_spec = 'buildspecs/{0}/{1}.xml'.format(v, m)
paladin_path = '{0}/versions/{1}'.format(temp, paladin_spec)
generic_path = '{0}/versions/{1}'.format(temp, generic_spec)
if os.path.exists(paladin_path):
ret = paladin_spec
elif os.path.exists(generic_path):
ret = generic_spec
else:
err_msg = 'No spec found for version {0}'.format(version)
ret = None
# Fall through to clean up.
commands = ['rm -rf {0}'.format(temp)]
cmd_executer.RunCommands(commands)
if err_msg:
logger.GetLogger().LogFatal(err_msg)
return ret
def LocateOrCopyImage(chromeos_root, image, board=None):
l = logger.GetLogger()
if board is None:
board_glob = '*'
else:
board_glob = board
chromeos_root_realpath = os.path.realpath(chromeos_root)
image = os.path.realpath(image)
if image.startswith('%s/' % chromeos_root_realpath):
return [True, image]
# First search within the existing build dirs for any matching files.
images_glob = ('%s/src/build/images/%s/*/*.bin' %
(chromeos_root_realpath, board_glob))
images_list = glob.glob(images_glob)
for potential_image in images_list:
if filecmp.cmp(potential_image, image):
l.LogOutput('Found matching image %s in chromeos_root.' %
potential_image)
return [True, potential_image]
# We did not find an image. Copy it in the src dir and return the copied
# file.
if board is None:
board = ''
base_dir = ('%s/src/build/images/%s' % (chromeos_root_realpath, board))
if not os.path.isdir(base_dir):
os.makedirs(base_dir)
temp_dir = tempfile.mkdtemp(prefix='%s/tmp' % base_dir)
new_image = '%s/%s' % (temp_dir, os.path.basename(image))
l.LogOutput('No matching image found. Copying %s to %s' %
(image, new_image))
shutil.copyfile(image, new_image)
return [False, new_image]
def __init__(self,
chromeos_root,
ndk_dir,
gcc_branch=None,
gcc_dir=None,
binutils_branch=None,
binutils_dir=None,
board=None,
disable_2nd_bootstrap=False,
setup_tool_ebuild_file_only=False):
self._chromeos_root = chromeos_root
self._ndk_dir = ndk_dir
self._gcc_branch = gcc_branch
self._gcc_branch_tree = None
self._gcc_dir = gcc_dir
self._gcc_ebuild_file = None
self._gcc_ebuild_file_name = None
self._binutils_branch = binutils_branch
self._binutils_branch_tree = None
self._binutils_dir = binutils_dir
self._binutils_ebuild_file = None
self._binutils_ebuild_file_name = None
self._setup_tool_ebuild_file_only = setup_tool_ebuild_file_only
self._ce = command_executer.GetCommandExecuter()
self._logger = logger.GetLogger()
self._board = board
self._disable_2nd_bootstrap = disable_2nd_bootstrap
def __init__(self, get_initial_items, switch_to_good, switch_to_bad,
test_setup_script, test_script, incremental, prune,
iterations, prune_iterations, verify, file_args, verbose):
"""BinarySearchState constructor, see Run for full args documentation."""
self.get_initial_items = get_initial_items
self.switch_to_good = switch_to_good
self.switch_to_bad = switch_to_bad
self.test_setup_script = test_setup_script
self.test_script = test_script
self.incremental = incremental
self.prune = prune
self.iterations = iterations
self.prune_iterations = prune_iterations
self.verify = verify
self.file_args = file_args
self.verbose = verbose
self.l = logger.GetLogger()
self.ce = command_executer.GetCommandExecuter()
self.resumed = False
self.prune_cycles = 0
self.search_cycles = 0
self.binary_search = None
self.all_items = None
self.PopulateItemsUsingCommand(self.get_initial_items)
self.currently_good_items = set([])
self.currently_bad_items = set([])
self.found_items = set([])
self.known_good = set([])
self.start_time = time.time()
def LoadState(cls):
"""Create BinarySearchState object from STATE_FILE."""
if not os.path.isfile(STATE_FILE):
return None
try:
bss = pickle.load(file(STATE_FILE))
bss.l = logger.GetLogger()
bss.ce = command_executer.GetCommandExecuter()
bss.binary_search.logger = bss.l
bss.start_time = time.time()
# Set resumed to be True so we can enter DoBinarySearch without the method
# resetting our current search_cycles to 0.
bss.resumed = True
# Set currently_good_items and currently_bad_items to empty so that the
# first iteration after resuming will always be non-incremental. This is
# just in case the environment changes, the user makes manual changes, or
# a previous switch_script corrupted the environment.
bss.currently_good_items = set([])
bss.currently_bad_items = set([])
binary_search_perforce.verbose = bss.verbose
return bss
except StandardError:
return None
def __init__(self, base_dir, mount, chromeos_root, remote, board, flags,
keep_intermediate_files, tools, cleanup):
self._l = logger.GetLogger()
self._chromeos_root = chromeos_root
self._chromeos_chroot = path.join(chromeos_root, 'chroot')
if mount:
self._gcc_source_dir_to_mount = mount
self._gcc_source_dir = path.join(constants.MOUNTED_TOOLCHAIN_ROOT,
'gcc')
else:
self._gcc_source_dir = None
self._remote = remote
self._board = board
## Compute target from board
self._target = misc.GetCtargetFromBoard(board, chromeos_root)
if not self._target:
raise ValueError('Unsupported board "%s"' % board)
self._executer = command_executer.GetCommandExecuter()
self._flags = flags or ''
self._base_dir = base_dir
self._tmp_abs = None
self._keep_intermediate_files = keep_intermediate_files
self._tools = tools.split(',')
self._cleanup = cleanup
def __init__(self, base_dir, source_dir, chromeos_root, remote, board,
cleanup):
self._l = logger.GetLogger()
self._chromeos_root = chromeos_root
self._chromeos_chroot = path.join(chromeos_root, 'chroot')
self._remote = remote
self._board = board
## Compute target from board
self._target = misc.GetCtargetFromBoard(board, chromeos_root)
if not self._target:
raise RuntimeError('Unsupported board "%s"' % board)
self._executer = command_executer.GetCommandExecuter()
self._base_dir = base_dir
self._tmp_abs = None
self._cleanup = cleanup
self._sshflag = ('-o StrictHostKeyChecking=no ' +
'-o CheckHostIP=no ' +
'-o UserKnownHostsFile=$(mktemp) ')
if source_dir:
self._source_dir = source_dir
self._mount_flag = 'USE="mounted_sources"'
self.MountSource()
else:
self._source_dir = None
self._mount_flag = ''
def __init__(self,
chromeos_root,
acquire_timeout,
log_level,
locks_dir,
cmd_exec=None,
lgr=None):
self._lock = threading.RLock()
self._all_machines = []
self._machines = []
self.image_lock = threading.Lock()
self.num_reimages = 0
self.chromeos_root = None
self.machine_checksum = {}
self.machine_checksum_string = {}
self.acquire_timeout = acquire_timeout
self.log_level = log_level
self.locks_dir = locks_dir
self.ce = cmd_exec or command_executer.GetCommandExecuter(
log_level=self.log_level)
self.logger = lgr or logger.GetLogger()
if self.locks_dir and not os.path.isdir(self.locks_dir):
raise MissingLocksDirectory('Cannot access locks directory: %s' %
self.locks_dir)
self._initialized_machines = []
self.chromeos_root = chromeos_root
def setUp(self):
self.status = []
self.called_ReadCache = None
self.log_error = []
self.log_output = []
self.err_msg = None
self.test_benchmark = Benchmark(
'page_cycler.netsim.top_10', # name
'page_cycler.netsim.top_10', # test_name
'', # test_args
1, # iterations
False, # rm_chroot_tmp
'', # perf_args
suite='telemetry_Crosperf') # suite
self.test_label = MockLabel('test1',
'image1',
'autotest_dir',
'/tmp/test_benchmark_run',
'x86-alex',
'chromeos2-row1-rack4-host9.cros',
image_args='',
cache_dir='',
cache_only=False,
log_level='average',
compiler='gcc')
self.test_cache_conditions = [
CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH
]
self.mock_logger = logger.GetLogger(log_dir='', mock=True)
self.mock_machine_manager = mock.Mock(spec=MachineManager)
def _InjectSuccesses(experiment, how_many, keyvals, for_benchmark=0,
label=None):
"""Injects successful experiment runs (for each label) into the experiment."""
# Defensive copy of keyvals, so if it's modified, we'll know.
keyvals = dict(keyvals)
num_configs = len(experiment.benchmarks) * len(experiment.labels)
num_runs = len(experiment.benchmark_runs) // num_configs
# TODO(gbiv): Centralize the mocking of these, maybe? (It's also done in
# benchmark_run_unittest)
bench = experiment.benchmarks[for_benchmark]
cache_conditions = []
log_level = 'average'
share_cache = ''
locks_dir = ''
log = logger.GetLogger()
machine_manager = MockMachineManager(
FakePath('chromeos_root'), 0, log_level, locks_dir)
machine_manager.AddMachine('testing_machine')
machine = next(m for m in machine_manager.GetMachines()
if m.name == 'testing_machine')
for label in experiment.labels:
def MakeSuccessfulRun(n):
run = MockBenchmarkRun('mock_success%d' % (n,), bench, label,
1 + n + num_runs, cache_conditions,
machine_manager, log, log_level, share_cache)
mock_result = MockResult(log, label, log_level, machine)
mock_result.keyvals = keyvals
run.result = mock_result
return run
experiment.benchmark_runs.extend(
MakeSuccessfulRun(n) for n in xrange(how_many))
return experiment
def __init__(self, inputs, output, chunk_size, merge_program, multipliers):
self._inputs = inputs
self._output = output
self._chunk_size = chunk_size
self._merge_program = merge_program
self._multipliers = multipliers
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
def CreateAutotestRuns(images,
autotests,
remote,
board,
exact_remote,
rerun,
rerun_if_failed,
main_chromeos_root=None):
autotest_runs = []
for image in images:
logger.GetLogger().LogOutput('Computing md5sum of: %s' % image)
image_checksum = FileUtils().Md5File(image)
logger.GetLogger().LogOutput('md5sum %s: %s' % (image, image_checksum))
### image_checksum = "abcdefghi"
chromeos_root = main_chromeos_root
if not main_chromeos_root:
image_chromeos_root = os.path.join(os.path.dirname(image),
'../../../../..')
chromeos_root = CanonicalizeChromeOSRoot(image_chromeos_root)
assert chromeos_root, 'chromeos_root: %s invalid' % image_chromeos_root
else:
chromeos_root = CanonicalizeChromeOSRoot(main_chromeos_root)
assert chromeos_root, 'chromeos_root: %s invalid' % main_chromeos_root
# We just need a single ChromeOS root in the MachineManagerSingleton. It is
# needed because we can save re-image time by checking the image checksum at
# the beginning and assigning autotests to machines appropriately.
if not MachineManagerSingleton().chromeos_root:
MachineManagerSingleton().chromeos_root = chromeos_root
for autotest in autotests:
for iteration in range(autotest.iterations):
autotest_run = AutotestRun(autotest,
chromeos_root=chromeos_root,
chromeos_image=image,
board=board,
remote=remote,
iteration=iteration,
image_checksum=image_checksum,
exact_remote=exact_remote,
rerun=rerun,
rerun_if_failed=rerun_if_failed)
autotest_runs.append(autotest_run)
return autotest_runs
def CreateSymlink(target, link_name):
logger.GetLogger().LogFatalIf(target.startswith('/'),
"Can't create symlink to absolute path!")
real_from_file = misc.GetRoot(link_name)[0] + '/' + target
if os.path.realpath(real_from_file) != os.path.realpath(link_name):
if os.path.exists(link_name):
command = 'rm -rf ' + link_name
command_executer.GetCommandExecuter().RunCommand(command)
os.symlink(target, link_name)
def CreateReposFromDict(self, main_dict):
for key, repo_list in main_dict.items():
for repo_dict in repo_list:
repo = self.CreateRepoFromDict(repo_dict)
if key == 'input':
self.input_repos.append(repo)
elif key == 'output':
self.output_repos.append(repo)
else:
logger.GetLogger().LogFatal('Unknown key: %s found' % key)
def RunCrosperf(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--noschedv2',
dest='noschedv2',
default=False,
action='store_true',
help=('Do not use new scheduler. '
'Use original scheduler instead.'))
parser.add_argument(
'-l',
'--log_dir',
dest='log_dir',
default='',
help='The log_dir, default is under <crosperf_logs>/logs')
SetupParserOptions(parser)
options, args = parser.parse_known_args(argv)
# Convert the relevant options that are passed in into a settings
# object which will override settings in the experiment file.
option_settings = ConvertOptionsToSettings(options)
log_dir = os.path.abspath(os.path.expanduser(options.log_dir))
logger.GetLogger(log_dir)
if len(args) == 2:
experiment_filename = args[1]
else:
parser.error('Invalid number arguments.')
working_directory = os.getcwd()
if options.dry_run:
test_flag.SetTestMode(True)
experiment_file = ExperimentFile(open(experiment_filename, 'rb'),
option_settings)
if not experiment_file.GetGlobalSettings().GetField('name'):
experiment_name = os.path.basename(experiment_filename)
experiment_file.GetGlobalSettings().SetField('name', experiment_name)
experiment = ExperimentFactory().GetExperiment(experiment_file,
working_directory, log_dir)
json_report = experiment_file.GetGlobalSettings().GetField('json_report')
signal.signal(signal.SIGTERM, CallExitHandler)
atexit.register(Cleanup, experiment)
if options.dry_run:
runner = MockExperimentRunner(experiment, json_report)
else:
runner = ExperimentRunner(experiment,
json_report,
using_schedv2=(not options.noschedv2))
runner.Run()
def __init__(self, image, chromeos_root, no_unmount):
self.image = image
self.chromeos_root = chromeos_root
self.mounted = False
self._ce = command_executer.GetCommandExecuter()
self.logger = logger.GetLogger()
self.elf_files = []
self.no_unmount = no_unmount
self.unmount_script = ''
self.stateful = ''
self.rootfs = ''
def __init__(self, chromeos_root, board, cycler, profile_dir, remote):
self._chromeos_root = chromeos_root
self._cycler = cycler
self._profile_dir = profile_dir
self._remote = remote
self._board = board
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
self._gcov_prefix = os.path.join(self.REMOTE_TMP_DIR,
self._GetProfileDir())
def __init__(self, no_create_tmp_dir=False):
self.repo_type = None
self.address = None
self.mappings = None
self.revision = None
self.ignores = ['.gitignore', '.p4config', 'README.google']
if no_create_tmp_dir:
self._root_dir = None
else:
self._root_dir = tempfile.mkdtemp()
self._ce = command_executer.GetCommandExecuter()
self._logger = logger.GetLogger()
def ParseResults(results_filename):
results = []
try:
results_file = open(results_filename, 'rb')
for line in results_file:
if line.strip() != '':
results.append(line.strip().split('\t'))
results_file.close()
except IOError:
logger.GetLogger().LogWarning('Could not open results file: ' +
results_filename)
return results