def _write_config_file(config_file, config_values, overwrite=False):
"""Writes out a configuration file.
@param config_file: The name of the configuration file.
@param config_values: The ConfigParser object.
@param ovewrite: Flag on if overwriting is allowed.
"""
if not config_file:
raise error.RPCException('Empty config file name.')
if not overwrite and os.path.exists(config_file):
raise error.RPCException('Config file already exists.')
if config_values:
with open(config_file, 'w') as config_file:
config_values.write(config_file)
def testFailedJobRetry(self):
"""Make sure the suite survives even if the retry failed."""
test_to_retry = self.files['seven']
fake_job = FakeJob(id=self._FAKE_JOB_ID)
test_results = self._createSuiteMockResults()
self.schedule_and_expect_these_results(
self.suite,
[test_results[0] + test_results[1]],
self.recorder)
self.mox.StubOutWithMock(self.suite._job_creator, 'create_job')
self.suite._job_creator.create_job(
test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise(
error.RPCException('Expected during test'))
# Do not file a bug.
self.mox.StubOutWithMock(self.suite, '_should_report')
self.suite._should_report(mox.IgnoreArg()).AndReturn(False)
self.mox.ReplayAll()
self.suite.schedule(self.recorder.record_entry)
self.suite._retry_handler._retry_map = {
self._FAKE_JOB_ID: {
'state': RetryHandler.States.NOT_ATTEMPTED,
'retry_max': 1}}
self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry
self.suite.wait(self.recorder.record_entry)
expected_retry_map = {
self._FAKE_JOB_ID: {
'state': RetryHandler.States.ATTEMPTED,
'retry_max': 1}}
expected_jobs_to_tests = self.suite._jobs_to_tests.copy()
self.assertEquals(self.suite._retry_handler._retry_map,
expected_retry_map)
self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests)
def add_shard(hostname, labels):
"""Add a shard and start running jobs on it.
@param hostname: The hostname of the shard to be added; needs to be unique.
@param labels: Board labels separated by a comma. Jobs of one of the labels
will be assigned to the shard.
@raises error.RPCException: If label provided doesn't start with `board:`
@raises model_logic.ValidationError: If a shard with the given hostname
already exists.
@raises models.Label.DoesNotExist: If the label specified doesn't exist.
"""
labels = labels.split(',')
label_models = []
for label in labels:
if not label.startswith('board:'):
raise error.RPCException('Sharding only supports for `board:.*` '
'labels.')
# Fetch label first, so shard isn't created when label doesn't exist.
label_models.append(models.Label.smart_get(label))
shard = models.Shard.add_object(hostname=hostname)
for label in label_models:
shard.labels.add(label)
return shard.id
def fanout_rpc(host_objs, rpc_name, include_hostnames=True, **kwargs):
"""Fanout the given rpc to shards of given hosts.
@param host_objs: Host objects for the rpc.
@param rpc_name: The name of the rpc.
@param include_hostnames: If True, include the hostnames in the kwargs.
Hostnames are not always necessary, this functions is designed to
send rpcs to the shard a host is on, the rpcs themselves could be
related to labels, acls etc.
@param kwargs: The kwargs for the rpc.
"""
# Figure out which hosts are on which shards.
shard_host_map = bucket_hosts_by_shard(host_objs)
# Execute the rpc against the appropriate shards.
for shard, hostnames in shard_host_map.iteritems():
if include_hostnames:
kwargs['hosts'] = hostnames
try:
run_rpc_on_multiple_hostnames(rpc_name, [shard], **kwargs)
except:
ei = sys.exc_info()
new_exc = error.RPCException(
'RPC %s failed on shard %s due to '
'%s: %s' % (rpc_name, shard, ei[0].__name__, ei[1]))
raise new_exc.__class__, new_exc, ei[2]
def set_boto_key(boto_key):
"""Update the boto_key file.
@param boto_key: File name of boto_key uploaded through handle_file_upload.
"""
if not os.path.exists(boto_key):
raise error.RPCException('Boto key: %s does not exist!' % boto_key)
shutil.copyfile(boto_key, moblab_host.MOBLAB_BOTO_LOCATION)
def set_service_account_credential(service_account_filename):
"""Update the service account credential file.
@param service_account_filename: Name of uploaded file through
handle_file_upload.
"""
if not os.path.exists(service_account_filename):
raise error.RPCException('Service account file: %s does not exist!' %
service_account_filename)
shutil.copyfile(service_account_filename,
moblab_host.MOBLAB_SERVICE_ACCOUNT_LOCATION)
def _get_builds_for_in_directory(directory_name,
milestone_limit=3,
build_limit=20):
""" Fetch the most recent builds for the last three milestones from gcs.
@param directory_name: The sub-directory under the configured GCS image
storage bucket to search.
@return: A string list no longer than <milestone_limit> x <build_limit>
items, containing the most recent <build_limit> builds from the
last milestone_limit milestones.
"""
output = StringIO.StringIO()
gs_image_location = _CONFIG.get_config_value('CROS', _IMAGE_STORAGE_SERVER)
try:
utils.run(GsUtil.get_gsutil_cmd(),
args=('ls', gs_image_location + directory_name),
stdout_tee=output)
except error.CmdError as e:
error_text = (
'Failed to list builds from %s.\n'
'Did you configure your boto key? Try running the config '
'wizard again.\n\n%s') % (
(gs_image_location + directory_name), e.result_obj.stderr)
raise error.RPCException(error_text)
lines = output.getvalue().split('\n')
output.close()
builds = [
line.replace(gs_image_location, '').strip('/ ') for line in lines
if line != ''
]
build_matcher = re.compile(r'^.*\/R([0-9]*)-.*')
build_map = {}
for build in builds:
match = build_matcher.match(build)
if match:
milestone = match.group(1)
if milestone not in build_map:
build_map[milestone] = []
build_map[milestone].append(build)
milestones = build_map.keys()
milestones.sort()
milestones.reverse()
build_list = []
for milestone in milestones[:milestone_limit]:
builds = build_map[milestone]
builds.sort(key=_get_sortable_build_number)
builds.reverse()
build_list.extend(builds[:build_limit])
return build_list
def set_launch_control_key(launch_control_key):
"""Update the launch_control_key file.
@param launch_control_key: File name of launch_control_key uploaded through
handle_file_upload.
"""
if not os.path.exists(launch_control_key):
raise error.RPCException('Launch Control key: %s does not exist!' %
launch_control_key)
shutil.copyfile(launch_control_key,
moblab_host.MOBLAB_LAUNCH_CONTROL_KEY_LOCATION)
# Restart the devserver service.
os.system('sudo restart moblab-devserver-init')
def _install_system_update():
""" Installs a ChromeOS update, will cause the system to reboot
"""
# sudo is required to run the update client
# first run a blocking command to check, fetch, prepare an update
# then check if a reboot is needed
try:
subprocess.check_call(['sudo', _UPDATE_ENGINE_CLIENT, '--update'])
# --is_reboot_needed returns 0 if a reboot is required
subprocess.check_call(
['sudo', _UPDATE_ENGINE_CLIENT, '--is_reboot_needed'])
subprocess.call(['sudo', _UPDATE_ENGINE_CLIENT, '--reboot'])
except subprocess.CalledProcessError as e:
update_error = subprocess.check_output(
['sudo', _UPDATE_ENGINE_CLIENT, '--last_attempt_error'])
raise error.RPCException(update_error)
def get_servers(hostname=None, role=None, status=None):
"""Get a list of servers with matching role and status.
@param hostname: FQDN of the server.
@param role: Name of the server role, e.g., drone, scheduler. Default to
None to match any role.
@param status: Status of the server, e.g., primary, backup, repair_required.
Default to None to match any server status.
@raises error.RPCException: If server database is not used.
@return: A list of server names for servers with matching role and status.
"""
if not server_manager_utils.use_server_db():
raise error.RPCException('Server database is not enabled. Please try '
'retrieve servers from global config.')
servers = server_manager_utils.get_servers(hostname=hostname,
role=role,
status=status)
return [s.get_details() for s in servers]
def validate_rpc_only_called_by_master(self, meth_name, remote_ip):
"""Validate whether the method name can be called by remote_ip.
This funcion checks whether the given method (meth_name) belongs to
_shard_rpc_module.
If True, it then checks whether the caller's IP (remote_ip) is autotest
master. An RPCException will be raised if an RPC method from
_shard_rpc_module is called by a caller that is not autotest master.
@param meth_name: the RPC method name which is called.
@param remote_ip: the caller's IP.
"""
if meth_name in self._shard_rpc_methods:
global_afe_ip = rpc_utils.get_ip(rpc_utils.GLOBAL_AFE_HOSTNAME)
if remote_ip != global_afe_ip:
raise error.RPCException(
'Shard RPC %r cannot be called by remote_ip %s. It '
'can only be called by global_afe: %s' %
(meth_name, remote_ip, global_afe_ip))
def update_config_handler(config_values):
"""Update config values and override shadow config.
@param config_values: See get_moblab_settings().
"""
original_config = _read_original_config()
new_shadow = ConfigParser.RawConfigParser()
for section, config_value_list in config_values.iteritems():
for key, value in config_value_list:
if original_config.get_config_value(
section, key, default='', allow_blank=True) != value:
if not new_shadow.has_section(section):
new_shadow.add_section(section)
new_shadow.set(section, key, value)
if not _CONFIG.shadow_file or not os.path.exists(_CONFIG.shadow_file):
raise error.RPCException('Shadow config file does not exist.')
_write_config_file(_CONFIG.shadow_file, new_shadow, True)
# TODO (sbasi) crbug.com/403916 - Remove the reboot command and
# instead restart the services that rely on the config values.
os.system('sudo reboot')
def verify(*args, **kwargs):
if not utils.is_moblab():
raise error.RPCException('RPC: %s can only run on Moblab Systems!',
func.__name__)
return func(*args, **kwargs)
def run_suite(board,
build,
suite,
model=None,
ro_firmware=None,
rw_firmware=None,
pool=None,
suite_args=None,
test_args=None,
bug_id=None,
part_id=None):
""" RPC handler to run a test suite.
@param board: a board name connected to the moblab.
@param build: a build name of a build in the GCS.
@param suite: the name of a suite to run
@param model: a board model name connected to the moblab.
@param ro_firmware: Optional ro firmware build number to use.
@param rw_firmware: Optional rw firmware build number to use.
@param pool: Optional pool name to run the suite in.
@param suite_args: Arguments to be used in the suite control file.
@param test_args: '\n' delimited key=val pairs passed to test control file.
@param bug_id: Optional bug ID used for AVL qualification process.
@param part_id: Optional part ID used for AVL qualification
process.
@return: None
"""
builds = {'cros-version': build}
# TODO(mattmallett b/92031054) Standardize bug id, part id passing for memory/storage qual
processed_suite_args = dict()
processed_test_args = dict()
if rw_firmware:
builds['fwrw-version'] = rw_firmware
if ro_firmware:
builds['fwro-version'] = ro_firmware
if suite_args:
processed_suite_args['tests'] = \
[s.strip() for s in suite_args.split(',')]
if bug_id:
processed_suite_args['bug_id'] = bug_id
if part_id:
processed_suite_args['part_id'] = part_id
processed_test_args['bug_id'] = bug_id or ''
processed_test_args['part_id'] = part_id or ''
# set processed_suite_args to None instead of empty dict when there is no
# argument in processed_suite_args
if len(processed_suite_args) == 0:
processed_suite_args = None
if test_args:
try:
processed_test_args['args'] = [test_args]
for line in test_args.split('\n'):
key, value = line.strip().split('=')
processed_test_args[key] = value
except:
raise error.RPCException('Could not parse test args.')
ap_name = _CONFIG.get_config_value('MOBLAB', _WIFI_AP_NAME, default=None)
processed_test_args['ssid'] = ap_name
ap_pass = _CONFIG.get_config_value('MOBLAB', _WIFI_AP_PASS, default='')
processed_test_args['wifipass'] = ap_pass
suite_timeout_mins = _SUITE_TIMEOUT_MAP.get(suite,
_DEFAULT_SUITE_TIMEOUT_MINS)
afe = frontend.AFE(user='******')
afe.run('create_suite_job',
board=board,
builds=builds,
name=suite,
pool=pool,
run_prod_code=False,
test_source_build=build,
wait_for_results=True,
suite_args=processed_suite_args,
test_args=processed_test_args,
job_retry=True,
max_retries=sys.maxint,
model=model,
timeout_mins=suite_timeout_mins,
max_runtime_mins=suite_timeout_mins)
