def _Delete(self, test_run_id):
"""Deletes a test run and all related files if it is in a final state."""
test_run_key = mtt_messages.ConvertToKey(ndb_models.TestRun,
test_run_id)
test_run = test_run_key.get()
if not test_run:
raise endpoints.NotFoundException('Test run %s not found' %
test_run_id)
if not test_run.IsFinal():
raise endpoints.BadRequestException(
'Cannot delete non-final test run %s' % test_run_id)
# Remove output files
if test_run.output_path is not None:
output_folder_url = file_util.GetAppStorageUrl(
[test_run.output_path])
output_folder = file_util.FileHandle.Get(output_folder_url)
output_folder.DeleteDir()
# Delete test results database
with sql_models.db.Session() as session:
session.query(sql_models.TestModuleResult).filter_by(
test_run_id=test_run_id).delete()
test_run_key.delete()
def GetCacheUrl(url=''):
"""Get the cache URL for a remote resource.
Args:
url: a resource URL.
Returns:
a cache URL.
"""
encoded_url = six.ensure_text(base64.b64encode(six.ensure_binary(url)))
n = 128
chunks = [encoded_url[i:i+n] for i in range(0, len(encoded_url), n)]
encoded_url = '/'.join(chunks)
return file_util.GetAppStorageUrl([TEST_RESOURCE_CACHE_DIR, encoded_url])
def _TestRunConverter(obj):
"""Converts a ndb_models.TestRun object to a message."""
# TestRun.test stores a snapshot copy of a test at test run creation time.
test_msg = Convert(obj.test, Test)
test_run_config_msg = Convert(obj.test_run_config, TestRunConfig)
# Grab a valid test id from test_run_config since a snapshot copy is missing
# a key.
if test_msg and test_run_config_msg:
test_msg.id = test_run_config_msg.test_id
return TestRun(
id=str(obj.key.id()),
prev_test_run_id=(str(obj.prev_test_run_key.id())
if obj.prev_test_run_key else None),
user=obj.user,
labels=obj.labels,
test_plan=(
Convert(obj.test_plan_key.get(), TestPlan)
if obj.test_plan_key else None),
test=test_msg,
test_run_config=test_run_config_msg,
test_resources=ConvertList(obj.test_resources, TestResourceObj),
state=obj.state,
state_info=_GetTestRunStateInfo(obj),
output_url=(file_util.GetAppStorageUrl([obj.output_path])
if obj.output_path else None),
prev_test_context=Convert(obj.prev_test_context, TestContextObj),
next_test_context=Convert(obj.next_test_context, TestContextObj),
request_id=obj.request_id,
sequence_id=obj.sequence_id,
total_test_count=obj.total_test_count,
failed_test_count=obj.failed_test_count,
failed_test_run_count=obj.failed_test_run_count,
create_time=_AddTimezone(obj.create_time),
update_time=_AddTimezone(obj.update_time),
before_device_actions=ConvertList(
obj.before_device_actions, DeviceAction),
test_run_actions=ConvertList(obj.test_run_actions, TestRunAction),
test_devices=ConvertList(obj.test_devices, TestDeviceInfo),
test_package_info=Convert(obj.test_package_info, TestPackageInfo),
log_entries=ConvertList(event_log.GetEntries(obj), EventLogEntry),
)
def _CreateTFCRequest(test_run_id):
"""Creates a TFC request.
Args:
test_run_id: a test run ID.
"""
test_run = ndb_models.TestRun.get_by_id(test_run_id)
if test_run.state == ndb_models.TestRunState.CANCELED:
logging.info(
'Test run %s is CANCELED; aborting _CreateTFCRequest()',
test_run_id)
return
assert test_run.state == ndb_models.TestRunState.PENDING
logging.info(
'Creating a TFC request: test=%s, test_run_config=%s',
test_run.test, test_run.test_run_config)
test_run.output_path = TEST_RUN_OUTPUT_PATH_FORMAT % test_run_id
test_run.output_url = file_util.GetAppStorageUrl([test_run.output_path])
# Construct command
if test_run.test_run_config.command:
command_line = test_run.test_run_config.command
else: # TODO: Deprecate extra_args
command_line = test_run.test.command
if test_run.test_run_config.extra_args:
command_line = ' '.join([command_line,
test_run.test_run_config.extra_args])
logging.warning(
'Test run %s missing command, using test.command and extra_args: %s',
test_run_id, command_line)
# Construct retry command
if test_run.test_run_config.retry_command:
retry_command_line = test_run.test_run_config.retry_command
else: # TODO: Deprecate extra_args
retry_command_line = test_run.test.retry_command_line
if retry_command_line and test_run.test_run_config.retry_extra_args:
retry_command_line = ' '.join([retry_command_line,
test_run.test_run_config.retry_extra_args])
logging.warning(
('Test run %s missing retry command, using test.retry_command and '
'retry_extra_args: %s'),
test_run_id, retry_command_line)
# Prepare TFC request parameters
run_target = test_run.test_run_config.run_target
if test_run.test_run_config.device_specs:
run_target = _DeviceSpecsToTFCRunTarget(
test_run.test_run_config.device_specs)
# Buid command infos
command_infos = []
max_concurrent_tasks = None
sharding_mode = (
test_run.test_run_config.sharding_mode or ndb_models.ShardingMode.RUNNER)
if sharding_mode == ndb_models.ShardingMode.RUNNER:
if (test_run.test_run_config.shard_count > 1 and
test_run.test.runner_sharding_args):
tmpl = string.Template(test_run.test.runner_sharding_args)
sharding_args = tmpl.safe_substitute({
'TF_SHARD_COUNT': str(test_run.test_run_config.shard_count)
})
command_line = ' '.join([command_line, sharding_args])
if retry_command_line:
retry_command_line = ' '.join([retry_command_line, sharding_args])
command_infos.append(
api_messages.CommandInfo(
command_line=command_line,
cluster=test_run.test_run_config.cluster,
run_target=run_target,
run_count=test_run.test_run_config.run_count,
shard_count=1))
elif sharding_mode == ndb_models.ShardingMode.MODULE:
test_package_urls = [
r.cache_url
for r in test_run.test_resources
if r.test_resource_type == ndb_models.TestResourceType.TEST_PACKAGE]
# get module infos
module_infos = file_util.GetTestModuleInfos(
file_util.OpenFile(test_package_urls[0]),
test_run.test.module_config_pattern)
tmpl = string.Template(test_run.test.module_execution_args)
for info in sorted(module_infos, key=lambda x: x.name):
module_args = tmpl.safe_substitute({'MODULE_NAME': info.name})
command_info = api_messages.CommandInfo(
name=info.name,
command_line=' '.join([command_line, module_args]),
cluster=test_run.test_run_config.cluster,
run_target=run_target,
run_count=test_run.test_run_config.run_count,
shard_count=1)
# Give a priority to CtsDeqpTestCases since it takes the longest time.
if info.name == 'CtsDeqpTestCases':
command_infos.insert(0, command_info)
else:
command_infos.append(command_info)
max_concurrent_tasks = test_run.test_run_config.shard_count
# Append extra command args to flag as a MTT run.
for info in command_infos:
info.command_line = ' '.join([info.command_line] + EXTRA_COMMAND_ARGS)
if retry_command_line:
retry_command_line = ' '.join([retry_command_line] + EXTRA_COMMAND_ARGS)
tradefed_config_objects = _GetTradefedConfigObjects(test_run)
prev_test_context = None
if test_run.prev_test_context:
prev_test_context = api_messages.TestContext(
env_vars=[
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.prev_test_context.env_vars
],
test_resources=[
_ConvertToTFCTestResource(r, r.url)
for r in test_run.prev_test_context.test_resources
])
# TODO: consider removing command_line from TestContext.
if not prev_test_context.test_resources:
prev_test_context.command_line = command_line
else:
prev_test_context.command_line = retry_command_line
test_resources = [
_ConvertToTFCTestResource(r, r.cache_url) for r in test_run.test_resources
]
# add metadata URL to the test resources
hostname = os.environ['DEFAULT_VERSION_HOSTNAME']
metadata_url = METADATA_API_FORMAT % (hostname, test_run.key.id())
metadata_url = file_util.GetWorkerAccessibleUrl(metadata_url)
test_resources.append(
api_messages.TestResource(name=METADATA_FILE, url=metadata_url))
# determine context file pattern
context_file_pattern = test_run.test.context_file_pattern
if test_run.test.context_file_dir and test_run.test.context_file_pattern:
context_file_pattern = os.path.join(
test_run.test.context_file_dir, test_run.test.context_file_pattern)
# Record metrics
_TrackTestRun(test_run)
new_request_msg = api_messages.NewMultiCommandRequestMessage(
type=api_messages.RequestType.MANAGED,
user='******',
command_infos=command_infos,
max_retry_on_test_failures=(
test_run.test_run_config.max_retry_on_test_failures),
queue_timeout_seconds=test_run.test_run_config.queue_timeout_seconds,
test_environment=api_messages.TestEnvironment(
env_vars=[
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.test.env_vars
],
setup_scripts=test_run.test.setup_scripts,
output_file_upload_url=file_util.GetWorkerAccessibleUrl(
test_run.output_url),
output_file_patterns=test_run.test.output_file_patterns,
use_subprocess_reporting=True,
invocation_timeout_millis=(
test_run.test_run_config.invocation_timeout_seconds * 1000),
output_idle_timeout_millis=(
test_run.test_run_config.output_idle_timeout_seconds * 1000),
jvm_options=test_run.test.jvm_options,
java_properties=[
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.test.java_properties
],
context_file_pattern=context_file_pattern,
extra_context_files=[METADATA_FILE], # append metadata to context
retry_command_line=retry_command_line,
log_level=common.LogLevel.INFO,
tradefed_config_objects=tradefed_config_objects,
use_parallel_setup=test_run.test_run_config.use_parallel_setup),
test_resources=test_resources,
prev_test_context=prev_test_context,
max_concurrent_tasks=max_concurrent_tasks)
logging.info('new_request_msg=%s', new_request_msg)
request = tfc_client.NewRequest(new_request_msg)
logging.info('TFC request %s is created', request.id)
test_run.request_id = request.id
test_run.state = ndb_models.TestRunState.QUEUED
test_run.put()
def testGetGetAppStorageUrl(self):
self.assertEqual('file:///root/file/path',
file_util.GetAppStorageUrl(['file', 'path']))
self.assertEqual(
'file://hostname.com/root/file/path',
file_util.GetAppStorageUrl(['file', 'path'], 'hostname.com'))