def testCreateCommandAttempt(self):
request_id = 'request_id'
command_id = 'command_id'
leased_tasks = [
command_task_api.CommandTask(
task_id='task_id0',
request_id=request_id,
command_id=command_id,
device_serials=['d1'],
plugin_data=[
api_messages.KeyValuePair(key='key0', value='value0'),
api_messages.KeyValuePair(key='key1', value='value1'),
api_messages.KeyValuePair(key='hostname', value='ahost'),
api_messages.KeyValuePair(key='lab_name', value='alab'),
api_messages.KeyValuePair(key='host_group', value='cluster'),
]),
command_task_api.CommandTask(
task_id='task_id1',
request_id=request_id,
command_id=command_id,
device_serials=['d2', 'd3'],
plugin_data=[
api_messages.KeyValuePair(key='key2', value='value2'),
api_messages.KeyValuePair(key='key3', value='value3'),
api_messages.KeyValuePair(key='hostname', value='ahost'),
api_messages.KeyValuePair(key='lab_name', value='alab'),
api_messages.KeyValuePair(key='host_group', value='agroup'),
])
]
command_task_api.CommandTaskApi()._CreateCommandAttempt(leased_tasks)
attempts = command_manager.GetCommandAttempts(request_id='request_id',
command_id='command_id')
self.assertEqual(2, len(attempts))
self.assertEqual('command_id', attempts[0].command_id)
self.assertEqual('task_id0', attempts[0].task_id)
self.assertEqual('ahost', attempts[0].hostname)
self.assertEqual(['d1'], attempts[0].device_serials)
self.assertEqual(
{'host_group': 'cluster', 'hostname': 'ahost', 'lab_name': 'alab',
'key0': 'value0', 'key1': 'value1'},
attempts[0].plugin_data)
self.assertIsNotNone(attempts[0].last_event_time)
self.assertEqual('command_id', attempts[1].command_id)
self.assertEqual('task_id1', attempts[1].task_id)
self.assertEqual('ahost', attempts[1].hostname)
self.assertEqual(['d2', 'd3'], attempts[1].device_serials)
self.assertEqual(
{'host_group': 'agroup', 'hostname': 'ahost', 'lab_name': 'alab',
'key2': 'value2', 'key3': 'value3'},
attempts[1].plugin_data)
self.assertIsNotNone(attempts[1].last_event_time)
def testGetCluster_withHostCountByHarnessVersion(self):
"""Tests GetCluster where the cluster has host count by harness version."""
api_request = {'cluster_id': 'free'}
api_response = self.testapp.post_json('/_ah/api/ClusterApi.GetCluster',
api_request)
cluster_info = protojson.decode_message(api_messages.ClusterInfo,
api_response.body)
expected_host_counts = [
api_messages.KeyValuePair(key='version1', value='72'),
api_messages.KeyValuePair(key='version2', value='20'),
]
self.assertCountEqual(
expected_host_counts, cluster_info.host_count_by_harness_version)
def testAfterTestRunHandler(self, mock_get_request, mock_get_test_context,
mock_add_task):
# configure mock TFC request
request_id = self.mock_test_run.request_id
mock_request = api_messages.RequestMessage(id=request_id)
mock_request.commands.append(api_messages.CommandMessage(id='bar'))
mock_get_request.return_value = mock_request
# configure mock TFC test context
mock_test_context = api_messages.TestContext(
command_line='command_line',
env_vars=[api_messages.KeyValuePair(key='key', value='value')],
test_resources=[
api_messages.TestResource(
url='url',
name='name',
path='path',
decompress=True,
decompress_dir='dir',
params=api_messages.TestResourceParameters(
decompress_files=['file'])),
api_messages.TestResource(url='url2',
name='name2',
path='path2')
])
mock_get_test_context.return_value = mock_test_context
expected_test_context = ndb_models.TestContextObj(
command_line='command_line',
env_vars=[ndb_models.NameValuePair(name='key', value='value')],
test_resources=[
ndb_models.TestResourceObj(
url='url',
name='name',
decompress=True,
decompress_dir='dir',
params=ndb_models.TestResourceParameters(
decompress_files=['file'])),
ndb_models.TestResourceObj(url='url2', name='name2')
])
tfc_event_handler._AfterTestRunHandler(self.mock_test_run)
# test run updated, run hooks invoked, output uploaded, and metrics tracked
mock_get_request.assert_called_with(request_id)
mock_get_test_context.assert_called_with(request_id, 'bar')
self.assertEqual(expected_test_context,
self.mock_test_run.next_test_context)
mock_add_task.assert_has_calls([
mock.call(test_run_hook.ExecuteHooks,
self.mock_test_run.key.id(),
ndb_models.TestRunPhase.AFTER_RUN,
_transactional=True),
mock.call(tfc_event_handler._TrackTestRun,
self.mock_test_run.key.id(),
_transactional=True),
])
def testGetLab_displayHostCountByHarnessVersion(self):
"""Test GetLab returns host count by harness version."""
host_count_by_harness_version = {
'version1': 12,
'version2': 1,
'version3': 668,
}
datastore_test_util.CreateLabInfo(
'lab4',
host_count_by_harness_version=host_count_by_harness_version)
api_request = {'lab_name': 'lab4'}
api_response = self.testapp.post_json(
'/_ah/api/LabManagementApi.GetLab', api_request)
lab_info = protojson.decode_message(api_messages.LabInfo,
api_response.body)
self.assertEqual('200 OK', api_response.status)
self.assertEqual('lab4', lab_info.lab_name)
expected_host_counts = [
api_messages.KeyValuePair(key='version1', value='12'),
api_messages.KeyValuePair(key='version2', value='1'),
api_messages.KeyValuePair(key='version3', value='668'),
]
self.assertCountEqual(expected_host_counts,
lab_info.host_count_by_harness_version)
def _CreateTFCRequest(test_run_id):
"""Creates a TFC request.
Args:
test_run_id: a test run ID.
"""
test_run = ndb_models.TestRun.get_by_id(test_run_id)
if test_run.state == ndb_models.TestRunState.CANCELED:
logging.info(
'Test run %s is CANCELED; aborting _CreateTFCRequest()',
test_run_id)
return
assert test_run.state == ndb_models.TestRunState.PENDING
logging.info(
'Creating a TFC request: test=%s, test_run_config=%s',
test_run.test, test_run.test_run_config)
test_run.output_path = TEST_RUN_OUTPUT_PATH_FORMAT % test_run_id
test_run.output_url = file_util.GetAppStorageUrl([test_run.output_path])
# Construct command
if test_run.test_run_config.command:
command_line = test_run.test_run_config.command
else: # TODO: Deprecate extra_args
command_line = test_run.test.command
if test_run.test_run_config.extra_args:
command_line = ' '.join([command_line,
test_run.test_run_config.extra_args])
logging.warning(
'Test run %s missing command, using test.command and extra_args: %s',
test_run_id, command_line)
# Construct retry command
if test_run.test_run_config.retry_command:
retry_command_line = test_run.test_run_config.retry_command
else: # TODO: Deprecate extra_args
retry_command_line = test_run.test.retry_command_line
if retry_command_line and test_run.test_run_config.retry_extra_args:
retry_command_line = ' '.join([retry_command_line,
test_run.test_run_config.retry_extra_args])
logging.warning(
('Test run %s missing retry command, using test.retry_command and '
'retry_extra_args: %s'),
test_run_id, retry_command_line)
# Prepare TFC request parameters
run_target = test_run.test_run_config.run_target
if test_run.test_run_config.device_specs:
run_target = _DeviceSpecsToTFCRunTarget(
test_run.test_run_config.device_specs)
# Buid command infos
command_infos = []
max_concurrent_tasks = None
sharding_mode = (
test_run.test_run_config.sharding_mode or ndb_models.ShardingMode.RUNNER)
if sharding_mode == ndb_models.ShardingMode.RUNNER:
if (test_run.test_run_config.shard_count > 1 and
test_run.test.runner_sharding_args):
tmpl = string.Template(test_run.test.runner_sharding_args)
sharding_args = tmpl.safe_substitute({
'TF_SHARD_COUNT': str(test_run.test_run_config.shard_count)
})
command_line = ' '.join([command_line, sharding_args])
if retry_command_line:
retry_command_line = ' '.join([retry_command_line, sharding_args])
command_infos.append(
api_messages.CommandInfo(
command_line=command_line,
cluster=test_run.test_run_config.cluster,
run_target=run_target,
run_count=test_run.test_run_config.run_count,
shard_count=1))
elif sharding_mode == ndb_models.ShardingMode.MODULE:
test_package_urls = [
r.cache_url
for r in test_run.test_resources
if r.test_resource_type == ndb_models.TestResourceType.TEST_PACKAGE]
# get module infos
module_infos = file_util.GetTestModuleInfos(
file_util.OpenFile(test_package_urls[0]),
test_run.test.module_config_pattern)
tmpl = string.Template(test_run.test.module_execution_args)
for info in sorted(module_infos, key=lambda x: x.name):
module_args = tmpl.safe_substitute({'MODULE_NAME': info.name})
command_info = api_messages.CommandInfo(
name=info.name,
command_line=' '.join([command_line, module_args]),
cluster=test_run.test_run_config.cluster,
run_target=run_target,
run_count=test_run.test_run_config.run_count,
shard_count=1)
# Give a priority to CtsDeqpTestCases since it takes the longest time.
if info.name == 'CtsDeqpTestCases':
command_infos.insert(0, command_info)
else:
command_infos.append(command_info)
max_concurrent_tasks = test_run.test_run_config.shard_count
# Append extra command args to flag as a MTT run.
for info in command_infos:
info.command_line = ' '.join([info.command_line] + EXTRA_COMMAND_ARGS)
if retry_command_line:
retry_command_line = ' '.join([retry_command_line] + EXTRA_COMMAND_ARGS)
tradefed_config_objects = _GetTradefedConfigObjects(test_run)
prev_test_context = None
if test_run.prev_test_context:
prev_test_context = api_messages.TestContext(
env_vars=[
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.prev_test_context.env_vars
],
test_resources=[
_ConvertToTFCTestResource(r, r.url)
for r in test_run.prev_test_context.test_resources
])
# TODO: consider removing command_line from TestContext.
if not prev_test_context.test_resources:
prev_test_context.command_line = command_line
else:
prev_test_context.command_line = retry_command_line
test_resources = [
_ConvertToTFCTestResource(r, r.cache_url) for r in test_run.test_resources
]
# add metadata URL to the test resources
hostname = os.environ['DEFAULT_VERSION_HOSTNAME']
metadata_url = METADATA_API_FORMAT % (hostname, test_run.key.id())
metadata_url = file_util.GetWorkerAccessibleUrl(metadata_url)
test_resources.append(
api_messages.TestResource(name=METADATA_FILE, url=metadata_url))
# determine context file pattern
context_file_pattern = test_run.test.context_file_pattern
if test_run.test.context_file_dir and test_run.test.context_file_pattern:
context_file_pattern = os.path.join(
test_run.test.context_file_dir, test_run.test.context_file_pattern)
# Record metrics
_TrackTestRun(test_run)
new_request_msg = api_messages.NewMultiCommandRequestMessage(
type=api_messages.RequestType.MANAGED,
user='******',
command_infos=command_infos,
max_retry_on_test_failures=(
test_run.test_run_config.max_retry_on_test_failures),
queue_timeout_seconds=test_run.test_run_config.queue_timeout_seconds,
test_environment=api_messages.TestEnvironment(
env_vars=[
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.test.env_vars
],
setup_scripts=test_run.test.setup_scripts,
output_file_upload_url=file_util.GetWorkerAccessibleUrl(
test_run.output_url),
output_file_patterns=test_run.test.output_file_patterns,
use_subprocess_reporting=True,
invocation_timeout_millis=(
test_run.test_run_config.invocation_timeout_seconds * 1000),
output_idle_timeout_millis=(
test_run.test_run_config.output_idle_timeout_seconds * 1000),
jvm_options=test_run.test.jvm_options,
java_properties=[
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.test.java_properties
],
context_file_pattern=context_file_pattern,
extra_context_files=[METADATA_FILE], # append metadata to context
retry_command_line=retry_command_line,
log_level=common.LogLevel.INFO,
tradefed_config_objects=tradefed_config_objects,
use_parallel_setup=test_run.test_run_config.use_parallel_setup),
test_resources=test_resources,
prev_test_context=prev_test_context,
max_concurrent_tasks=max_concurrent_tasks)
logging.info('new_request_msg=%s', new_request_msg)
request = tfc_client.NewRequest(new_request_msg)
logging.info('TFC request %s is created', request.id)
test_run.request_id = request.id
test_run.state = ndb_models.TestRunState.QUEUED
test_run.put()
def testLeaseHostTasks_nonExistHost(
self, create_command_attempt, ensure_consistency, mock_touch,
record_timing, mock_now):
mock_now.return_value = TIMESTAMP
mock_touch.side_effect = [mock.MagicMock(create_time=TIMESTAMP)]
ensure_consistency.return_value = True
self._AddCommand(
self.request.key.id(),
'2',
'command',
'non-exist-group1',
'run_target1',
ants_invocation_id='i123',
ants_work_unit_id='w123')
request = {
'hostname': 'non-exist-host',
'cluster': 'non-exist-group1',
'device_infos': [
{
'device_serial': 'd1',
'hostname': 'hostname',
'run_target': 'run_target1',
'state': common.DeviceState.AVAILABLE,
'group_name': 'non-exist-group1'
},
],
'next_cluster_ids': ['cluster2', 'cluster3']
}
response = self.testapp.post_json(
'/_ah/api/CommandTaskApi.LeaseHostTasks', request)
self.assertEqual('200 OK', response.status)
task_list = protojson.decode_message(command_task_api.CommandTaskList,
response.body)
self.assertEqual(1, len(task_list.tasks))
task = task_list.tasks[0]
self.assertEqual('command', task.command_line)
self.assertEqual('2', task.command_id)
self.assertEqual('%s-2-0' % REQUEST_ID, task.task_id)
self.assertEqual(['d1'], task.device_serials)
plugin_data = [
api_messages.KeyValuePair(key='ants_invocation_id', value='i123'),
api_messages.KeyValuePair(key='ants_work_unit_id', value='w123'),
api_messages.KeyValuePair(key='host_group', value='non-exist-group1'),
api_messages.KeyValuePair(key='hostname', value='non-exist-host'),
api_messages.KeyValuePair(key='lab_name', value='UNKNOWN'),
api_messages.KeyValuePair(
key='tfc_command_attempt_queue_end_timestamp',
value='1525071600000'),
api_messages.KeyValuePair(
key='tfc_command_attempt_queue_start_timestamp',
value='1525071600000'),
]
create_command_attempt.assert_has_calls([
mock.call(
[command_task_api.CommandTask(
task_id='1001-2-0',
request_id='1001',
command_id='2',
command_line='command',
device_serials=['d1'],
run_index=0,
attempt_index=0,
plugin_data=plugin_data)])])
ensure_consistency.assert_has_calls([
mock.call(REQUEST_ID, '2', '%s-2-0' % REQUEST_ID)])
mock_touch.assert_has_calls([
mock.call(REQUEST_ID, '2')])
record_timing.assert_has_calls([
mock.call(
cluster_id='non-exist-group1',
run_target='run_target1',
create_timestamp=TIMESTAMP,
command_action=metric.CommandAction.LEASE,
count=True)])
def _CheckNewRequestMessage(self,
msg,
test_run,
output_url,
test_resource_urls,
command_lines,
retry_command_line=None,
run_target=None,
shard_count=None,
max_concurrent_tasks=None):
"""Compare a new request message to its associated test run."""
test = test_run.test
test_run_config = test_run.test_run_config
# compare general options
self.assertEqual(api_messages.RequestType.MANAGED, msg.type)
expected_run_target = run_target
if not expected_run_target:
if test_run_config.device_specs:
expected_run_target = test_kicker._DeviceSpecsToTFCRunTarget(
test_run_config.device_specs)
else:
expected_run_target = test_run_config.run_target
expected_shard_count = shard_count or test_run_config.shard_count
for command_line, info in zip(command_lines, msg.command_infos):
self.assertEqual(command_line, info.command_line)
self.assertEqual(test_run_config.cluster, info.cluster)
self.assertEqual(expected_run_target, info.run_target)
self.assertEqual(test_run_config.run_count, info.run_count)
self.assertEqual(expected_shard_count, info.shard_count)
self.assertEqual(
retry_command_line, msg.test_environment.retry_command_line)
self.assertEqual(
test_run_config.max_retry_on_test_failures,
msg.max_retry_on_test_failures)
self.assertEqual(max_concurrent_tasks, msg.max_concurrent_tasks)
# compare test environment
self.assertTrue(msg.test_environment.use_subprocess_reporting)
self.assertEqual(output_url, msg.test_environment.output_file_upload_url)
self.assertEqual(test.context_file_pattern,
msg.test_environment.context_file_pattern)
self.assertEqual([test_kicker.METADATA_FILE],
msg.test_environment.extra_context_files)
self.assertEqual(common.LogLevel.INFO, msg.test_environment.log_level)
self.assertEqual(
test_run_config.use_parallel_setup,
msg.test_environment.use_parallel_setup)
# compare test resources, including additional metadata file
test_resources = [
api_messages.TestResource(name=r.name, url=test_resource_urls[r.name])
for r in test_run.test_resources
]
metadata_url = test_resource_urls['mtt.json']
test_resources.append(api_messages.TestResource(
name=test_kicker.METADATA_FILE, url=metadata_url))
self.assertEqual(test_resources, msg.test_resources)
# compare previous test context
if test_run.prev_test_context:
self.assertIsNotNone(msg.prev_test_context)
self.assertEqual([
api_messages.KeyValuePair(key=p.name, value=p.value)
for p in test_run.prev_test_context.env_vars
], msg.prev_test_context.env_vars)
self.assertEqual([
api_messages.TestResource(
name=r.name, url=file_util.GetWorkerAccessibleUrl(r.url))
for r in test_run.prev_test_context.test_resources
], msg.prev_test_context.test_resources)