def testUpdateClusters_calculateHostUpdateStateSummary(self):
cluster_name = 'cluster1'
host1 = datastore_test_util.CreateHost(cluster_name,
'host1.mtv',
lab_name='alab')
host2 = datastore_test_util.CreateHost(cluster_name,
'host2.mtv',
lab_name='alab')
host3 = datastore_test_util.CreateHost(cluster_name,
'host3.mtv',
lab_name='alab')
host4 = datastore_test_util.CreateHost(cluster_name,
'host4.mtv',
lab_name='alab')
host5 = datastore_test_util.CreateHost(cluster_name,
'host5.mtv',
lab_name='alab')
host6 = datastore_test_util.CreateHost(cluster_name,
'host6.mtv',
lab_name='alab')
host7 = datastore_test_util.CreateHost(cluster_name,
'host7.mtv',
lab_name='alab')
host8 = datastore_test_util.CreateHost(cluster_name,
'host8.mtv',
lab_name='alab')
host9 = datastore_test_util.CreateHost(cluster_name,
'host9.mtv',
lab_name='alab')
datastore_test_util.CreateHostUpdateState(
host1.hostname, state=api_messages.HostUpdateState.SYNCING)
datastore_test_util.CreateHostUpdateState(
host2.hostname, state=api_messages.HostUpdateState.SYNCING)
datastore_test_util.CreateHostUpdateState(
host3.hostname, state=api_messages.HostUpdateState.RESTARTING)
datastore_test_util.CreateHostUpdateState(
host4.hostname, state=api_messages.HostUpdateState.ERRORED)
datastore_test_util.CreateHostUpdateState(
host5.hostname, state=api_messages.HostUpdateState.PENDING)
datastore_test_util.CreateHostUpdateState(
host6.hostname, state=api_messages.HostUpdateState.TIMED_OUT)
datastore_test_util.CreateHostUpdateState(
host7.hostname, state=api_messages.HostUpdateState.SUCCEEDED)
datastore_test_util.CreateHostUpdateState(
host8.hostname, state=api_messages.HostUpdateState.SHUTTING_DOWN)
datastore_test_util.CreateHostUpdateState(
host9.hostname, state=api_messages.HostUpdateState.UNKNOWN)
device_monitor._UpdateClusters(
[host1, host2, host3, host4, host5, host6, host7, host8, host9])
cluster = datastore_entities.ClusterInfo.get_by_id(cluster_name)
self.assertEqual(9, cluster.host_update_state_summary.total)
self.assertEqual(1, cluster.host_update_state_summary.pending)
self.assertEqual(2, cluster.host_update_state_summary.syncing)
self.assertEqual(1, cluster.host_update_state_summary.shutting_down)
self.assertEqual(1, cluster.host_update_state_summary.restarting)
self.assertEqual(1, cluster.host_update_state_summary.errored)
self.assertEqual(1, cluster.host_update_state_summary.timed_out)
self.assertEqual(1, cluster.host_update_state_summary.succeeded)
self.assertEqual(1, cluster.host_update_state_summary.unknown)
def testMarkHostUpdateStateIfTimedOut_NonFinalStateUpdateIsTimedOut(
self, mock_now):
now = datetime.datetime(2021, 1, 15, 10, 10)
mock_now.return_value = now
state_updated_time = (
now - device_monitor._DEFAULT_HOST_UPDATE_STATE_TIMEOUT -
datetime.timedelta(minutes=5))
datastore_test_util.CreateHostUpdateState(
self.host1.hostname,
state=api_messages.HostUpdateState.PENDING,
update_timestamp=state_updated_time)
host_update_state = device_monitor._MarkHostUpdateStateIfTimedOut(
self.host1.hostname)
self.assertEqual(self.host1.hostname, host_update_state.hostname)
self.assertEqual(api_messages.HostUpdateState.TIMED_OUT,
host_update_state.state)
self.assertEqual(now, host_update_state.update_timestamp)
expected_display_message = (
'Host <atl-01.mtv> has HostUpdateState<PENDING> changed on '
'2021-01-15 08:05:00, which is 7500 sec ago, '
'exceeding timeouts 7200 sec. ')
self.assertEqual(expected_display_message,
host_update_state.display_message)
new_state_histories = device_manager.GetHostUpdateStateHistories(
self.host1.hostname)
self.assertLen(new_state_histories, 1)
self.assertEqual(api_messages.HostUpdateState.TIMED_OUT,
new_state_histories[0].state)
self.assertEqual(now, new_state_histories[0].update_timestamp)
def testMarkHostUpdateStateIfTimedOut_CustomizedTimedOutMarked(
self, mock_now):
customized_timeout_sec = 60
now = datetime.datetime(2021, 1, 15, 10, 10)
mock_now.return_value = now
state_updated_time = (
now - 5 * datetime.timedelta(seconds=customized_timeout_sec))
datastore_test_util.CreateHostUpdateState(
self.host1.hostname,
state=api_messages.HostUpdateState.PENDING,
update_timestamp=state_updated_time)
datastore_test_util.CreateHostConfig(
self.host1.hostname,
self.host1.lab_name,
shutdown_timeout_sec=customized_timeout_sec)
host_update_state = device_monitor._MarkHostUpdateStateIfTimedOut(
self.host1.hostname)
default_timeout_limit = (
now - device_monitor._DEFAULT_HOST_UPDATE_STATE_TIMEOUT)
# Assert that the last updated time did not reach default timeout limit yet.
self.assertLess(default_timeout_limit, state_updated_time)
self.assertEqual(self.host1.hostname, host_update_state.hostname)
self.assertEqual(api_messages.HostUpdateState.TIMED_OUT,
host_update_state.state)
self.assertEqual(now, host_update_state.update_timestamp)
expected_display_message = (
'Host <atl-01.mtv> has HostUpdateState<PENDING> changed on '
'2021-01-15 10:05:00, which is 300 sec ago, '
'exceeding timeouts 90 sec. ')
self.assertEqual(expected_display_message,
host_update_state.display_message)
new_state_histories = device_manager.GetHostUpdateStateHistories(
self.host1.hostname)
self.assertLen(new_state_histories, 1)
self.assertEqual(api_messages.HostUpdateState.TIMED_OUT,
new_state_histories[0].state)
self.assertEqual(now, new_state_histories[0].update_timestamp)
def testMarkHostUpdateStateIfTimedOut_AddMissingTimestamp(self, mock_now):
now = datetime.datetime(2021, 1, 15, 10, 10)
mock_now.return_value = now
datastore_test_util.CreateHostUpdateState(
self.host1.hostname,
state=api_messages.HostUpdateState.SYNCING,
update_timestamp=None)
host_update_state = device_monitor._MarkHostUpdateStateIfTimedOut(
self.host1.hostname)
self.assertEqual(self.host1.hostname, host_update_state.hostname)
self.assertEqual(api_messages.HostUpdateState.SYNCING,
host_update_state.state)
self.assertEqual(now, host_update_state.update_timestamp)
new_state_histories = device_manager.GetHostUpdateStateHistories(
self.host1.hostname)
self.assertLen(new_state_histories, 1)
self.assertEqual(api_messages.HostUpdateState.SYNCING,
new_state_histories[0].state)
self.assertEqual(now, new_state_histories[0].update_timestamp)
def testMarkHostUpdateStateIfTimedOut_FinalStateUpdateNotBeingChecked(
self, mock_now):
now = datetime.datetime(2021, 1, 15, 10, 10)
mock_now.return_value = now
state_updated_time = (
now - device_monitor._DEFAULT_HOST_UPDATE_STATE_TIMEOUT -
datetime.timedelta(minutes=5))
datastore_test_util.CreateHostUpdateState(
self.host1.hostname,
state=api_messages.HostUpdateState.SUCCEEDED,
update_timestamp=state_updated_time)
host_update_state = device_monitor._MarkHostUpdateStateIfTimedOut(
self.host1.hostname)
self.assertEqual(self.host1.hostname, host_update_state.hostname)
self.assertEqual(api_messages.HostUpdateState.SUCCEEDED,
host_update_state.state)
self.assertEqual(state_updated_time,
host_update_state.update_timestamp)
new_state_histories = device_manager.GetHostUpdateStateHistories(
self.host1.hostname)
self.assertEmpty(new_state_histories)
def testUpdateClusters_calculateHostUpdateStateSummaryPerVersion(self):
cluster_name = 'cluster1'
host1 = datastore_test_util.CreateHost(cluster_name,
'host1.mtv',
lab_name='alab')
host2 = datastore_test_util.CreateHost(cluster_name,
'host2.mtv',
lab_name='alab')
host3 = datastore_test_util.CreateHost(cluster_name,
'host3.mtv',
lab_name='alab')
host4 = datastore_test_util.CreateHost(cluster_name,
'host4.mtv',
lab_name='alab')
host5 = datastore_test_util.CreateHost(cluster_name,
'host5.mtv',
lab_name='alab')
host6 = datastore_test_util.CreateHost(cluster_name,
'host6.mtv',
lab_name='alab')
host7 = datastore_test_util.CreateHost(cluster_name,
'host7.mtv',
lab_name='alab')
host8 = datastore_test_util.CreateHost(cluster_name,
'host8.mtv',
lab_name='alab')
host9 = datastore_test_util.CreateHost(cluster_name,
'host9.mtv',
lab_name='alab')
datastore_test_util.CreateHostUpdateState(
host1.hostname,
state=api_messages.HostUpdateState.SYNCING,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host2.hostname,
state=api_messages.HostUpdateState.SYNCING,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host3.hostname,
state=api_messages.HostUpdateState.RESTARTING,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host4.hostname,
state=api_messages.HostUpdateState.ERRORED,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host5.hostname,
state=api_messages.HostUpdateState.PENDING,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host6.hostname,
state=api_messages.HostUpdateState.TIMED_OUT,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host7.hostname,
state=api_messages.HostUpdateState.SUCCEEDED,
target_version='v1')
datastore_test_util.CreateHostUpdateState(
host8.hostname,
state=api_messages.HostUpdateState.SHUTTING_DOWN,
target_version='v2')
datastore_test_util.CreateHostUpdateState(
host9.hostname,
state=api_messages.HostUpdateState.UNKNOWN,
target_version='v2')
device_monitor._UpdateClusters(
[host1, host2, host3, host4, host5, host6, host7, host8, host9])
cluster = datastore_entities.ClusterInfo.get_by_id(cluster_name)
version_to_summaries = {
summary.target_version: summary
for summary in cluster.host_update_state_summaries_by_version
}
self.assertEqual(1, version_to_summaries['v1'].total)
self.assertEqual(0, version_to_summaries['v1'].pending)
self.assertEqual(0, version_to_summaries['v1'].syncing)
self.assertEqual(0, version_to_summaries['v1'].shutting_down)
self.assertEqual(0, version_to_summaries['v1'].restarting)
self.assertEqual(0, version_to_summaries['v1'].errored)
self.assertEqual(0, version_to_summaries['v1'].timed_out)
self.assertEqual(1, version_to_summaries['v1'].succeeded)
self.assertEqual(0, version_to_summaries['v1'].unknown)
self.assertEqual(8, version_to_summaries['v2'].total)
self.assertEqual(1, version_to_summaries['v2'].pending)
self.assertEqual(2, version_to_summaries['v2'].syncing)
self.assertEqual(1, version_to_summaries['v2'].shutting_down)
self.assertEqual(1, version_to_summaries['v2'].restarting)
self.assertEqual(1, version_to_summaries['v2'].errored)
self.assertEqual(1, version_to_summaries['v2'].timed_out)
self.assertEqual(0, version_to_summaries['v2'].succeeded)
self.assertEqual(1, version_to_summaries['v2'].unknown)