def _revert():
LOG.fixture_step(
"Revert timezone to {} and ensure host created timestamp also reverted"
.format(DEFAULT_ZONE))
system_helper.modify_timezone(timezone=DEFAULT_ZONE,
auth_info=central_auth)
system_helper.modify_timezone(timezone=DEFAULT_ZONE,
auth_info=sub_auth)
wait_for_timestamp_update(auth_info=central_auth,
expt_time=prev_central_time)
wait_for_timestamp_update(auth_info=sub_auth, expt_time=prev_sub_time)
def test_modify_timezone_cli_timestamps(stx_openstack_required):
"""
Test correct timestamps in:
- ceilometer
- cinder
- glance
- neutron
- nova
- sysinv
Setups
- Get a random timezone for testing
- Create cinder volume
- Boot a vm
Test Steps
- Save the pre-timezone-change timestamps from each cli domain
- Modify the timezone
- Wait for out_of_date alarms to clear
- Save the post-timezone-change timestamps from each cli domain
- Verify the timestamps have changed to be in line with the timezone change
Teardown
- Deleted cinder volume
- Delete the vm
"""
services = ('sysinv', 'openstack')
prev_timezone = system_helper.get_timezone()
post_timezone = __select_diff_timezone(current_zone=prev_timezone)
# CHECK PRE TIMEZONE CHANGE CLI TIMESTAMPS
LOG.tc_step("Getting CLI timestamps before timezone change for: {}".format(services))
vol_id = cinder_helper.create_volume('timezone_test', cleanup='function')[1]
prev_timestamps = get_cli_timestamps(vol_id=vol_id)
LOG.tc_step("Modify timezone from {} to {}".format(prev_timezone, post_timezone))
system_helper.modify_timezone(post_timezone)
# CHECK POST TIMEZONE CHANGE CLI TIMESTAMPS
time.sleep(10)
LOG.tc_step("Getting CLI timestamps after timezone change for: {}".format(services))
post_timestamps = get_cli_timestamps(vol_id=vol_id)
LOG.tc_step("Comparing timestamps from before and after timezone change for: {}".format(services))
failed_services = []
for i in range(len(services) - 1): # -1 to ignore last item opentack cli (CGTS-10475)
if prev_timestamps[i] == post_timestamps[i]:
failed_services.append(services[i])
assert not failed_services, "{} timestamp did not update after timezone modify".format(failed_services)
def test_modify_timezone_sys_event_timestamp():
"""
Test alarm timestamps line up with a timezone change
Prerequisites
- N/A
Test Setups
- Get a random timezone for testing
Test Steps
- Get the UUID and timestamp from the most recent event
- Change the timezone and wait until the change is complete
- Wait for out_of_date alarms to clear
- Compare the timestamp from the event using the UUID
- Verify the timestamp changed with the timezone change
Test Teardown
- N/A
"""
LOG.tc_step("Gathering pre-modify timestamp for last event in system event-list")
event = system_helper.get_events(fields=('UUID', 'Event Log ID', 'Entity Instance ID', 'State', 'Time Stamp'),
limit=1, combine_entries=False)[0]
event_uuid, event_log_id, entity_instance_id, event_state, pre_timestamp = event
# Pick a random timezone to test that is not the current timezone
timezone_to_test = __select_diff_timezone()
LOG.tc_step("Modify timezone to {}".format(timezone_to_test))
system_helper.modify_timezone(timezone=timezone_to_test)
LOG.tc_step("Waiting for timezone for previous event to change in system event-list")
timeout = time.time() + 60
post_timestamp = None
while time.time() < timeout:
post_timestamp = system_helper.get_events(fields='Time Stamp', event_id=event_log_id, uuid=event_uuid,
entity_id=entity_instance_id, state=event_state)[0][0]
if pre_timestamp != post_timestamp:
break
time.sleep(5)
else:
assert pre_timestamp != post_timestamp, "Timestamp did not change with timezone change."
if not system_helper.is_aio_simplex():
LOG.tc_step("Swact and verify timezone persists")
host_helper.swact_host()
post_swact_timezone = system_helper.get_timezone()
assert post_swact_timezone == timezone_to_test
post_swact_timestamp = system_helper.get_events(fields='Time Stamp', event_id=event_log_id, uuid=event_uuid,
entity_id=entity_instance_id, state=event_state)[0][0]
assert post_swact_timestamp == post_timestamp
def prev_check(request, check_central_alarms_module):
LOG.fixture_step(
"(module) Ensure both central and subcloud are configured with {} timezone"
.format(DEFAULT_ZONE))
subcloud = ProjVar.get_var('PRIMARY_SUBCLOUD')
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
sub_auth = Tenant.get('admin_platform', dc_region=subcloud)
system_helper.modify_timezone(timezone=DEFAULT_ZONE,
auth_info=central_auth)
code = system_helper.modify_timezone(timezone=DEFAULT_ZONE,
auth_info=sub_auth)[0]
if code == 0:
# allow sometime for change to apply
time.sleep(30)
prev_central_time = system_helper.get_host_values(
host="controller-0", fields='created_at', auth_info=central_auth)[0]
prev_sub_time = system_helper.get_host_values(host="controller-0",
fields='created_at',
auth_info=sub_auth)[0]
LOG.fixture_step("prev_time: {}.".format(prev_central_time))
central_zone, sub_zone = __select_two_timezones(current_zone=DEFAULT_ZONE)
def _revert():
LOG.fixture_step(
"Revert timezone to {} and ensure host created timestamp also reverted"
.format(DEFAULT_ZONE))
system_helper.modify_timezone(timezone=DEFAULT_ZONE,
auth_info=central_auth)
system_helper.modify_timezone(timezone=DEFAULT_ZONE,
auth_info=sub_auth)
wait_for_timestamp_update(auth_info=central_auth,
expt_time=prev_central_time)
wait_for_timestamp_update(auth_info=sub_auth, expt_time=prev_sub_time)
request.addfinalizer(_revert)
return prev_central_time, prev_sub_time, central_zone, sub_zone, central_auth, sub_auth, \
subcloud
def test_dc_modify_timezone(prev_check):
"""
Test timezone modify on system controller and subcloud. Ensure timezone change is not
propagated.
Setups:
- Ensure both central and subcloud regions are configured with UTC
- Get the timestamps for host created_at before timezone modify
Test Steps
- Change the timezone in central region and wait until the change is applied
- Change the timezone to a different zone in subcloud and wait until the change is applied
- Verify host created_at timestamp updated according to the local timezone for the region
- Swact on subcloud and ensure timezone and host created_at timestamp persists locally
- Swact central controller and ensure timezone and host created_at timestamp persists
in central and subcloud
Teardown
- Change timezone to UTC in both central and subcloud regions
- Ensure host created_at timestamp is reverted to original
"""
prev_central_time, prev_sub_time, central_zone, sub_zone, central_auth, subcloud_auth, \
subcloud = prev_check
LOG.tc_step("Modify timezone to {} in central region".format(central_zone))
system_helper.modify_timezone(timezone=central_zone,
auth_info=central_auth)
LOG.tc_step(
"Waiting for timestamp for host created_at to update in central region"
)
post_central_time = wait_for_timestamp_update(
prev_timestamp=prev_central_time, auth_info=central_auth)
assert post_central_time != prev_central_time, \
"host created_at timestamp did not update after timezone changed " \
"to {} in central region".format(central_zone)
LOG.tc_step("Modify timezone to {} in {}".format(sub_zone, subcloud))
system_helper.modify_timezone(timezone=sub_zone, auth_info=subcloud_auth)
LOG.tc_step(
"Waiting for timestamp for same host created_at to update in {}".
format(subcloud))
post_sub_time = wait_for_timestamp_update(prev_timestamp=prev_sub_time,
auth_info=subcloud_auth)
assert post_sub_time != prev_sub_time, \
"host created_at timestamp did not update after timezone changed to {} " \
"in {}".format(sub_zone, subcloud)
assert post_sub_time != post_central_time, \
"Host created_at timestamp is the same on central and {} when configured with different " \
"timezones".format(subcloud)
LOG.tc_step(
"Ensure host created_at timestamp does not change after subcloud sync audit"
)
dc_helper.wait_for_sync_audit(subclouds=subcloud,
fail_ok=True,
timeout=660)
post_sync_sub_time = system_helper.get_host_values(
host='controller-0', fields='created_at', auth_info=subcloud_auth)[0]
assert post_sub_time == post_sync_sub_time, \
"Host created_at timestamp changed after sync audit on {}".format(subcloud)
if not system_helper.is_aio_simplex():
LOG.tc_step(
"Swact in {} region and verify timezone persists locally".format(
subcloud))
host_helper.swact_host(auth_info=subcloud_auth)
post_swact_sub_zone = system_helper.get_timezone(
auth_info=subcloud_auth)
assert post_swact_sub_zone == sub_zone
post_swact_sub_time = system_helper.get_host_values(
host='controller-0', fields='created_at',
auth_info=subcloud_auth)[0]
assert post_swact_sub_time == post_sub_time
if system_helper.get_standby_controller_name(auth_info=central_auth):
LOG.tc_step(
"Swact in central region, and ensure timezone persists locally in central"
" and subcloud")
host_helper.swact_host(auth_info=central_auth)
# Verify central timezone persists
post_swact_central_zone = system_helper.get_timezone(
auth_info=central_auth)
assert post_swact_central_zone == central_zone
post_swact_central_time = system_helper.get_host_values(
host='controller-0', fields='created_at',
auth_info=central_auth)[0]
assert post_swact_central_time == post_central_time
# Verify subcloud timezone persists
post_central_swact_sub_zone = system_helper.get_timezone(
auth_info=subcloud_auth)
assert post_central_swact_sub_zone == sub_zone
post_central_swact_sub_time = system_helper.get_host_values(
host='controller-0', fields='created_at',
auth_info=subcloud_auth)[0]
assert post_central_swact_sub_time == post_sub_time
def test_modify_timezone_log_timestamps():
"""
Test correct log timestamps after timezone change
Prerequisites
- N/A
Test Setups
- Get a random timezone for testing
- Get system time (epoch) before timezone change
Test Steps
- Modify timezone
- While to modification is pending get the last timestamp (epoch) from each log
- Wait for out_of_date alarms to clear
- Get system time (epoch) after timezone change
- Ensure the timezone change effected the date by comparing system time before and after timezone change
- For 3 minutes check each log for a new entry
- Ensure that the new entry in each log is in line with the timezone change
Test Teardown
- N/A
"""
logs = ('auth.log', 'daemon.log', 'fm-event.log', 'fsmond.log', 'kern.log', 'openstack.log',
'pmond.log', 'user.log')
# 'sm-scheduler.log' fails (CGTS-10475)
LOG.tc_step("Collect timezone, system time, and log timestamps before modify timezone")
LOG.info("Get timezones for testing")
prev_timezone = system_helper.get_timezone()
post_timezone = __select_diff_timezone(current_zone=prev_timezone)
con_ssh = ControllerClient.get_active_controller()
# Saving the last entry from the logs; If it is the same later, there is no new entry in the log.
LOG.tc_step("Get last entry from log files and compare with system time")
prev_timestamps = {}
time.sleep(5)
for log in logs:
last_line = con_ssh.exec_cmd('tail -n 1 /var/log/{}'.format(log))[1]
log_epoch = parse_log_time(last_line)
prev_timestamps[log] = log_epoch
prev_system_epoch = get_epoch_date(con_ssh=con_ssh)
prev_check_fail = []
for log, timestamp in prev_timestamps.items():
if timestamp > prev_system_epoch:
prev_check_fail.append(log)
assert not prev_check_fail, "{} timestamp does not match system time".format(prev_check_fail)
start_time = time.time()
LOG.tc_step("Modify timezone from {} to {}".format(prev_timezone, post_timezone))
system_helper.modify_timezone(timezone=post_timezone)
end_time = time.time()
mod_diff = end_time - start_time
LOG.tc_step("Verify system time is updated")
time.sleep(10)
post_system_epoch = get_epoch_date(con_ssh=con_ssh)
time_diff = abs(prev_system_epoch - post_system_epoch)
assert time_diff > 3600, "Timezone change did not affect the date"
LOG.info("system time is updated after timezone modify")
LOG.tc_step("Wait for new logs to generate and verify timestamps for new log entries are updated")
logs_to_test = {}
timeout = time.time() + 300
while time.time() < timeout:
for log_name, value in prev_timestamps.items():
last_line = con_ssh.exec_cmd('tail -n 1 /var/log/{}'.format(log_name))[1]
# If last line does not exist in last_log_entries; New line is in the log; Add log to logs_to_test
new_log_epoch = parse_log_time(last_line)
prev_log_epoch = prev_timestamps[log_name]
epoch_diff_prev_log_to_sys = prev_system_epoch - prev_log_epoch
epoch_diff_prev_log = new_log_epoch - prev_log_epoch
LOG.info('timezone modify time used: {}; pre-modify sys time - last log time: {}'.
format(mod_diff, epoch_diff_prev_log_to_sys))
if abs(epoch_diff_prev_log) > max(mod_diff, 180) + epoch_diff_prev_log_to_sys:
LOG.info("{} has new log entry. Adding to logs_to_test.".format(log_name))
logs_to_test[log_name] = new_log_epoch
del prev_timestamps[log_name]
break
if not prev_timestamps:
break
time.sleep(10)
# Get latest log entries, convert to epoch
failed_logs = {}
for log_name, value in logs_to_test.items():
time_diff = abs(value - post_system_epoch)
LOG.info("log: {} time diff: {}".format(log_name, time_diff))
if time_diff > 330:
failed_logs[log_name] = "time_diff: {}".format(time_diff)
assert not failed_logs, "Timestamp for following new logs are different than system time: {}".format(failed_logs)
def _revert():
LOG.fixture_step("Reverting timezone to UTC")
system_helper.modify_timezone(timezone='UTC')