def run_emu_experiment(emu_server, cmd_tmpl, job_cfg): # cleanup utils.cleanup([emu_server]) # 4. fill in the remaining varialbes expt_time = utils.utc_date() cmd_dict = emu_server_expand(emu_server, cmd_tmpl, job_cfg, expt_time) cmd = utils.safe_format(cmd_tmpl, cmd_dict) emu_addr = utils.get_host_addr(emu_server) final_cmd = ['ssh', emu_addr, cmd] check_call(final_cmd) # prepare parameters used in analyze_and_upload d = { 'title': path.basename(cmd_dict['data_dir']), 'data_dir': cmd_dict['data_dir'], 'job_log': cmd_dict['job_log'], 'master_addr': emu_addr, 'expt_time': expt_time, 'time': cmd_dict['time'], 'runs': cmd_dict['runs'], 'scenario': cmd_dict['scenario'], 'emu_scenario': job_cfg['scenario'], 'emu_cmd': cmd_dict['mm_cmd'], 'emu_desc': job_cfg['desc'], } analyze_and_upload(d) # cleanup utils.cleanup([emu_server])
def test_rtc9737_tc_er_013_vertical_grouping(stream, name, copy_type): """ Vertical Grouping - Enable vertical grouping on all profiles in MA. """ web_service_obj = None recording = None service_name = V2pc.MANIFEST_AGENT stream_name = nsa.get_stream(stream).json()[0][constants.STREAM_NAME] profile_data = v2pc.get_all_stream_profile_data(stream_name) assert len(profile_data)>=3, "Vertical grouping required minimum 3 profile for a stream" try: #Taking backup of v2pc pod config info and editing the config and then restarting the services is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error is_valid, error = v2pc_edit_manifest_config(V2pc.MANIFEST_AGENT, vertical_grouping="*") assert is_valid, error start_time = utils.get_formatted_time(constants.SECONDS * 30, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time(constants.SECONDS * 60, TimeFormat.TIME_FORMAT_MS, stream) recording = recording_model.Recording(StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) recording_id = recording.get_entry(0).RecordingId web_service_obj = notification_utils.get_web_service_object(recording_id) recording.get_entry(0).UpdateUrl = web_service_obj.get_url() LOGGER.debug("Recording instance created=%s", recording.serialize()) response = a8.create_recording(recording) is_valid, error = validate_common.validate_http_response_status_code(response, requests.codes.no_content) assert is_valid, error is_valid, error = validate_recordings.validate_recording(recording_id, web_service_obj) assert is_valid, error is_valid, error = verify_vertical_grouping(recording_id) assert is_valid, error is_valid, error = validate_recordings.validate_playback(recording_id) assert is_valid, error finally: #Revert back the v2pc config changes is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error if web_service_obj: web_service_obj.stop_server() if recording: response = a8.delete_recording(recording) LOGGER.debug("Recording clean up status code=%s", response.status_code)
def run_real_world_experiment(master, slave, cmd_tmpl): # cleanup utils.cleanup([master, slave]) # 4. fill in the remaining varialbes expt_time = utils.utc_date() cmd_dict = master_slave_expand(master, slave, cmd_tmpl, expt_time) cmd = utils.safe_format(cmd_tmpl, cmd_dict) master_addr = utils.get_host_addr(master) final_cmd = ['ssh', master_addr, cmd] check_call(final_cmd) # prepare parameters used in analyze_and_upload d = { 'title': path.basename(cmd_dict['data_dir']), 'data_dir': cmd_dict['data_dir'], 'job_log': cmd_dict['job_log'], 'master_addr': master_addr, 'slave_desc': cmd_dict['slave_desc'], 'expt_time': expt_time, 'time': cmd_dict['time'], 'runs': cmd_dict['runs'], 'scenario': cmd_dict['scenario'], } sender = cmd_dict['sender'] if expt_type == 'node': d['node'] = slave d['cloud'] = master d['to_node'] = True if sender == 'local' else False d['link'] = cmd_dict['link'] elif expt_type == 'cloud': if sender == 'local': d['src'] = master d['dst'] = slave else: d['src'] = slave d['dst'] = master analyze_and_upload(d) # cleanup utils.cleanup([master, slave])
def test_rtc9726_tc_er_018_private_copy(stream, name, copy_type): """ Schedule UNIQUE copy recording (20 - 10 same start/end time, 10 different start/end times), batch size 4. """ web_service_objects = [] recording_pool = None recording = None total_recording = 20 diff_start_time_recordings = 10 same_start_time_recordings = 10 try: #Taking backup of v2pc pod config info, editing the config and then restarting the services is_valid, error = cleanup(redeploy_config_map, V2pc.MANIFEST_AGENT, revert=True) assert is_valid, error is_valid, error = v2pc_edit_manifest_config(V2pc.MANIFEST_AGENT, batch_size='4') assert is_valid, error is_valid, error = verify_batch_size_update(V2pc.MANIFEST_AGENT, Component.VMR, "4") assert is_valid, error queue = Queue.Queue() start_time = utils.get_formatted_time(constants.SECONDS * 30, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time(constants.SECONDS * 80, TimeFormat.TIME_FORMAT_MS, stream) recording = recording_model.Recording(total_recordings=same_start_time_recordings, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) for i in range(diff_start_time_recordings, same_start_time_recordings+diff_start_time_recordings): start_time = utils.get_formatted_time((constants.SECONDS * 30)+i, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time((constants.SECONDS * 80)+i, TimeFormat.TIME_FORMAT_MS, stream) rec_with_diff_time = recording_model.Recording(total_recordings=1, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) rec_with_diff_time.Entries[0].RecordingId = RecordingAttribute.RECORDING_ID_PREFIX + \ rec_with_diff_time.RequestId + '_' + str(i) recording.Entries.append(rec_with_diff_time.get_entry(0)) last_recording_id = rec_with_diff_time.Entries[0].RecordingId for i in range(total_recording): recording_id = recording.get_entry(i).RecordingId web_service_objects.append(notification_utils.get_web_service_object(recording_id)) recording.get_entry(i).UpdateUrl = web_service_objects[i].get_url() LOGGER.debug("Recording instance created=%s", recording.serialize()) #Sending recording request to create recording response = a8.create_recording(recording) is_valid, error = validate_common.validate_http_response_status_code(response, requests.codes.no_content) assert is_valid, error response = rio.find_recording(last_recording_id).json() if not response: return False, ValidationError.RECORDING_RESPONSE_EMPTY.format(last_recording_id) start_time = utils.get_parsed_time(response[0][RecordingAttribute.START_TIME][:-1]) current_time = datetime.datetime.utcnow() wait_time = utils.add_time_to_secs((start_time - current_time), constants.SECONDS) if wait_time < 0: wait_time = 0 #Verifying recording is started or not recording_pool = mp_pool.ThreadPool() for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_notification, (web_service_objects[i], constants.RecordingStatus.STARTED, wait_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error #Verifying playback of recording in progress for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_vle, (recording.get_entry(i).RecordingId,), dict(in_progress = True), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_hls_checker, (recording.get_entry(i).RecordingId,), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error end_time = utils.get_parsed_time(response[0][RecordingAttribute.END_TIME][:-1]) current_time = datetime.datetime.utcnow() wait_time = utils.add_time_to_secs((end_time - current_time), constants.SECONDS) if wait_time < 0: wait_time = 0 #Verifying recording is completed or not recording_pool = mp_pool.ThreadPool() for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_notification, (web_service_objects[i], constants.RecordingStatus.COMPLETE, wait_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error #Playback verification after recording completed for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_vle, (recording.get_entry(i).RecordingId,), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_hls_checker, (recording.get_entry(i).RecordingId,), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error finally: #Revert back the v2pc config changes is_valid, error = cleanup(redeploy_config_map, V2pc.MANIFEST_AGENT, revert=True) assert is_valid, error if recording_pool: recording_pool.close() recording_pool.join() for web_service_obj in web_service_objects: if web_service_obj: web_service_obj.stop_server() if recording: response = a8.delete_recording(recording) LOGGER.debug("Recording clean up status code=%s", response.status_code)
def test_tc_er_006_007_incomplete_delete_playback_US62461(stream, name, copy_type): """ Create multiple recordings with copy type as COMMON """ web_service_objects = [] recording_pool = None recording = None total_recording = 20 diff_start_time_recordings = 10 same_start_time_recordings = 10 service_name = V2pc.MANIFEST_AGENT namespace = Component.VMR try: #Taking backup of v2pc pod config info, editing the config and then restarting the services is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error is_valid, error = v2pc_edit_manifest_config(V2pc.MANIFEST_AGENT, batch_size='4') assert is_valid, error is_valid, error = verify_batch_size_update(service_name, namespace, "4") assert is_valid, error queue = Queue.Queue() start_time = utils.get_formatted_time(constants.SECONDS * 30, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time(constants.SECONDS * 160, TimeFormat.TIME_FORMAT_MS, stream) recording = recording_model.Recording(total_recordings=same_start_time_recordings, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) for i in range(diff_start_time_recordings, same_start_time_recordings+diff_start_time_recordings): start_time = utils.get_formatted_time((constants.SECONDS * 30) + i, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time((constants.SECONDS * 160) + i, TimeFormat.TIME_FORMAT_MS, stream) rec_with_diff_time = recording_model.Recording(total_recordings=1, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) rec_with_diff_time.Entries[0].RecordingId = RecordingAttribute.RECORDING_ID_PREFIX + \ rec_with_diff_time.RequestId + '_' + str(i) recording.Entries.append(rec_with_diff_time.get_entry(0)) for i in range(total_recording): recording_id = recording.get_entry(i).RecordingId web_service_objects.append(notification_utils.get_web_service_object(recording_id)) recording.get_entry(i).UpdateUrl = web_service_objects[i].get_url() last_recording_id = rec_with_diff_time.Entries[0].RecordingId LOGGER.debug("Recording instance created=%s", recording.serialize()) #Sending recording request to create recording response = a8.create_recording(recording) is_valid, error = validate_common.validate_http_response_status_code(response, requests.codes.no_content) assert is_valid, error response = rio.find_recording(last_recording_id).json() if not response: return False, ValidationError.RECORDING_RESPONSE_EMPTY.format(last_recording_id) start_time = utils.get_parsed_time(response[0][RecordingAttribute.START_TIME][:-1]) current_time = datetime.datetime.utcnow() wait_time = utils.add_time_to_secs((start_time - current_time), constants.SECONDS) if wait_time < 0: wait_time = 0 recording_pool = mp_pool.ThreadPool() for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_notification, (web_service_objects[i], constants.RecordingStatus.STARTED, wait_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error #restarting segment recorder to make INCOMPLETE recording is_valid, error = delete_vmr_pods(V2pc.SEGMENT_RECORDER) assert is_valid, error #Verifying recording INCOMPLETE STATE for i in range(total_recording): recording_id = recording.get_entry(i).RecordingId recording_pool.apply_async(validate_recordings.validate_recording_end_state, (recording_id, [constants.RecordingStatus.INCOMPLETE]), dict(web_service_obj=web_service_objects[i], end_time=end_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error recording.Entries.pop(-1) # Deleting all but one recording response = a8.delete_recording(recording) #wait for georbage collect archive_helper.wait_for_archival(stream, recording.get_entry(-1).RecordingId, Archive.ARCHIVE, Archive.COMPLETE) #Verifying playback of recording is_valid, error = validate_recordings.validate_playback_using_vle(rec_with_diff_time.Entries[0].RecordingId,) assert is_valid, error finally: #Revert back the v2pc config changes is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error if recording_pool: recording_pool.close() recording_pool.join() for web_service_obj in web_service_objects: if web_service_obj: web_service_obj.stop_server() if recording: response = a8.delete_recording(rec_with_diff_time) LOGGER.debug("Recording clean up status code=%s", response.status_code)
#!/usr/bin/env python3 # coding: utf-8 import subprocess from helpers import config, shell, utils from ratlog import Log log = Log() log("Deployment Script") log("Detecting Changes") log("Deploying to Repos") log("Deploying Website") utils.deploy("website") log("Deploying Docs") utils.deploy("docs") utils.cleanup() log("Jobs Done")
def test_er_003_004_recording_incomplete(stream, name, copy_type): """ Create multiple recordings with copy type as COMMON """ web_service_objects = [] recording_pool = None recording = None total_recording = 20 diff_start_time_recordings = 10 same_start_time_recordings = 10 recording_duration = 30 # in sec response_a8 = None service_name = V2pc.MANIFEST_AGENT namespace = Component.VMR try: # backup v2pc master config is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error is_valid, error = v2pc_edit_manifest_config(V2pc.MANIFEST_AGENT, batch_size='4') assert is_valid, error is_valid, error = verify_batch_size_update(service_name, namespace, "4") assert is_valid, error queue = Queue.Queue() start_time = utils.get_formatted_time(constants.SECONDS * recording_duration, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time(constants.SECONDS * recording_duration * 5, TimeFormat.TIME_FORMAT_MS, stream) # same start time 10 recordings recording = recording_model.Recording(total_recordings=same_start_time_recordings, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) # different start time 10 recordings for i in range(diff_start_time_recordings, same_start_time_recordings + diff_start_time_recordings): start_time = utils.get_formatted_time((constants.SECONDS * recording_duration)+i, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time((constants.SECONDS * recording_duration * 5)+i, TimeFormat.TIME_FORMAT_MS, stream) rec_with_diff_time = recording_model.Recording(total_recordings=diff_start_time_recordings, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream ) rec_with_diff_time.Entries[0].RecordingId = RecordingAttribute.RECORDING_ID_PREFIX + \ rec_with_diff_time.RequestId + '_'+str(i) recording.Entries.append(rec_with_diff_time.get_entry(0)) last_recording_id = rec_with_diff_time.Entries[0].RecordingId # get recording id and update url for i in range(total_recording): recording_id = recording.get_entry(i).RecordingId web_service_objects.append(notification_utils.get_web_service_object(recording_id)) recording.get_entry(i).UpdateUrl = web_service_objects[i].get_url() # create recording LOGGER.debug("Recording instance created=%s", recording.serialize()) response_a8 = a8.create_recording(recording) is_valid, error = validate_common.validate_http_response_status_code(response_a8, requests.codes.no_content) assert is_valid, error response = rio.find_recording(last_recording_id).json() if not response: return False, ValidationError.RECORDING_RESPONSE_EMPTY.format(last_recording_id) start_time = utils.get_parsed_time(response[0][RecordingAttribute.START_TIME][:-1]) current_time = datetime.datetime.utcnow() wait_time = utils.add_time_to_secs((start_time - current_time), constants.SECONDS) if wait_time < 0: wait_time = 0 # validate recording is started recording_pool = mp_pool.ThreadPool() for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_notification, (web_service_objects[i], constants.RecordingStatus.STARTED, wait_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error #restarting segment recorder to make INCOMPLETE recording is_valid, error = delete_vmr_pods(V2pc.SEGMENT_RECORDER) assert is_valid, error # validate playback using vle # verify recording in progress for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_vle, (recording.get_entry(i).RecordingId,), dict(in_progress=True), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error # validate playback using hls checker while recording is in progress for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_hls_checker, (recording.get_entry(i).RecordingId,), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() if not is_valid: assert 'discontinuity' in error, error #Verifying recording INCOMPLETE STATE for i in range(total_recording): recording_id = recording.get_entry(i).RecordingId recording_pool.apply_async(validate_recordings.validate_recording_end_state, (recording_id, [constants.RecordingStatus.INCOMPLETE]), dict(web_service_obj=web_service_objects[i], end_time=end_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error # Playback using VLE for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_vle, (recording.get_entry(i).RecordingId,), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error # Playback using hls checker for i in range(total_recording): recording_pool.apply_async(validate_recordings.validate_playback_using_hls_checker, (recording.get_entry(i).RecordingId,), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() if not is_valid: assert 'discontinuity' in error, error finally: is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error if recording_pool: recording_pool.close() recording_pool.join() for web_service_obj in web_service_objects: if web_service_obj: web_service_obj.stop_server() if response_a8: response = a8.delete_recording(recording) LOGGER.debug("Recording clean up status code=%s", response.status_code)
def test_rtc9740_tc_er_016_horizontal_grouping(stream, name, copy_type): """ Horizontal Grouping - Enable horizotal grouping for 12s. """ web_service_obj = None recording = None grouping_duration = '12s' try: #Taking backup of v2pc pod config info and editing the config and then restarting the services is_valid, error = cleanup(redeploy_config_map, V2pc.MANIFEST_AGENT, revert=True) assert is_valid, error is_valid, error = v2pc_edit_manifest_config( V2pc.MANIFEST_AGENT, horizontal_grouping=grouping_duration) assert is_valid, error start_time = utils.get_formatted_time(constants.SECONDS * 30, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time(constants.SECONDS * 60, TimeFormat.TIME_FORMAT_MS, stream) recording = recording_model.Recording(StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) recording_id = recording.get_entry(0).RecordingId web_service_obj = notification_utils.get_web_service_object( recording_id) recording.get_entry(0).UpdateUrl = web_service_obj.get_url() LOGGER.debug("Recording instance created=%s", recording.serialize()) response = a8.create_recording(recording) is_valid, error = validate_common.validate_http_response_status_code( response, requests.codes.no_content) assert is_valid, error is_valid, error = validate_recordings.validate_recording( recording_id, web_service_obj) assert is_valid, error is_valid, error = verify_horizontal_grouping(recording_id, grouping_duration) assert is_valid, error is_valid, error = validate_recordings.validate_playback(recording_id) assert is_valid, error finally: #Revert back the v2pc config changes is_valid, error = cleanup(redeploy_config_map, V2pc.MANIFEST_AGENT, revert=True) assert is_valid, error if web_service_obj: web_service_obj.stop_server() if recording: response = a8.delete_recording(recording) LOGGER.debug("Recording clean up status code=%s", response.status_code)
def test_er_008_009_recording_recovery_manifest_restart_( stream, name, copy_type): """ UNIQUE and COMMON copy Recording recovery """ web_service_objects = [] recording_pool = None recording = None total_recording = 20 diff_start_time_recordings = 10 same_start_time_recordings = 10 service_name = V2pc.MANIFEST_AGENT namespace = Component.VMR try: #Taking backup of v2pc pod config info and editing the config and then restarting the services is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error is_valid, error = v2pc_edit_manifest_config(V2pc.MANIFEST_AGENT, batch_size='4') assert is_valid, error is_valid, error = verify_batch_size_update(service_name, namespace, "4") assert is_valid, error queue = Queue.Queue() start_time = utils.get_formatted_time(constants.SECONDS * 30, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time(constants.SECONDS * 150, TimeFormat.TIME_FORMAT_MS, stream) recording = recording_model.Recording( total_recordings=same_start_time_recordings, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) for i in range(diff_start_time_recordings, same_start_time_recordings + diff_start_time_recordings): start_time = utils.get_formatted_time((constants.SECONDS * 30) + i, TimeFormat.TIME_FORMAT_MS, stream) end_time = utils.get_formatted_time((constants.SECONDS * 150) + i, TimeFormat.TIME_FORMAT_MS, stream) rec_with_diff_time = recording_model.Recording( total_recordings=1, StartTime=start_time, EndTime=end_time, copyType=copy_type, StreamId=stream) rec_with_diff_time.Entries[0].RecordingId = RecordingAttribute.RECORDING_ID_PREFIX + \ rec_with_diff_time.RequestId + '_' + str(i) recording.Entries.append(rec_with_diff_time.get_entry(0)) last_recording_id = rec_with_diff_time.Entries[0].RecordingId for i in range(total_recording): recording_id = recording.get_entry(i).RecordingId web_service_objects.append( notification_utils.get_web_service_object(recording_id)) recording.get_entry(i).UpdateUrl = web_service_objects[i].get_url() LOGGER.debug("Recording instance created=%s", recording.serialize()) #Sending recording request to create recording response = a8.create_recording(recording) is_valid, error = validate_common.validate_http_response_status_code( response, requests.codes.no_content) assert is_valid, error response = rio.find_recording(last_recording_id).json() if not response: return False, ValidationError.RECORDING_RESPONSE_EMPTY.format( last_recording_id) start_time = utils.get_parsed_time( response[0][RecordingAttribute.START_TIME][:-1]) current_time = datetime.datetime.utcnow() wait_time = utils.add_time_to_secs((start_time - current_time), constants.SECONDS) if wait_time < 0: wait_time = 0 #Verifying recording is started or not recording_pool = mp_pool.ThreadPool() for i in range(total_recording): recording_pool.apply_async( validate_recordings.validate_notification, (web_service_objects[i], constants.RecordingStatus.STARTED, wait_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error is_valid, error = delete_vmr_pods(V2pc.MANIFEST_AGENT) assert is_valid, error response = rio.find_recording(last_recording_id).json() if not response: return False, ValidationError.RECORDING_RESPONSE_EMPTY.format( last_recording_id) end_time = utils.get_parsed_time( response[0][RecordingAttribute.END_TIME][:-1]) current_time = datetime.datetime.utcnow() wait_time = utils.add_time_to_secs((end_time - current_time), constants.SECONDS) if wait_time < 0: wait_time = 0 for i in range(total_recording): recording_pool.apply_async( validate_recordings.validate_notification, (web_service_objects[i], constants.RecordingStatus.COMPLETE, wait_time), callback=queue.put) for i in range(total_recording): is_valid, error = queue.get() assert is_valid, error #Verifying recording in storage for i in range(total_recording): response = rio.find_recording( recording.get_entry(i).RecordingId).json() is_valid, error = validate_storage.validate_recording_in_storage( response, Cos.ACTIVE_STORAGE, Cos.RECORDING_STORED) assert is_valid, error #Verfying Archive storage archive_helper.wait_for_archival(stream, recording.get_entry(0).RecordingId, Archive.ARCHIVE, Archive.COMPLETE) for i in range(total_recording): response = rio.find_recording( recording.get_entry(i).RecordingId).json() is_valid, error = validate_storage.validate_recording_in_storage( response, Cos.ARCHIVE_STORAGE, Cos.RECORDING_STORED) assert is_valid, error is_valid, error = validate_storage.validate_copy_count( response, Cos.ARCHIVE_STORAGE) assert is_valid, error finally: is_valid, error = cleanup(redeploy_config_map, service_name, revert=True) assert is_valid, error if recording_pool: recording_pool.close() recording_pool.join() for web_service_obj in web_service_objects: web_service_obj.stop_server() if recording: response = a8.delete_recording(recording) LOGGER.debug("Recording clean up status code=%s", response.status_code)