def make_observation(
self,
test_type,
mezzanine_qr_codes: List[MezzanineDecodedQr],
test_status_qr_codes: List[TestStatusDecodedQr],
parameters_dict: dict,
time_diff_file: str,
) -> Dict[str, str]:
"""Implements the logic:
sample_tolerance_in_recording = ct_frame_tolerance * 1000/mezzanine_frame_rate/(1000/camera_frame_rate)
= ct_frame_tolerance * camera_frame_rate/mezzanine_frame_rate
sample_tolerance = ct_frame_tolerance * 1000/mezzanine_frame_rate
target_camera_frame_num_of_ct_event
= ct_event.first_seen_camera_frame_num - (ct_event.d / camera_frame_duration_ms)
first_possible_camera_frame_num_of_target
= target_camera_frame_num_of_ct_event - CAMERA_FRAME_ADJUSTMENT - sample_tolerance_in_recording
last_possible_camera_frame_num_of_target
= target_camera_frame_num_of_ct_event + CAMERA_FRAME_ADJUSTMENT + sample_tolerance_in_recording
for first_possible_camera_frame_num_of_target to last_possible_camera_frame_num_of_target
foreach mezzanine_qr_code on camera_frame that within the range
if mezzanine_qr_code.media_time == (ct_event.current_time +/- (sample_tolerance + tolerance))
test is PASSED
"""
logger.info(f"Making observation {self.result['name']}...")
if not mezzanine_qr_codes:
self.result["status"] = "FAIL"
self.result["message"] = f"No QR mezzanine code detected."
logger.info(f"[{self.result['status']}] {self.result['message']}")
return self.result
camera_frame_rate = parameters_dict["camera_frame_rate"]
camera_frame_duration_ms = parameters_dict["camera_frame_duration_ms"]
allowed_tolerance = parameters_dict["tolerance"]
ct_frame_tolerance = parameters_dict["frame_tolerance"]
failure_report_count = 0
self.result[
"message"] += f" Allowed tolerance is {ct_frame_tolerance} frames, {allowed_tolerance}ms."
# for splicing test adjust media time in mezzanine_qr_codes
# media time for period 2 starts from 0 so the actual media time is += period_duration[0]
# media time for period 3 starts from where it was left but need to add the ad insertion duration
# so the actual media time is += period_duration[1]
if test_type == TestType.SPLICING:
period_list = PlayoutParser.get_splicing_period_list(
parameters_dict["playout"],
parameters_dict["fragment_duration_multi_mpd"])
change_type_list = PlayoutParser.get_change_type_list(
parameters_dict["playout"])
period_index = 0
change_count = 0
current_content_id = mezzanine_qr_codes[0].content_id
current_frame_rate = mezzanine_qr_codes[0].frame_rate
for i in range(1, len(mezzanine_qr_codes)):
if (mezzanine_qr_codes[i].content_id != current_content_id
or mezzanine_qr_codes[i].frame_rate !=
current_frame_rate):
# the content did change
change_count += 1
current_content_id = mezzanine_qr_codes[i].content_id
current_frame_rate = mezzanine_qr_codes[i].frame_rate
if change_type_list[change_count - 1] == "splicing":
period_index += 1
if period_index > 0:
mezzanine_qr_codes[i].media_time += period_list[
period_index - 1]
time_differences = []
for i in range(0, len(test_status_qr_codes)):
current_status = test_status_qr_codes[i]
if i + 1 < len(test_status_qr_codes):
if (current_status.status == "playing"
and current_status.last_action == "play"):
first_possible, last_possible = self._get_target_camera_frame_num(
current_status.camera_frame_num,
test_status_qr_codes[i + 1].delay,
camera_frame_duration_ms, camera_frame_rate,
mezzanine_qr_codes, ct_frame_tolerance)
diff_found, time_diff = self._find_diff_within_tolerance(
mezzanine_qr_codes, current_status, first_possible,
last_possible, allowed_tolerance, ct_frame_tolerance)
# The multiplication happens so that we get the results in ms
time_differences.append(
(current_status.current_time * 1000, time_diff))
if not diff_found:
self.result["status"] = "FAIL"
if failure_report_count == 0:
self.result["message"] += (
" Time difference between Test Runner reported media currentTime and actual media "
"time exceeded tolerance for following events:"
)
if failure_report_count < REPORT_NUM_OF_FAILURE:
self.result[
"message"] += f" currentTime={current_status.current_time} time_diff={round(time_diff, 4)}; "
failure_report_count += 1
if failure_report_count >= REPORT_NUM_OF_FAILURE:
self.result[
"message"] += f"...too many failures, reporting truncated. Total failure count is {failure_report_count}. "
if self.result["status"] != "FAIL":
self.result["status"] = "PASS"
logger.debug(f"[{self.result['status']}]: {self.result['message']}")
# Exporting time diff data to a CSV file
if time_diff_file and time_differences:
self._write_time_differences(time_diff_file, time_differences)
return self.result
def observe_splicing_mid_frame(
self, mezzanine_qr_codes: List[MezzanineDecodedQr],
playouts: List[List[int]],
fragment_duration_multi_mpd: dict) -> bool:
"""playout[i]: Provides the triple (Switching Set, CMAF track number, Fragment number)
for every playout position i=1,…,N that is be played out.
on each splicing point:
check previous ending and new starting frames are correct for each periods
check that the samples shall be rendered in increasing order within the same period
for QRb to QRn: QR[i-1].mezzanine_frame_num + 1 == QR[i].mezzanine_frame_num
"""
splice_start_frame_num_tolerance = self.tolerances[
"splice_start_frame_num_tolerance"]
splice_end_frame_num_tolerance = self.tolerances[
"splice_end_frame_num_tolerance"]
change_starting_index_list = self.get_content_change_position(
mezzanine_qr_codes)
change_type_list = PlayoutParser.get_change_type_list(playouts)
ending_playout_list = PlayoutParser.get_ending_playout_list(playouts)
starting_playout_list = PlayoutParser.get_starting_playout_list(
playouts)
# check if the configured content change and actual content change matches
# if not report error
actual_change_num = len(change_starting_index_list)
configured_change_num = len(change_type_list) + 1
if actual_change_num != configured_change_num:
self.result["message"] += (
f" Number of changes does not match the 'playout' configuration. "
f"Test is configured to change {configured_change_num} times. "
f"Actual number of change is {actual_change_num}. ")
return False
# check mid frames block by block based on the starting index of content change
mid_frame_result = True
for i, starting_index in enumerate(change_starting_index_list):
if starting_index == change_starting_index_list[-1]:
# check mid frames for last block
check_frame_result = mid_frame_result and self._check_every_frame(
mezzanine_qr_codes[change_starting_index_list[-1]:])
mid_frame_result = mid_frame_result and check_frame_result
else:
last_index = change_starting_index_list[i + 1] - 1
check_frame_result = mid_frame_result and self._check_every_frame(
mezzanine_qr_codes[starting_index:last_index])
mid_frame_result = mid_frame_result and check_frame_result
if i > 0:
# check previous ending frame and new starting frame numbers
ending_playout = ending_playout_list[i - 1]
ending_fragment_duration = fragment_duration_multi_mpd[(
ending_playout[0], ending_playout[1])]
ending_fragment_num = ending_playout[2]
previous_ending_frame_num = round(
ending_fragment_num * ending_fragment_duration / 1000 *
mezzanine_qr_codes[starting_index - 1].frame_rate)
# compare expected with the actual frame number detected at this splice point
diff_ending_frame = abs(mezzanine_qr_codes[starting_index -
1].frame_number -
previous_ending_frame_num)
if change_type_list[i - 1] == "splicing":
if diff_ending_frame > splice_end_frame_num_tolerance:
mid_frame_result = False
self.result["message"] += (
f" Ending with incorrect frame when splicing at period number {i}. "
f"Ending frame found is {mezzanine_qr_codes[starting_index -1].frame_number }, "
f"expected to end with {previous_ending_frame_num}. "
f"Splice end frame tolerance is {splice_end_frame_num_tolerance}."
)
else:
if diff_ending_frame > 0:
mid_frame_result = False
self.result["message"] += (
f" Ending with incorrect frame when switching at number {i}. "
f"Ending frame found is {mezzanine_qr_codes[starting_index -1].frame_number }, "
f"expected to end with {previous_ending_frame_num}. "
)
starting_playout = starting_playout_list[i - 1]
starting_fragment_duration = fragment_duration_multi_mpd[(
starting_playout[0], starting_playout[1])]
starting_fragment_num = starting_playout[2] - 1
current_starting_frame_num = (round(
starting_fragment_num * starting_fragment_duration / 1000 *
mezzanine_qr_codes[starting_index].frame_rate) + 1)
# compare expected with the actual frame number detected at this splice point
diff_starting_frame = abs(
mezzanine_qr_codes[starting_index].frame_number -
current_starting_frame_num)
if change_type_list[i - 1] == "splicing":
if diff_starting_frame > splice_start_frame_num_tolerance:
mid_frame_result = False
self.result["message"] += (
f" Starting from incorrect frame when splicing at period number {i + 1}. "
f"Starting frame found is {mezzanine_qr_codes[starting_index].frame_number }, "
f"expected to start from {current_starting_frame_num}. "
f"Splice start frame tolerance is {splice_start_frame_num_tolerance}."
)
else:
if diff_starting_frame > 0:
mid_frame_result = False
self.result["message"] += (
f" Starting from incorrect frame when switching at number {i + 1}. "
f"Starting frame found is {mezzanine_qr_codes[starting_index].frame_number }, "
f"expected to start from {current_starting_frame_num}. "
)
return mid_frame_result
def make_observation(
self,
test_type,
mezzanine_qr_codes: List[MezzanineDecodedQr],
_unused,
parameters_dict: dict,
_unused2,
) -> Dict[str, str]:
"""
make_observation for different test type
check 1st frame is present
QRa.mezzanine_frame_num == first_frame_num
check the last frame is present
QRn.mezzanine_frame_num == round(cmaf_track_duration * mezzanine_frame_rate)
Args:
test_type: SWITCHING|SPLICING|SEQUENTIAL
mezzanine_qr_codes: lists of MezzanineDecodedQr
_unused:
parameters_dict: parameter dictionary
_unused2
Returns:
Dict[str, str]: observation result
"""
logger.info(f"Making observation {self.result['name']}...")
if len(mezzanine_qr_codes) < 2:
self.result["status"] = "FAIL"
self.result[
"message"] = f"Too few mezzanine QR codes detected ({len(mezzanine_qr_codes)})."
logger.info(f"[{self.result['status']}] {self.result['message']}")
return self.result
first_frame_result = self._check_first_frame(
parameters_dict["first_frame_num"], mezzanine_qr_codes[0])
last_frame_result = self._check_last_frame(
parameters_dict["last_frame_num"], mezzanine_qr_codes[-1])
if test_type == TestType.SWITCHING:
switching_playout = PlayoutParser.get_switching_playout(
parameters_dict["playout"])
mid_frame_result = self.observe_switching_mid_frame(
mezzanine_qr_codes,
switching_playout,
parameters_dict["fragment_duration_list"],
)
elif test_type == TestType.SPLICING:
mid_frame_result = self.observe_splicing_mid_frame(
mezzanine_qr_codes,
parameters_dict["playout"],
parameters_dict["fragment_duration_multi_mpd"],
)
else:
# check that the samples shall be rendered in increasing order:
# for QRb to QRn: QR[i-1].mezzanine_frame_num + 1 == QR[i].mezzanine_frame_num
mid_frame_result = self._check_every_frame(mezzanine_qr_codes)
self.result["message"] += (
f" Total of missing frames is {self.missing_frame_count}.")
if first_frame_result and last_frame_result and mid_frame_result:
self.result["status"] = "PASS"
else:
self.result["status"] = "FAIL"
logger.debug(f"[{self.result['status']}]: {self.result['message']}")
return self.result
def observe_switching_mid_frame(
self,
mezzanine_qr_codes: List[MezzanineDecodedQr],
playout: List[int],
fragment_duration_list: Dict[int, float],
) -> bool:
"""observe switching set tests
playback more than one representations
Parse playout parameter to a list of switching point in media timeline
switching_positions: a list of switching position in media timeline
check every switching points starting frame and ending frame
check that the samples shall be rendered in increasing order within the same representations
for QRb to QRn: QR[i-1].mezzanine_frame_num + 1 == QR[i].mezzanine_frame_num
"""
switching_positions = []
switching_position = 0
switching_positions.append(switching_position)
for i in range(1, len(playout)):
switching_position += fragment_duration_list[playout[i]]
# when track change
if playout[i] != playout[i - 1]:
switching_positions.append(switching_position)
change_switching_index_list = self.get_content_change_position(
mezzanine_qr_codes)
# check configuration and actual switching matches
configured_switching_num = len(switching_positions)
actual_switching_num = len(change_switching_index_list)
if actual_switching_num != configured_switching_num:
self.result["message"] += (
f" Number of switches does not match. "
f"Test is configured to switch {configured_switching_num} times. "
f"Actual number of switches is {actual_switching_num}. ")
return False
# check mid frames block by block
playout_sequence = PlayoutParser.get_playout_sequence(playout)
mid_frame_result = self.check_every_frame_by_block(
mezzanine_qr_codes, change_switching_index_list, playout_sequence)
for i, starting_index in enumerate(change_switching_index_list):
if i > 0:
# check previous ending frame and new starting frame numbers
# the expected frame number position in the content being switched from is the expected relative time
# of the switch (derived from the test config) * that content's frames per second
previous_ending_frame_num = round(
switching_positions[i] / 1000 *
mezzanine_qr_codes[starting_index - 1].frame_rate)
# compare expected with the actual frame number detected at this switch point
diff_ending_frame = abs(mezzanine_qr_codes[starting_index -
1].frame_number -
previous_ending_frame_num)
if diff_ending_frame != 0:
mid_frame_result = False
self.result["message"] += (
f" Playout {playout_sequence[i - 1]} ending frame found is {mezzanine_qr_codes[starting_index - 1].frame_number },"
f" expected to end with {previous_ending_frame_num}.")
# the expected frame number position in the content being switched to is the expected relative time
# of the switch (derived from the test config) * that content's frames per second
# compare expected with the actual frame number detected at this switch point
current_starting_frame_num = (
round(switching_positions[i] / 1000 *
mezzanine_qr_codes[starting_index].frame_rate) + 1)
diff_starting_frame = abs(
mezzanine_qr_codes[starting_index].frame_number -
current_starting_frame_num)
if diff_starting_frame != 0:
mid_frame_result = False
self.result["message"] += (
f" Playout {playout_sequence[i]} starting frame found is {mezzanine_qr_codes[starting_index].frame_number },"
f" expected to start from {current_starting_frame_num}."
)
return mid_frame_result