def test_logical_or(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_left = ComparisonExpression(
ExpressionType.COMPARE_EQUAL, tpl_exp, const_exp)
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(1)
comparison_expression_right = ComparisonExpression(
ExpressionType.COMPARE_GREATER, tpl_exp, const_exp)
logical_expr = LogicalExpression(ExpressionType.LOGICAL_OR,
comparison_expression_left,
comparison_expression_right)
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
],
info=None)
expected_value = [[True], [True], [True]]
output_value = logical_expr.evaluate(input_batch)
self.assertEqual(expected_value, output_value)
def test_should_return_only_frames_satisfy_predicate(self):
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(1, 2 * np.ones((1, 1)), None)
frame_3 = Frame(1, 3 * np.ones((1, 1)), None)
outcome_1 = Prediction(frame_1, ["car", "bus"], [0.5, 0.6])
outcome_2 = Prediction(frame_2, ["bus"], [0.5, 0.6])
outcome_3 = Prediction(frame_3, ["car", "train"], [0.5, 0.6])
batch = FrameBatch(
frames=[
frame_1,
frame_2,
frame_3,
],
info=None,
outcomes={"test": [outcome_1, outcome_2, outcome_3]})
expression = type("AbstractExpression", (),
{"evaluate": lambda x: [False, False, True]})
plan = type("ScanPlan", (), {"predicate": expression})
predicate_executor = SequentialScanExecutor(plan)
predicate_executor.append_child(DummyExecutor([batch]))
expected = FrameBatch(frames=[frame_3],
info=None,
outcomes={"test": [outcome_3]})
filtered = list(predicate_executor.next())[0]
self.assertEqual(expected, filtered)
def test_should_return_all_frames_when_no_predicate_is_applied(self):
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(1, 2 * np.ones((1, 1)), None)
frame_3 = Frame(1, 3 * np.ones((1, 1)), None)
outcome_1 = Prediction(frame_1, ["car", "bus"], [0.5, 0.6])
outcome_2 = Prediction(frame_2, ["bus"], [0.5, 0.6])
outcome_3 = Prediction(frame_3, ["car", "train"], [0.5, 0.6])
batch = FrameBatch(
frames=[
frame_1,
frame_2,
frame_3,
],
info=None,
outcomes={"test": [outcome_1, outcome_2, outcome_3]})
plan = type("ScanPlan", (), {"predicate": None})
predicate_executor = SequentialScanExecutor(plan)
predicate_executor.append_child(DummyExecutor([batch]))
expected = FrameBatch(
frames=[frame_1, frame_2, frame_3],
info=None,
outcomes={"test": [outcome_1, outcome_2, outcome_3]})
filtered = list(predicate_executor.next())[0]
self.assertEqual(expected, filtered)
def test_set_outcomes_method_should_set_temp_outcome_when_bool_is_true(
self):
batch = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)], info=None)
batch.set_outcomes('test', [1], is_temp=True)
expected = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)],
info=None,
temp_outcomes={'test': [1]})
self.assertEqual(expected, batch)
def test_frames_as_numpy_array_should_frames_as_numpy_array(self):
batch = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, np.ones((1, 1)), None)
],
info=None)
expected = list(np.ones((2, 1, 1)))
actual = list(batch.frames_as_numpy_array())
self.assertEqual(expected, actual)
def test_return_only_frames_specified_in_the_indices(self):
batch = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, np.ones((1, 1)), None)
],
info=None)
expected = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)],
info=None)
output = batch[[0]]
self.assertEqual(expected, output)
def test_fetching_frames_by_index_should_also_return_outcomes(self):
batch = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, np.ones((1, 1)), None)
],
info=None,
outcomes={'test': [[None], [None]]})
expected = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)],
info=None,
outcomes={'test': [[None]]})
self.assertEqual(expected, batch[[0]])
def test_slicing_should_word_for_negative_stop_value(self):
batch = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, 2 * np.ones((1, 1)), None)
],
info=None,
outcomes={'test': [[None], [None]]})
expected = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)],
info=None,
outcomes={'test': [[None]]})
self.assertEqual(expected, batch[:-1])
def test_slicing_on_batched_should_return_new_batch_frame(self):
batch = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, 2 * np.ones((1, 1)), None)
],
info=None,
outcomes={'test': [[None], [None]]})
expected = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)],
info=None,
outcomes={'test': [[None]]})
self.assertEqual(batch, batch[:])
self.assertEqual(expected, batch[:-1])
def test_func_expr_with_cmpr_and_const_expr_should_work(self):
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(1, 2 * np.ones((1, 1)), None)
outcome_1 = Prediction(frame_1, ["car", "bus"], [0.5, 0.6])
outcome_2 = Prediction(frame_1, ["bus"], [0.6])
func = FunctionExpression(lambda x: [outcome_1, outcome_2])
value_expr = ConstantValueExpression("car")
expression_tree = ComparisonExpression(ExpressionType.COMPARE_EQUAL,
func, value_expr)
batch = FrameBatch(frames=[frame_1, frame_2])
self.assertEqual([True, False], expression_tree.evaluate(batch))
def test_slicing_should_work_with_skip_value(self):
batch = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, 2 * np.ones((1, 1)), None),
Frame(1, np.ones((1, 1)), None)
],
info=None,
outcomes={'test': [[None], [None], [None]]})
expected = FrameBatch(frames=[
Frame(1, np.ones((1, 1)), None),
Frame(1, np.ones((1, 1)), None)
],
info=None,
outcomes={'test': [[None], [None]]})
self.assertEqual(expected, batch[::2])
def test_should_return_batches_equivalent_to_number_of_frames(self):
frame_dog = Frame(
1,
self._load_image(os.path.join(self.base_path, 'data', 'dog.jpeg')),
None)
frame_dog_cat = Frame(
1,
self._load_image(
os.path.join(self.base_path, 'data', 'dog_cat.jpg')), None)
frame_batch = Batch([frame_dog, frame_dog_cat])
detector = FastRCNNObjectDetector()
result = detector.classify(frame_batch)
self.assertEqual(["dog"], result[0].labels)
self.assertEqual(["cat", "dog"], result[1].labels)
def test_frame_filtering_for_depth_estimation(self):
"""
Unit test method to test frame filtering functionality.
it loops over frames and sends them over to frame filtering
object's apply_filter method.
Finally it verifies that depth mask is applied to the frames
except every fifth one.
"""
# create two frames from kitti car dataset
frame_1 = Frame(
1,
self._load_image(
os.path.join(self.base_path, 'data', 'kitti_car_1.png')), None)
frame_2 = Frame(
1,
self._load_image(
os.path.join(self.base_path, 'data', 'kitti_car_2.png')), None)
# create a batch of 2 frames
frame_batch = FrameBatch([frame_1, frame_2], None)
frames = frame_batch.frames_as_numpy_array()
# initialize the frame filtering class object
frame_filter = FrameFilter()
# create a random depth mask array
depth_mask = np.random.rand(frames[0].shape[0], frames[0].shape[1],
frames[0].shape[2])
# iterate over frames in the batch
for i, img in enumerate(frames):
# apply frame filter on each frame
img = frame_filter.apply_filter(img, depth_mask)
# For every fifth frame the mask should not be applied. Hence, the
# frame returned by apply_filter method should be same as original
# frame
if i % 5 == 0:
self.assertTrue(np.array_equal(img, frames[0]))
else:
# Every other frame should be transformed after applying depth
# mask
self.assertTrue(
np.array_equal(img, frames[i] * depth_mask[:, :, None]))
def test_has_outcomes_returns_true_if_the_given_name_is_in_outcomes(self):
batch = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)], info=None)
batch.set_outcomes('test_temp', [1], is_temp=True)
batch.set_outcomes('test', [1])
self.assertTrue(batch.has_outcome('test'))
self.assertTrue(batch.has_outcome('test_temp'))
def load(self):
video = cv2.VideoCapture(self.video_metadata.file)
video_start = self.offset if self.offset else 0
video.set(cv2.CAP_PROP_POS_FRAMES, video_start)
LoggingManager().log("Loading frames", LoggingLevel.CRITICAL)
_, frame = video.read()
frame_ind = video_start - 1
info = None
if frame is not None:
(height, width, num_channels) = frame.shape
info = FrameInfo(height, width, num_channels, ColorSpace.BGR)
frames = []
while frame is not None:
frame_ind += 1
eva_frame = Frame(frame_ind, frame, info)
if self.skip_frames > 0 and frame_ind % self.skip_frames != 0:
_, frame = video.read()
continue
frames.append(eva_frame)
if self.limit and frame_ind >= self.limit:
return FrameBatch(frames, info)
if len(frames) % self.batch_size == 0:
yield FrameBatch(frames, info)
frames = []
_, frame = video.read()
if frames:
return FrameBatch(frames, info)
def create_dummy_frames(self, num_frames=NUM_FRAMES, filters=[]):
if not filters:
filters = range(num_frames)
for i in filters:
yield Frame(
i,
np.array(np.ones((2, 2, 3)) * 0.1 * float(i + 1) * 255,
dtype=np.uint8), FrameInfo(2, 2, 3, ColorSpace.BGR))
def test_should_check_if_batch_frames_equivalent_to_number_of_scores(self):
batch = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)], info=None)
predictions = [['A', 'B']]
scores = []
self.assertRaises(
AssertionError,
lambda x=None: Prediction.predictions_from_batch_and_lists(
batch, predictions, scores))
def test_should_return_list_of_predictions_for_each_frame_in_batch(self):
batch = FrameBatch(frames=[Frame(1, np.ones((1, 1)), None)], info=None)
predictions = [['A', 'B']]
scores = [[1, 1]]
expected = [Prediction(batch.frames[0], predictions[0], scores[0])]
actual = Prediction.predictions_from_batch_and_lists(
batch, predictions, scores)
self.assertEqual(expected, actual)
def test_should_check_if_batch_frames_equivalent_to_no_of_boxes_if_given(
self):
batch = Batch(frames=[Frame(1, np.ones((1, 1)), None)])
predictions = [['A', 'B']]
scores = [[1, 1]]
boxes = []
self.assertRaises(
AssertionError,
lambda x=None: Prediction.predictions_from_batch_and_lists(
batch, predictions, scores, boxes=boxes))
def test_should_return_only_frames_satisfy_predicate(self):
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(1, 2 * np.ones((1, 1)), None)
frame_3 = Frame(1, 3 * np.ones((1, 1)), None)
batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
], info=None)
expression = type("AbstractExpression", (),
{"evaluate": lambda x: [False, False, True]})
plan = type("PPScanPlan", (), {"predicate": expression})
predicate_executor = PPExecutor(plan)
predicate_executor.append_child(DummyExecutor([batch]))
expected = FrameBatch(frames=[frame_3], info=None)
filtered = list(predicate_executor.next())[0]
self.assertEqual(expected, filtered)
def test_comparison_compare_leq(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(2)
cmpr_exp = ComparisonExpression(ExpressionType.COMPARE_LEQ, tpl_exp,
const_exp)
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
],
info=None)
expected_value = [[True], [True], [False]]
output_value = cmpr_exp.evaluate(input_batch)
self.assertEqual(expected_value, output_value)
def test_divide(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
arithmetic_expr = ArithmeticExpression(
ExpressionType.ARITHMETIC_DIVIDE,
tpl_exp,
const_exp
)
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
], info=None)
expected_value = [[0.2], [0.4], [0.6]]
output_value = arithmetic_expr.evaluate(input_batch)
self.assertEqual(expected_value[0][0], output_value[0][0])
def _load_frames(self) -> Iterator[Frame]:
info = None
with make_reader(self.video_metadata.file_url,
shard_count=self.total_shards,
cur_shard=self.curr_shard) \
as reader:
for frame_ind, row in enumerate(reader):
if info is None:
(height, width, num_channels) = row.frame_data.shape
info = FrameInfo(height, width, num_channels,
ColorSpace.BGR)
yield Frame(row.frame_id, row.frame_data, info)
def test_aggregation_min(self):
columnName = TupleValueExpression(0)
aggr_expr = AggregationExpression(
ExpressionType.AGGREGATION_MIN,
None,
columnName
)
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
outcome_1 = Prediction(frame_1, ["car", "bus"], [0.5, 0.6])
outcome_2 = Prediction(frame_2, ["bus"], [0.5, 0.6])
outcome_3 = Prediction(frame_3, ["car", "train"], [0.5, 0.6])
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
], info=None)
expected_value = 1
output_value = aggr_expr.evaluate(input_batch)
self.assertEqual(expected_value, output_value)
def test_should_return_batches_equivalent_to_number_of_frames(self):
"""
Unit test method which creates a batch of frames, sends it for
model prediction.
It then checks if the returned object size is as expected.
"""
# create two frames from kitti car dataset
frame_1 = Frame(
1,
self._load_image(
os.path.join(self.base_path, 'data', 'kitti_car_1.png')), None)
frame_2 = Frame(
1,
self._load_image(
os.path.join(self.base_path, 'data', 'kitti_car_2.png')), None)
# create a batch of 2 frames
frame_batch = FrameBatch([frame_1, frame_2], None)
# process the batch frames for depth and segmentation prediction
estimator = DepthEstimator('ExpKITTI_joint.ckpt')
result = estimator.classify(frame_batch)
# assert if result size is same as the batch size
self.assertEqual(len(result), 2)
# assert if frame in result object is same as original frame
self.assertTrue(np.array_equal(result[0].frame.data, frame_1.data))
self.assertTrue(np.array_equal(result[1].frame.data, frame_2.data))
# assert that depth and segmentation results should not be null
assert result[0].depth is not None
assert result[0].segm is not None
assert result[1].depth is not None
assert result[1].segm is not None
def test_addition(self):
tpl_exp = TupleValueExpression(0)
const_exp = ConstantValueExpression(5)
arithmetic_expr = ArithmeticExpression(
ExpressionType.ARITHMETIC_ADD,
tpl_exp,
const_exp
)
tuple1 = [5, 2, 3]
frame_1 = Frame(1, np.ones((1, 1)), None)
frame_2 = Frame(2, 2 * np.ones((1, 1)), None)
frame_3 = Frame(3, 3 * np.ones((1, 1)), None)
input_batch = FrameBatch(frames=[
frame_1,
frame_2,
frame_3,
], info=None)
expected_value = [[6], [7], [8]]
output_value = arithmetic_expr.evaluate(input_batch)
self.assertEqual(expected_value[0][0], output_value[0][0])
def test_load_frame_load_frames_using_petastorm(self, mock):
mock.return_value = self.DummyReader(
map(lambda i: self.DummyRow(i,
np.ones((2, 2, 3)) * i), range(3)))
video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
video_loader = PetastormLoader(video_info,
curr_shard=3,
total_shards=3)
actual = list(video_loader._load_frames())
expected = [
Frame(i,
np.ones((2, 2, 3)) * i, FrameInfo(2, 2, 3, ColorSpace.BGR))
for i in range(3)
]
self.assertEqual(expected, actual)
def _load_frames(self) -> Iterator[Frame]:
video = cv2.VideoCapture(self.video_metadata.file_url)
video_start = self.offset if self.offset else 0
video.set(cv2.CAP_PROP_POS_FRAMES, video_start)
LoggingManager().log("Loading frames", LoggingLevel.INFO)
_, frame = video.read()
frame_ind = video_start - 1
info = None
if frame is not None:
(height, width, num_channels) = frame.shape
info = FrameInfo(height, width, num_channels, ColorSpace.BGR)
while frame is not None:
frame_ind += 1
yield Frame(frame_ind, frame, info)
_, frame = video.read()
def loadVideo(self, meta):
video = cv2.VideoCapture(meta.file)
video.set(cv2.CAP_PROP_POS_FRAMES, 0)
_, frame = video.read()
frame_ind = 0
info = None
if frame is not None:
(height, width, channels) = frame.shape
info = FrameInfo(height, width, channels, ColorSpace.RGB)
frames = []
while frame is not None:
# Save frame
eva_frame = Frame(frame_ind, frame, info)
frames.append(eva_frame)
# Read next frame
_, frame = video.read()
frame_ind += 1
return (frames, info)
def test_VidToFrameClassifier(self):
model = video_action_classification.VideoToFrameClassifier()
assert model is not None
X = np.random.random([240, 320, 3])
model.classify(FrameBatch([Frame(0, X, None)], None))
