def create_video_metadata(name: str) -> DataFrameMetadata:
"""Create video metadata object.
We have predefined columns for such a object
id: the frame id
data: the frame data
Arguments:
name (str): name of the metadata to be added to the catalog
Returns:
DataFrameMetadata: corresponding metadata for the input table info
"""
catalog = CatalogManager()
columns = [ColumnDefinition('id', ColumnType.INTEGER, None,
[], ColConstraintInfo(unique=True))]
# the ndarray dimensions are set as None. We need to fix this as we
# cannot assume. Either ask the user to provide this with load or
# we infer this from the provided video.
columns.append(
ColumnDefinition(
'data', ColumnType.NDARRAY, NdArrayType.UINT8, [None, None, None]
)
)
col_metadata = create_column_metadata(columns)
uri = str(generate_file_path(name))
metadata = catalog.create_metadata(
name, uri, col_metadata, identifier_column='id')
return metadata
def test_create_udf_statement(self):
parser = Parser()
create_udf_query = """CREATE UDF FastRCNN
INPUT (Frame_Array NDARRAY (3, 256, 256))
OUTPUT (Labels NDARRAY (10), Bbox NDARRAY (10, 4))
TYPE Classification
IMPL 'data/fastrcnn.py';
"""
eva_statement_list = parser.parse(create_udf_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type,
StatementType.CREATE_UDF)
create_udf_stmt = eva_statement_list[0]
self.assertEqual(create_udf_stmt.name, 'FastRCNN')
self.assertEqual(create_udf_stmt.if_not_exists, False)
self.assertEqual(
create_udf_stmt.inputs[0],
ColumnDefinition('Frame_Array', ParserColumnDataType.NDARRAY,
[3, 256, 256]))
self.assertEqual(
create_udf_stmt.outputs[0],
ColumnDefinition('Labels', ParserColumnDataType.NDARRAY, [10]))
self.assertEqual(
create_udf_stmt.outputs[1],
ColumnDefinition('Bbox', ParserColumnDataType.NDARRAY, [10, 4]))
self.assertEqual(create_udf_stmt.impl_path, Path('data/fastrcnn.py'))
self.assertEqual(create_udf_stmt.udf_type, 'Classification')
def test_create_udf_statement(self):
parser = Parser()
create_udf_query = """CREATE UDF FastRCNN
INPUT (Frame_Array NDARRAY (3, 256, 256))
OUTPUT (Labels NDARRAY (10), Bbox NDARRAY (10, 4))
TYPE Classification
IMPL 'data/fastrcnn.py';
"""
expected_stmt = CreateUDFStatement('FastRCNN', False, [
ColumnDefinition('Frame_Array', ParserColumnDataType.NDARRAY,
[3, 256, 256])
], [
ColumnDefinition('Labels', ParserColumnDataType.NDARRAY, [10]),
ColumnDefinition('Bbox', ParserColumnDataType.NDARRAY, [10, 4])
], Path('data/fastrcnn.py'), 'Classification')
eva_statement_list = parser.parse(create_udf_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type,
StatementType.CREATE_UDF)
create_udf_stmt = eva_statement_list[0]
self.assertEqual(create_udf_stmt, expected_stmt)
def test_should_return_false_for_unequal_expression(self):
table = TableRef(TableInfo('MyVideo'))
load_stmt = LoadDataStatement(table, Path('data/video.mp4'))
insert_stmt = InsertTableStatement(table)
create_udf = CreateUDFStatement('udf', False, [
ColumnDefinition('frame', ParserColumnDataType.NDARRAY,
[3, 256, 256])
], [ColumnDefinition('labels', ParserColumnDataType.NDARRAY, [10])],
Path('data/fastrcnn.py'),
'Classification')
select_stmt = SelectStatement()
self.assertNotEqual(load_stmt, insert_stmt)
self.assertNotEqual(insert_stmt, load_stmt)
self.assertNotEqual(create_udf, insert_stmt)
self.assertNotEqual(select_stmt, create_udf)
def visitColumnDeclaration(self,
ctx: evaql_parser.ColumnDeclarationContext):
data_type, dimensions = self.visit(ctx.columnDefinition())
column_name = self.visit(ctx.uid())
if column_name is not None:
return ColumnDefinition(column_name, data_type, dimensions)
def test_column_definition_to_udf_io(self):
col = ColumnDefinition('data', ColumnType.NDARRAY, NdArrayType.UINT8,
[None, None, None])
col_list = [col, col]
actual = column_definition_to_udf_io(col_list, True)
for io in actual:
self.assertEqual(io.name, 'data')
self.assertEqual(io.type, ColumnType.NDARRAY)
self.assertEqual(io.is_nullable, False)
self.assertEqual(io.array_type, NdArrayType.UINT8)
self.assertEqual(io.array_dimensions, [None, None, None])
self.assertEqual(io.is_input, True)
self.assertEqual(io.udf_id, None)
# input not list
actual2 = column_definition_to_udf_io(col, True)
for io in actual2:
self.assertEqual(io.name, 'data')
self.assertEqual(io.type, ColumnType.NDARRAY)
self.assertEqual(io.is_nullable, False)
self.assertEqual(io.array_type, NdArrayType.UINT8)
self.assertEqual(io.array_dimensions, [None, None, None])
self.assertEqual(io.is_input, True)
self.assertEqual(io.udf_id, None)