def test_blob_info_json_mapping(self):
"""Tests serialization of json ingestion blob info."""
validation_policy = ValidationPolicy(
ValidationOptions.ValidateCsvInputConstantColumns,
ValidationImplications.BestEffort)
properties = IngestionProperties(
database="database",
table="table",
dataFormat=DataFormat.JSON,
ingestionMapping=[
ColumnMapping("ColumnName", "datatype", path="jsonpath")
],
additionalTags=["tag"],
ingestIfNotExists=["ingestIfNotExistTags"],
ingestByTags=["ingestByTags"],
dropByTags=["dropByTags"],
flushImmediately=True,
reportLevel=ReportLevel.DoNotReport,
reportMethod=ReportMethod.Queue,
validationPolicy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = _IngestionBlobInfo(blob,
properties,
auth_context="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def test_blob_info_csv_mapping(self):
"""Tests serialization of csv ingestion blob info."""
validation_policy = ValidationPolicy(
ValidationOptions.ValidateCsvInputConstantColumns,
ValidationImplications.BestEffort)
column_mapping = ColumnMapping("ColumnName", "cslDataType", ordinal=1)
properties = IngestionProperties(
database="database",
table="table",
data_format=DataFormat.CSV,
column_mappings=[column_mapping],
additional_tags=["tag"],
ingest_if_not_exists=["ingestIfNotExistTags"],
ingest_by_tags=["ingestByTags"],
drop_by_tags=["dropByTags"],
flush_immediately=True,
report_level=ReportLevel.DoNotReport,
report_method=ReportMethod.Queue,
validation_policy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = IngestionBlobInfo(blob,
properties,
auth_context="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def test_with_constant_value():
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int", const_value="1")],
data_format=DataFormat.PARQUET,
ingestion_mapping_kind=IngestionMappingKind.PARQUET,
)
def test_duplicate_reference_and_column_mappings_raises():
"""Tests invalid ingestion properties."""
with pytest.raises(KustoDuplicateMappingError):
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int")],
ingestion_mapping_reference="ingestionMappingReference")
def test_mapping_kind_data_format_with_path():
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int", "path")],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_mapping_kind_data_format_with_no_properties():
with pytest.raises(KustoMappingError):
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("test", "int")],
data_format=DataFormat.AVRO,
ingestion_mapping_kind=IngestionMappingKind.AVRO,
)
def test_mapping_kind_data_format_invalid_no_name():
with pytest.raises(KustoMappingError):
IngestionProperties(
database="database",
table="table",
column_mappings=[ColumnMapping("", "int")],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_mapping_kind_data_format_with_transform():
IngestionProperties(
database="database",
table="table",
column_mappings=[
ColumnMapping("test",
"int",
transform=TransformationMethod.SOURCE_LINE_NUMBER)
],
data_format=DataFormat.JSON,
ingestion_mapping_kind=IngestionMappingKind.JSON,
)
def test_table_csv_mappings():
"""A method to define csv mappings to test table."""
mappings = list()
mappings.append(
ColumnMapping(column_name="rownumber",
column_type="int",
ordinal=0))
mappings.append(
ColumnMapping(column_name="rowguid",
column_type="string",
ordinal=1))
mappings.append(
ColumnMapping(column_name="xdouble", column_type="real",
ordinal=2))
mappings.append(
ColumnMapping(column_name="xfloat", column_type="real", ordinal=3))
mappings.append(
ColumnMapping(column_name="xbool", column_type="bool", ordinal=4))
mappings.append(
ColumnMapping(column_name="xint16", column_type="int", ordinal=5))
mappings.append(
ColumnMapping(column_name="xint32", column_type="int", ordinal=6))
mappings.append(
ColumnMapping(column_name="xint64", column_type="long", ordinal=7))
mappings.append(
ColumnMapping(column_name="xuint8", column_type="long", ordinal=8))
mappings.append(
ColumnMapping(column_name="xuint16", column_type="long",
ordinal=9))
mappings.append(
ColumnMapping(column_name="xuint32",
column_type="long",
ordinal=10))
mappings.append(
ColumnMapping(column_name="xuint64",
column_type="long",
ordinal=11))
mappings.append(
ColumnMapping(column_name="xdate",
column_type="datetime",
ordinal=12))
mappings.append(
ColumnMapping(column_name="xsmalltext",
column_type="string",
ordinal=13))
mappings.append(
ColumnMapping(column_name="xtext",
column_type="string",
ordinal=14))
mappings.append(
ColumnMapping(column_name="xnumberAsText",
column_type="string",
ordinal=15))
mappings.append(
ColumnMapping(column_name="xtime",
column_type="timespan",
ordinal=16))
mappings.append(
ColumnMapping(column_name="xtextWithNulls",
column_type="string",
ordinal=17))
mappings.append(
ColumnMapping(column_name="xdynamicWithNulls",
column_type="dynamic",
ordinal=18))
return mappings
def test_table_json_mappings():
"""A method to define json mappings to test table."""
mappings = list()
mappings.append(
ColumnMapping(column_name="rownumber",
path="$.rownumber",
column_type="int"))
mappings.append(
ColumnMapping(column_name="rowguid",
path="$.rowguid",
column_type="string"))
mappings.append(
ColumnMapping(column_name="xdouble",
path="$.xdouble",
column_type="real"))
mappings.append(
ColumnMapping(column_name="xfloat",
path="$.xfloat",
column_type="real"))
mappings.append(
ColumnMapping(column_name="xbool",
path="$.xbool",
column_type="bool"))
mappings.append(
ColumnMapping(column_name="xint16",
path="$.xint16",
column_type="int"))
mappings.append(
ColumnMapping(column_name="xint32",
path="$.xint32",
column_type="int"))
mappings.append(
ColumnMapping(column_name="xint64",
path="$.xint64",
column_type="long"))
mappings.append(
ColumnMapping(column_name="xuint8",
path="$.xuint8",
column_type="long"))
mappings.append(
ColumnMapping(column_name="xuint16",
path="$.xuint16",
column_type="long"))
mappings.append(
ColumnMapping(column_name="xuint32",
path="$.xuint32",
column_type="long"))
mappings.append(
ColumnMapping(column_name="xuint64",
path="$.xuint64",
column_type="long"))
mappings.append(
ColumnMapping(column_name="xdate",
path="$.xdate",
column_type="datetime"))
mappings.append(
ColumnMapping(column_name="xsmalltext",
path="$.xsmalltext",
column_type="string"))
mappings.append(
ColumnMapping(column_name="xtext",
path="$.xtext",
column_type="string"))
mappings.append(
ColumnMapping(column_name="xnumberAsText",
path="$.xnumberAsText",
column_type="string"))
mappings.append(
ColumnMapping(column_name="xtime",
path="$.xtime",
column_type="timespan"))
mappings.append(
ColumnMapping(column_name="xtextWithNulls",
path="$.xtextWithNulls",
column_type="string"))
mappings.append(
ColumnMapping(column_name="xdynamicWithNulls",
path="$.xdynamicWithNulls",
column_type="dynamic"))
return mappings
