def test_blob_info_csv_mapping(self):
""" Tests serialization of csv ingestion blob info. """
validation_policy = ValidationPolicy(
ValidationOptions.ValidateCsvInputConstantColumns,
ValidationImplications.BestEffort)
properties = IngestionProperties(
database="database",
table="table",
dataFormat=DataFormat.csv,
mapping=[CsvColumnMapping("ColumnName", "cslDataType", 1)],
additionalTags=["tag"],
ingestIfNotExists=["ingestIfNotExistTags"],
ingestByTags=["ingestByTags"],
dropByTags=["dropByTags"],
flushImmediately=True,
reportLevel=ReportLevel.DoNotReport,
reportMethod=ReportMethod.QueueAndTable,
validationPolicy=validation_policy,
)
blob = BlobDescriptor("somepath", 10)
blob_info = _IngestionBlobInfo(blob,
properties,
deleteSourcesOnSuccess=True,
authContext="authorizationContextText")
self._verify_ingestion_blob_info_result(blob_info.to_json())
def get_ingestion_mapping(
cls, data_format: str,
mapping: IngestionMapping) -> List[ColumnMapping]:
kusto_ingest_mapping = []
if data_format == "csv":
# TODO: need to add __str__ to columnMapping
mapping_func = lambda source_col, target_col: CsvColumnMapping(
target_col.name, target_col.data_type.value, source_col.index)
if data_format in ["json", "singlejson", "multijson"]:
# TODO: need to add __str__ to columnMapping
mapping_func = lambda source_col, target_col: JsonColumnMapping(
target_col.name,
f"$.{source_col.name}",
cslDataType=target_col.data_type.value)
for col in mapping.columns:
kusto_ingest_mapping.append(mapping_func(col.source, col.target))
return kusto_ingest_mapping
def create_deft_table_csv_mappings():
"""A method to define csv mappings to deft table."""
mappings = list()
mappings.append(
CsvColumnMapping(columnName="rownumber",
cslDataType="int",
ordinal=0))
mappings.append(
CsvColumnMapping(columnName="rowguid",
cslDataType="string",
ordinal=1))
mappings.append(
CsvColumnMapping(columnName="xdouble",
cslDataType="real",
ordinal=2))
mappings.append(
CsvColumnMapping(columnName="xfloat",
cslDataType="real",
ordinal=3))
mappings.append(
CsvColumnMapping(columnName="xbool", cslDataType="bool",
ordinal=4))
mappings.append(
CsvColumnMapping(columnName="xint16", cslDataType="int",
ordinal=5))
mappings.append(
CsvColumnMapping(columnName="xint32", cslDataType="int",
ordinal=6))
mappings.append(
CsvColumnMapping(columnName="xint64",
cslDataType="long",
ordinal=7))
mappings.append(
CsvColumnMapping(columnName="xuint8",
cslDataType="long",
ordinal=8))
mappings.append(
CsvColumnMapping(columnName="xuint16",
cslDataType="long",
ordinal=9))
mappings.append(
CsvColumnMapping(columnName="xuint32",
cslDataType="long",
ordinal=10))
mappings.append(
CsvColumnMapping(columnName="xuint64",
cslDataType="long",
ordinal=11))
mappings.append(
CsvColumnMapping(columnName="xdate",
cslDataType="datetime",
ordinal=12))
mappings.append(
CsvColumnMapping(columnName="xsmalltext",
cslDataType="string",
ordinal=13))
mappings.append(
CsvColumnMapping(columnName="xtext",
cslDataType="string",
ordinal=14))
mappings.append(
CsvColumnMapping(columnName="xnumberAsText",
cslDataType="string",
ordinal=15))
mappings.append(
CsvColumnMapping(columnName="xtime",
cslDataType="timespan",
ordinal=16))
mappings.append(
CsvColumnMapping(columnName="xtextWithNulls",
cslDataType="string",
ordinal=17))
mappings.append(
CsvColumnMapping(columnName="xdynamicWithNulls",
cslDataType="dynamic",
ordinal=18))
return mappings
