def test_streaming_ingest_from_io_streams():
ingestion_properties = IngestionProperties(database=db_name, table=table_name, dataFormat=DataFormat.CSV)
byte_sequence = b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
bytes_stream = io.BytesIO(byte_sequence)
ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
str_sequence = '0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
str_stream = io.StringIO(str_sequence)
ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
byte_sequence = b'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}'
bytes_stream = io.BytesIO(byte_sequence)
ingestion_properties.format = DataFormat.JSON
ingestion_properties.ingestion_mapping_reference = "JsonMapping"
ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
str_sequence = u'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}'
str_stream = io.StringIO(str_sequence)
ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
byte_sequence = (
b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
* 600000
)
bytes_stream = io.BytesIO(byte_sequence)
try:
ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
except KustoStreamMaxSizeExceededError:
pass
def test_streaming_ingest_from_stream(self):
responses.add_callback(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
callback=request_callback,
)
ingest_client = KustoStreamingIngestClient(
"https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database",
table="table",
dataFormat=DataFormat.CSV)
byte_sequence = b"56,56,56"
bytes_stream = io.BytesIO(byte_sequence)
ingest_client.ingest_from_stream(
bytes_stream, ingestion_properties=ingestion_properties)
str_sequence = u"57,57,57"
str_stream = io.StringIO(str_sequence)
ingest_client.ingest_from_stream(
str_stream, ingestion_properties=ingestion_properties)
byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}'
bytes_stream = io.BytesIO(byte_sequence)
ingestion_properties.format = DataFormat.JSON
try:
ingest_client.ingest_from_stream(
bytes_stream, ingestion_properties=ingestion_properties)
except KustoMissingMappingReferenceError:
pass
ingestion_properties.ingestion_mapping_reference = "JsonMapping"
ingest_client.ingest_from_stream(
bytes_stream, ingestion_properties=ingestion_properties)
str_sequence = u'{"Name":"Ben","Age":"56","Weight":"75"}'
str_stream = io.StringIO(str_sequence)
ingest_client.ingest_from_stream(
str_stream, ingestion_properties=ingestion_properties)
byte_sequence = b"56,56,56" * 600000
bytes_stream = io.BytesIO(byte_sequence)
try:
ingest_client.ingest_from_stream(
bytes_stream, ingestion_properties=ingestion_properties)
except KustoStreamMaxSizeExceededError:
pass
def test_streaming_ingest_from_stream(self, ingest_client_class):
responses.add_callback(
responses.POST,
"https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
callback=lambda r: request_callback(r, ingest_client_class))
ingest_client = ingest_client_class(
"https://somecluster.kusto.windows.net")
ingestion_properties = IngestionProperties(database="database",
table="table",
data_format=DataFormat.CSV)
byte_sequence = b"56,56,56"
bytes_stream = io.BytesIO(byte_sequence)
result = ingest_client.ingest_from_stream(
bytes_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
str_sequence = "57,57,57"
str_stream = io.StringIO(str_sequence)
result = ingest_client.ingest_from_stream(
str_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}'
bytes_stream = io.BytesIO(byte_sequence)
# Make sure we don't raise when not providing a mapping to JSON
ingestion_properties = IngestionProperties(database="database",
table="table",
data_format=DataFormat.JSON)
ingestion_properties.ingestion_mapping_reference = "JsonMapping"
result = ingest_client.ingest_from_stream(
bytes_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
str_sequence = '{"Name":"Ben","Age":"56","Weight":"75"}'
str_stream = io.StringIO(str_sequence)
result = ingest_client.ingest_from_stream(
str_stream, ingestion_properties=ingestion_properties)
assert result.status == IngestionStatus.SUCCESS
