def test_streaming_ingest_from_io_streams(): ingestion_properties = IngestionProperties(database=db_name, table=table_name, dataFormat=DataFormat.CSV) byte_sequence = b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null' bytes_stream = io.BytesIO(byte_sequence) ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) str_sequence = '0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null' str_stream = io.StringIO(str_sequence) ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) byte_sequence = b'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}' bytes_stream = io.BytesIO(byte_sequence) ingestion_properties.format = DataFormat.JSON ingestion_properties.ingestion_mapping_reference = "JsonMapping" ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) str_sequence = u'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}' str_stream = io.StringIO(str_sequence) ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) byte_sequence = ( b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null' * 600000 ) bytes_stream = io.BytesIO(byte_sequence) try: ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) except KustoStreamMaxSizeExceededError: pass
def test_streaming_ingest_from_stream(self): responses.add_callback( responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=request_callback, ) ingest_client = KustoStreamingIngestClient( "https://somecluster.kusto.windows.net") ingestion_properties = IngestionProperties(database="database", table="table", dataFormat=DataFormat.CSV) byte_sequence = b"56,56,56" bytes_stream = io.BytesIO(byte_sequence) ingest_client.ingest_from_stream( bytes_stream, ingestion_properties=ingestion_properties) str_sequence = u"57,57,57" str_stream = io.StringIO(str_sequence) ingest_client.ingest_from_stream( str_stream, ingestion_properties=ingestion_properties) byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}' bytes_stream = io.BytesIO(byte_sequence) ingestion_properties.format = DataFormat.JSON try: ingest_client.ingest_from_stream( bytes_stream, ingestion_properties=ingestion_properties) except KustoMissingMappingReferenceError: pass ingestion_properties.ingestion_mapping_reference = "JsonMapping" ingest_client.ingest_from_stream( bytes_stream, ingestion_properties=ingestion_properties) str_sequence = u'{"Name":"Ben","Age":"56","Weight":"75"}' str_stream = io.StringIO(str_sequence) ingest_client.ingest_from_stream( str_stream, ingestion_properties=ingestion_properties) byte_sequence = b"56,56,56" * 600000 bytes_stream = io.BytesIO(byte_sequence) try: ingest_client.ingest_from_stream( bytes_stream, ingestion_properties=ingestion_properties) except KustoStreamMaxSizeExceededError: pass
def test_streaming_ingest_from_stream(self, ingest_client_class): responses.add_callback( responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)) ingest_client = ingest_client_class( "https://somecluster.kusto.windows.net") ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV) byte_sequence = b"56,56,56" bytes_stream = io.BytesIO(byte_sequence) result = ingest_client.ingest_from_stream( bytes_stream, ingestion_properties=ingestion_properties) assert result.status == IngestionStatus.SUCCESS str_sequence = "57,57,57" str_stream = io.StringIO(str_sequence) result = ingest_client.ingest_from_stream( str_stream, ingestion_properties=ingestion_properties) assert result.status == IngestionStatus.SUCCESS byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}' bytes_stream = io.BytesIO(byte_sequence) # Make sure we don't raise when not providing a mapping to JSON ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON) ingestion_properties.ingestion_mapping_reference = "JsonMapping" result = ingest_client.ingest_from_stream( bytes_stream, ingestion_properties=ingestion_properties) assert result.status == IngestionStatus.SUCCESS str_sequence = '{"Name":"Ben","Age":"56","Weight":"75"}' str_stream = io.StringIO(str_sequence) result = ingest_client.ingest_from_stream( str_stream, ingestion_properties=ingestion_properties) assert result.status == IngestionStatus.SUCCESS