def test_null_values_in_data(self, mock_post):
"""Tests response with null values in non nullable column types"""
client = KustoClient("https://somecluster.kusto.windows.net")
query = "PrimaryResultName"
response = client.execute_query("PythonTest", query)
assert response is not None
def test_partial_results(self, mock_post):
"""Tests partial results."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = """set truncationmaxrecords = 1;
range x from 1 to 2 step 1"""
self.assertRaises(KustoServiceError, client.execute_query, "PythonTest", query)
response = client.execute_query("PythonTest", query, accept_partial_results=True)
self.assertEqual(response.errors_count, 1)
self.assertIn("E_QUERY_RESULT_SET_TOO_LARGE", response.get_exceptions()[0])
self.assertEqual(len(response), 3)
results = list(response.primary_results[0])
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["x"], 1)
def test_partial_results(self, mock_post):
"""Tests partial results."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = """set truncationmaxrecords = 5;
range x from 1 to 10 step 1"""
properties = ClientRequestProperties()
properties.set_option(
ClientRequestProperties.OptionDeferPartialQueryFailures, False)
self.assertRaises(KustoServiceError, client.execute_query,
"PythonTest", query, properties)
properties.set_option(
ClientRequestProperties.OptionDeferPartialQueryFailures, True)
response = client.execute_query("PythonTest", query, properties)
self.assertEqual(response.errors_count, 1)
self.assertIn("E_QUERY_RESULT_SET_TOO_LARGE",
response.get_exceptions()[0])
self.assertEqual(len(response), 3)
results = list(response.primary_results[0])
self.assertEqual(len(results), 5)
self.assertEqual(results[0]["x"], 1)
def test_partial_results(self, mock_post):
"""Tests partial results."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = """set truncationmaxrecords = 5;
range x from 1 to 10 step 1"""
properties = ClientRequestProperties()
properties.set_option(
ClientRequestProperties.
results_defer_partial_query_failures_option_name, False)
self.assertRaises(KustoServiceError, client.execute_query,
"PythonTest", query, properties)
properties.set_option(
ClientRequestProperties.
results_defer_partial_query_failures_option_name, True)
response = client.execute_query("PythonTest", query, properties)
assert response.errors_count == 1
assert "E_QUERY_RESULT_SET_TOO_LARGE" in response.get_exceptions()[0]
assert len(response) == 3
results = list(response.primary_results[0])
assert len(results) == 5
assert results[0]["x"] == 1
def test_dynamic(self, mock_post):
"""Tests dynamic responses."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = """print dynamic(123), dynamic("123"), dynamic("test bad json"), dynamic(null), dynamic('{"rowId":2,"arr":[0,2]}'), dynamic({"rowId":2,"arr":[0,2]})"""
row = client.execute_query("PythonTest",
query).primary_results[0].rows[0]
self.assertIsInstance(row[0], int)
self.assertEqual(row[0], 123)
self.assertIsInstance(row[1], text_type)
self.assertEqual(row[1], "123")
self.assertIsInstance(row[2], text_type)
self.assertEqual(row[2], "test bad json")
self.assertEqual(row[3], None)
self.assertIsInstance(row[4], text_type)
self.assertEqual(row[4], '{"rowId":2,"arr":[0,2]}')
self.assertIsInstance(row[5], dict)
self.assertEqual(row[5], {"rowId": 2, "arr": [0, 2]})
def test_dynamic(self, mock_post):
"""Tests dynamic responses."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = """print dynamic(123), dynamic("123"), dynamic("test bad json"),"""
""" dynamic(null), dynamic('{"rowId":2,"arr":[0,2]}'), dynamic({"rowId":2,"arr":[0,2]})"""
row = client.execute_query("PythonTest",
query).primary_results[0].rows[0]
assert isinstance(row[0], int)
assert row[0] == 123
assert isinstance(row[1], str)
assert row[1] == "123"
assert isinstance(row[2], str)
assert row[2] == "test bad json"
assert row[3] is None
assert isinstance(row[4], str)
assert row[4] == '{"rowId":2,"arr":[0,2]}'
assert isinstance(row[5], dict)
assert row[5] == {"rowId": 2, "arr": [0, 2]}
response = client.execute(db, query)
except KustoServiceError as error:
print("2. Error:", error)
print("2. Is semantic error:", error.is_semantic_error())
print("2. Has partial results:", error.has_partial_results())
print("2. Result size:", len(error.get_partial_results()))
response = client.execute(db, query, accept_partial_results=True)
print("3. Response error count: ", response.errors_count)
print("3. Exceptions:", response.get_exceptions())
print("3. Result size:", len(response.primary_results))
# Query has semantic error
query = "StormEvents | where foo = bar"
try:
response = client.execute(db, query)
except KustoServiceError as error:
print("4. Error:", error)
print("4. Is semantic error:", error.is_semantic_error())
print("4. Has partial results:", error.has_partial_results())
client = KustoClient("https://kustolab.kusto.windows.net")
response = client.execute("ML", ".show version")
query = """
let max_t = datetime(2016-09-03);
service_traffic
| make-series num=count() on TimeStamp in range(max_t-5d, max_t, 1h) by OsVer
"""
response = client.execute_query("ML", query).primary_results[0]
def test_sanity_query(self, mock_post, mock_aad):
"""Test query V2."""
client = KustoClient("https://somecluster.kusto.windows.net")
response = client.execute_query("PythonTest", "Deft")
expected = {
"rownumber": None,
"rowguid": text_type(""),
"xdouble": None,
"xfloat": None,
"xbool": None,
"xint16": None,
"xint32": None,
"xint64": None,
"xuint8": None,
"xuint16": None,
"xuint32": None,
"xuint64": None,
"xdate": None,
"xsmalltext": text_type(""),
"xtext": text_type(""),
"xnumberAsText": text_type(""),
"xtime": None,
"xtextWithNulls": text_type(""),
"xdynamicWithNulls": text_type(""),
}
for row in response.primary_results[0]:
self.assertEqual(row["rownumber"], expected["rownumber"])
self.assertEqual(row["rowguid"], expected["rowguid"])
self.assertEqual(row["xdouble"], expected["xdouble"])
self.assertEqual(row["xfloat"], expected["xfloat"])
self.assertEqual(row["xbool"], expected["xbool"])
self.assertEqual(row["xint16"], expected["xint16"])
self.assertEqual(row["xint32"], expected["xint32"])
self.assertEqual(row["xint64"], expected["xint64"])
self.assertEqual(row["xuint8"], expected["xuint8"])
self.assertEqual(row["xuint16"], expected["xuint16"])
self.assertEqual(row["xuint32"], expected["xuint32"])
self.assertEqual(row["xuint64"], expected["xuint64"])
self.assertEqual(row["xdate"], expected["xdate"])
self.assertEqual(row["xsmalltext"], expected["xsmalltext"])
self.assertEqual(row["xtext"], expected["xtext"])
self.assertEqual(row["xnumberAsText"], expected["xnumberAsText"])
self.assertEqual(row["xtime"], expected["xtime"])
self.assertEqual(row["xtextWithNulls"], expected["xtextWithNulls"])
self.assertEqual(row["xdynamicWithNulls"], expected["xdynamicWithNulls"])
self.assertEqual(type(row["rownumber"]), type(expected["rownumber"]))
self.assertEqual(type(row["rowguid"]), type(expected["rowguid"]))
self.assertEqual(type(row["xdouble"]), type(expected["xdouble"]))
self.assertEqual(type(row["xfloat"]), type(expected["xfloat"]))
self.assertEqual(type(row["xbool"]), type(expected["xbool"]))
self.assertEqual(type(row["xint16"]), type(expected["xint16"]))
self.assertEqual(type(row["xint32"]), type(expected["xint32"]))
self.assertEqual(type(row["xint64"]), type(expected["xint64"]))
self.assertEqual(type(row["xuint8"]), type(expected["xuint8"]))
self.assertEqual(type(row["xuint16"]), type(expected["xuint16"]))
self.assertEqual(type(row["xuint32"]), type(expected["xuint32"]))
self.assertEqual(type(row["xuint64"]), type(expected["xuint64"]))
self.assertEqual(type(row["xdate"]), type(expected["xdate"]))
self.assertEqual(type(row["xsmalltext"]), type(expected["xsmalltext"]))
self.assertEqual(type(row["xtext"]), type(expected["xtext"]))
self.assertEqual(type(row["xnumberAsText"]), type(expected["xnumberAsText"]))
self.assertEqual(type(row["xtime"]), type(expected["xtime"]))
self.assertEqual(type(row["xtextWithNulls"]), type(expected["xtextWithNulls"]))
self.assertEqual(type(row["xdynamicWithNulls"]), type(expected["xdynamicWithNulls"]))
expected["rownumber"] = 0 if expected["rownumber"] is None else expected["rownumber"] + 1
expected["rowguid"] = text_type("0000000{0}-0000-0000-0001-020304050607".format(expected["rownumber"]))
expected["xdouble"] = round(float(0) if expected["xdouble"] is None else expected["xdouble"] + 1.0001, 4)
expected["xfloat"] = round(float(0) if expected["xfloat"] is None else expected["xfloat"] + 1.01, 2)
expected["xbool"] = False if expected["xbool"] is None else not expected["xbool"]
expected["xint16"] = 0 if expected["xint16"] is None else expected["xint16"] + 1
expected["xint32"] = 0 if expected["xint32"] is None else expected["xint32"] + 1
expected["xint64"] = 0 if expected["xint64"] is None else expected["xint64"] + 1
expected["xuint8"] = 0 if expected["xuint8"] is None else expected["xuint8"] + 1
expected["xuint16"] = 0 if expected["xuint16"] is None else expected["xuint16"] + 1
expected["xuint32"] = 0 if expected["xuint32"] is None else expected["xuint32"] + 1
expected["xuint64"] = 0 if expected["xuint64"] is None else expected["xuint64"] + 1
expected["xdate"] = expected["xdate"] or datetime(2013, 1, 1, 1, 1, 1, 0, tzinfo=tzutc())
expected["xdate"] = expected["xdate"].replace(year=expected["xdate"].year + 1)
expected["xsmalltext"] = DIGIT_WORDS[int(expected["xint16"])]
expected["xtext"] = DIGIT_WORDS[int(expected["xint16"])]
expected["xnumberAsText"] = text_type(expected["xint16"])
microseconds = 1001 if expected["rownumber"] == 5 else 1000
expected["xtime"] = (
timedelta()
if expected["xtime"] is None
else (abs(expected["xtime"]) + timedelta(seconds=1, microseconds=microseconds))
* (-1) ** (expected["rownumber"] + 1)
)
if expected["xint16"] > 0:
expected["xdynamicWithNulls"] = text_type('{{"rowId":{0},"arr":[0,{0}]}}'.format(expected["xint16"]))
def test_sanity_data_frame(self, mock_post):
"""Tests KustoResponse to pandas.DataFrame."""
from pandas import DataFrame, Series
from pandas.util.testing import assert_frame_equal
client = KustoClient("https://somecluster.kusto.windows.net")
data_frame = dataframe_from_result_table(
client.execute_query("PythonTest", "Deft").primary_results[0], raise_errors=False
)
self.assertEqual(len(data_frame.columns), 19)
expected_dict = {
"rownumber": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"rowguid": Series(
[
"",
"00000000-0000-0000-0001-020304050607",
"00000001-0000-0000-0001-020304050607",
"00000002-0000-0000-0001-020304050607",
"00000003-0000-0000-0001-020304050607",
"00000004-0000-0000-0001-020304050607",
"00000005-0000-0000-0001-020304050607",
"00000006-0000-0000-0001-020304050607",
"00000007-0000-0000-0001-020304050607",
"00000008-0000-0000-0001-020304050607",
"00000009-0000-0000-0001-020304050607",
],
dtype=object,
),
"xdouble": Series([None, 0.0, 1.0001, 2.0002, 3.0003, 4.0004, 5.0005, 6.0006, 7.0007, 8.0008, 9.0009]),
"xfloat": Series([None, 0.0, 1.01, 2.02, 3.03, 4.04, 5.05, 6.06, 7.07, 8.08, 9.09]),
"xbool": Series([None, False, True, False, True, False, True, False, True, False, True], dtype=bool),
"xint16": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xint32": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xint64": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xuint8": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xuint16": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xuint32": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xuint64": Series([None, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]),
"xdate": Series(
[
"NaT",
"2014-01-01T01:01:01.000000000",
"2015-01-01T01:01:01.000000000",
"2016-01-01T01:01:01.000000000",
"2017-01-01T01:01:01.000000000",
"2018-01-01T01:01:01.000000000",
"2019-01-01T01:01:01.000000000",
"2020-01-01T01:01:01.000000000",
"2021-01-01T01:01:01.000000000",
"2022-01-01T01:01:01.000000000",
"2023-01-01T01:01:01.000000000",
],
dtype="datetime64[ns]",
),
"xsmalltext": Series(
["", "Zero", "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine"], dtype=object
),
"xtext": Series(
["", "Zero", "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine"], dtype=object
),
"xnumberAsText": Series(["", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"], dtype=object),
"xtime": Series(
[
"NaT",
0,
"00:00:01.0010001",
"-00:00:02.0020002",
"00:00:03.0030003",
"-00:00:04.0040004",
"00:00:05.0050005",
"-00:00:06.0060006",
"00:00:07.0070007",
"-00:00:08.0080008",
"00:00:09.0090009",
],
dtype="timedelta64[ns]",
),
"xtextWithNulls": Series(["", "", "", "", "", "", "", "", "", "", ""], dtype=object),
"xdynamicWithNulls": Series(
[
None,
None,
{"rowId": 1, "arr": [0, 1]},
{"rowId": 2, "arr": [0, 2]},
{"rowId": 3, "arr": [0, 3]},
{"rowId": 4, "arr": [0, 4]},
{"rowId": 5, "arr": [0, 5]},
{"rowId": 6, "arr": [0, 6]},
{"rowId": 7, "arr": [0, 7]},
{"rowId": 8, "arr": [0, 8]},
{"rowId": 9, "arr": [0, 9]},
],
dtype=object,
),
}
columns = [
"rownumber",
"rowguid",
"xdouble",
"xfloat",
"xbool",
"xint16",
"xint32",
"xint64",
"xuint8",
"xuint16",
"xuint32",
"xuint64",
"xdate",
"xsmalltext",
"xtext",
"xnumberAsText",
"xtime",
"xtextWithNulls",
"xdynamicWithNulls",
]
expected_data_frame = DataFrame(expected_dict, columns=columns, copy=True)
assert_frame_equal(data_frame, expected_data_frame)
def test_sanity_query(self, mock_post):
"""Test query V2."""
client = KustoClient("https://somecluster.kusto.windows.net")
response = client.execute_query("PythonTest", "Deft")
expected = {
"rownumber": None,
"rowguid": text_type(""),
"xdouble": None,
"xfloat": None,
"xbool": None,
"xint16": None,
"xint32": None,
"xint64": None,
"xuint8": None,
"xuint16": None,
"xuint32": None,
"xuint64": None,
"xdate": None,
"xsmalltext": text_type(""),
"xtext": text_type(""),
"xnumberAsText": text_type(""),
"xtime": None,
"xtextWithNulls": text_type(""),
"xdynamicWithNulls": text_type(""),
}
for row in response.primary_results[0]:
self.assertEqual(row["rownumber"], expected["rownumber"])
self.assertEqual(row["rowguid"], expected["rowguid"])
self.assertEqual(row["xdouble"], expected["xdouble"])
self.assertEqual(row["xfloat"], expected["xfloat"])
self.assertEqual(row["xbool"], expected["xbool"])
self.assertEqual(row["xint16"], expected["xint16"])
self.assertEqual(row["xint32"], expected["xint32"])
self.assertEqual(row["xint64"], expected["xint64"])
self.assertEqual(row["xuint8"], expected["xuint8"])
self.assertEqual(row["xuint16"], expected["xuint16"])
self.assertEqual(row["xuint32"], expected["xuint32"])
self.assertEqual(row["xuint64"], expected["xuint64"])
self.assertEqual(row["xdate"], expected["xdate"])
self.assertEqual(row["xsmalltext"], expected["xsmalltext"])
self.assertEqual(row["xtext"], expected["xtext"])
self.assertEqual(row["xnumberAsText"], expected["xnumberAsText"])
self.assertEqual(row["xtime"], expected["xtime"])
self.assertEqual(row["xtextWithNulls"], expected["xtextWithNulls"])
self.assertEqual(row["xdynamicWithNulls"],
expected["xdynamicWithNulls"])
self.assertEqual(type(row["rownumber"]),
type(expected["rownumber"]))
self.assertEqual(type(row["rowguid"]), type(expected["rowguid"]))
self.assertEqual(type(row["xdouble"]), type(expected["xdouble"]))
self.assertEqual(type(row["xfloat"]), type(expected["xfloat"]))
self.assertEqual(type(row["xbool"]), type(expected["xbool"]))
self.assertEqual(type(row["xint16"]), type(expected["xint16"]))
self.assertEqual(type(row["xint32"]), type(expected["xint32"]))
self.assertEqual(type(row["xint64"]), type(expected["xint64"]))
self.assertEqual(type(row["xuint8"]), type(expected["xuint8"]))
self.assertEqual(type(row["xuint16"]), type(expected["xuint16"]))
self.assertEqual(type(row["xuint32"]), type(expected["xuint32"]))
self.assertEqual(type(row["xuint64"]), type(expected["xuint64"]))
self.assertEqual(type(row["xdate"]), type(expected["xdate"]))
self.assertEqual(type(row["xsmalltext"]),
type(expected["xsmalltext"]))
self.assertEqual(type(row["xtext"]), type(expected["xtext"]))
self.assertEqual(type(row["xnumberAsText"]),
type(expected["xnumberAsText"]))
self.assertEqual(type(row["xtime"]), type(expected["xtime"]))
self.assertEqual(type(row["xtextWithNulls"]),
type(expected["xtextWithNulls"]))
self.assertEqual(type(row["xdynamicWithNulls"]),
type(expected["xdynamicWithNulls"]))
expected["rownumber"] = 0 if expected[
"rownumber"] is None else expected["rownumber"] + 1
expected["rowguid"] = text_type(
"0000000{0}-0000-0000-0001-020304050607".format(
expected["rownumber"]))
expected["xdouble"] = round(
float(0) if expected["xdouble"] is None else
expected["xdouble"] + 1.0001, 4)
expected["xfloat"] = round(
float(0) if expected["xfloat"] is None else
expected["xfloat"] + 1.01, 2)
expected["xbool"] = False if expected[
"xbool"] is None else not expected["xbool"]
expected["xint16"] = 0 if expected[
"xint16"] is None else expected["xint16"] + 1
expected["xint32"] = 0 if expected[
"xint32"] is None else expected["xint32"] + 1
expected["xint64"] = 0 if expected[
"xint64"] is None else expected["xint64"] + 1
expected["xuint8"] = 0 if expected[
"xuint8"] is None else expected["xuint8"] + 1
expected["xuint16"] = 0 if expected[
"xuint16"] is None else expected["xuint16"] + 1
expected["xuint32"] = 0 if expected[
"xuint32"] is None else expected["xuint32"] + 1
expected["xuint64"] = 0 if expected[
"xuint64"] is None else expected["xuint64"] + 1
expected["xdate"] = expected["xdate"] or datetime(
2013, 1, 1, 1, 1, 1, 0, tzinfo=UTC)
expected["xdate"] = expected["xdate"].replace(
year=expected["xdate"].year + 1)
expected["xsmalltext"] = DIGIT_WORDS[int(expected["xint16"])]
expected["xtext"] = DIGIT_WORDS[int(expected["xint16"])]
expected["xnumberAsText"] = text_type(expected["xint16"])
next_time = (timedelta() if expected["xtime"] is None else
(abs(expected["xtime"]) +
timedelta(days=1, seconds=1, microseconds=1000)) *
(-1)**(expected["rownumber"] + 1))
# hacky tests - because time here is relative to previous row, after we pass a time where we have > 500 nanoseconds,
# another microseconds digit is needed
if expected["rownumber"] + 1 == 6:
next_time += timedelta(microseconds=1)
expected["xtime"] = next_time
if expected["xint16"] > 0:
expected["xdynamicWithNulls"] = {
"rowId": expected["xint16"],
"arr": [0, expected["xint16"]]
}
def test_empty_result(self, mock_post):
"""Tests dynamic responses."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = """print 'a' | take 0"""
response = client.execute_query("PythonTest", query)
self.assertTrue(response.primary_results[0])
print("2. Is semantic error:", error.is_semantic_error())
print("2. Has partial results:", error.has_partial_results())
print("2. Result size:", len(error.get_partial_results()))
RESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE,
KUSTO_QUERY,
accept_partial_results=True)
print("3. Response error count: ", RESPONSE.errors_count)
print("3. Exceptions:", RESPONSE.get_exceptions())
print("3. Result size:", len(RESPONSE.primary_results))
# Query has semantic error
KUSTO_QUERY = "StormEvents | where foo = bar"
try:
RESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)
except KustoServiceError as error:
print("4. Error:", error)
print("4. Is semantic error:", error.is_semantic_error())
print("4. Has partial results:", error.has_partial_results())
# Testing data frames
KUSTO_CLIENT = KustoClient("https://kustolab.kusto.windows.net")
RESPONSE = KUSTO_CLIENT.execute("ML", ".show version")
QUERY = """
let max_t = datetime(2016-09-03);
service_traffic
| make-series num=count() on TimeStamp in range(max_t-5d, max_t, 1h) by OsVer
"""
DATA_FRAME = KUSTO_CLIENT.execute_query(
"ML", QUERY).primary_results[0].to_dataframe()
