def createKustoConnection(uri, tenantID, appID, appKey):
connectionString = None
try:
connectionString = KustoConnectionStringBuilder.with_aad_application_key_authentication(uri, appID, appKey, tenantID)
except Exception as e:
logging.error("Could not create a connection string:%s"%e)
return connectionString
def __init__(
self,
kusto_cluster,
client_id=None,
client_secret=None,
username=None,
password=None,
certificate=None,
certificate_thumbprint=None,
authority=None,
):
"""
Kusto Client constructor.
Parameters
----------
kusto_cluster : str
Kusto cluster endpoint. Example: https://help.kusto.windows.net
client_id : str
The AAD application ID of the application making the request to Kusto
client_secret : str
The AAD application key of the application making the request to Kusto.
if this is given, then username/password should not be.
username : str
The username of the user making the request to Kusto.
if this is given, then password must follow and the client_secret should not be given.
password : str
The password matching the username of the user making the request to Kusto
authority : 'microsoft.com', optional
In case your tenant is not microsoft please use this param.
"""
if all([username, password]):
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(
kusto_cluster, username, password)
elif all([client_id, client_secret]):
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
kusto_cluster, client_id, client_secret)
elif all([client_id, certificate, certificate_thumbprint]):
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
kusto_cluster, client_id, certificate, certificate_thumbprint)
else:
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(
kusto_cluster)
if authority:
kcsb.authority_id = authority
self.client = KustoClient(kcsb)
# replace aadhelper to use remote browser in interactive mode
self.client._aad_helper = _MyAadHelper(kcsb)
self.mgmt_endpoint_version = "v2" if self.client._mgmt_endpoint.endswith(
"v2/rest/query") else "v1"
self.query_endpoint_version = "v2" if self.client._query_endpoint.endswith(
"v2/rest/query") else "v1"
def get_conn(self) -> KustoClient:
"""Return a KustoClient object."""
conn = self.get_connection(self.conn_id)
cluster = conn.host
if not cluster:
raise AirflowException('Host connection option is required')
def get_required_param(name: str) -> str:
"""Extract required parameter from extra JSON, raise exception if not found"""
value = conn.extra_dejson.get(name)
if not value:
raise AirflowException(
f'Extra connection option is missing required parameter: `{name}`'
)
return value
auth_method = get_required_param('auth_method') or get_required_param(
'extra__azure_data_explorer__auth_method')
if auth_method == 'AAD_APP':
tenant = get_required_param('tenant') or get_required_param(
'extra__azure_data_explorer__tenant')
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, conn.login, conn.password, tenant)
elif auth_method == 'AAD_APP_CERT':
certificate = get_required_param(
'certificate') or get_required_param(
'extra__azure_data_explorer__certificate')
thumbprint = get_required_param(
'thumbprint') or get_required_param(
'extra__azure_data_explorer__thumbprint')
tenant = get_required_param('tenant') or get_required_param(
'extra__azure_data_explorer__tenant')
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
cluster,
conn.login,
certificate,
thumbprint,
tenant,
)
elif auth_method == 'AAD_CREDS':
tenant = get_required_param('tenant') or get_required_param(
'extra__azure_data_explorer__tenant')
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(
cluster, conn.login, conn.password, tenant)
elif auth_method == 'AAD_DEVICE':
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(
cluster)
else:
raise AirflowException(
f'Unknown authentication method: {auth_method}')
return KustoClient(kcsb)
def __init__(self, trainingConfig, trainingId):
self.trainingId = trainingId
cluster = "https://usage360.kusto.windows.net"
authority_id = "72f988bf-86f1-41af-91ab-2d7cd011db47"
client_id = kustoClientId
client_secret = kustoClientSecret
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id)
self.kustoClient = KustoClient(kcsb)
self.garbageList = [x.strip() for x in open("metadata/garbagePhrases.txt", "r").readlines()]
self.striptrailers = [x.strip() for x in open("metadata/stripTrailers.txt", "r").readlines()]
self.shortPhrases = [x.strip() for x in open("metadata/shortPhrasesList.txt", "r").readlines()]
self.trainingConfig = trainingConfig
def run_query(self, query, user):
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
connection_string=self.configuration["cluster"],
aad_app_id=self.configuration["azure_ad_client_id"],
app_key=self.configuration["azure_ad_client_secret"],
authority_id=self.configuration["azure_ad_tenant_id"],
)
client = KustoClient(kcsb)
db = self.configuration["database"]
try:
response = client.execute(db, query)
result_cols = response.primary_results[0].columns
result_rows = response.primary_results[0].rows
columns = []
rows = []
for c in result_cols:
columns.append(
{
"name": c.column_name,
"friendly_name": c.column_name,
"type": TYPES_MAP.get(c.column_type, None),
}
)
# rows must be [{'column1': value, 'column2': value}]
for row in result_rows:
rows.append(row.to_dict())
error = None
data = {"columns": columns, "rows": rows}
json_data = json_dumps(data)
except KustoServiceError as err:
json_data = None
try:
error = err.args[1][0]["error"]["@message"]
except (IndexError, KeyError):
error = err.args[1]
except KeyboardInterrupt:
json_data = None
error = "Query cancelled by user."
return json_data, error
def __init__(self, conn_kv):
"""
Kusto Client constructor.
Parameters
----------
kusto_cluster : str
Kusto cluster endpoint. Example: https://help.kusto.windows.net
client_id : str
The AAD application ID of the application making the request to Kusto
client_secret : str
The AAD application key of the application making the request to Kusto.
if this is given, then username/password should not be.
username : str
The username of the user making the request to Kusto.
if this is given, then password must follow and the client_secret should not be given.
password : str
The password matching the username of the user making the request to Kusto
authority : 'microsoft.com', optional
In case your tenant is not microsoft please use this param.
"""
kusto_cluster = "https://{0}.kusto.windows.net".format(conn_kv["cluster"])
if all([conn_kv.get("username"), conn_kv.get("password")]):
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(kusto_cluster, conn_kv.get("username"), conn_kv.get("password"))
if conn_kv.get("tenant") is not None: kcsb.authority_id = conn_kv.get("tenant")
elif all([conn_kv.get("clientid"), conn_kv.get("clientsecret")]):
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
kusto_cluster, conn_kv.get("clientid"), conn_kv.get("clientsecret"), conn_kv.get("tenant"))
elif all([conn_kv.get("clientid"), conn_kv.get("certificate"), conn_kv.get("certificate_thumbprint")]):
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
kusto_cluster, conn_kv.get("clientid"), conn_kv.get("certificate"), conn_kv.get("certificate_thumbprint", conn_kv.get("tenant"))
)
else:
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(kusto_cluster)
if conn_kv.get("tenant") is not None: kcsb.authority_id = conn_kv.get("tenant")
self.client = KustoClient(kcsb)
# replace aadhelper to use remote browser in interactive mode
self.client._aad_helper = _MyAadHelper(kcsb, self._DEFAULT_CLIENTID)
self.mgmt_endpoint_version = "v2" if self.client._mgmt_endpoint.endswith("v2/rest/query") else "v1"
self.query_endpoint_version = "v2" if self.client._query_endpoint.endswith("v2/rest/query") else "v1"
def get_conn(self) -> KustoClient:
"""Return a KustoClient object."""
conn = self.get_connection(self.conn_id)
cluster = conn.host
if not cluster:
raise AirflowException('Host connection option is required')
def get_required_param(name):
"""Extract required parameter from extra JSON, raise exception if not found"""
value = conn.extra_dejson.get(name)
if not value:
raise AirflowException(
'Extra connection option is missing required parameter: `{}`'
.format(name))
return value
auth_method = get_required_param('auth_method')
if auth_method == 'AAD_APP':
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, conn.login, conn.password,
get_required_param('tenant'))
elif auth_method == 'AAD_APP_CERT':
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
cluster,
conn.login,
get_required_param('certificate'),
get_required_param('thumbprint'),
get_required_param('tenant'),
)
elif auth_method == 'AAD_CREDS':
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(
cluster, conn.login, conn.password,
get_required_param('tenant'))
elif auth_method == 'AAD_DEVICE':
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(
cluster)
else:
raise AirflowException(
'Unknown authentication method: {}'.format(auth_method))
return KustoClient(kcsb)
def kusto_output():
try:
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
SOCAlertConsumer.cluster, SOCAlertConsumer.client_id,
SOCAlertConsumer.client_secret, SOCAlertConsumer.authority_id)
logger.AppLogging.auditlogger.info(
"Successfully loaded Kusto Connection Strings" + " " +
SOCAlertConsumer.cluster + " with AppID " +
SOCAlertConsumer.client_id)
except RuntimeError as ie:
logger.AppLogging.auditlogger.error(
"Unable to import Kusto Connection Strings. "
"Please check your configuration" + SOCAlertConsumer.cluster +
"with AppID" + SOCAlertConsumer.client_id)
logger.AppLogging.auditlogger.error(str(ie))
client = KustoClient(kcsb)
kusto_query = open(".//KustoQuery//SOCAlerts.csl",
"r") # change to forward slash for linux (//)
query = kusto_query.read()
kusto_query.close()
try:
response = client.execute(SOCAlertConsumer.db, query)
logger.AppLogging.auditlogger.info(
"Successfully received response from Kusto")
logger.AppLogging.auditlogger.info(
"Query output is saved to output folder")
except RuntimeError as r:
logger.AppLogging.auditlogger.error(
"There was an error in receiving response from Kusto")
logger.AppLogging.auditlogger.error(str(r))
for row in response.primary_results:
filetime = time.strftime("%Y%m%d-%H%M%S")
file = open(".//output//alerts-" + filetime + ".json",
"x") # forward slash works fine on Win & nix
file.write(str(row))
file.close()
def test_aad_app(self):
"""Checks kcsb that is created with AAD application credentials."""
uuid = str(uuid4())
key = "key of application"
kcsbs = [
KustoConnectionStringBuilder(
"localhost;Application client Id={0};application Key={1};Authority Id={2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder(
"Data Source=localhost ; Application Client Id={0}; Appkey ={1};Authority Id= {2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder(
" Addr = localhost ; AppClientId = {0} ; AppKey ={1}; Authority Id={2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder(
"Network Address = localhost; AppClientId = {0} ; AppKey ={1};AuthorityId={2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder.
with_aad_application_key_authentication("localhost", uuid, key,
"microsoft.com"),
]
try:
KustoConnectionStringBuilder.with_aad_application_key_authentication(
"localhost", uuid, key, None)
except Exception as e:
# make sure error is raised when authoriy_id i none
assert isinstance(e, ValueError) == True
kcsb1 = KustoConnectionStringBuilder("server=localhost")
kcsb1[KustoConnectionStringBuilder.ValidKeywords.
application_client_id] = uuid
kcsb1[KustoConnectionStringBuilder.ValidKeywords.application_key] = key
kcsb1[KustoConnectionStringBuilder.ValidKeywords.
authority_id] = "microsoft.com"
kcsb1[KustoConnectionStringBuilder.ValidKeywords.
aad_federated_security] = True
kcsbs.append(kcsb1)
kcsb2 = KustoConnectionStringBuilder("Server=localhost")
kcsb2["AppclientId"] = uuid
kcsb2["Application key"] = key
kcsb2["Authority Id"] = "microsoft.com"
kcsb2["aad federated security"] = True
kcsbs.append(kcsb2)
for kcsb in kcsbs:
self.assertEqual(kcsb.data_source, "localhost")
self.assertTrue(kcsb.aad_federated_security)
self.assertIsNone(kcsb.aad_user_id)
self.assertIsNone(kcsb.password)
self.assertEqual(kcsb.application_client_id, uuid)
self.assertEqual(kcsb.application_key, key)
self.assertEqual(kcsb.authority_id, "microsoft.com")
self.assertEqual(
repr(kcsb),
"Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}"
.format(uuid, key, "microsoft.com"),
)
self.assertEqual(
str(kcsb),
"Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}"
.format(uuid, self.PASSWORDS_REPLACEMENT, "microsoft.com"),
)
)
##################################################################
## AUTH ##
##################################################################
cluster = "https://ingest-{cluster_name}.kusto.windows.net"
# In case you want to authenticate with AAD application.
client_id = "<insert here your AAD application id>"
client_secret = "<insert here your AAD application key>"
# read more at https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id
authority_id = "<insert here your tenant id>"
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, client_id, client_secret, authority_id
)
# In case you want to authenticate with AAD application certificate.
filename = "path to a PEM certificate"
with open(filename, "r") as pem_file:
PEM = pem_file.read()
thumbprint = "certificate's thumbprint"
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
cluster, client_id, PEM, thumbprint, authority_id
)
# In case you want to authenticate with AAD username and password
username = "******"
password = "******"
kusto_ingest_uri = "https://ingest-{}.{}.kusto.windows.net".format(
cluster_name, cluster_region)
aad_tenant_id = adx_cfg_json.get('AAD_TENANT_ID')
app_client_secret = adx_cfg_json.get('APP_CLIENT_SECRET')
app_client_id = adx_cfg_json.get('APP_CLIENT_ID')
container = adx_cfg_json.get('BLOB_CONTAINER_NAME')
account_name = adx_cfg_json.get('STORAGE_ACCOUNT_NAME')
account_key = adx_cfg_json.get('STORAGE_ACCOUNT_KEY')
sas_token = adx_cfg_json.get('SAS_TOKEN')
# kcsb_ingest = KustoConnectionStringBuilder.with_aad_device_authentication(kusto_ingest_uri, aad_tenant_id)
# kcsb_data = KustoConnectionStringBuilder.with_aad_device_authentication(kusto_uri, aad_tenant_id)
kcsb_ingest = KustoConnectionStringBuilder.with_aad_application_key_authentication(
kusto_ingest_uri, app_client_id, app_client_secret, aad_tenant_id)
kcsb_data = KustoConnectionStringBuilder.with_aad_application_key_authentication(
kusto_uri, app_client_id, app_client_secret, aad_tenant_id)
kusto_client = KustoClient(kcsb_data)
column_mapping = col_map_as_json_dict.get('OBJECT_CONFIG')
blob_service = BlockBlobService(account_name, account_key)
# Function to find latest blob from all available
# Assumes format of <source> + '/' + <YYYYMMDDHHmm> + '_' + <object name>
blob_names_generator = blob_service.list_blobs(container)
# for blob in blob_names_generator:
# length = BlockBlobService.get_blob_properties(blob_service, container, blob.name).properties.content_length
from azure.kusto.ingest import (
KustoIngestClient,
IngestionProperties,
FileDescriptor,
BlobDescriptor,
DataFormat,
)
INGESTION_PROPERTIES = IngestionProperties(database="database name",
table="table name",
dataFormat=DataFormat.csv)
INGEST_CLIENT = KustoIngestClient(
"https://ingest-<clustername>.kusto.windows.net")
KCSB = KustoConnectionStringBuilder.with_aad_application_key_authentication(
"https://ingest-<clustername>.kusto.windows.net", "aad app id", "secret")
INGEST_CLIENT = KustoIngestClient(KCSB)
FILE_DESCRIPTOR = FileDescriptor(
"E:\\filePath.csv", 3333) # 3333 is the raw size of the data in bytes.
INGEST_CLIENT.ingest_from_multiple_files(
[FILE_DESCRIPTOR],
delete_sources_on_success=True,
ingestion_properties=INGESTION_PROPERTIES)
INGEST_CLIENT.ingest_from_multiple_files(
["E:\\filePath.csv"],
delete_sources_on_success=True,
ingestion_properties=INGESTION_PROPERTIES)
BLOB_DESCRIPTOR = BlobDescriptor(
"""A simple example how to use KustoClient."""
from azure.kusto.data.request import KustoClient, KustoConnectionStringBuilder
from azure.kusto.data.exceptions import KustoServiceError
# TODO: this should become functional test at some point.
KUSTO_CLUSTER = "https://help.kusto.windows.net"
# In case you want to authenticate with AAD application.
CLIENT_ID = "<insert here your AAD application id>"
CLIENT_SECRET = "<insert here your AAD application key>"
KCSB = KustoConnectionStringBuilder.with_aad_application_key_authentication(
KUSTO_CLUSTER, CLIENT_ID, CLIENT_SECRET)
# In case you want to authenticate with AAD application certificate.
FILENAME = "path to a PEM certificate"
with open(FILENAME, "r") as pem_file:
PEM = pem_file.read()
THUMBPRINT = "certificate's thumbprint"
KCSB = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
KUSTO_CLUSTER, CLIENT_ID, PEM, THUMBPRINT)
KUSTO_CLIENT = KustoClient(KCSB)
# In case you want to authenticate with the logged in AAD user.
KUSTO_CLIENT = KustoClient(KUSTO_CLUSTER)
KUSTO_DATABASE = "Samples"
KUSTO_QUERY = "StormEvents | take 10"
' { "column" : "xdynamicWithNulls", "datatype" : "dynamic", "Properties":{"Path":"$.xdynamicWithNulls"}},'
']'"""
# Get environment variables
engine_cs = os.environ.get("ENGINE_CONECTION_STRING")
dm_cs = os.environ.get("DM_CONECTION_STRING")
db_name = os.environ.get(
"TEST_DATABASE") # Existed db with streaming ingestion enabled
app_id = os.environ.get("APP_ID")
app_key = os.environ.get("APP_KEY")
tenant_id = os.environ.get("TENANT_ID")
# Init clients
table_name = "python_test"
engine_kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
engine_cs, app_id, app_key, tenant_id)
dm_kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
dm_cs, app_id, app_key, tenant_id)
client = KustoClient(engine_kcsb)
ingest_client = KustoIngestClient(dm_kcsb)
ingest_status_q = KustoIngestStatusQueues(ingest_client)
streaming_ingest_client = KustoStreamingIngestClient(engine_kcsb)
# Clean previous test
client.execute(db_name, ".drop table {} ifexists".format(table_name))
while not ingest_status_q.success.is_empty():
ingest_status_q.success.pop()
# Get files paths
current_dir = os.getcwd()
path_parts = ["azure-kusto-ingest", "tests", "input"]
