def createKustoConnection(uri, tenantID, appID, appKey): connectionString = None try: connectionString = KustoConnectionStringBuilder.with_aad_application_key_authentication(uri, appID, appKey, tenantID) except Exception as e: logging.error("Could not create a connection string:%s"%e) return connectionString
def __init__( self, kusto_cluster, client_id=None, client_secret=None, username=None, password=None, certificate=None, certificate_thumbprint=None, authority=None, ): """ Kusto Client constructor. Parameters ---------- kusto_cluster : str Kusto cluster endpoint. Example: https://help.kusto.windows.net client_id : str The AAD application ID of the application making the request to Kusto client_secret : str The AAD application key of the application making the request to Kusto. if this is given, then username/password should not be. username : str The username of the user making the request to Kusto. if this is given, then password must follow and the client_secret should not be given. password : str The password matching the username of the user making the request to Kusto authority : 'microsoft.com', optional In case your tenant is not microsoft please use this param. """ if all([username, password]): kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication( kusto_cluster, username, password) elif all([client_id, client_secret]): kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( kusto_cluster, client_id, client_secret) elif all([client_id, certificate, certificate_thumbprint]): kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication( kusto_cluster, client_id, certificate, certificate_thumbprint) else: kcsb = KustoConnectionStringBuilder.with_aad_device_authentication( kusto_cluster) if authority: kcsb.authority_id = authority self.client = KustoClient(kcsb) # replace aadhelper to use remote browser in interactive mode self.client._aad_helper = _MyAadHelper(kcsb) self.mgmt_endpoint_version = "v2" if self.client._mgmt_endpoint.endswith( "v2/rest/query") else "v1" self.query_endpoint_version = "v2" if self.client._query_endpoint.endswith( "v2/rest/query") else "v1"
def get_conn(self) -> KustoClient: """Return a KustoClient object.""" conn = self.get_connection(self.conn_id) cluster = conn.host if not cluster: raise AirflowException('Host connection option is required') def get_required_param(name: str) -> str: """Extract required parameter from extra JSON, raise exception if not found""" value = conn.extra_dejson.get(name) if not value: raise AirflowException( f'Extra connection option is missing required parameter: `{name}`' ) return value auth_method = get_required_param('auth_method') or get_required_param( 'extra__azure_data_explorer__auth_method') if auth_method == 'AAD_APP': tenant = get_required_param('tenant') or get_required_param( 'extra__azure_data_explorer__tenant') kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, conn.login, conn.password, tenant) elif auth_method == 'AAD_APP_CERT': certificate = get_required_param( 'certificate') or get_required_param( 'extra__azure_data_explorer__certificate') thumbprint = get_required_param( 'thumbprint') or get_required_param( 'extra__azure_data_explorer__thumbprint') tenant = get_required_param('tenant') or get_required_param( 'extra__azure_data_explorer__tenant') kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication( cluster, conn.login, certificate, thumbprint, tenant, ) elif auth_method == 'AAD_CREDS': tenant = get_required_param('tenant') or get_required_param( 'extra__azure_data_explorer__tenant') kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication( cluster, conn.login, conn.password, tenant) elif auth_method == 'AAD_DEVICE': kcsb = KustoConnectionStringBuilder.with_aad_device_authentication( cluster) else: raise AirflowException( f'Unknown authentication method: {auth_method}') return KustoClient(kcsb)
def __init__(self, trainingConfig, trainingId): self.trainingId = trainingId cluster = "https://usage360.kusto.windows.net" authority_id = "72f988bf-86f1-41af-91ab-2d7cd011db47" client_id = kustoClientId client_secret = kustoClientSecret kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id) self.kustoClient = KustoClient(kcsb) self.garbageList = [x.strip() for x in open("metadata/garbagePhrases.txt", "r").readlines()] self.striptrailers = [x.strip() for x in open("metadata/stripTrailers.txt", "r").readlines()] self.shortPhrases = [x.strip() for x in open("metadata/shortPhrasesList.txt", "r").readlines()] self.trainingConfig = trainingConfig
def run_query(self, query, user): kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( connection_string=self.configuration["cluster"], aad_app_id=self.configuration["azure_ad_client_id"], app_key=self.configuration["azure_ad_client_secret"], authority_id=self.configuration["azure_ad_tenant_id"], ) client = KustoClient(kcsb) db = self.configuration["database"] try: response = client.execute(db, query) result_cols = response.primary_results[0].columns result_rows = response.primary_results[0].rows columns = [] rows = [] for c in result_cols: columns.append( { "name": c.column_name, "friendly_name": c.column_name, "type": TYPES_MAP.get(c.column_type, None), } ) # rows must be [{'column1': value, 'column2': value}] for row in result_rows: rows.append(row.to_dict()) error = None data = {"columns": columns, "rows": rows} json_data = json_dumps(data) except KustoServiceError as err: json_data = None try: error = err.args[1][0]["error"]["@message"] except (IndexError, KeyError): error = err.args[1] except KeyboardInterrupt: json_data = None error = "Query cancelled by user." return json_data, error
def __init__(self, conn_kv): """ Kusto Client constructor. Parameters ---------- kusto_cluster : str Kusto cluster endpoint. Example: https://help.kusto.windows.net client_id : str The AAD application ID of the application making the request to Kusto client_secret : str The AAD application key of the application making the request to Kusto. if this is given, then username/password should not be. username : str The username of the user making the request to Kusto. if this is given, then password must follow and the client_secret should not be given. password : str The password matching the username of the user making the request to Kusto authority : 'microsoft.com', optional In case your tenant is not microsoft please use this param. """ kusto_cluster = "https://{0}.kusto.windows.net".format(conn_kv["cluster"]) if all([conn_kv.get("username"), conn_kv.get("password")]): kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(kusto_cluster, conn_kv.get("username"), conn_kv.get("password")) if conn_kv.get("tenant") is not None: kcsb.authority_id = conn_kv.get("tenant") elif all([conn_kv.get("clientid"), conn_kv.get("clientsecret")]): kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( kusto_cluster, conn_kv.get("clientid"), conn_kv.get("clientsecret"), conn_kv.get("tenant")) elif all([conn_kv.get("clientid"), conn_kv.get("certificate"), conn_kv.get("certificate_thumbprint")]): kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication( kusto_cluster, conn_kv.get("clientid"), conn_kv.get("certificate"), conn_kv.get("certificate_thumbprint", conn_kv.get("tenant")) ) else: kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(kusto_cluster) if conn_kv.get("tenant") is not None: kcsb.authority_id = conn_kv.get("tenant") self.client = KustoClient(kcsb) # replace aadhelper to use remote browser in interactive mode self.client._aad_helper = _MyAadHelper(kcsb, self._DEFAULT_CLIENTID) self.mgmt_endpoint_version = "v2" if self.client._mgmt_endpoint.endswith("v2/rest/query") else "v1" self.query_endpoint_version = "v2" if self.client._query_endpoint.endswith("v2/rest/query") else "v1"
def get_conn(self) -> KustoClient: """Return a KustoClient object.""" conn = self.get_connection(self.conn_id) cluster = conn.host if not cluster: raise AirflowException('Host connection option is required') def get_required_param(name): """Extract required parameter from extra JSON, raise exception if not found""" value = conn.extra_dejson.get(name) if not value: raise AirflowException( 'Extra connection option is missing required parameter: `{}`' .format(name)) return value auth_method = get_required_param('auth_method') if auth_method == 'AAD_APP': kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, conn.login, conn.password, get_required_param('tenant')) elif auth_method == 'AAD_APP_CERT': kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication( cluster, conn.login, get_required_param('certificate'), get_required_param('thumbprint'), get_required_param('tenant'), ) elif auth_method == 'AAD_CREDS': kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication( cluster, conn.login, conn.password, get_required_param('tenant')) elif auth_method == 'AAD_DEVICE': kcsb = KustoConnectionStringBuilder.with_aad_device_authentication( cluster) else: raise AirflowException( 'Unknown authentication method: {}'.format(auth_method)) return KustoClient(kcsb)
def kusto_output(): try: kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( SOCAlertConsumer.cluster, SOCAlertConsumer.client_id, SOCAlertConsumer.client_secret, SOCAlertConsumer.authority_id) logger.AppLogging.auditlogger.info( "Successfully loaded Kusto Connection Strings" + " " + SOCAlertConsumer.cluster + " with AppID " + SOCAlertConsumer.client_id) except RuntimeError as ie: logger.AppLogging.auditlogger.error( "Unable to import Kusto Connection Strings. " "Please check your configuration" + SOCAlertConsumer.cluster + "with AppID" + SOCAlertConsumer.client_id) logger.AppLogging.auditlogger.error(str(ie)) client = KustoClient(kcsb) kusto_query = open(".//KustoQuery//SOCAlerts.csl", "r") # change to forward slash for linux (//) query = kusto_query.read() kusto_query.close() try: response = client.execute(SOCAlertConsumer.db, query) logger.AppLogging.auditlogger.info( "Successfully received response from Kusto") logger.AppLogging.auditlogger.info( "Query output is saved to output folder") except RuntimeError as r: logger.AppLogging.auditlogger.error( "There was an error in receiving response from Kusto") logger.AppLogging.auditlogger.error(str(r)) for row in response.primary_results: filetime = time.strftime("%Y%m%d-%H%M%S") file = open(".//output//alerts-" + filetime + ".json", "x") # forward slash works fine on Win & nix file.write(str(row)) file.close()
def test_aad_app(self): """Checks kcsb that is created with AAD application credentials.""" uuid = str(uuid4()) key = "key of application" kcsbs = [ KustoConnectionStringBuilder( "localhost;Application client Id={0};application Key={1};Authority Id={2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder( "Data Source=localhost ; Application Client Id={0}; Appkey ={1};Authority Id= {2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder( " Addr = localhost ; AppClientId = {0} ; AppKey ={1}; Authority Id={2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder( "Network Address = localhost; AppClientId = {0} ; AppKey ={1};AuthorityId={2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder. with_aad_application_key_authentication("localhost", uuid, key, "microsoft.com"), ] try: KustoConnectionStringBuilder.with_aad_application_key_authentication( "localhost", uuid, key, None) except Exception as e: # make sure error is raised when authoriy_id i none assert isinstance(e, ValueError) == True kcsb1 = KustoConnectionStringBuilder("server=localhost") kcsb1[KustoConnectionStringBuilder.ValidKeywords. application_client_id] = uuid kcsb1[KustoConnectionStringBuilder.ValidKeywords.application_key] = key kcsb1[KustoConnectionStringBuilder.ValidKeywords. authority_id] = "microsoft.com" kcsb1[KustoConnectionStringBuilder.ValidKeywords. aad_federated_security] = True kcsbs.append(kcsb1) kcsb2 = KustoConnectionStringBuilder("Server=localhost") kcsb2["AppclientId"] = uuid kcsb2["Application key"] = key kcsb2["Authority Id"] = "microsoft.com" kcsb2["aad federated security"] = True kcsbs.append(kcsb2) for kcsb in kcsbs: self.assertEqual(kcsb.data_source, "localhost") self.assertTrue(kcsb.aad_federated_security) self.assertIsNone(kcsb.aad_user_id) self.assertIsNone(kcsb.password) self.assertEqual(kcsb.application_client_id, uuid) self.assertEqual(kcsb.application_key, key) self.assertEqual(kcsb.authority_id, "microsoft.com") self.assertEqual( repr(kcsb), "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}" .format(uuid, key, "microsoft.com"), ) self.assertEqual( str(kcsb), "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}" .format(uuid, self.PASSWORDS_REPLACEMENT, "microsoft.com"), )
) ################################################################## ## AUTH ## ################################################################## cluster = "https://ingest-{cluster_name}.kusto.windows.net" # In case you want to authenticate with AAD application. client_id = "<insert here your AAD application id>" client_secret = "<insert here your AAD application key>" # read more at https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id authority_id = "<insert here your tenant id>" kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, client_id, client_secret, authority_id ) # In case you want to authenticate with AAD application certificate. filename = "path to a PEM certificate" with open(filename, "r") as pem_file: PEM = pem_file.read() thumbprint = "certificate's thumbprint" kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication( cluster, client_id, PEM, thumbprint, authority_id ) # In case you want to authenticate with AAD username and password username = "******" password = "******"
kusto_ingest_uri = "https://ingest-{}.{}.kusto.windows.net".format( cluster_name, cluster_region) aad_tenant_id = adx_cfg_json.get('AAD_TENANT_ID') app_client_secret = adx_cfg_json.get('APP_CLIENT_SECRET') app_client_id = adx_cfg_json.get('APP_CLIENT_ID') container = adx_cfg_json.get('BLOB_CONTAINER_NAME') account_name = adx_cfg_json.get('STORAGE_ACCOUNT_NAME') account_key = adx_cfg_json.get('STORAGE_ACCOUNT_KEY') sas_token = adx_cfg_json.get('SAS_TOKEN') # kcsb_ingest = KustoConnectionStringBuilder.with_aad_device_authentication(kusto_ingest_uri, aad_tenant_id) # kcsb_data = KustoConnectionStringBuilder.with_aad_device_authentication(kusto_uri, aad_tenant_id) kcsb_ingest = KustoConnectionStringBuilder.with_aad_application_key_authentication( kusto_ingest_uri, app_client_id, app_client_secret, aad_tenant_id) kcsb_data = KustoConnectionStringBuilder.with_aad_application_key_authentication( kusto_uri, app_client_id, app_client_secret, aad_tenant_id) kusto_client = KustoClient(kcsb_data) column_mapping = col_map_as_json_dict.get('OBJECT_CONFIG') blob_service = BlockBlobService(account_name, account_key) # Function to find latest blob from all available # Assumes format of <source> + '/' + <YYYYMMDDHHmm> + '_' + <object name> blob_names_generator = blob_service.list_blobs(container) # for blob in blob_names_generator: # length = BlockBlobService.get_blob_properties(blob_service, container, blob.name).properties.content_length
from azure.kusto.ingest import ( KustoIngestClient, IngestionProperties, FileDescriptor, BlobDescriptor, DataFormat, ) INGESTION_PROPERTIES = IngestionProperties(database="database name", table="table name", dataFormat=DataFormat.csv) INGEST_CLIENT = KustoIngestClient( "https://ingest-<clustername>.kusto.windows.net") KCSB = KustoConnectionStringBuilder.with_aad_application_key_authentication( "https://ingest-<clustername>.kusto.windows.net", "aad app id", "secret") INGEST_CLIENT = KustoIngestClient(KCSB) FILE_DESCRIPTOR = FileDescriptor( "E:\\filePath.csv", 3333) # 3333 is the raw size of the data in bytes. INGEST_CLIENT.ingest_from_multiple_files( [FILE_DESCRIPTOR], delete_sources_on_success=True, ingestion_properties=INGESTION_PROPERTIES) INGEST_CLIENT.ingest_from_multiple_files( ["E:\\filePath.csv"], delete_sources_on_success=True, ingestion_properties=INGESTION_PROPERTIES) BLOB_DESCRIPTOR = BlobDescriptor(
"""A simple example how to use KustoClient.""" from azure.kusto.data.request import KustoClient, KustoConnectionStringBuilder from azure.kusto.data.exceptions import KustoServiceError # TODO: this should become functional test at some point. KUSTO_CLUSTER = "https://help.kusto.windows.net" # In case you want to authenticate with AAD application. CLIENT_ID = "<insert here your AAD application id>" CLIENT_SECRET = "<insert here your AAD application key>" KCSB = KustoConnectionStringBuilder.with_aad_application_key_authentication( KUSTO_CLUSTER, CLIENT_ID, CLIENT_SECRET) # In case you want to authenticate with AAD application certificate. FILENAME = "path to a PEM certificate" with open(FILENAME, "r") as pem_file: PEM = pem_file.read() THUMBPRINT = "certificate's thumbprint" KCSB = KustoConnectionStringBuilder.with_aad_application_certificate_authentication( KUSTO_CLUSTER, CLIENT_ID, PEM, THUMBPRINT) KUSTO_CLIENT = KustoClient(KCSB) # In case you want to authenticate with the logged in AAD user. KUSTO_CLIENT = KustoClient(KUSTO_CLUSTER) KUSTO_DATABASE = "Samples" KUSTO_QUERY = "StormEvents | take 10"
' { "column" : "xdynamicWithNulls", "datatype" : "dynamic", "Properties":{"Path":"$.xdynamicWithNulls"}},' ']'""" # Get environment variables engine_cs = os.environ.get("ENGINE_CONECTION_STRING") dm_cs = os.environ.get("DM_CONECTION_STRING") db_name = os.environ.get( "TEST_DATABASE") # Existed db with streaming ingestion enabled app_id = os.environ.get("APP_ID") app_key = os.environ.get("APP_KEY") tenant_id = os.environ.get("TENANT_ID") # Init clients table_name = "python_test" engine_kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( engine_cs, app_id, app_key, tenant_id) dm_kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( dm_cs, app_id, app_key, tenant_id) client = KustoClient(engine_kcsb) ingest_client = KustoIngestClient(dm_kcsb) ingest_status_q = KustoIngestStatusQueues(ingest_client) streaming_ingest_client = KustoStreamingIngestClient(engine_kcsb) # Clean previous test client.execute(db_name, ".drop table {} ifexists".format(table_name)) while not ingest_status_q.success.is_empty(): ingest_status_q.success.pop() # Get files paths current_dir = os.getcwd() path_parts = ["azure-kusto-ingest", "tests", "input"]