def engine_kcsb_from_env() -> KustoConnectionStringBuilder: engine_cs = os.environ.get("ENGINE_CONNECTION_STRING") app_id = os.environ.get("APP_ID") app_key = os.environ.get("APP_KEY") auth_id = os.environ.get("AUTH_ID") return KustoConnectionStringBuilder.with_aad_application_key_authentication( engine_cs, app_id, app_key, auth_id)
def authenticate_to_kusto_ingress(cluster): """Authenticate and return kusto connection client""" kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, CLIENT_ID, CLIENT_SECRET, AUTHORITY_ID) # The authentication method will be taken from the chosen KustoConnectionStringBuilder. kusto_client = KustoIngestClient(kcsb) return kusto_client
def dm_kcsb_from_env() -> KustoConnectionStringBuilder: engine_cs = os.environ.get("ENGINE_CONNECTION_STRING") dm_cs = os.environ.get("DM_CONNECTION_STRING") or engine_cs.replace( "//", "//ingest-") app_id = os.environ.get("APP_ID") app_key = os.environ.get("APP_KEY") auth_id = os.environ.get("AUTH_ID") return KustoConnectionStringBuilder.with_aad_application_key_authentication( dm_cs, app_id, app_key, auth_id)
def initialize_kusto_client(): """initialize kusto client """ global KUSTO_INGESTION_CLIENT if not KUSTO_INGESTION_CLIENT: kcsb_ingest = KustoConnectionStringBuilder.with_aad_application_key_authentication( \ INGESTION_SERVER_URI, APP_CLIENT_ID, APP_CLIENT_SECRETS, APP_AAD_TENANT_ID) KUSTO_INGESTION_CLIENT = KustoIngestClient(kcsb_ingest) logging.info(f"{LOG_MESSAGE_HEADER} Build KUSTO_INGESTION_CLIENT") else: logging.info(f"{LOG_MESSAGE_HEADER} KUSTO_INGESTION_CLIENT exist")
def __init__(self, db_name: str): """Initialize a Kusto report DB connector. Args: db_name: The Kusto database to connect to. """ self.db_name = db_name ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER") tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID") service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID") service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY") if not ingest_cluster or not tenant_id or not service_id or not service_key: raise RuntimeError("Could not load Kusto Credentials from environment") kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(ingest_cluster, service_id, service_key, tenant_id) self._ingestion_client = KustoIngestClient(kcsb) """ Kusto performance depends on the work load of cluster, to improve the high availability of test result data service by hosting a backup cluster, which is optional. """ ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER_BACKUP") tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID_BACKUP") service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID_BACKUP") service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY_BACKUP") if not ingest_cluster or not tenant_id or not service_id or not service_key: print("Could not load backup Kusto Credentials from environment") self._ingestion_client_backup = None else: kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(ingest_cluster, service_id, service_key, tenant_id) self._ingestion_client_backup = KustoIngestClient(kcsb)
def __init__(self, cluster, database, table, clientId, clientSecret, authority="microsoft.com", resetTable=False): """ Parameters ---------- cluster : str Azure Data Explorer (ADX) cluster address. eg, 'CDOC.kusto.windows.net' database : str Azure Data Explorer (ADX) database name. eg, 'TestDb' table : str Azure Data Explorer (ADX) table name. eg, 'OutputTable' clientId : str Azure Data Explorer (ADX) client Id that has permissions to access ADX. clientSecret : str Azure Data Explorer (ADX) access key. Used along with client Id. authority : str Azure Data Explorer (ADX) authority. Optional. When not specified, 'microsoft.com' is used. resetTable : bool Default is False. If True, the existing data in the destination table is dropped before new data is logged. """ self.running = True self.batchSize = 10000 self.flushDuration = timedelta(milliseconds = 1000) self.lastUploadTime = datetime.utcnow() self.initTable = False self.nextBatch = list() self.currentBatch = None self.lock = threading.Lock() self.resetTable = resetTable self.database = database self.table = table self.kcsbData = KustoConnectionStringBuilder.with_aad_application_key_authentication(f"https://{cluster}:443/", clientId, clientSecret, authority) self.kcsbIngest = KustoConnectionStringBuilder.with_aad_application_key_authentication(f"https://ingest-{cluster}:443/", clientId, clientSecret, authority) self.dataClient = KustoClient(self.kcsbData) self.ingestClient = QueuedIngestClient(self.kcsbIngest) self.ingestionProps = IngestionProperties(database=database, table=table,)
def __init__(self, db_name: str): """Initialize a Kusto report DB connector. Args: db_name: The Kusto database to connect to. """ self.db_name = db_name ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER") tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID") service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID") service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY") if not ingest_cluster or not tenant_id or not service_id or not service_key: raise RuntimeError( "Could not load Kusto Credentials from environment") kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( ingest_cluster, service_id, service_key, tenant_id) self._ingestion_client = KustoIngestClient(kcsb)
def generate_connection_string( cls, cluster_url: str, authentication_mode: AuthenticationModeOptions ) -> KustoConnectionStringBuilder: """ Generates Kusto Connection String based on given Authentication Mode. :param cluster_url: Cluster to connect to. :param authentication_mode: User Authentication Mode, Options: (UserPrompt|ManagedIdentity|AppKey|AppCertificate) :return: A connection string to be used when creating a Client """ # Learn More: For additional information on how to authorize users and apps in Kusto, # see: https://docs.microsoft.com/azure/data-explorer/manage-database-permissions if authentication_mode == AuthenticationModeOptions.UserPrompt.name: # Prompt user for credentials return KustoConnectionStringBuilder.with_interactive_login( cluster_url) elif authentication_mode == AuthenticationModeOptions.ManagedIdentity.name: # Authenticate using a System-Assigned managed identity provided to an azure service, or using a User-Assigned managed identity. # For more information, see https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview return cls.create_managed_identity_connection_string( cluster_url) elif authentication_mode == AuthenticationModeOptions.AppKey.name: # Learn More: For information about how to procure an AAD Application, # see: https://docs.microsoft.com/azure/data-explorer/provision-azure-ad-app # TODO (config - optional): App ID & tenant, and App Key to authenticate with return KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster_url, os.environ.get("APP_ID"), os.environ.get("APP_KEY"), os.environ.get("APP_TENANT")) elif authentication_mode == AuthenticationModeOptions.AppCertificate.name: return cls.create_application_certificate_connection_string( cluster_url) else: Utils.error_handler( f"Authentication mode '{authentication_mode}' is not supported" )
def proxy_kcsb(request) -> Tuple[KustoConnectionStringBuilder, bool]: cluster = KustoClientTestsMixin.HOST user = "******" password = "******" authority_id = "13456" uuid = "11111111-1111-1111-1111-111111111111" key = "key of application" token = "The app hardest token ever" return { "user_password": (KustoConnectionStringBuilder.with_aad_user_password_authentication( cluster, user, password, authority_id), True), "application_key": (KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, uuid, key, "microsoft.com"), True), "application_token": (KustoConnectionStringBuilder. with_aad_application_token_authentication( cluster, application_token=token), False), "device": (KustoConnectionStringBuilder.with_aad_device_authentication(cluster), True), "user_token": (KustoConnectionStringBuilder.with_aad_user_token_authentication( cluster, user_token=token), False), "managed_identity": (KustoConnectionStringBuilder. with_aad_managed_service_identity_authentication(cluster), False), "token_provider": (KustoConnectionStringBuilder.with_token_provider( cluster, lambda x: x), False), "async_token_provider": (KustoConnectionStringBuilder.with_async_token_provider( cluster, lambda x: x), False), "az_cli": (KustoConnectionStringBuilder.with_az_cli_authentication(cluster), True), "interactive_login": (KustoConnectionStringBuilder.with_interactive_login(cluster), True), }[request.param]
def get_dataset(): in_use = microsoft_config kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( in_use['cluster'], in_use['client_id'], in_use["client_secret"], in_use["authority_id"]) query = ''' RawEventsAzCli | where EventTimestamp > ago(2h) and EventTimestamp < ago(1h) // Remove pre 2.0.28 telemetry as it has a different schema | where toint(split(tostring(parse_json(Properties).["context.default.vs.core.telemetryapi.productversion"]), '.')[2]) > 28 | where tostring(parse_json(Properties).["reserved.datamodel.action.result"]) == "Success" | sort by UserId | project UserId,command=parse_json(Properties)["context.default.azurecli.rawcommand"], params = parse_json(Properties).["context.default.azurecli.params"], os_type = tostring(parse_json(Properties).["context.default.vs.core.os.type"]), source = tostring(parse_json(Properties).["context.default.azurecli.source"]), start_time_str= todatetime(parse_json(Properties).["context.default.azurecli.starttime"]), end_time_str = todatetime(parse_json(Properties).["context.default.azurecli.endtime"]) | take 500 ''' client = KustoClient(kcsb) response = client.execute("AzureCli", query) dataset = {} for row in response.primary_results[0]: uid, command, params, os_type, source, begin, end = row params = sort_params(params) if uid not in dataset: dataset[uid] = [(command, params, os_type, source, begin, end)] else: last_cmd, last_params, *_ = dataset[uid][-1] last_params = sort_params(last_params) if last_cmd == command and last_params == params: continue # duplicate dataset[uid].append( (command, params, os_type, source, begin, end)) return dataset
def initialize_kusto_client(): """initialize kusto client """ try: global SERVICE_CLIENT, KUSTO_MGMT_CLIENT credentials = ServicePrincipalCredentials( client_id=CLIENT_ID, secret=CLIENT_SECRET, tenant=TENANT_ID ) KUSTO_MGMT_CLIENT = KustoManagementClient(credentials, SUBSCRIPTION_ID) kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( CLUSTER, CLIENT_ID, CLIENT_SECRET, TENANT_ID) SERVICE_CLIENT = KustoClient(kcsb) except Exception as err: print("error for KustoClient") print(err) raise
import os from azure.kusto.data.helpers import dataframe_from_result_table from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from generatedDataset.adxClient.adxConfig import Config ###################################################### ## AUTH ## ###################################################### cluster = Config.cluster client_id = Config.client_id client_secret = Config.client_secret authority_id = Config.authority_id kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, client_id, client_secret, authority_id) client = KustoClient(kcsb) ###################################################### ## QUERY ## ###################################################### # Observability ID UUID = 'obsid_338d0e82-98dd-467c-8a15-5443f50e9e4a/v-60b28f6a-8509-4922-988f-d52029566468' # The ADX database name db = "dev-observability" # Query to get all required metrics def get_observability_metrics():
def get_kusto_client(client_id: str, client_secret: str, tenant_id: str, region: str, cluster_name: str) -> KustoClient: cluster = f"https://{cluster_name}.{region}.kusto.windows.net" kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, client_id, client_secret, tenant_id) return KustoClient(kcsb)
def engine_kcsb_from_env(cls, app_insights=False) -> KustoConnectionStringBuilder: engine = cls.engine_cs if not app_insights else cls.ai_engine_cs if all([cls.app_id, cls.app_key, cls.auth_id]): return KustoConnectionStringBuilder.with_aad_application_key_authentication(engine, cls.app_id, cls.app_key, cls.auth_id) else: return KustoConnectionStringBuilder.with_interactive_login(engine)
def authenticate(cluster_url, aad_app_id, aad_app_secret, tenant_id): kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster_url, aad_app_id, aad_app_secret, tenant_id) client = KustoClient(kcsb) return client
def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder: return KustoConnectionStringBuilder.with_aad_application_key_authentication( cls.dm_cs, cls.app_id, cls.app_key, cls.auth_id)
def test_aad_app(self): """Checks kcsb that is created with AAD application credentials.""" uuid = str(uuid4()) key = "key of application" kcsbs = [ KustoConnectionStringBuilder( "localhost;Application client Id={0};application Key={1};Authority Id={2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder( "Data Source=localhost ; Application Client Id={0}; Appkey ={1};Authority Id= {2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder( " Addr = localhost ; AppClientId = {0} ; AppKey ={1}; Authority Id={2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder( "Network Address = localhost; AppClientId = {0} ; AppKey ={1};AuthorityId={2} ; aad federated security = {3}" .format(uuid, key, "microsoft.com", True)), KustoConnectionStringBuilder. with_aad_application_key_authentication("localhost", uuid, key, "microsoft.com"), ] try: KustoConnectionStringBuilder.with_aad_application_key_authentication( "localhost", uuid, key, None) except Exception as e: # make sure error is raised when authority_id i none assert isinstance(e, ValueError) kcsb1 = KustoConnectionStringBuilder("server=localhost") kcsb1[KustoConnectionStringBuilder.ValidKeywords. application_client_id] = uuid kcsb1[KustoConnectionStringBuilder.ValidKeywords.application_key] = key kcsb1[KustoConnectionStringBuilder.ValidKeywords. authority_id] = "microsoft.com" kcsb1[KustoConnectionStringBuilder.ValidKeywords. aad_federated_security] = True kcsbs.append(kcsb1) kcsb2 = KustoConnectionStringBuilder("Server=localhost") kcsb2["AppclientId"] = uuid kcsb2["Application key"] = key kcsb2["Authority Id"] = "microsoft.com" kcsb2["aad federated security"] = True kcsbs.append(kcsb2) for kcsb in kcsbs: assert kcsb.data_source == "localhost" assert kcsb.aad_federated_security assert kcsb.aad_user_id is None assert kcsb.password is None assert kcsb.application_client_id == uuid assert kcsb.application_key == key assert kcsb.authority_id == "microsoft.com" assert repr( kcsb ) == "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}".format( uuid, key, "microsoft.com") assert str( kcsb ) == "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}".format( uuid, self.PASSWORDS_REPLACEMENT, "microsoft.com")
def main(): # Kusto cluster inputs data = os.environ["INPUT_DATA"] tenantId = os.environ["INPUT_TENANTID"] databaseName = os.environ["INPUT_DATABASE"] clusterName = os.environ["INPUT_CLUSTERNAME"] region = os.environ["INPUT_CLUSTERREGION"] clientId = os.environ["INPUT_CLIENTID"] clientSecret = os.environ["INPUT_CLIENTSECRET"] destinationTable = os.environ["INPUT_TABLE"] mapping = os.environ['INPUT_MAPPING'] try: print(data) # file creation fileName = "sample.json" filePath = os.path.join(os.environ["GITHUB_WORKSPACE"], fileName) deploymentData = {} deploymentData["Timestamp"] = str(datetime.now()) deploymentData["DeploymentDetails"] = data with open(filePath, "w") as targetFile: json.dump(deploymentData, targetFile) # cluster client connection and auth httpsPrefix = "https://" suffixKustoUri = "kusto.windows.net:443/" clusterIngestUri = "{0}ingest-{1}.{2}.{3}".format(httpsPrefix, clusterName, region, suffixKustoUri) kcsb_ingest = KustoConnectionStringBuilder.with_aad_application_key_authentication( clusterIngestUri, clientId, clientSecret, tenantId) print(mapping) # Cluster ingestion parameters ingestionClient = KustoIngestClient(kcsb_ingest) ingestionProperties = IngestionProperties(database=databaseName, table=destinationTable, dataFormat=DataFormat.JSON, ingestion_mapping_reference=mapping, report_level=ReportLevel.FailuresAndSuccesses) fileDescriptor = FileDescriptor(filePath, 1000) print('Payload to dump') with open(filePath, "r") as targetFile: parsed = json.load(targetFile) print(json.dumps(parsed, indent=2, sort_keys=True)) ingestionClient.ingest_from_file(fileDescriptor, ingestion_properties=ingestionProperties) print('Queued up ingestion with Azure Data Explorer') # Remove the temporary file os.remove(filePath) """ # Repeated pinging to wait for success/failure message qs = KustoIngestStatusQueues(ingestionClient) # Interval to ping MAX_BACKOFF = 5 backoff = 1 while True: if qs.success.is_empty() and qs.failure.is_empty(): time.sleep(backoff) backoff = min(backoff * 2, MAX_BACKOFF) print("No new messages. backing off for {} seconds".format(backoff)) continue backoff = 1 success_messages = qs.success.pop(10) failure_messages = qs.failure.pop(10) pprint.pprint("SUCCESS : {}".format(success_messages)) pprint.pprint("FAILURE : {}".format(failure_messages)) break """ except Exception as e: raise Exception(e)