def engine_kcsb_from_env() -> KustoConnectionStringBuilder:
engine_cs = os.environ.get("ENGINE_CONNECTION_STRING")
app_id = os.environ.get("APP_ID")
app_key = os.environ.get("APP_KEY")
auth_id = os.environ.get("AUTH_ID")
return KustoConnectionStringBuilder.with_aad_application_key_authentication(
engine_cs, app_id, app_key, auth_id)
def authenticate_to_kusto_ingress(cluster):
"""Authenticate and return kusto connection client"""
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, CLIENT_ID, CLIENT_SECRET, AUTHORITY_ID)
# The authentication method will be taken from the chosen KustoConnectionStringBuilder.
kusto_client = KustoIngestClient(kcsb)
return kusto_client
def dm_kcsb_from_env() -> KustoConnectionStringBuilder:
engine_cs = os.environ.get("ENGINE_CONNECTION_STRING")
dm_cs = os.environ.get("DM_CONNECTION_STRING") or engine_cs.replace(
"//", "//ingest-")
app_id = os.environ.get("APP_ID")
app_key = os.environ.get("APP_KEY")
auth_id = os.environ.get("AUTH_ID")
return KustoConnectionStringBuilder.with_aad_application_key_authentication(
dm_cs, app_id, app_key, auth_id)
def initialize_kusto_client():
"""initialize kusto client
"""
global KUSTO_INGESTION_CLIENT
if not KUSTO_INGESTION_CLIENT:
kcsb_ingest = KustoConnectionStringBuilder.with_aad_application_key_authentication( \
INGESTION_SERVER_URI, APP_CLIENT_ID, APP_CLIENT_SECRETS, APP_AAD_TENANT_ID)
KUSTO_INGESTION_CLIENT = KustoIngestClient(kcsb_ingest)
logging.info(f"{LOG_MESSAGE_HEADER} Build KUSTO_INGESTION_CLIENT")
else:
logging.info(f"{LOG_MESSAGE_HEADER} KUSTO_INGESTION_CLIENT exist")
def __init__(self, db_name: str):
"""Initialize a Kusto report DB connector.
Args:
db_name: The Kusto database to connect to.
"""
self.db_name = db_name
ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER")
tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID")
service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID")
service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY")
if not ingest_cluster or not tenant_id or not service_id or not service_key:
raise RuntimeError("Could not load Kusto Credentials from environment")
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(ingest_cluster,
service_id,
service_key,
tenant_id)
self._ingestion_client = KustoIngestClient(kcsb)
"""
Kusto performance depends on the work load of cluster, to improve the high availability of test result data service
by hosting a backup cluster, which is optional.
"""
ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER_BACKUP")
tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID_BACKUP")
service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID_BACKUP")
service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY_BACKUP")
if not ingest_cluster or not tenant_id or not service_id or not service_key:
print("Could not load backup Kusto Credentials from environment")
self._ingestion_client_backup = None
else:
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(ingest_cluster,
service_id,
service_key,
tenant_id)
self._ingestion_client_backup = KustoIngestClient(kcsb)
def __init__(self, cluster, database, table, clientId, clientSecret, authority="microsoft.com", resetTable=False):
"""
Parameters
----------
cluster : str
Azure Data Explorer (ADX) cluster address. eg, 'CDOC.kusto.windows.net'
database : str
Azure Data Explorer (ADX) database name. eg, 'TestDb'
table : str
Azure Data Explorer (ADX) table name. eg, 'OutputTable'
clientId : str
Azure Data Explorer (ADX) client Id that has permissions to access ADX.
clientSecret : str
Azure Data Explorer (ADX) access key. Used along with client Id.
authority : str
Azure Data Explorer (ADX) authority. Optional. When not specified, 'microsoft.com' is used.
resetTable : bool
Default is False. If True, the existing data in the destination table is dropped before new data is logged.
"""
self.running = True
self.batchSize = 10000
self.flushDuration = timedelta(milliseconds = 1000)
self.lastUploadTime = datetime.utcnow()
self.initTable = False
self.nextBatch = list()
self.currentBatch = None
self.lock = threading.Lock()
self.resetTable = resetTable
self.database = database
self.table = table
self.kcsbData = KustoConnectionStringBuilder.with_aad_application_key_authentication(f"https://{cluster}:443/", clientId, clientSecret, authority)
self.kcsbIngest = KustoConnectionStringBuilder.with_aad_application_key_authentication(f"https://ingest-{cluster}:443/", clientId, clientSecret, authority)
self.dataClient = KustoClient(self.kcsbData)
self.ingestClient = QueuedIngestClient(self.kcsbIngest)
self.ingestionProps = IngestionProperties(database=database, table=table,)
def __init__(self, db_name: str):
"""Initialize a Kusto report DB connector.
Args:
db_name: The Kusto database to connect to.
"""
self.db_name = db_name
ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER")
tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID")
service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID")
service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY")
if not ingest_cluster or not tenant_id or not service_id or not service_key:
raise RuntimeError(
"Could not load Kusto Credentials from environment")
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
ingest_cluster, service_id, service_key, tenant_id)
self._ingestion_client = KustoIngestClient(kcsb)
def generate_connection_string(
cls, cluster_url: str,
authentication_mode: AuthenticationModeOptions
) -> KustoConnectionStringBuilder:
"""
Generates Kusto Connection String based on given Authentication Mode.
:param cluster_url: Cluster to connect to.
:param authentication_mode: User Authentication Mode, Options: (UserPrompt|ManagedIdentity|AppKey|AppCertificate)
:return: A connection string to be used when creating a Client
"""
# Learn More: For additional information on how to authorize users and apps in Kusto,
# see: https://docs.microsoft.com/azure/data-explorer/manage-database-permissions
if authentication_mode == AuthenticationModeOptions.UserPrompt.name:
# Prompt user for credentials
return KustoConnectionStringBuilder.with_interactive_login(
cluster_url)
elif authentication_mode == AuthenticationModeOptions.ManagedIdentity.name:
# Authenticate using a System-Assigned managed identity provided to an azure service, or using a User-Assigned managed identity.
# For more information, see https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview
return cls.create_managed_identity_connection_string(
cluster_url)
elif authentication_mode == AuthenticationModeOptions.AppKey.name:
# Learn More: For information about how to procure an AAD Application,
# see: https://docs.microsoft.com/azure/data-explorer/provision-azure-ad-app
# TODO (config - optional): App ID & tenant, and App Key to authenticate with
return KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster_url, os.environ.get("APP_ID"),
os.environ.get("APP_KEY"), os.environ.get("APP_TENANT"))
elif authentication_mode == AuthenticationModeOptions.AppCertificate.name:
return cls.create_application_certificate_connection_string(
cluster_url)
else:
Utils.error_handler(
f"Authentication mode '{authentication_mode}' is not supported"
)
def proxy_kcsb(request) -> Tuple[KustoConnectionStringBuilder, bool]:
cluster = KustoClientTestsMixin.HOST
user = "******"
password = "******"
authority_id = "13456"
uuid = "11111111-1111-1111-1111-111111111111"
key = "key of application"
token = "The app hardest token ever"
return {
"user_password":
(KustoConnectionStringBuilder.with_aad_user_password_authentication(
cluster, user, password, authority_id), True),
"application_key":
(KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, uuid, key, "microsoft.com"), True),
"application_token": (KustoConnectionStringBuilder.
with_aad_application_token_authentication(
cluster, application_token=token), False),
"device":
(KustoConnectionStringBuilder.with_aad_device_authentication(cluster),
True),
"user_token":
(KustoConnectionStringBuilder.with_aad_user_token_authentication(
cluster, user_token=token), False),
"managed_identity":
(KustoConnectionStringBuilder.
with_aad_managed_service_identity_authentication(cluster), False),
"token_provider": (KustoConnectionStringBuilder.with_token_provider(
cluster, lambda x: x), False),
"async_token_provider":
(KustoConnectionStringBuilder.with_async_token_provider(
cluster, lambda x: x), False),
"az_cli":
(KustoConnectionStringBuilder.with_az_cli_authentication(cluster),
True),
"interactive_login":
(KustoConnectionStringBuilder.with_interactive_login(cluster), True),
}[request.param]
def get_dataset():
in_use = microsoft_config
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
in_use['cluster'], in_use['client_id'], in_use["client_secret"], in_use["authority_id"])
query = '''
RawEventsAzCli
| where EventTimestamp > ago(2h) and EventTimestamp < ago(1h)
// Remove pre 2.0.28 telemetry as it has a different schema
| where toint(split(tostring(parse_json(Properties).["context.default.vs.core.telemetryapi.productversion"]), '.')[2]) > 28
| where tostring(parse_json(Properties).["reserved.datamodel.action.result"]) == "Success"
| sort by UserId
| project UserId,command=parse_json(Properties)["context.default.azurecli.rawcommand"],
params = parse_json(Properties).["context.default.azurecli.params"],
os_type = tostring(parse_json(Properties).["context.default.vs.core.os.type"]),
source = tostring(parse_json(Properties).["context.default.azurecli.source"]),
start_time_str= todatetime(parse_json(Properties).["context.default.azurecli.starttime"]),
end_time_str = todatetime(parse_json(Properties).["context.default.azurecli.endtime"])
| take 500
'''
client = KustoClient(kcsb)
response = client.execute("AzureCli", query)
dataset = {}
for row in response.primary_results[0]:
uid, command, params, os_type, source, begin, end = row
params = sort_params(params)
if uid not in dataset:
dataset[uid] = [(command, params, os_type, source, begin, end)]
else:
last_cmd, last_params, *_ = dataset[uid][-1]
last_params = sort_params(last_params)
if last_cmd == command and last_params == params:
continue # duplicate
dataset[uid].append(
(command, params, os_type, source, begin, end))
return dataset
def initialize_kusto_client():
"""initialize kusto client
"""
try:
global SERVICE_CLIENT, KUSTO_MGMT_CLIENT
credentials = ServicePrincipalCredentials(
client_id=CLIENT_ID,
secret=CLIENT_SECRET,
tenant=TENANT_ID
)
KUSTO_MGMT_CLIENT = KustoManagementClient(credentials, SUBSCRIPTION_ID)
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
CLUSTER,
CLIENT_ID,
CLIENT_SECRET,
TENANT_ID)
SERVICE_CLIENT = KustoClient(kcsb)
except Exception as err:
print("error for KustoClient")
print(err)
raise
import os
from azure.kusto.data.helpers import dataframe_from_result_table
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
from generatedDataset.adxClient.adxConfig import Config
######################################################
## AUTH ##
######################################################
cluster = Config.cluster
client_id = Config.client_id
client_secret = Config.client_secret
authority_id = Config.authority_id
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, client_id, client_secret, authority_id)
client = KustoClient(kcsb)
######################################################
## QUERY ##
######################################################
# Observability ID
UUID = 'obsid_338d0e82-98dd-467c-8a15-5443f50e9e4a/v-60b28f6a-8509-4922-988f-d52029566468'
# The ADX database name
db = "dev-observability"
# Query to get all required metrics
def get_observability_metrics():
def get_kusto_client(client_id: str, client_secret: str, tenant_id: str,
region: str, cluster_name: str) -> KustoClient:
cluster = f"https://{cluster_name}.{region}.kusto.windows.net"
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(
cluster, client_id, client_secret, tenant_id)
return KustoClient(kcsb)
def engine_kcsb_from_env(cls, app_insights=False) -> KustoConnectionStringBuilder:
engine = cls.engine_cs if not app_insights else cls.ai_engine_cs
if all([cls.app_id, cls.app_key, cls.auth_id]):
return KustoConnectionStringBuilder.with_aad_application_key_authentication(engine, cls.app_id, cls.app_key, cls.auth_id)
else:
return KustoConnectionStringBuilder.with_interactive_login(engine)
def authenticate(cluster_url, aad_app_id, aad_app_secret, tenant_id):
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster_url, aad_app_id, aad_app_secret, tenant_id)
client = KustoClient(kcsb)
return client
def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder:
return KustoConnectionStringBuilder.with_aad_application_key_authentication(
cls.dm_cs, cls.app_id, cls.app_key, cls.auth_id)
def test_aad_app(self):
"""Checks kcsb that is created with AAD application credentials."""
uuid = str(uuid4())
key = "key of application"
kcsbs = [
KustoConnectionStringBuilder(
"localhost;Application client Id={0};application Key={1};Authority Id={2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder(
"Data Source=localhost ; Application Client Id={0}; Appkey ={1};Authority Id= {2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder(
" Addr = localhost ; AppClientId = {0} ; AppKey ={1}; Authority Id={2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder(
"Network Address = localhost; AppClientId = {0} ; AppKey ={1};AuthorityId={2} ; aad federated security = {3}"
.format(uuid, key, "microsoft.com", True)),
KustoConnectionStringBuilder.
with_aad_application_key_authentication("localhost", uuid, key,
"microsoft.com"),
]
try:
KustoConnectionStringBuilder.with_aad_application_key_authentication(
"localhost", uuid, key, None)
except Exception as e:
# make sure error is raised when authority_id i none
assert isinstance(e, ValueError)
kcsb1 = KustoConnectionStringBuilder("server=localhost")
kcsb1[KustoConnectionStringBuilder.ValidKeywords.
application_client_id] = uuid
kcsb1[KustoConnectionStringBuilder.ValidKeywords.application_key] = key
kcsb1[KustoConnectionStringBuilder.ValidKeywords.
authority_id] = "microsoft.com"
kcsb1[KustoConnectionStringBuilder.ValidKeywords.
aad_federated_security] = True
kcsbs.append(kcsb1)
kcsb2 = KustoConnectionStringBuilder("Server=localhost")
kcsb2["AppclientId"] = uuid
kcsb2["Application key"] = key
kcsb2["Authority Id"] = "microsoft.com"
kcsb2["aad federated security"] = True
kcsbs.append(kcsb2)
for kcsb in kcsbs:
assert kcsb.data_source == "localhost"
assert kcsb.aad_federated_security
assert kcsb.aad_user_id is None
assert kcsb.password is None
assert kcsb.application_client_id == uuid
assert kcsb.application_key == key
assert kcsb.authority_id == "microsoft.com"
assert repr(
kcsb
) == "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}".format(
uuid, key, "microsoft.com")
assert str(
kcsb
) == "Data Source=localhost;AAD Federated Security=True;Application Client Id={0};Application Key={1};Authority Id={2}".format(
uuid, self.PASSWORDS_REPLACEMENT, "microsoft.com")
def main():
# Kusto cluster inputs
data = os.environ["INPUT_DATA"]
tenantId = os.environ["INPUT_TENANTID"]
databaseName = os.environ["INPUT_DATABASE"]
clusterName = os.environ["INPUT_CLUSTERNAME"]
region = os.environ["INPUT_CLUSTERREGION"]
clientId = os.environ["INPUT_CLIENTID"]
clientSecret = os.environ["INPUT_CLIENTSECRET"]
destinationTable = os.environ["INPUT_TABLE"]
mapping = os.environ['INPUT_MAPPING']
try:
print(data)
# file creation
fileName = "sample.json"
filePath = os.path.join(os.environ["GITHUB_WORKSPACE"], fileName)
deploymentData = {}
deploymentData["Timestamp"] = str(datetime.now())
deploymentData["DeploymentDetails"] = data
with open(filePath, "w") as targetFile:
json.dump(deploymentData, targetFile)
# cluster client connection and auth
httpsPrefix = "https://"
suffixKustoUri = "kusto.windows.net:443/"
clusterIngestUri = "{0}ingest-{1}.{2}.{3}".format(httpsPrefix, clusterName, region, suffixKustoUri)
kcsb_ingest = KustoConnectionStringBuilder.with_aad_application_key_authentication(
clusterIngestUri, clientId, clientSecret, tenantId)
print(mapping)
# Cluster ingestion parameters
ingestionClient = KustoIngestClient(kcsb_ingest)
ingestionProperties = IngestionProperties(database=databaseName, table=destinationTable, dataFormat=DataFormat.JSON, ingestion_mapping_reference=mapping, report_level=ReportLevel.FailuresAndSuccesses)
fileDescriptor = FileDescriptor(filePath, 1000)
print('Payload to dump')
with open(filePath, "r") as targetFile:
parsed = json.load(targetFile)
print(json.dumps(parsed, indent=2, sort_keys=True))
ingestionClient.ingest_from_file(fileDescriptor, ingestion_properties=ingestionProperties)
print('Queued up ingestion with Azure Data Explorer')
# Remove the temporary file
os.remove(filePath)
"""
# Repeated pinging to wait for success/failure message
qs = KustoIngestStatusQueues(ingestionClient)
# Interval to ping
MAX_BACKOFF = 5
backoff = 1
while True:
if qs.success.is_empty() and qs.failure.is_empty():
time.sleep(backoff)
backoff = min(backoff * 2, MAX_BACKOFF)
print("No new messages. backing off for {} seconds".format(backoff))
continue
backoff = 1
success_messages = qs.success.pop(10)
failure_messages = qs.failure.pop(10)
pprint.pprint("SUCCESS : {}".format(success_messages))
pprint.pprint("FAILURE : {}".format(failure_messages))
break
"""
except Exception as e:
raise Exception(e)