def write_to_db(coverage_data, args): # connect to database cluster = "https://ingest-onnxruntimedashboarddb.southcentralus.kusto.windows.net" kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(cluster) # The authentication method will be taken from the chosen KustoConnectionStringBuilder. client = QueuedIngestClient(kcsb) fields = [ "UploadTime", "CommitId", "Coverage", "LinesCovered", "TotalLines", "OS", "Arch", "BuildConfig", "ReportURL", "Branch" ] now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") rows = [[ now_str, args.commit_hash, coverage_data['coverage'], coverage_data['lines_covered'], coverage_data['lines_valid'], args.os.lower(), args.arch.lower(), args.build_config.lower(), args.report_url.lower(), args.branch.lower() ]] ingestion_props = IngestionProperties( database="powerbi", table="test_coverage", data_format=DataFormat.CSV, report_level=ReportLevel.FailuresAndSuccesses) df = pandas.DataFrame(data=rows, columns=fields) client.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
def write_to_db(binary_size_data, args): # connect to database cluster = "https://ingest-onnxruntimedashboarddb.southcentralus.kusto.windows.net" kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(cluster) # The authentication method will be taken from the chosen KustoConnectionStringBuilder. client = QueuedIngestClient(kcsb) fields = ["build_time", "build_id", "build_project", "commit_id", "os", "arch", "build_config", "size", "Branch"] now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") branch_name = os.environ.get("BUILD_SOURCEBRANCHNAME", "main") rows = [] for row in binary_size_data: rows.append( [ now_str, args.build_id, args.build_project, args.commit_hash, row["os"], row["arch"], row["build_config"], row["size"], branch_name.lower(), ] ) ingestion_props = IngestionProperties( database="powerbi", table="binary_size", data_format=DataFormat.CSV, report_level=ReportLevel.FailuresAndSuccesses, ) df = pandas.DataFrame(data=rows, columns=fields) client.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
def _get_client_for_cluster(cluster: str) -> KustoClient: # If we call 'with_az_cli_authentication' directly, in case of failure we will get an un-informative exception. # As a workaround, we first attempt to manually get the Azure CLI token, and see if it works. # Get rid of this workaround once this is resolved: https://github.com/Azure/azure-kusto-python/issues/240 stored_token = _get_azure_cli_auth_token() if stored_token is None: _logger.info("Failed to get Azure CLI token, falling back to AAD device authentication") connection_string_builder = KustoConnectionStringBuilder.with_aad_device_authentication(cluster) else: connection_string_builder = KustoConnectionStringBuilder.with_az_cli_authentication(cluster) return KustoClient(connection_string_builder)
def main(): args = parse_arguments() # connect to database kcsb_ingest = KustoConnectionStringBuilder.with_az_cli_authentication(cluster_ingest) ingest_client = QueuedIngestClient(kcsb_ingest) date_time = args.datetime identifier = get_identifier(date_time, args.commit_hash, args.trt_version, args.branch) try: result_file = args.report_folder folders = os.listdir(result_file) os.chdir(result_file) tables = [fail_name, memory_name, latency_name, status_name, latency_over_time_name, specs_name, session_name] table_results = {} for table_name in tables: table_results[table_name] = pd.DataFrame() for model_group in folders: os.chdir(model_group) csv_filenames = os.listdir() for csv in csv_filenames: table = parse_csv(csv) if session_name in csv: table_results[session_name] = table_results[session_name].append(get_session(table, model_group), ignore_index=True) if specs_name in csv: table_results[specs_name] = table_results[specs_name].append(get_specs(table, args.branch, args.commit_hash, date_time), ignore_index=True) if fail_name in csv: table_results[fail_name] = table_results[fail_name].append(get_failures(table, model_group), ignore_index=True) if latency_name in csv: table_results[memory_name] = table_results[memory_name].append(get_memory(table, model_group), ignore_index=True) table_results[latency_name] = table_results[latency_name].append(get_latency(table, model_group), ignore_index=True) table_results[latency_over_time_name] = table_results[latency_over_time_name].append(get_latency_over_time(args.commit_hash, args.report_url, args.branch, table_results[latency_name]), ignore_index=True) if status_name in csv: table_results[status_name] = table_results[status_name].append(get_status(table, model_group), ignore_index=True) os.chdir(result_file) for table in tables: print('writing ' + table + ' to database') db_table_name = 'ep_model_' + table write_table(ingest_client, table_results[table], db_table_name, date_time, identifier) except BaseException as e: print(str(e)) sys.exit(1)
def get_kusto_client(server: str, database: str) -> KustoClient: """ Helper to get an authenticated KustoClient. Try to use Az CLI cached credentials, fall back to device code auth. :param server: The (short) name of a Kusto server cluster, not the full URI :param database: The name of the initial catalog to connect to """ logger = logging.getLogger(__name__) server_uri = f"https://{server}.kusto.windows.net" try: kcsb = KustoConnectionStringBuilder.with_az_cli_authentication( server_uri) client = KustoClient(kcsb) # hit the server to force authentication client.execute_query(database, "print('hi')") return client except KustoAuthenticationError: kcsb = KustoConnectionStringBuilder.with_aad_device_authentication( server_uri) client = KustoClient(kcsb) client.execute_query(database, "print('hi')") return client
def proxy_kcsb(request) -> Tuple[KustoConnectionStringBuilder, bool]: cluster = KustoClientTestsMixin.HOST user = "******" password = "******" authority_id = "13456" uuid = "11111111-1111-1111-1111-111111111111" key = "key of application" token = "The app hardest token ever" return { "user_password": (KustoConnectionStringBuilder.with_aad_user_password_authentication( cluster, user, password, authority_id), True), "application_key": (KustoConnectionStringBuilder.with_aad_application_key_authentication( cluster, uuid, key, "microsoft.com"), True), "application_token": (KustoConnectionStringBuilder. with_aad_application_token_authentication( cluster, application_token=token), False), "device": (KustoConnectionStringBuilder.with_aad_device_authentication(cluster), True), "user_token": (KustoConnectionStringBuilder.with_aad_user_token_authentication( cluster, user_token=token), False), "managed_identity": (KustoConnectionStringBuilder. with_aad_managed_service_identity_authentication(cluster), False), "token_provider": (KustoConnectionStringBuilder.with_token_provider( cluster, lambda x: x), False), "async_token_provider": (KustoConnectionStringBuilder.with_async_token_provider( cluster, lambda x: x), False), "az_cli": (KustoConnectionStringBuilder.with_az_cli_authentication(cluster), True), "interactive_login": (KustoConnectionStringBuilder.with_interactive_login(cluster), True), }[request.param]
kcsb = KustoConnectionStringBuilder() # Managed Identity - automatically injected into your machine by azure when running on an azure service. # It's the best way for any code that does such - it's automatic, and requires no saving of secrets. # In case you want to authenticate with a System Assigned Managed Service Identity (MSI) kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster) # In case you want to authenticate with a User Assigned Managed Service Identity (MSI) user_assigned_client_id = "the AAD identity client id" kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster, client_id=user_assigned_client_id) # In case you want to authenticate with Azure CLI. # Users are required to be in a logged in state in az-cli, for this authentication method to succeed. Run `az login` to login to azure cli. kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(cluster) # In case you want to authenticate with AAD username and password username = "******" password = "******" kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, username, password, authority_id) # In case you want to authenticate with AAD device code. # Please note that if you choose this option, you'll need to authenticate for every new instance that is initialized. # It is highly recommended to create one instance and use it for all of your queries. kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster) # The authentication method will be taken from the chosen KustoConnectionStringBuilder. client = KustoClient(kcsb) ######################################################
def main(): """ Entry point of this script. Uploads data produced by benchmarking scripts to the database. """ args = parse_arguments() # connect to database kcsb_ingest = KustoConnectionStringBuilder.with_az_cli_authentication( CLUSTER_INGEST) ingest_client = QueuedIngestClient(kcsb_ingest) identifier = get_identifier(args.commit_datetime, args.commit_hash, args.trt_version, args.branch) upload_time = datetime.datetime.now(tz=datetime.timezone.utc).replace( microsecond=0) try: result_file = args.report_folder folders = os.listdir(result_file) os.chdir(result_file) tables = [ fail_name, memory_name, latency_name, status_name, latency_over_time_name, specs_name, session_name, ] table_results = {} for table_name in tables: table_results[table_name] = pd.DataFrame() for model_group in folders: os.chdir(model_group) csv_filenames = os.listdir() for csv in csv_filenames: table = pd.read_csv(csv) if session_name in csv: table_results[session_name] = table_results[ session_name].append(get_session(table, model_group), ignore_index=True) elif specs_name in csv: table_results[specs_name] = table_results[ specs_name].append( get_specs(table, args.branch, args.commit_hash, args.commit_datetime), ignore_index=True, ) elif fail_name in csv: table_results[fail_name] = table_results[fail_name].append( get_failures(table, model_group), ignore_index=True) elif latency_name in csv: table_results[memory_name] = table_results[ memory_name].append(get_memory(table, model_group), ignore_index=True) table_results[latency_name] = table_results[ latency_name].append(get_latency(table, model_group), ignore_index=True) if not table_results[latency_name].empty: table_results[latency_over_time_name] = table_results[ latency_over_time_name].append( get_latency_over_time( args.commit_hash, args.report_url, args.branch, table_results[latency_name], ), ignore_index=True, ) elif status_name in csv: table_results[status_name] = table_results[ status_name].append(get_status(table, model_group), ignore_index=True) os.chdir(result_file) for table in tables: print("writing " + table + " to database") db_table_name = "ep_model_" + table write_table( ingest_client, table_results[table], db_table_name, upload_time, identifier, ) except BaseException as e: print(str(e)) sys.exit(1)