def Ingest(Tag): # setting AUTHORITY_ID = "6babcaad-604b-40ac-a9d7-9fd97c0b779f" INGESTCLUSTER = "https://ingest-cgadataout.kusto.windows.net" KUSTOCLUSTER = "https://cgadataout.kusto.windows.net" DATABASE = "DevRelWorkArea" # Create table KCSB_DATA = KustoConnectionStringBuilder.with_aad_device_authentication( KUSTOCLUSTER) DESTINATION_TABLE = "RepoContributors" DESTINATION_TABLE_COLUMN_MAPPING = "RepoContributors_CSV_Mapping" KUSTO_CLIENT = KustoClient(KCSB_DATA) DROP_TABLE_IF_EXIST = ".drop table RepoContributors ifexists" RESPONSE = KUSTO_CLIENT.execute_mgmt(DATABASE, DROP_TABLE_IF_EXIST) CREATE_TABLE_COMMAND = ".create table RepoContributors (Article: string, Contributors: int64, Data: string)" RESPONSE = KUSTO_CLIENT.execute_mgmt(DATABASE, CREATE_TABLE_COMMAND) print("RepoContributors table is created") # Create mapping CREATE_MAPPING_COMMAND = """.create table RepoContributors ingestion csv mapping 'RepoContributors_CSV_Mapping' '[{"Name": "Article","datatype": "string","Ordinal": 0},{"Name": "Contributors","datatype": "int64","Ordinal": 1},{"Name": "Data","datatype": "string","Ordinal": 2}]'""" RESPONSE = KUSTO_CLIENT.execute_mgmt(DATABASE, CREATE_MAPPING_COMMAND) print("mapping is created") # Ingest # The authentication method will be taken from the chosen KustoConnectionStringBuilder. ingestion_props = IngestionProperties( database="DevRelWorkArea", table="RepoContributors", dataFormat=DataFormat.CSV, ingestByTags=[Tag], dropByTags=[Tag], mappingReference=DESTINATION_TABLE_COLUMN_MAPPING, reportLevel=ReportLevel.FailuresAndSuccesses, additionalProperties={'ignoreFirstRecord': 'true'}) kcsb = KustoConnectionStringBuilder.with_aad_device_authentication( INGESTCLUSTER) client = KustoIngestClient(kcsb) # ingest from file file_descriptor = FileDescriptor( r"D:\test\Results\log_data_merge\merge_microsoftdocs_sql-docs-pr.txt", 3333) # 3333 is the raw size of the data in bytes. client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_props) # if status updates are required, something like this can be done return 1
def test_admin_then_query(self, mock_post): """Tests partial results.""" client = KustoClient("https://somecluster.kusto.windows.net") query = ".show tables | project DatabaseName, TableName" response = client.execute_mgmt("PythonTest", query) self.assertEqual(response.errors_count, 0) self.assertEqual(len(response), 4) results = list(response.primary_results[0]) self.assertEqual(len(results), 2) self.assertEqual(response[0].table_kind, WellKnownDataSet.PrimaryResult) self.assertEqual(response[1].table_kind, WellKnownDataSet.QueryProperties) self.assertEqual(response[2].table_kind, WellKnownDataSet.QueryCompletionInformation) self.assertEqual(response[3].table_kind, WellKnownDataSet.TableOfContents)
def test_admin_then_query(self, mock_post): """Tests admin then query.""" client = KustoClient("https://somecluster.kusto.windows.net") query = ".show tables | project DatabaseName, TableName" response = client.execute_mgmt("PythonTest", query) assert response.errors_count == 0 assert len(response) == 4 results = list(response.primary_results[0]) assert len(results) == 2 assert response[0].table_kind == WellKnownDataSet.PrimaryResult assert response[1].table_kind == WellKnownDataSet.QueryProperties assert response[ 2].table_kind == WellKnownDataSet.QueryCompletionInformation assert response[3].table_kind == WellKnownDataSet.TableOfContents
def test_sanity_control_command(self, mock_post, mock_aad): """Tests contol command.""" client = KustoClient("https://somecluster.kusto.windows.net") response = client.execute_mgmt("NetDefaultDB", ".show version") self.assertEqual(len(response), 1) primary_table = response.primary_results[0] row_count = 0 for _ in primary_table: row_count += 1 self.assertEqual(row_count, 1) result = primary_table[0] self.assertEqual(result["BuildVersion"], "1.0.6693.14577") self.assertEqual( result["BuildTime"], datetime(year=2018, month=4, day=29, hour=8, minute=5, second=54, tzinfo=tzutc()) ) self.assertEqual(result["ServiceType"], "Engine") self.assertEqual(result["ProductVersion"], "KustoMain_2018.04.29.5")
""".format(DESTINATION_TABLE=destination_table, CREATE_COLUMN_LIST=obj.get('CREATE_COLUMN_LIST')) check_for_mapping_command = """ .show table {DESTINATION_TABLE} ingestion csv mapping '{COLUMN_MAPPING_NAME}' """.format(DESTINATION_TABLE=destination_table, COLUMN_MAPPING_NAME=column_mapping_name) create_mapping_command = """ .create table {DESTINATION_TABLE} ingestion csv mapping '{COLUMN_MAPPING_NAME}' '{COLUMN_MAPPING}' """.format(DESTINATION_TABLE=destination_table, COLUMN_MAPPING_NAME=column_mapping_name, COLUMN_MAPPING=json.dumps(obj.get('COLUMN_MAPPING'))) # Drop table if exists, then create. kusto_client.execute_mgmt(kusto_database, drop_table_if_exists_command) kusto_client.execute_mgmt(kusto_database, create_table_command) #NOTE: this may be backwards... try: response = kusto_client.execute_mgmt(kusto_database, check_for_mapping_command) except: # Because check_for_mapping_command should throw error if mapping already exists response = kusto_client.execute_mgmt(kusto_database, create_mapping_command) ingestion_client = KustoIngestClient(kcsb_ingest) # All ingestion properties: https://docs.microsoft.com/en-us/azure/kusto/management/data-ingestion/#ingestion-properties ingestion_props = IngestionProperties( reportLevel=reportLevel,
import argparse from azure.kusto.data.request import KustoClient, KustoConnectionStringBuilder arg_parser = argparse.ArgumentParser() arg_parser.add_argument('--cluster', '-cluster', help="Kusto cluster URL", type=str, required=True) arg_parser.add_argument('--database', '-db', help="Kusto database", type=str, required=True) args = arg_parser.parse_args() cluster = args.cluster database = args.database # define the command to create our first table and mapping in the database LANDING_TABLE_CMD = ".create-merge table landing (rawdata: dynamic)" LANDING_TABLE_MAPPING_CMD = ".create table landing ingestion json mapping \"raw_mapping\" '[{\"column\":\"rawdata\",\"path\":\"$\"}]'" # create a connection string with a device code (=interactive) authentication kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster) # create the client and do the actual authentication client = KustoClient(kcsb) # run the command to create the table response = client.execute_mgmt(database, LANDING_TABLE_CMD) # run the command to create the table mapping response = client.execute_mgmt(database, LANDING_TABLE_MAPPING_CMD)