def Ingest(Tag):
# setting
AUTHORITY_ID = "6babcaad-604b-40ac-a9d7-9fd97c0b779f"
INGESTCLUSTER = "https://ingest-cgadataout.kusto.windows.net"
KUSTOCLUSTER = "https://cgadataout.kusto.windows.net"
DATABASE = "DevRelWorkArea"
# Create table
KCSB_DATA = KustoConnectionStringBuilder.with_aad_device_authentication(
KUSTOCLUSTER)
DESTINATION_TABLE = "RepoContributors"
DESTINATION_TABLE_COLUMN_MAPPING = "RepoContributors_CSV_Mapping"
KUSTO_CLIENT = KustoClient(KCSB_DATA)
DROP_TABLE_IF_EXIST = ".drop table RepoContributors ifexists"
RESPONSE = KUSTO_CLIENT.execute_mgmt(DATABASE, DROP_TABLE_IF_EXIST)
CREATE_TABLE_COMMAND = ".create table RepoContributors (Article: string, Contributors: int64, Data: string)"
RESPONSE = KUSTO_CLIENT.execute_mgmt(DATABASE, CREATE_TABLE_COMMAND)
print("RepoContributors table is created")
# Create mapping
CREATE_MAPPING_COMMAND = """.create table RepoContributors ingestion csv mapping 'RepoContributors_CSV_Mapping' '[{"Name": "Article","datatype": "string","Ordinal": 0},{"Name": "Contributors","datatype": "int64","Ordinal": 1},{"Name": "Data","datatype": "string","Ordinal": 2}]'"""
RESPONSE = KUSTO_CLIENT.execute_mgmt(DATABASE, CREATE_MAPPING_COMMAND)
print("mapping is created")
# Ingest
# The authentication method will be taken from the chosen KustoConnectionStringBuilder.
ingestion_props = IngestionProperties(
database="DevRelWorkArea",
table="RepoContributors",
dataFormat=DataFormat.CSV,
ingestByTags=[Tag],
dropByTags=[Tag],
mappingReference=DESTINATION_TABLE_COLUMN_MAPPING,
reportLevel=ReportLevel.FailuresAndSuccesses,
additionalProperties={'ignoreFirstRecord': 'true'})
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(
INGESTCLUSTER)
client = KustoIngestClient(kcsb)
# ingest from file
file_descriptor = FileDescriptor(
r"D:\test\Results\log_data_merge\merge_microsoftdocs_sql-docs-pr.txt",
3333) # 3333 is the raw size of the data in bytes.
client.ingest_from_file(file_descriptor,
ingestion_properties=ingestion_props)
# if status updates are required, something like this can be done
return 1
def test_admin_then_query(self, mock_post):
"""Tests partial results."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = ".show tables | project DatabaseName, TableName"
response = client.execute_mgmt("PythonTest", query)
self.assertEqual(response.errors_count, 0)
self.assertEqual(len(response), 4)
results = list(response.primary_results[0])
self.assertEqual(len(results), 2)
self.assertEqual(response[0].table_kind, WellKnownDataSet.PrimaryResult)
self.assertEqual(response[1].table_kind, WellKnownDataSet.QueryProperties)
self.assertEqual(response[2].table_kind, WellKnownDataSet.QueryCompletionInformation)
self.assertEqual(response[3].table_kind, WellKnownDataSet.TableOfContents)
def test_admin_then_query(self, mock_post):
"""Tests admin then query."""
client = KustoClient("https://somecluster.kusto.windows.net")
query = ".show tables | project DatabaseName, TableName"
response = client.execute_mgmt("PythonTest", query)
assert response.errors_count == 0
assert len(response) == 4
results = list(response.primary_results[0])
assert len(results) == 2
assert response[0].table_kind == WellKnownDataSet.PrimaryResult
assert response[1].table_kind == WellKnownDataSet.QueryProperties
assert response[
2].table_kind == WellKnownDataSet.QueryCompletionInformation
assert response[3].table_kind == WellKnownDataSet.TableOfContents
def test_sanity_control_command(self, mock_post, mock_aad):
"""Tests contol command."""
client = KustoClient("https://somecluster.kusto.windows.net")
response = client.execute_mgmt("NetDefaultDB", ".show version")
self.assertEqual(len(response), 1)
primary_table = response.primary_results[0]
row_count = 0
for _ in primary_table:
row_count += 1
self.assertEqual(row_count, 1)
result = primary_table[0]
self.assertEqual(result["BuildVersion"], "1.0.6693.14577")
self.assertEqual(
result["BuildTime"], datetime(year=2018, month=4, day=29, hour=8, minute=5, second=54, tzinfo=tzutc())
)
self.assertEqual(result["ServiceType"], "Engine")
self.assertEqual(result["ProductVersion"], "KustoMain_2018.04.29.5")
""".format(DESTINATION_TABLE=destination_table,
CREATE_COLUMN_LIST=obj.get('CREATE_COLUMN_LIST'))
check_for_mapping_command = """
.show table {DESTINATION_TABLE} ingestion csv mapping '{COLUMN_MAPPING_NAME}'
""".format(DESTINATION_TABLE=destination_table,
COLUMN_MAPPING_NAME=column_mapping_name)
create_mapping_command = """
.create table {DESTINATION_TABLE} ingestion csv mapping '{COLUMN_MAPPING_NAME}' '{COLUMN_MAPPING}'
""".format(DESTINATION_TABLE=destination_table,
COLUMN_MAPPING_NAME=column_mapping_name,
COLUMN_MAPPING=json.dumps(obj.get('COLUMN_MAPPING')))
# Drop table if exists, then create.
kusto_client.execute_mgmt(kusto_database, drop_table_if_exists_command)
kusto_client.execute_mgmt(kusto_database, create_table_command)
#NOTE: this may be backwards...
try:
response = kusto_client.execute_mgmt(kusto_database,
check_for_mapping_command)
except: # Because check_for_mapping_command should throw error if mapping already exists
response = kusto_client.execute_mgmt(kusto_database,
create_mapping_command)
ingestion_client = KustoIngestClient(kcsb_ingest)
# All ingestion properties: https://docs.microsoft.com/en-us/azure/kusto/management/data-ingestion/#ingestion-properties
ingestion_props = IngestionProperties(
reportLevel=reportLevel,
import argparse
from azure.kusto.data.request import KustoClient, KustoConnectionStringBuilder
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--cluster', '-cluster', help="Kusto cluster URL", type=str, required=True)
arg_parser.add_argument('--database', '-db', help="Kusto database", type=str, required=True)
args = arg_parser.parse_args()
cluster = args.cluster
database = args.database
# define the command to create our first table and mapping in the database
LANDING_TABLE_CMD = ".create-merge table landing (rawdata: dynamic)"
LANDING_TABLE_MAPPING_CMD = ".create table landing ingestion json mapping \"raw_mapping\" '[{\"column\":\"rawdata\",\"path\":\"$\"}]'"
# create a connection string with a device code (=interactive) authentication
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster)
# create the client and do the actual authentication
client = KustoClient(kcsb)
# run the command to create the table
response = client.execute_mgmt(database, LANDING_TABLE_CMD)
# run the command to create the table mapping
response = client.execute_mgmt(database, LANDING_TABLE_MAPPING_CMD)
