def main():
# Put your customer id and shared key below. For these info, see also
# https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api#sample-requests
customer_id = '<Customer ID aka WorkspaceID String>'
shared_key = '<Primary Key String>'
log_type = 'ApacheAccessLog'
json_records = []
json_records.append({
"Log_ID":
"5cdad72f-c848-4df0-8aaa-ffe033e75d57",
"date":
"2016-12-03 09:44:32 JST",
"processing_time":
"372",
"remote":
"101.202.74.59",
"user":
"******",
"method":
"GET / HTTP/1.1",
"status":
"304",
"size":
"-",
"referer":
"-",
"agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0"
})
json_records.append({
"Log_ID":
"7260iswx-8034-4cc3-uirtx-f068dd4cd659",
"date":
"2016-12-03 09:45:14 JST",
"processing_time":
"105",
"remote":
"201.78.74.59",
"user":
"******",
"method":
"GET /manager/html HTTP/1.1",
"status":
"200",
"size":
"-",
"referer":
"-",
"agent":
"Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0"
})
api = client.DataCollectorAPIClient(customer_id, shared_key)
response = api.post_data(log_type, json_records)
if (helper.is_success(response.status_code)):
print('Succeeded in posting data to Data Collector API!!')
else:
print("Failure: Error code:{}".format(response.status_code))
from datacollectorapi import helper
from ast import literal_eval
import os
# Create the QueueClient
con_string = os.environ['CONNECTION_STRING'].rstrip("\n\r")
queue_name = os.environ['QUEUE_NAME'].rstrip("\n\r")
customer_id = os.environ['CUSTOMER_ID'].rstrip("\n\r")
shared_key = os.environ['SHARED_KEY'].rstrip("\n\r")
log_type = os.environ['LOG_TYPE'].rstrip("\n\r") # 'SyslogTest'
batch_size = int(os.environ['BATCH_SIZE'].rstrip("\n\r"))
batch, batch_size = [], batch_size
print(con_string, queue_name, customer_id, shared_key, log_type, batch_size)
queue_client = QueueClient.from_connection_string(con_string, queue_name)
api = client.DataCollectorAPIClient(customer_id, shared_key)
while True:
# Receive the message from the queue
with queue_client.get_receiver() as queue_receiver:
#https://docs.microsoft.com/en-us/python/api/azure-servicebus/azure.servicebus.receive_handler.receiver?view=azure-python#fetch-next-max-batch-size-none--timeout-none-
#https://docs.microsoft.com/en-us/python/api/azure-servicebus/azure.servicebus.common.message.message?view=azure-python
messages = queue_receiver.fetch_next(timeout=3)
print("{} messages".format(len(messages)))
for message in messages:
message_eval = literal_eval(str(message))
print(message_eval)
batch.append(message_eval) #literal_eval()
message.complete()
print("batch {}".format(len(batch)))
if len(batch) >= batch_size:
print("SENT {}".format(len(batch)))