def upload(data_to_save):
from applicationinsights import TelemetryClient
from applicationinsights.exceptions import enable
client = TelemetryClient(INSTRUMENTATION_KEY)
enable(INSTRUMENTATION_KEY)
if in_diagnostic_mode():
sys.stdout.write('Telemetry upload begins\n')
try:
data_to_save = json.loads(data_to_save.replace("'", '"'))
except Exception as err: # pylint: disable=broad-except
if in_diagnostic_mode():
sys.stdout.write('{}/n'.format(str(err)))
sys.stdout.write('Raw [{}]/n'.format(data_to_save))
for record in data_to_save:
client.track_event(record['name'], record['properties'])
client.flush()
if in_diagnostic_mode():
json.dump(data_to_save, sys.stdout, indent=2, sort_keys=True)
sys.stdout.write('\nTelemetry upload completes\n')
def log_event(self, message):
""" Log simple message in strinbg """
obfuscate = ObfuscationUtility(self.seed)
client = TelemetryClient(obfuscate.deobfuscate_text(self.ai_code))
client.track_event(message)
client.flush()
def test_track_event_modifes_options(self):
def process(data, context):
context.user.id = "BOTUSER"
context.session.id = "BOTSESSION"
return True
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient(
'99999999-9999-9999-9999-999999999999',
channel.TelemetryChannel(context=None, queue=queue))
client.add_telemetry_processor(process)
client.context.device = None
client.context.properties['foo'] = 'bar'
client.track_event('test')
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags[
'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
def test_track_event_with_merged_context_properties_works_as_expected(
self):
key = '99999999-9999-9999-9999-999999999999'
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
chan = channel.TelemetryChannel(queue=queue)
chan.context.properties['foo'] = 'bar'
client1 = TelemetryClient(key, chan)
client1.context.device = None
client1.context.properties['x'] = 42
client2 = TelemetryClient(key, chan)
client2.context.device = None
client2.context.properties['x'] = 84
client1.track_event('test 1')
client1.flush()
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
actual = json.dumps(sender.data.write())
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 1", "properties": {"foo": "bar", "x": 42}}}}'
self.maxDiff = None
self.assertEqual(expected, actual)
client2.track_event('test 2')
client2.flush()
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
actual = json.dumps(sender.data.write())
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 2", "properties": {"foo": "bar", "x": 84}}}}'
self.assertEqual(expected, actual)
def gen_metadata_msg_enqueue_tasks(queue_msg_list: List[str],
queue_client_list: List[QueueClient],
tc: TelemetryClient) -> None:
""" Send queue messages to target queues """
tasks = []
for idx, queue_msg in enumerate(queue_msg_list):
output_obj = json.loads(queue_msg)
queue_index = idx % len(queue_client_list)
logging.debug(
f"{HEADER} Try to send message to ingest queue {queue_index}, queue_msg: {queue_msg}")
base64_message = base64.b64encode(queue_msg.encode('ascii')).decode('ascii')
file_url = output_obj['data']['url']
size = int(output_obj['data']['contentLength'])
tc.track_event(METADATA_HANDLE_EVENT_NAME,
{'FILE_URL': file_url},
{METADATA_HANDLE_EVENT_NAME + '_SIZE': size,
METADATA_HANDLE_EVENT_NAME + '_COUNT': 1})
# round robin to enqueue message
task = asyncio.ensure_future(send_queue_messages(
queue_client_list[queue_index], base64_message, queue_msg))
tasks.append(task)
tc.flush()
return tasks
def send_specific():
filename = filedialog.askopenfilename(initialdir="/",
title="selectfile",
filetypes=(("executables", "*.exe"),
("allfiles", "*.*")))
tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>')
tc.track_event(filename, {'foo': 'bar'}, {'baz': 42})
tc.flush()
messagebox.showinfo("updates", "sent")
class InsightsLogStream(MedexLogStream):
# InsightsLogStream sources code from
# https://microsoft.github.io/ApplicationInsights-Python/
def __init__(self, key):
from applicationinsights import TelemetryClient
self.tc = TelemetryClient(key)
def log(self, event_type, data):
self.tc.track_event(event_type, data)
self.tc.flush()
def TrackEvent(File, Event, startTime,endTime):
try:
tc = TelemetryClient(Constants.AppInsight_Key)
host_name = socket.gethostname()
host_ip = socket.gethostbyname(host_name)
tc.track_event(Event, { 'Server': host_name , 'IP': host_ip ,'Event':Event,'File':File}, { 'TimeTaken': TimeDiffMinutes(startTime,endTime)})
tc.flush()
except:
tc.track_exception()
def test_track_event_processor_filtered(self):
def process(data, context):
return False # Filter the event
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.add_telemetry_processor(process)
client.context.device = None
client.track_event('test', { 'foo': 'bar' }, { 'x': 42 })
client.flush()
self.assertEqual(None, sender.data)
def test_track_event_works_as_expected(self):
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.context.device = None
client.track_event('test', { 'foo': 'bar' }, { 'x': 42 })
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}, "measurements": {"x": 42}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
def test_track_event_works_as_expected(self):
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.context.device = None
client.track_event('test', { 'foo': 'bar' }, { 'x': 42 })
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}, "measurements": {"x": 42}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
def app_insights_track_event(self, name, props=None, measurments=None):
if self.inst_key:
tc = TelemetryClient(self.inst_key)
tc.context.instrumentation_key = self.inst_key
tc.context.application.id = 'autoscaler'
tc.context.application.ver = '0.3.0'
tc.context.device.id = 'k8s'
tc.context.user.id = 'autoscaler-serivce-account'
tc.track_event(name, props, measurments)
logger.debug('APP INSIGHTS: sent %s event', name)
tc.flush()
else:
logger.debug('APP_INSIGHTS not configured.')
def main(event: func.EventHubEvent):
handler = LoggingHandler(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'])
logging.basicConfig(handlers=[ handler ], format='%(levelname)s: %(message)s', level=logging.DEBUG)
tc = TelemetryClient(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'])
tc.track_event("Incoming event")
tc.flush()
logging.info('Function triggered to process a message: %s', event)
logging.info(' body: %s', event.get_body())
logging.info(' EnqueuedTimeUtc: %s', event.enqueued_time)
logging.info(' SequenceNumber: %s', event.sequence_number)
logging.info(' Offset: %s', event.offset)
logging.info(' Partition: %s', event.partition_key)
logging.info(' Metadata: %s', event.iothub_metadata)
table_service = TableService(connection_string=os.environ['AzureTableConnectionString'])
for datapoint in json.loads(event.get_body()):
# Expected data format:
# {"timestamp": 1564598054, "deviceid": "Node1", "scale": 2, "temperature": 1.1,"weight": 10000}
if datapoint is not None and 'deviceid' in datapoint and \
'timestamp' in datapoint and 'scale' in datapoint and \
'weight' in datapoint:
logging.debug(' datapoint: %s', (datapoint))
# deviceid is used as partition key.
# {timestamp}-{scale} is used as RowKey
# timestamp and scale number are duplicated as an int columns
# to keep them searchable. The rest of the datapoint elements
# are added as columns as well.
history = {}
history['PartitionKey'] = datapoint.pop('deviceid')
history['RowKey'] = str(datapoint['timestamp']) + '-' + str(datapoint['scale'])
history.update(datapoint.items())
logging.debug('history: %s' % (history))
table_service.insert_entity(TABLE_NAME_HISTORICAL_DATA, history)
logging.info('Added historical table data: %s', (history))
# Touch/create the row in the config table for each reported scale with latest weight
configupdate = {}
configupdate['PartitionKey'] = history['PartitionKey']
configupdate['RowKey'] = str(history['scale'])
configupdate['weight'] = history['weight']
if 'temperature' in history:
configupdate['temperature'] = history['temperature']
logging.info('config update: %s' % (configupdate))
logging.info('Writing to table: %s' % (TABLE_NAME_CONFIGURATION))
table_service.insert_or_merge_entity(TABLE_NAME_CONFIGURATION, configupdate)
logging.info('Updated configuration table entry: %s', (configupdate))
else:
logging.info(' Invalid datapoint: %s', (datapoint))
def _send_feedback(score, response_what_changes, response_do_well, email_address):
from applicationinsights import TelemetryClient
tc = TelemetryClient(INSTRUMENTATION_KEY)
tc.context.application.ver = core_version
version_components, version_python = _get_version_info()
tc.track_event(
EVENT_NAME,
{'response_what_changes': response_what_changes,
'response_do_well': response_do_well,
'response_email_address': email_address,
'version_components': version_components,
'version_python': version_python},
{'response_net_promoter_score': score})
tc.flush()
def test_track_event_processor_filtered(self):
def process(data, context):
return False # Filter the event
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient(
'99999999-9999-9999-9999-999999999999',
channel.TelemetryChannel(context=None, queue=queue))
client.add_telemetry_processor(process)
client.context.device = None
client.track_event('test', {'foo': 'bar'}, {'x': 42})
client.flush()
self.assertEqual(None, sender.data)
def _send_feedback(score, response_what_changes, response_do_well, email_address):
from applicationinsights import TelemetryClient
tc = TelemetryClient(INSTRUMENTATION_KEY)
tc.context.application.ver = core_version
version_components, version_python = _get_version_info()
tc.track_event(
EVENT_NAME,
{'response_what_changes': response_what_changes,
'response_do_well': response_do_well,
'response_email_address': email_address,
'version_components': version_components,
'version_python': version_python},
{'response_net_promoter_score': score})
tc.flush()
def test_track_event_with_common_processor_two_clients(self):
def process(data, context):
data.properties["NEW_PROP"] = "MYPROP"
context.user.id = "BOTUSER"
context.session.id = "BOTSESSION"
return True
key = '99999999-9999-9999-9999-999999999999'
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
chan = channel.TelemetryChannel(queue=queue)
chan.context.properties['foo'] = 'bar'
client1 = TelemetryClient(key, chan)
client1.add_telemetry_processor(process)
client1.context.device = None
client1.context.properties['x'] = 42
client2 = TelemetryClient(key, chan)
client2.add_telemetry_processor(process)
client2.context.device = None
client2.context.properties['x'] = 84
client1.track_event('test 1')
client1.flush()
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags[
'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 1", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 42}}}}'
self.maxDiff = None
self.assertEqual(expected, actual)
client2.track_event('test 2')
client2.flush()
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags[
'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 2", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 84}}}}'
self.assertEqual(expected, actual)
def test_track_event_with_common_processor_two_clients(self):
def process(data, context):
data.properties["NEW_PROP"] = "MYPROP"
context.user.id = "BOTUSER"
context.session.id = "BOTSESSION"
return True
key = '99999999-9999-9999-9999-999999999999'
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
chan = channel.TelemetryChannel(queue=queue)
chan.context.properties['foo'] = 'bar'
client1 = TelemetryClient(key, chan)
client1.add_telemetry_processor(process)
client1.context.device = None
client1.context.properties['x'] = 42
client2 = TelemetryClient(key, chan)
client2.add_telemetry_processor(process)
client2.context.device = None
client2.context.properties['x'] = 84
client1.track_event('test 1')
client1.flush()
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 1", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 42}}}}'
self.maxDiff = None
self.assertEqual(expected, actual)
client2.track_event('test 2')
client2.flush()
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 2", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 84}}}}'
self.assertEqual(expected, actual)
def main(msg: func.QueueMessage) -> None:
"""
Main function, triggered by Azure Storage Queue, parsed queue content
:param msg: func.QueueMessage
:return: None
"""
logging.info('Python queue trigger function processed a queue item: %s',
msg.get_body().decode('utf-8'))
get_config_values()
# Get blob file content
content = json.loads(msg.get_body().decode('utf-8'))
filepath = content['data']['url']
container_name, blob_file_path = get_blob_info_from_url(filepath)
dest_container_name, dest_blob_file_path = get_new_blob_move_file_path(container_name, blob_file_path)
retry_times = get_blob_retry_times(filepath)
retry_times += 1
# Initialize Track Event/Metrics to App insight
tc = TelemetryClient(APP_INSIGHT_KEY)
tc.context.application.ver = '1.0'
tc.context.properties["PROCESS_PROGRAM"] = "XDR_SDL_INGESTION_ERR_HANDLER_V01A"
tc.context.properties["PROCESS_START"] = time.time()
# Do retry (move file to retry folder)
# TODO: Should filter out the non-retry case
logging.info("Retry the blob ingest to ADX, blob_path: %s", filepath)
retry_blob_ingest_to_adx(container_name, blob_file_path, dest_container_name, dest_blob_file_path)
if retry_times > MAX_INGEST_RETRIES_TIMES:
logging.error("Retry blob ingest to ADX hit the retries limit %s, blob_path: %s",
MAX_INGEST_RETRIES_TIMES, filepath)
tc.track_event(RETRY_END_IN_FAIL_EVENT_NAME,
{'FILE_PATH': filepath},
{RETRY_END_IN_FAIL_EVENT_NAME + '_COUNT': 1})
tc.flush()
return
tc.track_event(RETRY_EVENT_NAME,
{'FILE_PATH': filepath},
{RETRY_EVENT_NAME + '_COUNT': 1})
tc.flush()
logging.info("ADX error handler execution succeeded, blob path: %s, trial count: %s",
filepath, retry_times)
def test_track_event_with_context_properties_works_as_expected(self):
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.context.device = None
client.context.properties['foo'] = 'bar'
client.track_event('test')
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
def init():
global model, scaler, input_name, label_name, inputs_dc, prediction_dc, tc
# Add your telemetry key
tc = TelemetryClient('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx')
scaler_path = os.path.join(os.getenv('AZUREML_MODEL_DIR'),
'scaler/1/scaler.pkl')
# deserialize the model file back into a sklearn model
try:
scaler = joblib.load(scaler_path)
except Exception as e:
tc.track_event('FileNotFoundException', {'error_message': str(e)},
{'FileNotFoundError': 101})
tc.flush()
model_onnx = os.path.join(os.getenv('AZUREML_MODEL_DIR'),
'support-vector-classifier/1/svc.onnx')
try:
model = onnxruntime.InferenceSession(model_onnx, None)
except Exception as e:
tc.track_event('FileNotFoundException', {'error_message': str(e)},
{'FileNotFoundError': 101})
tc.flush()
input_name = model.get_inputs()[0].name
label_name = model.get_outputs()[0].name
# variables to monitor model input and output data
inputs_dc = ModelDataCollector("Support vector classifier model",
designation="inputs",
feature_names=[
"Temperature_C", "Humidity",
"Wind_speed_kmph",
"Wind_bearing_degrees", "Visibility_km",
"Pressure_millibars",
"Current_weather_condition"
])
prediction_dc = ModelDataCollector(
"Support vector classifier model",
designation="predictions",
feature_names=["Future_weather_condition"])
def upload(data_to_save):
data_to_save = json.loads(data_to_save)
for instrumentation_key in data_to_save:
client = TelemetryClient(instrumentation_key=instrumentation_key,
telemetry_channel=TelemetryChannel(queue=SynchronousQueue(LimitedRetrySender())))
enable(instrumentation_key)
for record in data_to_save[instrumentation_key]:
name = record['name']
raw_properties = record['properties']
properties = {}
measurements = {}
for k, v in raw_properties.items():
if isinstance(v, str):
properties[k] = v
else:
measurements[k] = v
client.track_event(name, properties, measurements)
client.flush()
def upload(data_to_save):
from applicationinsights import TelemetryClient
from applicationinsights.exceptions import enable
client = TelemetryClient(INSTRUMENTATION_KEY)
enable(INSTRUMENTATION_KEY)
if in_diagnostic_mode():
sys.stdout.write('Telemetry upload begins\n')
sys.stdout.write('Got data {}\n'.format(
json.dumps(json.loads(data_to_save), indent=2)))
try:
data_to_save = json.loads(data_to_save.replace("'", '"'))
except Exception as err: # pylint: disable=broad-except
if in_diagnostic_mode():
sys.stdout.write('{}/n'.format(str(err)))
sys.stdout.write('Raw [{}]/n'.format(data_to_save))
for record in data_to_save:
name = record['name']
raw_properties = record['properties']
properties = {}
measurements = {}
for k in raw_properties:
v = raw_properties[k]
if isinstance(v, six.string_types):
properties[k] = v
else:
measurements[k] = v
client.track_event(record['name'], properties, measurements)
if in_diagnostic_mode():
sys.stdout.write(
'\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.format(
name, json.dumps(properties, indent=2),
json.dumps(measurements, indent=2)))
client.flush()
if in_diagnostic_mode():
sys.stdout.write('\nTelemetry upload completes\n')
def upload(data_to_save):
if in_diagnostic_mode():
sys.stdout.write('Telemetry upload begins\n')
sys.stdout.write('Got data {}\n'.format(
json.dumps(json.loads(data_to_save), indent=2)))
try:
data_to_save = json.loads(data_to_save.replace("'", '"'))
except Exception as err: # pylint: disable=broad-except
if in_diagnostic_mode():
sys.stdout.write('ERROR: {}/n'.format(str(err)))
sys.stdout.write('Raw [{}]/n'.format(data_to_save))
for instrumentation_key in data_to_save:
client = TelemetryClient(
instrumentation_key=instrumentation_key,
telemetry_channel=TelemetryChannel(
queue=SynchronousQueue(LimitedRetrySender())))
enable(instrumentation_key)
for record in data_to_save[instrumentation_key]:
name = record['name']
raw_properties = record['properties']
properties = {}
measurements = {}
for k, v in raw_properties.items():
if isinstance(v, six.string_types):
properties[k] = v
else:
measurements[k] = v
client.track_event(record['name'], properties, measurements)
if in_diagnostic_mode():
sys.stdout.write(
'\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.
format(name, json.dumps(properties, indent=2),
json.dumps(measurements, indent=2)))
client.flush()
if in_diagnostic_mode():
sys.stdout.write('\nTelemetry upload completes\n')
class Telemetry:
def __init__(self):
# State of the telemetry
self.__enable_telemetry = True
self.telemetry_client = TelemetryClient("__AIKEY__")
self.telemetry_state = dict.fromkeys(
[name for name, _ in TelemetryEvent.__members__.items()], False)
self.extension_name = "Device Simulator Express"
def send_telemetry(self, event_name: TelemetryEvent):
if (self.__enable_telemetry and self.telemetry_available()
and not self.telemetry_state[event_name.name]
and not sys.platform.startswith(CONSTANTS.MAC_OS)):
self.telemetry_client.track_event(
f"{self.extension_name}/{event_name.value}")
self.telemetry_client.flush()
self.telemetry_state[event_name.name] = True
def telemetry_available(self):
return self.telemetry_client.context.instrumentation_key == "__AIKEY__"
def send_telemetry_best_attempt():
"""
Sends telemetry data from file TELEMETRY_FILE_PATH. The data is preserved if the process
is terminated before telemetry is sent, since removal of the TELEMETRY_FILE_PATH is called
only once that is completed.
There is a possibility of duplicate data, since if the telemetry data is sent out, and the
process is terminated before the file is deleted, we may end up resending.
This is acceptable since it will be rare, and since we have unique IDs for each telemetry
event.
There is a possibility that the process is terminated after telemetry send and file delete,
but before we can resent the telemetry send retry counter. This case will also be rare.
It will also take care of itself over time, since we will keep filling up new data, and
eventually it will send. We may lose some data, but that can be fixed in future changes.
We are also not currently setting time on the telemetry. We are counting the time as the time
sent. There is an entry to track the time, but the default sorting will not work with time
necessarily. We accept this right now, since there is not much we do with time data.
:return: None
"""
telemetry_tuples = read_telemetry_entries()
telemetry_client = TelemetryClient(INSTRUMENTATION)
for tup in telemetry_tuples:
telemetry_client.track_event(tup[0], tup[1])
# This will never end if there is no internet connection, for example.
telemetry_client.flush()
# After send has completed, delete the file
try:
os.remove(TELEMETRY_FILE_PATH)
except: # pylint: disable=bare-except
pass
# After send has completed, clear the telemetry retry counter
set_telemetry_send_retry_count(0)
def upload(data_to_save):
from applicationinsights import TelemetryClient
from applicationinsights.exceptions import enable
client = TelemetryClient(INSTRUMENTATION_KEY)
enable(INSTRUMENTATION_KEY)
if in_diagnostic_mode():
sys.stdout.write('Telemetry upload begins\n')
sys.stdout.write('Got data {}\n'.format(json.dumps(json.loads(data_to_save), indent=2)))
try:
data_to_save = json.loads(data_to_save.replace("'", '"'))
except Exception as err: # pylint: disable=broad-except
if in_diagnostic_mode():
sys.stdout.write('{}/n'.format(str(err)))
sys.stdout.write('Raw [{}]/n'.format(data_to_save))
for record in data_to_save:
name = record['name']
raw_properties = record['properties']
properties = {}
measurements = {}
for k in raw_properties:
v = raw_properties[k]
if isinstance(v, six.string_types):
properties[k] = v
else:
measurements[k] = v
client.track_event(record['name'], properties, measurements)
if in_diagnostic_mode():
sys.stdout.write('\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.format(
name, json.dumps(properties, indent=2), json.dumps(measurements, indent=2)))
client.flush()
if in_diagnostic_mode():
sys.stdout.write('\nTelemetry upload completes\n')
class Telemetry:
def __init__(self):
try:
self.telemetry = TelemetryClient(IKEY)
if os.path.exists("telemetry.config"):
config_file = open("telemetry.config", "r")
if config_file.read() == "1":
self.enable_telemetry = True
else:
self.enable_telemetry = False
else:
self.enable_telemetry = self._query_yes_no(PROMPT_TEXT)
config_file = open("telemetry.config", "w")
if self.enable_telemetry:
config_file.write("1")
self.telemetry.track_event("yes", {"device": DEVICE, "language": LANGUAGE})
else:
config_file.write("0")
self.telemetry.context.location.ip = "0.0.0.0"
self.telemetry.track_event("no", {"device": DEVICE, "language": LANGUAGE})
self.telemetry.flush()
except:
pass
def send_telemetry_data(self, iot_hub_name, event, message):
try:
if self.enable_telemetry:
hash_mac = self._get_mac_hash()
hash_iot_hub_name = hashlib.sha256(iot_hub_name.encode("utf-8")).hexdigest()
self.telemetry.track_event(event, {"iothub": hash_iot_hub_name, "message": message,
"language": LANGUAGE, "device": DEVICE, "mac": hash_mac,
"osType": platform.system(), "osPlatform": platform.dist()[0],
"osRelease": platform.dist()[1]})
self.telemetry.flush()
except:
pass
def _get_mac_hash(self):
mac = ":".join(re.findall("..", "%012x" % uuid.getnode()))
return hashlib.sha256(mac.encode("utf-8")).hexdigest()
def _query_yes_no(self, question):
global input
default = "y"
valid = {"y": True, "n": False}
prompt = " [Y/n] "
while True:
sys.stdout.write(question + prompt)
try:
input = raw_input
except NameError:
pass
choice = input().lower()
if default is not None and choice == "":
return valid[default]
elif choice in valid:
return valid[choice]
else:
sys.stdout.write("Please respond with 'y' or 'n' ")
class Telemetry:
def __init__(self):
try:
self.telemetry = TelemetryClient(IKEY)
if os.path.exists("telemetry.config"):
config_file = open("telemetry.config", "r")
if config_file.read() == "1":
self.enable_telemetry = True
else:
self.enable_telemetry = False
else:
self.enable_telemetry = self._query_yes_no(PROMPT_TEXT)
config_file = open("telemetry.config", "w")
if self.enable_telemetry:
config_file.write("1")
self.telemetry.track_event("yes", {"device": DEVICE, "language": LANGUAGE})
else:
config_file.write("0")
self.telemetry.context.location.ip = "0.0.0.0"
self.telemetry.track_event("no", {"device": DEVICE, "language": LANGUAGE})
self.telemetry.flush()
except:
pass
def send_telemetry_data(self, iot_hub_name, event, message):
try:
if self.enable_telemetry:
hash_mac = self._get_mac_hash()
hash_iot_hub_name = hashlib.sha256(iot_hub_name.encode("utf-8")).hexdigest()
self.telemetry.track_event(event, {"iothub": hash_iot_hub_name, "message": message,
"language": LANGUAGE, "device": DEVICE, "mac": hash_mac,
"osType": platform.system(), "osPlatform": platform.dist()[0],
"osRelease": platform.dist()[1]})
self.telemetry.flush()
except:
pass
def _get_mac_hash(self):
mac = ":".join(re.findall("..", "%012x" % uuid.getnode()))
return hashlib.sha256(mac.encode("utf-8")).hexdigest()
def _query_yes_no(self, question):
global input
default = "y"
valid = {"y": True, "n": False}
prompt = " [Y/n] "
while True:
sys.stdout.write(question + prompt)
try:
input = raw_input
except NameError:
pass
choice = input().lower()
if default is not None and choice == "":
return valid[default]
elif choice in valid:
return valid[choice]
else:
sys.stdout.write("Please respond with 'y' or 'n' ")
def upload(data_to_save):
if in_diagnostic_mode():
sys.stdout.write('Telemetry upload begins\n')
sys.stdout.write('Got data {}\n'.format(json.dumps(json.loads(data_to_save), indent=2)))
try:
data_to_save = json.loads(data_to_save.replace("'", '"'))
except Exception as err: # pylint: disable=broad-except
if in_diagnostic_mode():
sys.stdout.write('ERROR: {}/n'.format(str(err)))
sys.stdout.write('Raw [{}]/n'.format(data_to_save))
for instrumentation_key in data_to_save:
client = TelemetryClient(instrumentation_key=instrumentation_key,
telemetry_channel=TelemetryChannel(queue=SynchronousQueue(LimitedRetrySender())))
enable(instrumentation_key)
for record in data_to_save[instrumentation_key]:
name = record['name']
raw_properties = record['properties']
properties = {}
measurements = {}
for k, v in raw_properties.items():
if isinstance(v, six.string_types):
properties[k] = v
else:
measurements[k] = v
client.track_event(record['name'], properties, measurements)
if in_diagnostic_mode():
sys.stdout.write(
'\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.format(name, json.dumps(properties, indent=2),
json.dumps(measurements, indent=2)))
client.flush()
if in_diagnostic_mode():
sys.stdout.write('\nTelemetry upload completes\n')
def test_track_event_processor_modifies_data(self):
def process(data, context):
data.name = "WOWTESTING"
data.ver = 30000
return True
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.add_telemetry_processor(process)
client.context.device = None
client.track_event('test', { 'foo': 'bar' }, { 'x': 42 })
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 30000, "name": "WOWTESTING", "properties": {"foo": "bar"}, "measurements": {"x": 42}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
class AppInsightsHook:
def __init__(self):
self.tc = TelemetryClient(APP_INSIGHTS_KEY)
self.user_id = USER_ID
self.app_version = pkg_resources.require('oml')[0].version
self.python_version = platform.python_version()
self.platform = platform.platform()
self.timestamp = str(datetime.utcnow())
def track_event(self, event_type, action):
"""Sends Custom Event to Application Insights"""
event_data = {
'user_id': self.user_id,
'action': action,
'app_version': self.app_version,
'event_type': event_type,
'python_version': self.python_version,
'platform': self.platform,
'timestamp': self.timestamp
}
self.tc.track_event(APP_INSIGHTS_EVENT_TITLE, event_data)
self.tc.flush()
}
return jsonify({'bird': response}), 201
@app.route('/', methods=['GET'])
def get_bird():
return 'Version 1'
if __name__ == '__main__':
t0 = time.time()
tc = TelemetryClient(os.environ["TELEMETRY"])
# gpu based
cfg.CUDA = os.environ["GPU"].lower() == 'true'
tc.track_event('container initializing', {"CUDA": str(cfg.CUDA)})
# load word dictionaries
wordtoix, ixtoword = word_index()
# lead models
text_encoder, netG = models(len(wordtoix))
# load blob service
blob_service = BlockBlobService(account_name='attgan',
account_key=os.environ["BLOB_KEY"])
seed = 100
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if cfg.CUDA:
torch.cuda.manual_seed_all(seed)
load_id, loaded_on)
# Insert new rows
nr_fact_parking.write.mode("append").insertInto("dw.fact_parking")
# COMMAND ----------
# MAGIC %md
# MAGIC ### Metrics
# COMMAND ----------
new_dim_parkingbay_count = spark.read.table("dw.dim_parking_bay").count()
new_dim_location_count = spark.read.table("dw.dim_location").count()
new_dim_st_marker_count = spark.read.table("dw.dim_st_marker").count()
nr_fact_parking_count = nr_fact_parking.count()
tc.track_event('Transform : Completed load',
properties={'load_id': load_id},
measurements={
'new_dim_parkingbay_count': new_dim_parkingbay_count,
'new_dim_location_count': new_dim_location_count,
'new_dim_st_marker_count': new_dim_st_marker_count,
'newrecords_fact_parking_count': nr_fact_parking_count
})
tc.flush()
# COMMAND ----------
dbutils.notebook.exit("success")
def ocr(body, response, language: "The language(s) to use for OCR" = "eng"):
logger.info(os.environ)
azureAccountName = os.environ['SCANOCR_STORAGE_ACCOUNT_NAME']
azureAccountKey = os.environ['SCANOCR_STORAGE_ACCOUNT_KEY']
appInsightsTelemetryKey = os.environ['SCANOCR_APP_INSIGHTS_TELEMETRY_KEY']
fileSetId = body["file_set_id"]
zipFileUrl = body["zip_file_url"]
fileName = body["file_name"]
# initiate app insights
tc = TelemetryClient(appInsightsTelemetryKey)
tc.context.operation.id = str(uuid.uuid4())
tc.track_event('ocr', {
'zip_file_url': zipFileUrl,
'file_set_id': fileSetId
})
tc.flush()
# download zip, extract
zipRequestBody = requests.get(zipFileUrl)
z = zipfile.ZipFile(io.BytesIO(zipRequestBody.content))
tempDir = '/tmp/' + fileSetId + '/' + fileName
if (os.path.isdir(tempDir)):
shutil.rmtree(tempDir)
z.extractall(tempDir)
# grab all PNG images from zip extract results
image_files = []
for root, dirs, files in os.walk(tempDir):
for file in files:
if file.endswith(".png"):
image_files.append(os.path.join(root, file))
# log file count to app insights
tc.track_event('ocr_zip_extracted', {'file_count': len(image_files)})
tc.flush()
with open(tempDir + '/output.pdf', 'w+') as output:
# convert PNGs to (non-OCR) PDF
pdf_bytes = img2pdf.convert(image_files)
file = open(tempDir + '/input.pdf', "wb")
file.write(pdf_bytes)
# log progress to app insights
tc.track_event('ocr_pdf_created')
tc.flush()
# launch OCR process
proc = subprocess.Popen("ocrmypdf --jobs 4 --output-type pdf " +
tempDir + "/input.pdf " + tempDir +
"/output.pdf",
stdin=subprocess.PIPE,
shell=True)
# wait for OCR completion
code = proc.wait()
# log OCR completion to app insights
tc.track_event('ocr_output_pdf_complete')
tc.flush()
# upload resulting PDF to Azure
blob_service = BlockBlobService(account_name=azureAccountName,
account_key=azureAccountKey)
blob_service.create_blob_from_path(
'images/' + fileSetId,
fileName + ".pdf",
tempDir + '/output.pdf',
content_settings=ContentSettings(content_type='application/pdf'))
# log upload completion to app insights
tc.track_event('ocr_uploaded_to_azure')
tc.flush()
# obtain download signature from Azure
sas_url = blob_service.generate_blob_shared_access_signature(
'images/' + fileSetId, fileName + "pdf", BlobPermissions.READ,
datetime.utcnow() + timedelta(hours=12))
download_url = 'https://' + azureAccountName + '.blob.core.windows.net/' + 'images/' + fileSetId + '/' + fileName + ".pdf" + '?' + sas_url
# return results
return {
'filename': zipFileUrl,
'files': z.namelist(),
'download_url': download_url
}
def send_events(client: TelemetryClient, num_events: int):
for _ in range(num_events):
event = generate_event_name()
properties = generate_event_properties()
client.track_event(event, properties)
LOG.info('sent event %s %r', event, properties)
def process(filesrootfolder, forceinsert):
# Create process id as identify of this process
process_id = time.time()
tc = TelemetryClient('')
tc.context.application.ver = '1.0'
tc.context.properties["PROCESS_PROGRAM"] = "BATCH_CSV_V001a"
tc.context.properties["PROCESS_START"] = time.time()
tc.context.properties["DATA_FOLDER"] = filesrootfolder
tc.context.properties["PROCESS_ID"] = process_id
tc.track_trace('STRAT RUN BATHCH INGEST CSV DATA from folder ' +
filesrootfolder)
tc.track_event('BATHCH_INGEST_CSV_START', {
'PROCESS_ID': process_id,
'DATA_FOLDER': filesrootfolder
}, {})
tc.flush()
tc.flush()
#print (vm_uuid,deploy_uuid,config_uuid)
# Prepare COSMOS Link
url = COSMOS_URL
#key = os.environ['ACCOUNT_KEY']
key = COSMOS_KEY
client = cosmos_client.CosmosClient(url, {'masterKey': key})
database_id = COSMOS_DATABASE
container_id = COSMOS_CONTAINER
database_link = 'dbs/' + database_id
collection_link = database_link + '/colls/' + container_id
doc_id = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_Metric'
doc_link = collection_link + '/docs/' + doc_id
options = {}
options['enableCrossPartitionQuery'] = True
options['maxItemCount'] = 5
options['partitionKey'] = vm_uuid
proc_log_doc = None
try:
proc_log_doc = client.ReadItem(doc_link, options)
except:
print("New Process Metric Doc")
if (proc_log_doc is not None):
print("Find Existing Metric Doc ")
if str(forceinsert).lower(
) != 'true': # Stop Proccess if data is already been proccessed
return 400, doc_id + " is already been processed"
else: # New process log
proc_log_doc = {}
proc_log_doc["PROCESSES"] = []
proc_log_doc["DOC_TYPE"] = "PROCESS_METRIC"
proc_log_doc["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a"
proc_log_doc['id'] = doc_id
tc.track_event('BATHCH_INGEST_METRIC_CSV', {'PROCESS_ID': process_id},
{'DATA_FOLDER': filesrootfolder})
#+'_'+config_uuid+'_'+deploy_uuid , { 'DATA_FOLDER': telemetriespath }
tc.flush()
proc_log_this = {}
proc_log_this["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a"
proc_log_this["PROCESS_START"] = time.time()
proc_log_this["DATA_FOLDER"] = filesrootfolder
proc_log_this[
'id'] = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_' + str(
process_id)
error_files, merged_files, source_files = merge_rename_core_columns_CSV(
vm_uuid, deploy_uuid, config_uuid, 'defualt_metrics_csv_001A', 0,
SOURCE_CSV_CONTAINER, filesrootfolder, FILE_OUTPUT_FOLDER, process_id)
# ToDo ...
proc_log_this["PROCESS_ID"] = process_id
proc_log_this["ERROR_SOURCE_FILES_COUNT"] = len(error_files)
proc_log_this["SOURCE_FILES_COUNT"] = len(source_files)
tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_FILES_COUNT',
len(error_files))
tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_SOURCE_FILES_COUNT',
len(source_files))
tc.flush()
# print(str(len(error_files)),' ',str(len(merged_files)))
proc_log_this["PROCESS_END"] = time.time()
proc_log_this["STATUS"] = "OK"
proc_log_this["STATUS_MESSAGE"] = (
"It takes %s seconds to ingest CSV file from Blob Storage") % (
proc_log_this["PROCESS_END"] - proc_log_this["PROCESS_START"])
proc_log_doc["PROCESSES"].append(proc_log_this)
proc_log_doc['LATEST_UPDATE_TIMESTAMP'] = time.time()
# Update Process Log
client.UpsertItem(collection_link, proc_log_doc, options)
tc.track_trace('END RUN BATHCH INGEST METRIC CSV DATA from folder ' +
filesrootfolder)
tc.track_event('BATHCH_INGEST_METRIC_CSV_END', {
'PROCESS_ID': process_id,
'DATA_FOLDER': filesrootfolder
}, {
'DEFECT_FILES_COUNT': len(error_files),
'MERGED_FILES_COUNT': len(merged_files),
'SOURCE_FILES_COUNT': len(source_files)
})
tc.flush()
# COMMAND ----------
# MAGIC %md
# MAGIC ### Metrics
# COMMAND ----------
parkingbay_count = t_parkingbay_sdf.count()
sensordata_count = t_sensordata_sdf.count()
parkingbay_malformed_count = t_parkingbay_malformed_sdf.count()
sensordata_malformed_count = t_sensordata_malformed_sdf.count()
tc.track_event('Standardize : Completed load',
properties={
'parkingbay_filepath': parkingbay_filepath,
'sensors_filepath': sensors_filepath,
'load_id': load_id
},
measurements={
'parkingbay_count': parkingbay_count,
'sensordata_count': sensordata_count,
'parkingbay_malformed_count': parkingbay_malformed_count,
'sensordata_malformed_count': sensordata_malformed_count
})
tc.flush()
# COMMAND ----------
dbutils.notebook.exit("success")
def main(msg: func.QueueMessage) -> None:
"""
Main function, triggered by Azure Storage Queue, parsed queue content and
try to download the databricks output metadata file to get each succefully processed file location.
Then enqueue to ingest queue for ingestion to ADX on later Azure function.
If the file is checkpoint compact file, the code will shrink the file size.
:param msg: func.QueueMessage
:return: None
"""
code_start_time = time.time()
logging.info('Python queue trigger function processed a queue item: %s',
msg.get_body().decode('utf-8'))
# modify the log level of azure sdk requests
logging.getLogger('azure').setLevel(logging.WARNING)
init_config_values()
tc = TelemetryClient(APPINSIGHTS_INSTRUMENTATIONKEY)
tc.context.application.ver = '1.0'
tc.context.properties["PROCESS_PROGRAM"] = PROCESS_PROGRAM_NAME
tc.context.properties["PROCESS_START"] = time.time()
# 1. Get trigger file content (rename event)
content_json = json.loads(msg.get_body().decode('utf-8'))
logging.info("meta-data event content: {}".format(msg.get_body().decode('utf-8')))
file_url = content_json['data']['destinationUrl']
logging.info(f"file_url: {file_url}")
event_time = content_json['eventTime']
# 2. Download metadata blob content
logging.info(f"{HEADER} Download blob file from {file_url}")
temp_blob_client = BlobClient.from_blob_url(blob_url=file_url, logging_enable=False)
blob_path = temp_blob_client.blob_name
container_name = temp_blob_client.container_name
try:
metadata_file_content = get_blob_content(container_name, blob_path)
except Exception:
logging.exception(f"Failed to download blob from url {file_url}")
raise
# 3. Parse split output file from the metadata
queue_msg_list = generate_metadata_queue_messages(event_time, metadata_file_content)
logging.info(
f"{HEADER} Generate metadata queue_messages from {file_url}, {len(queue_msg_list)} messages")
# 4. Loop to enqueue msg to ADX ingest queue
queue_client_list = []
for q_url in ADX_INGEST_QUEUE_URL_LIST:
queue_client = get_queue_client(q_url)
queue_client_list.append(queue_client)
asyncio.set_event_loop(asyncio.new_event_loop())
loop = asyncio.get_event_loop()
tasks = gen_metadata_msg_enqueue_tasks(queue_msg_list, queue_client_list, tc)
loop.run_until_complete(gather_with_concurrency(CONCURRENT_ENQUEUE_TASKS, tasks))
close_queue_clients(queue_client_list, loop)
loop.close()
logging.info(f"{HEADER} Done queuing up messages to Ingestion queue")
if file_url.endswith(".compact"): # reduce compact file size
update_blob_content(container_name,
blob_path,
get_shrinked_checkpoint_content(
metadata_file_content, MAX_COMPACT_FILE_RECORDS))
logging.info(f"{HEADER} Reduced checkpoint files {file_url}, max lines is {MAX_COMPACT_FILE_RECORDS}")
code_duration = time.time() - code_start_time
tc.track_event(METADATA_HANDLE_EVENT_NAME,
{'FILE_URL': file_url},
{METADATA_HANDLE_EVENT_NAME + '_DURATION_SEC': code_duration})
tc.flush()
