def upload(data_to_save): from applicationinsights import TelemetryClient from applicationinsights.exceptions import enable client = TelemetryClient(INSTRUMENTATION_KEY) enable(INSTRUMENTATION_KEY) if in_diagnostic_mode(): sys.stdout.write('Telemetry upload begins\n') try: data_to_save = json.loads(data_to_save.replace("'", '"')) except Exception as err: # pylint: disable=broad-except if in_diagnostic_mode(): sys.stdout.write('{}/n'.format(str(err))) sys.stdout.write('Raw [{}]/n'.format(data_to_save)) for record in data_to_save: client.track_event(record['name'], record['properties']) client.flush() if in_diagnostic_mode(): json.dump(data_to_save, sys.stdout, indent=2, sort_keys=True) sys.stdout.write('\nTelemetry upload completes\n')
def log_event(self, message): """ Log simple message in strinbg """ obfuscate = ObfuscationUtility(self.seed) client = TelemetryClient(obfuscate.deobfuscate_text(self.ai_code)) client.track_event(message) client.flush()
def test_track_event_modifes_options(self): def process(data, context): context.user.id = "BOTUSER" context.session.id = "BOTSESSION" return True sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient( '99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.add_telemetry_processor(process) client.context.device = None client.context.properties['foo'] = 'bar' client.track_event('test') client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags[ 'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
def test_track_event_with_merged_context_properties_works_as_expected( self): key = '99999999-9999-9999-9999-999999999999' sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) chan = channel.TelemetryChannel(queue=queue) chan.context.properties['foo'] = 'bar' client1 = TelemetryClient(key, chan) client1.context.device = None client1.context.properties['x'] = 42 client2 = TelemetryClient(key, chan) client2.context.device = None client2.context.properties['x'] = 84 client1.track_event('test 1') client1.flush() sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' actual = json.dumps(sender.data.write()) expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 1", "properties": {"foo": "bar", "x": 42}}}}' self.maxDiff = None self.assertEqual(expected, actual) client2.track_event('test 2') client2.flush() sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' actual = json.dumps(sender.data.write()) expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 2", "properties": {"foo": "bar", "x": 84}}}}' self.assertEqual(expected, actual)
def gen_metadata_msg_enqueue_tasks(queue_msg_list: List[str], queue_client_list: List[QueueClient], tc: TelemetryClient) -> None: """ Send queue messages to target queues """ tasks = [] for idx, queue_msg in enumerate(queue_msg_list): output_obj = json.loads(queue_msg) queue_index = idx % len(queue_client_list) logging.debug( f"{HEADER} Try to send message to ingest queue {queue_index}, queue_msg: {queue_msg}") base64_message = base64.b64encode(queue_msg.encode('ascii')).decode('ascii') file_url = output_obj['data']['url'] size = int(output_obj['data']['contentLength']) tc.track_event(METADATA_HANDLE_EVENT_NAME, {'FILE_URL': file_url}, {METADATA_HANDLE_EVENT_NAME + '_SIZE': size, METADATA_HANDLE_EVENT_NAME + '_COUNT': 1}) # round robin to enqueue message task = asyncio.ensure_future(send_queue_messages( queue_client_list[queue_index], base64_message, queue_msg)) tasks.append(task) tc.flush() return tasks
def send_specific(): filename = filedialog.askopenfilename(initialdir="/", title="selectfile", filetypes=(("executables", "*.exe"), ("allfiles", "*.*"))) tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') tc.track_event(filename, {'foo': 'bar'}, {'baz': 42}) tc.flush() messagebox.showinfo("updates", "sent")
class InsightsLogStream(MedexLogStream): # InsightsLogStream sources code from # https://microsoft.github.io/ApplicationInsights-Python/ def __init__(self, key): from applicationinsights import TelemetryClient self.tc = TelemetryClient(key) def log(self, event_type, data): self.tc.track_event(event_type, data) self.tc.flush()
def TrackEvent(File, Event, startTime,endTime): try: tc = TelemetryClient(Constants.AppInsight_Key) host_name = socket.gethostname() host_ip = socket.gethostbyname(host_name) tc.track_event(Event, { 'Server': host_name , 'IP': host_ip ,'Event':Event,'File':File}, { 'TimeTaken': TimeDiffMinutes(startTime,endTime)}) tc.flush() except: tc.track_exception()
def test_track_event_processor_filtered(self): def process(data, context): return False # Filter the event sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.add_telemetry_processor(process) client.context.device = None client.track_event('test', { 'foo': 'bar' }, { 'x': 42 }) client.flush() self.assertEqual(None, sender.data)
def test_track_event_works_as_expected(self): sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.context.device = None client.track_event('test', { 'foo': 'bar' }, { 'x': 42 }) client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}, "measurements": {"x": 42}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
def app_insights_track_event(self, name, props=None, measurments=None): if self.inst_key: tc = TelemetryClient(self.inst_key) tc.context.instrumentation_key = self.inst_key tc.context.application.id = 'autoscaler' tc.context.application.ver = '0.3.0' tc.context.device.id = 'k8s' tc.context.user.id = 'autoscaler-serivce-account' tc.track_event(name, props, measurments) logger.debug('APP INSIGHTS: sent %s event', name) tc.flush() else: logger.debug('APP_INSIGHTS not configured.')
def main(event: func.EventHubEvent): handler = LoggingHandler(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY']) logging.basicConfig(handlers=[ handler ], format='%(levelname)s: %(message)s', level=logging.DEBUG) tc = TelemetryClient(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY']) tc.track_event("Incoming event") tc.flush() logging.info('Function triggered to process a message: %s', event) logging.info(' body: %s', event.get_body()) logging.info(' EnqueuedTimeUtc: %s', event.enqueued_time) logging.info(' SequenceNumber: %s', event.sequence_number) logging.info(' Offset: %s', event.offset) logging.info(' Partition: %s', event.partition_key) logging.info(' Metadata: %s', event.iothub_metadata) table_service = TableService(connection_string=os.environ['AzureTableConnectionString']) for datapoint in json.loads(event.get_body()): # Expected data format: # {"timestamp": 1564598054, "deviceid": "Node1", "scale": 2, "temperature": 1.1,"weight": 10000} if datapoint is not None and 'deviceid' in datapoint and \ 'timestamp' in datapoint and 'scale' in datapoint and \ 'weight' in datapoint: logging.debug(' datapoint: %s', (datapoint)) # deviceid is used as partition key. # {timestamp}-{scale} is used as RowKey # timestamp and scale number are duplicated as an int columns # to keep them searchable. The rest of the datapoint elements # are added as columns as well. history = {} history['PartitionKey'] = datapoint.pop('deviceid') history['RowKey'] = str(datapoint['timestamp']) + '-' + str(datapoint['scale']) history.update(datapoint.items()) logging.debug('history: %s' % (history)) table_service.insert_entity(TABLE_NAME_HISTORICAL_DATA, history) logging.info('Added historical table data: %s', (history)) # Touch/create the row in the config table for each reported scale with latest weight configupdate = {} configupdate['PartitionKey'] = history['PartitionKey'] configupdate['RowKey'] = str(history['scale']) configupdate['weight'] = history['weight'] if 'temperature' in history: configupdate['temperature'] = history['temperature'] logging.info('config update: %s' % (configupdate)) logging.info('Writing to table: %s' % (TABLE_NAME_CONFIGURATION)) table_service.insert_or_merge_entity(TABLE_NAME_CONFIGURATION, configupdate) logging.info('Updated configuration table entry: %s', (configupdate)) else: logging.info(' Invalid datapoint: %s', (datapoint))
def _send_feedback(score, response_what_changes, response_do_well, email_address): from applicationinsights import TelemetryClient tc = TelemetryClient(INSTRUMENTATION_KEY) tc.context.application.ver = core_version version_components, version_python = _get_version_info() tc.track_event( EVENT_NAME, {'response_what_changes': response_what_changes, 'response_do_well': response_do_well, 'response_email_address': email_address, 'version_components': version_components, 'version_python': version_python}, {'response_net_promoter_score': score}) tc.flush()
def test_track_event_processor_filtered(self): def process(data, context): return False # Filter the event sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient( '99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.add_telemetry_processor(process) client.context.device = None client.track_event('test', {'foo': 'bar'}, {'x': 42}) client.flush() self.assertEqual(None, sender.data)
def test_track_event_with_common_processor_two_clients(self): def process(data, context): data.properties["NEW_PROP"] = "MYPROP" context.user.id = "BOTUSER" context.session.id = "BOTSESSION" return True key = '99999999-9999-9999-9999-999999999999' sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) chan = channel.TelemetryChannel(queue=queue) chan.context.properties['foo'] = 'bar' client1 = TelemetryClient(key, chan) client1.add_telemetry_processor(process) client1.context.device = None client1.context.properties['x'] = 42 client2 = TelemetryClient(key, chan) client2.add_telemetry_processor(process) client2.context.device = None client2.context.properties['x'] = 84 client1.track_event('test 1') client1.flush() sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags[ 'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 1", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 42}}}}' self.maxDiff = None self.assertEqual(expected, actual) client2.track_event('test 2') client2.flush() sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags[ 'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 2", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 84}}}}' self.assertEqual(expected, actual)
def test_track_event_with_common_processor_two_clients(self): def process(data, context): data.properties["NEW_PROP"] = "MYPROP" context.user.id = "BOTUSER" context.session.id = "BOTSESSION" return True key = '99999999-9999-9999-9999-999999999999' sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) chan = channel.TelemetryChannel(queue=queue) chan.context.properties['foo'] = 'bar' client1 = TelemetryClient(key, chan) client1.add_telemetry_processor(process) client1.context.device = None client1.context.properties['x'] = 42 client2 = TelemetryClient(key, chan) client2.add_telemetry_processor(process) client2.context.device = None client2.context.properties['x'] = 84 client1.track_event('test 1') client1.flush() sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 1", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 42}}}}' self.maxDiff = None self.assertEqual(expected, actual) client2.track_event('test 2') client2.flush() sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test 2", "properties": {"NEW_PROP": "MYPROP", "foo": "bar", "x": 84}}}}' self.assertEqual(expected, actual)
def main(msg: func.QueueMessage) -> None: """ Main function, triggered by Azure Storage Queue, parsed queue content :param msg: func.QueueMessage :return: None """ logging.info('Python queue trigger function processed a queue item: %s', msg.get_body().decode('utf-8')) get_config_values() # Get blob file content content = json.loads(msg.get_body().decode('utf-8')) filepath = content['data']['url'] container_name, blob_file_path = get_blob_info_from_url(filepath) dest_container_name, dest_blob_file_path = get_new_blob_move_file_path(container_name, blob_file_path) retry_times = get_blob_retry_times(filepath) retry_times += 1 # Initialize Track Event/Metrics to App insight tc = TelemetryClient(APP_INSIGHT_KEY) tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = "XDR_SDL_INGESTION_ERR_HANDLER_V01A" tc.context.properties["PROCESS_START"] = time.time() # Do retry (move file to retry folder) # TODO: Should filter out the non-retry case logging.info("Retry the blob ingest to ADX, blob_path: %s", filepath) retry_blob_ingest_to_adx(container_name, blob_file_path, dest_container_name, dest_blob_file_path) if retry_times > MAX_INGEST_RETRIES_TIMES: logging.error("Retry blob ingest to ADX hit the retries limit %s, blob_path: %s", MAX_INGEST_RETRIES_TIMES, filepath) tc.track_event(RETRY_END_IN_FAIL_EVENT_NAME, {'FILE_PATH': filepath}, {RETRY_END_IN_FAIL_EVENT_NAME + '_COUNT': 1}) tc.flush() return tc.track_event(RETRY_EVENT_NAME, {'FILE_PATH': filepath}, {RETRY_EVENT_NAME + '_COUNT': 1}) tc.flush() logging.info("ADX error handler execution succeeded, blob path: %s, trial count: %s", filepath, retry_times)
def test_track_event_with_context_properties_works_as_expected(self): sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.context.device = None client.context.properties['foo'] = 'bar' client.track_event('test') client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 2, "name": "test", "properties": {"foo": "bar"}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
def init(): global model, scaler, input_name, label_name, inputs_dc, prediction_dc, tc # Add your telemetry key tc = TelemetryClient('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx') scaler_path = os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'scaler/1/scaler.pkl') # deserialize the model file back into a sklearn model try: scaler = joblib.load(scaler_path) except Exception as e: tc.track_event('FileNotFoundException', {'error_message': str(e)}, {'FileNotFoundError': 101}) tc.flush() model_onnx = os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'support-vector-classifier/1/svc.onnx') try: model = onnxruntime.InferenceSession(model_onnx, None) except Exception as e: tc.track_event('FileNotFoundException', {'error_message': str(e)}, {'FileNotFoundError': 101}) tc.flush() input_name = model.get_inputs()[0].name label_name = model.get_outputs()[0].name # variables to monitor model input and output data inputs_dc = ModelDataCollector("Support vector classifier model", designation="inputs", feature_names=[ "Temperature_C", "Humidity", "Wind_speed_kmph", "Wind_bearing_degrees", "Visibility_km", "Pressure_millibars", "Current_weather_condition" ]) prediction_dc = ModelDataCollector( "Support vector classifier model", designation="predictions", feature_names=["Future_weather_condition"])
def upload(data_to_save): data_to_save = json.loads(data_to_save) for instrumentation_key in data_to_save: client = TelemetryClient(instrumentation_key=instrumentation_key, telemetry_channel=TelemetryChannel(queue=SynchronousQueue(LimitedRetrySender()))) enable(instrumentation_key) for record in data_to_save[instrumentation_key]: name = record['name'] raw_properties = record['properties'] properties = {} measurements = {} for k, v in raw_properties.items(): if isinstance(v, str): properties[k] = v else: measurements[k] = v client.track_event(name, properties, measurements) client.flush()
def upload(data_to_save): from applicationinsights import TelemetryClient from applicationinsights.exceptions import enable client = TelemetryClient(INSTRUMENTATION_KEY) enable(INSTRUMENTATION_KEY) if in_diagnostic_mode(): sys.stdout.write('Telemetry upload begins\n') sys.stdout.write('Got data {}\n'.format( json.dumps(json.loads(data_to_save), indent=2))) try: data_to_save = json.loads(data_to_save.replace("'", '"')) except Exception as err: # pylint: disable=broad-except if in_diagnostic_mode(): sys.stdout.write('{}/n'.format(str(err))) sys.stdout.write('Raw [{}]/n'.format(data_to_save)) for record in data_to_save: name = record['name'] raw_properties = record['properties'] properties = {} measurements = {} for k in raw_properties: v = raw_properties[k] if isinstance(v, six.string_types): properties[k] = v else: measurements[k] = v client.track_event(record['name'], properties, measurements) if in_diagnostic_mode(): sys.stdout.write( '\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.format( name, json.dumps(properties, indent=2), json.dumps(measurements, indent=2))) client.flush() if in_diagnostic_mode(): sys.stdout.write('\nTelemetry upload completes\n')
def upload(data_to_save): if in_diagnostic_mode(): sys.stdout.write('Telemetry upload begins\n') sys.stdout.write('Got data {}\n'.format( json.dumps(json.loads(data_to_save), indent=2))) try: data_to_save = json.loads(data_to_save.replace("'", '"')) except Exception as err: # pylint: disable=broad-except if in_diagnostic_mode(): sys.stdout.write('ERROR: {}/n'.format(str(err))) sys.stdout.write('Raw [{}]/n'.format(data_to_save)) for instrumentation_key in data_to_save: client = TelemetryClient( instrumentation_key=instrumentation_key, telemetry_channel=TelemetryChannel( queue=SynchronousQueue(LimitedRetrySender()))) enable(instrumentation_key) for record in data_to_save[instrumentation_key]: name = record['name'] raw_properties = record['properties'] properties = {} measurements = {} for k, v in raw_properties.items(): if isinstance(v, six.string_types): properties[k] = v else: measurements[k] = v client.track_event(record['name'], properties, measurements) if in_diagnostic_mode(): sys.stdout.write( '\nTrack Event: {}\nProperties: {}\nMeasurements: {}'. format(name, json.dumps(properties, indent=2), json.dumps(measurements, indent=2))) client.flush() if in_diagnostic_mode(): sys.stdout.write('\nTelemetry upload completes\n')
class Telemetry: def __init__(self): # State of the telemetry self.__enable_telemetry = True self.telemetry_client = TelemetryClient("__AIKEY__") self.telemetry_state = dict.fromkeys( [name for name, _ in TelemetryEvent.__members__.items()], False) self.extension_name = "Device Simulator Express" def send_telemetry(self, event_name: TelemetryEvent): if (self.__enable_telemetry and self.telemetry_available() and not self.telemetry_state[event_name.name] and not sys.platform.startswith(CONSTANTS.MAC_OS)): self.telemetry_client.track_event( f"{self.extension_name}/{event_name.value}") self.telemetry_client.flush() self.telemetry_state[event_name.name] = True def telemetry_available(self): return self.telemetry_client.context.instrumentation_key == "__AIKEY__"
def send_telemetry_best_attempt(): """ Sends telemetry data from file TELEMETRY_FILE_PATH. The data is preserved if the process is terminated before telemetry is sent, since removal of the TELEMETRY_FILE_PATH is called only once that is completed. There is a possibility of duplicate data, since if the telemetry data is sent out, and the process is terminated before the file is deleted, we may end up resending. This is acceptable since it will be rare, and since we have unique IDs for each telemetry event. There is a possibility that the process is terminated after telemetry send and file delete, but before we can resent the telemetry send retry counter. This case will also be rare. It will also take care of itself over time, since we will keep filling up new data, and eventually it will send. We may lose some data, but that can be fixed in future changes. We are also not currently setting time on the telemetry. We are counting the time as the time sent. There is an entry to track the time, but the default sorting will not work with time necessarily. We accept this right now, since there is not much we do with time data. :return: None """ telemetry_tuples = read_telemetry_entries() telemetry_client = TelemetryClient(INSTRUMENTATION) for tup in telemetry_tuples: telemetry_client.track_event(tup[0], tup[1]) # This will never end if there is no internet connection, for example. telemetry_client.flush() # After send has completed, delete the file try: os.remove(TELEMETRY_FILE_PATH) except: # pylint: disable=bare-except pass # After send has completed, clear the telemetry retry counter set_telemetry_send_retry_count(0)
def upload(data_to_save): from applicationinsights import TelemetryClient from applicationinsights.exceptions import enable client = TelemetryClient(INSTRUMENTATION_KEY) enable(INSTRUMENTATION_KEY) if in_diagnostic_mode(): sys.stdout.write('Telemetry upload begins\n') sys.stdout.write('Got data {}\n'.format(json.dumps(json.loads(data_to_save), indent=2))) try: data_to_save = json.loads(data_to_save.replace("'", '"')) except Exception as err: # pylint: disable=broad-except if in_diagnostic_mode(): sys.stdout.write('{}/n'.format(str(err))) sys.stdout.write('Raw [{}]/n'.format(data_to_save)) for record in data_to_save: name = record['name'] raw_properties = record['properties'] properties = {} measurements = {} for k in raw_properties: v = raw_properties[k] if isinstance(v, six.string_types): properties[k] = v else: measurements[k] = v client.track_event(record['name'], properties, measurements) if in_diagnostic_mode(): sys.stdout.write('\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.format( name, json.dumps(properties, indent=2), json.dumps(measurements, indent=2))) client.flush() if in_diagnostic_mode(): sys.stdout.write('\nTelemetry upload completes\n')
class Telemetry: def __init__(self): try: self.telemetry = TelemetryClient(IKEY) if os.path.exists("telemetry.config"): config_file = open("telemetry.config", "r") if config_file.read() == "1": self.enable_telemetry = True else: self.enable_telemetry = False else: self.enable_telemetry = self._query_yes_no(PROMPT_TEXT) config_file = open("telemetry.config", "w") if self.enable_telemetry: config_file.write("1") self.telemetry.track_event("yes", {"device": DEVICE, "language": LANGUAGE}) else: config_file.write("0") self.telemetry.context.location.ip = "0.0.0.0" self.telemetry.track_event("no", {"device": DEVICE, "language": LANGUAGE}) self.telemetry.flush() except: pass def send_telemetry_data(self, iot_hub_name, event, message): try: if self.enable_telemetry: hash_mac = self._get_mac_hash() hash_iot_hub_name = hashlib.sha256(iot_hub_name.encode("utf-8")).hexdigest() self.telemetry.track_event(event, {"iothub": hash_iot_hub_name, "message": message, "language": LANGUAGE, "device": DEVICE, "mac": hash_mac, "osType": platform.system(), "osPlatform": platform.dist()[0], "osRelease": platform.dist()[1]}) self.telemetry.flush() except: pass def _get_mac_hash(self): mac = ":".join(re.findall("..", "%012x" % uuid.getnode())) return hashlib.sha256(mac.encode("utf-8")).hexdigest() def _query_yes_no(self, question): global input default = "y" valid = {"y": True, "n": False} prompt = " [Y/n] " while True: sys.stdout.write(question + prompt) try: input = raw_input except NameError: pass choice = input().lower() if default is not None and choice == "": return valid[default] elif choice in valid: return valid[choice] else: sys.stdout.write("Please respond with 'y' or 'n' ")
def upload(data_to_save): if in_diagnostic_mode(): sys.stdout.write('Telemetry upload begins\n') sys.stdout.write('Got data {}\n'.format(json.dumps(json.loads(data_to_save), indent=2))) try: data_to_save = json.loads(data_to_save.replace("'", '"')) except Exception as err: # pylint: disable=broad-except if in_diagnostic_mode(): sys.stdout.write('ERROR: {}/n'.format(str(err))) sys.stdout.write('Raw [{}]/n'.format(data_to_save)) for instrumentation_key in data_to_save: client = TelemetryClient(instrumentation_key=instrumentation_key, telemetry_channel=TelemetryChannel(queue=SynchronousQueue(LimitedRetrySender()))) enable(instrumentation_key) for record in data_to_save[instrumentation_key]: name = record['name'] raw_properties = record['properties'] properties = {} measurements = {} for k, v in raw_properties.items(): if isinstance(v, six.string_types): properties[k] = v else: measurements[k] = v client.track_event(record['name'], properties, measurements) if in_diagnostic_mode(): sys.stdout.write( '\nTrack Event: {}\nProperties: {}\nMeasurements: {}'.format(name, json.dumps(properties, indent=2), json.dumps(measurements, indent=2))) client.flush() if in_diagnostic_mode(): sys.stdout.write('\nTelemetry upload completes\n')
def test_track_event_processor_modifies_data(self): def process(data, context): data.name = "WOWTESTING" data.ver = 30000 return True sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.add_telemetry_processor(process) client.context.device = None client.track_event('test', { 'foo': 'bar' }, { 'x': 42 }) client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "EventData", "baseData": {"ver": 30000, "name": "WOWTESTING", "properties": {"foo": "bar"}, "measurements": {"x": 42}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
class AppInsightsHook: def __init__(self): self.tc = TelemetryClient(APP_INSIGHTS_KEY) self.user_id = USER_ID self.app_version = pkg_resources.require('oml')[0].version self.python_version = platform.python_version() self.platform = platform.platform() self.timestamp = str(datetime.utcnow()) def track_event(self, event_type, action): """Sends Custom Event to Application Insights""" event_data = { 'user_id': self.user_id, 'action': action, 'app_version': self.app_version, 'event_type': event_type, 'python_version': self.python_version, 'platform': self.platform, 'timestamp': self.timestamp } self.tc.track_event(APP_INSIGHTS_EVENT_TITLE, event_data) self.tc.flush()
} return jsonify({'bird': response}), 201 @app.route('/', methods=['GET']) def get_bird(): return 'Version 1' if __name__ == '__main__': t0 = time.time() tc = TelemetryClient(os.environ["TELEMETRY"]) # gpu based cfg.CUDA = os.environ["GPU"].lower() == 'true' tc.track_event('container initializing', {"CUDA": str(cfg.CUDA)}) # load word dictionaries wordtoix, ixtoword = word_index() # lead models text_encoder, netG = models(len(wordtoix)) # load blob service blob_service = BlockBlobService(account_name='attgan', account_key=os.environ["BLOB_KEY"]) seed = 100 random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) if cfg.CUDA: torch.cuda.manual_seed_all(seed)
load_id, loaded_on) # Insert new rows nr_fact_parking.write.mode("append").insertInto("dw.fact_parking") # COMMAND ---------- # MAGIC %md # MAGIC ### Metrics # COMMAND ---------- new_dim_parkingbay_count = spark.read.table("dw.dim_parking_bay").count() new_dim_location_count = spark.read.table("dw.dim_location").count() new_dim_st_marker_count = spark.read.table("dw.dim_st_marker").count() nr_fact_parking_count = nr_fact_parking.count() tc.track_event('Transform : Completed load', properties={'load_id': load_id}, measurements={ 'new_dim_parkingbay_count': new_dim_parkingbay_count, 'new_dim_location_count': new_dim_location_count, 'new_dim_st_marker_count': new_dim_st_marker_count, 'newrecords_fact_parking_count': nr_fact_parking_count }) tc.flush() # COMMAND ---------- dbutils.notebook.exit("success")
def ocr(body, response, language: "The language(s) to use for OCR" = "eng"): logger.info(os.environ) azureAccountName = os.environ['SCANOCR_STORAGE_ACCOUNT_NAME'] azureAccountKey = os.environ['SCANOCR_STORAGE_ACCOUNT_KEY'] appInsightsTelemetryKey = os.environ['SCANOCR_APP_INSIGHTS_TELEMETRY_KEY'] fileSetId = body["file_set_id"] zipFileUrl = body["zip_file_url"] fileName = body["file_name"] # initiate app insights tc = TelemetryClient(appInsightsTelemetryKey) tc.context.operation.id = str(uuid.uuid4()) tc.track_event('ocr', { 'zip_file_url': zipFileUrl, 'file_set_id': fileSetId }) tc.flush() # download zip, extract zipRequestBody = requests.get(zipFileUrl) z = zipfile.ZipFile(io.BytesIO(zipRequestBody.content)) tempDir = '/tmp/' + fileSetId + '/' + fileName if (os.path.isdir(tempDir)): shutil.rmtree(tempDir) z.extractall(tempDir) # grab all PNG images from zip extract results image_files = [] for root, dirs, files in os.walk(tempDir): for file in files: if file.endswith(".png"): image_files.append(os.path.join(root, file)) # log file count to app insights tc.track_event('ocr_zip_extracted', {'file_count': len(image_files)}) tc.flush() with open(tempDir + '/output.pdf', 'w+') as output: # convert PNGs to (non-OCR) PDF pdf_bytes = img2pdf.convert(image_files) file = open(tempDir + '/input.pdf', "wb") file.write(pdf_bytes) # log progress to app insights tc.track_event('ocr_pdf_created') tc.flush() # launch OCR process proc = subprocess.Popen("ocrmypdf --jobs 4 --output-type pdf " + tempDir + "/input.pdf " + tempDir + "/output.pdf", stdin=subprocess.PIPE, shell=True) # wait for OCR completion code = proc.wait() # log OCR completion to app insights tc.track_event('ocr_output_pdf_complete') tc.flush() # upload resulting PDF to Azure blob_service = BlockBlobService(account_name=azureAccountName, account_key=azureAccountKey) blob_service.create_blob_from_path( 'images/' + fileSetId, fileName + ".pdf", tempDir + '/output.pdf', content_settings=ContentSettings(content_type='application/pdf')) # log upload completion to app insights tc.track_event('ocr_uploaded_to_azure') tc.flush() # obtain download signature from Azure sas_url = blob_service.generate_blob_shared_access_signature( 'images/' + fileSetId, fileName + "pdf", BlobPermissions.READ, datetime.utcnow() + timedelta(hours=12)) download_url = 'https://' + azureAccountName + '.blob.core.windows.net/' + 'images/' + fileSetId + '/' + fileName + ".pdf" + '?' + sas_url # return results return { 'filename': zipFileUrl, 'files': z.namelist(), 'download_url': download_url }
def send_events(client: TelemetryClient, num_events: int): for _ in range(num_events): event = generate_event_name() properties = generate_event_properties() client.track_event(event, properties) LOG.info('sent event %s %r', event, properties)
def process(filesrootfolder, forceinsert): # Create process id as identify of this process process_id = time.time() tc = TelemetryClient('') tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = "BATCH_CSV_V001a" tc.context.properties["PROCESS_START"] = time.time() tc.context.properties["DATA_FOLDER"] = filesrootfolder tc.context.properties["PROCESS_ID"] = process_id tc.track_trace('STRAT RUN BATHCH INGEST CSV DATA from folder ' + filesrootfolder) tc.track_event('BATHCH_INGEST_CSV_START', { 'PROCESS_ID': process_id, 'DATA_FOLDER': filesrootfolder }, {}) tc.flush() tc.flush() #print (vm_uuid,deploy_uuid,config_uuid) # Prepare COSMOS Link url = COSMOS_URL #key = os.environ['ACCOUNT_KEY'] key = COSMOS_KEY client = cosmos_client.CosmosClient(url, {'masterKey': key}) database_id = COSMOS_DATABASE container_id = COSMOS_CONTAINER database_link = 'dbs/' + database_id collection_link = database_link + '/colls/' + container_id doc_id = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_Metric' doc_link = collection_link + '/docs/' + doc_id options = {} options['enableCrossPartitionQuery'] = True options['maxItemCount'] = 5 options['partitionKey'] = vm_uuid proc_log_doc = None try: proc_log_doc = client.ReadItem(doc_link, options) except: print("New Process Metric Doc") if (proc_log_doc is not None): print("Find Existing Metric Doc ") if str(forceinsert).lower( ) != 'true': # Stop Proccess if data is already been proccessed return 400, doc_id + " is already been processed" else: # New process log proc_log_doc = {} proc_log_doc["PROCESSES"] = [] proc_log_doc["DOC_TYPE"] = "PROCESS_METRIC" proc_log_doc["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a" proc_log_doc['id'] = doc_id tc.track_event('BATHCH_INGEST_METRIC_CSV', {'PROCESS_ID': process_id}, {'DATA_FOLDER': filesrootfolder}) #+'_'+config_uuid+'_'+deploy_uuid , { 'DATA_FOLDER': telemetriespath } tc.flush() proc_log_this = {} proc_log_this["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a" proc_log_this["PROCESS_START"] = time.time() proc_log_this["DATA_FOLDER"] = filesrootfolder proc_log_this[ 'id'] = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_' + str( process_id) error_files, merged_files, source_files = merge_rename_core_columns_CSV( vm_uuid, deploy_uuid, config_uuid, 'defualt_metrics_csv_001A', 0, SOURCE_CSV_CONTAINER, filesrootfolder, FILE_OUTPUT_FOLDER, process_id) # ToDo ... proc_log_this["PROCESS_ID"] = process_id proc_log_this["ERROR_SOURCE_FILES_COUNT"] = len(error_files) proc_log_this["SOURCE_FILES_COUNT"] = len(source_files) tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_FILES_COUNT', len(error_files)) tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_SOURCE_FILES_COUNT', len(source_files)) tc.flush() # print(str(len(error_files)),' ',str(len(merged_files))) proc_log_this["PROCESS_END"] = time.time() proc_log_this["STATUS"] = "OK" proc_log_this["STATUS_MESSAGE"] = ( "It takes %s seconds to ingest CSV file from Blob Storage") % ( proc_log_this["PROCESS_END"] - proc_log_this["PROCESS_START"]) proc_log_doc["PROCESSES"].append(proc_log_this) proc_log_doc['LATEST_UPDATE_TIMESTAMP'] = time.time() # Update Process Log client.UpsertItem(collection_link, proc_log_doc, options) tc.track_trace('END RUN BATHCH INGEST METRIC CSV DATA from folder ' + filesrootfolder) tc.track_event('BATHCH_INGEST_METRIC_CSV_END', { 'PROCESS_ID': process_id, 'DATA_FOLDER': filesrootfolder }, { 'DEFECT_FILES_COUNT': len(error_files), 'MERGED_FILES_COUNT': len(merged_files), 'SOURCE_FILES_COUNT': len(source_files) }) tc.flush()
# COMMAND ---------- # MAGIC %md # MAGIC ### Metrics # COMMAND ---------- parkingbay_count = t_parkingbay_sdf.count() sensordata_count = t_sensordata_sdf.count() parkingbay_malformed_count = t_parkingbay_malformed_sdf.count() sensordata_malformed_count = t_sensordata_malformed_sdf.count() tc.track_event('Standardize : Completed load', properties={ 'parkingbay_filepath': parkingbay_filepath, 'sensors_filepath': sensors_filepath, 'load_id': load_id }, measurements={ 'parkingbay_count': parkingbay_count, 'sensordata_count': sensordata_count, 'parkingbay_malformed_count': parkingbay_malformed_count, 'sensordata_malformed_count': sensordata_malformed_count }) tc.flush() # COMMAND ---------- dbutils.notebook.exit("success")
def main(msg: func.QueueMessage) -> None: """ Main function, triggered by Azure Storage Queue, parsed queue content and try to download the databricks output metadata file to get each succefully processed file location. Then enqueue to ingest queue for ingestion to ADX on later Azure function. If the file is checkpoint compact file, the code will shrink the file size. :param msg: func.QueueMessage :return: None """ code_start_time = time.time() logging.info('Python queue trigger function processed a queue item: %s', msg.get_body().decode('utf-8')) # modify the log level of azure sdk requests logging.getLogger('azure').setLevel(logging.WARNING) init_config_values() tc = TelemetryClient(APPINSIGHTS_INSTRUMENTATIONKEY) tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = PROCESS_PROGRAM_NAME tc.context.properties["PROCESS_START"] = time.time() # 1. Get trigger file content (rename event) content_json = json.loads(msg.get_body().decode('utf-8')) logging.info("meta-data event content: {}".format(msg.get_body().decode('utf-8'))) file_url = content_json['data']['destinationUrl'] logging.info(f"file_url: {file_url}") event_time = content_json['eventTime'] # 2. Download metadata blob content logging.info(f"{HEADER} Download blob file from {file_url}") temp_blob_client = BlobClient.from_blob_url(blob_url=file_url, logging_enable=False) blob_path = temp_blob_client.blob_name container_name = temp_blob_client.container_name try: metadata_file_content = get_blob_content(container_name, blob_path) except Exception: logging.exception(f"Failed to download blob from url {file_url}") raise # 3. Parse split output file from the metadata queue_msg_list = generate_metadata_queue_messages(event_time, metadata_file_content) logging.info( f"{HEADER} Generate metadata queue_messages from {file_url}, {len(queue_msg_list)} messages") # 4. Loop to enqueue msg to ADX ingest queue queue_client_list = [] for q_url in ADX_INGEST_QUEUE_URL_LIST: queue_client = get_queue_client(q_url) queue_client_list.append(queue_client) asyncio.set_event_loop(asyncio.new_event_loop()) loop = asyncio.get_event_loop() tasks = gen_metadata_msg_enqueue_tasks(queue_msg_list, queue_client_list, tc) loop.run_until_complete(gather_with_concurrency(CONCURRENT_ENQUEUE_TASKS, tasks)) close_queue_clients(queue_client_list, loop) loop.close() logging.info(f"{HEADER} Done queuing up messages to Ingestion queue") if file_url.endswith(".compact"): # reduce compact file size update_blob_content(container_name, blob_path, get_shrinked_checkpoint_content( metadata_file_content, MAX_COMPACT_FILE_RECORDS)) logging.info(f"{HEADER} Reduced checkpoint files {file_url}, max lines is {MAX_COMPACT_FILE_RECORDS}") code_duration = time.time() - code_start_time tc.track_event(METADATA_HANDLE_EVENT_NAME, {'FILE_URL': file_url}, {METADATA_HANDLE_EVENT_NAME + '_DURATION_SEC': code_duration}) tc.flush()