def main(event: func.EventGridEvent): result = json.dumps({ 'id': event.id, 'data': event.get_json(), 'topic': event.topic, 'subject': event.subject, 'event_type': event.event_type, }) logging.info('{} Python EventGrid trigger processed an event:{}'.format(LOG_MESSAGE_HEADER ,result)) get_config_values() regexp = re.compile(EVENT_SUBJECT_FILTER_REGEX) if regexp.search(event.subject): # Check if file path match criteria tc = TelemetryClient(APP_INSIGHT_ID) tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"]=PROCESS_PROGRAM_NAME tc.context.properties["PROCESS_START"]=time.time() tc.track_trace('{} STRAT RUN EVENTGRID INGEST TELEMETRY JSON DATA from folder {} '.format(LOG_MESSAGE_HEADER,result)) tc.flush() configfile_block_blob_service = BlockBlobService(account_name=CONFIG_FILE_BLOB_ACCOUNT, account_key=CONFIG_FILE_BLOB_KEY) cleanfile_block_blob_service = BlockBlobService(account_name=CLEAN_FILE_BLOB_ACCOUNT, account_key=CLEAN_FILE_BLOB_KEY) filepath=get_file_path(event.subject) container_name=get_container_name(event.subject) config_file_name=get_filename(filepath) logging.info('{} filepath: {}'.format(LOG_MESSAGE_HEADER,filepath)) logging.info('{} container_name: {}'.format(LOG_MESSAGE_HEADER, container_name)) config_file_name_utf8, filesize,vm_uuid,deploy_uuid,config_uuid=generate_UTF8_config_json(configfile_block_blob_service,CONFIG_FILE_CONTAINER,filepath,cleanfile_block_blob_service,CLEAN_FILE_CONTAINER ) ingest_to_ADX(config_file_name_utf8, cleanfile_block_blob_service, CLEAN_FILE_CONTAINER, CLEAN_FILE_BLOB_ACCOUNT,filesize,tc,vm_uuid,deploy_uuid,config_uuid) else: logging.info("{} Subject : {} does not match regular express {}. Skip process. ".format(LOG_MESSAGE_HEADER,event.subject,EVENT_SUBJECT_FILTER_REGEX))
def log(self, telemetry_instance): """ Log Telemetry instance """ obfuscate = ObfuscationUtility(self.seed) client = TelemetryClient(obfuscate.deobfuscate_text(self.ai_code)) client.track_trace(str(telemetry_instance)) client.flush()
def test_track_trace_works_as_expected(self): def process(data, context): data.properties["NEW_PROP"] = "MYPROP" context.user.id = "BOTUSER" context.session.id = "BOTSESSION" return True sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient( '99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.add_telemetry_processor(process) client.context.device = None client.track_trace('test', {'foo': 'bar'}, severity='WARNING') client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Message", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "MessageData", "baseData": {"ver": 2, "message": "test", "severityLevel": 2, "properties": {"NEW_PROP": "MYPROP", "foo": "bar"}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags[ 'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') metricspath = req.params.get('metricspath') forceinsert = req.params.get('forceinsert') if not metricspath: try: req_body = req.get_json() except ValueError: pass else: metricspath = req_body.get('metricspath') forceinsert = req_body.get('forceinsert') if forceinsert is None: forceinsert = '' if metricspath: scode = 400 msg = 'Something Wrong, this is Default Message' #print (metricspath) try: scode, msg = process(metricspath, forceinsert) except Exception as e: error_class = e.__class__.__name__ detail = e.args[0] cl, exc, tb = sys.exc_info() lastCallStack = traceback.extract_tb(tb)[-1] fileName = lastCallStack[0] lineNum = lastCallStack[1] funcName = lastCallStack[2] errMsg = "File \"{}\", line {}, in {}: [{}] {}".format( fileName, lineNum, funcName, error_class, detail) print("Unexpected error:", sys.exc_info()[0]) traceback.print_exc() msg = errMsg + traceback.format_exc() tc = TelemetryClient('') tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = "BATCH_CSV_V001a" tc.context.properties["DATA_FOLDER"] = metricspath tc.track_trace(msg) tc.flush() scode = 500 return func.HttpResponse(f"Proccessed {metricspath}! " + msg, status_code=scode) else: return func.HttpResponse( "Please pass a metricspath on the query string or in the request body", status_code=400)
class AI4EAppInsights(object): def __init__(self): self.grantee_key = None raw_key = getenv(CONF_KEY_GRANTEE, None) if (raw_key and len(raw_key.strip()) > 0): self.grantee_key = raw_key.strip() if (self.grantee_key): self.sender = AsynchronousSender() self.r_queue = AsynchronousQueue(self.sender) self.r_context = AI4ETelemetryContext() self.r_channel = TelemetryChannel(self.r_context, self.r_queue) self.appinsights_grantee_client = TelemetryClient( getenv(CONF_KEY_GRANTEE), self.r_channel) self.appinsights_ai4e_client = None if (getenv(CONF_KEY_AI4E)): self.appinsights_ai4e_client = TelemetryClient( getenv(CONF_KEY_AI4E), self.r_channel) def _log(self, message, sev, taskId=None, additionalProperties=None): if (self.grantee_key): if (taskId): if (additionalProperties is None): additionalProperties = {'task_id': taskId} else: additionalProperties['task_id'] = taskId self.appinsights_grantee_client.track_trace( message, severity=sev, properties=additionalProperties) self.appinsights_grantee_client.flush() if (self.appinsights_ai4e_client): self.appinsights_ai4e_client.track_trace( message, severity=sev, properties=additionalProperties) self.appinsights_ai4e_client.flush() def log_debug(self, message, taskId=None, additionalProperties=None): self._log(message, "DEBUG", taskId, additionalProperties) def log_info(self, message, taskId=None, additionalProperties=None): self._log(message, "INFO", taskId, additionalProperties) def log_error(self, message, taskId=None, additionalProperties=None): self._log(message, "ERROR", taskId, additionalProperties) def log_warn(self, message, taskId=None, additionalProperties=None): self._log(message, "WARNING", taskId, additionalProperties) def log_exception(self, message, taskId=None, additionalProperties=None): self._log(message, "CRITICAL", taskId, additionalProperties)
def test_track_trace_works_as_expected(self): sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.context.device = None client.track_trace('test', { 'foo': 'bar' }) client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Message", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "MessageData", "baseData": {"ver": 2, "message": "test", "properties": {"foo": "bar"}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
def test_track_trace_works_as_expected(self): sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.context.device = None client.track_trace('test', { 'foo': 'bar' }, severity='WARNING') client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Message", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "MessageData", "baseData": {"ver": 2, "message": "test", "severityLevel": 2, "properties": {"foo": "bar"}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
def main(mytimer: func.TimerRequest) -> None: get_config_values() utc_timestamp = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc).isoformat() if mytimer.past_due: logging.info('The timer is past due!') logging.info('Python timer trigger function ran at %s', utc_timestamp) tc = TelemetryClient(APP_INSIGHT_ID) tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = PROCESS_PROGRAM_NAME tc.context.properties["PROCESS_START"] = time.time() tc.track_trace( '{} Start to Check ADX Ingest Log'.format(LOG_MESSAGE_HEADER)) tc.flush() update_ADX_ingest_status(tc)
def test_track_trace_works_as_expected(self): def process(data, context): data.properties["NEW_PROP"] = "MYPROP" context.user.id = "BOTUSER" context.session.id = "BOTSESSION" return True sender = MockTelemetrySender() queue = channel.SynchronousQueue(sender) client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue)) client.add_telemetry_processor(process) client.context.device = None client.track_trace('test', { 'foo': 'bar' }, severity='WARNING') client.flush() expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Message", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "MessageData", "baseData": {"ver": 2, "message": "test", "severityLevel": 2, "properties": {"NEW_PROP": "MYPROP", "foo": "bar"}}}}' sender.data.time = 'TIME_PLACEHOLDER' sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER' sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER" sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER" sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER" sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER" actual = json.dumps(sender.data.write()) self.maxDiff = None self.assertEqual(expected, actual)
class RaftUtils(): def __init__(self): work_directory = os.environ['RAFT_WORK_DIRECTORY'] with open(os.path.join(work_directory, 'task-config.json'), 'r') as task_config: self.config = json.load(task_config) connection_str = os.environ['RAFT_SB_OUT_SAS'] self.topic_client = TopicClient.from_connection_string(connection_str) self.telemetry_client = TelemetryClient( instrumentation_key=os.environ['RAFT_APP_INSIGHTS_KEY']) self.job_id = os.environ['RAFT_JOB_ID'] self.container_name = os.environ['RAFT_CONTAINER_NAME'] self.telemetry_properties = { "jobId": self.job_id, "taskIndex": os.environ['RAFT_TASK_INDEX'], "containerName": self.container_name } def report_status(self, state, details): m = { 'eventType': 'JobStatus', 'message': { 'tool': 'ZAP', 'jobId': self.job_id, 'agentName': self.container_name, 'details': details, 'utcEventTime': time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), 'state': state } } msg = Message(str.encode(json.dumps(m))) self.topic_client.send(msg) def report_status_created(self, details=None): self.report_status('Created', details) def report_status_running(self, details=None): self.report_status('Running', details) def report_status_error(self, details=None): self.report_status('Error', details) def report_status_completed(self, details=None): self.report_status('Completed', details) def log_trace(self, trace): self.telemetry_client.track_trace(trace, properties=self.telemetry_properties) def log_exception(self): self.telemetry_client.track_exception( properties=self.telemetry_properties) def flush(self): self.telemetry_client.flush() def get_swagger_target(self): swagger = self.config.get("swaggerLocation") if swagger and swagger.get("url"): return swagger["url"] elif swagger.get("filePath"): return swagger["filePath"]
def merge_rename_core_columns_CSV(vm_uuid, deploy_uuid, config_uuid, schema_ver, inject_ver, container_name, filesrootfolder, fileoutputfolder, process_id): #block_blob_service = BlockBlobService(account_name=SOURCE_CSV_BLOB_ACCOUNT, sas_token=SOURCE_CSV_BLOB_TOKEN) block_blob_service = BlockBlobService(account_name=SOURCE_CSV_BLOB_ACCOUNT, account_key=SOURCE_CSV_BLOB_KEY) tc = TelemetryClient('') print("Start merge CSV ", vm_uuid, ' ', deploy_uuid, ' ', config_uuid) blobs = [] marker = None while True: batch = block_blob_service.list_blobs(container_name, prefix=filesrootfolder) blobs.extend(batch) if not batch.next_marker: break marker = batch.next_marker i = 0 blobpaths = [] for blob in blobs: blobpaths.append(blob.name) matchers = ['.csv'] matching = [s for s in blobpaths if any(xs in s for xs in matchers)] mergelog = {} mergelog["vm_uuid"] = vm_uuid mergelog["process_type"] = "MERGE_METRIC_CSV" mergelog["DOC_TYPE"] = "MERGE_METRIC_FILES_LOG" mergelog["file_folder"] = filesrootfolder mergelog["process_time"] = time.time() mergelog["files"] = [] mergelog["defect_files"] = [] a_mergelog = copy.deepcopy(mergelog) dfagg = pd.DataFrame(columns=[]) mixagg = AGGREGATION_FILES_NUM aggcount = 0 aggcount_total = 0 aggoutcount = 0 aggsize = 0 error_files = [] merged_files = [] totoal_rows = 0 alldfs = [] outfilenamebase = fileoutputfolder + filesrootfolder + "_aggr_" t1 = time.time() #print (outfilenamebase) source_col = [''] target_col = [''] tc.track_trace('Prepare to process ' + str(len(matching)) + ' Metric CSV files ') tc.flush() for fname in matching: #print(aggcount) head, tail = os.path.split(fname) aggcount += 1 aggcount_total += 1 blobstring = block_blob_service.get_blob_to_text( container_name, fname).content aggsize += len(blobstring) #print('Prepare to merge '+str(aggcount_total)+' / '+str(len(matching)) +' Memeory '+str(aggsize)+' File Name: '+tail) #tc.track_trace('Prepare to merge '+tail) #tc.flush() try: # Rread CSV And Try Processing dfone = pd.read_csv(StringIO(blobstring)) dfAll_cols = dfone.columns #colname0=dfAll_cols dfAll_newcols = [] pc_name = re.search(r'(\\{2}.*\\)(.*\\)', dfAll_cols[1]).group(1) for col in dfAll_cols: dfAll_newcols.append( col.replace(pc_name, '').replace('`', '').replace( '\\', '').replace(' ', '').replace('/', '').replace( '.', '').replace('-', '').replace('%', '').replace( '(', '').replace(')', '')) dfAll_newcols[0] = "Universal_datetime" # Rename all columns dfone.columns = dfAll_newcols alldfs.append(dfone) a_mergelog['files'].append(tail) #if (aggcount>=mixagg) or (aggcount_total==len(matching)): if (aggsize > MAX_FILESIZE) or (aggcount_total == len(matching)): if (aggcount_total == len(matching)): print("Processing Final File") tc.track_trace('Processing Final File') tc.flush() alldfs.append(pd.DataFrame(columns=source_col)) dfagg = pd.concat(alldfs, ignore_index=True) dfagg_out = dfagg[source_col] dfagg_out.columns = target_col dfagg_out['schema_ver'] = schema_ver dfagg_out['inject_ver'] = inject_ver output = dfagg_out.to_csv(index=False, encoding="utf-8") outfile = outfilenamebase + str(aggoutcount) + ".csv" block_blob_service.create_blob_from_text( container_name, outfile, output) print( "Output aggregated file to " + container_name, outfile + " Data Shape " + str(dfagg.shape) + ' uuid: ' + str(vm_uuid) + str(deploy_uuid) + str(config_uuid)) totoal_rows += dfagg_out.shape[0] merged_files.append(outfile) a_mergelog['output_file'] = outfile a_mergelog['merged_files_num'] = len(a_mergelog['files']) a_mergelog['defect_files_num'] = len( a_mergelog['defect_files']) # Insert Process Log to COSMOS DB insert_json_cosmos(a_mergelog) a_mergelog = copy.deepcopy(mergelog) t2 = time.time() print(("It takes %s seconds to merge " + str(aggcount) + " CSV Metrics") % (t2 - t1)) aggoutcount += 1 aggcount = 0 aggsize = 0 alldfs = [] t1 = time.time() file_size = BlockBlobService.get_blob_properties( block_blob_service, container_name, outfile).properties.content_length print(outfile + " File Size " + str(file_size)) # Ingest to AXX ingest_to_ADX(outfile, file_size) except Exception as e: print('Error While process ' + fname) error_class = e.__class__.__name__ detail = e.args[0] cl, exc, tb = sys.exc_info() lastCallStack = traceback.extract_tb(tb)[-1] fileName = lastCallStack[0] lineNum = lastCallStack[1] funcName = lastCallStack[2] errMsg = "File \"{}\", line {}, in {}: [{}] {}".format( fileName, lineNum, funcName, error_class, detail) print("Unexpected error:", sys.exc_info()[0]) traceback.print_exc() msg = errMsg + traceback.format_exc() tc = TelemetryClient('') tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a" tc.context.properties["DATA_FOLDER"] = metricspath tc.track_trace(msg) tc.flush() # print("Unexpected error:", sys.exc_info()[0]) a_mergelog["defect_files"].append(tail) error_files.append(fname) # Add No-Well Formed JSON to error file print('Total Rows ' + str(totoal_rows)) tc.track_trace('Proccessed Rows: ' + str(totoal_rows)) tc.track_metric('BATHCH_INGEST_METRIC_CSV_TOTAL_ROWS', str(totoal_rows)) tc.flush() return error_files, merged_files, matching
class AppinsightsBotTelemetryClient(BotTelemetryClient): def __init__(self, instrumentation_key: str): self._instrumentation_key = instrumentation_key self._context = TelemetryContext() context.instrumentation_key = self._instrumentation_key # context.user.id = 'BOTID' # telemetry_channel.context.session. # context.session.id = 'BOTSESSION' # set up channel with context self._channel = TelemetryChannel(context) # telemetry_channel.context.properties['my_property'] = 'my_value' self._client = TelemetryClient(self._instrumentation_key, self._channel) def track_pageview(self, name: str, url: str, duration: int = 0, properties: Dict[str, object] = None, measurements: Dict[str, object] = None) -> None: """ Send information about the page viewed in the application (a web page for instance). :param name: the name of the page that was viewed. :param url: the URL of the page that was viewed. :param duration: the duration of the page view in milliseconds. (defaults to: 0) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) """ self._client.track_pageview(name, url, duration, properties, measurements) def track_exception(self, type_exception: type = None, value: Exception = None, tb: traceback = None, properties: Dict[str, object] = None, measurements: Dict[str, object] = None) -> None: """ Send information about a single exception that occurred in the application. :param type_exception: the type of the exception that was thrown. :param value: the exception that the client wants to send. :param tb: the traceback information as returned by :func:`sys.exc_info`. :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) """ self._client.track_exception(type_exception, value, tb, properties, measurements) def track_event(self, name: str, properties: Dict[str, object] = None, measurements: Dict[str, object] = None) -> None: """ Send information about a single event that has occurred in the context of the application. :param name: the data to associate to this event. :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) """ self._client.track_event(name, properties, measurements) def track_metric(self, name: str, value: float, type: TelemetryDataPointType = None, count: int = None, min: float = None, max: float = None, std_dev: float = None, properties: Dict[str, object] = None) -> NotImplemented: """ Send information about a single metric data point that was captured for the application. :param name: The name of the metric that was captured. :param value: The value of the metric that was captured. :param type: The type of the metric. (defaults to: TelemetryDataPointType.aggregation`) :param count: the number of metrics that were aggregated into this data point. (defaults to: None) :param min: the minimum of all metrics collected that were aggregated into this data point. (defaults to: None) :param max: the maximum of all metrics collected that were aggregated into this data point. (defaults to: None) :param std_dev: the standard deviation of all metrics collected that were aggregated into this data point. (defaults to: None) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) """ self._client.track_metric(name, value, type, count, min, max, std_dev, properties) def track_trace(self, name: str, properties: Dict[str, object] = None, severity=None): """ Sends a single trace statement. :param name: the trace statement.\n :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)\n :param severity: the severity level of this trace, one of DEBUG, INFO, WARNING, ERROR, CRITICAL """ self._client.track_trace(name, properties, severity) def track_request(self, name: str, url: str, success: bool, start_time: str = None, duration: int = None, response_code: str = None, http_method: str = None, properties: Dict[str, object] = None, measurements: Dict[str, object] = None, request_id: str = None): """ Sends a single request that was captured for the application. :param name: The name for this request. All requests with the same name will be grouped together. :param url: The actual URL for this request (to show in individual request instances). :param success: True if the request ended in success, False otherwise. :param start_time: the start time of the request. The value should look the same as the one returned by :func:`datetime.isoformat()` (defaults to: None) :param duration: the number of milliseconds that this request lasted. (defaults to: None) :param response_code: the response code that this request returned. (defaults to: None) :param http_method: the HTTP method that triggered this request. (defaults to: None) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) :param request_id: the id for this request. If None, a new uuid will be generated. (defaults to: None) """ self._client.track_request(name, url, success, start_time, duration, response_code, http_method, properties, measurements, request_id) def track_dependency(self, name: str, data: str, type: str = None, target: str = None, duration: int = None, success: bool = None, result_code: str = None, properties: Dict[str, object] = None, measurements: Dict[str, object] = None, dependency_id: str = None): """ Sends a single dependency telemetry that was captured for the application. :param name: the name of the command initiated with this dependency call. Low cardinality value. Examples are stored procedure name and URL path template. :param data: the command initiated by this dependency call. Examples are SQL statement and HTTP URL with all query parameters. :param type: the dependency type name. Low cardinality value for logical grouping of dependencies and interpretation of other fields like commandName and resultCode. Examples are SQL, Azure table, and HTTP. (default to: None) :param target: the target site of a dependency call. Examples are server name, host address. (default to: None) :param duration: the number of milliseconds that this dependency call lasted. (defaults to: None) :param success: true if the dependency call ended in success, false otherwise. (defaults to: None) :param result_code: the result code of a dependency call. Examples are SQL error code and HTTP status code. (defaults to: None) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) :param id: the id for this dependency call. If None, a new uuid will be generated. (defaults to: None) """ self._client.track_dependency(name, data, type, target, duration, success, result_code, properties, measurements, dependency_id)
def send_logs(client: TelemetryClient, num_traces: int): for _ in range(num_traces): trace = generate_log_message() severity = generate_log_severity() client.track_trace(trace, severity=severity) LOG.info('sent trace %s %d', trace, severity)
class RaftUtils(): def __init__(self, tool_name): from applicationinsights import TelemetryClient from azure.servicebus import ServiceBusClient, ServiceBusMessage self.config = task_config() connection_str = os.environ['RAFT_SB_OUT_SAS'] self.sb_client = ServiceBusClient.from_connection_string( connection_str) self.topic_client = self.sb_client.get_topic_sender( self.sb_client._entity_name) self.telemetry_client = TelemetryClient( instrumentation_key=os.environ['RAFT_APP_INSIGHTS_KEY']) self.job_id = os.environ['RAFT_JOB_ID'] self.container_name = os.environ['RAFT_CONTAINER_NAME'] self.tool_name = tool_name self.telemetry_properties = { "jobId": self.job_id, "taskIndex": os.environ['RAFT_TASK_INDEX'], "containerName": self.container_name } self.newSbMessage = ServiceBusMessage def report_bug(self, bugDetails): m = { 'eventType': 'BugFound', 'message': { 'tool': self.tool_name, 'jobId': self.job_id, 'agentName': self.container_name, 'bugDetails': bugDetails } } msg = self.newSbMessage(str.encode(json.dumps(m))) self.topic_client.send_messages([msg]) def report_status(self, state, details): m = { 'eventType': 'JobStatus', 'message': { 'tool': self.tool_name, 'jobId': self.job_id, 'agentName': self.container_name, 'details': details, 'utcEventTime': time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), 'state': state } } msg = self.newSbMessage(str.encode(json.dumps(m))) self.topic_client.send_messages([msg]) def report_status_created(self, details=None): self.report_status('Created', details) def report_status_running(self, details=None): self.report_status('Running', details) def report_status_error(self, details=None): self.report_status('Error', details) def report_status_completed(self, details=None): self.report_status('Completed', details) def log_trace(self, trace): self.telemetry_client.track_trace(trace, properties=self.telemetry_properties) def log_exception(self): self.telemetry_client.track_exception( properties=self.telemetry_properties) def flush(self): self.telemetry_client.flush() self.sb_client.close()
class ApplicationInsightsTelemetryClient(BotTelemetryClient): def __init__(self, instrumentation_key: str): self._instrumentation_key = instrumentation_key self._client = TelemetryClient(self._instrumentation_key) # Telemetry Processor def telemetry_processor(data, context): post_data = IntegrationPostData().activity_json # Override session and user id from_prop = post_data['from'] if 'from' in post_data else None user_id = from_prop['id'] if from_prop != None else None channel_id = post_data[ 'channelId'] if 'channelId' in post_data else None conversation = post_data[ 'conversation'] if 'conversation' in post_data else None conversation_id = conversation[ 'id'] if 'id' in conversation else None context.user.id = channel_id + user_id context.session.id = conversation_id # Additional bot-specific properties if 'activityId' in post_data: data.properties["activityId"] = post_data['activityId'] if 'channelId' in post_data: data.properties["channelId"] = post_data['channelId'] if 'activityType' in post_data: data.properties["activityType"] = post_data['activityType'] self._client.add_telemetry_processor(telemetry_processor) def track_pageview(self, name: str, url: str, duration: int = 0, properties: Dict[str, object] = None, measurements: Dict[str, object] = None) -> None: """ Send information about the page viewed in the application (a web page for instance). :param name: the name of the page that was viewed. :param url: the URL of the page that was viewed. :param duration: the duration of the page view in milliseconds. (defaults to: 0) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) """ self._client.track_pageview(name, url, duration, properties, measurements) def track_exception(self, type_exception: type = None, value: Exception = None, tb: traceback = None, properties: Dict[str, object] = None, measurements: Dict[str, object] = None) -> None: """ Send information about a single exception that occurred in the application. :param type_exception: the type of the exception that was thrown. :param value: the exception that the client wants to send. :param tb: the traceback information as returned by :func:`sys.exc_info`. :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) """ self._client.track_exception(type_exception, value, tb, properties, measurements) def track_event(self, name: str, properties: Dict[str, object] = None, measurements: Dict[str, object] = None) -> None: """ Send information about a single event that has occurred in the context of the application. :param name: the data to associate to this event. :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) """ self._client.track_event(name, properties, measurements) def track_metric(self, name: str, value: float, type: TelemetryDataPointType = None, count: int = None, min: float = None, max: float = None, std_dev: float = None, properties: Dict[str, object] = None) -> NotImplemented: """ Send information about a single metric data point that was captured for the application. :param name: The name of the metric that was captured. :param value: The value of the metric that was captured. :param type: The type of the metric. (defaults to: TelemetryDataPointType.aggregation`) :param count: the number of metrics that were aggregated into this data point. (defaults to: None) :param min: the minimum of all metrics collected that were aggregated into this data point. (defaults to: None) :param max: the maximum of all metrics collected that were aggregated into this data point. (defaults to: None) :param std_dev: the standard deviation of all metrics collected that were aggregated into this data point. (defaults to: None) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) """ self._client.track_metric(name, value, type, count, min, max, std_dev, properties) def track_trace(self, name: str, properties: Dict[str, object] = None, severity=None): """ Sends a single trace statement. :param name: the trace statement.\n :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)\n :param severity: the severity level of this trace, one of DEBUG, INFO, WARNING, ERROR, CRITICAL """ self._client.track_trace(name, properties, severity) def track_request(self, name: str, url: str, success: bool, start_time: str = None, duration: int = None, response_code: str = None, http_method: str = None, properties: Dict[str, object] = None, measurements: Dict[str, object] = None, request_id: str = None): """ Sends a single request that was captured for the application. :param name: The name for this request. All requests with the same name will be grouped together. :param url: The actual URL for this request (to show in individual request instances). :param success: True if the request ended in success, False otherwise. :param start_time: the start time of the request. The value should look the same as the one returned by :func:`datetime.isoformat()` (defaults to: None) :param duration: the number of milliseconds that this request lasted. (defaults to: None) :param response_code: the response code that this request returned. (defaults to: None) :param http_method: the HTTP method that triggered this request. (defaults to: None) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) :param request_id: the id for this request. If None, a new uuid will be generated. (defaults to: None) """ self._client.track_request(name, url, success, start_time, duration, response_code, http_method, properties, measurements, request_id) def track_dependency(self, name: str, data: str, type: str = None, target: str = None, duration: int = None, success: bool = None, result_code: str = None, properties: Dict[str, object] = None, measurements: Dict[str, object] = None, dependency_id: str = None): """ Sends a single dependency telemetry that was captured for the application. :param name: the name of the command initiated with this dependency call. Low cardinality value. Examples are stored procedure name and URL path template. :param data: the command initiated by this dependency call. Examples are SQL statement and HTTP URL with all query parameters. :param type: the dependency type name. Low cardinality value for logical grouping of dependencies and interpretation of other fields like commandName and resultCode. Examples are SQL, Azure table, and HTTP. (default to: None) :param target: the target site of a dependency call. Examples are server name, host address. (default to: None) :param duration: the number of milliseconds that this dependency call lasted. (defaults to: None) :param success: true if the dependency call ended in success, false otherwise. (defaults to: None) :param result_code: the result code of a dependency call. Examples are SQL error code and HTTP status code. (defaults to: None) :param properties: the set of custom properties the client wants attached to this data item. (defaults to: None) :param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None) :param id: the id for this dependency call. If None, a new uuid will be generated. (defaults to: None) """ self._client.track_dependency(name, data, type, target, duration, success, result_code, properties, measurements, dependency_id) def flush(self): """Flushes data in the queue. Data in the queue will be sent either immediately irrespective of what sender is being used. """ self._client.flush()
def trace_item(): tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') tc.track_trace('Test trace', {'foo': 'bar'}) #trace function to be given tc.flush()
def process(filesrootfolder, forceinsert): # Create process id as identify of this process process_id = time.time() tc = TelemetryClient('') tc.context.application.ver = '1.0' tc.context.properties["PROCESS_PROGRAM"] = "BATCH_CSV_V001a" tc.context.properties["PROCESS_START"] = time.time() tc.context.properties["DATA_FOLDER"] = filesrootfolder tc.context.properties["PROCESS_ID"] = process_id tc.track_trace('STRAT RUN BATHCH INGEST CSV DATA from folder ' + filesrootfolder) tc.track_event('BATHCH_INGEST_CSV_START', { 'PROCESS_ID': process_id, 'DATA_FOLDER': filesrootfolder }, {}) tc.flush() tc.flush() #print (vm_uuid,deploy_uuid,config_uuid) # Prepare COSMOS Link url = COSMOS_URL #key = os.environ['ACCOUNT_KEY'] key = COSMOS_KEY client = cosmos_client.CosmosClient(url, {'masterKey': key}) database_id = COSMOS_DATABASE container_id = COSMOS_CONTAINER database_link = 'dbs/' + database_id collection_link = database_link + '/colls/' + container_id doc_id = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_Metric' doc_link = collection_link + '/docs/' + doc_id options = {} options['enableCrossPartitionQuery'] = True options['maxItemCount'] = 5 options['partitionKey'] = vm_uuid proc_log_doc = None try: proc_log_doc = client.ReadItem(doc_link, options) except: print("New Process Metric Doc") if (proc_log_doc is not None): print("Find Existing Metric Doc ") if str(forceinsert).lower( ) != 'true': # Stop Proccess if data is already been proccessed return 400, doc_id + " is already been processed" else: # New process log proc_log_doc = {} proc_log_doc["PROCESSES"] = [] proc_log_doc["DOC_TYPE"] = "PROCESS_METRIC" proc_log_doc["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a" proc_log_doc['id'] = doc_id tc.track_event('BATHCH_INGEST_METRIC_CSV', {'PROCESS_ID': process_id}, {'DATA_FOLDER': filesrootfolder}) #+'_'+config_uuid+'_'+deploy_uuid , { 'DATA_FOLDER': telemetriespath } tc.flush() proc_log_this = {} proc_log_this["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a" proc_log_this["PROCESS_START"] = time.time() proc_log_this["DATA_FOLDER"] = filesrootfolder proc_log_this[ 'id'] = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_' + str( process_id) error_files, merged_files, source_files = merge_rename_core_columns_CSV( vm_uuid, deploy_uuid, config_uuid, 'defualt_metrics_csv_001A', 0, SOURCE_CSV_CONTAINER, filesrootfolder, FILE_OUTPUT_FOLDER, process_id) # ToDo ... proc_log_this["PROCESS_ID"] = process_id proc_log_this["ERROR_SOURCE_FILES_COUNT"] = len(error_files) proc_log_this["SOURCE_FILES_COUNT"] = len(source_files) tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_FILES_COUNT', len(error_files)) tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_SOURCE_FILES_COUNT', len(source_files)) tc.flush() # print(str(len(error_files)),' ',str(len(merged_files))) proc_log_this["PROCESS_END"] = time.time() proc_log_this["STATUS"] = "OK" proc_log_this["STATUS_MESSAGE"] = ( "It takes %s seconds to ingest CSV file from Blob Storage") % ( proc_log_this["PROCESS_END"] - proc_log_this["PROCESS_START"]) proc_log_doc["PROCESSES"].append(proc_log_this) proc_log_doc['LATEST_UPDATE_TIMESTAMP'] = time.time() # Update Process Log client.UpsertItem(collection_link, proc_log_doc, options) tc.track_trace('END RUN BATHCH INGEST METRIC CSV DATA from folder ' + filesrootfolder) tc.track_event('BATHCH_INGEST_METRIC_CSV_END', { 'PROCESS_ID': process_id, 'DATA_FOLDER': filesrootfolder }, { 'DEFECT_FILES_COUNT': len(error_files), 'MERGED_FILES_COUNT': len(merged_files), 'SOURCE_FILES_COUNT': len(source_files) }) tc.flush()
from applicationinsights import TelemetryClient tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') tc.track_event('Test event') tc.flush() #Sending an event telemetry item with custom properties and measurements from applicationinsights import TelemetryClient tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') tc.track_event('Test event', { 'foo': 'bar' }, { 'baz': 42 }) tc.flush() #Sending a trace telemetry item with custom properties from applicationinsights import TelemetryClient tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') tc.track_trace('Test trace', { 'foo': 'bar' }) tc.flush() #Sending a metric telemetry item from applicationinsights import TelemetryClient tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') tc.track_metric('My Metric', 42) tc.flush() #Sending an exception telemetry item with custom properties and measurements import sys from applicationinsights import TelemetryClient tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>') try: raise Exception('blah') except: