def _init_trace_logging(self, app):
"""
Sets up trace logging unless ``APPINSIGHTS_DISABLE_TRACE_LOGGING`` is
set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_TRACE_LOGGING, False)
if not enabled:
return
if self._key_grantee:
self._trace_log_handler_grantee = LoggingHandler(
self._key_grantee, telemetry_channel=self._channel)
app.logger.addHandler(self._trace_log_handler_grantee)
if self._key_ai4e:
print("Starting trace logging")
self._trace_log_handler_ai4e = LoggingHandler(
self._key_ai4e, telemetry_channel=self._channel)
app.logger.addHandler(self._trace_log_handler_ai4e)
def setup_logging(ikey):
telemetry_channel = channel.TelemetryChannel()
telemetry_channel.context.application.ver = __version__
telemetry_channel.context.properties['worker.id'] = str(uuid.uuid4())
handler = LoggingHandler(ikey, telemetry_channel=telemetry_channel)
logging.basicConfig(handlers=[handler],
format='%(levelname)s: %(message)s',
level=logging.DEBUG)
def InitializeAppInsights(self):
# AppInsights initialization
cur_thread = threading.current_thread()
logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-7.7s]: %(message)s")
appInsightsKey = os.environ['APPINSIGHTS_KEY']
self.rootLogger.info("AppInsights key: '" + appInsightsKey + "'") # log locally
# create a child logger per thread so that we can set the SessionId without collision during concurrent execution
# by default logging will propagate to the parent rootLogger
self.telemetryLogger = self.rootLogger.getChild('AppInsights.{0}'.format(cur_thread) )
telemetryhandler = LoggingHandler(appInsightsKey)
telemetryhandler.setFormatter(logFormatter)
telemetryhandler.client.context.application.id = "DiskInspect-Service"
telemetryhandler.client.context.application.ver = self.containerVersion
telemetryhandler.client.context.properties['releaseName'] = self.releaseName
self.telemetryLogger.addHandler(telemetryhandler)
self.telemetryClient = telemetryhandler.client
def __init__(self, bright_host_ip: str, metrics: Iterable[str], instrumentation_key: str):
self.__set_bright_host_ip(bright_host_ip)
self.__set_metrics(metrics)
self.__set_instrumentation_key(instrumentation_key)
bright_cluster = self.__create_bright_cluster()
self.__set_bright_cluster(bright_cluster)
emit_handler = LoggingHandler(instrumentation_key)
emit_handler.setLevel(logging.DEBUG)
self.__set_emit_handler(emit_handler)
emitter = logging.getLogger('application-insights')
emitter.setLevel(logging.DEBUG)
emitter.addHandler(emit_handler)
self.__set_emitter(emitter)
mutex = threading.Lock()
self.__set_mutex(mutex)
def main(event: func.EventHubEvent):
handler = LoggingHandler(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'])
logging.basicConfig(handlers=[ handler ], format='%(levelname)s: %(message)s', level=logging.DEBUG)
tc = TelemetryClient(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'])
tc.track_event("Incoming event")
tc.flush()
logging.info('Function triggered to process a message: %s', event)
logging.info(' body: %s', event.get_body())
logging.info(' EnqueuedTimeUtc: %s', event.enqueued_time)
logging.info(' SequenceNumber: %s', event.sequence_number)
logging.info(' Offset: %s', event.offset)
logging.info(' Partition: %s', event.partition_key)
logging.info(' Metadata: %s', event.iothub_metadata)
table_service = TableService(connection_string=os.environ['AzureTableConnectionString'])
for datapoint in json.loads(event.get_body()):
# Expected data format:
# {"timestamp": 1564598054, "deviceid": "Node1", "scale": 2, "temperature": 1.1,"weight": 10000}
if datapoint is not None and 'deviceid' in datapoint and \
'timestamp' in datapoint and 'scale' in datapoint and \
'weight' in datapoint:
logging.debug(' datapoint: %s', (datapoint))
# deviceid is used as partition key.
# {timestamp}-{scale} is used as RowKey
# timestamp and scale number are duplicated as an int columns
# to keep them searchable. The rest of the datapoint elements
# are added as columns as well.
history = {}
history['PartitionKey'] = datapoint.pop('deviceid')
history['RowKey'] = str(datapoint['timestamp']) + '-' + str(datapoint['scale'])
history.update(datapoint.items())
logging.debug('history: %s' % (history))
table_service.insert_entity(TABLE_NAME_HISTORICAL_DATA, history)
logging.info('Added historical table data: %s', (history))
# Touch/create the row in the config table for each reported scale with latest weight
configupdate = {}
configupdate['PartitionKey'] = history['PartitionKey']
configupdate['RowKey'] = str(history['scale'])
configupdate['weight'] = history['weight']
if 'temperature' in history:
configupdate['temperature'] = history['temperature']
logging.info('config update: %s' % (configupdate))
logging.info('Writing to table: %s' % (TABLE_NAME_CONFIGURATION))
table_service.insert_or_merge_entity(TABLE_NAME_CONFIGURATION, configupdate)
logging.info('Updated configuration table entry: %s', (configupdate))
else:
logging.info(' Invalid datapoint: %s', (datapoint))
def _init_trace_logging(self, app):
"""
Sets up trace logging unless ``APPINSIGHTS_DISABLE_TRACE_LOGGING`` is
set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_TRACE_LOGGING, False)
if not enabled:
return
self._trace_log_handler = LoggingHandler(
self._key, telemetry_channel=self._channel)
app.logger.addHandler(self._trace_log_handler)
def attach_appinsights(logger, instrumentation_key: str):
if not instrumentation_key:
logger.warning(
"appinsights instrumentation key is null; not writing to app insights"
)
return
handler = LoggingHandler(instrumentation_key)
# TODO: extend this collection to jobid, pipelineid or farmid etc.
handler.client._context.properties['Collection'] = 'ADF_LOGS'
# Removing all PIO information from context.
# Due to a bug in LoggingHanlder constructor, its not honoring context passed.
# so trying to set values once handler is created. Overriding with 'None' string
# instead of None as for later value, its taking default value from constructor.
handler.client._context.device = contracts.Device()
handler.client._context.device.os_version = NONE_STRING
handler.client._context.device.locale = NONE_STRING
handler.client._context.device.id = NONE_STRING
handler.client._context.device.type = NONE_STRING
handler.client._context.device.oem_name = NONE_STRING
handler.client._context.device.model = NONE_STRING
handler.client._context.location = contracts.Location()
handler.client._context.location.ip = NONE_STRING
handler.client._context.user = contracts.User()
handler.client._context.user.id = NONE_STRING
handler.client._context.user.account_id = NONE_STRING
handler.client._context.user.auth_user_id = NONE_STRING
handler.client._context.session = contracts.Session()
handler.client._context.session.id = NONE_STRING
handler.client._context.cloud = contracts.Cloud()
handler.client._context.cloud.role = NONE_STRING
handler.setLevel(Logger.get_logging_level())
# Log formatter looks similar to default python logging statement.
handler.setFormatter(
logging.Formatter(
'[%(asctime)s %(levelname)s %(filename)s:%(lineno)d %(process)d:%(thread)d]: %(message)s'
))
# enable exceptions to app insights
enable(instrumentation_key)
logger.addHandler(handler)
logger.info("Attached app insights handler to logger")
def setup_logger():
logging.setLoggerClass(CustomLogger)
_logger = logging.getLogger()
_logger.setLevel(logging.DEBUG)
# Stdout handler
c_handler = logging.StreamHandler()
c_handler.setLevel(logging.DEBUG)
c_handler.setFormatter(logging.Formatter(format_str))
_logger.addHandler(c_handler)
# Application insight handler
if utils.convert_to_boolean(os.environ["AZURE_USEAPPINSIGHT"]):
from applicationinsights import TelemetryClient
from applicationinsights.logging import LoggingHandler
a_handler = LoggingHandler(os.environ["AZURE_INSTRUMENTATION_KEY"])
a_handler.setLevel(logging.DEBUG)
a_handler.setFormatter(logging.Formatter(ai_str))
_logger.addHandler(a_handler)
tc = TelemetryClient(os.environ["AZURE_INSTRUMENTATION_KEY"])
tc.channel.queue.max_queue_length = 2
class AppInsights(object):
""" This class represents a Flask extension that enables request telemetry,
trace logging and exception logging for a Flask application. The telemetry
will be sent to Application Insights service using the supplied
instrumentation key.
The following Flask config variables can be used to configure the extension:
- Set ``APPINSIGHTS_INSTRUMENTATIONKEY`` to a string to provide the
instrumentation key to send telemetry to application insights.
Alternatively, this value can also be provided via an environment variable
of the same name.
- Set ``APPINSIGHTS_ENDPOINT_URI`` to a string to customize the telemetry
endpoint to which Application Insights will send the telemetry.
- Set ``APPINSIGHTS_DISABLE_REQUEST_LOGGING`` to ``False`` to disable
logging of Flask requests to Application Insights.
- Set ``APPINSIGHTS_DISABLE_TRACE_LOGGING`` to ``False`` to disable logging
of all log traces to Application Insights.
- Set ``APPINSIGHTS_DISABLE_EXCEPTION_LOGGING`` to ``False`` to disable
logging of all exceptions to Application Insights.
.. code:: python
from flask import Flask
from applicationinsights.flask.ext import AppInsights
# instantiate the Flask application
app = Flask(__name__)
app.config['APPINSIGHTS_INSTRUMENTATIONKEY'] = '<YOUR INSTRUMENTATION KEY GOES HERE>'
# log requests, traces and exceptions to the Application Insights service
appinsights = AppInsights(app)
# define a simple route
@app.route('/')
def hello_world():
return 'Hello World!'
# run the application
if __name__ == '__main__':
app.run()
"""
def __init__(self, app=None):
"""
Initialize a new instance of the extension.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
self._key = None
self._endpoint_uri = None
self._channel = None
self._requests_middleware = None
self._trace_log_handler = None
self._exception_telemetry_client = None
if app:
self.init_app(app)
def init_app(self, app):
"""
Initializes the extension for the provided Flask application.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
self._key = app.config.get(CONF_KEY) or getenv(CONF_KEY)
if not self._key:
return
self._endpoint_uri = app.config.get(CONF_ENDPOINT_URI)
sender = AsynchronousSender(self._endpoint_uri)
queue = AsynchronousQueue(sender)
self._channel = TelemetryChannel(None, queue)
self._init_request_logging(app)
self._init_trace_logging(app)
self._init_exception_logging(app)
@property
def context(self):
"""
Accesses the telemetry context.
Returns:
(applicationinsights.channel.TelemetryContext). The Application Insights telemetry context.
"""
return self._channel.context
def _init_request_logging(self, app):
"""
Sets up request logging unless ``APPINSIGHTS_DISABLE_REQUEST_LOGGING``
is set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_REQUEST_LOGGING, False)
if not enabled:
return
self._requests_middleware = WSGIApplication(
self._key, app.wsgi_app, telemetry_channel=self._channel)
app.wsgi_app = self._requests_middleware
def _init_trace_logging(self, app):
"""
Sets up trace logging unless ``APPINSIGHTS_DISABLE_TRACE_LOGGING`` is
set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_TRACE_LOGGING, False)
if not enabled:
return
self._trace_log_handler = LoggingHandler(
self._key, telemetry_channel=self._channel)
app.logger.addHandler(self._trace_log_handler)
def _init_exception_logging(self, app):
"""
Sets up exception logging unless ``APPINSIGHTS_DISABLE_EXCEPTION_LOGGING``
is set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_EXCEPTION_LOGGING, False)
if not enabled:
return
exception_telemetry_client = TelemetryClient(
self._key, telemetry_channel=self._channel)
@app.errorhandler(Exception)
def exception_handler(exception):
if HTTPException and isinstance(exception, HTTPException):
return exception
try:
raise exception
except Exception:
exception_telemetry_client.track_exception()
finally:
raise exception
self._exception_telemetry_client = exception_telemetry_client
def flush(self):
"""Flushes the queued up telemetry to the service.
"""
if self._requests_middleware:
self._requests_middleware.flush()
if self._trace_log_handler:
self._trace_log_handler.flush()
if self._exception_telemetry_client:
self._exception_telemetry_client.flush()
instrumentation_key = '<Instrumentation key>'
#Enable unhandled exception logging
enable(instrumentation_key)
#setup other needed variables
tc = TelemetryClient(instrumentation_key)
tc.context.application.ver = '0.0.1'
tc.context.device.id = 'Sample notebook'
telemetry_channel = channel.TelemetryChannel()
telemetry_channel.context.application.ver = '1.0.0'
telemetry_channel.context.properties['application_name'] = 'sample_notebook'
telemetry_channel.context.properties['application_id'] = sc.applicationId
handler = LoggingHandler(instrumentation_key,
telemetry_channel=telemetry_channel)
handler.setLevel(logging.DEBUG)
handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
logger = logging.getLogger('simple_logger')
logger.setLevel(logging.INFO)
logger.addHandler(handler)
logger.info('Starting sample app ... ')
def getReadConfig():
readConfig = {
"Endpoint": "https://nomier-test-sql.documents.azure.com:443/",
"Masterkey": "<MK>",
"Database": "partitionIdTestDB",
"Collection": "sourcecollection",
class AppInsights(object):
"""This class represents a Flask extension that enables request telemetry,
trace logging and exception logging for a Flask application. The telemetry
will be sent to Application Insights service using the supplied
instrumentation key.
The following Flask config variables can be used to configure the extension:
- Set ``APPINSIGHTS_INSTRUMENTATIONKEY`` to a string to provide the
instrumentation key to send telemetry to application insights.
Alternatively, this value can also be provided via an environment variable
of the same name.
- Set ``APPINSIGHTS_ENDPOINT_URI`` to a string to customize the telemetry
endpoint to which Application Insights will send the telemetry.
- Set ``APPINSIGHTS_DISABLE_REQUEST_LOGGING`` to ``False`` to disable
logging of Flask requests to Application Insights.
- Set ``APPINSIGHTS_DISABLE_TRACE_LOGGING`` to ``False`` to disable logging
of all log traces to Application Insights.
- Set ``APPINSIGHTS_DISABLE_EXCEPTION_LOGGING`` to ``False`` to disable
logging of all exceptions to Application Insights.
.. code:: python
from flask import Flask
from ai4e_app_insights import AppInsights
# instantiate the Flask application
app = Flask(__name__)
app.config['APPINSIGHTS_INSTRUMENTATIONKEY'] = '<YOUR INSTRUMENTATION KEY GOES HERE>'
# log requests, traces and exceptions to the Application Insights service
appinsights = AppInsights(app)
# define a simple route
@app.route('/')
def hello_world():
return 'Hello World!'
# run the application
if __name__ == '__main__':
app.run()
"""
def __init__(self, app=None, context=None):
"""
Initialize a new instance of the extension.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
socket.setdefaulttimeout(30)
self._appinsights_key = None
self._endpoint_uri = None
self._channel = None
self._requests_middleware = None
self._trace_log_handler_grantee = None
self._trace_log_handler_ai4e = None
self._exception_telemetry_client_grantee = None
self._exception_telemetry_client_ai4e = None
if app:
self.init_app(app, context)
def init_app(self, app, context):
"""
Initializes the extension for the provided Flask application.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
print("Starting application insights module.")
self._appinsights_key = app.config.get(
APPINSIGHTS_INSTRUMENTATIONKEY) or getenv(
APPINSIGHTS_INSTRUMENTATIONKEY)
if self._appinsights_key and len(self._appinsights_key.strip()) > 0:
self._appinsights_key = self._appinsights_key.strip()
else:
self._appinsights_key = None
# Set the application insights key for production use.
if self._appinsights_key:
print("Application insights key set.")
self._endpoint_uri = app.config.get(CONF_ENDPOINT_URI)
if self._endpoint_uri:
sender = AsynchronousSender(self._endpoint_uri)
else:
sender = AsynchronousSender()
queue = AsynchronousQueue(sender)
if not context:
context = AI4ETelemetryContext()
self._channel = TelemetryChannel(context, queue)
self._init_request_logging(app)
self._init_trace_logging(app)
self._init_exception_logging(app)
def _init_request_logging(self, app):
"""
Sets up request logging unless ``APPINSIGHTS_DISABLE_REQUEST_LOGGING``
is set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_REQUEST_LOGGING, False)
if not enabled:
return
wsgi_key = self._appinsights_key
self._requests_middleware = WSGIApplication(
wsgi_key, app.wsgi_app, telemetry_channel=self._channel)
app.wsgi_app = self._requests_middleware
def _init_trace_logging(self, app):
"""
Sets up trace logging unless ``APPINSIGHTS_DISABLE_TRACE_LOGGING`` is
set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_TRACE_LOGGING, False)
if not enabled:
return
if self._appinsights_key:
self._trace_log_handler_grantee = LoggingHandler(
self._appinsights_key, telemetry_channel=self._channel)
app.logger.addHandler(self._trace_log_handler_grantee)
def _init_exception_logging(self, app):
"""
Sets up exception logging unless ``APPINSIGHTS_DISABLE_EXCEPTION_LOGGING``
is set in the Flask config.
Args:
app (flask.Flask). the Flask application for which to initialize the extension.
"""
enabled = not app.config.get(CONF_DISABLE_EXCEPTION_LOGGING, False)
if not enabled:
return
if self._appinsights_key:
self._exception_telemetry_client_grantee = TelemetryClient(
self._appinsights_key, telemetry_channel=self._channel)
@app.errorhandler(Exception)
def exception_handler(exception):
try:
raise exception
except Exception:
if self._exception_telemetry_client_grantee:
self._exception_telemetry_client_grantee.track_exception()
if self._exception_telemetry_client_ai4e:
self._exception_telemetry_client_ai4e.track_exception()
finally:
raise exception
def flush(self):
"""Flushes the queued up telemetry to the service."""
print("trying all flush")
if self._requests_middleware:
self._requests_middleware.flush()
if self._trace_log_handler_grantee:
self._trace_log_handler_grantee.flush()
if self._trace_log_handler_ai4e:
print("Trying trace flush...")
self._trace_log_handler_ai4e.flush()
print("Trace flush finsihed.")
if self._exception_telemetry_client_grantee:
self._exception_telemetry_client_grantee.flush()
if self._exception_telemetry_client_ai4e:
self._exception_telemetry_client_ai4e.flush()
sys.settrace(trace)
"""
"""Setup Azure Application Insights
____ _ _ ____ ____ ___ ____ __ _ ____
( __| \/ | __| _ \/ __| __| ( (_ _)
) _)/ \/ \) _) ) ( (_ \) _)/ / )(
(____)_)(_(____|__\_)\___(____)_)__)(__)
"""
# set up channel with context
telemetry_channel = channel.TelemetryChannel()
telemetry_channel.context.application.ver = '0.0.0.0'
#telemetry_channel.context.properties['my_property'] = 'my_value'
# set up logging
az_handler = LoggingHandler('1bd7b388-4afd-4b58-8b2f-060ac172d00d',
telemetry_channel=telemetry_channel)
az_handler.setLevel(logging.DEBUG)
hash = hashlib.sha1()
hash.update(str(time.time()).encode("utf-8", "strict"))
az_handler.setFormatter(
logging.Formatter(f'{hash.hexdigest()[:16]} ||'
'%(name)s - %(levelname)s: %(message)s')
)
f_handler = logging.FileHandler('output.log')
f_handler.setLevel(logging.DEBUG)
f_handler.setFormatter(
logging.Formatter('%(name)s - %(levelname)s: %(message)s')
)
c_handler = logging.StreamHandler()
}
# Sysout Logging Setup
logger = logging.getLogger("monitofi")
logger.setLevel(logging.INFO)
syshandler = logging.StreamHandler(sys.stdout)
syshandler.setLevel(logging.INFO)
formatter = json_log_formatter.JSONFormatter()
syshandler.setFormatter(formatter)
logger.addHandler(syshandler)
if IKEY != "REPLACE_ME":
# Logging unhandled exceptions with Appinsights
enable(IKEY)
# Applications Insights Logging Setup
handler = LoggingHandler(IKEY)
handler.setFormatter(formatter)
logger.addHandler(handler)
iclient = InfluxDBClient(INFLUXDB_SERVER, INFLUXDB_PORT, INFLUXDB_USERNAME,
INFLUXDB_PASSWORD, INFLUXDB_DATABASE)
iclient.create_database(INFLUXDB_DATABASE)
def flattening(nested, prefix, ignore_list):
field = {}
flatten(True, nested, field, prefix, ignore_list)
return field
# set up logging
enable('<YOUR INSTRUMENTATION KEY GOES HERE>')
# log something (this will be sent to the Application Insights service as a trace)
logging.info('This is a message')
# logging shutdown will cause a flush of all un-sent telemetry items
logging.shutdown()
Basic logging configuration (second option)
import logging
from applicationinsights.logging import LoggingHandler
# set up logging
handler = LoggingHandler('<YOUR INSTRUMENTATION KEY GOES HERE>')
logging.basicConfig(handlers=[ handler ], format='%(levelname)s: %(message)s', level=logging.DEBUG)
# log something (this will be sent to the Application Insights service as a trace)
logging.debug('This is a message')
try:
raise Exception('Some exception')
except:
# this will send an exception to the Application Insights service
logging.exception('Code went boom!')
# logging shutdown will cause a flush of all un-sent telemetry items
# alternatively flush manually via handler.flush()
logging.shutdown()
Advanced logging configuration
