def bind(app, schedule=True):
"""
Binding the app to this object should happen before importing the routing-
methods below. Thus, the importing statement is part of this function.
:param app: the app for which the performance has to be tracked
:param schedule: flag telling if the background scheduler should be started
"""
config.app = app
# Provide a secret-key for using WTF-forms
if not app.secret_key:
log('WARNING: You should provide a security key.')
app.secret_key = 'my-secret-key'
# Add all route-functions to the blueprint
from flask_monitoringdashboard.views import deployment, custom, endpoint, outlier, request, profiler, version, auth
import flask_monitoringdashboard.views
# Add wrappers to the endpoints that have to be monitored
from flask_monitoringdashboard.core.measurement import init_measurement
from flask_monitoringdashboard.core import custom_graph
blueprint.before_app_first_request(init_measurement)
if schedule:
custom_graph.init(app)
# register the blueprint to the app
app.register_blueprint(blueprint, url_prefix='/' + config.link)
def parse_version(parser, header, version):
"""
Parse the version given in the config-file.
If both GIT and VERSION are used, the GIT argument is used.
:param parser: the parser to be used for parsing
:param header: name of the header in the configuration file
:param version: the default version
"""
version = parse_string(parser, header, 'APP_VERSION', version)
if parser.has_option(header, 'GIT'):
git = parser.get(header, 'GIT')
try:
# current hash can be found in the link in HEAD-file in git-folder
# The file is specified by: 'ref: <location>'
git_file = (open(os.path.join(git, 'HEAD')).read().rsplit(
': ', 1)[1]).rstrip()
# read the git-version
version = open(git + '/' + git_file).read()
# cut version to at most 6 chars
return version[:6]
except IOError:
log("Error reading one of the files to retrieve the current git-version."
)
raise
return version
def run(self):
# sleep for average * ODC ms
average = get_avg_endpoint(
self._endpoint.name) * config.outlier_detection_constant
self._exit.wait(average / 1000)
if not self._exit.is_set():
stack_list = []
try:
frame = sys._current_frames()[self._current_thread]
except KeyError:
log('Can\'t get the stacktrace of the main thread.')
return
in_endpoint_code = False
# filename, line number, function name, source code line
for fn, ln, fun, line in traceback.extract_stack(frame):
if self._endpoint.name == fun:
in_endpoint_code = True
if in_endpoint_code:
stack_list.append(
'File: "{}", line {}, in "{}": "{}"'.format(
fn, ln, fun, line))
# Set the values in the object
self._stacktrace = '<br />'.join(stack_list)
self._cpu_percent = str(
psutil.cpu_percent(interval=None, percpu=True))
self._memory = str(psutil.virtual_memory())
def __init__(self):
"""Sets the default values for the project."""
# dashboard
self.version = '1.0'
self.blueprint_name = 'dashboard'
self.link = 'dashboard'
self.monitor_level = 1
self.outlier_detection_constant = 2.5
self.sampling_period = 5 / 1000.0
self.enable_logging = False
# database
self.database_name = 'sqlite:///flask_monitoringdashboard.db'
self.table_prefix = ''
# authentication
self.username = '******'
self.password = '******'
self.security_token = 'cc83733cb0af8b884ff6577086b87909'
# visualization
self.colors = {}
try:
self.timezone = pytz.timezone(str(get_localzone()))
except pytz.UnknownTimeZoneError:
log('Using default timezone, which is UTC')
self.timezone = pytz.timezone('UTC')
# define a custom function to retrieve the session_id or username
self.group_by = None
# store the Flask app
self.app = None
def parse_version(parser, header, version):
"""
Parse the version given in the config-file.
If both GIT and VERSION are used, the GIT argument is used.
:param parser: the parser to be used for parsing
:param header: name of the header in the configuration file
:param version: the default version
"""
version = parse_string(parser, header, 'APP_VERSION', version)
if parser.has_option(header, 'GIT'):
git = parser.get(header, 'GIT')
try:
# current hash can be found in the link in HEAD-file in git-folder
# The file is specified by: 'ref: <location>'
git_head = os.path.join(git, 'HEAD')
if os.path.isfile(git_head):
git_file = (open(git_head).read().rsplit(': ', 1)[1]).rstrip()
# read the git-version
version_file = os.path.join(git, git_file)
if os.path.exists(version_file):
version = open(version_file).read()
# cut version to at most 6 chars
return version[:6]
else:
# Return "dummy" version in case of no git version file found
log("Folder {} not found, using dummy version: {}".format(
git_head, version))
return version
except IOError:
log("Error reading one of the files to retrieve the current git-version."
)
raise
return version
def bind(app, schedule=True, include_dashboard=True):
"""Binding the app to this object should happen before importing the routing-
methods below. Thus, the importing statement is part of this function.
:param app: the app for which the performance has to be tracked
:param schedule: flag telling if the background scheduler should be started
:param include_dashboard: flag telling if the views should be added or not.
"""
blueprint.name = config.blueprint_name
config.app = app
# Provide a secret-key for using WTF-forms
if not app.secret_key:
log('WARNING: You should provide a security key.')
app.secret_key = 'my-secret-key'
# Add all route-functions to the blueprint
if include_dashboard:
from flask_monitoringdashboard.views import (
deployment,
custom,
endpoint,
outlier,
request,
profiler,
version,
auth,
reporting,
)
import flask_monitoringdashboard.views
# Add wrappers to the endpoints that have to be monitored
from flask_monitoringdashboard.core.measurement import init_measurement
from flask_monitoringdashboard.core.cache import init_cache
from flask_monitoringdashboard.core import custom_graph
blueprint.before_app_first_request(init_measurement)
blueprint.before_app_first_request(init_cache)
if schedule:
custom_graph.init(app)
# register the blueprint to the app
app.register_blueprint(blueprint, url_prefix='/' + config.link)
# flush cache to db before shutdown
import atexit
from flask_monitoringdashboard.core.cache import flush_cache
atexit.register(flush_cache)
if not include_dashboard:
@app.teardown_request
def teardown(_):
flush_cache()
def get_ip():
"""
:return: the ip address associated with the current request context
"""
if config.get_ip:
try:
return config.get_ip()
except Exception as e:
log('Failed to execute provided get_ip function: {}'.format(e))
# This is a reasonable fallback, but will not work for clients behind proxies.
return request.environ['REMOTE_ADDR']
def get_group_by():
"""
:return: a string with the value
"""
group_by = None
try:
if config.group_by:
group_by = recursive_group_by(config.group_by)
except Exception as e:
log('Can\'t execute group_by function: {}'.format(e))
return str(group_by)
def get_outlier_table(session, endpoint_id, offset, per_page):
"""
:param session: session for the database
:param endpoint_id: id of the endpoint
:param offset: number of items to be skipped
:param per_page: maximum number of items to be returned
:return: a list of length at most 'per_page' with data about each outlier
"""
table = get_outliers_sorted(session, endpoint_id, offset, per_page)
for idx, row in enumerate(table):
row.request.time_requested = to_local_datetime(row.request.time_requested)
try:
row.request_url = row.request_url.decode('utf-8')
except Exception as e:
log(e)
dict_request = row2dict(row.request)
table[idx] = row2dict(row)
table[idx]['request'] = dict_request
return table
def run(self):
"""
Continuously takes a snapshot from the stacktrace (only the main-thread). Filters everything before the
endpoint has been called (i.e. the Flask library).
Directly computes the histogram, since this is more efficient for performance
:return:
"""
current_time = time.time()
while self._keeprunning:
newcurrent_time = time.time()
duration = newcurrent_time - current_time
current_time = newcurrent_time
try:
frame = sys._current_frames()[self._thread_to_monitor]
except KeyError:
log('Can\'t get the stacktrace of the main thread. Stopping StacktraceProfiler'
)
break
in_endpoint_code = False
self._path_hash.set_path('')
# filename, line number, function name, source code line
for fn, ln, fun, line in traceback.extract_stack(frame):
if self._endpoint.name == fun:
in_endpoint_code = True
if in_endpoint_code:
key = (self._path_hash.get_path(fn, ln), fun, line)
self._histogram[key] += duration
if len(fn) > FILENAME_LEN and fn[
-FILENAME_LEN:] == FILENAME and fun == "wrapper":
in_endpoint_code = True
if in_endpoint_code:
self._total += duration
elapsed = time.time() - current_time
if config.sampling_period > elapsed:
time.sleep(config.sampling_period - elapsed)
self._on_thread_stopped()
def bind(app):
"""
Binding the app to this object should happen before importing the routing-
methods below. Thus, the importing statement is part of this function.
:param app: the app for which the performance has to be tracked
"""
from flask_monitoringdashboard.core.logger import log
assert app is not None
global user_app, blueprint
user_app = app
# Provide a secret-key for using WTF-forms
if not user_app.secret_key:
log('WARNING: You should provide a security key.')
user_app.secret_key = 'my-secret-key'
import os
# Only initialize unit test logging when running on Travis.
if 'TRAVIS' in os.environ:
import datetime
from flask import request
@user_app.before_first_request
def log_current_version():
"""
Logs the version of the user app that is currently being tested.
:return:
"""
home = os.path.expanduser("~")
with open(home + '/app_version.log', 'w') as app_log:
app_log.write(config.version)
@user_app.before_request
def log_start_endpoint_hit():
"""
Add log_start_endpoint_hit as before_request function that logs the start of the endpoint hit.
:return:
"""
hit_time_stamp = str(datetime.datetime.utcnow())
home = os.path.expanduser("~")
with open(home + '/start_endpoint_hits.log', 'a') as hits_log:
hits_log.write('"{}","{}"\n'.format(hit_time_stamp,
request.endpoint))
@user_app.after_request
def log_finish_endpoint_hit(response):
"""
Add log_finish_endpoint_hit as after_request function that logs the finish of the endpoint hit.
:param response: the response object that the actual endpoint returns
:return: the unchanged response of the original endpoint
"""
hit_time_stamp = str(datetime.datetime.utcnow())
home = os.path.expanduser("~")
with open(home + '/finish_endpoint_hits.log', 'a') as hits2_log:
hits2_log.write('"{}","{}"\n'.format(hit_time_stamp,
request.endpoint))
return response
else:
# Add all route-functions to the blueprint
import flask_monitoringdashboard.views
# Add wrappers to the endpoints that have to be monitored
from flask_monitoringdashboard.core.measurement import init_measurement
blueprint.before_app_first_request(init_measurement)
# register the blueprint to the app
app.register_blueprint(blueprint, url_prefix='/' + config.link)
def init_from(self, file=None, envvar=None, log_verbose=False):
"""
The config_file must at least contains the following variables in section 'dashboard':
- APP_VERSION: the version of the app that you use. Updating the version helps in
showing differences in execution times of a function over a period of time.
- GIT = If you're using git, then it is easier to set the location to the .git-folder,
The location is relative to the config-file.
- CUSTOM_LINK: The dashboard can be visited at localhost:5000/{CUSTOM_LINK}.
- MONITOR_LEVEL: The level for monitoring your endpoints. The default value is 3.
- OUTLIER_DETECTION_CONSTANT: When the execution time is more than this constant *
average, extra information is logged into the database. A default value for this
variable is 2.5.
- SAMPLING_PERIOD: Time between two profiler-samples. The time must be specified in ms.
If this value is not set, the profiler continuously monitors.
- ENABLE_LOGGING: Boolean if you want additional logs to be printed to the console. Default
value is False
The config_file must at least contains the following variables in section 'authentication':
- USERNAME: for logging into the dashboard, a username and password is required. The
username can be set using this variable.
- PASSWORD: same as for the username, but this is the password variable.
- GUEST_USERNAME: A guest can only see the results, but cannot configure/download data.
- GUEST_PASSWORD: A guest can only see the results, but cannot configure/download data.
- SECURITY_TOKEN: Used for getting the data in /get_json_data
The config_file must at least contains the following variables in section 'database':
- DATABASE: Suppose you have multiple projects where you're working on and want to
separate the results. Then you can specify different database_names, such that the
result of each project is stored in its own database.
- TABLE_PREFIX: A prefix to every table that the Flask-MonitoringDashboard uses, to ensure
that there are no conflicts with the user of the dashboard.
The config_file must at least contains the following variables in section 'visualization':
- TIMEZONE: The timezone for converting a UTC timestamp to a local timestamp.
for a list of all timezones, use the following:
>>> import pytz # pip install pytz
>>> print(pytz.all_timezones)
- COLORS: A dictionary to override the colors used per endpoint.
:param file: a string pointing to the location of the config-file.
:param envvar: a string specifying which environment variable holds the config file location
"""
if envvar:
file = os.getenv(envvar)
if log_verbose:
log("Running with config from: " + (str(file)))
if not file:
# Travis does not need a config file.
if '/home/travis/build/' in os.getcwd():
return
log("No configuration file specified. Please do so.")
return
try:
parser = configparser.RawConfigParser()
parser.read(file)
# parse 'dashboard'
self.version = parse_version(parser, 'dashboard', self.version)
self.link = parse_string(parser, 'dashboard', 'CUSTOM_LINK', self.link)
self.monitor_level = parse_literal(parser, 'dashboard', 'MONITOR_LEVEL', self.monitor_level)
self.outlier_detection_constant = parse_literal(parser, 'dashboard', 'OUTlIER_DETECTION_CONSTANT',
self.outlier_detection_constant)
self.sampling_period = parse_literal(parser, 'dashboard', 'SAMPLING_RATE', self.sampling_period) / 1000
self.enable_logging = parse_bool(parser, 'dashboard', 'ENABLE_LOGGING', self.enable_logging)
# parse 'authentication'
self.username = parse_string(parser, 'authentication', 'USERNAME', self.username)
self.password = parse_string(parser, 'authentication', 'PASSWORD', self.password)
self.security_token = parse_string(parser, 'authentication', 'SECURITY_TOKEN', self.security_token)
self.guest_username = parse_string(parser, 'authentication', 'GUEST_USERNAME', self.guest_username)
self.guest_password = parse_literal(parser, 'authentication', 'GUEST_PASSWORD', self.guest_password)
# database
self.database_name = parse_string(parser, 'database', 'DATABASE', self.database_name)
self.table_prefix = parse_string(parser, 'database', 'TABLE_PREFIX', self.table_prefix)
# visualization
self.colors = parse_literal(parser, 'visualization', 'COLORS', self.colors)
self.timezone = pytz.timezone(parse_string(parser, 'visualization', 'TIMEZONE', self.timezone.zone))
if log_verbose:
log("version: " + self.version)
log("username: " + self.username)
except AttributeError:
log('Cannot use configparser in python2.7')
raise
