def bind(app, schedule=True): """ Binding the app to this object should happen before importing the routing- methods below. Thus, the importing statement is part of this function. :param app: the app for which the performance has to be tracked :param schedule: flag telling if the background scheduler should be started """ config.app = app # Provide a secret-key for using WTF-forms if not app.secret_key: log('WARNING: You should provide a security key.') app.secret_key = 'my-secret-key' # Add all route-functions to the blueprint from flask_monitoringdashboard.views import deployment, custom, endpoint, outlier, request, profiler, version, auth import flask_monitoringdashboard.views # Add wrappers to the endpoints that have to be monitored from flask_monitoringdashboard.core.measurement import init_measurement from flask_monitoringdashboard.core import custom_graph blueprint.before_app_first_request(init_measurement) if schedule: custom_graph.init(app) # register the blueprint to the app app.register_blueprint(blueprint, url_prefix='/' + config.link)
def parse_version(parser, header, version): """ Parse the version given in the config-file. If both GIT and VERSION are used, the GIT argument is used. :param parser: the parser to be used for parsing :param header: name of the header in the configuration file :param version: the default version """ version = parse_string(parser, header, 'APP_VERSION', version) if parser.has_option(header, 'GIT'): git = parser.get(header, 'GIT') try: # current hash can be found in the link in HEAD-file in git-folder # The file is specified by: 'ref: <location>' git_file = (open(os.path.join(git, 'HEAD')).read().rsplit( ': ', 1)[1]).rstrip() # read the git-version version = open(git + '/' + git_file).read() # cut version to at most 6 chars return version[:6] except IOError: log("Error reading one of the files to retrieve the current git-version." ) raise return version
def run(self): # sleep for average * ODC ms average = get_avg_endpoint( self._endpoint.name) * config.outlier_detection_constant self._exit.wait(average / 1000) if not self._exit.is_set(): stack_list = [] try: frame = sys._current_frames()[self._current_thread] except KeyError: log('Can\'t get the stacktrace of the main thread.') return in_endpoint_code = False # filename, line number, function name, source code line for fn, ln, fun, line in traceback.extract_stack(frame): if self._endpoint.name == fun: in_endpoint_code = True if in_endpoint_code: stack_list.append( 'File: "{}", line {}, in "{}": "{}"'.format( fn, ln, fun, line)) # Set the values in the object self._stacktrace = '<br />'.join(stack_list) self._cpu_percent = str( psutil.cpu_percent(interval=None, percpu=True)) self._memory = str(psutil.virtual_memory())
def __init__(self): """Sets the default values for the project.""" # dashboard self.version = '1.0' self.blueprint_name = 'dashboard' self.link = 'dashboard' self.monitor_level = 1 self.outlier_detection_constant = 2.5 self.sampling_period = 5 / 1000.0 self.enable_logging = False # database self.database_name = 'sqlite:///flask_monitoringdashboard.db' self.table_prefix = '' # authentication self.username = '******' self.password = '******' self.security_token = 'cc83733cb0af8b884ff6577086b87909' # visualization self.colors = {} try: self.timezone = pytz.timezone(str(get_localzone())) except pytz.UnknownTimeZoneError: log('Using default timezone, which is UTC') self.timezone = pytz.timezone('UTC') # define a custom function to retrieve the session_id or username self.group_by = None # store the Flask app self.app = None
def parse_version(parser, header, version): """ Parse the version given in the config-file. If both GIT and VERSION are used, the GIT argument is used. :param parser: the parser to be used for parsing :param header: name of the header in the configuration file :param version: the default version """ version = parse_string(parser, header, 'APP_VERSION', version) if parser.has_option(header, 'GIT'): git = parser.get(header, 'GIT') try: # current hash can be found in the link in HEAD-file in git-folder # The file is specified by: 'ref: <location>' git_head = os.path.join(git, 'HEAD') if os.path.isfile(git_head): git_file = (open(git_head).read().rsplit(': ', 1)[1]).rstrip() # read the git-version version_file = os.path.join(git, git_file) if os.path.exists(version_file): version = open(version_file).read() # cut version to at most 6 chars return version[:6] else: # Return "dummy" version in case of no git version file found log("Folder {} not found, using dummy version: {}".format( git_head, version)) return version except IOError: log("Error reading one of the files to retrieve the current git-version." ) raise return version
def bind(app, schedule=True, include_dashboard=True): """Binding the app to this object should happen before importing the routing- methods below. Thus, the importing statement is part of this function. :param app: the app for which the performance has to be tracked :param schedule: flag telling if the background scheduler should be started :param include_dashboard: flag telling if the views should be added or not. """ blueprint.name = config.blueprint_name config.app = app # Provide a secret-key for using WTF-forms if not app.secret_key: log('WARNING: You should provide a security key.') app.secret_key = 'my-secret-key' # Add all route-functions to the blueprint if include_dashboard: from flask_monitoringdashboard.views import ( deployment, custom, endpoint, outlier, request, profiler, version, auth, reporting, ) import flask_monitoringdashboard.views # Add wrappers to the endpoints that have to be monitored from flask_monitoringdashboard.core.measurement import init_measurement from flask_monitoringdashboard.core.cache import init_cache from flask_monitoringdashboard.core import custom_graph blueprint.before_app_first_request(init_measurement) blueprint.before_app_first_request(init_cache) if schedule: custom_graph.init(app) # register the blueprint to the app app.register_blueprint(blueprint, url_prefix='/' + config.link) # flush cache to db before shutdown import atexit from flask_monitoringdashboard.core.cache import flush_cache atexit.register(flush_cache) if not include_dashboard: @app.teardown_request def teardown(_): flush_cache()
def get_ip(): """ :return: the ip address associated with the current request context """ if config.get_ip: try: return config.get_ip() except Exception as e: log('Failed to execute provided get_ip function: {}'.format(e)) # This is a reasonable fallback, but will not work for clients behind proxies. return request.environ['REMOTE_ADDR']
def get_group_by(): """ :return: a string with the value """ group_by = None try: if config.group_by: group_by = recursive_group_by(config.group_by) except Exception as e: log('Can\'t execute group_by function: {}'.format(e)) return str(group_by)
def get_outlier_table(session, endpoint_id, offset, per_page): """ :param session: session for the database :param endpoint_id: id of the endpoint :param offset: number of items to be skipped :param per_page: maximum number of items to be returned :return: a list of length at most 'per_page' with data about each outlier """ table = get_outliers_sorted(session, endpoint_id, offset, per_page) for idx, row in enumerate(table): row.request.time_requested = to_local_datetime(row.request.time_requested) try: row.request_url = row.request_url.decode('utf-8') except Exception as e: log(e) dict_request = row2dict(row.request) table[idx] = row2dict(row) table[idx]['request'] = dict_request return table
def run(self): """ Continuously takes a snapshot from the stacktrace (only the main-thread). Filters everything before the endpoint has been called (i.e. the Flask library). Directly computes the histogram, since this is more efficient for performance :return: """ current_time = time.time() while self._keeprunning: newcurrent_time = time.time() duration = newcurrent_time - current_time current_time = newcurrent_time try: frame = sys._current_frames()[self._thread_to_monitor] except KeyError: log('Can\'t get the stacktrace of the main thread. Stopping StacktraceProfiler' ) break in_endpoint_code = False self._path_hash.set_path('') # filename, line number, function name, source code line for fn, ln, fun, line in traceback.extract_stack(frame): if self._endpoint.name == fun: in_endpoint_code = True if in_endpoint_code: key = (self._path_hash.get_path(fn, ln), fun, line) self._histogram[key] += duration if len(fn) > FILENAME_LEN and fn[ -FILENAME_LEN:] == FILENAME and fun == "wrapper": in_endpoint_code = True if in_endpoint_code: self._total += duration elapsed = time.time() - current_time if config.sampling_period > elapsed: time.sleep(config.sampling_period - elapsed) self._on_thread_stopped()
def bind(app): """ Binding the app to this object should happen before importing the routing- methods below. Thus, the importing statement is part of this function. :param app: the app for which the performance has to be tracked """ from flask_monitoringdashboard.core.logger import log assert app is not None global user_app, blueprint user_app = app # Provide a secret-key for using WTF-forms if not user_app.secret_key: log('WARNING: You should provide a security key.') user_app.secret_key = 'my-secret-key' import os # Only initialize unit test logging when running on Travis. if 'TRAVIS' in os.environ: import datetime from flask import request @user_app.before_first_request def log_current_version(): """ Logs the version of the user app that is currently being tested. :return: """ home = os.path.expanduser("~") with open(home + '/app_version.log', 'w') as app_log: app_log.write(config.version) @user_app.before_request def log_start_endpoint_hit(): """ Add log_start_endpoint_hit as before_request function that logs the start of the endpoint hit. :return: """ hit_time_stamp = str(datetime.datetime.utcnow()) home = os.path.expanduser("~") with open(home + '/start_endpoint_hits.log', 'a') as hits_log: hits_log.write('"{}","{}"\n'.format(hit_time_stamp, request.endpoint)) @user_app.after_request def log_finish_endpoint_hit(response): """ Add log_finish_endpoint_hit as after_request function that logs the finish of the endpoint hit. :param response: the response object that the actual endpoint returns :return: the unchanged response of the original endpoint """ hit_time_stamp = str(datetime.datetime.utcnow()) home = os.path.expanduser("~") with open(home + '/finish_endpoint_hits.log', 'a') as hits2_log: hits2_log.write('"{}","{}"\n'.format(hit_time_stamp, request.endpoint)) return response else: # Add all route-functions to the blueprint import flask_monitoringdashboard.views # Add wrappers to the endpoints that have to be monitored from flask_monitoringdashboard.core.measurement import init_measurement blueprint.before_app_first_request(init_measurement) # register the blueprint to the app app.register_blueprint(blueprint, url_prefix='/' + config.link)
def init_from(self, file=None, envvar=None, log_verbose=False): """ The config_file must at least contains the following variables in section 'dashboard': - APP_VERSION: the version of the app that you use. Updating the version helps in showing differences in execution times of a function over a period of time. - GIT = If you're using git, then it is easier to set the location to the .git-folder, The location is relative to the config-file. - CUSTOM_LINK: The dashboard can be visited at localhost:5000/{CUSTOM_LINK}. - MONITOR_LEVEL: The level for monitoring your endpoints. The default value is 3. - OUTLIER_DETECTION_CONSTANT: When the execution time is more than this constant * average, extra information is logged into the database. A default value for this variable is 2.5. - SAMPLING_PERIOD: Time between two profiler-samples. The time must be specified in ms. If this value is not set, the profiler continuously monitors. - ENABLE_LOGGING: Boolean if you want additional logs to be printed to the console. Default value is False The config_file must at least contains the following variables in section 'authentication': - USERNAME: for logging into the dashboard, a username and password is required. The username can be set using this variable. - PASSWORD: same as for the username, but this is the password variable. - GUEST_USERNAME: A guest can only see the results, but cannot configure/download data. - GUEST_PASSWORD: A guest can only see the results, but cannot configure/download data. - SECURITY_TOKEN: Used for getting the data in /get_json_data The config_file must at least contains the following variables in section 'database': - DATABASE: Suppose you have multiple projects where you're working on and want to separate the results. Then you can specify different database_names, such that the result of each project is stored in its own database. - TABLE_PREFIX: A prefix to every table that the Flask-MonitoringDashboard uses, to ensure that there are no conflicts with the user of the dashboard. The config_file must at least contains the following variables in section 'visualization': - TIMEZONE: The timezone for converting a UTC timestamp to a local timestamp. for a list of all timezones, use the following: >>> import pytz # pip install pytz >>> print(pytz.all_timezones) - COLORS: A dictionary to override the colors used per endpoint. :param file: a string pointing to the location of the config-file. :param envvar: a string specifying which environment variable holds the config file location """ if envvar: file = os.getenv(envvar) if log_verbose: log("Running with config from: " + (str(file))) if not file: # Travis does not need a config file. if '/home/travis/build/' in os.getcwd(): return log("No configuration file specified. Please do so.") return try: parser = configparser.RawConfigParser() parser.read(file) # parse 'dashboard' self.version = parse_version(parser, 'dashboard', self.version) self.link = parse_string(parser, 'dashboard', 'CUSTOM_LINK', self.link) self.monitor_level = parse_literal(parser, 'dashboard', 'MONITOR_LEVEL', self.monitor_level) self.outlier_detection_constant = parse_literal(parser, 'dashboard', 'OUTlIER_DETECTION_CONSTANT', self.outlier_detection_constant) self.sampling_period = parse_literal(parser, 'dashboard', 'SAMPLING_RATE', self.sampling_period) / 1000 self.enable_logging = parse_bool(parser, 'dashboard', 'ENABLE_LOGGING', self.enable_logging) # parse 'authentication' self.username = parse_string(parser, 'authentication', 'USERNAME', self.username) self.password = parse_string(parser, 'authentication', 'PASSWORD', self.password) self.security_token = parse_string(parser, 'authentication', 'SECURITY_TOKEN', self.security_token) self.guest_username = parse_string(parser, 'authentication', 'GUEST_USERNAME', self.guest_username) self.guest_password = parse_literal(parser, 'authentication', 'GUEST_PASSWORD', self.guest_password) # database self.database_name = parse_string(parser, 'database', 'DATABASE', self.database_name) self.table_prefix = parse_string(parser, 'database', 'TABLE_PREFIX', self.table_prefix) # visualization self.colors = parse_literal(parser, 'visualization', 'COLORS', self.colors) self.timezone = pytz.timezone(parse_string(parser, 'visualization', 'TIMEZONE', self.timezone.zone)) if log_verbose: log("version: " + self.version) log("username: " + self.username) except AttributeError: log('Cannot use configparser in python2.7') raise