def testReadConfig(loggingMixin, configType, configTypeString): """ Integration tests for reading a configuration for a particular module. """ # This could be different than configType if we want to use a string. # We use a different object because we still want to use the standard config type later in the test. configTypeForReadingConfig = configType if configTypeString: configTypeForReadingConfig = configType.name (parameters, filesRead) = config.readConfig(configTypeForReadingConfig) filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configTestFiles", "{}ConfigRef.yaml".format(configType.name)) # We need to treat whether the file exists with a bit of care. # NOTE: Since the parametization causes this to run mulitple times, some will pass and # and some will fail, even when creating the configuration files. This is fine. if os.path.exists(filename): # Access the expected values with open(filename, "r") as f: expected = yaml.load(f.read(), Loader=yaml.SafeLoader) else: # For making the reference with open(filename, "w") as f: yaml.dump(parameters, f) logger.warning("Creating configuration reference for {} module".format( configType.name)) # We don't want to go further - we're just creating the reference. assert False # Don't compare the full "_users" values because they will always be different due to differences in hashing paramUsers = parameters.pop("_users", None) expectedUsers = expected.pop("_users", None) # However, the beginning should match (same idea as in `testBcrypt`) lengthToCheck = 7 # It won't always exist, so we need to check for it first. if paramUsers: for k, v in iteritems(paramUsers): assert v[:lengthToCheck] == expectedUsers[k][:lengthToCheck] # Apparently the order of these lists can vary between different systems. We don't care about the order # - just the values themselves - so we compare them as sets, which don't depend on order. paramTemplates = parameters.pop("availableRunPageTemplates", None) expectedTemplates = expected.pop("availableRunPageTemplates", None) # It won't always exist, so we need to check for it first. if paramTemplates: assert set(paramTemplates) == set(expectedTemplates) # Everything else should be identical. assert parameters == expected
""" WSGI Server for handling POST requests containing DQM data. .. codeauthor:: Raymond Ehlers <*****@*****.**>, Yale University """ # For python 3 support from __future__ import print_function # Python logging system import logging from overwatch.base import config # For configuring logger from overwatch.base import utilities (receiverParameters, filesRead) = config.readConfig(config.configurationType.dqmReceiver) # Setup logger # When imported, we just want it to take on it normal name logger = logging.getLogger(__name__) # Alternatively, we could set "overwatch.receiver" to get everything derived from that #logger = logging.getLogger("overwatch.receiver") from flask import Flask, url_for, request, render_template, redirect, flash, send_from_directory, Markup, jsonify, session from werkzeug.utils import secure_filename import os import time import functools import ROOT # Fix Flask debug mode with ROOT 5 issue.
#!/usr/bin/env python # For python 3 support from __future__ import print_function import os # Python logging system import logging # Configuration from overwatch.base import config ## For configuring logger from overwatch.base import utilities (apiParameters, filesRead) = config.readConfig(config.configurationType.apiConfig) # Setup logger # When imported, we just want it to take on it normal name logger = logging.getLogger(__name__) # Alternatively, we could set "overwatch.receiver" to get everything derived from that #logger = logging.getLogger("overwatch.receiver") from flask import Flask, url_for, request, render_template, redirect, flash, send_from_directory, Markup, jsonify, session, make_response import flask_restful import flask_zodb from werkzeug.utils import secure_filename app = Flask(__name__) api = flask_restful.Api(app) app.config["ZODB_STORAGE"] = apiParameters["databaseLocation"]
#!/usr/bin/env python import logging import pprint # Config #from config.processingParams import processingParameters from overwatch.base import config from overwatch.base import utilities (processingParameters, filesRead) = config.readConfig(config.configurationType.processing) print("Configuration files read: {0}".format(filesRead)) print("processingParameters: {0}".format(pprint.pformat(processingParameters))) # By not setting a name, we get everything! logger = logging.getLogger("") # Alternatively, we could set processRuns to get everything derived from that #logger = logging.getLogger("processRuns") # Setup logging utilities.setupLogging(logger, processingParameters["loggingLevel"], processingParameters["debug"], "processRuns") # Log settings logger.info(processingParameters) # Imports are below here so that they can be logged from overwatch.processing import processRuns def run(): # Process all of the run data
""" .. code-author: Mateusz Piwowarczyk <>, AGH University of Science and Technology """ import aenum from overwatch.base import config from overwatch.database.mongoDatabaseFactory import MongoDatabaseFactory from overwatch.database.zodbDatabaseFactory import ZodbDatabaseFactory (databaseParameters, _) = config.readConfig(config.configurationType.database) class databaseTypes(aenum.OrderedEnum): mongodb = 0 zodb = 1 def getDatabaseFactory(): """ Creates database factory object using parameters specified in config.yaml. Args: None Returns: Database factory object. """ databaseType = databaseParameters["databaseType"] if databaseTypes[databaseType] == databaseTypes.mongodb: return MongoDatabaseFactory( databaseName=databaseParameters["databaseName"], host=databaseParameters["mongoHost"], port=databaseParameters["mongoPort"])
#!/usr/bin/env python import logging import socket import pprint # Config from overwatch.base import config # For configuring logger from overwatch.base import utilities (serverParameters, filesRead) = config.readConfig(config.configurationType.webApp) print("Configuration files read: {0}".format(filesRead)) print("serverParameters: {0}".format(pprint.pformat(serverParameters))) # By not setting a name, we get everything! logger = logging.getLogger("") # Alternatively, we could set "webApp" to get everything derived from that #logger = logging.getLogger("webApp") # Setup logger utilities.setupLogging(logger, serverParameters["loggingLevel"], serverParameters["debug"], "webApp") # Log server settings logger.info(serverParameters) # Imports are below here so that they can be logged from overwatch.webApp.webApp import app # Set the secret key here if not serverParameters["debug"]: # Connect to database ourselves and grab the secret key (dbRoot, connection) = utilities.getDB(serverParameters["databaseLocation"])
import os import pendulum import pprint import shutil import transaction import logging # We want to log everything, so we give it empty quotes. logger = logging.getLogger("") import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration # Config from overwatch.base import config from overwatch.base import utilities (parameters, filesRead) = config.readConfig(config.configurationType.base) print("Configuration files read: {filesRead}".format(filesRead = filesRead)) print("parameters: {parameters}".format( parameters = pprint.pformat(parameters))) # Setup logging utilities.setupLogging(logger = logger, logLevel = parameters["loggingLevel"], debug = parameters["debug"]) # Log settings logger.info(parameters) # Setup sentry to create alerts for warning level messages. Those will include info level breadcrumbs. sentry_logging = LoggingIntegration(level = logging.INFO, event_level = logging.WARNING) # Usually, we want the module specific DSN, but we will take a generic one if it's the only one available. sentryDSN = os.getenv("SENTRY_DSN_DATA_TRANSFER") or os.getenv("SENTRY_DSN")
.. codeauthor:: Raymond Ehlers <*****@*****.**>, Yale University """ # For python 3 support from __future__ import print_function from future.utils import iteritems import functools import os import logging logger = logging.getLogger(__name__) import pendulum from overwatch.base import config (receiverParameters, filesRead) = config.readConfig(config.configurationType.receiver) from flask import Flask, request, send_from_directory, jsonify, url_for from werkzeug.utils import secure_filename import ROOT # Fix Flask debug mode with ROOT 5 issue. # See: https://root-forum.cern.ch/t/pyroot-and-spyder-re-running-error/20926/5 ROOT.std.__file__ = "ROOT.std.py" #import rootpy.io #import rootpy.ROOT as ROOT app = Flask(__name__) # From: http://flask.pocoo.org/docs/0.12/patterns/apierrors/ class InvalidUsage(Exception):
#!/usr/bin/env python """ Minimal executable wrapper to update users in the ZODB database. ``__main__`` is implemented to allow for this function to be executed directly, while ``updateDBUsers()`` is defined to allow for execution via ``entry_points`` defined in the python package setup. .. codeauthor:: Raymond Ehlers <*****@*****.**>, Yale University """ # Server configuration from overwatch.base import config (serverParameters, filesRead) = config.readConfig(config.configurationType.webApp) # Get the most useful fucntions from overwatch.base import utilities import logging # By not setting a name, we get everything! logger = logging.getLogger("") # Alternatively, we could set processRuns to get everything derived from that #logger = logging.getLogger("processRuns") import pprint # Setup logging utilities.setupLogging(logger=logger, logLevel=serverParameters["loggingLevel"], debug=serverParameters["debug"]) # Log settings logger.info("Settings: {serverParameters}".format(
#!/usr/bin/env python import logging import pprint # Config #from config.processingParams import processingParameters from overwatch.base import config from overwatch.base import utilities (processingParameters, filesRead) = config.readConfig(config.configurationType.processing) print("Configuration files read: {0}".format(filesRead)) print("processingParameters: {0}".format(pprint.pformat(processingParameters))) # By not setting a name, we get everything! logger = logging.getLogger("") # Alternatively, we could set processRuns to get everything derived from that #logger = logging.getLogger("processRuns") # Setup logging utilities.setupLogging(logger, processingParameters["loggingLevel"], processingParameters["debug"], "processRuns") # Log settings logger.info(processingParameters) # Imports are below here so that they can be logged from overwatch.processing import processRuns def run(): # Process all of the run data processRuns.processAllRuns() # Function calls that be used for debugging
#!/usr/bin/env python # For python 3 support from __future__ import print_function from future.utils import iteritems from future.utils import itervalues import os # Python logging system import logging # Configuration from overwatch.base import config from overwatch.base import storageWrapper (apiParameters, filesRead) = config.readConfig(config.configurationType.api) # Setup logger # When imported, we just want it to take on it normal name logger = logging.getLogger(__name__) # Alternatively, we could set "overwatch.receiver" to get everything derived from that #logger = logging.getLogger("overwatch.receiver") from flask import Flask, request, send_file, make_response import flask_restful import flask_zodb from io import StringIO from werkzeug.utils import secure_filename app = Flask(__name__) api = flask_restful.Api(app)