def testReadConfig(loggingMixin, configType, configTypeString):
""" Integration tests for reading a configuration for a particular module. """
# This could be different than configType if we want to use a string.
# We use a different object because we still want to use the standard config type later in the test.
configTypeForReadingConfig = configType
if configTypeString:
configTypeForReadingConfig = configType.name
(parameters, filesRead) = config.readConfig(configTypeForReadingConfig)
filename = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"configTestFiles",
"{}ConfigRef.yaml".format(configType.name))
# We need to treat whether the file exists with a bit of care.
# NOTE: Since the parametization causes this to run mulitple times, some will pass and
# and some will fail, even when creating the configuration files. This is fine.
if os.path.exists(filename):
# Access the expected values
with open(filename, "r") as f:
expected = yaml.load(f.read(), Loader=yaml.SafeLoader)
else:
# For making the reference
with open(filename, "w") as f:
yaml.dump(parameters, f)
logger.warning("Creating configuration reference for {} module".format(
configType.name))
# We don't want to go further - we're just creating the reference.
assert False
# Don't compare the full "_users" values because they will always be different due to differences in hashing
paramUsers = parameters.pop("_users", None)
expectedUsers = expected.pop("_users", None)
# However, the beginning should match (same idea as in `testBcrypt`)
lengthToCheck = 7
# It won't always exist, so we need to check for it first.
if paramUsers:
for k, v in iteritems(paramUsers):
assert v[:lengthToCheck] == expectedUsers[k][:lengthToCheck]
# Apparently the order of these lists can vary between different systems. We don't care about the order
# - just the values themselves - so we compare them as sets, which don't depend on order.
paramTemplates = parameters.pop("availableRunPageTemplates", None)
expectedTemplates = expected.pop("availableRunPageTemplates", None)
# It won't always exist, so we need to check for it first.
if paramTemplates:
assert set(paramTemplates) == set(expectedTemplates)
# Everything else should be identical.
assert parameters == expected
""" WSGI Server for handling POST requests containing DQM data.
.. codeauthor:: Raymond Ehlers <*****@*****.**>, Yale University
"""
# For python 3 support
from __future__ import print_function
# Python logging system
import logging
from overwatch.base import config
# For configuring logger
from overwatch.base import utilities
(receiverParameters,
filesRead) = config.readConfig(config.configurationType.dqmReceiver)
# Setup logger
# When imported, we just want it to take on it normal name
logger = logging.getLogger(__name__)
# Alternatively, we could set "overwatch.receiver" to get everything derived from that
#logger = logging.getLogger("overwatch.receiver")
from flask import Flask, url_for, request, render_template, redirect, flash, send_from_directory, Markup, jsonify, session
from werkzeug.utils import secure_filename
import os
import time
import functools
import ROOT
# Fix Flask debug mode with ROOT 5 issue.
#!/usr/bin/env python
# For python 3 support
from __future__ import print_function
import os
# Python logging system
import logging
# Configuration
from overwatch.base import config
## For configuring logger
from overwatch.base import utilities
(apiParameters,
filesRead) = config.readConfig(config.configurationType.apiConfig)
# Setup logger
# When imported, we just want it to take on it normal name
logger = logging.getLogger(__name__)
# Alternatively, we could set "overwatch.receiver" to get everything derived from that
#logger = logging.getLogger("overwatch.receiver")
from flask import Flask, url_for, request, render_template, redirect, flash, send_from_directory, Markup, jsonify, session, make_response
import flask_restful
import flask_zodb
from werkzeug.utils import secure_filename
app = Flask(__name__)
api = flask_restful.Api(app)
app.config["ZODB_STORAGE"] = apiParameters["databaseLocation"]
#!/usr/bin/env python
import logging
import pprint
# Config
#from config.processingParams import processingParameters
from overwatch.base import config
from overwatch.base import utilities
(processingParameters,
filesRead) = config.readConfig(config.configurationType.processing)
print("Configuration files read: {0}".format(filesRead))
print("processingParameters: {0}".format(pprint.pformat(processingParameters)))
# By not setting a name, we get everything!
logger = logging.getLogger("")
# Alternatively, we could set processRuns to get everything derived from that
#logger = logging.getLogger("processRuns")
# Setup logging
utilities.setupLogging(logger, processingParameters["loggingLevel"],
processingParameters["debug"], "processRuns")
# Log settings
logger.info(processingParameters)
# Imports are below here so that they can be logged
from overwatch.processing import processRuns
def run():
# Process all of the run data
"""
.. code-author: Mateusz Piwowarczyk <>, AGH University of Science and Technology
"""
import aenum
from overwatch.base import config
from overwatch.database.mongoDatabaseFactory import MongoDatabaseFactory
from overwatch.database.zodbDatabaseFactory import ZodbDatabaseFactory
(databaseParameters, _) = config.readConfig(config.configurationType.database)
class databaseTypes(aenum.OrderedEnum):
mongodb = 0
zodb = 1
def getDatabaseFactory():
""" Creates database factory object using parameters specified in config.yaml.
Args:
None
Returns:
Database factory object.
"""
databaseType = databaseParameters["databaseType"]
if databaseTypes[databaseType] == databaseTypes.mongodb:
return MongoDatabaseFactory(
databaseName=databaseParameters["databaseName"],
host=databaseParameters["mongoHost"],
port=databaseParameters["mongoPort"])
#!/usr/bin/env python
import logging
import socket
import pprint
# Config
from overwatch.base import config
# For configuring logger
from overwatch.base import utilities
(serverParameters, filesRead) = config.readConfig(config.configurationType.webApp)
print("Configuration files read: {0}".format(filesRead))
print("serverParameters: {0}".format(pprint.pformat(serverParameters)))
# By not setting a name, we get everything!
logger = logging.getLogger("")
# Alternatively, we could set "webApp" to get everything derived from that
#logger = logging.getLogger("webApp")
# Setup logger
utilities.setupLogging(logger, serverParameters["loggingLevel"], serverParameters["debug"], "webApp")
# Log server settings
logger.info(serverParameters)
# Imports are below here so that they can be logged
from overwatch.webApp.webApp import app
# Set the secret key here
if not serverParameters["debug"]:
# Connect to database ourselves and grab the secret key
(dbRoot, connection) = utilities.getDB(serverParameters["databaseLocation"])
import os
import pendulum
import pprint
import shutil
import transaction
import logging
# We want to log everything, so we give it empty quotes.
logger = logging.getLogger("")
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
# Config
from overwatch.base import config
from overwatch.base import utilities
(parameters, filesRead) = config.readConfig(config.configurationType.base)
print("Configuration files read: {filesRead}".format(filesRead = filesRead))
print("parameters: {parameters}".format(
parameters = pprint.pformat(parameters)))
# Setup logging
utilities.setupLogging(logger = logger,
logLevel = parameters["loggingLevel"],
debug = parameters["debug"])
# Log settings
logger.info(parameters)
# Setup sentry to create alerts for warning level messages. Those will include info level breadcrumbs.
sentry_logging = LoggingIntegration(level = logging.INFO, event_level = logging.WARNING)
# Usually, we want the module specific DSN, but we will take a generic one if it's the only one available.
sentryDSN = os.getenv("SENTRY_DSN_DATA_TRANSFER") or os.getenv("SENTRY_DSN")
.. codeauthor:: Raymond Ehlers <*****@*****.**>, Yale University """ # For python 3 support from __future__ import print_function from future.utils import iteritems import functools import os import logging logger = logging.getLogger(__name__) import pendulum from overwatch.base import config (receiverParameters, filesRead) = config.readConfig(config.configurationType.receiver) from flask import Flask, request, send_from_directory, jsonify, url_for from werkzeug.utils import secure_filename import ROOT # Fix Flask debug mode with ROOT 5 issue. # See: https://root-forum.cern.ch/t/pyroot-and-spyder-re-running-error/20926/5 ROOT.std.__file__ = "ROOT.std.py" #import rootpy.io #import rootpy.ROOT as ROOT app = Flask(__name__) # From: http://flask.pocoo.org/docs/0.12/patterns/apierrors/ class InvalidUsage(Exception):
#!/usr/bin/env python
""" Minimal executable wrapper to update users in the ZODB database.
``__main__`` is implemented to allow for this function to be executed directly,
while ``updateDBUsers()`` is defined to allow for execution via ``entry_points``
defined in the python package setup.
.. codeauthor:: Raymond Ehlers <*****@*****.**>, Yale University
"""
# Server configuration
from overwatch.base import config
(serverParameters,
filesRead) = config.readConfig(config.configurationType.webApp)
# Get the most useful fucntions
from overwatch.base import utilities
import logging
# By not setting a name, we get everything!
logger = logging.getLogger("")
# Alternatively, we could set processRuns to get everything derived from that
#logger = logging.getLogger("processRuns")
import pprint
# Setup logging
utilities.setupLogging(logger=logger,
logLevel=serverParameters["loggingLevel"],
debug=serverParameters["debug"])
# Log settings
logger.info("Settings: {serverParameters}".format(
#!/usr/bin/env python
import logging
import pprint
# Config
#from config.processingParams import processingParameters
from overwatch.base import config
from overwatch.base import utilities
(processingParameters, filesRead) = config.readConfig(config.configurationType.processing)
print("Configuration files read: {0}".format(filesRead))
print("processingParameters: {0}".format(pprint.pformat(processingParameters)))
# By not setting a name, we get everything!
logger = logging.getLogger("")
# Alternatively, we could set processRuns to get everything derived from that
#logger = logging.getLogger("processRuns")
# Setup logging
utilities.setupLogging(logger, processingParameters["loggingLevel"], processingParameters["debug"], "processRuns")
# Log settings
logger.info(processingParameters)
# Imports are below here so that they can be logged
from overwatch.processing import processRuns
def run():
# Process all of the run data
processRuns.processAllRuns()
# Function calls that be used for debugging
#!/usr/bin/env python
# For python 3 support
from __future__ import print_function
from future.utils import iteritems
from future.utils import itervalues
import os
# Python logging system
import logging
# Configuration
from overwatch.base import config
from overwatch.base import storageWrapper
(apiParameters, filesRead) = config.readConfig(config.configurationType.api)
# Setup logger
# When imported, we just want it to take on it normal name
logger = logging.getLogger(__name__)
# Alternatively, we could set "overwatch.receiver" to get everything derived from that
#logger = logging.getLogger("overwatch.receiver")
from flask import Flask, request, send_file, make_response
import flask_restful
import flask_zodb
from io import StringIO
from werkzeug.utils import secure_filename
app = Flask(__name__)
api = flask_restful.Api(app)
