def load_graph_from_db(time_limit):
config = Config('./')
config.from_pyfile('web_config.cfg')
with NodeDB(config) as db:
nodes = db.get_nodes(time_limit)
edges = db.get_edges(nodes, 60*60*24*7)
return (nodes, edges)
def test():
config = Config('application/config/')
config.from_pyfile('testing.cfg')
#db('createall', database=config['POSTGRESQL_DATABASE_DB'])
print ' [START] Testing server'
proc = subprocess.Popen(["python", "runserver.py", "testing"])
time.sleep(5)
call(["python", "application/tests/testsuite.py"])
print 'KILLING THE TESTING SERVER...'
time.sleep(5)
os.kill(proc.pid, signal.SIGTERM)
for pid in children_pid(proc.pid):
os.kill(pid, signal.SIGTERM)
def main():
if len(sys.argv) > 1 and len(sys.argv) != 4:
print "You must either specify no parameters or exactly 3: <username> <password> <tenant>.\n" \
"If you specify no parameters, credentials and tenant will be taken from config"
exit
scriptDir = os.path.dirname(os.path.realpath(__file__))
config = Config(scriptDir + '/../etc')
config.from_pyfile('local.cfg')
print "Configuration has been loaded from 'etc/local.cfg'"
if len(sys.argv) == 4:
user = sys.argv[1]
password = sys.argv[2]
tenant = sys.argv[3]
else:
print "You didn't provided credentials, using ones found in config"
user = config['OS_ADMIN_USER']
password = config['OS_ADMIN_PASSWORD']
tenant = config['OS_ADMIN_TENANT']
protocol = config['OS_AUTH_PROTOCOL']
host = config['OS_AUTH_HOST']
port = config['OS_AUTH_PORT']
auth_url = "%s://%s:%s/v2.0/" % (protocol, host, port)
print "User: %s" % user
print "Password: %s" % password
print "Tenant: %s" % tenant
print "Auth URL: %s" % auth_url
keystone = keystone_client(
username=user,
password=password,
tenant_name=tenant,
auth_url=auth_url
)
result = keystone.authenticate()
print "Auth succeed: %s" % result
print "Auth token: %s" % keystone.auth_token
print "Tenant [%s] id: %s" % (tenant, keystone.tenant_id)
def run():
"""Parse the command line arguments and run the application"""
args = parser.parse_args()
config = Config(getcwd())
# Set the configuration options from the file given
if args.conf:
config.from_pyfile(args.conf)
# Load the rest of the arguments, overriding the conf file
config['DEBUG'] = args.debug
# Create the application and create the database
app = create_app(config)
# Show the application config
if args.show:
from pprint import pprint
pprint(dict(app.config))
# Run the application
app.run(threaded=True)
class Configurator(object):
"""
Object that takes care of loading the different configurations from the
different sources. There are 3 types of settings:
* Project: The basic set of settings needed by the system. These are
shipped with Shiva.
* Local: Specific to each instance, useful for overwriting system
settings. Some of them must be defined before running Shiva, like the
DB URI.
* Debug: This setting will only be loaded if ``DEBUG`` is set to True
in the local settings.
There are also 3 different places where Shiva looks for this config files:
* A ``local.py`` file inside the ``config/`` directory, relative to
Shiva.
* The ``$SHIVA_CONFIG`` environment variable. It's assumed to be
pointing to a file (not a dir) if exists.
* The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If
``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as
defined by the `XDG Base Directory Specification
<http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html>`_.
"""
def __init__(self):
self._config = FlaskConfig('')
# project
_project = self.load_project()
# local
_xdg_config = self.from_xdg_config()
_env = self.from_env()
_local = self.from_local()
# debug
_debug = self.load_debug()
self.extract_conf(self._config)
if not (_xdg_config or _env or _local):
raise NoConfigFoundError
def load_project(self):
return self._config.from_object(project)
def get_xdg_path(self):
default_config_home = os.path.join(os.getenv('HOME'), '.config')
return os.getenv('XDG_CONFIG_HOME') or default_config_home
def from_xdg_config(self):
local_py = os.path.join(self.get_xdg_path(), 'shiva/config.py')
if not os.path.exists(local_py):
return False
return self._config.from_pyfile(local_py)
def from_env(self):
if not os.getenv('SHIVA_CONFIG'):
return False
return self._config.from_envvar('SHIVA_CONFIG')
def from_local(self):
with ignored(ImportError):
self._config.from_object('shiva.config.local')
return True
return False
def load_debug(self):
if not self._config.get('DEBUG'):
return False
loaded = False
with ignored(ImportError):
from shiva.config import debug
loaded = self._config.from_object(debug)
debug_py = os.path.join(self.get_xdg_path(), 'shiva/debug.py')
if not os.path.exists(debug_py):
return False
return self._config.from_pyfile(debug_py) or loaded
def extract_conf(self, *args):
"""
Receives one or more objects, iterates over their elements, extracts
all the uppercase properties and injects them into the Configurator
object.
"""
for obj in args:
for key in dir(obj):
if key.isupper():
setattr(self, key, getattr(obj, key))
if hasattr(obj, 'iterkeys') and callable(getattr(obj, 'iterkeys')):
for key in obj.iterkeys():
if key.isupper():
setattr(self, key, obj[key])
client_ip_fingerprint,
browser_fingerprint,
client_ip,
user_agent,
accept,
accept_encoding,
accept_language)
return is_valid
if __name__ == '__main__':
conf = Config('..')
test_request_data = dict(
accept_encoding='Accept-Encoding: gzip, deflate, sdch',
accept_language='Accept-Language: en-US,en;q=0.8',
accept='Accept: application/json, text/javascript, */*; q=0.01',
user_agent='User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
vm_ip='128.196.64.214',
client_ip='127.0.0.1'
)
conf.from_pyfile('default_settings.py')
# conf.from_pyfile('local_settings.py')
signature = generate_signature_01(conf, test_request_data)
print signature
values, timestamp = decode_signature_02(conf, signature)
print values
is_valid = validate_fingerprints_03(conf, values, test_request_data)
print is_valid
console_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s [in %(module)s @ %(pathname)s:%(lineno)d]"))
logger = logging.Logger("PROXY", level=logging.DEBUG)
logger.addHandler(file_handler)
logger.addHandler(console_handler)
# Log startup
logger.info('---------- Proxy starting up!')
try:
# Load configuration
logger.info('Loading configuration')
config_file = os.path.join(instance_path, 'app.cfg')
default_config_file = os.path.join(resource_path, 'default_app.cfg')
config = Config(instance_path)
config.from_pyfile(default_config_file)
config.from_pyfile(config_file)
environment = config['ENVIRONMENT']
logLevel = None
if environment == 'PRODUCTION':
file_handler.setLevel(logging.WARNING)
console_handler.setLevel(logging.WARNING)
logger.setLevel(logging.WARNING)
except:
logger.exception('Exception during startup actions')
raise
finalized = False
proxy = None
def get_user_config():
from flask import Config
config = Config('/')
config.from_object('alerta.settings')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
if 'DEBUG' in os.environ:
config['DEBUG'] = True
if 'BASE_URL' in os.environ:
config['BASE_URL'] = os.environ['BASE_URL']
if 'SECRET_KEY' in os.environ:
config['SECRET_KEY'] = os.environ['SECRET_KEY']
database_url = (
os.environ.get('DATABASE_URL', None) or
# The following database settings are deprecated.
os.environ.get('MONGO_URI', None) or
os.environ.get('MONGODB_URI', None) or
os.environ.get('MONGOHQ_URL', None) or
os.environ.get('MONGOLAB_URI', None)
)
# Use app config for DATABASE_URL if no env var from above override it
config['DATABASE_URL'] = database_url or config['DATABASE_URL']
if 'DATABASE_NAME' in os.environ:
config['DATABASE_NAME'] = os.environ['DATABASE_NAME']
if 'AUTH_REQUIRED' in os.environ:
config['AUTH_REQUIRED'] = True if os.environ['AUTH_REQUIRED'] == 'True' else False
if 'AUTH_PROVIDER' in os.environ:
config['AUTH_PROVIDER'] = os.environ['AUTH_PROVIDER']
if 'ADMIN_USERS' in os.environ:
config['ADMIN_USERS'] = os.environ['ADMIN_USERS'].split(',')
if 'CUSTOMER_VIEWS' in os.environ:
config['CUSTOMER_VIEWS'] = True if os.environ['CUSTOMER_VIEWS'] == 'True' else False
if 'OAUTH2_CLIENT_ID' in os.environ:
config['OAUTH2_CLIENT_ID'] = os.environ['OAUTH2_CLIENT_ID']
if 'OAUTH2_CLIENT_SECRET' in os.environ:
config['OAUTH2_CLIENT_SECRET'] = os.environ['OAUTH2_CLIENT_SECRET']
if 'ALLOWED_EMAIL_DOMAINS' in os.environ:
config['ALLOWED_EMAIL_DOMAINS'] = os.environ['ALLOWED_EMAIL_DOMAINS'].split(',')
if 'GITHUB_URL' in os.environ:
config['GITHUB_URL'] = os.environ['GITHUB_URL']
if 'ALLOWED_GITHUB_ORGS' in os.environ:
config['ALLOWED_GITHUB_ORGS'] = os.environ['ALLOWED_GITHUB_ORGS'].split(',')
if 'GITLAB_URL' in os.environ:
config['GITLAB_URL'] = os.environ['GITLAB_URL']
if 'ALLOWED_GITLAB_GROUPS' in os.environ:
config['ALLOWED_GITLAB_GROUPS'] = os.environ['ALLOWED_GITLAB_GROUPS'].split(',')
if 'KEYCLOAK_URL' in os.environ:
config['KEYCLOAK_URL'] = os.environ['KEYCLOAK_URL']
if 'KEYCLOAK_REALM' in os.environ:
config['KEYCLOAK_REALM'] = os.environ['KEYCLOAK_REALM']
if 'ALLOWED_KEYCLOAK_ROLES' in os.environ:
config['ALLOWED_KEYCLOAK_ROLES'] = os.environ['ALLOWED_KEYCLOAK_ROLES'].split(',')
if 'PINGFEDERATE_OPENID_ACCESS_TOKEN_URL' in os.environ:
config['PINGFEDERATE_OPENID_ACCESS_TOKEN_URL'] = os.environ['PINGFEDERATE_OPENID_ACCESS_TOKEN_URL'].split(
',')
if 'PINGFEDERATE_OPENID_PAYLOAD_USERNAME' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_USERNAME'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_USERNAME'].split(
',')
if 'PINGFEDERATE_OPENID_PAYLOAD_EMAIL' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_EMAIL'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_EMAIL'].split(',')
if 'PINGFEDERATE_OPENID_PAYLOAD_GROUP' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_GROUP'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_GROUP'].split(',')
if 'PINGFEDERATE_PUBKEY_LOCATION' in os.environ:
config['PINGFEDERATE_PUBKEY_LOCATION'] = os.environ['PINGFEDERATE_PUBKEY_LOCATION'].split(',')
if 'PINGFEDERATE_TOKEN_ALGORITHM' in os.environ:
config['PINGFEDERATE_TOKEN_ALGORITHM'] = os.environ['PINGFEDERATE_TOKEN_ALGORITHM'].split(',')
if 'CORS_ORIGINS' in os.environ:
config['CORS_ORIGINS'] = os.environ['CORS_ORIGINS'].split(',')
if 'MAIL_FROM' in os.environ:
config['MAIL_FROM'] = os.environ['MAIL_FROM']
if 'SMTP_PASSWORD' in os.environ:
config['SMTP_PASSWORD'] = os.environ['SMTP_PASSWORD']
if 'GOOGLE_TRACKING_ID' in os.environ:
config['GOOGLE_TRACKING_ID'] = os.environ['GOOGLE_TRACKING_ID']
if 'PLUGINS' in os.environ:
config['PLUGINS'] = os.environ['PLUGINS'].split(',')
# Runtime config check
if config['CUSTOMER_VIEWS'] and not config['AUTH_REQUIRED']:
raise RuntimeError('Must enable authentication to use customer views')
if config['CUSTOMER_VIEWS'] and not config['ADMIN_USERS']:
raise RuntimeError('Customer views is enabled but there are no admin users')
return config
def create_app(config_file=None, **kwargs):
"""
Create a new eve app object and initialize everything.
User configuration can be loaded in the following order:
1. Use the `config_file` arg to specify a file
2. If `config_file` is `None`, you set the environment variable
`AMIVAPI_CONFIG` to the path of your config file
3. If no environment variable is set either, `config.py` in the current
working directory is used
Args:
config (path): Specify config file to use.
kwargs: All other key-value arguments will be used to update the config
Returns:
(Eve): The Eve application
"""
# Load config
config = Config(getcwd())
config.from_object("amivapi.settings")
# Specified path > environment var > default path; abspath for better log
user_config = abspath(config_file or getenv('AMIVAPI_CONFIG', 'config.py'))
try:
config.from_pyfile(user_config)
config_status = "Config loaded: %s" % user_config
except IOError:
config_status = "No config found."
config.update(kwargs)
# Initialize empty domain to create Eve object, register resources later
config['DOMAIN'] = {}
app = Eve(
"amivapi", # Flask needs this name to find the static folder
settings=config,
validator=ValidatorAMIV)
app.logger.info(config_status)
# Set up error logging with sentry
init_sentry(app)
# Create LDAP connector
ldap.init_app(app)
# Initialize modules to register resources, validation, hooks, auth, etc.
users.init_app(app)
auth.init_app(app)
events.init_app(app)
groups.init_app(app)
blacklist.init_app(app)
joboffers.init_app(app)
studydocs.init_app(app)
cascade.init_app(app)
cron.init_app(app)
documentation.init_app(app)
# Fix that eve doesn't run hooks on embedded documents
app.on_fetched_item += utils.run_embedded_hooks_fetched_item
app.on_fetched_resource += utils.run_embedded_hooks_fetched_resource
return app
from asyncio import get_event_loop, coroutine, gather, wait
from io import BytesIO
import logging
from os.path import dirname
from aiohttp import web
from flask import Config
import arrow
import pygal
from .redmine import AsyncRedmine
config = Config(dirname(__file__))
config.from_pyfile('default.cfg')
config.from_pyfile('settings.cfg')
pygal_config = pygal.Config(style=config['PYGAL_STYLE'])
redmine = AsyncRedmine(config['REDMINE_URL'], key=config['API_KEY'])
access_logger = logging.getLogger('aiohttp.access')
access_logger.addHandler(logging.StreamHandler())
access_logger.setLevel(logging.INFO)
issue_statuses = []
def main():
loop = get_event_loop()
loop.run_until_complete(redminecharts(loop))
import requests
from bs4 import BeautifulSoup as bs
import csv
from zipfile import ZipFile
import dateutil.parser
from datetime import datetime
import mysql.connector
from flask import Config
config = Config('..')
config.from_pyfile('settings.cfg')
conn = mysql.connector.connect(**config['BRAZIL_DB'])
url = 'http://www.portaltransparencia.gov.br/download-de-dados/ceaf'
def get_date_info():
page = requests.get(url)
soup = bs(page.content, 'html.parser')
script_load = ''
date = {'ano': '', 'mes': '', 'dia': ''}
for script in soup.find_all('script'):
for s in script.contents:
if 'arquivos.push' in s:
script_load = s.split('"')
date['ano'] = script_load[script_load.index('ano') + 2]
date['mes'] = script_load[script_load.index('mes') + 2]
date['dia'] = script_load[script_load.index('dia') + 2]
def create_app(config_file=None, **kwargs):
"""
Create a new eve app object and initialize everything.
User configuration can be loaded in the following order:
1. Use the `config_file` arg to specify a file
2. If `config_file` is `None`, you set the environment variable
`AMIVAPI_CONFIG` to the path of your config file
3. If no environment variable is set either, `config.py` in the current
working directory is used
Args:
config (path): Specify config file to use.
kwargs: All other key-value arguments will be used to update the config
Returns:
(Eve): The Eve application
"""
# Load config
config = Config(getcwd())
config.from_object("amivapi.settings")
# Specified path > environment var > default path; abspath for better log
user_config = abspath(config_file or getenv('AMIVAPI_CONFIG', 'config.py'))
try:
config.from_pyfile(user_config)
config_status = "Config loaded: %s" % user_config
except IOError:
config_status = "No config found."
config.update(kwargs)
# Initialize empty domain to create Eve object, register resources later
config['DOMAIN'] = {}
app = Eve("amivapi", # Flask needs this name to find the static folder
settings=config,
validator=ValidatorAMIV)
app.logger.info(config_status)
# Set up error logging with sentry
init_sentry(app)
# Create LDAP connector
ldap.init_app(app)
# Initialize modules to register resources, validation, hooks, auth, etc.
users.init_app(app)
auth.init_app(app)
events.init_app(app)
groups.init_app(app)
blacklist.init_app(app)
joboffers.init_app(app)
beverages.init_app(app)
studydocs.init_app(app)
cascade.init_app(app)
cron.init_app(app)
documentation.init_app(app)
# Fix that eve doesn't run hooks on embedded documents
app.on_fetched_item += utils.run_embedded_hooks_fetched_item
app.on_fetched_resource += utils.run_embedded_hooks_fetched_resource
return app
def main(argconfig):
# Load the config file the same way Flask does which Chill uses.
config_file = argconfig if argconfig[0] == os.sep else os.path.join(os.getcwd(), argconfig)
config = Config(os.getcwd())
config.from_pyfile(config_file)
pyrax.set_credential_file(config["RACKSPACE_CREDENTIAL_FILE"])
cf = pyrax.cloudfiles
# sync sync_folder_to_container
local = config["FREEZER_DESTINATION"]
container_name = config.get("RACKSPACE_CONTAINER_NAME", os.path.basename(os.getcwd()))
prefix = config.get("PUBLIC_URL_PREFIX", None)
if prefix:
for page in (INDEX_PAGE, ERROR_PAGE, "401{0}".format(ERROR_PAGE), "404{0}".format(ERROR_PAGE)):
# If no index.html or error pages then link to the one found in
# PUBLIC_URL_PREFIX
if not os.path.exists(os.path.join(local, page)) and os.path.exists(os.path.join(local, prefix[1:], page)):
print "Creating hard link for {0}".format(page)
print "{0} -> {1}".format(os.path.join(local, page), os.path.join(local, prefix[1:], page))
os.link(os.path.join(local, prefix[1:], page), os.path.join(local, page))
confirm = raw_input(
"\nsync the folder: {local} \nto rackspace cloudfiles container: {container_name}\n[n]/y\n".format(**locals())
)
if confirm != "y":
return
remote = cf.create_container(container_name)
local_files = set()
for root, dir, files in os.walk(local):
for f in files:
local_files.add(os.path.join(root[len(local) + 1 :], f))
cf.sync_folder_to_container(local, remote)
# Mark all objects on remote to be deleted if not in local
# The limit here is arbitrary, but can not be higher then 10000.
limit = 1000
marker = ""
remote_objects_list = remote.get_objects(limit=limit, marker=marker)
while remote_objects_list:
marker = remote_objects_list[-1].name
for obj in remote_objects_list:
if obj.name not in local_files:
obj.delete_in_seconds(DELETE_OBJECTS_DELAY)
remote_objects_list = remote.get_objects(limit=limit, marker=marker)
# publish
cf.make_container_public(container_name, ttl=TTL)
# cdn website
remote.set_web_index_page(INDEX_PAGE)
remote.set_web_error_page(ERROR_PAGE)
# Totally copied from the docs
print
print "After Making Public"
print "cdn_enabled", remote.cdn_enabled
print "cdn_ttl", remote.cdn_ttl
print "cdn_log_retention", remote.cdn_log_retention
print "cdn_uri", remote.cdn_uri
print "cdn_ssl_uri", remote.cdn_ssl_uri
print "cdn_streaming_uri", remote.cdn_streaming_uri
print "cdn_ios_uri", remote.cdn_ios_uri
# -*- coding: utf-8 -*-
from flask import Config
from qianka.sqlalchemy import QKSQLAlchemy
db = QKSQLAlchemy()
config = Config('.')
config.from_pyfile('config.py')
db.configure(config)
class Configurator(object):
"""
Object that takes care of loading the different configurations from the
different sources. There are 3 types of settings:
* Project: The basic set of settings needed by the system. These are
shipped with Shiva.
* Local: Specific to each instance, useful for overwriting system
settings. Some of them must be defined before running Shiva, like the
DB URI.
* Debug: This setting will only be loaded if ``DEBUG`` is set to True
in the local settings.
There are also 3 different places where Shiva looks for this config files:
* A ``local.py`` file inside the ``config/`` directory, relative to
Shiva.
* The ``$SHIVA_CONFIG`` environment variable. It's assumed to be
pointing to a file (not a dir) if exists.
* The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If
``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as
defined by the `XDG Base Directory Specification
<http://standards.freedesktop.org/basedir-spec/basedir-spec-latest\
.html>`_.
"""
def __init__(self):
self._config = FlaskConfig("")
# project
_project = self.load_project()
# local
_xdg_config = self.from_xdg_config()
_env = self.from_env()
_local = self.from_local()
# debug
_debug = self.load_debug()
self.extract_conf(self._config)
if not (_xdg_config or _env or _local):
raise NoConfigFoundError
def load_project(self):
return self._config.from_object(project)
def get_xdg_path(self):
path_home = os.getenv("HOME")
if not path_home:
return None
default_config_home = os.path.join(path_home, ".config")
return os.getenv("XDG_CONFIG_HOME") or default_config_home
def from_xdg_config(self):
xdg_path = self.get_xdg_path()
if not xdg_path:
return False
local_py = os.path.join(xdg_path, "shiva/config.py")
if not os.path.exists(local_py):
return False
return self._config.from_pyfile(local_py)
def from_env(self):
if not os.getenv("SHIVA_CONFIG"):
return False
return self._config.from_envvar("SHIVA_CONFIG")
def from_local(self):
with ignored(ImportError):
self._config.from_object("shiva.config.local")
return True
return False
def load_debug(self):
if not self._config.get("DEBUG"):
return False
loaded = False
with ignored(ImportError):
from shiva.config import debug
loaded = self._config.from_object(debug)
xdg_path = self.get_xdg_path()
if not xdg_path:
return False
debug_py = os.path.join(xdg_path, "shiva/debug.py")
if not os.path.exists(debug_py):
return False
return self._config.from_pyfile(debug_py) or loaded
def extract_conf(self, *args):
"""
Receives one or more objects, iterates over their elements, extracts
all the uppercase properties and injects them into the Configurator
object.
"""
for obj in args:
for key in dir(obj):
if key.isupper():
setattr(self, key, getattr(obj, key))
if hasattr(obj, "iterkeys") and callable(getattr(obj, "iterkeys")):
for key in obj.iterkeys():
if key.isupper():
setattr(self, key, obj[key])
def get_user_config():
from flask import Config
config = Config('/')
config.from_object('alerta.settings')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
if 'DEBUG' in os.environ:
config['DEBUG'] = True
if 'BASE_URL' in os.environ:
config['BASE_URL'] = os.environ['BASE_URL']
if 'USE_PROXYFIX' in os.environ:
config['USE_PROXYFIX'] = True if os.environ['USE_PROXYFIX'] == 'True' else False
if 'SECRET_KEY' in os.environ:
config['SECRET_KEY'] = os.environ['SECRET_KEY']
database_url = (
os.environ.get('DATABASE_URL', None) or
# The following database settings are deprecated.
os.environ.get('MONGO_URI', None) or
os.environ.get('MONGODB_URI', None) or
os.environ.get('MONGOHQ_URL', None) or
os.environ.get('MONGOLAB_URI', None)
)
# Use app config for DATABASE_URL if no env var from above override it
config['DATABASE_URL'] = database_url or config['DATABASE_URL']
if 'DATABASE_NAME' in os.environ:
config['DATABASE_NAME'] = os.environ['DATABASE_NAME']
if 'AUTH_REQUIRED' in os.environ:
config['AUTH_REQUIRED'] = True if os.environ['AUTH_REQUIRED'] == 'True' else False
if 'AUTH_PROVIDER' in os.environ:
config['AUTH_PROVIDER'] = os.environ['AUTH_PROVIDER']
if 'ADMIN_USERS' in os.environ:
config['ADMIN_USERS'] = os.environ['ADMIN_USERS'].split(',')
if 'SIGNUP_ENABLED' in os.environ:
config['SIGNUP_ENABLED'] = True if os.environ['SIGNUP_ENABLED'] == 'True' else False
if 'CUSTOMER_VIEWS' in os.environ:
config['CUSTOMER_VIEWS'] = True if os.environ['CUSTOMER_VIEWS'] == 'True' else False
if 'OAUTH2_CLIENT_ID' in os.environ:
config['OAUTH2_CLIENT_ID'] = os.environ['OAUTH2_CLIENT_ID']
if 'OAUTH2_CLIENT_SECRET' in os.environ:
config['OAUTH2_CLIENT_SECRET'] = os.environ['OAUTH2_CLIENT_SECRET']
if 'ALLOWED_EMAIL_DOMAINS' in os.environ:
config['ALLOWED_EMAIL_DOMAINS'] = os.environ['ALLOWED_EMAIL_DOMAINS'].split(',')
if 'AZURE_TENANT' in os.environ:
config['AZURE_TENANT'] = os.environ['AZURE_TENANT']
if 'GITHUB_URL' in os.environ:
config['GITHUB_URL'] = os.environ['GITHUB_URL']
if 'ALLOWED_GITHUB_ORGS' in os.environ:
config['ALLOWED_GITHUB_ORGS'] = os.environ['ALLOWED_GITHUB_ORGS'].split(',')
if 'GITLAB_URL' in os.environ:
config['GITLAB_URL'] = os.environ['GITLAB_URL']
if 'ALLOWED_GITLAB_GROUPS' in os.environ:
config['ALLOWED_OIDC_ROLES'] = os.environ['ALLOWED_GITLAB_GROUPS'].split(',')
if 'KEYCLOAK_URL' in os.environ:
config['KEYCLOAK_URL'] = os.environ['KEYCLOAK_URL']
if 'KEYCLOAK_REALM' in os.environ:
config['KEYCLOAK_REALM'] = os.environ['KEYCLOAK_REALM']
if 'ALLOWED_KEYCLOAK_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = os.environ['ALLOWED_KEYCLOAK_ROLES'].split(',')
if 'OIDC_ISSUER_URL' in os.environ:
config['OIDC_ISSUER_URL'] = os.environ['OIDC_ISSUER_URL']
if 'ALLOWED_OIDC_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = os.environ['ALLOWED_OIDC_ROLES'].split(',')
if 'PINGFEDERATE_OPENID_ACCESS_TOKEN_URL' in os.environ:
config['PINGFEDERATE_OPENID_ACCESS_TOKEN_URL'] = os.environ['PINGFEDERATE_OPENID_ACCESS_TOKEN_URL'].split(
',')
if 'PINGFEDERATE_OPENID_PAYLOAD_USERNAME' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_USERNAME'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_USERNAME'].split(
',')
if 'PINGFEDERATE_OPENID_PAYLOAD_EMAIL' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_EMAIL'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_EMAIL'].split(',')
if 'PINGFEDERATE_OPENID_PAYLOAD_GROUP' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_GROUP'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_GROUP'].split(',')
if 'PINGFEDERATE_PUBKEY_LOCATION' in os.environ:
config['PINGFEDERATE_PUBKEY_LOCATION'] = os.environ['PINGFEDERATE_PUBKEY_LOCATION'].split(',')
if 'PINGFEDERATE_TOKEN_ALGORITHM' in os.environ:
config['PINGFEDERATE_TOKEN_ALGORITHM'] = os.environ['PINGFEDERATE_TOKEN_ALGORITHM'].split(',')
if 'CORS_ORIGINS' in os.environ:
config['CORS_ORIGINS'] = os.environ['CORS_ORIGINS'].split(',')
if 'MAIL_FROM' in os.environ:
config['MAIL_FROM'] = os.environ['MAIL_FROM']
if 'SMTP_PASSWORD' in os.environ:
config['SMTP_PASSWORD'] = os.environ['SMTP_PASSWORD']
if 'GOOGLE_TRACKING_ID' in os.environ:
config['GOOGLE_TRACKING_ID'] = os.environ['GOOGLE_TRACKING_ID']
if 'PLUGINS' in os.environ:
config['PLUGINS'] = os.environ['PLUGINS'].split(',')
# Runtime config check
if config['CUSTOMER_VIEWS'] and not config['AUTH_REQUIRED']:
raise RuntimeError('Must enable authentication to use customer views')
if config['CUSTOMER_VIEWS'] and not config['ADMIN_USERS']:
raise RuntimeError('Customer views is enabled but there are no admin users')
return config
* OpenShift
* DATABASE_URI
"""
import os
import json
import re
from distutils.util import strtobool
from flask import Config
from sqlalchemy.engine.url import URL
config = Config('.')
config.from_object('compair.settings')
config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../config.py'), silent=True)
db_conn_options = ''
if os.environ.get('DB_CONN_OPTIONS'):
options_dict = json.loads(os.environ.get('DB_CONN_OPTIONS'))
for key in options_dict:
db_conn_options = db_conn_options + ('?' if db_conn_options == '' else '&') + \
key + '=' + options_dict.get(key)
if os.environ.get('OPENSHIFT_MYSQL_DB_HOST'):
config['SQLALCHEMY_DATABASE_URI'] = URL(
'mysql+pymysql',
host=os.getenv('OPENSHIFT_MYSQL_DB_HOST', 'localhost'),
port=os.getenv('OPENSHIFT_MYSQL_DB_PORT', '3306'),
username=os.getenv('OPENSHIFT_MYSQL_DB_USERNAME', 'compair'),
password=os.getenv('OPENSHIFT_MYSQL_DB_PASSWORD', 'compair'),
dest="config_path",
help=u'Path to the configuration file')
parser.add_argument("--log-level",
default='INFO',
dest="log_level",
choices=['DEBUG', 'INFO', 'WARNINGS', 'ERROR'],
help=u"Logging level")
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s',
stream=sys.stdout,
level=getattr(logging, args.log_level))
# load config
config = Config('')
config.from_pyfile(args.config_path)
# redis
r = Redis(**config['REDIS_CONFIG'])
logging.info(u"Warming up redis")
# clarifai
clarifai_keys = [json.dumps(pair) for pair in config["INIT_CLARIFAI_KEYS"]]
with r.pipeline() as pipe:
key = config["CLARIFAI_KEY"]
pipe.delete(key)
pipe.rpush(key, *clarifai_keys)
pipe.execute()
logging.info(u"%s clarifai keys have been inserted", len(clarifai_keys))
def FromEmails():
#Config, init variables
config = Config(os.getcwd())
config.from_pyfile('settings.cfg')
logger = logging.getLogger('ICONZ.HelpdeskDashboard.OutageParse')
logger.addHandler(config['HANDLER'])
def GetBody(msg):
if msg.is_multipart():
return(msg.get_payload(0)).get_payload(decode=True)
else:
return(msg.get_payload(decode=True))
def Passthrough(msg_list):
try:
smtp_conn = smtplib.SMTP(config['OUTAGES_SMTP_SERVER'],
config['OUTAGES_SMTP_PORT'])
for msg in msg_list:
smtp_conn.sendmail(config['OUTAGES_SMTP_FROM_ADDR'],
config['OUTAGES_SMTP_TO_ADDR'], msg.as_string())
logger.info('Succesfully passed through ' + str(len(msg_list)) + ' messages')
except Exception as e:
logger.error('Faliure to passthrough all messages.', exc_info = True)
def DBInsert(data):
#Could tidy this up so it's done as a batch operation. Probably unecessary
conn = connect_db()
cur = conn.cursor()
try:
cur.execute('select rowid from outage_texts where provider_ref_id match(?)',
[data['reference']])
db_outage = cur.fetchone()
db_outage_id = None
db_outage_text_id = None
if db_outage == None:
cur.execute('insert into outage_texts (provider_ref_id, equipment_list, full_text) values (?,?,?)',
[unicode(data['reference']), unicode(data['equipment']), unicode(data['full_text'])])
db_outage_text_id = cur.lastrowid
cur.execute('insert into outages (provider, num_affected, start_time, end_time, issues, text_id) values (?,?,?,?,?,?)',
[data['provider'], data['num_affected'], data['start_datetime'], data['end_datetime'], data['issues'], db_outage_text_id])
db_outage_id = cur.lastrowid
conn.commit()
logger.debug('Created new outage with id ' + str(db_outage_id))
else:
cur.execute('update outage_texts set equipment_list=(?), full_text=(?) where rowid==?',
[unicode(data['equipment']), unicode(data['full_text']), db_outage[0]])
db_outage_text_id = db_outage[0]
cur.execute('select id from outages where text_id==?', [db_outage_text_id])
db_outage_id = cur.fetchone()[0]
cur.execute('update outages set num_affected=(?), start_time=(?), end_time=(?), issues=(?) where id==?',
[data['num_affected'], data['start_datetime'], data['end_datetime'], data['issues'], db_outage_id])
conn.commit()
logger.debug('Updated outage with id ' + str(db_outage_id))
users = list()
for line in data['equipment'].split('\n'):
cur.execute('select id, name from dslams where name==?', [line])
dslam = cur.fetchone()
if dslam is not None:
cur.execute('select id, asid from dslusers where dslam_id==?', [dslam[0]])
users.append(cur.fetchall())
for db_block in users:
for user in db_block:
if user is not None:
cur.execute('select * from outages_dslusers_rel where (outages_id==?) and (dslusers_id==?)', [db_outage_id, user[0]])
if cur.fetchone() is None:
cur.execute('insert into outages_dslusers_rel (outages_id, dslusers_id) values (?,?)',
[db_outage_id, user[0]])
conn.commit()
#Clears connection so that program can continue in case of exception
except sqlite3.IntegrityError as e:
conn.close()
conn = connect_db()
raise(e)
def ChorusHTMLFat(data):
def FindContent(fieldname, soup, one = True):
if one:
out = [string for string in soup.find(
"td", text=re.compile(fieldname), style=re.compile('(;color:|; COLOR: )rgb\(0,72,97\);')
).parent.find(
"td", style=re.compile('(;color:|; COLOR: )rgb\(0,0,0\);')
).stripped_strings]
if len(out) < 1: return None
else: return out[0]
else:
return [string for string in soup.find(
"td", text=re.compile(fieldname), style=re.compile('(;color:|; COLOR: )rgb\(0,72,97\);')
).parent.find(
"td", style=re.compile('(;color:|; COLOR: )rgb\(0,0,0\);')
).stripped_strings]
soup = BeautifulSoup(data)
results = dict({'provider' : 'Chorus', 'reference' : FindContent('Ticket Number:', soup), 'issues' : False})
try:
start_date = FindContent('Date:', soup)
start_time = FindContent('Start Time:', soup)
if start_time is not None:
if start_time.endswith('(amended)'):
start_time = start_time[0:6]
else:
results['issues'] = True
start_time = '00:00'
results['start_datetime'] = datetime.datetime.strptime(start_date + ' ' +
start_time, '%d/%m/%Y %H:%M')
except (AttributeError, ValueError):
results['issues'] = True
results['start_datetime'] = None
try:
end_date = FindContent('Resolution Date:', soup)
end_time = FindContent('Resolution Time:', soup)
if end_date is None:
if end_time is None:
results['end_datetime'] = None
else:
results['end_datetime'] = datetime.datetime.combine(start_datetime.date,
datetime.time.strptime(end_time, '%H:%M'))
else:
results['end_datetime'] = datetime.datetime.strptime(end_date + ' ' +
end_time, '%d/%m/%Y %H:%M')
except (AttributeError, ValueError):
results['end_datetime'] = None
try:
results['num_affected'] = int(re.search(('(\d+)'), FindContent('Customer Impact:',
soup)).group(1))
except AttributeError:
results['issues'] = True
results['num_affected'] = 0
#Might be useful later if I can get ASID -> DSLAM mappings
equipment = FindContent('Equipment List:', soup, False)
results['equipment'] = ''
for string in equipment:
if re.match('[A-Z]{2,3}[-|_|/]([A-Z]{1,3}-)?DSLAM-\d{0,2}', string):
results['equipment'] += string + '\n'
else:
results['issues'] = True
results['full_text'] = ''
for string in soup.stripped_strings:
if re.match('^VF \w{1,3}$', string):
string = ''
results['full_text'] = results['full_text'] + string + '\n'
return results
def ChorusPlainText(f):
results = dict({'provider' : 'Chorus',
'reference' : re.search(('SED Ref.+:.+(\d{6})'), f).group(1),
'issues' : False})
start_date = re.search(('DATE.+:.+(\d{2}/\d{2}/\d{2})'), f)
start_time = re.search(('NZ Standard Time.+:.+(\d{2}:\d{2}:\d{2}.(AM|PM))'), f)
results['start_datetime'] = datetime.datetime.strptime(start_date.group(1) + ' ' +
start_time.group(1), '%d/%m/%y %I:%M:%S %p')
duration = re.search(('DURATION \(HR:MIN:SEC\).+:.+(\d{3}):(\d{2}):(\d{2})'), f)
results['end_datetime'] = results['start_datetime'] + datetime.timedelta(hours = int(duration.group(1)),
minutes = int(duration.group(2)), seconds = int(duration.group(3)))
try:
results['num_affected'] = int(re.search('Up to (\d+) DSL customers', f).group(1))
except AttributeError:
results['issues'] = True
results['num_affected'] = 0
logger.warning('ChorusPlainText could not extract num_affected')
equipment = re.findall(('([A-Z]{2,3}[-|_|/]([A-Z]{1,3}-)?DSLAM-\d{0,2})'), f)
results['equipment'] = ''
for regex_groups in equipment:
results['equipment'] += regex_groups[0] + '\n'
if 'DSLAM' not in results['equipment']:
results['issues'] = True
logger.warning('ChorusPlainText could not extract equipment_list')
results['full_text'] = str(f)
if results['issues'] == False:
logger.debug('ChorusPlainText parsed message without issue')
return results
def ChorusHTMLSkinny(outage):
soup = BeautifulSoup(outage)
results = dict({'provider' : 'Chorus', 'issues' : False})
text = ''
#Mandated by shitty HTML
stringgen = soup.stripped_strings
for string in stringgen:
if ('Field' not in string) and ('FIELD' not in string):
if 'Ticket Number:' in string:
stringgen.next()
results['reference'] = stringgen.next()
text += results['reference'] + '\n'
elif 'Location:' in string:
stringgen.next()
results['location'] = stringgen.next()
text += results['location'] + '\n'
elif 'Start Date:' in string:
stringgen.next()
results['start_datetime'] = stringgen.next()
text += results['start_datetime'] + '\n'
elif 'Start Time:' in string:
stringgen.next()
temp = stringgen.next()
results['start_datetime'] = results['start_datetime'] + ' ' + temp
text += temp + '\n'
elif 'Resolution Date:' in string:
stringgen.next()
results['end_datetime'] = stringgen.next()
text += results['end_datetime'] + '\n'
elif 'Resolution Time:' in string:
stringgen.next()
temp = stringgen.next()
results['end_datetime'] += ' ' + temp
text += temp + '\n'
elif 'Customer Impact:' in string:
stringgen.next()
temp = stringgen.next()
try:
results['num_affected'] = int(re.search('Up to (\d+) DSL customers', temp).group(1))
except AttributeError:
results['issues'] = True
results['num_affected'] = 0
logger.warning('ChorusHTMLSkinny could not extract num_affected')
text += temp + '\n'
elif 'Equipment List:' in string:
results['equipment'] = ''
stringgen.next()
temp = stringgen.next()
while 'Additional Information:' not in temp:
results['equipment'] += temp
text += temp + '\n'
temp = stringgen.next()
elif 'Cable Pair & Range:' in string:
results['equipment'] = ''
stringgen.next()
temp = stringgen.next()
while 'Additional Information:' not in temp:
results['equipment'] += temp
text += temp + '\n'
temp = stringgen.next()
else:
text += string + '\n'
if (results['start_datetime'] is not None) and (len(results['start_datetime']) == 16):
try:
results['start_datetime'] = datetime.datetime.strptime(results['start_datetime'], '%d/%m/%Y %H:%M')
except ValueError:
try:
results['start_datetime'] = datetime.datetime.strptime(results['start_datetime'], '%H:%M %d/%m/%Y')
except ValueError:
results['issues'] = True
results['start_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract start_datetime')
elif results['start_datetime'] is not None:
results['issues'] = True
results['start_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract start_datetime')
else:
results['issues'] = True
logger.error('Fallthrough condition in ChorusHTMLSkinny when parsing start_datetime')
if (results['end_datetime'] is not None) and (len(results['end_datetime']) == 16):
try:
results['end_datetime'] = datetime.datetime.strptime(results['end_datetime'], '%d/%m/%Y %H:%M')
except ValueError:
try:
results['end_datetime'] = datetime.datetime.strptime(results['end_datetime'], '%H:%M %d/%m/%Y')
except ValueError:
results['issues'] = True
results['end_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract end_datetime')
else:
results['end_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract end_datetime')
results['full_text'] = text
if results['issues'] == False:
logger.debug('ChorusHTMLSkinny parsed message without issue')
return results
#Actual script begins here.
passthrough_list = list()
#IMAP login
imap = imaplib.IMAP4(config['OUTAGES_POP_SERVER'])
imap.login(config['OUTAGES_POP_USER'], config['OUTAGES_POP_PASS'])
imap.select('INBOX')
ids = None
if config['DEBUG'] is True:
ids = imap.search(None, '(SUBJECT "")')[1][0].split()
else:
ids = imap.search(None, '(UNSEEN)')[1][0].split()
if len(ids) > 0:
logger.info('Collected ' + str(len(ids)) + 'messages from inbox')
#Fetch messages
for id in ids:
typ, msg_data = imap.fetch(id, '(RFC822)')
for response_part in msg_data:
if isinstance(response_part, tuple):
msg = email.message_from_string(response_part[1])
data = None
#Choose parser
if (msg.has_key('from')) and (msg.has_key('subject')):
if (config['CHORUS_PLAIN_TEXT_FROM_ADDRS_RE'].search(msg['from'])) and (config['CHORUS_PLAIN_TEXT_RE'].match(msg['subject'])):
logger.debug('Message ' + msg['subject'] + 'passed to ChorusPlainText parser')
data = ChorusPlainText(GetBody(msg))
elif (config['CHORUS_HTML_FAT_FROM_ADDRS_RE'].search(msg['from'])) and (config['CHORUS_HTML_FAT_RE'].match(msg['subject'])):
logger.debug('Message ' + msg['subject'] + 'passed to ChorusHTMLFat parser')
data = ChorusHTMLFat(GetBody(msg))
elif (config['CHORUS_HTML_SKINNY_FROM_ADDRS_RE'].search(msg['from'])) and (config['CHORUS_HTML_SKINNY_RE'].match(msg['subject'])):
logger.debug('Message ' + msg['subject'] + 'passed to ChorusHTMLSkinny parser')
data = ChorusHTMLSkinny(GetBody(msg))
else:
passthrough_list.append(msg)
logger.warning('Could not select parser for message ' + msg['subject'])
else:
logger.error('Unable to select parser due to missing headers')
if data is not None:
try:
DBInsert(data)
except (sqlite3.IntegrityError, UnicodeDecodeError) as e:
logger.warning('Unable to insert data into db from message ' + msg['subject'], exc_info = True)
passthrough_list.append(msg)
Passthrough(passthrough_list)
def get_user_config():
valid = Validate()
from flask import Config
config = Config('/')
config.from_object('alerta.settings')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
if 'DEBUG' in os.environ:
config['DEBUG'] = valid.validate_boolean(os.environ['DEBUG'])
if 'BASE_URL' in os.environ:
config['BASE_URL'] = valid.validate_url(os.environ['BASE_URL'])
if 'USE_PROXYFIX' in os.environ:
config['USE_PROXYFIX'] = valid.validate_boolean(
os.environ['USE_PROXYFIX'])
if 'SECRET_KEY' in os.environ:
config['SECRET_KEY'] = valid.validate_string(
os.environ['SECRET_KEY'])
database_url = (
os.environ.get('DATABASE_URL', None)
# The following database settings are deprecated.
or os.environ.get('MONGO_URI', None) or
os.environ.get('MONGODB_URI', None) or os.environ.get(
'MONGOHQ_URL', None) or os.environ.get('MONGOLAB_URI', None))
# Use app config for DATABASE_URL if no env var from above override it
config['DATABASE_URL'] = database_url or config['DATABASE_URL']
if config['DATABASE_URL'] in os.environ:
config['DATABASE_URL'] = valid.validate_url(
os.environ['DATABASE_URL'])
if 'DATABASE_NAME' in os.environ:
config['DATABASE_NAME'] = valid.validate_string(
os.environ['DATABASE_NAME'])
if 'AUTH_REQUIRED' in os.environ:
config['AUTH_REQUIRED'] = valid.validate_boolean(
os.environ['AUTH_REQUIRED'])
if 'AUTH_PROVIDER' in os.environ:
config['AUTH_PROVIDER'] = valid.validate_string(
os.environ['AUTH_PROVIDER'])
if 'ADMIN_USERS' in os.environ:
config['ADMIN_USERS'] = valid.validate_list(
os.environ['ADMIN_USERS'], 'string')
if 'SIGNUP_ENABLED' in os.environ:
config['SIGNUP_ENABLED'] = valid.validate_boolean(
os.environ['SIGNUP_ENABLED'])
if 'CUSTOMER_VIEWS' in os.environ:
config['CUSTOMER_VIEWS'] = valid.validate_boolean(
os.environ['CUSTOMER_VIEWS'])
if 'OAUTH2_CLIENT_ID' in os.environ:
config['OAUTH2_CLIENT_ID'] = valid.validate_string(
os.environ['OAUTH2_CLIENT_ID'])
if 'OAUTH2_CLIENT_SECRET' in os.environ:
config['OAUTH2_CLIENT_SECRET'] = valid.validate_string(
os.environ['OAUTH2_CLIENT_SECRET'])
if 'ALLOWED_EMAIL_DOMAINS' in os.environ:
config['ALLOWED_EMAIL_DOMAINS'] = valid.validate_list(
os.environ['ALLOWED_EMAIL_DOMAINS'], 'string')
if 'AZURE_TENANT' in os.environ:
config['AZURE_TENANT'] = valid.validate_string(
os.environ['AZURE_TENANT'])
if 'GITHUB_URL' in os.environ:
config['GITHUB_URL'] = valid.validate_url(os.environ['GITHUB_URL'])
if 'ALLOWED_GITHUB_ORGS' in os.environ:
config['ALLOWED_GITHUB_ORGS'] = valid.validate_list(
os.environ['ALLOWED_GITHUB_ORGS'], 'string')
if 'GITLAB_URL' in os.environ:
config['GITLAB_URL'] = valid.validate_url(os.environ['GITLAB_URL'])
if 'ALLOWED_GITLAB_GROUPS' in os.environ:
config['ALLOWED_OIDC_ROLES'] = valid.validate_list(
os.environ['ALLOWED_GITLAB_GROUPS'], 'string')
if 'KEYCLOAK_URL' in os.environ:
config['KEYCLOAK_URL'] = valid.validate_url(
os.environ['KEYCLOAK_URL'])
if 'KEYCLOAK_REALM' in os.environ:
config['KEYCLOAK_REALM'] = valid.validate_string(
os.environ['KEYCLOAK_REALM'])
if 'ALLOWED_KEYCLOAK_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = valid.validate_list(
os.environ['ALLOWED_KEYCLOAK_ROLES'], 'string')
if 'OIDC_ISSUER_URL' in os.environ:
config['OIDC_ISSUER_URL'] = valid.validate_url(
os.environ['OIDC_ISSUER_URL'])
if 'ALLOWED_OIDC_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = valid.validate_list(
os.environ['ALLOWED_OIDC_ROLES'], 'string')
if 'CORS_ORIGINS' in os.environ:
config['CORS_ORIGINS'] = valid.validate_list(
os.environ['CORS_ORIGINS'], 'url')
if 'MAIL_FROM' in os.environ:
config['MAIL_FROM'] = valid.validate_email(os.environ['MAIL_FROM'])
if 'SMTP_PASSWORD' in os.environ:
config['SMTP_PASSWORD'] = valid.validate_string(
os.environ['SMTP_PASSWORD'])
if 'GOOGLE_TRACKING_ID' in os.environ:
config['GOOGLE_TRACKING_ID'] = valid.validate_string(
os.environ['GOOGLE_TRACKING_ID'])
if 'PLUGINS' in os.environ:
config['PLUGINS'] = valid.validate_list(os.environ['PLUGINS'],
'string')
if 'ALERT_TIMEOUT' in os.environ:
config['ALERT_TIMEOUT'] = valid.validate_integer(
os.environ['ALERT_TIMEOUT'])
if 'HEARTBEAT_TIMEOUT' in os.environ:
config['HEARTBEAT_TIMEOUT'] = valid.validate_integer(
os.environ['HEARTBEAT_TIMEOUT'])
if 'API_KEY_EXPIRE_DAYS' in os.environ:
config['API_KEY_EXPIRE_DAYS'] = valid.validate_integer(
os.environ['API_KEY_EXPIRE_DAYS'])
# Runtime config check
if config['CUSTOMER_VIEWS'] and not config['AUTH_REQUIRED']:
raise RuntimeError(
'Must enable authentication to use customer views')
if config['CUSTOMER_VIEWS'] and not config['ADMIN_USERS']:
raise RuntimeError(
'Customer views is enabled but there are no admin users')
return config
import pygraphviz as pgv
import time
import json
import copy
import os.path
import networkx as nx
from networkx.algorithms import centrality
from networkx.algorithms.community import k_clique_communities
from flask import Config
config = Config("./")
config.from_pyfile('web_config.cfg')
def position_nodes(nodes, edges):
G = pgv.AGraph(strict=True, directed=False, size='10!')
for n in nodes.values():
G.add_node(n.asn, label=n.label)
for e in edges:
G.add_edge(e.a.asn, e.b.asn, len=1.0)
G.layout(prog='neato', args='-Gepsilon=0.0001 -Gmaxiter=100000')
return G
def compute_betweenness(G):
ng = nx.Graph()
for start in G.iternodes():
import logging
import os
from flask.ext.security import RoleMixin
from flask.ext.sqlalchemy import SQLAlchemy
from sqlalchemy import create_engine, Column, Integer, String, Table, DateTime, exists, ForeignKey, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, backref
from sqlalchemy.orm import scoped_session
from . import jsonalchemy
from flask import Config
cfg = Config(os.path.abspath("."))
cfg.from_pyfile("config.py")
db = SQLAlchemy()
logger = logging.getLogger("Gunrskite::Database")
# Check if we're inside the UDP listener, or not.
if os.environ.get("INSIDE_LISTENER", "n") == "y":
print("Using declarative_base() for model base.")
Base = declarative_base()
else:
# Switch to using Flask-SQLAlchemy's model.
print("Using db.Model for model base.")
Base = db.Model
engine = create_engine(cfg["SQLALCHEMY_DATABASE_URI"], pool_recycle=3600)
# Function for creating the engine and session.
def create_sess():
def get_user_config():
from flask import Config
config = Config('/')
config.from_object('alerta.settings')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
config['DEBUG'] = get_config('DEBUG',
default=True,
type=bool,
config=config)
config['BASE_URL'] = get_config('BASE_URL',
default='',
type=str,
config=config)
config['USE_PROXYFIX'] = get_config('USE_PROXYFIX',
default=False,
type=bool,
config=config)
config['SECRET_KEY'] = get_config('SECRET_KEY',
default='',
type=str,
config=config)
database_url = (
# The following database settings are deprecated.
os.environ.get('MONGO_URI', None)
or os.environ.get('MONGODB_URI', None)
or os.environ.get('MONGOHQ_URL', None)
or os.environ.get('MONGOLAB_URI', None))
# Use app config for DATABASE_URL if no env var from above override it
config['DATABASE_URL'] = get_config('DATABASE_URL',
default=database_url,
type=str,
config=config)
config['DATABASE_NAME'] = get_config('DATABASE_NAME',
default=None,
type=str,
config=config)
config['AUTH_REQUIRED'] = get_config('AUTH_REQUIRED',
default=None,
type=bool,
config=config)
config['AUTH_PROVIDER'] = get_config('AUTH_PROVIDER',
default=None,
type=str,
config=config)
config['ADMIN_USERS'] = get_config('ADMIN_USERS',
default=[],
type=list,
config=config)
config['SIGNUP_ENABLED'] = get_config('SIGNUP_ENABLED',
default=True,
type=bool,
config=config)
config['CUSTOMER_VIEWS'] = get_config('CUSTOMER_VIEWS',
default=False,
type=bool,
config=config)
config['OAUTH2_CLIENT_ID'] = get_config('OAUTH2_CLIENT_ID',
default=None,
type=str,
config=config)
config['OAUTH2_CLIENT_SECRET'] = get_config('OAUTH2_CLIENT_SECRET',
default=None,
type=str,
config=config)
config['ALLOWED_EMAIL_DOMAINS'] = get_config('ALLOWED_EMAIL_DOMAINS',
default=[],
type=list,
config=config)
config['AZURE_TENANT'] = get_config('AZURE_TENANT',
default=None,
type=str,
config=config)
config['GITHUB_URL'] = get_config('GITHUB_URL',
default=None,
type=str,
config=config)
config['ALLOWED_GITHUB_ORGS'] = get_config('ALLOWED_GITHUB_ORGS',
default=[],
type=list,
config=config)
config['GITLAB_URL'] = get_config('GITLAB_URL',
default=None,
type=str,
config=config)
if 'ALLOWED_GITLAB_GROUPS' in os.environ:
config['ALLOWED_OIDC_ROLES'] = get_config('ALLOWED_GITLAB_GROUPS',
default=[],
type=list,
config=config)
config['KEYCLOAK_URL'] = get_config('KEYCLOAK_URL',
default=None,
type=str,
config=config)
config['KEYCLOAK_REALM'] = get_config('KEYCLOAK_REALM',
default=None,
type=str,
config=config)
if 'ALLOWED_KEYCLOAK_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = get_config('ALLOWED_KEYCLOAK_ROLES',
default=[],
type=list,
config=config)
config['OIDC_ISSUER_URL'] = get_config('OIDC_ISSUER_URL',
default=None,
type=str,
config=config)
config['ALLOWED_OIDC_ROLES'] = get_config('ALLOWED_OIDC_ROLES',
default=[],
type=list,
config=config)
config['CORS_ORIGINS'] = get_config('CORS_ORIGINS',
default=[],
type=list,
config=config)
config['MAIL_FROM'] = get_config('MAIL_FROM',
default=None,
type=str,
config=config)
config['SMTP_PASSWORD'] = get_config('SMTP_PASSWORD',
default=None,
type=str,
config=config)
config['GOOGLE_TRACKING_ID'] = get_config('GOOGLE_TRACKING_ID',
default=None,
type=str,
config=config)
config['PLUGINS'] = get_config('PLUGINS',
default=[],
type=list,
config=config)
# blackout plugin
config['BLACKOUT_DURATION'] = get_config('BLACKOUT_DURATION',
default=None,
type=int,
config=config)
config['NOTIFICATION_BLACKOUT'] = get_config('NOTIFICATION_BLACKOUT',
default=None,
type=bool,
config=config)
config['BLACKOUT_ACCEPT'] = get_config('BLACKOUT_ACCEPT',
default=[],
type=list,
config=config)
# reject plugin
config['ORIGIN_BLACKLIST'] = get_config('ORIGIN_BLACKLIST',
default=[],
type=list,
config=config)
config['ALLOWED_ENVIRONMENTS'] = get_config('ALLOWED_ENVIRONMENTS',
default=[],
type=list,
config=config)
# Runtime config check
if config['CUSTOMER_VIEWS'] and not config['AUTH_REQUIRED']:
raise RuntimeError(
'Must enable authentication to use customer views')
if config['CUSTOMER_VIEWS'] and not config['ADMIN_USERS']:
raise RuntimeError(
'Customer views is enabled but there are no admin users')
return config
import datetime
import os
import re
import redis
import urlparse
from flask import Blueprint, Config, flash, g, jsonify, request, render_template
from flask import session, url_for
from flask_bibframe.models import CoverArt
blueprint_folder = os.path.abspath(os.path.dirname(__file__))
app_folder = os.path.split(blueprint_folder)[0]
metrics_config = Config(app_folder)
metrics_config.from_pyfile('catalog.cfg')
redis_ds = redis.StrictRedis(metrics_config.get('REDIS_HOST'))
def parse_logfile(log_path, redis_ds=redis_ds, tz='-0700'):
"""Takes a nginx access log, opens the file, and parses through and creates
metrics for use by other Blueprints and apps.
Parameters
----------
log_path : str
Full path and filename of the nginx access log
redis_ds : StrictRedis instance
defaults to Redis datastore for CPP's metrics
tz : str
reader=parsing.get_variants_from_sites_vcf_only_dp_gq),
get_file_contig_pairs(variants_files))
if __name__ == '__main__':
global mongo_host
global mongo_port
global mongo_db_name
args = argparser.parse_args()
config = Config(os.path.dirname(os.path.realpath(__file__)))
# Load default config
config.from_object('config.default')
# Load instance configuration if exists
config.from_pyfile('config.py', silent=True)
# Load configuration file specified in BRAVO_CONFIG_FILE environment variable if exists
config.from_envvar('BRAVO_CONFIG_FILE', silent=True)
mongo_host = config['MONGO']['host']
mongo_port = config['MONGO']['port']
mongo_db_name = config['MONGO']['name']
igv_cache_collection_name = config['IGV_CACHE_COLLECTION']
if args.command == 'genes':
sys.stdout.write(
'Start loading genes to {} database.\n'.format(mongo_db_name))
load_gene_models(args.canonical_transcripts_file, args.omim_file,
args.genenames_file, args.gencode_file)
sys.stdout.write(
'Done loading genes to {} database.\n'.format(mongo_db_name))
def make_config(filename=None):
config = Config(getcwd())
if filename is not None:
config.from_pyfile(filename)
return config
is_valid = validate_fingerprints(conf['WEB_DESKTOP_FP_SECRET_KEY'],
conf['WEB_DESKTOP_FP_SALT'],
client_ip_fingerprint,
browser_fingerprint, client_ip,
user_agent, accept, accept_encoding,
accept_language)
return is_valid
if __name__ == '__main__':
conf = Config('..')
test_request_data = dict(
accept_encoding='Accept-Encoding: gzip, deflate, sdch',
accept_language='Accept-Language: en-US,en;q=0.8',
accept='Accept: application/json, text/javascript, */*; q=0.01',
user_agent=
'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
vm_ip='128.196.64.214',
client_ip='127.0.0.1')
conf.from_pyfile('default_settings.py')
# conf.from_pyfile('local_settings.py')
signature = generate_signature_01(conf, test_request_data)
print signature
values, timestamp = decode_signature_02(conf, signature)
print values
is_valid = validate_fingerprints_03(conf, values, test_request_data)
print is_valid
Currently the supported environment variables:
* OpenShift
* DATABASE_URI
"""
import os
from flask import Config
from sqlalchemy.engine.url import URL
config = Config(".")
config.from_object("compair.settings")
config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../config.py"), silent=True)
if os.environ.get("OPENSHIFT_MYSQL_DB_HOST"):
config["SQLALCHEMY_DATABASE_URI"] = URL(
"mysql+pymysql",
host=os.getenv("OPENSHIFT_MYSQL_DB_HOST", "localhost"),
port=os.getenv("OPENSHIFT_MYSQL_DB_PORT", "3306"),
username=os.getenv("OPENSHIFT_MYSQL_DB_USERNAME", "compair"),
password=os.getenv("OPENSHIFT_MYSQL_DB_PASSWORD", "compair"),
database=os.getenv("OPENSHIFT_GEAR_NAME", "compair"),
)
elif (
os.environ.get("DB_HOST")
or os.environ.get("DB_PORT")
or os.environ.get("DB_USERNAME")
or os.environ.get("DB_PASSWORD")
console_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s [in %(module)s @ %(pathname)s:%(lineno)d]"))
logger = logging.Logger("CONTROL", level=logging.DEBUG)
logger.addHandler(file_handler)
logger.addHandler(console_handler)
# Log startup
logger.info('---------- Backend starting up!')
try:
# Load configuration
logger.info('Loading configuration')
config_file = os.path.join(instance_path, 'app.cfg')
default_config_file = os.path.join(resource_path, 'default_app.cfg')
config = Config(instance_path)
config.from_pyfile(default_config_file)
config.from_pyfile(config_file)
#config = SimpleConfigParser(config_file, default_config_file)
# Set up iftt events if a maker key is present
if config['IFTTT_MAKER_KEY']:
logger.info('Creating IFTTT events')
changed_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_changed', logger)
opened_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_opened', logger)
closed_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_closed', logger)
warning_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_warning', logger)
else:
logger.info('No IFTTT maker key provided. No events will be raised.')
changed_event = None # type: IftttEvent
opened_event = None # type: IftttEvent
closed_event = None # type: IftttEvent
