def loadConfig(argconfig):
"Load the config file the same way Flask does which Chill uses."
config_file = (argconfig if argconfig[0] == os.sep else os.path.join(
os.getcwd(), argconfig))
config = Config(os.getcwd())
config.from_pyfile(config_file)
return config
def test_config():
" how can you read the config if do not create an app? " ""
config = Config(".")
print(config)
config.from_object("cryptoadvance.specter.config.DevelopmentConfig")
print(config)
assert config["PORT"] == 25441
def getcollection(cfgfile='config.default.py'):
cfg = Config('')
cfg.from_pyfile(cfgfile)
client = pymongo.MongoClient(cfg['SIMDB_URI'])
db = client[cfg['SIMDB_DATABASE']]
collection = db[cfg['SIMDB_COLLECTION']]
return collection
def setup(context=None, config=None, app_factory=get_app):
app_abspath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = Config(app_abspath)
app_config.from_object('superdesk.tests.test_settings')
app_config['APP_ABSPATH'] = app_abspath
app_config.update(get_test_settings())
app_config.update(config or {})
app_config.update({
'DEBUG': True,
'TESTING': True,
})
app = app_factory(app_config)
logger = logging.getLogger('superdesk')
logger.setLevel(logging.ERROR)
logger = logging.getLogger('elasticsearch')
logger.setLevel(logging.ERROR)
logger = logging.getLogger('urllib3')
logger.setLevel(logging.ERROR)
drop_elastic(app)
drop_mongo(app)
# create index again after dropping it
app.data.init_elastic(app)
if context:
context.app = app
context.client = app.test_client()
def get_config(
self,
bundle: Bundle,
env: Union[DEV, PROD, STAGING, TEST],
) -> AttrDict:
bundle_config_module = self.import_bundle_module(bundle)
base_config = getattr(bundle_config_module, BASE_CONFIG, None)
env_config = getattr(bundle_config_module, ENV_CONFIGS[env], None)
if (isinstance(bundle, AppBundle) and
(not base_config or not issubclass(base_config, AppConfig))):
raise Exception("Could not find an AppConfig subclass in your app "
"bundle's config module.")
merged = Config(None)
for config in [base_config, env_config]:
if config:
merged.from_object(config)
if isinstance(bundle, AppBundle) and 'SECRET_KEY' not in merged:
raise Exception(
"The `SECRET_KEY` config option is required. "
"Please set it in your app bundle's base `Config` class.")
return AttrDict(merged)
def worker(verbose, queues, worker_ttl, job_monitoring_interval, config_file):
"""
Run an RQ worker, with config loaded from CONFIG_FILE,
to process commands on the QUEUES.
"""
from sentry_sdk.integrations.rq import RqIntegration
sentry_sdk.init(send_default_pii=False, integrations=[RqIntegration()])
from rq import Connection, Worker
import solarforecastarbiter # NOQA preload
from sfa_api.utils.queuing import make_redis_connection
config = Config(Path.cwd())
config.from_pyfile(config_file)
worker_loglevel = _get_log_level(config, verbose, key='WORKER_LOG_LEVEL')
_setup_logging(_get_log_level(config, verbose))
if 'QUEUES' in config: # pragma: no cover
queues = config['QUEUES']
# will likely want to add prometheus in here somewhere,
# perhaps as custom worker class
# if possible, get len report obj, time range
red = make_redis_connection(config)
with Connection(red):
w = Worker(queues,
default_worker_ttl=worker_ttl,
job_monitoring_interval=job_monitoring_interval)
w.work(logging_level=worker_loglevel)
def __init__(self, args):
self.env = args.env
self.action = args.action
self.kill_delay = args.kill_delay
self.conn = None
# Load settings
self.config = Config(os.path.dirname(os.path.realpath(__file__)))
# There may be overiding settings specific to the server we are running on
servername = socket.gethostname().split('.')[0]
if servername and os.path.isfile(f'settings/workers/{self.env}_{servername}.py'):
self.config.from_object(f'settings.workers.{self.env}_{servername}.Config')
else:
self.config.from_object(f'settings.workers.{self.env}.Config')
# Redis
if self.config['REDIS_USE_SENTINEL']:
sentinel = redis.sentinel.Sentinel(
self.config['REDIS_ADDRESS'],
db=self.config['REDIS_DB'],
password=self.config['REDIS_PASSWORD'],
decode_responses=True
)
self.conn = sentinel.master_for(self.config['REDIS_SENTINEL_MASTER'])
else:
self.conn = redis.StrictRedis(
host=self.config['REDIS_ADDRESS'][0],
port=self.config['REDIS_ADDRESS'][1],
db=self.config['REDIS_DB'],
password=self.config['REDIS_PASSWORD'],
decode_responses=True
)
def Config(envvar_silent=True) -> FlaskConfig:
global _Config_instance
if _Config_instance is None:
_Config_instance = FlaskConfig('.')
_Config_instance.from_object('ledslie.defaults')
_Config_instance.from_envvar('LEDSLIE_CONFIG', silent=envvar_silent)
return _Config_instance
def __init__(self,
app_name='enhanced-flask-app',
config_file=None,
js_filters=[],
css_filters=[],
**flask_kwargs):
"""
"""
config = Config('.', Flask.default_config)
if config_file:
config.from_object(config_file)
self.app = Flask(app_name, **flask_kwargs)
self.app.config = config
# Create webassets
self.assets_env = Environment(self.app)
self.assets_env.url_expire = True
self.assets_env.url = '/static'
# Initialize additional jinja stuff
self.enhance_jinja(self.app.jinja_env)
# Flask assets related
self.js_filters = []
self.css_filters = []
self.depends_scss = []
self.enhance_assets()
def load_graph_from_db(time_limit):
config = Config('./')
config.from_pyfile('web_config.cfg')
with NodeDB(config) as db:
nodes = db.get_nodes(time_limit)
edges = db.get_edges(nodes, 60*60*24*7)
return (nodes, edges)
def load_graph_from_db(time_limit):
config = Config('./')
config.from_pyfile('web_config.cfg')
with NodeDB(config) as db:
nodes = db.get_nodes(time_limit)
edges = db.get_edges(nodes, 60 * 60 * 24 * 7)
return (nodes, edges)
def app():
from flask import Config
from newsroom import Newsroom
cfg = Config(root)
cfg.from_object('newsroom.default_settings')
update_config(cfg)
return Newsroom(config=cfg)
def app():
from flask import Config
from newsroom.news_api.app import NewsroomNewsAPI
cfg = Config(root)
cfg.from_object('newsroom.news_api.settings')
update_config(cfg)
return NewsroomNewsAPI(config=cfg, testing=True)
def initialize_service(self, config: Config):
logger.info(
f'''Connection details: {config.get('REDIS_HOST')}:{config.get('REDIS_PORT')}'''
)
self.__redis_connection = Redis(host=config.get('REDIS_HOST'),
port=config.get('REDIS_PORT'),
password=config.get('REDIS_P'),
health_check_interval=15)
def app():
from flask import Config
from newsroom.web import NewsroomWebApp
cfg = Config(root)
cfg.from_object('newsroom.default_settings')
update_config(cfg)
return NewsroomWebApp(config=cfg, testing=True)
def get_neptune_graph_traversal_source_factory_from_config(
config: Config) -> Callable[[], GraphTraversalSource]:
session = config.get('NEPTUNE_SESSION')
assert session is not None
neptune_url = config.get('NEPTUNE_URL')
assert neptune_url is not None
return get_neptune_graph_traversal_source_factory(neptune_url=neptune_url,
session=session)
def get_config():
"""Get the configuration file and return a flask.Config instance."""
config = Config("/")
if os.path.exists("/etc/upload_rest_api.conf"):
config.from_pyfile("/etc/upload_rest_api.conf")
else:
logging.error("/etc/upload_rest_api.conf not found!")
return config
def get(pipeline_name):
config = Config(None)
config.from_object('brainminer.settings')
pipelines = config['PIPELINES']
if pipeline_name not in pipelines.keys():
return None
pipeline_cls = getattr(importlib.import_module(
pipelines[pipeline_name]['module_path']), pipelines[pipeline_name]['class_name'])
pipeline_obj = pipeline_cls()
return pipeline_obj
def _init_config():
import os
import sys
from flask import Config
from . import defconfig
config = Config(os.path.abspath(os.path.curdir))
config.from_object(defconfig)
del sys.modules[__name__]._init_config
return config
def setup(context=None, config=None, app_factory=get_app):
app_abspath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = Config(app_abspath)
app_config.from_object('superdesk.tests.test_settings')
app_config['APP_ABSPATH'] = app_abspath
app_config.update(get_test_settings())
app_config.update(config or {})
app_config.update({
'DEBUG': True,
'TESTING': True,
})
app = app_factory(app_config)
logger = logging.getLogger('superdesk')
logger.setLevel(logging.ERROR)
logger = logging.getLogger('elasticsearch')
logger.setLevel(logging.ERROR)
logger = logging.getLogger('urllib3')
logger.setLevel(logging.ERROR)
drop_elastic(app)
drop_mongo(app)
# create index again after dropping it
app.data.init_elastic(app)
if context:
context.app = app
context.client = app.test_client()
def setup_config(config):
app_abspath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = Config(app_abspath)
app_config.from_object("superdesk.default_settings")
cwd = Path.cwd()
for p in [cwd] + list(cwd.parents):
settings = p / "settings.py"
if settings.is_file():
logger.info(f"using local settings from {settings}")
app_config.from_pyfile(settings)
break
else:
logger.warning("Can't find local settings")
update_config(app_config)
app_config.update(
config or {},
**{
"APP_ABSPATH": app_abspath,
"DEBUG": True,
"TESTING": True,
},
)
logging.getLogger("apps").setLevel(logging.WARNING)
logging.getLogger("elastic").setLevel(logging.WARNING) # elastic datalayer
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("celery").setLevel(logging.WARNING)
logging.getLogger("superdesk").setLevel(logging.ERROR)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
logging.getLogger("superdesk.errors").setLevel(logging.CRITICAL)
return {key: deepcopy(val) for key, val in app_config.items()}
def __init__(self, *args, **kwargs):
"""
LibConfig is essentially just a wrapper around
a ConfigParser that reads the combined configuration
files from the command line (typically).
"""
self.localconf = ""
self.baseconf = ""
self.parser = None
self.tainted = False
Config.__init__(self, *args, **kwargs)
def get_config(path=None, filename=None):
"""
Used to make the config globally accessible for jobs
Has to be accessed once with a path and config filename.
After that it will return the same object everytime without the need to pass a path or a filename
:param path: The path to the containing folder of the config file
:param filename: The filename of the config file (Needs to be a .py file)
:return: The loaded :class:`~flask.Config` instance
"""
if get_config.cfg is None and path is not None and filename is not None:
cfg = Config(path)
cfg.from_pyfile(filename=filename)
get_config.cfg = cfg
return get_config.cfg
def create_app(config_file=None, **kwargs):
"""
Create a new eve app object and initialize everything.
User configuration can be loaded in the following order:
1. Use the `config_file` arg to specify a file
2. If `config_file` is `None`, you set the environment variable
`AMIVAPI_CONFIG` to the path of your config file
3. If no environment variable is set either, `config.py` in the current
working directory is used
Args:
config (path): Specify config file to use.
kwargs: All other key-value arguments will be used to update the config
Returns:
(Eve): The Eve application
"""
# Load config
config = Config(getcwd())
config.from_object("amivapi.settings")
# Specified path > environment var > default path; abspath for better log
user_config = abspath(config_file or getenv('AMIVAPI_CONFIG', 'config.py'))
try:
config.from_pyfile(user_config)
config_status = "Config loaded: %s" % user_config
except IOError:
config_status = "No config found."
config.update(kwargs)
app = Eve(settings=config, validator=ValidatorAMIV)
app.logger.info(config_status)
# Set up error logging with sentry
init_sentry(app)
# Create LDAP connector
ldap.init_app(app)
# Initialize modules to register resources, validation, hooks, auth, etc.
users.init_app(app)
auth.init_app(app)
events.init_app(app)
groups.init_app(app)
joboffers.init_app(app)
beverages.init_app(app)
studydocs.init_app(app)
cascade.init_app(app)
cron.init_app(app)
documentation.init_app(app)
# Fix that eve doesn't run hooks on embedded documents
app.on_fetched_item += utils.run_embedded_hooks_fetched_item
app.on_fetched_resource += utils.run_embedded_hooks_fetched_resource
return app
def __init__(self, app, registry_namespace=None):
super(ConfigurationRegistry, self).__init__(
'config',
registry_namespace=registry_namespace,
with_setup=False,
)
# Create a new configuration module to collect configuration in.
self.new_config = Config(app.config.root_path)
# Auto-discover configuration in packages
self.discover(app)
# Overwrite default configuration with user specified configuration
self.new_config.update(app.config)
app.config.update(self.new_config)
def test():
config = Config('application/config/')
config.from_pyfile('testing.cfg')
#db('createall', database=config['POSTGRESQL_DATABASE_DB'])
print ' [START] Testing server'
proc = subprocess.Popen(["python", "runserver.py", "testing"])
time.sleep(5)
call(["python", "application/tests/testsuite.py"])
print 'KILLING THE TESTING SERVER...'
time.sleep(5)
os.kill(proc.pid, signal.SIGTERM)
for pid in children_pid(proc.pid):
os.kill(pid, signal.SIGTERM)
def __init__(self) -> None:
self.plugins = OrderedDict() # type: OrderedDict[str, PluginBase]
self.rules = None # entry point
self.config = Config('/')
app.init_app() # fake app for plugin config (deprecated)
def setup_config(config):
app_abspath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = Config(app_abspath)
app_config.from_object('superdesk.default_settings')
update_config(app_config)
app_config.update(config or {}, **{
'APP_ABSPATH': app_abspath,
'DEBUG': True,
'TESTING': True,
})
logging.getLogger('superdesk').setLevel(logging.WARNING)
logging.getLogger('elastic').setLevel(logging.WARNING) # elastic datalayer
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
return app_config
def main():
if len(sys.argv) > 1 and len(sys.argv) != 4:
print "You must either specify no parameters or exactly 3: <username> <password> <tenant>.\n" \
"If you specify no parameters, credentials and tenant will be taken from config"
exit
scriptDir = os.path.dirname(os.path.realpath(__file__))
config = Config(scriptDir + '/../etc')
config.from_pyfile('local.cfg')
print "Configuration has been loaded from 'etc/local.cfg'"
if len(sys.argv) == 4:
user = sys.argv[1]
password = sys.argv[2]
tenant = sys.argv[3]
else:
print "You didn't provided credentials, using ones found in config"
user = config['OS_ADMIN_USER']
password = config['OS_ADMIN_PASSWORD']
tenant = config['OS_ADMIN_TENANT']
protocol = config['OS_AUTH_PROTOCOL']
host = config['OS_AUTH_HOST']
port = config['OS_AUTH_PORT']
auth_url = "%s://%s:%s/v2.0/" % (protocol, host, port)
print "User: %s" % user
print "Password: %s" % password
print "Tenant: %s" % tenant
print "Auth URL: %s" % auth_url
keystone = keystone_client(
username=user,
password=password,
tenant_name=tenant,
auth_url=auth_url
)
result = keystone.authenticate()
print "Auth succeed: %s" % result
print "Auth token: %s" % keystone.auth_token
print "Tenant [%s] id: %s" % (tenant, keystone.tenant_id)
def create_app(config_file=None, **kwargs):
"""Create a new eve app object and initialize everything.
User configuration can be loaded in the following order:
1. Use the `config_file` arg to specify a file
2. If `config_file` is `None`, you set the environment variable
`PVK_CONFIG` to the path of your config file
3. If no environment variable is set either, `config.py` in the current
working directory is used
Args:
config (path): Specify config file to use.
kwargs: All other key-value arguments will be used to update the config
Returns:
(Eve): The Eve application
"""
# Load config
config = Config(getcwd())
config.from_object("backend.settings")
# Specified path > environment var > default path; abspath for better log
user_config = abspath(config_file or getenv('PVK_CONFIG', 'config.py'))
try:
config.from_pyfile(user_config)
config_status = "Config loaded: %s" % user_config
except IOError:
config_status = "No config found."
config.update(kwargs)
# Create the app object
application = Eve(auth=APIAuth, validator=APIValidator, settings=config)
application.logger.info(config_status)
# Eve provides hooks at several points of the request,
# we use this do add dynamic filtering
for resource in ['signups', 'selections']:
for method in ['GET', 'PATCH', 'DELETE']:
event = getattr(application, 'on_pre_%s_%s' % (method, resource))
event += only_own_nethz
# Also use hooks to add pre- and postprocessing to resources
application.on_post_POST_signups += new_signups
application.on_deleted_item_signups += deleted_signup
application.on_updated_signups += patched_signup
application.on_updated_courses += patched_course
application.on_delete_item_courses += block_course_deletion
application.on_insert_payments += create_payment
application.on_inserted_payments += mark_as_paid
application.on_deleted_item_payments += mark_as_unpaid
return application
def setup_config(config):
app_abspath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = Config(app_abspath)
app_config.from_object("superdesk.default_settings")
update_config(app_config)
app_config.update(
config or {},
**{
"APP_ABSPATH": app_abspath,
"DEBUG": True,
"TESTING": True,
},
)
logging.getLogger("apps").setLevel(logging.WARNING)
logging.getLogger("elastic").setLevel(logging.WARNING) # elastic datalayer
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("celery").setLevel(logging.WARNING)
logging.getLogger("superdesk").setLevel(logging.ERROR)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
logging.getLogger("superdesk.errors").setLevel(logging.CRITICAL)
return {key: deepcopy(val) for key, val in app_config.items()}
def main():
config = Config(__name__)
config.from_envvar('COMPOWSR_SETTINGS', silent=True)
# get the praw stuff
reddit = praw.Reddit(config['PRAW_SITE_NAME'],
user_agent='flair_updater by /u/Witchtower')
subreddit = reddit.subreddit(config['PRAW_SUBREDDIT_NAME'])
# get reddit users with rank flair set
print "getting all users with rank-flair from reddit"
to_update = [(user['user'].name, user['flair_css_class'])
for user in subreddit.flair()
if user['flair_css_class'] in config['OW_RANKS'].keys()]
print "got %i users with a rank flair" % len(to_update)
# check who really needs to be updated
db = connect_db(config)
to_update_usernames = [i[0] for i in to_update]
statement = "SELECT * FROM acc_links " #\
#% ','.join(['?']*len(to_update_usernames))
# AND last_update > datetime('now', '+14 days')" \
print statement
print to_update_usernames
cursor = db.execute(statement) #, to_update_usernames)
to_update2 = cursor.fetchall()
print "%i users haven't been updated in the last 14 days" % len(to_update2)
for row in to_update2:
# pull sr and update in db
new_rank = playoverwatch.CareerProfile('eu', row['bnet_name']).rank
db.execute(
"UPDATE acc_links SET \
last_rank = ?, \
last_update = datetime('now') \
WHERE bnet_name = ?", [new_rank, row['bnet_name']])
subreddit.flair.set(row['reddit_name'], css_class=new_rank)
print "flair '%s' set for '%s'" % (new_rank, row['reddit_name'])
print "all done"
def create(
self,
telegram_service: telegram.TelegramService,
links: model_link.Links,
extractor: services.TitleExtractor,
config: flask.Config,
) -> incoming.Incoming:
return incoming.Incoming(telegram_service, links, extractor,
config.get("PROJECT_ID"))
class ConfigurationRegistry(ModuleDiscoveryRegistry):
"""
Specialized ``ModuleDiscoveryRegistry`` that search for ``config`` modules
in a list of Python packages and merge them into the Flask application
config without overwriting already set variables.
:param app: A Flask application
:param registry_namespace: The registry namespace of an
``ImportPathRegistry`` with a list Python packages to search for
``config`` modules in. Defaults to ``packages``.
"""
def __init__(self, app, registry_namespace=None):
super(ConfigurationRegistry, self).__init__(
'config',
registry_namespace=registry_namespace,
with_setup=False,
)
# Create a new configuration module to collect configuration in.
from flask import Config
self.new_config = Config(app.config.root_path)
# Auto-discover configuration in packages
self.discover(app)
# Overwrite default configuration with user specified configuration
self.new_config.update(app.config)
app.config = self.new_config
def register(self, new_object):
"""
Register a new ``config`` module.
:param new_object: The configuration module.
``app.config.from_object()`` will be called on it.
"""
self.new_config.from_object(new_object)
super(ConfigurationRegistry, self).register(new_object)
def unregister(self, *args, **kwargs):
"""
It is not possible to unregister configuration.
"""
raise NotImplementedError()
def init_app(cls, app):
Config.init_app(app)
# email errors to the administrators
import logging
from logging.handlers import SMTPHandler
credentials = None
secure = None
if getattr(cls, 'MAIL_USERNAME', None) is not None:
credentials = (cls.MAIL_USERNAME, cls.MAIL_PASSWORD)
if getattr(cls, 'MAIL_USE_TLS', None):
secure = ()
mail_handler = SMTPHandler(mailhost=(cls.MAIL_SERVER, cls.MAIL_PORT),
fromaddr=cls.FLASKY_MAIL_SENDER,
toaddrs=[cls.FLASKY_ADMIN],
subject=cls.FLASKY_MAIL_SUBJECT_PREFIX + ' Application Error',
credentials=credentials,
secure=secure)
mail_handler.setLevel(logging.ERROR)
app.logger.addHandler(mail_handler)
class ConfigurationRegistry(ModuleDiscoveryRegistry):
"""
Specialized ``ModuleDiscoveryRegistry`` that search for ``config`` modules
in a list of Python packages and merge them into the Flask application
config without overwriting already set variables.
:param app: A Flask application
:param registry_namespace: The registry namespace of an
``ImportPathRegistry`` with a list Python packages to search for
``config`` modules in. Defaults to ``packages``.
"""
def __init__(self, app, registry_namespace=None):
super(ConfigurationRegistry, self).__init__(
'config',
registry_namespace=registry_namespace,
with_setup=False,
)
# Create a new configuration module to collect configuration in.
self.new_config = Config(app.config.root_path)
# Auto-discover configuration in packages
self.discover(app)
# Overwrite default configuration with user specified configuration
self.new_config.update(app.config)
app.config.update(self.new_config)
def register(self, new_object):
"""
Register a new ``config`` module.
:param new_object: The configuration module.
``app.config.from_object()`` will be called on it.
"""
self.new_config.from_object(new_object)
super(ConfigurationRegistry, self).register(new_object)
def unregister(self, *args, **kwargs):
"""
It is not possible to unregister configuration.
"""
raise NotImplementedError()
def run():
"""Parse the command line arguments and run the application"""
args = parser.parse_args()
config = Config(getcwd())
# Set the configuration options from the file given
if args.conf:
config.from_pyfile(args.conf)
# Load the rest of the arguments, overriding the conf file
config['DEBUG'] = args.debug
# Create the application and create the database
app = create_app(config)
# Show the application config
if args.show:
from pprint import pprint
pprint(dict(app.config))
# Run the application
app.run(threaded=True)
def main(expected):
config = Config(here)
config.from_object('ooi_status.default_settings')
if 'OOISTATUS_SETTINGS' in os.environ:
config.from_envvar('OOISTATUS_SETTINGS')
for key in config:
log.info('OOI_STATUS CONFIG: %r: %r', key, config[key])
monitor = StatusMonitor(config)
if expected:
monitor.read_expected_csv(expected)
else:
scheduler = BlockingScheduler()
log.info('adding jobs')
# notify on change every minute
scheduler.add_job(monitor.check_all, 'cron', second=0)
scheduler.add_job(monitor.notify_all, 'cron', second=10)
log.info('starting jobs')
scheduler.start()
class ConfigurationRegistry(ModuleDiscoveryRegistry):
"""
Specialized import path registry that takes the initial list of import
paths from ``PACKAGES`` configuration variable.
Example::
app.extensions['registry']['packages'] = PackageRegistry()
app.extendsions['registry']['config'] = ConfigurationRegistry(
_app, base_config='invenio.core.config'
)
"""
def __init__(self, app, registry_namespace=None):
super(ConfigurationRegistry, self).__init__(
'config',
registry_namespace=registry_namespace,
with_setup=False,
)
# Create a new configuration module to collect configuration in.
from flask import Config
self.new_config = Config(app.config.root_path)
# Auto-discover configuration in packages
self.discover(app)
# Overwrite default configuration with user specified configuration
self.new_config.update(app.config)
app.config = self.new_config
def register(self, new_object):
self.new_config.from_object(new_object)
super(ConfigurationRegistry, self).register(new_object)
def unregister(self, *args, **kwargs):
raise NotImplementedError()
def __init__(self, app, registry_namespace=None):
super(ConfigurationRegistry, self).__init__(
'config',
registry_namespace=registry_namespace,
with_setup=False,
)
# Create a new configuration module to collect configuration in.
self.new_config = Config(app.config.root_path)
# Auto-discover configuration in packages
self.discover(app)
# Overwrite default configuration with user specified configuration
self.new_config.update(app.config)
app.config.update(self.new_config)
def __init__(self):
self._config = FlaskConfig("")
# project
_project = self.load_project()
# local
_xdg_config = self.from_xdg_config()
_env = self.from_env()
_local = self.from_local()
# debug
_debug = self.load_debug()
self.extract_conf(self._config)
if not (_xdg_config or _env or _local):
raise NoConfigFoundError
def create_app(config_file=None, **kwargs):
"""
Create a new eve app object and initialize everything.
User configuration can be loaded in the following order:
1. Use the `config_file` arg to specify a file
2. If `config_file` is `None`, you set the environment variable
`AMIVAPI_CONFIG` to the path of your config file
3. If no environment variable is set either, `config.py` in the current
working directory is used
Args:
config (path): Specify config file to use.
kwargs: All other key-value arguments will be used to update the config
Returns:
(Eve): The Eve application
"""
# Load config
config = Config(getcwd())
config.from_object("amivapi.settings")
# Specified path > environment var > default path; abspath for better log
user_config = abspath(config_file or getenv('AMIVAPI_CONFIG', 'config.py'))
try:
config.from_pyfile(user_config)
config_status = "Config loaded: %s" % user_config
except IOError:
config_status = "No config found."
config.update(kwargs)
# Initialize empty domain to create Eve object, register resources later
config['DOMAIN'] = {}
app = Eve("amivapi", # Flask needs this name to find the static folder
settings=config,
validator=ValidatorAMIV)
app.logger.info(config_status)
# Set up error logging with sentry
init_sentry(app)
# Create LDAP connector
ldap.init_app(app)
# Initialize modules to register resources, validation, hooks, auth, etc.
users.init_app(app)
auth.init_app(app)
events.init_app(app)
groups.init_app(app)
blacklist.init_app(app)
joboffers.init_app(app)
beverages.init_app(app)
studydocs.init_app(app)
cascade.init_app(app)
cron.init_app(app)
documentation.init_app(app)
# Fix that eve doesn't run hooks on embedded documents
app.on_fetched_item += utils.run_embedded_hooks_fetched_item
app.on_fetched_resource += utils.run_embedded_hooks_fetched_resource
return app
from flask import redirect, url_for, request, g, Config
from flask_oauthlib.client import OAuth
from flask.ext.login import LoginManager, login_required, \
login_user, logout_user
from matchmaker import app
from models import User, BotIdentity
# Wire everything up
login_manager = LoginManager()
login_manager.init_app(app)
oauth = OAuth(app)
# Casino GitHub OAuth
oauth_config = Config("")
oauth_config.from_object('matchmaker.default_oauth')
oauth_config.from_envvar('CASINO_OAUTH', silent=True)
github = oauth.remote_app(
'github',
**{k.lower(): v for k, v in oauth_config.items()}
)
@app.route('/login')
def login():
return github.authorize(
callback=url_for('authorized', _external=True,
_scheme=app.config.get("PREFERRED_URL_SCHEME"))
)
def make_config(filename=None):
config = Config(getcwd())
if filename is not None:
config.from_pyfile(filename)
return config
from asyncio import get_event_loop, coroutine, gather, wait
from io import BytesIO
import logging
from os.path import dirname
from aiohttp import web
from flask import Config
import arrow
import pygal
from .redmine import AsyncRedmine
config = Config(dirname(__file__))
config.from_pyfile('default.cfg')
config.from_pyfile('settings.cfg')
pygal_config = pygal.Config(style=config['PYGAL_STYLE'])
redmine = AsyncRedmine(config['REDMINE_URL'], key=config['API_KEY'])
access_logger = logging.getLogger('aiohttp.access')
access_logger.addHandler(logging.StreamHandler())
access_logger.setLevel(logging.INFO)
issue_statuses = []
def main():
loop = get_event_loop()
loop.run_until_complete(redminecharts(loop))
import re
from flask import Config
engine = None
sessionmaker = sa.orm.sessionmaker()
session = sa.orm.scoped_session(sessionmaker)
def configure_engine(url):
global sessionmaker, engine, session
engine = sa.create_engine(url)
session.remove()
sessionmaker.configure(bind=engine)
_config = Config('')
_config.from_object('config')
_config.from_envvar('MULCHN_CONFIG', silent=True)
configure_engine(_config.get('DATABASE_URL'))
class _Base(object):
@declared_attr
def __tablename__(cls):
"""
Convert CamelCase class name to underscores_between_words
table name.
"""
name = cls.__name__
return (
name[0].lower() +
Binds to 0.0.0.0:23887 by default.
"""
import asyncio
import logging
import os
os.environ["INSIDE_LISTENER"] = "y"
import sqlalchemy
from gunrskite import parser as gparse
from gunrskite import consumer as consumer
from flask import Config
cfg = Config(os.path.abspath("."))
cfg.from_pyfile("config.py")
loop = asyncio.get_event_loop()
formatter = logging.Formatter('%(asctime)s - [%(levelname)s] %(name)s -> %(message)s')
root = logging.getLogger()
root.setLevel(cfg.get("LOG_LEVEL", logging.INFO))
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(formatter)
root.addHandler(consoleHandler)
logger = logging.getLogger("Gunrskite::Listener")
#-------------------------------------------------------------------------------
import datetime
import os
import re
import redis
import urlparse
from flask import Blueprint, Config, flash, g, jsonify, request, render_template
from flask import session, url_for
from flask_bibframe.models import CoverArt
blueprint_folder = os.path.abspath(os.path.dirname(__file__))
app_folder = os.path.split(blueprint_folder)[0]
metrics_config = Config(app_folder)
metrics_config.from_pyfile('catalog.cfg')
redis_ds = redis.StrictRedis(metrics_config.get('REDIS_HOST'))
def parse_logfile(log_path, redis_ds=redis_ds, tz='-0700'):
"""Takes a nginx access log, opens the file, and parses through and creates
metrics for use by other Blueprints and apps.
Parameters
----------
log_path : str
Full path and filename of the nginx access log
redis_ds : StrictRedis instance
defaults to Redis datastore for CPP's metrics
class Configurator(object):
"""
Object that takes care of loading the different configurations from the
different sources. There are 3 types of settings:
* Project: The basic set of settings needed by the system. These are
shipped with Shiva.
* Local: Specific to each instance, useful for overwriting system
settings. Some of them must be defined before running Shiva, like the
DB URI.
* Debug: This setting will only be loaded if ``DEBUG`` is set to True
in the local settings.
There are also 3 different places where Shiva looks for this config files:
* A ``local.py`` file inside the ``config/`` directory, relative to
Shiva.
* The ``$SHIVA_CONFIG`` environment variable. It's assumed to be
pointing to a file (not a dir) if exists.
* The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If
``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as
defined by the `XDG Base Directory Specification
<http://standards.freedesktop.org/basedir-spec/basedir-spec-latest\
.html>`_.
"""
def __init__(self):
self._config = FlaskConfig("")
# project
_project = self.load_project()
# local
_xdg_config = self.from_xdg_config()
_env = self.from_env()
_local = self.from_local()
# debug
_debug = self.load_debug()
self.extract_conf(self._config)
if not (_xdg_config or _env or _local):
raise NoConfigFoundError
def load_project(self):
return self._config.from_object(project)
def get_xdg_path(self):
path_home = os.getenv("HOME")
if not path_home:
return None
default_config_home = os.path.join(path_home, ".config")
return os.getenv("XDG_CONFIG_HOME") or default_config_home
def from_xdg_config(self):
xdg_path = self.get_xdg_path()
if not xdg_path:
return False
local_py = os.path.join(xdg_path, "shiva/config.py")
if not os.path.exists(local_py):
return False
return self._config.from_pyfile(local_py)
def from_env(self):
if not os.getenv("SHIVA_CONFIG"):
return False
return self._config.from_envvar("SHIVA_CONFIG")
def from_local(self):
with ignored(ImportError):
self._config.from_object("shiva.config.local")
return True
return False
def load_debug(self):
if not self._config.get("DEBUG"):
return False
loaded = False
with ignored(ImportError):
from shiva.config import debug
loaded = self._config.from_object(debug)
xdg_path = self.get_xdg_path()
if not xdg_path:
return False
debug_py = os.path.join(xdg_path, "shiva/debug.py")
if not os.path.exists(debug_py):
return False
return self._config.from_pyfile(debug_py) or loaded
def extract_conf(self, *args):
"""
Receives one or more objects, iterates over their elements, extracts
all the uppercase properties and injects them into the Configurator
object.
"""
for obj in args:
for key in dir(obj):
if key.isupper():
setattr(self, key, getattr(obj, key))
if hasattr(obj, "iterkeys") and callable(getattr(obj, "iterkeys")):
for key in obj.iterkeys():
if key.isupper():
setattr(self, key, obj[key])
* DATABASE_URI
* DATABASE
Currently the supported environment variables:
* OpenShift
* DATABASE_URI
"""
import os
from flask import Config
from sqlalchemy.engine.url import URL
config = Config('.')
config.from_object('acj.settings')
config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../config.py'), silent=True)
if os.environ.get('OPENSHIFT_MYSQL_DB_HOST'):
config['SQLALCHEMY_DATABASE_URI'] = URL({
'drivername': 'mysql',
'host': os.environ.get('OPENSHIFT_MYSQL_DB_HOST'),
'port': os.environ.get('OPENSHIFT_MYSQL_DB_PORT'),
'username': os.environ.get('OPENSHIFT_MYSQL_DB_USERNAME'),
'password': os.environ.get('OPENSHIFT_MYSQL_DB_PASSWORD'),
'database': os.environ.get('OPENSHIFT_GEAR_NAME'),
})
elif os.environ.get('DATABASE_URI'):
config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI')
elif "DATABASE" in config and 'DATABASE_URI' not in config:
from flask import Config
cfg = Config('.')
cfg.from_envvar('TIGERGRADER_SETTINGS')
def get_user_config():
from flask import Config
config = Config('/')
config.from_object('alerta.settings')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
if 'DEBUG' in os.environ:
config['DEBUG'] = True
if 'BASE_URL' in os.environ:
config['BASE_URL'] = os.environ['BASE_URL']
if 'USE_PROXYFIX' in os.environ:
config['USE_PROXYFIX'] = True if os.environ['USE_PROXYFIX'] == 'True' else False
if 'SECRET_KEY' in os.environ:
config['SECRET_KEY'] = os.environ['SECRET_KEY']
database_url = (
os.environ.get('DATABASE_URL', None) or
# The following database settings are deprecated.
os.environ.get('MONGO_URI', None) or
os.environ.get('MONGODB_URI', None) or
os.environ.get('MONGOHQ_URL', None) or
os.environ.get('MONGOLAB_URI', None)
)
# Use app config for DATABASE_URL if no env var from above override it
config['DATABASE_URL'] = database_url or config['DATABASE_URL']
if 'DATABASE_NAME' in os.environ:
config['DATABASE_NAME'] = os.environ['DATABASE_NAME']
if 'AUTH_REQUIRED' in os.environ:
config['AUTH_REQUIRED'] = True if os.environ['AUTH_REQUIRED'] == 'True' else False
if 'AUTH_PROVIDER' in os.environ:
config['AUTH_PROVIDER'] = os.environ['AUTH_PROVIDER']
if 'ADMIN_USERS' in os.environ:
config['ADMIN_USERS'] = os.environ['ADMIN_USERS'].split(',')
if 'SIGNUP_ENABLED' in os.environ:
config['SIGNUP_ENABLED'] = True if os.environ['SIGNUP_ENABLED'] == 'True' else False
if 'CUSTOMER_VIEWS' in os.environ:
config['CUSTOMER_VIEWS'] = True if os.environ['CUSTOMER_VIEWS'] == 'True' else False
if 'OAUTH2_CLIENT_ID' in os.environ:
config['OAUTH2_CLIENT_ID'] = os.environ['OAUTH2_CLIENT_ID']
if 'OAUTH2_CLIENT_SECRET' in os.environ:
config['OAUTH2_CLIENT_SECRET'] = os.environ['OAUTH2_CLIENT_SECRET']
if 'ALLOWED_EMAIL_DOMAINS' in os.environ:
config['ALLOWED_EMAIL_DOMAINS'] = os.environ['ALLOWED_EMAIL_DOMAINS'].split(',')
if 'AZURE_TENANT' in os.environ:
config['AZURE_TENANT'] = os.environ['AZURE_TENANT']
if 'GITHUB_URL' in os.environ:
config['GITHUB_URL'] = os.environ['GITHUB_URL']
if 'ALLOWED_GITHUB_ORGS' in os.environ:
config['ALLOWED_GITHUB_ORGS'] = os.environ['ALLOWED_GITHUB_ORGS'].split(',')
if 'GITLAB_URL' in os.environ:
config['GITLAB_URL'] = os.environ['GITLAB_URL']
if 'ALLOWED_GITLAB_GROUPS' in os.environ:
config['ALLOWED_OIDC_ROLES'] = os.environ['ALLOWED_GITLAB_GROUPS'].split(',')
if 'KEYCLOAK_URL' in os.environ:
config['KEYCLOAK_URL'] = os.environ['KEYCLOAK_URL']
if 'KEYCLOAK_REALM' in os.environ:
config['KEYCLOAK_REALM'] = os.environ['KEYCLOAK_REALM']
if 'ALLOWED_KEYCLOAK_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = os.environ['ALLOWED_KEYCLOAK_ROLES'].split(',')
if 'OIDC_ISSUER_URL' in os.environ:
config['OIDC_ISSUER_URL'] = os.environ['OIDC_ISSUER_URL']
if 'ALLOWED_OIDC_ROLES' in os.environ:
config['ALLOWED_OIDC_ROLES'] = os.environ['ALLOWED_OIDC_ROLES'].split(',')
if 'PINGFEDERATE_OPENID_ACCESS_TOKEN_URL' in os.environ:
config['PINGFEDERATE_OPENID_ACCESS_TOKEN_URL'] = os.environ['PINGFEDERATE_OPENID_ACCESS_TOKEN_URL'].split(
',')
if 'PINGFEDERATE_OPENID_PAYLOAD_USERNAME' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_USERNAME'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_USERNAME'].split(
',')
if 'PINGFEDERATE_OPENID_PAYLOAD_EMAIL' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_EMAIL'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_EMAIL'].split(',')
if 'PINGFEDERATE_OPENID_PAYLOAD_GROUP' in os.environ:
config['PINGFEDERATE_OPENID_PAYLOAD_GROUP'] = os.environ['PINGFEDERATE_OPENID_PAYLOAD_GROUP'].split(',')
if 'PINGFEDERATE_PUBKEY_LOCATION' in os.environ:
config['PINGFEDERATE_PUBKEY_LOCATION'] = os.environ['PINGFEDERATE_PUBKEY_LOCATION'].split(',')
if 'PINGFEDERATE_TOKEN_ALGORITHM' in os.environ:
config['PINGFEDERATE_TOKEN_ALGORITHM'] = os.environ['PINGFEDERATE_TOKEN_ALGORITHM'].split(',')
if 'CORS_ORIGINS' in os.environ:
config['CORS_ORIGINS'] = os.environ['CORS_ORIGINS'].split(',')
if 'MAIL_FROM' in os.environ:
config['MAIL_FROM'] = os.environ['MAIL_FROM']
if 'SMTP_PASSWORD' in os.environ:
config['SMTP_PASSWORD'] = os.environ['SMTP_PASSWORD']
if 'GOOGLE_TRACKING_ID' in os.environ:
config['GOOGLE_TRACKING_ID'] = os.environ['GOOGLE_TRACKING_ID']
if 'PLUGINS' in os.environ:
config['PLUGINS'] = os.environ['PLUGINS'].split(',')
# Runtime config check
if config['CUSTOMER_VIEWS'] and not config['AUTH_REQUIRED']:
raise RuntimeError('Must enable authentication to use customer views')
if config['CUSTOMER_VIEWS'] and not config['ADMIN_USERS']:
raise RuntimeError('Customer views is enabled but there are no admin users')
return config
from flask import Config
from os import path
config = Config('../')
config.from_object('config')
if path.exists(config["IMAGEMAGICK_CONVERT"]) is False:
raise StandardError("No imagemagick 'convert' found. Check your config or/and install imagemagick.")
from ifsApprover.DB import DB
db = DB(config["DB_NAME"])
def main(argconfig):
# Load the config file the same way Flask does which Chill uses.
config_file = argconfig if argconfig[0] == os.sep else os.path.join(os.getcwd(), argconfig)
config = Config(os.getcwd())
config.from_pyfile(config_file)
pyrax.set_credential_file(config["RACKSPACE_CREDENTIAL_FILE"])
cf = pyrax.cloudfiles
# sync sync_folder_to_container
local = config["FREEZER_DESTINATION"]
container_name = config.get("RACKSPACE_CONTAINER_NAME", os.path.basename(os.getcwd()))
prefix = config.get("PUBLIC_URL_PREFIX", None)
if prefix:
for page in (INDEX_PAGE, ERROR_PAGE, "401{0}".format(ERROR_PAGE), "404{0}".format(ERROR_PAGE)):
# If no index.html or error pages then link to the one found in
# PUBLIC_URL_PREFIX
if not os.path.exists(os.path.join(local, page)) and os.path.exists(os.path.join(local, prefix[1:], page)):
print "Creating hard link for {0}".format(page)
print "{0} -> {1}".format(os.path.join(local, page), os.path.join(local, prefix[1:], page))
os.link(os.path.join(local, prefix[1:], page), os.path.join(local, page))
confirm = raw_input(
"\nsync the folder: {local} \nto rackspace cloudfiles container: {container_name}\n[n]/y\n".format(**locals())
)
if confirm != "y":
return
remote = cf.create_container(container_name)
local_files = set()
for root, dir, files in os.walk(local):
for f in files:
local_files.add(os.path.join(root[len(local) + 1 :], f))
cf.sync_folder_to_container(local, remote)
# Mark all objects on remote to be deleted if not in local
# The limit here is arbitrary, but can not be higher then 10000.
limit = 1000
marker = ""
remote_objects_list = remote.get_objects(limit=limit, marker=marker)
while remote_objects_list:
marker = remote_objects_list[-1].name
for obj in remote_objects_list:
if obj.name not in local_files:
obj.delete_in_seconds(DELETE_OBJECTS_DELAY)
remote_objects_list = remote.get_objects(limit=limit, marker=marker)
# publish
cf.make_container_public(container_name, ttl=TTL)
# cdn website
remote.set_web_index_page(INDEX_PAGE)
remote.set_web_error_page(ERROR_PAGE)
# Totally copied from the docs
print
print "After Making Public"
print "cdn_enabled", remote.cdn_enabled
print "cdn_ttl", remote.cdn_ttl
print "cdn_log_retention", remote.cdn_log_retention
print "cdn_uri", remote.cdn_uri
print "cdn_ssl_uri", remote.cdn_ssl_uri
print "cdn_streaming_uri", remote.cdn_streaming_uri
print "cdn_ios_uri", remote.cdn_ios_uri
def FromEmails():
#Config, init variables
config = Config(os.getcwd())
config.from_pyfile('settings.cfg')
logger = logging.getLogger('ICONZ.HelpdeskDashboard.OutageParse')
logger.addHandler(config['HANDLER'])
def GetBody(msg):
if msg.is_multipart():
return(msg.get_payload(0)).get_payload(decode=True)
else:
return(msg.get_payload(decode=True))
def Passthrough(msg_list):
try:
smtp_conn = smtplib.SMTP(config['OUTAGES_SMTP_SERVER'],
config['OUTAGES_SMTP_PORT'])
for msg in msg_list:
smtp_conn.sendmail(config['OUTAGES_SMTP_FROM_ADDR'],
config['OUTAGES_SMTP_TO_ADDR'], msg.as_string())
logger.info('Succesfully passed through ' + str(len(msg_list)) + ' messages')
except Exception as e:
logger.error('Faliure to passthrough all messages.', exc_info = True)
def DBInsert(data):
#Could tidy this up so it's done as a batch operation. Probably unecessary
conn = connect_db()
cur = conn.cursor()
try:
cur.execute('select rowid from outage_texts where provider_ref_id match(?)',
[data['reference']])
db_outage = cur.fetchone()
db_outage_id = None
db_outage_text_id = None
if db_outage == None:
cur.execute('insert into outage_texts (provider_ref_id, equipment_list, full_text) values (?,?,?)',
[unicode(data['reference']), unicode(data['equipment']), unicode(data['full_text'])])
db_outage_text_id = cur.lastrowid
cur.execute('insert into outages (provider, num_affected, start_time, end_time, issues, text_id) values (?,?,?,?,?,?)',
[data['provider'], data['num_affected'], data['start_datetime'], data['end_datetime'], data['issues'], db_outage_text_id])
db_outage_id = cur.lastrowid
conn.commit()
logger.debug('Created new outage with id ' + str(db_outage_id))
else:
cur.execute('update outage_texts set equipment_list=(?), full_text=(?) where rowid==?',
[unicode(data['equipment']), unicode(data['full_text']), db_outage[0]])
db_outage_text_id = db_outage[0]
cur.execute('select id from outages where text_id==?', [db_outage_text_id])
db_outage_id = cur.fetchone()[0]
cur.execute('update outages set num_affected=(?), start_time=(?), end_time=(?), issues=(?) where id==?',
[data['num_affected'], data['start_datetime'], data['end_datetime'], data['issues'], db_outage_id])
conn.commit()
logger.debug('Updated outage with id ' + str(db_outage_id))
users = list()
for line in data['equipment'].split('\n'):
cur.execute('select id, name from dslams where name==?', [line])
dslam = cur.fetchone()
if dslam is not None:
cur.execute('select id, asid from dslusers where dslam_id==?', [dslam[0]])
users.append(cur.fetchall())
for db_block in users:
for user in db_block:
if user is not None:
cur.execute('select * from outages_dslusers_rel where (outages_id==?) and (dslusers_id==?)', [db_outage_id, user[0]])
if cur.fetchone() is None:
cur.execute('insert into outages_dslusers_rel (outages_id, dslusers_id) values (?,?)',
[db_outage_id, user[0]])
conn.commit()
#Clears connection so that program can continue in case of exception
except sqlite3.IntegrityError as e:
conn.close()
conn = connect_db()
raise(e)
def ChorusHTMLFat(data):
def FindContent(fieldname, soup, one = True):
if one:
out = [string for string in soup.find(
"td", text=re.compile(fieldname), style=re.compile('(;color:|; COLOR: )rgb\(0,72,97\);')
).parent.find(
"td", style=re.compile('(;color:|; COLOR: )rgb\(0,0,0\);')
).stripped_strings]
if len(out) < 1: return None
else: return out[0]
else:
return [string for string in soup.find(
"td", text=re.compile(fieldname), style=re.compile('(;color:|; COLOR: )rgb\(0,72,97\);')
).parent.find(
"td", style=re.compile('(;color:|; COLOR: )rgb\(0,0,0\);')
).stripped_strings]
soup = BeautifulSoup(data)
results = dict({'provider' : 'Chorus', 'reference' : FindContent('Ticket Number:', soup), 'issues' : False})
try:
start_date = FindContent('Date:', soup)
start_time = FindContent('Start Time:', soup)
if start_time is not None:
if start_time.endswith('(amended)'):
start_time = start_time[0:6]
else:
results['issues'] = True
start_time = '00:00'
results['start_datetime'] = datetime.datetime.strptime(start_date + ' ' +
start_time, '%d/%m/%Y %H:%M')
except (AttributeError, ValueError):
results['issues'] = True
results['start_datetime'] = None
try:
end_date = FindContent('Resolution Date:', soup)
end_time = FindContent('Resolution Time:', soup)
if end_date is None:
if end_time is None:
results['end_datetime'] = None
else:
results['end_datetime'] = datetime.datetime.combine(start_datetime.date,
datetime.time.strptime(end_time, '%H:%M'))
else:
results['end_datetime'] = datetime.datetime.strptime(end_date + ' ' +
end_time, '%d/%m/%Y %H:%M')
except (AttributeError, ValueError):
results['end_datetime'] = None
try:
results['num_affected'] = int(re.search(('(\d+)'), FindContent('Customer Impact:',
soup)).group(1))
except AttributeError:
results['issues'] = True
results['num_affected'] = 0
#Might be useful later if I can get ASID -> DSLAM mappings
equipment = FindContent('Equipment List:', soup, False)
results['equipment'] = ''
for string in equipment:
if re.match('[A-Z]{2,3}[-|_|/]([A-Z]{1,3}-)?DSLAM-\d{0,2}', string):
results['equipment'] += string + '\n'
else:
results['issues'] = True
results['full_text'] = ''
for string in soup.stripped_strings:
if re.match('^VF \w{1,3}$', string):
string = ''
results['full_text'] = results['full_text'] + string + '\n'
return results
def ChorusPlainText(f):
results = dict({'provider' : 'Chorus',
'reference' : re.search(('SED Ref.+:.+(\d{6})'), f).group(1),
'issues' : False})
start_date = re.search(('DATE.+:.+(\d{2}/\d{2}/\d{2})'), f)
start_time = re.search(('NZ Standard Time.+:.+(\d{2}:\d{2}:\d{2}.(AM|PM))'), f)
results['start_datetime'] = datetime.datetime.strptime(start_date.group(1) + ' ' +
start_time.group(1), '%d/%m/%y %I:%M:%S %p')
duration = re.search(('DURATION \(HR:MIN:SEC\).+:.+(\d{3}):(\d{2}):(\d{2})'), f)
results['end_datetime'] = results['start_datetime'] + datetime.timedelta(hours = int(duration.group(1)),
minutes = int(duration.group(2)), seconds = int(duration.group(3)))
try:
results['num_affected'] = int(re.search('Up to (\d+) DSL customers', f).group(1))
except AttributeError:
results['issues'] = True
results['num_affected'] = 0
logger.warning('ChorusPlainText could not extract num_affected')
equipment = re.findall(('([A-Z]{2,3}[-|_|/]([A-Z]{1,3}-)?DSLAM-\d{0,2})'), f)
results['equipment'] = ''
for regex_groups in equipment:
results['equipment'] += regex_groups[0] + '\n'
if 'DSLAM' not in results['equipment']:
results['issues'] = True
logger.warning('ChorusPlainText could not extract equipment_list')
results['full_text'] = str(f)
if results['issues'] == False:
logger.debug('ChorusPlainText parsed message without issue')
return results
def ChorusHTMLSkinny(outage):
soup = BeautifulSoup(outage)
results = dict({'provider' : 'Chorus', 'issues' : False})
text = ''
#Mandated by shitty HTML
stringgen = soup.stripped_strings
for string in stringgen:
if ('Field' not in string) and ('FIELD' not in string):
if 'Ticket Number:' in string:
stringgen.next()
results['reference'] = stringgen.next()
text += results['reference'] + '\n'
elif 'Location:' in string:
stringgen.next()
results['location'] = stringgen.next()
text += results['location'] + '\n'
elif 'Start Date:' in string:
stringgen.next()
results['start_datetime'] = stringgen.next()
text += results['start_datetime'] + '\n'
elif 'Start Time:' in string:
stringgen.next()
temp = stringgen.next()
results['start_datetime'] = results['start_datetime'] + ' ' + temp
text += temp + '\n'
elif 'Resolution Date:' in string:
stringgen.next()
results['end_datetime'] = stringgen.next()
text += results['end_datetime'] + '\n'
elif 'Resolution Time:' in string:
stringgen.next()
temp = stringgen.next()
results['end_datetime'] += ' ' + temp
text += temp + '\n'
elif 'Customer Impact:' in string:
stringgen.next()
temp = stringgen.next()
try:
results['num_affected'] = int(re.search('Up to (\d+) DSL customers', temp).group(1))
except AttributeError:
results['issues'] = True
results['num_affected'] = 0
logger.warning('ChorusHTMLSkinny could not extract num_affected')
text += temp + '\n'
elif 'Equipment List:' in string:
results['equipment'] = ''
stringgen.next()
temp = stringgen.next()
while 'Additional Information:' not in temp:
results['equipment'] += temp
text += temp + '\n'
temp = stringgen.next()
elif 'Cable Pair & Range:' in string:
results['equipment'] = ''
stringgen.next()
temp = stringgen.next()
while 'Additional Information:' not in temp:
results['equipment'] += temp
text += temp + '\n'
temp = stringgen.next()
else:
text += string + '\n'
if (results['start_datetime'] is not None) and (len(results['start_datetime']) == 16):
try:
results['start_datetime'] = datetime.datetime.strptime(results['start_datetime'], '%d/%m/%Y %H:%M')
except ValueError:
try:
results['start_datetime'] = datetime.datetime.strptime(results['start_datetime'], '%H:%M %d/%m/%Y')
except ValueError:
results['issues'] = True
results['start_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract start_datetime')
elif results['start_datetime'] is not None:
results['issues'] = True
results['start_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract start_datetime')
else:
results['issues'] = True
logger.error('Fallthrough condition in ChorusHTMLSkinny when parsing start_datetime')
if (results['end_datetime'] is not None) and (len(results['end_datetime']) == 16):
try:
results['end_datetime'] = datetime.datetime.strptime(results['end_datetime'], '%d/%m/%Y %H:%M')
except ValueError:
try:
results['end_datetime'] = datetime.datetime.strptime(results['end_datetime'], '%H:%M %d/%m/%Y')
except ValueError:
results['issues'] = True
results['end_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract end_datetime')
else:
results['end_datetime'] = None
logger.warning('ChorusHTMLSkinny could not extract end_datetime')
results['full_text'] = text
if results['issues'] == False:
logger.debug('ChorusHTMLSkinny parsed message without issue')
return results
#Actual script begins here.
passthrough_list = list()
#IMAP login
imap = imaplib.IMAP4(config['OUTAGES_POP_SERVER'])
imap.login(config['OUTAGES_POP_USER'], config['OUTAGES_POP_PASS'])
imap.select('INBOX')
ids = None
if config['DEBUG'] is True:
ids = imap.search(None, '(SUBJECT "")')[1][0].split()
else:
ids = imap.search(None, '(UNSEEN)')[1][0].split()
if len(ids) > 0:
logger.info('Collected ' + str(len(ids)) + 'messages from inbox')
#Fetch messages
for id in ids:
typ, msg_data = imap.fetch(id, '(RFC822)')
for response_part in msg_data:
if isinstance(response_part, tuple):
msg = email.message_from_string(response_part[1])
data = None
#Choose parser
if (msg.has_key('from')) and (msg.has_key('subject')):
if (config['CHORUS_PLAIN_TEXT_FROM_ADDRS_RE'].search(msg['from'])) and (config['CHORUS_PLAIN_TEXT_RE'].match(msg['subject'])):
logger.debug('Message ' + msg['subject'] + 'passed to ChorusPlainText parser')
data = ChorusPlainText(GetBody(msg))
elif (config['CHORUS_HTML_FAT_FROM_ADDRS_RE'].search(msg['from'])) and (config['CHORUS_HTML_FAT_RE'].match(msg['subject'])):
logger.debug('Message ' + msg['subject'] + 'passed to ChorusHTMLFat parser')
data = ChorusHTMLFat(GetBody(msg))
elif (config['CHORUS_HTML_SKINNY_FROM_ADDRS_RE'].search(msg['from'])) and (config['CHORUS_HTML_SKINNY_RE'].match(msg['subject'])):
logger.debug('Message ' + msg['subject'] + 'passed to ChorusHTMLSkinny parser')
data = ChorusHTMLSkinny(GetBody(msg))
else:
passthrough_list.append(msg)
logger.warning('Could not select parser for message ' + msg['subject'])
else:
logger.error('Unable to select parser due to missing headers')
if data is not None:
try:
DBInsert(data)
except (sqlite3.IntegrityError, UnicodeDecodeError) as e:
logger.warning('Unable to insert data into db from message ' + msg['subject'], exc_info = True)
passthrough_list.append(msg)
Passthrough(passthrough_list)
(vm_ip, client_ip_fingerprint, browser_fingerprint) = values
is_valid = validate_fingerprints(conf['WEB_DESKTOP_FP_SECRET_KEY'],
conf['WEB_DESKTOP_FP_SALT'],
client_ip_fingerprint,
browser_fingerprint,
client_ip,
user_agent,
accept,
accept_encoding,
accept_language)
return is_valid
if __name__ == '__main__':
conf = Config('..')
test_request_data = dict(
accept_encoding='Accept-Encoding: gzip, deflate, sdch',
accept_language='Accept-Language: en-US,en;q=0.8',
accept='Accept: application/json, text/javascript, */*; q=0.01',
user_agent='User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
vm_ip='128.196.64.214',
client_ip='127.0.0.1'
)
conf.from_pyfile('default_settings.py')
# conf.from_pyfile('local_settings.py')
signature = generate_signature_01(conf, test_request_data)
print signature
values, timestamp = decode_signature_02(conf, signature)
import logging
import os
from flask.ext.security import RoleMixin
from flask.ext.sqlalchemy import SQLAlchemy
from sqlalchemy import create_engine, Column, Integer, String, Table, DateTime, exists, ForeignKey, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, backref
from sqlalchemy.orm import scoped_session
from . import jsonalchemy
from flask import Config
cfg = Config(os.path.abspath("."))
cfg.from_pyfile("config.py")
db = SQLAlchemy()
logger = logging.getLogger("Gunrskite::Database")
# Check if we're inside the UDP listener, or not.
if os.environ.get("INSIDE_LISTENER", "n") == "y":
print("Using declarative_base() for model base.")
Base = declarative_base()
else:
# Switch to using Flask-SQLAlchemy's model.
print("Using db.Model for model base.")
Base = db.Model
engine = create_engine(cfg["SQLALCHEMY_DATABASE_URI"], pool_recycle=3600)
# Function for creating the engine and session.
def create_sess():
* DATABASE_URI
* DATABASE
Currently the supported environment variables:
* OpenShift
* DATABASE_URI
"""
import os
from flask import Config
from sqlalchemy.engine.url import URL
config = Config(".")
config.from_object("compair.settings")
config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../config.py"), silent=True)
if os.environ.get("OPENSHIFT_MYSQL_DB_HOST"):
config["SQLALCHEMY_DATABASE_URI"] = URL(
"mysql+pymysql",
host=os.getenv("OPENSHIFT_MYSQL_DB_HOST", "localhost"),
port=os.getenv("OPENSHIFT_MYSQL_DB_PORT", "3306"),
username=os.getenv("OPENSHIFT_MYSQL_DB_USERNAME", "compair"),
password=os.getenv("OPENSHIFT_MYSQL_DB_PASSWORD", "compair"),
database=os.getenv("OPENSHIFT_GEAR_NAME", "compair"),
)
elif (
os.environ.get("DB_HOST")
or os.environ.get("DB_PORT")
console_handler.setLevel(logging.DEBUG)
console_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s [in %(module)s @ %(pathname)s:%(lineno)d]"))
logger = logging.Logger("CONTROL", level=logging.DEBUG)
logger.addHandler(file_handler)
logger.addHandler(console_handler)
# Log startup
logger.info('---------- Backend starting up!')
try:
# Load configuration
logger.info('Loading configuration')
config_file = os.path.join(instance_path, 'app.cfg')
default_config_file = os.path.join(resource_path, 'default_app.cfg')
config = Config(instance_path)
config.from_pyfile(default_config_file)
config.from_pyfile(config_file)
#config = SimpleConfigParser(config_file, default_config_file)
# Set up iftt events if a maker key is present
if config['IFTTT_MAKER_KEY']:
logger.info('Creating IFTTT events')
changed_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_changed', logger)
opened_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_opened', logger)
closed_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_closed', logger)
warning_event = IftttEvent(config['IFTTT_MAKER_KEY'], 'garage_door_warning', logger)
else:
logger.info('No IFTTT maker key provided. No events will be raised.')
changed_event = None # type: IftttEvent
opened_event = None # type: IftttEvent
