def main(ctx, verbosity):
configure_logger(verbosity)
# Load the configurations from file
config = Config(root_path=".")
config.from_object(default_settings)
config.from_envvar(ZUBBI_SETTINGS_ENV)
# Validate the configuration
tenant_sources_repo = config.get("TENANT_SOURCES_REPO")
tenant_sources_file = config.get("TENANT_SOURCES_FILE")
# Fail if both are set or none of both is set
if (
not tenant_sources_file
and not tenant_sources_repo
or (tenant_sources_file and tenant_sources_repo)
):
raise ScraperConfigurationError(
"Either one of 'TENANT_SOURCES_REPO' "
"and 'TENANT_SOURCES_FILE' must be set, "
"but not both."
)
# Store the config in click's context object to be available for subcommands
ctx.obj = {"config": config}
if ctx.invoked_subcommand is None:
ctx.invoke(scrape)
def app_config(postgres_user_conf):
from flask.config import Config
from datacat.settings import testing
conf = Config('')
conf.from_object(testing)
conf['DATABASE'] = postgres_user_conf
return conf
def _get_config():
# Workaround to get an available config object before the app is initiallized
# Only needed/used in top-level and class statements
# https://stackoverflow.com/a/18138250/7597273
root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
config = Config(root_path)
config.from_object('config')
return config
def app_config(postgres_user_conf):
from flask.config import Config
from datacat.settings import testing
conf = Config('')
conf.from_object(testing)
conf['DATABASE'] = postgres_user_conf
return conf
def load_config():
cfg = Config(dirname(dirname(__file__)))
cfg.from_object("autobit.settings")
if "AUTOBIT_SETTINGS" in os.environ:
cfg.from_envvar("AUTOBIT_SETTINGS")
if not exists(cfg['WATCH_DIR']):
logger.info("Creating watch dir: {}".format(cfg['WATCH_DIR']))
os.makedirs(cfg['WATCH_DIR'])
return cfg
class WorkProc:
"""A container process for our worker threads that can receive
notifications from a Unix domain socket.
"""
def __init__(self, basedir, db=None):
"""Create a container using a given base directory for the
storage and socket. Optionally, provide a database object to use
that instead of creating a new one (to, for example, reuse its
internal locks).
"""
self.basedir = os.path.abspath(basedir)
# Load the configuration. We're just reusing Flask's simple
# configuration component here.
self.config = Config(self.basedir)
self.config.from_object('buildbot.config_default')
self.config.from_pyfile('buildbot.cfg', silent=True)
# Create the database.
self.db = db or JobDB(self.basedir)
def start(self):
"""Create and start the worker threads.
"""
threads = worker.work_threads(self.db, self.config)
for thread in threads:
if not thread.is_alive():
thread.start()
async def handle(self, client, addr):
"""Handle an incoming socket connection.
"""
async for line in client.makefile('rb'):
# Each line is a job name.
job_name = line.decode('utf8').strip()
print(job_name)
# Just notify the database that something changed.
with self.db.cv:
self.db.cv.notify_all()
def serve(self):
"""Start listening on a Unix domain socket for incoming
messages. Run indefinitely (until the server is interrupted).
"""
sockpath = os.path.join(self.basedir, SOCKNAME)
if os.path.exists(sockpath):
os.unlink(sockpath)
try:
curio.run(curio.unix_server, sockpath, self.handle)
except KeyboardInterrupt:
pass
finally:
os.unlink(sockpath)
def make_config(app=None):
if app is not None:
cfg = app.config
else:
from flask.config import Config
root_path = os.path.dirname(__file__).rsplit('/', 1)[0]
cfg = Config(root_path)
# customize config here
cfg.from_object(default_config)
cfg.from_pyfile('myapp.cfg', silent=True)
cfg.from_envvar('MYAPP_CONFIG', silent=True)
cfg['BABEL_DEFAULT_LOCALE'] = cfg['LANG']
return cfg
def load_config(config_obj=None):
"""
Load Greenwave configuration. It will load the configuration based on how the environment is
configured.
:return: A dict of Greenwave configuration.
"""
# Load default config, then override that with a config file
config = Config(__name__)
if config_obj is None:
if os.getenv('TEST') == 'true':
config_obj = 'greenwave.config.TestingConfig'
elif os.getenv('DEV') == 'true' or os.getenv('DOCS') == 'true':
config_obj = 'greenwave.config.DevelopmentConfig'
else:
config_obj = 'greenwave.config.ProductionConfig'
if os.getenv('TEST') == 'true':
default_config_file = os.path.join(os.getcwd(), 'conf',
'settings.py.example')
elif os.getenv('DEV') == 'true':
default_config_file = os.path.join(os.getcwd(), 'conf', 'settings.py')
elif os.getenv('DOCS') == 'true':
default_config_file = os.path.normpath(
os.path.join(os.getcwd(), '..', 'conf', 'settings.py.example'))
else:
default_config_file = '/etc/greenwave/settings.py'
# 1. Load default configuration.
log.debug("config: Loading config from %r", config_obj)
config.from_object(config_obj)
# 2. Override default configuration with environment variables.
if os.environ.get('GREENWAVE_SUBJECT_TYPES_DIR'):
config['SUBJECT_TYPES_DIR'] = os.environ['GREENWAVE_SUBJECT_TYPES_DIR']
if os.environ.get('GREENWAVE_POLICIES_DIR'):
config['POLICIES_DIR'] = os.environ['GREENWAVE_POLICIES_DIR']
# 3. Override default configuration and environment variables with custom config file.
config_file = os.environ.get('GREENWAVE_CONFIG', default_config_file)
log.debug("config: Extending config with %r", config_file)
config.from_pyfile(config_file)
if os.environ.get('SECRET_KEY'):
config['SECRET_KEY'] = os.environ['SECRET_KEY']
return config
class CartesiusSuite :
#: Default configuration parameters.
default_config = ImmutableDict({
'DEBUG': False,
'SERVER_NAME': None,
'APPLICATION_ROOT': None
})
def __init__(self, configuration = None) :
self.make_config(configuration)
self.make_servers()
def make_config(self, configuration):
self.config = Config(None, self.default_config)
if (configuration) :
self.config.from_object(configuration)
def make_servers(self) :
self.servers = {}
for server in self.config['SERVERS'] :
server_path = self.config['SERVER_PATH'] + "." + server['path'] + "."
if( 'app_name' in server ) :
server_path += server['app_name']
else :
server_path += 'app'
self.servers[server['name']] = CartesiusServer(server_path)
def start(self, server) :
self.servers[server].start()
def start_all(self) :
for server in self.servers :
self.servers[server].start()
def stop(self, server) :
self.servers[server].stop()
def stop_all(self) :
for server in self.servers :
server.stop()
from __future__ import unicode_literals
import os
from flask import Flask
from flask.config import Config
from flaskext.csrf import csrf
# config
config = Config(None, Flask.default_config)
config.from_object('rentmybike.settings.default')
if os.getenv('RENTMYBIKE_ENV'):
config.from_object('rentmybike.settings.' + os.getenv('RENTMYBIKE_ENV'))
# app
from application import RentMyBike # deferred
app = RentMyBike()
if app.config['DUMMY_DATA']:
app.add_dummy_data()
csrf(app)
# controllers
import controllers # deferred
import datetime
from flask_login import UserMixin
from flask.config import Config
#from werkzeug.security import check_password_hash, generate_password_hash
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
import os
from pytwis import Pytwis
from .config import config_by_name
# BUGBUG: Read the configuration of the Flask app again since we can't
# find a way to access the configuration outside an application context.
config_name = os.getenv('PYTWASK_ENV', 'dev')
app_config = Config(None)
app_config.from_object(config_by_name[config_name])
# Connect to the local Redis database.
twis = Pytwis(hostname=app_config['REDIS_DB_HOSTNAME'],
port=app_config['REDIS_DB_PORT'],
db=app_config['REDIS_DB_INDEX'],
password=app_config['REDIS_DB_PASSWORD'])
class Tweet():
"""Tweet class"""
def __init__(self, username, post_unix_time, body):
self.username = username
self.post_datetime = datetime.datetime.fromtimestamp(post_unix_time)
self.body = body
import boto3
from boto.s3.connection import S3Connection
from depot.fields.sqlalchemy import UploadedFileField
import inflection
from flask.config import Config
from flask_migrate import Migrate
from sqlalchemy import Column, Integer, String
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy_wrapper import SQLAlchemy
from sa_types import EnumChoiceType
config_name = 'coding_challenge_restful.settings.Config'
config = Config('')
config.from_object(config_name)
s3_client = boto3.client(
's3',
# Hard coded strings as credentials, not recommended.
aws_access_key_id=config['AWS_ACCESS_KEY'],
aws_secret_access_key=config['AWS_SECRET_KEY'])
isolation_level = 'READ COMMITTED'
db = SQLAlchemy(uri=config['DATABASE_URL'], isolation_level=isolation_level)
migrate = Migrate(compare_type=True)
# Create Models
Model = db.Model
# Models are defined here
class WorkProc:
"""A container process for our worker threads that can receive
notifications from a Unix domain socket.
"""
def __init__(self, basedir, db=None):
"""Create a container using a given base directory for the
storage and socket. Optionally, provide a database object to use
that instead of creating a new one (to, for example, reuse its
internal locks).
"""
self.basedir = os.path.abspath(basedir)
# Load the configuration. We're just reusing Flask's simple
# configuration component here.
self.config = Config(self.basedir)
self.config.from_object('polyphemus.config_default')
self.config.from_pyfile('polyphemus.cfg', silent=True)
# Create the database.
self.db = db or JobDB(self.basedir)
def start(self, stages_conf=None):
"""Create and start the worker threads. If stages_conf is None, create the
default workers for the given toolchain. If stages_confg is a list of
strings in worker.KNOWN_STAGES then create workers mapping to those.
"""
if stages_conf is None:
stages = worker.default_work_stages(self.config)
else:
stages = [worker.KNOWN_STAGES[stage] for stage in stages_conf]
print(stages)
for thread in worker.work_threads(stages, self.config, self.db):
if not thread.is_alive():
thread.start()
async def handle(self, client, addr):
"""Handle an incoming socket connection.
"""
async for line in client.makefile('rb'):
# Each line is a job name.
job_name = line.decode('utf8').strip()
print(job_name)
# Just notify the database that something changed.
with self.db.cv:
self.db.cv.notify_all()
def serve(self):
"""Start listening on a Unix domain socket for incoming
messages. Run indefinitely (until the server is interrupted).
"""
sockpath = os.path.join(self.basedir, SOCKNAME)
if os.path.exists(sockpath):
os.unlink(sockpath)
try:
curio.run(curio.unix_server, sockpath, self.handle)
except KeyboardInterrupt:
print ("Shutting down worker.")
finally:
os.unlink(sockpath)
def poll(self):
"""Continously poll the work directory for open jobs.
"""
try:
while True:
with self.db.cv:
self.db.cv.notify_all()
time.sleep(2)
except KeyboardInterrupt:
print ("Shutting down worker.")
def make_config():
cfg = Config('')
cfg.from_object('datacat.settings.default')
cfg.from_envvar('DATACAT_SETTINGS', silent=True)
return cfg
def make_config():
cfg = Config('')
cfg.from_object('datacat.settings.default')
cfg.from_envvar('DATACAT_SETTINGS', silent=True)
return cfg
from __future__ import unicode_literals
import os
from flask import Flask
from flask.config import Config
from flaskext.csrf import csrf
# config
config = Config(None, Flask.default_config)
config.from_object('rentmybike.settings.default')
if os.getenv('RENTMYBIKE_ENV'):
config.from_object('rentmybike.settings.' + os.getenv('RENTMYBIKE_ENV'))
else:
config.from_object('rentmybike.settings.custom')
# app
from application import RentMyBike # deferred
app = RentMyBike()
if app.config['DUMMY_DATA']:
app.add_dummy_data()
csrf(app)
# controllers
import controllers # deferred
from flask.config import Config
from celery.utils.log import get_task_logger
from ckanpackager.tasks.url_package_task import UrlPackageTask
from ckanpackager.tasks.datastore_package_task import DatastorePackageTask
from ckanpackager.tasks.dwc_archive_package_task import DwcArchivePackageTask
config = Config(__file__)
config.from_object('ckanpackager.config_defaults')
config.from_envvar('CKANPACKAGER_CONFIG')
from celery import Celery
app = Celery('tasks', broker=config['CELERY_BROKER'])
app.conf.CELERY_DISABLE_RATE_LIMITS = True
app.conf.CELERY_ACCEPT_CONTENT = ['json']
app.conf.CELERY_TASK_SERIALIZER = 'json'
app.conf.CELERY_CREATE_MISSING_QUEUES = True
app.conf.CELERY_DEFAULT_QUEUE = 'slow'
@app.task
def run_task(task, request):
""" Run/enqueue the given task for the given request
Note that the request should be validated before
this is called.
@param task: Name of the task. One of package_url,
package_dwc_archive or package_datastore
@param request: Dictionary containing the request
"""
from flask.config import Config
import os
ds_settings = os.getenv(
"DS_SETTINGS", "project.config.data_science_config.DsDevelopmentConfig")
ds_config = Config(None)
ds_config.from_object(ds_settings)
class OdataServer(object) :
#: Default configuration parameters.
default_config = ImmutableDict({
'DEBUG': False,
'SERVER_NAME': 'ODATA SERVER',
'APPLICATION_ROOT': None,
'READ_ONLY' : True,
'SERVICE_ROOT' : 'http://localhost',
'SERVICE_PORT' : '8080',
'CORS' : True,
'METADATA' : 'metadata.xml'
})
functions = None
def __init__(self, import_name, configuration = None):
self.make_config(configuration)
conf = self.config
service_url = self.config['SERVICE_ROOT'] + ':' + self.config['SERVICE_PORT']
if self.config['READ_ONLY'] :
self.server = ReadOnlyServer(service_url)
else:
self.server = Server(serviceRoot=service_url)
self.load_metadata()
self.make_containers()
def __call__(self, environ, start_response):
if self.config['CORS'] :
return CORS(self.server).__call__(environ, start_response)
else :
return self.server.__call__(environ, start_response)
def make_config(self, configuration):
self.config = Config(None, self.default_config)
if (configuration) :
self.config.from_object(configuration)
def load_metadata(self) :
"""Loads the metadata file from the current directory."""
doc = edmx.Document()
with open(self.config['SERVER_ROOT'] + '/' + self.config['SERVER_NAME'] + '/' + self.config['METADATA'], 'rb') as f:
doc.Read(f)
if doc :
self.metadata = doc
self.server.SetModel(doc)
else:
pass
#raise error
def add_function_import(self, rule, endpoint=None, view_func=None, **options):
# This seems terribly hacky but I can't think of another way...
funcImport = self.server.model.DataServices[rule]
if funcImport :
funcImport.bind(view_func, **options)
def function_import(self, rule, **options):
def decorator(f):
endpoint = options.pop('endpoint', None)
self.add_function_import(rule, endpoint, f, **options)
return f
return decorator
def make_containers(self) :
for schema in self.metadata.root.DataServices.Schema :
self.make_container(schema)
def make_container(self, schema) :
for entity_container in schema.EntityContainer :
entity_name = schema.Name + '.' + entity_container.Name
if (self.config["DATASOURCES"][entity_name]) :
ctype = self.config["DATASOURCES"][entity_name]['dbapi']
if (ctype == 'psycopg2') :
container = PgSQLEntityContainer(
pgsql_options = self.config["DATASOURCES"][entity_name]['config'],
container=self.metadata.root.DataServices[entity_name]
)
def _get_config():
root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
cfg = Config(root_path)
cfg.from_object('config')
return cfg
},
'short': {
'format':
'[BIS %(asctime)s%(msecs)d-%(levelname).1s][%(name)-9s]%(message)s',
'datefmt':
'%M%S' # miliseconds will be added by the format string
},
'long': {
'format': '[%(asctime)s:%(levelname)s][%(name)-10s]%(message)s'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'simple',
'stream': 'ext://sys.stderr'
}
},
'loggers': {
'harpgest': {
'handlers': ['console'],
'level': logging.INFO,
}
}
}
config = Config(os.getcwd())
config.from_object(DefaultConfig)
config.from_pyfile(os.path.join(os.getcwd(), 'app.cfg'), silent=True)
