def cli(ctx):
'''
Access and modify data from a PISS server on the command line.
'''
current_dir = os.path.dirname(os.path.realpath(__file__))
config_file = os.path.splitext(os.path.basename(__file__))[0] + ".cfg"
ctx.obj = {}
ctx.obj['current_dir'] = current_dir
ctx.obj['config_file'] = config_file
if ctx.args[0] != 'register':
if not os.path.isfile(os.path.join(current_dir, config_file)):
click.echo('No configuration file found! Use `register <url>` to\
create one!')
ctx.abort()
try:
app_config = Config(current_dir)
app_config.from_pyfile(config_file)
except Exception as e:
click.echo('Could not load config from file. Use `register <url>`\
to create a new one!')
ctx.abort()
try:
ctx.obj['credentials'] = app_config['CREDENTIALS']
ctx.obj['meta_url'] = app_config['META_URL']
meta_post = app_config['META_POST']
ctx.obj['meta_post'] = meta_post
ctx.obj['url'] = meta_post['server']['urls']['posts_feed']
except Exception as e:
click.echo('Parameters missing from configuration file! Use\
`register <url>` to fix!')
ctx.abort()
def __init__(self, settings=None):
self.settings = settings
log_cfg = Config('')
log_cfg.from_pyfile( settings['LOGGING_CONFIG'])
self.log = logging_factory.get_logger(settings['APP_NAME'], settings['LOGGING_PATH'], log_cfg['LOGGING'])
self.dbattr_dict = {'ORA':'ora_db','PG':'pg_db', 'DB':'db'}
self.name_dict = {'ORA':'oracle','PG':'postgresql', 'DB':'oracle'}
#import dbsession after logger initialized
from mabolab.database import dbfactory
for db_type in settings['DB_TYPE']:
if not hasattr(self, self.dbattr_dict[db_type]):
#self.log.debug("set %s" %(db_type) )
setattr(self, self.dbattr_dict[db_type], dbfactory.get_db(self.name_dict[db_type], settings) )
if not hasattr(self, 'db'):
setattr(self, 'db', dbfactory.get_db(self.name_dict['DB'], settings) )
def app_config(postgres_user_conf):
from flask.config import Config
from datacat.settings import testing
conf = Config('')
conf.from_object(testing)
conf['DATABASE'] = postgres_user_conf
return conf
def _get_config():
# Workaround to get an available config object before the app is initiallized
# Only needed/used in top-level and class statements
# https://stackoverflow.com/a/18138250/7597273
root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
config = Config(root_path)
config.from_object('config')
return config
def app_config(postgres_user_conf):
from flask.config import Config
from datacat.settings import testing
conf = Config('')
conf.from_object(testing)
conf['DATABASE'] = postgres_user_conf
return conf
def __init__(self, _settings, _app, *args, **kwargs):
"""perform the initial load"""
super().__init__(*args, **kwargs)
# Bring Dynaconf instance value to Flask Config
Config.update(self, _settings.store)
self._settings = _settings
self._app = _app
def setUp(self):
tmp = '../config/logging_config.py'
log_cfg = Config('')
log_cfg.from_pyfile(tmp)
self.logging_cfg = log_cfg['LOGGING']
def load_config():
cfg = Config(dirname(dirname(__file__)))
cfg.from_object("autobit.settings")
if "AUTOBIT_SETTINGS" in os.environ:
cfg.from_envvar("AUTOBIT_SETTINGS")
if not exists(cfg['WATCH_DIR']):
logger.info("Creating watch dir: {}".format(cfg['WATCH_DIR']))
os.makedirs(cfg['WATCH_DIR'])
return cfg
class WorkProc:
"""A container process for our worker threads that can receive
notifications from a Unix domain socket.
"""
def __init__(self, basedir, db=None):
"""Create a container using a given base directory for the
storage and socket. Optionally, provide a database object to use
that instead of creating a new one (to, for example, reuse its
internal locks).
"""
self.basedir = os.path.abspath(basedir)
# Load the configuration. We're just reusing Flask's simple
# configuration component here.
self.config = Config(self.basedir)
self.config.from_object('buildbot.config_default')
self.config.from_pyfile('buildbot.cfg', silent=True)
# Create the database.
self.db = db or JobDB(self.basedir)
def start(self):
"""Create and start the worker threads.
"""
threads = worker.work_threads(self.db, self.config)
for thread in threads:
if not thread.is_alive():
thread.start()
async def handle(self, client, addr):
"""Handle an incoming socket connection.
"""
async for line in client.makefile('rb'):
# Each line is a job name.
job_name = line.decode('utf8').strip()
print(job_name)
# Just notify the database that something changed.
with self.db.cv:
self.db.cv.notify_all()
def serve(self):
"""Start listening on a Unix domain socket for incoming
messages. Run indefinitely (until the server is interrupted).
"""
sockpath = os.path.join(self.basedir, SOCKNAME)
if os.path.exists(sockpath):
os.unlink(sockpath)
try:
curio.run(curio.unix_server, sockpath, self.handle)
except KeyboardInterrupt:
pass
finally:
os.unlink(sockpath)
def load(config_filename='settings.py'):
"""Create a Flask config that will be used to update the application
config when it is created."""
config = Config("pjuu")
# Load the default settings
config.from_pyfile(config_filename)
# Load the setting from the Environment variable
config.from_envvar('PJUU_SETTINGS', silent=True)
return config
def download_users():
"""
Download users.xml file.
"""
import urllib2
from flask.config import Config
config = Config(etc())
config.from_pyfile("deploy.cfg")
response = urllib2.urlopen(config['USERS_URL'])
users_xml = os.path.join('runtime', 'data', 'users.xml')
if response.code == 200:
with open(users_xml, 'w') as f:
f.write(response.read())
def read_config() -> Config:
config = Config("")
env_settings_file = os.environ.get('DEBUGPROXY_SETTING_FILE',
'./config/default_settings.py')
config.from_pyfile(env_settings_file)
# update the config with any values found in the environment
for key in config.keys():
env_value = os.environ.get(key)
if env_value:
config[key] = env_value
return config
def configs(self):
if not hasattr(self, '_configs'):
configs = Config(ROOT)
resoure = ResourceLoader.get().get_resoure('settings.py')
config_files = resoure.as_list()
if config_files:
for path in config_files:
configs.from_pyfile(path)
else:
raise Exception('need a configuration file to start app')
self._configs = configs
return self._configs
def download_users():
"""
Download users.xml file.
"""
import urllib2
from flask.config import Config
config = Config(etc())
config.from_pyfile("deploy.cfg")
response = urllib2.urlopen(config['USERS_URL'])
users_xml = os.path.join('runtime', 'data', 'users.xml')
if response.code == 200:
with open(users_xml, 'w') as f:
f.write(response.read())
def configs(self):
if not hasattr(self, '_configs'):
configs = Config(ROOT)
resoure = ResourceLoader.get().get_resoure('settings.py')
config_files = resoure.as_list()
if config_files:
for path in config_files:
configs.from_pyfile(path)
else:
raise Exception('need a configuration file to start app')
self._configs = configs
return self._configs
def load_config(settings_file='./test_settings.py'):
"""
Loads the config files merging the defaults
with the file defined in environ.PULLSBURY_SETTINGS if it exists.
"""
config = Config(os.getcwd())
if 'PULLSBURY_SETTINGS' in os.environ:
config.from_envvar('PULLSBURY_SETTINGS')
else:
config.from_pyfile(settings_file)
if config.get('LOGGING_CONFIG'):
logging.config.fileConfig(
config.get('LOGGING_CONFIG'),
disable_existing_loggers=False)
json_values = [
'TEAMS',
'HAPPY_SLACK_EMOJIS',
'REPO_BLACKLIST',
'SLACK_CUSTOM_EMOJI_MAPPING'
]
for value in json_values:
config.update({
value: json.loads(config.get(value, '{}'))
})
return config
def __init__(self, basedir, db=None):
"""Create a container using a given base directory for the
storage and socket. Optionally, provide a database object to use
that instead of creating a new one (to, for example, reuse its
internal locks).
"""
self.basedir = os.path.abspath(basedir)
# Load the configuration. We're just reusing Flask's simple
# configuration component here.
self.config = Config(self.basedir)
self.config.from_object('buildbot.config_default')
self.config.from_pyfile('buildbot.cfg', silent=True)
# Create the database.
self.db = db or JobDB(self.basedir)
def load_config():
"""
Loads the config files merging the defaults
with the file defined in environ.LINTREVIEW_SETTINGS if it exists.
"""
config = Config(os.getcwd())
if 'LINTREVIEW_SETTINGS' in os.environ:
config.from_envvar('LINTREVIEW_SETTINGS')
elif os.path.exists(os.path.join(os.getcwd(), 'settings.py')):
config.from_pyfile('settings.py')
else:
msg = ("Unable to load configuration file. Please "
"either create ./settings.py or set LINTREVIEW_SETTINGS "
"in your environment before running.")
raise ImportError(msg)
if config.get('LOGGING_CONFIG'):
logging.config.fileConfig(
config.get('LOGGING_CONFIG'),
disable_existing_loggers=False)
if config.get('SSL_CA_BUNDLE'):
os.environ['REQUESTS_CA_BUNDLE'] = config.get('SSL_CA_BUNDLE')
return config
def main(ctx, verbosity):
configure_logger(verbosity)
# Load the configurations from file
config = Config(root_path=".")
config.from_object(default_settings)
config.from_envvar(ZUBBI_SETTINGS_ENV)
# Validate the configuration
tenant_sources_repo = config.get("TENANT_SOURCES_REPO")
tenant_sources_file = config.get("TENANT_SOURCES_FILE")
# Fail if both are set or none of both is set
if (
not tenant_sources_file
and not tenant_sources_repo
or (tenant_sources_file and tenant_sources_repo)
):
raise ScraperConfigurationError(
"Either one of 'TENANT_SOURCES_REPO' "
"and 'TENANT_SOURCES_FILE' must be set, "
"but not both."
)
# Store the config in click's context object to be available for subcommands
ctx.obj = {"config": config}
if ctx.invoked_subcommand is None:
ctx.invoke(scrape)
def __init__(self, import_name, middlewares=None, loop=None, session=None):
self.import_name = import_name
self.root_path = get_root_path(import_name)
self.config = Config(self.root_path, default_config)
self._context = {}
self._loop = loop or asyncio.get_event_loop()
self._middlewares = SpiderMiddlewareManager(self, middlewares)
self._session = session or aiohttp.ClientSession(loop=self._loop)
def __init__(self, debug=False):
self.valid = True
# loggerを初期化する
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s')
logger = logging.getLogger(self.service_name)
logger.setLevel(logging.DEBUG)
self.logger = logger
# 設定ファイルからAPIkeyとかを読み込む
current_dir = os.path.dirname(__file__)
config = Config('../../')
if os.path.exists(activate_this):
config.from_pyfile("wsgi_dev.cfg")
else:
config.from_pyfile("dev.cfg")
#self.logger.debug(config)
self.validate(self.require_config, config['EXTERNAL_CONFIG'])
self.config = config['EXTERNAL_CONFIG']
self.debug = config['POST_DEBUG']
def __init__(self):
self.config = Config(root_path="/")
self.config.from_envvar('APP_CONFIG')
self.redis = searcher.redis_connect(self.config)
db.init_db(self.config['SQLALCHEMY_DATABASE_URI'])
self.searcher = searcher.WhooshSearcher(
index_dir=self.config['WHOOSH_INDEX_PATH'])
self.running = False
def test_settings_abs_path(self):
"""Check if the config obj is updated with default_settings when it is
passed as a python file absolute path
"""
abs_path = getcwd() + '/arachne/tests/test_settings.py'
test_app = self.create_app(settings=abs_path)
# load config from pyfile
flask_app = Flask(__name__)
flask_app.config.from_object('arachne.default_settings')
config_cls = Config(__name__)
config_cls.from_pyfile(abs_path)
# update config with the arachne default settings
flask_app.config.update(config_cls)
# test if config dicts are same
self.assertEquals(test_app.config, flask_app.config)
class Settings(object):
config = Config(__name__)
DEBUG = ConfigAttribute('DEBUG')
APP_NAME = ConfigAttribute('APP_NAME')
DJAODJIN_SECRET_KEY = ConfigAttribute('DJAODJIN_SECRET_KEY')
def update(self, **updates):
return self.config.update(updates)
def test_settings_abs_path(self):
"""Check if the config obj is updated with default_settings when it is
passed as a python file absolute path
"""
abs_path = getcwd() + '/arachne/tests/test_settings.py'
test_app = self.create_app(settings=abs_path)
# load config from pyfile
flask_app = Flask(__name__)
flask_app.config.from_object('arachne.default_settings')
config_cls = Config(__name__)
config_cls.from_pyfile(abs_path)
# update config with the arachne default settings
flask_app.config.update(config_cls)
# test if config dicts are same
self.assertEquals(test_app.config, flask_app.config)
class IndexerDaemon(object):
def __init__(self):
self.config = Config(root_path="/")
self.config.from_envvar('APP_CONFIG')
self.redis = searcher.redis_connect(self.config)
db.init_db(self.config['SQLALCHEMY_DATABASE_URI'])
self.searcher = searcher.WhooshSearcher(
index_dir=self.config['WHOOSH_INDEX_PATH'])
self.running = False
def _get_writer(self):
return self.searcher.ix.writer()
def main_loop(self):
self.running = True
while self.running:
self.work_on_queue()
def work_on_queue(self):
"""Queue operations (redis reliable queue pattern)
Atomically pop an item from the work queue and put it on the
processing queue; after processing the item is removed from
the processing queue.
The "processing" step fetch the specified bookmark from the SQL
database and run the specified operation (index, update, delete);
after completing the operation it updates the bookmark `indexed_on`
field.
"""
conn = self.redis
msg = conn.brpoplpush(searcher.QUEUE_INDEX, searcher.QUEUE_WORK)
log.debug("Processing new operation: {}".format(msg))
try:
payload = json.loads(msg)
op, _id = payload
except ValueError, ex:
log.error("Invalid job in queue: {0}: {1}".format(msg, str(ex)))
else:
class CartesiusSuite :
#: Default configuration parameters.
default_config = ImmutableDict({
'DEBUG': False,
'SERVER_NAME': None,
'APPLICATION_ROOT': None
})
def __init__(self, configuration = None) :
self.make_config(configuration)
self.make_servers()
def make_config(self, configuration):
self.config = Config(None, self.default_config)
if (configuration) :
self.config.from_object(configuration)
def make_servers(self) :
self.servers = {}
for server in self.config['SERVERS'] :
server_path = self.config['SERVER_PATH'] + "." + server['path'] + "."
if( 'app_name' in server ) :
server_path += server['app_name']
else :
server_path += 'app'
self.servers[server['name']] = CartesiusServer(server_path)
def start(self, server) :
self.servers[server].start()
def start_all(self) :
for server in self.servers :
self.servers[server].start()
def stop(self, server) :
self.servers[server].stop()
def stop_all(self) :
for server in self.servers :
server.stop()
def test_settings_abs_path(self):
"""Check if the config obj is updated with default_settings when it is
passed as a python file absolute path
"""
abs_path = getcwd() + '/arachne/tests/test_settings.py'
test_app = self.create_app(settings=abs_path)
# since the object initialized created is always different
# we ignore CRAWLER_PROCESS setting for test
if SCRAPY_VERSION >= (1, 0, 0):
del test_app.config['CRAWLER_PROCESS']
# load config from pyfile
flask_app = Flask(__name__)
flask_app.config.from_object('arachne.default_settings')
config_cls = Config(__name__)
config_cls.from_pyfile(abs_path)
# update config with the arachne default settings
flask_app.config.update(config_cls)
# test if config dicts are same
self.assertEquals(test_app.config, flask_app.config)
def test_settings_abs_path(self):
"""Check if the config obj is updated with default_settings when it is
passed as a python file absolute path
"""
abs_path = getcwd() + '/arachneserver/tests/test_settings.py'
test_app = self.create_app(settings=abs_path)
# since the object initialized created is always different
# we ignore CRAWLER_PROCESS setting for test
# if SCRAPY_VERSION >= (1, 0, 0):
del test_app.config['CRAWLER_PROCESS']
# load config from pyfile
flask_app = Flask(__name__)
flask_app.config.from_object('arachneserver.default_settings')
config_cls = Config(__name__)
config_cls.from_pyfile(abs_path)
# update config with the server default settings
flask_app.config.update(config_cls)
# test if config dicts are same
self.assertEquals(test_app.config, flask_app.config)
def make_config(app=None):
if app is not None:
cfg = app.config
else:
from flask.config import Config
root_path = os.path.dirname(__file__).rsplit('/', 1)[0]
cfg = Config(root_path)
# customize config here
cfg.from_object(default_config)
cfg.from_pyfile('myapp.cfg', silent=True)
cfg.from_envvar('MYAPP_CONFIG', silent=True)
cfg['BABEL_DEFAULT_LOCALE'] = cfg['LANG']
return cfg
def load_config(config_obj=None):
"""
Load Greenwave configuration. It will load the configuration based on how the environment is
configured.
:return: A dict of Greenwave configuration.
"""
# Load default config, then override that with a config file
config = Config(__name__)
if config_obj is None:
if os.getenv('TEST') == 'true':
config_obj = 'greenwave.config.TestingConfig'
elif os.getenv('DEV') == 'true' or os.getenv('DOCS') == 'true':
config_obj = 'greenwave.config.DevelopmentConfig'
else:
config_obj = 'greenwave.config.ProductionConfig'
if os.getenv('TEST') == 'true':
default_config_file = os.path.join(os.getcwd(), 'conf',
'settings.py.example')
elif os.getenv('DEV') == 'true':
default_config_file = os.path.join(os.getcwd(), 'conf', 'settings.py')
elif os.getenv('DOCS') == 'true':
default_config_file = os.path.normpath(
os.path.join(os.getcwd(), '..', 'conf', 'settings.py.example'))
else:
default_config_file = '/etc/greenwave/settings.py'
# 1. Load default configuration.
log.debug("config: Loading config from %r", config_obj)
config.from_object(config_obj)
# 2. Override default configuration with environment variables.
if os.environ.get('GREENWAVE_SUBJECT_TYPES_DIR'):
config['SUBJECT_TYPES_DIR'] = os.environ['GREENWAVE_SUBJECT_TYPES_DIR']
if os.environ.get('GREENWAVE_POLICIES_DIR'):
config['POLICIES_DIR'] = os.environ['GREENWAVE_POLICIES_DIR']
# 3. Override default configuration and environment variables with custom config file.
config_file = os.environ.get('GREENWAVE_CONFIG', default_config_file)
log.debug("config: Extending config with %r", config_file)
config.from_pyfile(config_file)
if os.environ.get('SECRET_KEY'):
config['SECRET_KEY'] = os.environ['SECRET_KEY']
return config
def __init__(self, _config, *args, **kwargs):
super(_Config, self).__init__(*args, **kwargs)
Config.update(self, _config)
def __init__(self, config=None):
self.config = Config()
if config is not None:
self.config.update(config)
class DatacatCore(object):
def __init__(self, config=None):
self.config = Config()
if config is not None:
self.config.update(config)
@property
def db(self):
if getattr(self, '_db', None) is None:
self._db = connect(**self.config['DATABASE'])
self._db.autocommit = False
return self._db
@property
def admin_db(self):
if getattr(self, '_admin_db', None) is None:
self._admin_db = connect(**self.config['DATABASE'])
self._admin_db.autocommit = True
return self._admin_db
def create_tables(self):
create_tables(self.admin_db)
def drop_tables(self):
drop_tables(self.admin_db)
# ------------------------------------------------------------
# Resource data CRUD
# ------------------------------------------------------------
def resource_data_iter(self):
"""Iterate over resource attributes"""
with self.db, self.db.cursor() as cur:
cur.execute("SELECT * FROM resource_data")
for row in cur.fetchall():
yield row
def resource_data_create(self, stream, metadata=None, mimetype=None):
"""Create resource data from a stream"""
resource_hash = hashlib.sha1()
with self.db, self.db.cursor() as cur:
lobj = self.db.lobject(oid=0, mode='wb')
oid = lobj.oid
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data = {
'ctime': datetime.now(),
'mtime': datetime.now(),
'metadata': json.dumps(metadata),
'mimetype': mimetype or 'application/octet-stream',
'data_oid': oid,
'hash': 'sha1:{0}'.format(resource_hash.hexdigest()),
}
query = querybuilder.insert('resource_data', data)
cur.execute(query, data)
resource_data_id = cur.fetchone()[0]
return resource_data_id
def resource_data_get_info(self, objid):
query = querybuilder.select_pk('resource_data')
with self.db, self.db.cursor() as cur:
cur.execute(query, {'id': objid})
return cur.fetchone()
def resource_data_read(self, objid):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
return lobj.read()
def resource_data_copy(self, objid, dest):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
for chunk in file_read_chunks(lobj):
dest.write(chunk)
def resource_data_update(self, objid, stream=None, metadata=None,
mimetype=None):
# Get the original object, to check for its existence
# and to get the oid of the lobject holding the data.
original = self.resource_data_get_info(objid)
data = {
'id': objid,
'mtime': datetime.now(),
}
if metadata is not None:
data['metadata'] = json.dumps(metadata)
if mimetype is not None:
data['mimetype'] = mimetype
with self.db, self.db.cursor() as cur:
if stream is not None:
# Update the lobject with data from the stream
resource_hash = hashlib.sha1()
lobj = self.db.lobject(oid=original['data_oid'], mode='wb')
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data['hash'] = 'sha1:{0}'.format(resource_hash.hexdigest())
query = querybuilder.update('resource_data', data)
cur.execute(query, data)
def resource_data_remove(self, objid):
# We need the OID to remove the lobject
original = self.resource_data_get_info(objid)
with self.db, self.db.cursor() as cur:
lobject = db.lobject(oid=resource['data_oid'], mode='rb')
data = lobject.read()
lobject.close()
cur.execute(querybuilder.delete('resource_data'), {'id': objid})
# ------------------------------------------------------------
# Dataset / resource CRUD
# ------------------------------------------------------------
def create_resource(self, resource):
return self._dsres_create('resource', resource)
def update_resource(self, resource_id, resource):
return self._dsres_update('resource', resource_id, resource)
def get_resource(self, resource_id):
return self._dsres_get('resource', resource_id)
def list_resources(self, offset=None, limit=None):
return self._dsres_list('resource', offset=offset, limit=limit)
def delete_resource(self, resource_id):
return self._dsres_delete('resource', resource_id)
def create_dataset(self, dataset):
return self._dsres_create('dataset', dataset)
def update_dataset(self, dataset_id, dataset):
return self._dsres_update('dataset', dataset_id, dataset)
def get_dataset(self, dataset_id):
return self._dsres_get('dataset', dataset_id)
def list_datasets(self, offset=None, limit=None):
return self._dsres_list('dataset', offset=offset, limit=limit)
def delete_dataset(self, dataset_id):
return self._dsres_delete('dataset', dataset_id)
def add_dataset_resource(self, dataset_id, resource_id, order=0):
data = {
'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order}
query = querybuilder.insert(
'dataset_resource', data=data, table_key=None)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def delete_dataset_resource(self, dataset_id, resource_id):
data = {'dataset_id': dataset_id, 'resource_id': resource_id}
query = ("DELETE FROM dataset_resource"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def move_dataset_resource(self, dataset_id, resource_id, order):
data = {'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order}
query = ("UPDATE dataset_resource"
" SET order=%(order)s"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
# ------------------------------------------------------------
# Rows and datasets have the same schema:
# let's use some common functions for them..
# ------------------------------------------------------------
def _dsres_create(self, name, obj):
data = {
'configuration': json.dumps(obj),
'ctime': datetime.now(),
'mtime': datetime.now(),
}
query = querybuilder.insert(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
return cur.fetchone()[0]
def _dsres_update(self, name, obj_id, obj):
data = {
'id': obj_id,
'configuration': json.dumps(obj),
'mtime': datetime.now(),
}
query = querybuilder.update(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def _dsres_get(self, name, obj_id):
query = querybuilder.select_pk(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
row = cur.fetchone()
if row is None:
raise NotFound()
return self._row_to_obj(row)
def _dsres_list(self, name, offset=None, limit=None):
query = querybuilder.select_paged(
name, offset=offset, limit=limit)
with self.db, self.db.cursor() as cur:
cur.execute(query)
for row in cur.fetchall():
yield self._row_to_obj(row)
def _dsres_from_row(self, row):
obj = row['configuration']
obj['_id'] = row['id']
obj['_ctime'] = row['ctime']
obj['_mtime'] = row['mtime']
return obj
def _dsres_delete(self, name, obj_id):
query = querybuilder.delete(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
def __getitem__(self, key):
try:
return self._settings[key]
except KeyError:
return Config.__getitem__(self, key)
),
('/magazine/', _('Magazine')),
('http://tieba.baidu.com/f?ie=utf-8&kw=%E4%BD%9F%E5%A4%A7%E4%B8%BA',
_('Fans')),
]
FOOTER_MENU = [
('/about', _('About David Tong Studio')),
('/contact', _('Contact')),
('/jobs', _('Join us'))
]
import os
from flask.config import Config
APP_ROOT = os.path.abspath(os.path.join(__path__[0], '../'))
config = Config(APP_ROOT)
# load from environment
if 'DAVID_CONFIG_FILE' in os.environ:
config.from_pyfile(os.environ['DAVID_CONFIG_FILE'])
# load from local config
config.from_pyfile('local_config.py', silent=True)
globals().update(config)
class WorkProc:
"""A container process for our worker threads that can receive
notifications from a Unix domain socket.
"""
def __init__(self, basedir, db=None):
"""Create a container using a given base directory for the
storage and socket. Optionally, provide a database object to use
that instead of creating a new one (to, for example, reuse its
internal locks).
"""
self.basedir = os.path.abspath(basedir)
# Load the configuration. We're just reusing Flask's simple
# configuration component here.
self.config = Config(self.basedir)
self.config.from_object('polyphemus.config_default')
self.config.from_pyfile('polyphemus.cfg', silent=True)
# Create the database.
self.db = db or JobDB(self.basedir)
def start(self, stages_conf=None):
"""Create and start the worker threads. If stages_conf is None, create the
default workers for the given toolchain. If stages_confg is a list of
strings in worker.KNOWN_STAGES then create workers mapping to those.
"""
if stages_conf is None:
stages = worker.default_work_stages(self.config)
else:
stages = [worker.KNOWN_STAGES[stage] for stage in stages_conf]
print(stages)
for thread in worker.work_threads(stages, self.config, self.db):
if not thread.is_alive():
thread.start()
async def handle(self, client, addr):
"""Handle an incoming socket connection.
"""
async for line in client.makefile('rb'):
# Each line is a job name.
job_name = line.decode('utf8').strip()
print(job_name)
# Just notify the database that something changed.
with self.db.cv:
self.db.cv.notify_all()
def serve(self):
"""Start listening on a Unix domain socket for incoming
messages. Run indefinitely (until the server is interrupted).
"""
sockpath = os.path.join(self.basedir, SOCKNAME)
if os.path.exists(sockpath):
os.unlink(sockpath)
try:
curio.run(curio.unix_server, sockpath, self.handle)
except KeyboardInterrupt:
print ("Shutting down worker.")
finally:
os.unlink(sockpath)
def poll(self):
"""Continously poll the work directory for open jobs.
"""
try:
while True:
with self.db.cv:
self.db.cv.notify_all()
time.sleep(2)
except KeyboardInterrupt:
print ("Shutting down worker.")
from __future__ import unicode_literals
import os
from flask import Flask
from flask.config import Config
from flaskext.csrf import csrf
# config
config = Config(None, Flask.default_config)
config.from_object('rentmybike.settings.default')
if os.getenv('RENTMYBIKE_ENV'):
config.from_object('rentmybike.settings.' + os.getenv('RENTMYBIKE_ENV'))
# app
from application import RentMyBike # deferred
app = RentMyBike()
if app.config['DUMMY_DATA']:
app.add_dummy_data()
csrf(app)
# controllers
import controllers # deferred
def load_config(self):
config = Config(current_app.root_path)
config.from_pyfile(path.join(path.dirname(current_app.root_path),
'config.py'))
for option, value in config.items():
setattr(self, option, value)
from __future__ import unicode_literals
import os
from flask import Flask
from flask.config import Config
from flaskext.csrf import csrf
# config
config = Config(None, Flask.default_config)
config.from_object('rentmybike.settings.default')
if os.getenv('RENTMYBIKE_ENV'):
config.from_object('rentmybike.settings.' + os.getenv('RENTMYBIKE_ENV'))
else:
config.from_object('rentmybike.settings.custom')
# app
from application import RentMyBike # deferred
app = RentMyBike()
if app.config['DUMMY_DATA']:
app.add_dummy_data()
csrf(app)
# controllers
import controllers # deferred
def get(self, key, default=None):
"""Gets config from dynaconf variables
if variables does not exists in dynaconf try getting from
app.config to support runtime settings."""
return self._settings.get(key, Config.get(self, key, default))
def make_config(self, configuration): self.config = Config(None, self.default_config) if (configuration) : self.config.from_object(configuration)
import time
from time import strftime, localtime
from datetime import datetime
from sqlalchemy import String, Integer
from sqlalchemy.sql.expression import text , bindparam, outparam
from mabolab.core.base import Base
from flask.config import Config
settings = Config("" )
settings.from_pyfile( 'C:/MTP/mabotech/maboss1.2/maboss/configuration/central_config.py')
settings['APP_NAME'] = "next_seq"
base = Base( settings)
db = base.get_db("oracle")
def get_next_seq():
""" call stored procedure """
sql = """BP_SP_GETNEXTCERTNO (:I_FACILITY, :O_CertNo, :O_Message )"""
import os
import datetime
from flask_login import UserMixin
from flask.config import Config
# from werkzeug.security import check_password_hash, generate_password_hash
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
from pytwis import Pytwis, pytwis_constants
from .config import config_by_name
# BUGBUG: Read the configuration of the Flask app again since we can't
# find a way to access the configuration outside an application context.
config_name = os.getenv('PYTWASK_ENV', 'dev') # pylint: disable=invalid-name
app_config = Config(None) # pylint: disable=invalid-name
app_config.from_object(config_by_name[config_name])
# Connect to the local Redis database.
twis = Pytwis(hostname=app_config['REDIS_DB_HOSTNAME'], # pylint: disable=invalid-name
port=app_config['REDIS_DB_PORT'],
socket=app_config['REDIS_DB_SOCKET'],
db=app_config['REDIS_DB_INDEX'],
password=app_config['REDIS_DB_PASSWORD'])
class Tweet(): # pylint: disable=too-few-public-methods
"""This 'Tweet' class encapsulates all the information related to one tweet.
Note that it will convert the posted UNIX timestamp into a datetime.
"""
def __init__(self, username, post_unix_time, body):
import logging
import logging.handlers
import logging.config
#import profile
import time
from time import strftime, localtime
from Cheetah.Template import Template
from mabolab.core.base import Base
from flask.config import Config
from schedule import Schedule
settings = Config("" )
settings.from_pyfile( 'C:/MTP/mabotech/maboss1.3.0/maboss/conf/tools_config.py')
settings['APP_NAME'] = "mabozen"
base = Base( settings)
db_type = 'postgresql'
db = base.get_db(db_type)
def render(template_file, name_space):
"""
template file render
import numpy as np
import imagehash
from jinja2 import Environment, PackageLoader
import luigi
from luigi.contrib import redis_store
import networkx as nx
from PIL import Image
from flask.config import Config
import requests
import twarc
import json2csv
config = Config(os.path.dirname(__file__))
config.from_pyfile('dnflow.cfg')
logging.getLogger().setLevel(logging.WARN)
logging.getLogger('').setLevel(logging.WARN)
logging.getLogger('luigi-interface').setLevel(logging.WARN)
def time_hash(digits=6):
"""Generate an arbitrary hash based on the current time for filenames."""
hash = hashlib.sha1()
hash.update(str(time.time()).encode())
t = time.localtime()
dt = '%s%02d%02d%02d%02d' % (t.tm_year, t.tm_mon, t.tm_mday,
t.tm_hour, t.tm_min)
return '%s-%s' % (dt, hash.hexdigest()[:digits])
def config_to_dict(root_path, config_file):
config = FlaskConfig(root_path)
config.from_pyfile(config_file)
return dict((k, v) for k, v in config.iteritems())
import boto3
from boto.s3.connection import S3Connection
from depot.fields.sqlalchemy import UploadedFileField
import inflection
from flask.config import Config
from flask_migrate import Migrate
from sqlalchemy import Column, Integer, String
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy_wrapper import SQLAlchemy
from sa_types import EnumChoiceType
config_name = 'coding_challenge_restful.settings.Config'
config = Config('')
config.from_object(config_name)
s3_client = boto3.client(
's3',
# Hard coded strings as credentials, not recommended.
aws_access_key_id=config['AWS_ACCESS_KEY'],
aws_secret_access_key=config['AWS_SECRET_KEY'])
isolation_level = 'READ COMMITTED'
db = SQLAlchemy(uri=config['DATABASE_URL'], isolation_level=isolation_level)
migrate = Migrate(compare_type=True)
# Create Models
Model = db.Model
def __init__(self, _settings, *args, **kwargs):
"""perform the initial load"""
super(DynaconfConfig, self).__init__(*args, **kwargs)
Config.update(self, _settings.store)
self._settings = _settings
from flask.config import Config py = 'config.py' root_path = "" setting = Config(root_path) setting.from_pyfile(py) print setting['LOGGING'] print setting['PG_URL'] print setting['ORA_URL']
import datetime
from flask_login import UserMixin
from flask.config import Config
#from werkzeug.security import check_password_hash, generate_password_hash
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
import os
from pytwis import Pytwis
from .config import config_by_name
# BUGBUG: Read the configuration of the Flask app again since we can't
# find a way to access the configuration outside an application context.
config_name = os.getenv('PYTWASK_ENV', 'dev')
app_config = Config(None)
app_config.from_object(config_by_name[config_name])
# Connect to the local Redis database.
twis = Pytwis(hostname=app_config['REDIS_DB_HOSTNAME'],
port=app_config['REDIS_DB_PORT'],
db=app_config['REDIS_DB_INDEX'],
password=app_config['REDIS_DB_PASSWORD'])
class Tweet():
"""Tweet class"""
def __init__(self, username, post_unix_time, body):
self.username = username
self.post_datetime = datetime.datetime.fromtimestamp(post_unix_time)
self.body = body
class DatacatCore(object):
def __init__(self, config=None):
self.config = Config()
if config is not None:
self.config.update(config)
@property
def db(self):
if getattr(self, '_db', None) is None:
self._db = connect(**self.config['DATABASE'])
self._db.autocommit = False
return self._db
@property
def admin_db(self):
if getattr(self, '_admin_db', None) is None:
self._admin_db = connect(**self.config['DATABASE'])
self._admin_db.autocommit = True
return self._admin_db
def create_tables(self):
create_tables(self.admin_db)
def drop_tables(self):
drop_tables(self.admin_db)
# ------------------------------------------------------------
# Resource data CRUD
# ------------------------------------------------------------
def resource_data_iter(self):
"""Iterate over resource attributes"""
with self.db, self.db.cursor() as cur:
cur.execute("SELECT * FROM resource_data")
for row in cur.fetchall():
yield row
def resource_data_create(self, stream, metadata=None, mimetype=None):
"""Create resource data from a stream"""
resource_hash = hashlib.sha1()
with self.db, self.db.cursor() as cur:
lobj = self.db.lobject(oid=0, mode='wb')
oid = lobj.oid
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data = {
'ctime': datetime.now(),
'mtime': datetime.now(),
'metadata': json.dumps(metadata),
'mimetype': mimetype or 'application/octet-stream',
'data_oid': oid,
'hash': 'sha1:{0}'.format(resource_hash.hexdigest()),
}
query = querybuilder.insert('resource_data', data)
cur.execute(query, data)
resource_data_id = cur.fetchone()[0]
return resource_data_id
def resource_data_get_info(self, objid):
query = querybuilder.select_pk('resource_data')
with self.db, self.db.cursor() as cur:
cur.execute(query, {'id': objid})
return cur.fetchone()
def resource_data_read(self, objid):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
return lobj.read()
def resource_data_copy(self, objid, dest):
resource = self.resource_data_get_info(objid)
with self.db:
lobj = self.db.lobject(oid=resource['data_oid'], mode='rb')
for chunk in file_read_chunks(lobj):
dest.write(chunk)
def resource_data_update(self,
objid,
stream=None,
metadata=None,
mimetype=None):
# Get the original object, to check for its existence
# and to get the oid of the lobject holding the data.
original = self.resource_data_get_info(objid)
data = {
'id': objid,
'mtime': datetime.now(),
}
if metadata is not None:
data['metadata'] = json.dumps(metadata)
if mimetype is not None:
data['mimetype'] = mimetype
with self.db, self.db.cursor() as cur:
if stream is not None:
# Update the lobject with data from the stream
resource_hash = hashlib.sha1()
lobj = self.db.lobject(oid=original['data_oid'], mode='wb')
for chunk in file_read_chunks(stream):
lobj.write(chunk)
resource_hash.update(chunk)
lobj.close()
data['hash'] = 'sha1:{0}'.format(resource_hash.hexdigest())
query = querybuilder.update('resource_data', data)
cur.execute(query, data)
def resource_data_remove(self, objid):
# We need the OID to remove the lobject
original = self.resource_data_get_info(objid)
with self.db, self.db.cursor() as cur:
lobject = db.lobject(oid=resource['data_oid'], mode='rb')
data = lobject.read()
lobject.close()
cur.execute(querybuilder.delete('resource_data'), {'id': objid})
# ------------------------------------------------------------
# Dataset / resource CRUD
# ------------------------------------------------------------
def create_resource(self, resource):
return self._dsres_create('resource', resource)
def update_resource(self, resource_id, resource):
return self._dsres_update('resource', resource_id, resource)
def get_resource(self, resource_id):
return self._dsres_get('resource', resource_id)
def list_resources(self, offset=None, limit=None):
return self._dsres_list('resource', offset=offset, limit=limit)
def delete_resource(self, resource_id):
return self._dsres_delete('resource', resource_id)
def create_dataset(self, dataset):
return self._dsres_create('dataset', dataset)
def update_dataset(self, dataset_id, dataset):
return self._dsres_update('dataset', dataset_id, dataset)
def get_dataset(self, dataset_id):
return self._dsres_get('dataset', dataset_id)
def list_datasets(self, offset=None, limit=None):
return self._dsres_list('dataset', offset=offset, limit=limit)
def delete_dataset(self, dataset_id):
return self._dsres_delete('dataset', dataset_id)
def add_dataset_resource(self, dataset_id, resource_id, order=0):
data = {
'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order
}
query = querybuilder.insert('dataset_resource',
data=data,
table_key=None)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def delete_dataset_resource(self, dataset_id, resource_id):
data = {'dataset_id': dataset_id, 'resource_id': resource_id}
query = ("DELETE FROM dataset_resource"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def move_dataset_resource(self, dataset_id, resource_id, order):
data = {
'dataset_id': dataset_id,
'resource_id': resource_id,
'order': order
}
query = ("UPDATE dataset_resource"
" SET order=%(order)s"
" WHERE dataset_id=%(dataset_id)s"
" AND resource_id=%(resource_id)s")
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
# ------------------------------------------------------------
# Rows and datasets have the same schema:
# let's use some common functions for them..
# ------------------------------------------------------------
def _dsres_create(self, name, obj):
data = {
'configuration': json.dumps(obj),
'ctime': datetime.now(),
'mtime': datetime.now(),
}
query = querybuilder.insert(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
return cur.fetchone()[0]
def _dsres_update(self, name, obj_id, obj):
data = {
'id': obj_id,
'configuration': json.dumps(obj),
'mtime': datetime.now(),
}
query = querybuilder.update(name, data)
with self.db, self.db.cursor() as cur:
cur.execute(query, data)
def _dsres_get(self, name, obj_id):
query = querybuilder.select_pk(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
row = cur.fetchone()
if row is None:
raise NotFound()
return self._row_to_obj(row)
def _dsres_list(self, name, offset=None, limit=None):
query = querybuilder.select_paged(name, offset=offset, limit=limit)
with self.db, self.db.cursor() as cur:
cur.execute(query)
for row in cur.fetchall():
yield self._row_to_obj(row)
def _dsres_from_row(self, row):
obj = row['configuration']
obj['_id'] = row['id']
obj['_ctime'] = row['ctime']
obj['_mtime'] = row['mtime']
return obj
def _dsres_delete(self, name, obj_id):
query = querybuilder.delete(name)
with self.db, self.db.cursor() as cur:
cur.execute(query, dict(id=obj_id))
#!/usr/bin/env python
import os
import sys
import time
import logging
from flask.config import Config
import boto.sqs
from boto.sqs.message import RawMessage
from boto import exception
LOG = logging.getLogger('alerta.sqs')
config = Config('/')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
DEFAULT_AWS_REGION = 'eu-west-1'
DEFAULT_AWS_SQS_QUEUE = 'alerts'
AWS_REGION = os.environ.get('AWS_REGION') or config.get('AWS_REGION', DEFAULT_AWS_REGION)
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get('AWS_SECRET_ACCESS_KEY')
AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get('AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE)
class Worker(object):
def __init__(self):
def __init__(self, config=None):
self.config = Config()
if config is not None:
self.config.update(config)
from flask.config import Config
from mabolab.core.global_obj import Global
CENTRAL_CONFIG = 'C:/MTP/mabotech/maboss1.3.0/maboss/conf/maboss_config.py'
settings = Config("")
settings.from_pyfile(CENTRAL_CONFIG)
settings['APP_NAME'] = "monitor_bli"
g = Global(settings)
db = g.get_db('postgresql')
ora = g.get_db('oracle')
log = g.get_logger()
def dbtest(serialno):
sql = """select status, lastupdateon from cob_t_serial_no_workstation
where serialno = '%s' and workstation = '42700'
order by id desc""" % (serialno)
rtn = ora.execute(sql)
print rtn.fetchone()
from flask.config import Config
from mabolab.core.global_obj import Global
CENTRAL_CONFIG = 'C:/MTP/mabotech/maboss1.3.0/maboss/conf/maboss_config.py'
settings = Config("")
settings.from_pyfile(CENTRAL_CONFIG)
settings['APP_NAME'] = "monitor_bli"
g = Global(settings)
db = g.get_db('postgresql')
ora = g.get_db('oracle')
log = g.get_logger()
def dbtest(serialno):
sql = """select status, lastupdateon from cob_t_serial_no_workstation
where serialno = '%s' and workstation = '42700'
order by id desc""" % (serialno)
rtn = ora.execute(sql)
#!/usr/bin/env python
import os
import sys
import time
import logging
from flask.config import Config
import boto.sqs
from boto.sqs.message import RawMessage
from boto import exception
LOG = logging.getLogger('alerta.sqs')
config = Config('/')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
DEFAULT_AWS_REGION = 'eu-west-1'
DEFAULT_AWS_SQS_QUEUE = 'alerts'
AWS_REGION = os.environ.get('AWS_REGION') or config.get(
'AWS_REGION', DEFAULT_AWS_REGION)
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get(
'AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get(
'AWS_SECRET_ACCESS_KEY')
AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get(
'AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE)
def make_config():
cfg = Config('')
cfg.from_object('datacat.settings.default')
cfg.from_envvar('DATACAT_SETTINGS', silent=True)
return cfg
