def cli(ctx): ''' Access and modify data from a PISS server on the command line. ''' current_dir = os.path.dirname(os.path.realpath(__file__)) config_file = os.path.splitext(os.path.basename(__file__))[0] + ".cfg" ctx.obj = {} ctx.obj['current_dir'] = current_dir ctx.obj['config_file'] = config_file if ctx.args[0] != 'register': if not os.path.isfile(os.path.join(current_dir, config_file)): click.echo('No configuration file found! Use `register <url>` to\ create one!') ctx.abort() try: app_config = Config(current_dir) app_config.from_pyfile(config_file) except Exception as e: click.echo('Could not load config from file. Use `register <url>`\ to create a new one!') ctx.abort() try: ctx.obj['credentials'] = app_config['CREDENTIALS'] ctx.obj['meta_url'] = app_config['META_URL'] meta_post = app_config['META_POST'] ctx.obj['meta_post'] = meta_post ctx.obj['url'] = meta_post['server']['urls']['posts_feed'] except Exception as e: click.echo('Parameters missing from configuration file! Use\ `register <url>` to fix!') ctx.abort()
def __init__(self, settings=None): self.settings = settings log_cfg = Config('') log_cfg.from_pyfile( settings['LOGGING_CONFIG']) self.log = logging_factory.get_logger(settings['APP_NAME'], settings['LOGGING_PATH'], log_cfg['LOGGING']) self.dbattr_dict = {'ORA':'ora_db','PG':'pg_db', 'DB':'db'} self.name_dict = {'ORA':'oracle','PG':'postgresql', 'DB':'oracle'} #import dbsession after logger initialized from mabolab.database import dbfactory for db_type in settings['DB_TYPE']: if not hasattr(self, self.dbattr_dict[db_type]): #self.log.debug("set %s" %(db_type) ) setattr(self, self.dbattr_dict[db_type], dbfactory.get_db(self.name_dict[db_type], settings) ) if not hasattr(self, 'db'): setattr(self, 'db', dbfactory.get_db(self.name_dict['DB'], settings) )
def app_config(postgres_user_conf): from flask.config import Config from datacat.settings import testing conf = Config('') conf.from_object(testing) conf['DATABASE'] = postgres_user_conf return conf
def _get_config(): # Workaround to get an available config object before the app is initiallized # Only needed/used in top-level and class statements # https://stackoverflow.com/a/18138250/7597273 root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) config = Config(root_path) config.from_object('config') return config
def __init__(self, _settings, _app, *args, **kwargs): """perform the initial load""" super().__init__(*args, **kwargs) # Bring Dynaconf instance value to Flask Config Config.update(self, _settings.store) self._settings = _settings self._app = _app
def setUp(self): tmp = '../config/logging_config.py' log_cfg = Config('') log_cfg.from_pyfile(tmp) self.logging_cfg = log_cfg['LOGGING']
def load_config(): cfg = Config(dirname(dirname(__file__))) cfg.from_object("autobit.settings") if "AUTOBIT_SETTINGS" in os.environ: cfg.from_envvar("AUTOBIT_SETTINGS") if not exists(cfg['WATCH_DIR']): logger.info("Creating watch dir: {}".format(cfg['WATCH_DIR'])) os.makedirs(cfg['WATCH_DIR']) return cfg
class WorkProc: """A container process for our worker threads that can receive notifications from a Unix domain socket. """ def __init__(self, basedir, db=None): """Create a container using a given base directory for the storage and socket. Optionally, provide a database object to use that instead of creating a new one (to, for example, reuse its internal locks). """ self.basedir = os.path.abspath(basedir) # Load the configuration. We're just reusing Flask's simple # configuration component here. self.config = Config(self.basedir) self.config.from_object('buildbot.config_default') self.config.from_pyfile('buildbot.cfg', silent=True) # Create the database. self.db = db or JobDB(self.basedir) def start(self): """Create and start the worker threads. """ threads = worker.work_threads(self.db, self.config) for thread in threads: if not thread.is_alive(): thread.start() async def handle(self, client, addr): """Handle an incoming socket connection. """ async for line in client.makefile('rb'): # Each line is a job name. job_name = line.decode('utf8').strip() print(job_name) # Just notify the database that something changed. with self.db.cv: self.db.cv.notify_all() def serve(self): """Start listening on a Unix domain socket for incoming messages. Run indefinitely (until the server is interrupted). """ sockpath = os.path.join(self.basedir, SOCKNAME) if os.path.exists(sockpath): os.unlink(sockpath) try: curio.run(curio.unix_server, sockpath, self.handle) except KeyboardInterrupt: pass finally: os.unlink(sockpath)
def load(config_filename='settings.py'): """Create a Flask config that will be used to update the application config when it is created.""" config = Config("pjuu") # Load the default settings config.from_pyfile(config_filename) # Load the setting from the Environment variable config.from_envvar('PJUU_SETTINGS', silent=True) return config
def download_users(): """ Download users.xml file. """ import urllib2 from flask.config import Config config = Config(etc()) config.from_pyfile("deploy.cfg") response = urllib2.urlopen(config['USERS_URL']) users_xml = os.path.join('runtime', 'data', 'users.xml') if response.code == 200: with open(users_xml, 'w') as f: f.write(response.read())
def read_config() -> Config: config = Config("") env_settings_file = os.environ.get('DEBUGPROXY_SETTING_FILE', './config/default_settings.py') config.from_pyfile(env_settings_file) # update the config with any values found in the environment for key in config.keys(): env_value = os.environ.get(key) if env_value: config[key] = env_value return config
def configs(self): if not hasattr(self, '_configs'): configs = Config(ROOT) resoure = ResourceLoader.get().get_resoure('settings.py') config_files = resoure.as_list() if config_files: for path in config_files: configs.from_pyfile(path) else: raise Exception('need a configuration file to start app') self._configs = configs return self._configs
def load_config(settings_file='./test_settings.py'): """ Loads the config files merging the defaults with the file defined in environ.PULLSBURY_SETTINGS if it exists. """ config = Config(os.getcwd()) if 'PULLSBURY_SETTINGS' in os.environ: config.from_envvar('PULLSBURY_SETTINGS') else: config.from_pyfile(settings_file) if config.get('LOGGING_CONFIG'): logging.config.fileConfig( config.get('LOGGING_CONFIG'), disable_existing_loggers=False) json_values = [ 'TEAMS', 'HAPPY_SLACK_EMOJIS', 'REPO_BLACKLIST', 'SLACK_CUSTOM_EMOJI_MAPPING' ] for value in json_values: config.update({ value: json.loads(config.get(value, '{}')) }) return config
def __init__(self, basedir, db=None): """Create a container using a given base directory for the storage and socket. Optionally, provide a database object to use that instead of creating a new one (to, for example, reuse its internal locks). """ self.basedir = os.path.abspath(basedir) # Load the configuration. We're just reusing Flask's simple # configuration component here. self.config = Config(self.basedir) self.config.from_object('buildbot.config_default') self.config.from_pyfile('buildbot.cfg', silent=True) # Create the database. self.db = db or JobDB(self.basedir)
def load_config(): """ Loads the config files merging the defaults with the file defined in environ.LINTREVIEW_SETTINGS if it exists. """ config = Config(os.getcwd()) if 'LINTREVIEW_SETTINGS' in os.environ: config.from_envvar('LINTREVIEW_SETTINGS') elif os.path.exists(os.path.join(os.getcwd(), 'settings.py')): config.from_pyfile('settings.py') else: msg = ("Unable to load configuration file. Please " "either create ./settings.py or set LINTREVIEW_SETTINGS " "in your environment before running.") raise ImportError(msg) if config.get('LOGGING_CONFIG'): logging.config.fileConfig( config.get('LOGGING_CONFIG'), disable_existing_loggers=False) if config.get('SSL_CA_BUNDLE'): os.environ['REQUESTS_CA_BUNDLE'] = config.get('SSL_CA_BUNDLE') return config
def main(ctx, verbosity): configure_logger(verbosity) # Load the configurations from file config = Config(root_path=".") config.from_object(default_settings) config.from_envvar(ZUBBI_SETTINGS_ENV) # Validate the configuration tenant_sources_repo = config.get("TENANT_SOURCES_REPO") tenant_sources_file = config.get("TENANT_SOURCES_FILE") # Fail if both are set or none of both is set if ( not tenant_sources_file and not tenant_sources_repo or (tenant_sources_file and tenant_sources_repo) ): raise ScraperConfigurationError( "Either one of 'TENANT_SOURCES_REPO' " "and 'TENANT_SOURCES_FILE' must be set, " "but not both." ) # Store the config in click's context object to be available for subcommands ctx.obj = {"config": config} if ctx.invoked_subcommand is None: ctx.invoke(scrape)
def __init__(self, import_name, middlewares=None, loop=None, session=None): self.import_name = import_name self.root_path = get_root_path(import_name) self.config = Config(self.root_path, default_config) self._context = {} self._loop = loop or asyncio.get_event_loop() self._middlewares = SpiderMiddlewareManager(self, middlewares) self._session = session or aiohttp.ClientSession(loop=self._loop)
def __init__(self, debug=False): self.valid = True # loggerを初期化する logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s') logger = logging.getLogger(self.service_name) logger.setLevel(logging.DEBUG) self.logger = logger # 設定ファイルからAPIkeyとかを読み込む current_dir = os.path.dirname(__file__) config = Config('../../') if os.path.exists(activate_this): config.from_pyfile("wsgi_dev.cfg") else: config.from_pyfile("dev.cfg") #self.logger.debug(config) self.validate(self.require_config, config['EXTERNAL_CONFIG']) self.config = config['EXTERNAL_CONFIG'] self.debug = config['POST_DEBUG']
def __init__(self): self.config = Config(root_path="/") self.config.from_envvar('APP_CONFIG') self.redis = searcher.redis_connect(self.config) db.init_db(self.config['SQLALCHEMY_DATABASE_URI']) self.searcher = searcher.WhooshSearcher( index_dir=self.config['WHOOSH_INDEX_PATH']) self.running = False
def test_settings_abs_path(self): """Check if the config obj is updated with default_settings when it is passed as a python file absolute path """ abs_path = getcwd() + '/arachne/tests/test_settings.py' test_app = self.create_app(settings=abs_path) # load config from pyfile flask_app = Flask(__name__) flask_app.config.from_object('arachne.default_settings') config_cls = Config(__name__) config_cls.from_pyfile(abs_path) # update config with the arachne default settings flask_app.config.update(config_cls) # test if config dicts are same self.assertEquals(test_app.config, flask_app.config)
class Settings(object): config = Config(__name__) DEBUG = ConfigAttribute('DEBUG') APP_NAME = ConfigAttribute('APP_NAME') DJAODJIN_SECRET_KEY = ConfigAttribute('DJAODJIN_SECRET_KEY') def update(self, **updates): return self.config.update(updates)
class IndexerDaemon(object): def __init__(self): self.config = Config(root_path="/") self.config.from_envvar('APP_CONFIG') self.redis = searcher.redis_connect(self.config) db.init_db(self.config['SQLALCHEMY_DATABASE_URI']) self.searcher = searcher.WhooshSearcher( index_dir=self.config['WHOOSH_INDEX_PATH']) self.running = False def _get_writer(self): return self.searcher.ix.writer() def main_loop(self): self.running = True while self.running: self.work_on_queue() def work_on_queue(self): """Queue operations (redis reliable queue pattern) Atomically pop an item from the work queue and put it on the processing queue; after processing the item is removed from the processing queue. The "processing" step fetch the specified bookmark from the SQL database and run the specified operation (index, update, delete); after completing the operation it updates the bookmark `indexed_on` field. """ conn = self.redis msg = conn.brpoplpush(searcher.QUEUE_INDEX, searcher.QUEUE_WORK) log.debug("Processing new operation: {}".format(msg)) try: payload = json.loads(msg) op, _id = payload except ValueError, ex: log.error("Invalid job in queue: {0}: {1}".format(msg, str(ex))) else:
class CartesiusSuite : #: Default configuration parameters. default_config = ImmutableDict({ 'DEBUG': False, 'SERVER_NAME': None, 'APPLICATION_ROOT': None }) def __init__(self, configuration = None) : self.make_config(configuration) self.make_servers() def make_config(self, configuration): self.config = Config(None, self.default_config) if (configuration) : self.config.from_object(configuration) def make_servers(self) : self.servers = {} for server in self.config['SERVERS'] : server_path = self.config['SERVER_PATH'] + "." + server['path'] + "." if( 'app_name' in server ) : server_path += server['app_name'] else : server_path += 'app' self.servers[server['name']] = CartesiusServer(server_path) def start(self, server) : self.servers[server].start() def start_all(self) : for server in self.servers : self.servers[server].start() def stop(self, server) : self.servers[server].stop() def stop_all(self) : for server in self.servers : server.stop()
def test_settings_abs_path(self): """Check if the config obj is updated with default_settings when it is passed as a python file absolute path """ abs_path = getcwd() + '/arachne/tests/test_settings.py' test_app = self.create_app(settings=abs_path) # since the object initialized created is always different # we ignore CRAWLER_PROCESS setting for test if SCRAPY_VERSION >= (1, 0, 0): del test_app.config['CRAWLER_PROCESS'] # load config from pyfile flask_app = Flask(__name__) flask_app.config.from_object('arachne.default_settings') config_cls = Config(__name__) config_cls.from_pyfile(abs_path) # update config with the arachne default settings flask_app.config.update(config_cls) # test if config dicts are same self.assertEquals(test_app.config, flask_app.config)
def test_settings_abs_path(self): """Check if the config obj is updated with default_settings when it is passed as a python file absolute path """ abs_path = getcwd() + '/arachneserver/tests/test_settings.py' test_app = self.create_app(settings=abs_path) # since the object initialized created is always different # we ignore CRAWLER_PROCESS setting for test # if SCRAPY_VERSION >= (1, 0, 0): del test_app.config['CRAWLER_PROCESS'] # load config from pyfile flask_app = Flask(__name__) flask_app.config.from_object('arachneserver.default_settings') config_cls = Config(__name__) config_cls.from_pyfile(abs_path) # update config with the server default settings flask_app.config.update(config_cls) # test if config dicts are same self.assertEquals(test_app.config, flask_app.config)
def make_config(app=None): if app is not None: cfg = app.config else: from flask.config import Config root_path = os.path.dirname(__file__).rsplit('/', 1)[0] cfg = Config(root_path) # customize config here cfg.from_object(default_config) cfg.from_pyfile('myapp.cfg', silent=True) cfg.from_envvar('MYAPP_CONFIG', silent=True) cfg['BABEL_DEFAULT_LOCALE'] = cfg['LANG'] return cfg
def load_config(config_obj=None): """ Load Greenwave configuration. It will load the configuration based on how the environment is configured. :return: A dict of Greenwave configuration. """ # Load default config, then override that with a config file config = Config(__name__) if config_obj is None: if os.getenv('TEST') == 'true': config_obj = 'greenwave.config.TestingConfig' elif os.getenv('DEV') == 'true' or os.getenv('DOCS') == 'true': config_obj = 'greenwave.config.DevelopmentConfig' else: config_obj = 'greenwave.config.ProductionConfig' if os.getenv('TEST') == 'true': default_config_file = os.path.join(os.getcwd(), 'conf', 'settings.py.example') elif os.getenv('DEV') == 'true': default_config_file = os.path.join(os.getcwd(), 'conf', 'settings.py') elif os.getenv('DOCS') == 'true': default_config_file = os.path.normpath( os.path.join(os.getcwd(), '..', 'conf', 'settings.py.example')) else: default_config_file = '/etc/greenwave/settings.py' # 1. Load default configuration. log.debug("config: Loading config from %r", config_obj) config.from_object(config_obj) # 2. Override default configuration with environment variables. if os.environ.get('GREENWAVE_SUBJECT_TYPES_DIR'): config['SUBJECT_TYPES_DIR'] = os.environ['GREENWAVE_SUBJECT_TYPES_DIR'] if os.environ.get('GREENWAVE_POLICIES_DIR'): config['POLICIES_DIR'] = os.environ['GREENWAVE_POLICIES_DIR'] # 3. Override default configuration and environment variables with custom config file. config_file = os.environ.get('GREENWAVE_CONFIG', default_config_file) log.debug("config: Extending config with %r", config_file) config.from_pyfile(config_file) if os.environ.get('SECRET_KEY'): config['SECRET_KEY'] = os.environ['SECRET_KEY'] return config
def __init__(self, _config, *args, **kwargs): super(_Config, self).__init__(*args, **kwargs) Config.update(self, _config)
def __init__(self, config=None): self.config = Config() if config is not None: self.config.update(config)
class DatacatCore(object): def __init__(self, config=None): self.config = Config() if config is not None: self.config.update(config) @property def db(self): if getattr(self, '_db', None) is None: self._db = connect(**self.config['DATABASE']) self._db.autocommit = False return self._db @property def admin_db(self): if getattr(self, '_admin_db', None) is None: self._admin_db = connect(**self.config['DATABASE']) self._admin_db.autocommit = True return self._admin_db def create_tables(self): create_tables(self.admin_db) def drop_tables(self): drop_tables(self.admin_db) # ------------------------------------------------------------ # Resource data CRUD # ------------------------------------------------------------ def resource_data_iter(self): """Iterate over resource attributes""" with self.db, self.db.cursor() as cur: cur.execute("SELECT * FROM resource_data") for row in cur.fetchall(): yield row def resource_data_create(self, stream, metadata=None, mimetype=None): """Create resource data from a stream""" resource_hash = hashlib.sha1() with self.db, self.db.cursor() as cur: lobj = self.db.lobject(oid=0, mode='wb') oid = lobj.oid for chunk in file_read_chunks(stream): lobj.write(chunk) resource_hash.update(chunk) lobj.close() data = { 'ctime': datetime.now(), 'mtime': datetime.now(), 'metadata': json.dumps(metadata), 'mimetype': mimetype or 'application/octet-stream', 'data_oid': oid, 'hash': 'sha1:{0}'.format(resource_hash.hexdigest()), } query = querybuilder.insert('resource_data', data) cur.execute(query, data) resource_data_id = cur.fetchone()[0] return resource_data_id def resource_data_get_info(self, objid): query = querybuilder.select_pk('resource_data') with self.db, self.db.cursor() as cur: cur.execute(query, {'id': objid}) return cur.fetchone() def resource_data_read(self, objid): resource = self.resource_data_get_info(objid) with self.db: lobj = self.db.lobject(oid=resource['data_oid'], mode='rb') return lobj.read() def resource_data_copy(self, objid, dest): resource = self.resource_data_get_info(objid) with self.db: lobj = self.db.lobject(oid=resource['data_oid'], mode='rb') for chunk in file_read_chunks(lobj): dest.write(chunk) def resource_data_update(self, objid, stream=None, metadata=None, mimetype=None): # Get the original object, to check for its existence # and to get the oid of the lobject holding the data. original = self.resource_data_get_info(objid) data = { 'id': objid, 'mtime': datetime.now(), } if metadata is not None: data['metadata'] = json.dumps(metadata) if mimetype is not None: data['mimetype'] = mimetype with self.db, self.db.cursor() as cur: if stream is not None: # Update the lobject with data from the stream resource_hash = hashlib.sha1() lobj = self.db.lobject(oid=original['data_oid'], mode='wb') for chunk in file_read_chunks(stream): lobj.write(chunk) resource_hash.update(chunk) lobj.close() data['hash'] = 'sha1:{0}'.format(resource_hash.hexdigest()) query = querybuilder.update('resource_data', data) cur.execute(query, data) def resource_data_remove(self, objid): # We need the OID to remove the lobject original = self.resource_data_get_info(objid) with self.db, self.db.cursor() as cur: lobject = db.lobject(oid=resource['data_oid'], mode='rb') data = lobject.read() lobject.close() cur.execute(querybuilder.delete('resource_data'), {'id': objid}) # ------------------------------------------------------------ # Dataset / resource CRUD # ------------------------------------------------------------ def create_resource(self, resource): return self._dsres_create('resource', resource) def update_resource(self, resource_id, resource): return self._dsres_update('resource', resource_id, resource) def get_resource(self, resource_id): return self._dsres_get('resource', resource_id) def list_resources(self, offset=None, limit=None): return self._dsres_list('resource', offset=offset, limit=limit) def delete_resource(self, resource_id): return self._dsres_delete('resource', resource_id) def create_dataset(self, dataset): return self._dsres_create('dataset', dataset) def update_dataset(self, dataset_id, dataset): return self._dsres_update('dataset', dataset_id, dataset) def get_dataset(self, dataset_id): return self._dsres_get('dataset', dataset_id) def list_datasets(self, offset=None, limit=None): return self._dsres_list('dataset', offset=offset, limit=limit) def delete_dataset(self, dataset_id): return self._dsres_delete('dataset', dataset_id) def add_dataset_resource(self, dataset_id, resource_id, order=0): data = { 'dataset_id': dataset_id, 'resource_id': resource_id, 'order': order} query = querybuilder.insert( 'dataset_resource', data=data, table_key=None) with self.db, self.db.cursor() as cur: cur.execute(query, data) def delete_dataset_resource(self, dataset_id, resource_id): data = {'dataset_id': dataset_id, 'resource_id': resource_id} query = ("DELETE FROM dataset_resource" " WHERE dataset_id=%(dataset_id)s" " AND resource_id=%(resource_id)s") with self.db, self.db.cursor() as cur: cur.execute(query, data) def move_dataset_resource(self, dataset_id, resource_id, order): data = {'dataset_id': dataset_id, 'resource_id': resource_id, 'order': order} query = ("UPDATE dataset_resource" " SET order=%(order)s" " WHERE dataset_id=%(dataset_id)s" " AND resource_id=%(resource_id)s") with self.db, self.db.cursor() as cur: cur.execute(query, data) # ------------------------------------------------------------ # Rows and datasets have the same schema: # let's use some common functions for them.. # ------------------------------------------------------------ def _dsres_create(self, name, obj): data = { 'configuration': json.dumps(obj), 'ctime': datetime.now(), 'mtime': datetime.now(), } query = querybuilder.insert(name, data) with self.db, self.db.cursor() as cur: cur.execute(query, data) return cur.fetchone()[0] def _dsres_update(self, name, obj_id, obj): data = { 'id': obj_id, 'configuration': json.dumps(obj), 'mtime': datetime.now(), } query = querybuilder.update(name, data) with self.db, self.db.cursor() as cur: cur.execute(query, data) def _dsres_get(self, name, obj_id): query = querybuilder.select_pk(name) with self.db, self.db.cursor() as cur: cur.execute(query, dict(id=obj_id)) row = cur.fetchone() if row is None: raise NotFound() return self._row_to_obj(row) def _dsres_list(self, name, offset=None, limit=None): query = querybuilder.select_paged( name, offset=offset, limit=limit) with self.db, self.db.cursor() as cur: cur.execute(query) for row in cur.fetchall(): yield self._row_to_obj(row) def _dsres_from_row(self, row): obj = row['configuration'] obj['_id'] = row['id'] obj['_ctime'] = row['ctime'] obj['_mtime'] = row['mtime'] return obj def _dsres_delete(self, name, obj_id): query = querybuilder.delete(name) with self.db, self.db.cursor() as cur: cur.execute(query, dict(id=obj_id))
def __getitem__(self, key): try: return self._settings[key] except KeyError: return Config.__getitem__(self, key)
), ('/magazine/', _('Magazine')), ('http://tieba.baidu.com/f?ie=utf-8&kw=%E4%BD%9F%E5%A4%A7%E4%B8%BA', _('Fans')), ] FOOTER_MENU = [ ('/about', _('About David Tong Studio')), ('/contact', _('Contact')), ('/jobs', _('Join us')) ] import os from flask.config import Config APP_ROOT = os.path.abspath(os.path.join(__path__[0], '../')) config = Config(APP_ROOT) # load from environment if 'DAVID_CONFIG_FILE' in os.environ: config.from_pyfile(os.environ['DAVID_CONFIG_FILE']) # load from local config config.from_pyfile('local_config.py', silent=True) globals().update(config)
class WorkProc: """A container process for our worker threads that can receive notifications from a Unix domain socket. """ def __init__(self, basedir, db=None): """Create a container using a given base directory for the storage and socket. Optionally, provide a database object to use that instead of creating a new one (to, for example, reuse its internal locks). """ self.basedir = os.path.abspath(basedir) # Load the configuration. We're just reusing Flask's simple # configuration component here. self.config = Config(self.basedir) self.config.from_object('polyphemus.config_default') self.config.from_pyfile('polyphemus.cfg', silent=True) # Create the database. self.db = db or JobDB(self.basedir) def start(self, stages_conf=None): """Create and start the worker threads. If stages_conf is None, create the default workers for the given toolchain. If stages_confg is a list of strings in worker.KNOWN_STAGES then create workers mapping to those. """ if stages_conf is None: stages = worker.default_work_stages(self.config) else: stages = [worker.KNOWN_STAGES[stage] for stage in stages_conf] print(stages) for thread in worker.work_threads(stages, self.config, self.db): if not thread.is_alive(): thread.start() async def handle(self, client, addr): """Handle an incoming socket connection. """ async for line in client.makefile('rb'): # Each line is a job name. job_name = line.decode('utf8').strip() print(job_name) # Just notify the database that something changed. with self.db.cv: self.db.cv.notify_all() def serve(self): """Start listening on a Unix domain socket for incoming messages. Run indefinitely (until the server is interrupted). """ sockpath = os.path.join(self.basedir, SOCKNAME) if os.path.exists(sockpath): os.unlink(sockpath) try: curio.run(curio.unix_server, sockpath, self.handle) except KeyboardInterrupt: print ("Shutting down worker.") finally: os.unlink(sockpath) def poll(self): """Continously poll the work directory for open jobs. """ try: while True: with self.db.cv: self.db.cv.notify_all() time.sleep(2) except KeyboardInterrupt: print ("Shutting down worker.")
from __future__ import unicode_literals import os from flask import Flask from flask.config import Config from flaskext.csrf import csrf # config config = Config(None, Flask.default_config) config.from_object('rentmybike.settings.default') if os.getenv('RENTMYBIKE_ENV'): config.from_object('rentmybike.settings.' + os.getenv('RENTMYBIKE_ENV')) # app from application import RentMyBike # deferred app = RentMyBike() if app.config['DUMMY_DATA']: app.add_dummy_data() csrf(app) # controllers import controllers # deferred
def load_config(self): config = Config(current_app.root_path) config.from_pyfile(path.join(path.dirname(current_app.root_path), 'config.py')) for option, value in config.items(): setattr(self, option, value)
from __future__ import unicode_literals import os from flask import Flask from flask.config import Config from flaskext.csrf import csrf # config config = Config(None, Flask.default_config) config.from_object('rentmybike.settings.default') if os.getenv('RENTMYBIKE_ENV'): config.from_object('rentmybike.settings.' + os.getenv('RENTMYBIKE_ENV')) else: config.from_object('rentmybike.settings.custom') # app from application import RentMyBike # deferred app = RentMyBike() if app.config['DUMMY_DATA']: app.add_dummy_data() csrf(app) # controllers import controllers # deferred
def get(self, key, default=None): """Gets config from dynaconf variables if variables does not exists in dynaconf try getting from app.config to support runtime settings.""" return self._settings.get(key, Config.get(self, key, default))
def make_config(self, configuration): self.config = Config(None, self.default_config) if (configuration) : self.config.from_object(configuration)
import time from time import strftime, localtime from datetime import datetime from sqlalchemy import String, Integer from sqlalchemy.sql.expression import text , bindparam, outparam from mabolab.core.base import Base from flask.config import Config settings = Config("" ) settings.from_pyfile( 'C:/MTP/mabotech/maboss1.2/maboss/configuration/central_config.py') settings['APP_NAME'] = "next_seq" base = Base( settings) db = base.get_db("oracle") def get_next_seq(): """ call stored procedure """ sql = """BP_SP_GETNEXTCERTNO (:I_FACILITY, :O_CertNo, :O_Message )"""
import os import datetime from flask_login import UserMixin from flask.config import Config # from werkzeug.security import check_password_hash, generate_password_hash from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired from pytwis import Pytwis, pytwis_constants from .config import config_by_name # BUGBUG: Read the configuration of the Flask app again since we can't # find a way to access the configuration outside an application context. config_name = os.getenv('PYTWASK_ENV', 'dev') # pylint: disable=invalid-name app_config = Config(None) # pylint: disable=invalid-name app_config.from_object(config_by_name[config_name]) # Connect to the local Redis database. twis = Pytwis(hostname=app_config['REDIS_DB_HOSTNAME'], # pylint: disable=invalid-name port=app_config['REDIS_DB_PORT'], socket=app_config['REDIS_DB_SOCKET'], db=app_config['REDIS_DB_INDEX'], password=app_config['REDIS_DB_PASSWORD']) class Tweet(): # pylint: disable=too-few-public-methods """This 'Tweet' class encapsulates all the information related to one tweet. Note that it will convert the posted UNIX timestamp into a datetime. """ def __init__(self, username, post_unix_time, body):
import logging import logging.handlers import logging.config #import profile import time from time import strftime, localtime from Cheetah.Template import Template from mabolab.core.base import Base from flask.config import Config from schedule import Schedule settings = Config("" ) settings.from_pyfile( 'C:/MTP/mabotech/maboss1.3.0/maboss/conf/tools_config.py') settings['APP_NAME'] = "mabozen" base = Base( settings) db_type = 'postgresql' db = base.get_db(db_type) def render(template_file, name_space): """ template file render
import numpy as np import imagehash from jinja2 import Environment, PackageLoader import luigi from luigi.contrib import redis_store import networkx as nx from PIL import Image from flask.config import Config import requests import twarc import json2csv config = Config(os.path.dirname(__file__)) config.from_pyfile('dnflow.cfg') logging.getLogger().setLevel(logging.WARN) logging.getLogger('').setLevel(logging.WARN) logging.getLogger('luigi-interface').setLevel(logging.WARN) def time_hash(digits=6): """Generate an arbitrary hash based on the current time for filenames.""" hash = hashlib.sha1() hash.update(str(time.time()).encode()) t = time.localtime() dt = '%s%02d%02d%02d%02d' % (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min) return '%s-%s' % (dt, hash.hexdigest()[:digits])
def config_to_dict(root_path, config_file): config = FlaskConfig(root_path) config.from_pyfile(config_file) return dict((k, v) for k, v in config.iteritems())
import boto3 from boto.s3.connection import S3Connection from depot.fields.sqlalchemy import UploadedFileField import inflection from flask.config import Config from flask_migrate import Migrate from sqlalchemy import Column, Integer, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.declarative import declared_attr from sqlalchemy_wrapper import SQLAlchemy from sa_types import EnumChoiceType config_name = 'coding_challenge_restful.settings.Config' config = Config('') config.from_object(config_name) s3_client = boto3.client( 's3', # Hard coded strings as credentials, not recommended. aws_access_key_id=config['AWS_ACCESS_KEY'], aws_secret_access_key=config['AWS_SECRET_KEY']) isolation_level = 'READ COMMITTED' db = SQLAlchemy(uri=config['DATABASE_URL'], isolation_level=isolation_level) migrate = Migrate(compare_type=True) # Create Models Model = db.Model
def __init__(self, _settings, *args, **kwargs): """perform the initial load""" super(DynaconfConfig, self).__init__(*args, **kwargs) Config.update(self, _settings.store) self._settings = _settings
from flask.config import Config py = 'config.py' root_path = "" setting = Config(root_path) setting.from_pyfile(py) print setting['LOGGING'] print setting['PG_URL'] print setting['ORA_URL']
import datetime from flask_login import UserMixin from flask.config import Config #from werkzeug.security import check_password_hash, generate_password_hash from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired import os from pytwis import Pytwis from .config import config_by_name # BUGBUG: Read the configuration of the Flask app again since we can't # find a way to access the configuration outside an application context. config_name = os.getenv('PYTWASK_ENV', 'dev') app_config = Config(None) app_config.from_object(config_by_name[config_name]) # Connect to the local Redis database. twis = Pytwis(hostname=app_config['REDIS_DB_HOSTNAME'], port=app_config['REDIS_DB_PORT'], db=app_config['REDIS_DB_INDEX'], password=app_config['REDIS_DB_PASSWORD']) class Tweet(): """Tweet class""" def __init__(self, username, post_unix_time, body): self.username = username self.post_datetime = datetime.datetime.fromtimestamp(post_unix_time) self.body = body
class DatacatCore(object): def __init__(self, config=None): self.config = Config() if config is not None: self.config.update(config) @property def db(self): if getattr(self, '_db', None) is None: self._db = connect(**self.config['DATABASE']) self._db.autocommit = False return self._db @property def admin_db(self): if getattr(self, '_admin_db', None) is None: self._admin_db = connect(**self.config['DATABASE']) self._admin_db.autocommit = True return self._admin_db def create_tables(self): create_tables(self.admin_db) def drop_tables(self): drop_tables(self.admin_db) # ------------------------------------------------------------ # Resource data CRUD # ------------------------------------------------------------ def resource_data_iter(self): """Iterate over resource attributes""" with self.db, self.db.cursor() as cur: cur.execute("SELECT * FROM resource_data") for row in cur.fetchall(): yield row def resource_data_create(self, stream, metadata=None, mimetype=None): """Create resource data from a stream""" resource_hash = hashlib.sha1() with self.db, self.db.cursor() as cur: lobj = self.db.lobject(oid=0, mode='wb') oid = lobj.oid for chunk in file_read_chunks(stream): lobj.write(chunk) resource_hash.update(chunk) lobj.close() data = { 'ctime': datetime.now(), 'mtime': datetime.now(), 'metadata': json.dumps(metadata), 'mimetype': mimetype or 'application/octet-stream', 'data_oid': oid, 'hash': 'sha1:{0}'.format(resource_hash.hexdigest()), } query = querybuilder.insert('resource_data', data) cur.execute(query, data) resource_data_id = cur.fetchone()[0] return resource_data_id def resource_data_get_info(self, objid): query = querybuilder.select_pk('resource_data') with self.db, self.db.cursor() as cur: cur.execute(query, {'id': objid}) return cur.fetchone() def resource_data_read(self, objid): resource = self.resource_data_get_info(objid) with self.db: lobj = self.db.lobject(oid=resource['data_oid'], mode='rb') return lobj.read() def resource_data_copy(self, objid, dest): resource = self.resource_data_get_info(objid) with self.db: lobj = self.db.lobject(oid=resource['data_oid'], mode='rb') for chunk in file_read_chunks(lobj): dest.write(chunk) def resource_data_update(self, objid, stream=None, metadata=None, mimetype=None): # Get the original object, to check for its existence # and to get the oid of the lobject holding the data. original = self.resource_data_get_info(objid) data = { 'id': objid, 'mtime': datetime.now(), } if metadata is not None: data['metadata'] = json.dumps(metadata) if mimetype is not None: data['mimetype'] = mimetype with self.db, self.db.cursor() as cur: if stream is not None: # Update the lobject with data from the stream resource_hash = hashlib.sha1() lobj = self.db.lobject(oid=original['data_oid'], mode='wb') for chunk in file_read_chunks(stream): lobj.write(chunk) resource_hash.update(chunk) lobj.close() data['hash'] = 'sha1:{0}'.format(resource_hash.hexdigest()) query = querybuilder.update('resource_data', data) cur.execute(query, data) def resource_data_remove(self, objid): # We need the OID to remove the lobject original = self.resource_data_get_info(objid) with self.db, self.db.cursor() as cur: lobject = db.lobject(oid=resource['data_oid'], mode='rb') data = lobject.read() lobject.close() cur.execute(querybuilder.delete('resource_data'), {'id': objid}) # ------------------------------------------------------------ # Dataset / resource CRUD # ------------------------------------------------------------ def create_resource(self, resource): return self._dsres_create('resource', resource) def update_resource(self, resource_id, resource): return self._dsres_update('resource', resource_id, resource) def get_resource(self, resource_id): return self._dsres_get('resource', resource_id) def list_resources(self, offset=None, limit=None): return self._dsres_list('resource', offset=offset, limit=limit) def delete_resource(self, resource_id): return self._dsres_delete('resource', resource_id) def create_dataset(self, dataset): return self._dsres_create('dataset', dataset) def update_dataset(self, dataset_id, dataset): return self._dsres_update('dataset', dataset_id, dataset) def get_dataset(self, dataset_id): return self._dsres_get('dataset', dataset_id) def list_datasets(self, offset=None, limit=None): return self._dsres_list('dataset', offset=offset, limit=limit) def delete_dataset(self, dataset_id): return self._dsres_delete('dataset', dataset_id) def add_dataset_resource(self, dataset_id, resource_id, order=0): data = { 'dataset_id': dataset_id, 'resource_id': resource_id, 'order': order } query = querybuilder.insert('dataset_resource', data=data, table_key=None) with self.db, self.db.cursor() as cur: cur.execute(query, data) def delete_dataset_resource(self, dataset_id, resource_id): data = {'dataset_id': dataset_id, 'resource_id': resource_id} query = ("DELETE FROM dataset_resource" " WHERE dataset_id=%(dataset_id)s" " AND resource_id=%(resource_id)s") with self.db, self.db.cursor() as cur: cur.execute(query, data) def move_dataset_resource(self, dataset_id, resource_id, order): data = { 'dataset_id': dataset_id, 'resource_id': resource_id, 'order': order } query = ("UPDATE dataset_resource" " SET order=%(order)s" " WHERE dataset_id=%(dataset_id)s" " AND resource_id=%(resource_id)s") with self.db, self.db.cursor() as cur: cur.execute(query, data) # ------------------------------------------------------------ # Rows and datasets have the same schema: # let's use some common functions for them.. # ------------------------------------------------------------ def _dsres_create(self, name, obj): data = { 'configuration': json.dumps(obj), 'ctime': datetime.now(), 'mtime': datetime.now(), } query = querybuilder.insert(name, data) with self.db, self.db.cursor() as cur: cur.execute(query, data) return cur.fetchone()[0] def _dsres_update(self, name, obj_id, obj): data = { 'id': obj_id, 'configuration': json.dumps(obj), 'mtime': datetime.now(), } query = querybuilder.update(name, data) with self.db, self.db.cursor() as cur: cur.execute(query, data) def _dsres_get(self, name, obj_id): query = querybuilder.select_pk(name) with self.db, self.db.cursor() as cur: cur.execute(query, dict(id=obj_id)) row = cur.fetchone() if row is None: raise NotFound() return self._row_to_obj(row) def _dsres_list(self, name, offset=None, limit=None): query = querybuilder.select_paged(name, offset=offset, limit=limit) with self.db, self.db.cursor() as cur: cur.execute(query) for row in cur.fetchall(): yield self._row_to_obj(row) def _dsres_from_row(self, row): obj = row['configuration'] obj['_id'] = row['id'] obj['_ctime'] = row['ctime'] obj['_mtime'] = row['mtime'] return obj def _dsres_delete(self, name, obj_id): query = querybuilder.delete(name) with self.db, self.db.cursor() as cur: cur.execute(query, dict(id=obj_id))
#!/usr/bin/env python import os import sys import time import logging from flask.config import Config import boto.sqs from boto.sqs.message import RawMessage from boto import exception LOG = logging.getLogger('alerta.sqs') config = Config('/') config.from_pyfile('/etc/alertad.conf', silent=True) config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True) DEFAULT_AWS_REGION = 'eu-west-1' DEFAULT_AWS_SQS_QUEUE = 'alerts' AWS_REGION = os.environ.get('AWS_REGION') or config.get('AWS_REGION', DEFAULT_AWS_REGION) AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get('AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get('AWS_SECRET_ACCESS_KEY') AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get('AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE) class Worker(object): def __init__(self):
from flask.config import Config from mabolab.core.global_obj import Global CENTRAL_CONFIG = 'C:/MTP/mabotech/maboss1.3.0/maboss/conf/maboss_config.py' settings = Config("") settings.from_pyfile(CENTRAL_CONFIG) settings['APP_NAME'] = "monitor_bli" g = Global(settings) db = g.get_db('postgresql') ora = g.get_db('oracle') log = g.get_logger() def dbtest(serialno): sql = """select status, lastupdateon from cob_t_serial_no_workstation where serialno = '%s' and workstation = '42700' order by id desc""" % (serialno) rtn = ora.execute(sql) print rtn.fetchone()
from flask.config import Config from mabolab.core.global_obj import Global CENTRAL_CONFIG = 'C:/MTP/mabotech/maboss1.3.0/maboss/conf/maboss_config.py' settings = Config("") settings.from_pyfile(CENTRAL_CONFIG) settings['APP_NAME'] = "monitor_bli" g = Global(settings) db = g.get_db('postgresql') ora = g.get_db('oracle') log = g.get_logger() def dbtest(serialno): sql = """select status, lastupdateon from cob_t_serial_no_workstation where serialno = '%s' and workstation = '42700' order by id desc""" % (serialno) rtn = ora.execute(sql)
#!/usr/bin/env python import os import sys import time import logging from flask.config import Config import boto.sqs from boto.sqs.message import RawMessage from boto import exception LOG = logging.getLogger('alerta.sqs') config = Config('/') config.from_pyfile('/etc/alertad.conf', silent=True) config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True) DEFAULT_AWS_REGION = 'eu-west-1' DEFAULT_AWS_SQS_QUEUE = 'alerts' AWS_REGION = os.environ.get('AWS_REGION') or config.get( 'AWS_REGION', DEFAULT_AWS_REGION) AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get( 'AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get( 'AWS_SECRET_ACCESS_KEY') AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get( 'AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE)
def make_config(): cfg = Config('') cfg.from_object('datacat.settings.default') cfg.from_envvar('DATACAT_SETTINGS', silent=True) return cfg