def initialize_service(self, config: Config): logger.info( f'''Connection details: {config.get('REDIS_HOST')}:{config.get('REDIS_PORT')}''' ) self.__redis_connection = Redis(host=config.get('REDIS_HOST'), port=config.get('REDIS_PORT'), password=config.get('REDIS_P'), health_check_interval=15)
def get_neptune_graph_traversal_source_factory_from_config( config: Config) -> Callable[[], GraphTraversalSource]: session = config.get('NEPTUNE_SESSION') assert session is not None neptune_url = config.get('NEPTUNE_URL') assert neptune_url is not None return get_neptune_graph_traversal_source_factory(neptune_url=neptune_url, session=session)
def create( self, telegram_service: telegram.TelegramService, links: model_link.Links, extractor: services.TitleExtractor, config: flask.Config, ) -> incoming.Incoming: return incoming.Incoming(telegram_service, links, extractor, config.get("PROJECT_ID"))
def create(self, config: flask.Config) -> datastore.Client: return datastore.Client(config.get("PROJECT_ID"))
def create(self, config: flask.Config) -> telegram.TelegramService: return telegram.TelegramService(config.get("TG_URL"))
class Application(swm.servers.Application): """ The API application. """ def __init__(self, env): # Load settings self.config = Config(os.path.dirname(os.path.realpath(__file__))) self.config.from_object(f'settings.servers.{env}.Config') # Sentry (logging) if self.config.get('SENTRY_DSN'): sentry_logging = LoggingIntegration(level=logging.INFO, event_level=logging.WARNING) self.sentry = sentry_sdk.init(self.config.get('SENTRY_DSN'), integrations=[ sentry_logging, TornadoIntegration(), RedisIntegration() ]) # Initialize application super().__init__( [ # Assets (collection) (r'/assets', api.assets.CollectionHandler), (r'/assets/analyze', api.assets.AnalyzeManyHandler), (r'/assets/expire', api.assets.ExpireManyHandler), (r'/assets/persist', api.assets.PersistManyHandler), (r'/assets/transform', api.assets.variations.TransformManyHandler), # Assets (document) (r'/assets/(\w+)', api.assets.DocumentHandler), (r'/assets/(\w+)/analyze', api.assets.AnalyzeHandler), (r'/assets/(\w+)/download', api.assets.DownloadHandler), (r'/assets/(\w+)/expire', api.assets.ExpireHandler), (r'/assets/(\w+)/persist', api.assets.PersistHandler), # Assets > Variations (r'/assets/(\w+)/variations', api.assets.variations.CollectionHandler), (r'/assets/(\w+)/variations/(\w+)', api.assets.variations.DocumentHandler), (r'/assets/(\w+)/variations/(\w+)/download', api.assets.variations.DownloadHandler) ], debug=self.config.get('DEBUG'), default_handler_class=api.APIErrorHandler, default_handler_args={'status_code': 404}) loop = asyncio.get_event_loop() # Set up redis connections self._redis = loop.run_until_complete(self._get_redis(loop)) self._redis_sub = loop.run_until_complete(self._get_redis(loop)) # Set up the event listener loop.run_until_complete( self.listen_for_task_events(self._redis_sub, 'h51_events')) # Mongo database self.mongo = pymongo.MongoClient(self.config.get('MONGO_URI')) self.db = self.mongo.get_default_database() mongoframes.Frame._client = self.mongo if self.config.get('MONGO_PASSWORD'): self.db.authenticate(self.config.get('MONGO_USERNAME'), self.config.get('MONGO_PASSWORD')) async def _get_redis(self, loop): if self.config['REDIS_USE_SENTINEL']: sentinel = await aioredis.create_sentinel( self.config.get('REDIS_ADDRESS'), db=self.config.get('REDIS_DB'), password=self.config['REDIS_PASSWORD'], loop=loop) return sentinel.master_for(self.config['REDIS_SENTINEL_MASTER']) else: return await aioredis.create_redis_pool( self.config.get('REDIS_ADDRESS'), db=self.config.get('REDIS_DB'), password=self.config['REDIS_PASSWORD'], loop=loop)
# -*- coding: utf-8 -*- """Extensions module. Each extension is initialized in the apps factory located in apps/__init__.py.""" import os from flask import Config from flask_caching import Cache from flask_seasurf import SeaSurf config = Config(root_path='') config.from_object(os.getenv('FLASK_CONFIG') or 'config') cache = Cache(config=config.get('CACHE')) csrf = SeaSurf() SECRET_KEY = config.get('SECRET_KEY', 'super_secret_key') if config.get('REDIS_URL', None): from redis import StrictRedis db_redis = StrictRedis.from_url( config.get("REDIS_URL", "redis://*****:*****@" + release_version sentry_sdk.init(dsn=config.get('SENTRY_DSL', ''), integrations=[FlaskIntegration()],
engine = None sessionmaker = sa.orm.sessionmaker() session = sa.orm.scoped_session(sessionmaker) def configure_engine(url): global sessionmaker, engine, session engine = sa.create_engine(url) session.remove() sessionmaker.configure(bind=engine) _config = Config('') _config.from_object('config') _config.from_envvar('MULCHN_CONFIG', silent=True) configure_engine(_config.get('DATABASE_URL')) class _Base(object): @declared_attr def __tablename__(cls): """ Convert CamelCase class name to underscores_between_words table name. """ name = cls.__name__ return ( name[0].lower() + re.sub(r'([A-Z])', lambda m:"_" + m.group(0).lower(), name[1:]) )
class Configurator(object): """ Object that takes care of loading the different configurations from the different sources. There are 3 types of settings: * Project: The basic set of settings needed by the system. These are shipped with Shiva. * Local: Specific to each instance, useful for overwriting system settings. Some of them must be defined before running Shiva, like the DB URI. * Debug: This setting will only be loaded if ``DEBUG`` is set to True in the local settings. There are also 3 different places where Shiva looks for this config files: * A ``local.py`` file inside the ``config/`` directory, relative to Shiva. * The ``$SHIVA_CONFIG`` environment variable. It's assumed to be pointing to a file (not a dir) if exists. * The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If ``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as defined by the `XDG Base Directory Specification <http://standards.freedesktop.org/basedir-spec/basedir-spec-latest\ .html>`_. """ def __init__(self): self._config = FlaskConfig('') # project _project = self.load_project() # local _xdg_config = self.from_xdg_config() _env = self.from_env() _local = self.from_local() # debug _debug = self.load_debug() self.extract_conf(self._config) if not (_xdg_config or _env or _local): raise NoConfigFoundError def load_project(self): return self._config.from_object(project) def get_xdg_path(self): default_config_home = os.path.join(os.getenv('HOME'), '.config') return os.getenv('XDG_CONFIG_HOME') or default_config_home def from_xdg_config(self): local_py = os.path.join(self.get_xdg_path(), 'shiva/config.py') if not os.path.exists(local_py): return False return self._config.from_pyfile(local_py) def from_env(self): if not os.getenv('SHIVA_CONFIG'): return False return self._config.from_envvar('SHIVA_CONFIG') def from_local(self): with ignored(ImportError): self._config.from_object('shiva.config.local') return True return False def load_debug(self): if not self._config.get('DEBUG'): return False loaded = False with ignored(ImportError): from shiva.config import debug loaded = self._config.from_object(debug) debug_py = os.path.join(self.get_xdg_path(), 'shiva/debug.py') if not os.path.exists(debug_py): return False return self._config.from_pyfile(debug_py) or loaded def extract_conf(self, *args): """ Receives one or more objects, iterates over their elements, extracts all the uppercase properties and injects them into the Configurator object. """ for obj in args: for key in dir(obj): if key.isupper(): setattr(self, key, getattr(obj, key)) if hasattr(obj, 'iterkeys') and callable(getattr(obj, 'iterkeys')): for key in obj.iterkeys(): if key.isupper(): setattr(self, key, obj[key])
class Configurator(object): """ Object that takes care of loading the different configurations from the different sources. There are 3 types of settings: * Project: The basic set of settings needed by the system. These are shipped with Shiva. * Local: Specific to each instance, useful for overwriting system settings. Some of them must be defined before running Shiva, like the DB URI. * Debug: This setting will only be loaded if ``DEBUG`` is set to True in the local settings. There are also 3 different places where Shiva looks for this config files: * A ``local.py`` file inside the ``config/`` directory, relative to Shiva. * The ``$SHIVA_CONFIG`` environment variable. It's assumed to be pointing to a file (not a dir) if exists. * The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If ``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as defined by the `XDG Base Directory Specification <http://standards.freedesktop.org/basedir-spec/basedir-spec-latest\ .html>`_. """ def __init__(self): self._config = FlaskConfig("") # project _project = self.load_project() # local _xdg_config = self.from_xdg_config() _env = self.from_env() _local = self.from_local() # debug _debug = self.load_debug() self.extract_conf(self._config) if not (_xdg_config or _env or _local): raise NoConfigFoundError def load_project(self): return self._config.from_object(project) def get_xdg_path(self): path_home = os.getenv("HOME") if not path_home: return None default_config_home = os.path.join(path_home, ".config") return os.getenv("XDG_CONFIG_HOME") or default_config_home def from_xdg_config(self): xdg_path = self.get_xdg_path() if not xdg_path: return False local_py = os.path.join(xdg_path, "shiva/config.py") if not os.path.exists(local_py): return False return self._config.from_pyfile(local_py) def from_env(self): if not os.getenv("SHIVA_CONFIG"): return False return self._config.from_envvar("SHIVA_CONFIG") def from_local(self): with ignored(ImportError): self._config.from_object("shiva.config.local") return True return False def load_debug(self): if not self._config.get("DEBUG"): return False loaded = False with ignored(ImportError): from shiva.config import debug loaded = self._config.from_object(debug) xdg_path = self.get_xdg_path() if not xdg_path: return False debug_py = os.path.join(xdg_path, "shiva/debug.py") if not os.path.exists(debug_py): return False return self._config.from_pyfile(debug_py) or loaded def extract_conf(self, *args): """ Receives one or more objects, iterates over their elements, extracts all the uppercase properties and injects them into the Configurator object. """ for obj in args: for key in dir(obj): if key.isupper(): setattr(self, key, getattr(obj, key)) if hasattr(obj, "iterkeys") and callable(getattr(obj, "iterkeys")): for key in obj.iterkeys(): if key.isupper(): setattr(self, key, obj[key])
def make_celery_redis_url(config: flask.Config, *, envvar: str) -> str: if var := config.get(envvar): return var
def create(self, config: flask.Config) -> DirectDriver: uri_db = config.get(URI_KEY) user_db = config.get(USER_KEY) password_db = config.get(PASSWORD_KEY) return GraphDatabase.driver(uri_db, auth=(user_db, password_db))
import sqlalchemy from gunrskite import parser as gparse from gunrskite import consumer as consumer from flask import Config cfg = Config(os.path.abspath(".")) cfg.from_pyfile("config.py") loop = asyncio.get_event_loop() formatter = logging.Formatter('%(asctime)s - [%(levelname)s] %(name)s -> %(message)s') root = logging.getLogger() root.setLevel(cfg.get("LOG_LEVEL", logging.INFO)) consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(formatter) root.addHandler(consoleHandler) logger = logging.getLogger("Gunrskite::Listener") logging.getLogger("sqlalchemy").setLevel(cfg.get("SQLALCHEMY_LOG_LEVEL", logging.CRITICAL)) # Load database from gunrskite import db session = db.create_sess()
def create( self, config: flask.Config, ) -> redis.StrictRedis: return redis.StrictRedis(host=config.get('redis_host'), port=config.get('redis_port'))
import re import redis import urlparse from flask import Blueprint, Config, flash, g, jsonify, request, render_template from flask import session, url_for from flask_bibframe.models import CoverArt blueprint_folder = os.path.abspath(os.path.dirname(__file__)) app_folder = os.path.split(blueprint_folder)[0] metrics_config = Config(app_folder) metrics_config.from_pyfile('catalog.cfg') redis_ds = redis.StrictRedis(metrics_config.get('REDIS_HOST')) def parse_logfile(log_path, redis_ds=redis_ds, tz='-0700'): """Takes a nginx access log, opens the file, and parses through and creates metrics for use by other Blueprints and apps. Parameters ---------- log_path : str Full path and filename of the nginx access log redis_ds : StrictRedis instance defaults to Redis datastore for CPP's metrics tz : str Timezone offset, defaults to -0700 """
from celery import Celery from flask import Config config_name = 'coding_challenge_restful.settings.Config' config = Config("") config.from_object(config_name) CELERY_CONFIG = dict( task_serializer='json', accept_content=['json'], result_serializer='json', enable_utc=True, broker_url=config.get('CLOUDAMQP_URL'), broker_pool_limit=1, # Will decrease connection usage broker_heartbeat=None, # We're using TCP keep-alive instead broker_connection_timeout=30, # May require a long timeout due to Linux DNS timeouts etc result_backend=None, # AMQP is not recommended as result backend as it creates thousands of queues event_queue_expires=60, # Will delete all celeryev. queues without consumers after 1 minute. worker_prefetch_multiplier=1, # Disable prefetching, it's causes problems and doesn't help performance worker_concurrency=50, # If you tasks are CPU bound, then limit to the number of cores, otherwise increase substainally ) celery_app = Celery('task_csv_import', backend=config.get("CELERY_BACKEND"), broker=config.get('CLOUDAMQP_URL')) celery_app.conf.update(**CELERY_CONFIG)
class AssetWorker(BaseWorker): """ A worker for performing asset tasks (running analyzers and filters). """ def __init__(self, env, idle_lifespan): # Load settings self.config = Config(os.path.dirname(os.path.realpath(__file__))) # There may be overiding settings specific to the server we are running on servername = socket.gethostname().split('.')[0] if servername and os.path.isfile(f'settings/workers/{env}_{servername}.py'): self.config.from_object(f'settings.workers.{env}_{servername}.Config') else: self.config.from_object(f'settings.workers.{env}.Config') # Sentry (logging) if self.config.get('SENTRY_DSN'): sentry_logging = LoggingIntegration( level=logging.INFO, event_level=logging.WARNING ) self.sentry = sentry_sdk.init( self.config.get('SENTRY_DSN'), integrations=[ sentry_logging, RedisIntegration() ] ) # Mongo database self.mongo = pymongo.MongoClient(self.config.get('MONGO_URI')) self.db = self.mongo.get_default_database() mongoframes.Frame._client = self.mongo if self.config.get('MONGO_PASSWORD'): self.db.authenticate( self.config.get('MONGO_USERNAME'), self.config.get('MONGO_PASSWORD') ) # Redis if self.config['REDIS_USE_SENTINEL']: sentinel = redis.sentinel.Sentinel( self.config['REDIS_ADDRESS'], db=self.config['REDIS_DB'], password=self.config['REDIS_PASSWORD'], decode_responses=True ) conn = sentinel.master_for(self.config['REDIS_SENTINEL_MASTER']) else: conn = redis.StrictRedis( host=self.config['REDIS_ADDRESS'][0], port=self.config['REDIS_ADDRESS'][1], db=self.config['REDIS_DB'], password=self.config['REDIS_PASSWORD'], decode_responses=True ) super().__init__( conn, [AnalyzeTask, GenerateVariationTask], broadcast_channel='h51_events', max_status_interval=self.config['ASSET_WORKER_MAX_STATUS_INTERVAL'], max_spawn_time=self.config['ASSET_WORKER_MAX_SPAWN_TIME'], sleep_interval=self.config['ASSET_WORKER_SLEEP_INTERVAL'], idle_lifespan=idle_lifespan, population_control=self.config['ASSET_WORKER_POPULATION_CONTROL'], population_spawner=self.config['ASSET_WORKER_POPULATION_SPAWNER'] ) def analyze(self, task): """Analyze an asset""" asset = task.get_asset( projection={ 'variations': {'$sub.': Variation} } ) file = task.get_file() history = [] for analyzer in task.get_analyzers(asset): analyzer.analyze(self.config, asset, file, history) history.append(analyzer) if task.notification_url: # POST the result to the notification URL account = Account.by_id( asset.account, projection={'api_key': True} ) task.post_notification( account.api_key, json.dumps(asset.to_json_type()) ) return {} def do_task(self, task): if isinstance(task, AnalyzeTask): return self.analyze(task) elif isinstance(task, GenerateVariationTask): return self.generate_variation(task) def generate_variation(self, task): """Generate variation for the asset""" asset = task.get_asset( projection={ 'variations': { '$sub.': Variation } } ) file = task.get_file() native_file = None history = [] for transform in task.get_transforms(asset): native_file = transform.transform( self.config, asset, file, task.variation_name, native_file, history ) history.append(transform) if task.notification_url: # POST the result to the notification URL account = Account.by_id( asset.account, projection={'api_key': True} ) task.post_notification( account.api_key, json.dumps(asset.to_json_type()) ) return {} def get_tasks(self): tasks = super().get_tasks() pairs = list(tasks.items()) # Randomize tasks to prevent large tasks hogging all the workers random.shuffle(pairs) tasks = dict(pairs) return tasks def on_error(self, task_id, error): super().on_error(task_id, error) if self.config['DEBUG']: traceback.print_exc() print(error) else: if self.config.get('SENTRY_DSN'): sentry_sdk.capture_exception(error) def on_spawn_error(self, error): if self.config['DEBUG']: super().on_spawn_error(error) else: if self.config.get('SENTRY_DSN'): sentry_sdk.capture_exception(error) @classmethod def get_id_prefix(cls): return 'h51_asset_worker'
import json import sys import time from RPi import GPIO from flask import Config from GetTemperature import read_temp from SimpleDaemon import Daemon from thermostat_controls import air_conditioning, system_off config = Config('config') FAN_PIN = config.get('FAN_PIN') HEATER_PIN = config.get('HEATER_PIN') AC_PIN = config.get('AC_PIN') check_time = 15 def get_target_temp(): with open('settings.json', 'r') as f: data = json.load(f) return data['TARGET_TEMP'] def get_current_mode(): with open('settings.json', 'r') as f: data = json.load(f) return data['TARGET_MODE']
def main(argconfig): # Load the config file the same way Flask does which Chill uses. config_file = argconfig if argconfig[0] == os.sep else os.path.join(os.getcwd(), argconfig) config = Config(os.getcwd()) config.from_pyfile(config_file) pyrax.set_credential_file(config["RACKSPACE_CREDENTIAL_FILE"]) cf = pyrax.cloudfiles # sync sync_folder_to_container local = config["FREEZER_DESTINATION"] container_name = config.get("RACKSPACE_CONTAINER_NAME", os.path.basename(os.getcwd())) prefix = config.get("PUBLIC_URL_PREFIX", None) if prefix: for page in (INDEX_PAGE, ERROR_PAGE, "401{0}".format(ERROR_PAGE), "404{0}".format(ERROR_PAGE)): # If no index.html or error pages then link to the one found in # PUBLIC_URL_PREFIX if not os.path.exists(os.path.join(local, page)) and os.path.exists(os.path.join(local, prefix[1:], page)): print "Creating hard link for {0}".format(page) print "{0} -> {1}".format(os.path.join(local, page), os.path.join(local, prefix[1:], page)) os.link(os.path.join(local, prefix[1:], page), os.path.join(local, page)) confirm = raw_input( "\nsync the folder: {local} \nto rackspace cloudfiles container: {container_name}\n[n]/y\n".format(**locals()) ) if confirm != "y": return remote = cf.create_container(container_name) local_files = set() for root, dir, files in os.walk(local): for f in files: local_files.add(os.path.join(root[len(local) + 1 :], f)) cf.sync_folder_to_container(local, remote) # Mark all objects on remote to be deleted if not in local # The limit here is arbitrary, but can not be higher then 10000. limit = 1000 marker = "" remote_objects_list = remote.get_objects(limit=limit, marker=marker) while remote_objects_list: marker = remote_objects_list[-1].name for obj in remote_objects_list: if obj.name not in local_files: obj.delete_in_seconds(DELETE_OBJECTS_DELAY) remote_objects_list = remote.get_objects(limit=limit, marker=marker) # publish cf.make_container_public(container_name, ttl=TTL) # cdn website remote.set_web_index_page(INDEX_PAGE) remote.set_web_error_page(ERROR_PAGE) # Totally copied from the docs print print "After Making Public" print "cdn_enabled", remote.cdn_enabled print "cdn_ttl", remote.cdn_ttl print "cdn_log_retention", remote.cdn_log_retention print "cdn_uri", remote.cdn_uri print "cdn_ssl_uri", remote.cdn_ssl_uri print "cdn_streaming_uri", remote.cdn_streaming_uri print "cdn_ios_uri", remote.cdn_ios_uri