def initialize_service(self, config: Config):
logger.info(
f'''Connection details: {config.get('REDIS_HOST')}:{config.get('REDIS_PORT')}'''
)
self.__redis_connection = Redis(host=config.get('REDIS_HOST'),
port=config.get('REDIS_PORT'),
password=config.get('REDIS_P'),
health_check_interval=15)
def get_neptune_graph_traversal_source_factory_from_config(
config: Config) -> Callable[[], GraphTraversalSource]:
session = config.get('NEPTUNE_SESSION')
assert session is not None
neptune_url = config.get('NEPTUNE_URL')
assert neptune_url is not None
return get_neptune_graph_traversal_source_factory(neptune_url=neptune_url,
session=session)
def create(
self,
telegram_service: telegram.TelegramService,
links: model_link.Links,
extractor: services.TitleExtractor,
config: flask.Config,
) -> incoming.Incoming:
return incoming.Incoming(telegram_service, links, extractor,
config.get("PROJECT_ID"))
def create(self, config: flask.Config) -> datastore.Client:
return datastore.Client(config.get("PROJECT_ID"))
def create(self, config: flask.Config) -> telegram.TelegramService:
return telegram.TelegramService(config.get("TG_URL"))
class Application(swm.servers.Application):
"""
The API application.
"""
def __init__(self, env):
# Load settings
self.config = Config(os.path.dirname(os.path.realpath(__file__)))
self.config.from_object(f'settings.servers.{env}.Config')
# Sentry (logging)
if self.config.get('SENTRY_DSN'):
sentry_logging = LoggingIntegration(level=logging.INFO,
event_level=logging.WARNING)
self.sentry = sentry_sdk.init(self.config.get('SENTRY_DSN'),
integrations=[
sentry_logging,
TornadoIntegration(),
RedisIntegration()
])
# Initialize application
super().__init__(
[
# Assets (collection)
(r'/assets', api.assets.CollectionHandler),
(r'/assets/analyze', api.assets.AnalyzeManyHandler),
(r'/assets/expire', api.assets.ExpireManyHandler),
(r'/assets/persist', api.assets.PersistManyHandler),
(r'/assets/transform',
api.assets.variations.TransformManyHandler),
# Assets (document)
(r'/assets/(\w+)', api.assets.DocumentHandler),
(r'/assets/(\w+)/analyze', api.assets.AnalyzeHandler),
(r'/assets/(\w+)/download', api.assets.DownloadHandler),
(r'/assets/(\w+)/expire', api.assets.ExpireHandler),
(r'/assets/(\w+)/persist', api.assets.PersistHandler),
# Assets > Variations
(r'/assets/(\w+)/variations',
api.assets.variations.CollectionHandler),
(r'/assets/(\w+)/variations/(\w+)',
api.assets.variations.DocumentHandler),
(r'/assets/(\w+)/variations/(\w+)/download',
api.assets.variations.DownloadHandler)
],
debug=self.config.get('DEBUG'),
default_handler_class=api.APIErrorHandler,
default_handler_args={'status_code': 404})
loop = asyncio.get_event_loop()
# Set up redis connections
self._redis = loop.run_until_complete(self._get_redis(loop))
self._redis_sub = loop.run_until_complete(self._get_redis(loop))
# Set up the event listener
loop.run_until_complete(
self.listen_for_task_events(self._redis_sub, 'h51_events'))
# Mongo database
self.mongo = pymongo.MongoClient(self.config.get('MONGO_URI'))
self.db = self.mongo.get_default_database()
mongoframes.Frame._client = self.mongo
if self.config.get('MONGO_PASSWORD'):
self.db.authenticate(self.config.get('MONGO_USERNAME'),
self.config.get('MONGO_PASSWORD'))
async def _get_redis(self, loop):
if self.config['REDIS_USE_SENTINEL']:
sentinel = await aioredis.create_sentinel(
self.config.get('REDIS_ADDRESS'),
db=self.config.get('REDIS_DB'),
password=self.config['REDIS_PASSWORD'],
loop=loop)
return sentinel.master_for(self.config['REDIS_SENTINEL_MASTER'])
else:
return await aioredis.create_redis_pool(
self.config.get('REDIS_ADDRESS'),
db=self.config.get('REDIS_DB'),
password=self.config['REDIS_PASSWORD'],
loop=loop)
# -*- coding: utf-8 -*-
"""Extensions module. Each extension is initialized in the apps factory located in apps/__init__.py."""
import os
from flask import Config
from flask_caching import Cache
from flask_seasurf import SeaSurf
config = Config(root_path='')
config.from_object(os.getenv('FLASK_CONFIG') or 'config')
cache = Cache(config=config.get('CACHE'))
csrf = SeaSurf()
SECRET_KEY = config.get('SECRET_KEY', 'super_secret_key')
if config.get('REDIS_URL', None):
from redis import StrictRedis
db_redis = StrictRedis.from_url(
config.get("REDIS_URL", "redis://*****:*****@" + release_version
sentry_sdk.init(dsn=config.get('SENTRY_DSL', ''),
integrations=[FlaskIntegration()],
engine = None
sessionmaker = sa.orm.sessionmaker()
session = sa.orm.scoped_session(sessionmaker)
def configure_engine(url):
global sessionmaker, engine, session
engine = sa.create_engine(url)
session.remove()
sessionmaker.configure(bind=engine)
_config = Config('')
_config.from_object('config')
_config.from_envvar('MULCHN_CONFIG', silent=True)
configure_engine(_config.get('DATABASE_URL'))
class _Base(object):
@declared_attr
def __tablename__(cls):
"""
Convert CamelCase class name to underscores_between_words
table name.
"""
name = cls.__name__
return (
name[0].lower() +
re.sub(r'([A-Z])', lambda m:"_" + m.group(0).lower(), name[1:])
)
class Configurator(object):
"""
Object that takes care of loading the different configurations from the
different sources. There are 3 types of settings:
* Project: The basic set of settings needed by the system. These are
shipped with Shiva.
* Local: Specific to each instance, useful for overwriting system
settings. Some of them must be defined before running Shiva, like the
DB URI.
* Debug: This setting will only be loaded if ``DEBUG`` is set to True
in the local settings.
There are also 3 different places where Shiva looks for this config files:
* A ``local.py`` file inside the ``config/`` directory, relative to
Shiva.
* The ``$SHIVA_CONFIG`` environment variable. It's assumed to be
pointing to a file (not a dir) if exists.
* The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If
``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as
defined by the `XDG Base Directory Specification
<http://standards.freedesktop.org/basedir-spec/basedir-spec-latest\
.html>`_.
"""
def __init__(self):
self._config = FlaskConfig('')
# project
_project = self.load_project()
# local
_xdg_config = self.from_xdg_config()
_env = self.from_env()
_local = self.from_local()
# debug
_debug = self.load_debug()
self.extract_conf(self._config)
if not (_xdg_config or _env or _local):
raise NoConfigFoundError
def load_project(self):
return self._config.from_object(project)
def get_xdg_path(self):
default_config_home = os.path.join(os.getenv('HOME'), '.config')
return os.getenv('XDG_CONFIG_HOME') or default_config_home
def from_xdg_config(self):
local_py = os.path.join(self.get_xdg_path(), 'shiva/config.py')
if not os.path.exists(local_py):
return False
return self._config.from_pyfile(local_py)
def from_env(self):
if not os.getenv('SHIVA_CONFIG'):
return False
return self._config.from_envvar('SHIVA_CONFIG')
def from_local(self):
with ignored(ImportError):
self._config.from_object('shiva.config.local')
return True
return False
def load_debug(self):
if not self._config.get('DEBUG'):
return False
loaded = False
with ignored(ImportError):
from shiva.config import debug
loaded = self._config.from_object(debug)
debug_py = os.path.join(self.get_xdg_path(), 'shiva/debug.py')
if not os.path.exists(debug_py):
return False
return self._config.from_pyfile(debug_py) or loaded
def extract_conf(self, *args):
"""
Receives one or more objects, iterates over their elements, extracts
all the uppercase properties and injects them into the Configurator
object.
"""
for obj in args:
for key in dir(obj):
if key.isupper():
setattr(self, key, getattr(obj, key))
if hasattr(obj, 'iterkeys') and callable(getattr(obj, 'iterkeys')):
for key in obj.iterkeys():
if key.isupper():
setattr(self, key, obj[key])
class Configurator(object):
"""
Object that takes care of loading the different configurations from the
different sources. There are 3 types of settings:
* Project: The basic set of settings needed by the system. These are
shipped with Shiva.
* Local: Specific to each instance, useful for overwriting system
settings. Some of them must be defined before running Shiva, like the
DB URI.
* Debug: This setting will only be loaded if ``DEBUG`` is set to True
in the local settings.
There are also 3 different places where Shiva looks for this config files:
* A ``local.py`` file inside the ``config/`` directory, relative to
Shiva.
* The ``$SHIVA_CONFIG`` environment variable. It's assumed to be
pointing to a file (not a dir) if exists.
* The ``$XDG_CONFIG_HOME/shiva/config.py`` file. If
``$XDG_CONFIG_HOME`` is not set, defaults to ``$HOME/.config``, as
defined by the `XDG Base Directory Specification
<http://standards.freedesktop.org/basedir-spec/basedir-spec-latest\
.html>`_.
"""
def __init__(self):
self._config = FlaskConfig("")
# project
_project = self.load_project()
# local
_xdg_config = self.from_xdg_config()
_env = self.from_env()
_local = self.from_local()
# debug
_debug = self.load_debug()
self.extract_conf(self._config)
if not (_xdg_config or _env or _local):
raise NoConfigFoundError
def load_project(self):
return self._config.from_object(project)
def get_xdg_path(self):
path_home = os.getenv("HOME")
if not path_home:
return None
default_config_home = os.path.join(path_home, ".config")
return os.getenv("XDG_CONFIG_HOME") or default_config_home
def from_xdg_config(self):
xdg_path = self.get_xdg_path()
if not xdg_path:
return False
local_py = os.path.join(xdg_path, "shiva/config.py")
if not os.path.exists(local_py):
return False
return self._config.from_pyfile(local_py)
def from_env(self):
if not os.getenv("SHIVA_CONFIG"):
return False
return self._config.from_envvar("SHIVA_CONFIG")
def from_local(self):
with ignored(ImportError):
self._config.from_object("shiva.config.local")
return True
return False
def load_debug(self):
if not self._config.get("DEBUG"):
return False
loaded = False
with ignored(ImportError):
from shiva.config import debug
loaded = self._config.from_object(debug)
xdg_path = self.get_xdg_path()
if not xdg_path:
return False
debug_py = os.path.join(xdg_path, "shiva/debug.py")
if not os.path.exists(debug_py):
return False
return self._config.from_pyfile(debug_py) or loaded
def extract_conf(self, *args):
"""
Receives one or more objects, iterates over their elements, extracts
all the uppercase properties and injects them into the Configurator
object.
"""
for obj in args:
for key in dir(obj):
if key.isupper():
setattr(self, key, getattr(obj, key))
if hasattr(obj, "iterkeys") and callable(getattr(obj, "iterkeys")):
for key in obj.iterkeys():
if key.isupper():
setattr(self, key, obj[key])
def make_celery_redis_url(config: flask.Config, *, envvar: str) -> str:
if var := config.get(envvar):
return var
def create(self, config: flask.Config) -> DirectDriver:
uri_db = config.get(URI_KEY)
user_db = config.get(USER_KEY)
password_db = config.get(PASSWORD_KEY)
return GraphDatabase.driver(uri_db, auth=(user_db, password_db))
import sqlalchemy
from gunrskite import parser as gparse
from gunrskite import consumer as consumer
from flask import Config
cfg = Config(os.path.abspath("."))
cfg.from_pyfile("config.py")
loop = asyncio.get_event_loop()
formatter = logging.Formatter('%(asctime)s - [%(levelname)s] %(name)s -> %(message)s')
root = logging.getLogger()
root.setLevel(cfg.get("LOG_LEVEL", logging.INFO))
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(formatter)
root.addHandler(consoleHandler)
logger = logging.getLogger("Gunrskite::Listener")
logging.getLogger("sqlalchemy").setLevel(cfg.get("SQLALCHEMY_LOG_LEVEL", logging.CRITICAL))
# Load database
from gunrskite import db
session = db.create_sess()
def create(
self,
config: flask.Config,
) -> redis.StrictRedis:
return redis.StrictRedis(host=config.get('redis_host'),
port=config.get('redis_port'))
import re
import redis
import urlparse
from flask import Blueprint, Config, flash, g, jsonify, request, render_template
from flask import session, url_for
from flask_bibframe.models import CoverArt
blueprint_folder = os.path.abspath(os.path.dirname(__file__))
app_folder = os.path.split(blueprint_folder)[0]
metrics_config = Config(app_folder)
metrics_config.from_pyfile('catalog.cfg')
redis_ds = redis.StrictRedis(metrics_config.get('REDIS_HOST'))
def parse_logfile(log_path, redis_ds=redis_ds, tz='-0700'):
"""Takes a nginx access log, opens the file, and parses through and creates
metrics for use by other Blueprints and apps.
Parameters
----------
log_path : str
Full path and filename of the nginx access log
redis_ds : StrictRedis instance
defaults to Redis datastore for CPP's metrics
tz : str
Timezone offset, defaults to -0700
"""
from celery import Celery
from flask import Config
config_name = 'coding_challenge_restful.settings.Config'
config = Config("")
config.from_object(config_name)
CELERY_CONFIG = dict(
task_serializer='json',
accept_content=['json'],
result_serializer='json',
enable_utc=True,
broker_url=config.get('CLOUDAMQP_URL'),
broker_pool_limit=1, # Will decrease connection usage
broker_heartbeat=None, # We're using TCP keep-alive instead
broker_connection_timeout=30, # May require a long timeout due to Linux DNS timeouts etc
result_backend=None, # AMQP is not recommended as result backend as it creates thousands of queues
event_queue_expires=60, # Will delete all celeryev. queues without consumers after 1 minute.
worker_prefetch_multiplier=1, # Disable prefetching, it's causes problems and doesn't help performance
worker_concurrency=50,
# If you tasks are CPU bound, then limit to the number of cores, otherwise increase substainally
)
celery_app = Celery('task_csv_import', backend=config.get("CELERY_BACKEND"), broker=config.get('CLOUDAMQP_URL'))
celery_app.conf.update(**CELERY_CONFIG)
class AssetWorker(BaseWorker):
"""
A worker for performing asset tasks (running analyzers and filters).
"""
def __init__(self, env, idle_lifespan):
# Load settings
self.config = Config(os.path.dirname(os.path.realpath(__file__)))
# There may be overiding settings specific to the server we are running on
servername = socket.gethostname().split('.')[0]
if servername and os.path.isfile(f'settings/workers/{env}_{servername}.py'):
self.config.from_object(f'settings.workers.{env}_{servername}.Config')
else:
self.config.from_object(f'settings.workers.{env}.Config')
# Sentry (logging)
if self.config.get('SENTRY_DSN'):
sentry_logging = LoggingIntegration(
level=logging.INFO,
event_level=logging.WARNING
)
self.sentry = sentry_sdk.init(
self.config.get('SENTRY_DSN'),
integrations=[
sentry_logging,
RedisIntegration()
]
)
# Mongo database
self.mongo = pymongo.MongoClient(self.config.get('MONGO_URI'))
self.db = self.mongo.get_default_database()
mongoframes.Frame._client = self.mongo
if self.config.get('MONGO_PASSWORD'):
self.db.authenticate(
self.config.get('MONGO_USERNAME'),
self.config.get('MONGO_PASSWORD')
)
# Redis
if self.config['REDIS_USE_SENTINEL']:
sentinel = redis.sentinel.Sentinel(
self.config['REDIS_ADDRESS'],
db=self.config['REDIS_DB'],
password=self.config['REDIS_PASSWORD'],
decode_responses=True
)
conn = sentinel.master_for(self.config['REDIS_SENTINEL_MASTER'])
else:
conn = redis.StrictRedis(
host=self.config['REDIS_ADDRESS'][0],
port=self.config['REDIS_ADDRESS'][1],
db=self.config['REDIS_DB'],
password=self.config['REDIS_PASSWORD'],
decode_responses=True
)
super().__init__(
conn,
[AnalyzeTask, GenerateVariationTask],
broadcast_channel='h51_events',
max_status_interval=self.config['ASSET_WORKER_MAX_STATUS_INTERVAL'],
max_spawn_time=self.config['ASSET_WORKER_MAX_SPAWN_TIME'],
sleep_interval=self.config['ASSET_WORKER_SLEEP_INTERVAL'],
idle_lifespan=idle_lifespan,
population_control=self.config['ASSET_WORKER_POPULATION_CONTROL'],
population_spawner=self.config['ASSET_WORKER_POPULATION_SPAWNER']
)
def analyze(self, task):
"""Analyze an asset"""
asset = task.get_asset(
projection={
'variations': {'$sub.': Variation}
}
)
file = task.get_file()
history = []
for analyzer in task.get_analyzers(asset):
analyzer.analyze(self.config, asset, file, history)
history.append(analyzer)
if task.notification_url:
# POST the result to the notification URL
account = Account.by_id(
asset.account,
projection={'api_key': True}
)
task.post_notification(
account.api_key,
json.dumps(asset.to_json_type())
)
return {}
def do_task(self, task):
if isinstance(task, AnalyzeTask):
return self.analyze(task)
elif isinstance(task, GenerateVariationTask):
return self.generate_variation(task)
def generate_variation(self, task):
"""Generate variation for the asset"""
asset = task.get_asset(
projection={
'variations': {
'$sub.': Variation
}
}
)
file = task.get_file()
native_file = None
history = []
for transform in task.get_transforms(asset):
native_file = transform.transform(
self.config,
asset,
file,
task.variation_name,
native_file,
history
)
history.append(transform)
if task.notification_url:
# POST the result to the notification URL
account = Account.by_id(
asset.account,
projection={'api_key': True}
)
task.post_notification(
account.api_key,
json.dumps(asset.to_json_type())
)
return {}
def get_tasks(self):
tasks = super().get_tasks()
pairs = list(tasks.items())
# Randomize tasks to prevent large tasks hogging all the workers
random.shuffle(pairs)
tasks = dict(pairs)
return tasks
def on_error(self, task_id, error):
super().on_error(task_id, error)
if self.config['DEBUG']:
traceback.print_exc()
print(error)
else:
if self.config.get('SENTRY_DSN'):
sentry_sdk.capture_exception(error)
def on_spawn_error(self, error):
if self.config['DEBUG']:
super().on_spawn_error(error)
else:
if self.config.get('SENTRY_DSN'):
sentry_sdk.capture_exception(error)
@classmethod
def get_id_prefix(cls):
return 'h51_asset_worker'
import json
import sys
import time
from RPi import GPIO
from flask import Config
from GetTemperature import read_temp
from SimpleDaemon import Daemon
from thermostat_controls import air_conditioning, system_off
config = Config('config')
FAN_PIN = config.get('FAN_PIN')
HEATER_PIN = config.get('HEATER_PIN')
AC_PIN = config.get('AC_PIN')
check_time = 15
def get_target_temp():
with open('settings.json', 'r') as f:
data = json.load(f)
return data['TARGET_TEMP']
def get_current_mode():
with open('settings.json', 'r') as f:
data = json.load(f)
return data['TARGET_MODE']
def main(argconfig):
# Load the config file the same way Flask does which Chill uses.
config_file = argconfig if argconfig[0] == os.sep else os.path.join(os.getcwd(), argconfig)
config = Config(os.getcwd())
config.from_pyfile(config_file)
pyrax.set_credential_file(config["RACKSPACE_CREDENTIAL_FILE"])
cf = pyrax.cloudfiles
# sync sync_folder_to_container
local = config["FREEZER_DESTINATION"]
container_name = config.get("RACKSPACE_CONTAINER_NAME", os.path.basename(os.getcwd()))
prefix = config.get("PUBLIC_URL_PREFIX", None)
if prefix:
for page in (INDEX_PAGE, ERROR_PAGE, "401{0}".format(ERROR_PAGE), "404{0}".format(ERROR_PAGE)):
# If no index.html or error pages then link to the one found in
# PUBLIC_URL_PREFIX
if not os.path.exists(os.path.join(local, page)) and os.path.exists(os.path.join(local, prefix[1:], page)):
print "Creating hard link for {0}".format(page)
print "{0} -> {1}".format(os.path.join(local, page), os.path.join(local, prefix[1:], page))
os.link(os.path.join(local, prefix[1:], page), os.path.join(local, page))
confirm = raw_input(
"\nsync the folder: {local} \nto rackspace cloudfiles container: {container_name}\n[n]/y\n".format(**locals())
)
if confirm != "y":
return
remote = cf.create_container(container_name)
local_files = set()
for root, dir, files in os.walk(local):
for f in files:
local_files.add(os.path.join(root[len(local) + 1 :], f))
cf.sync_folder_to_container(local, remote)
# Mark all objects on remote to be deleted if not in local
# The limit here is arbitrary, but can not be higher then 10000.
limit = 1000
marker = ""
remote_objects_list = remote.get_objects(limit=limit, marker=marker)
while remote_objects_list:
marker = remote_objects_list[-1].name
for obj in remote_objects_list:
if obj.name not in local_files:
obj.delete_in_seconds(DELETE_OBJECTS_DELAY)
remote_objects_list = remote.get_objects(limit=limit, marker=marker)
# publish
cf.make_container_public(container_name, ttl=TTL)
# cdn website
remote.set_web_index_page(INDEX_PAGE)
remote.set_web_error_page(ERROR_PAGE)
# Totally copied from the docs
print
print "After Making Public"
print "cdn_enabled", remote.cdn_enabled
print "cdn_ttl", remote.cdn_ttl
print "cdn_log_retention", remote.cdn_log_retention
print "cdn_uri", remote.cdn_uri
print "cdn_ssl_uri", remote.cdn_ssl_uri
print "cdn_streaming_uri", remote.cdn_streaming_uri
print "cdn_ios_uri", remote.cdn_ios_uri
