async def upload(self):
"""Upload documentation to amazon s3
"""
if not self.cfg.docs_bucket:
raise ImproperlyConfigured('Please specify the "docs_bucket" '
'in your config file')
docs = self.cfg.docs
path = os.path.join(self.repo_path, 'docs', '_build', docs)
if not os.path.isdir(path):
raise ImproperlyConfigured('path "%s" missing' % path)
self.logger.info('Docs at "%s"', path)
mod = import_module(self.cfg.app_module)
version = mod.__version__
name = mod.__name__
url = '%s/%s' % (name, version)
if docs != 'html':
url = '%s/%s' % (docs, url)
self.logger.info('Preparing to upload to "%s/%s"',
self.cfg.docs_bucket, url)
aws_config = self.config['docs'].get('aws_config', {})
s3 = aws.AsyncioBotocore('s3',
http_session=self.gitapi.http,
**aws_config)
await s3.upload_folder(self.cfg.docs_bucket,
path,
url,
skip=['environment.pickle', 'last_build'],
content_types=content_types)
async def monitor_start(self, monitor):
'''Create the socket listening to the ``bind`` address.
If the platform does not support multiprocessing sockets set the
number of workers to 0.
'''
cfg = self.cfg
loop = monitor._loop
if (not pulsar.platform.has_multiProcessSocket
or cfg.concurrency == 'thread'):
cfg.set('workers', 0)
if not cfg.address:
raise ImproperlyConfigured('Could not open a socket. '
'No address to bind to')
address = parse_address(self.cfg.address)
if cfg.cert_file or cfg.key_file:
if not ssl:
raise RuntimeError('No support for ssl')
if cfg.cert_file and not os.path.exists(cfg.cert_file):
raise ImproperlyConfigured('cert_file "%s" does not exist' %
cfg.cert_file)
if cfg.key_file and not os.path.exists(cfg.key_file):
raise ImproperlyConfigured('key_file "%s" does not exist' %
cfg.key_file)
# First create the sockets
try:
server = await loop.create_server(asyncio.Protocol, *address)
except socket.error as e:
raise ImproperlyConfigured(e)
else:
self.monitor_sockets(monitor, server.sockets)
def create_cache(app, url):
if isinstance(url, Cache):
return url
scheme, _, _ = parse_store_url(url)
dotted_path = data_caches.get(scheme)
if not dotted_path:
raise ImproperlyConfigured('%s cache not available' % scheme)
store_class = module_attribute(dotted_path)
if not store_class:
raise ImproperlyConfigured('"%s" store not available' % dotted_path)
return store_class(app, scheme, url)
def register_broker(name, factory=None):
if factory is None:
dotted_path = brokers.get(name)
if not dotted_path:
raise ImproperlyConfigured('No such message broker: %s' % name)
factory = module_attribute(dotted_path, safe=True)
if not factory:
raise ImproperlyConfigured(
'"%s" store not available' % dotted_path)
else:
brokers[name] = factory
return factory
def create_store(url, **kw):
'''Create a new :class:`Store` for a valid ``url``.
:param url: a valid ``url`` takes the following forms:
:ref:`Pulsar datastore <store_pulsar>`::
pulsar://user:[email protected]:6410
:ref:`Redis <store_redis>`::
redis://user:[email protected]:6500/11?namespace=testdb
:param kw: additional key-valued parameters to pass to the :class:`.Store`
initialisation method. It can contains parameters such as
``database``, ``user`` and ``password`` to override the
``url`` values. Additional parameters are processed by the
:meth:`.Store._init` method.
:return: a :class:`Store`.
'''
if isinstance(url, Store):
return url
scheme, address, params = parse_store_url(url)
dotted_path = data_stores.get(scheme)
if not dotted_path:
raise NoSuchStore('%s store not available' % scheme)
store_class = module_attribute(dotted_path, safe=True)
if not store_class:
raise ImproperlyConfigured('"%s" store not available' % dotted_path)
if not store_class.registered:
store_class.registered = True
store_class.register()
params.update(kw)
return store_class(scheme, address, **params)
def get_param(self, name):
value = self.cfg.get(name)
if not value:
raise ImproperlyConfigured(
'Please specify the "%s" parameter in your %s file' %
(name, self.cfg.config))
return value
def __init__(self, binds):
# Setup mdoels and engines
if not binds:
binds = {}
elif isinstance(binds, str):
binds = {'default': binds}
if 'default' not in binds:
raise ImproperlyConfigured('default datastore not specified')
self._engines = {}
self._declarative_register = {}
self._bases = {}
self._base_declarative = declarative_base(name='OdmBase',
metaclass=DeclarativeMeta)
self.binds = {}
self.is_green = False
for name, bind in tuple(binds.items()):
key = None if name == 'default' else name
engine = create_engine(bind)
dialect = engine.dialect
# Dialect requires Green Pool
if getattr(dialect, 'is_green', False):
self.is_green = True
self._engines[key] = engine
def authenticate(self, request, **data):
if not jwt:
raise ImproperlyConfigured('JWT library not available')
api = request.app.api
try:
# TODO: add address from request
# client = request.get_client_address()
response = api.post('authorizations', data=data)
if response.status_code == 201:
token = response.json().get('token')
payload = jwt.decode(token, verify=False)
user = User(payload)
user.encoded = token
return user
else:
request.response.status_code = response.status_code
messages = response.json()
msg = None
for error in messages.get('errors', ()):
if 'field' not in error:
msg = error.get('message')
raise AuthenticationError(msg or 'Could not login')
except AuthenticationError:
raise
except Exception:
if data.get('username'):
raise AuthenticationError('Invalid username or password')
elif data.get('email'):
raise AuthenticationError('Invalid email or password')
else:
raise AuthenticationError('Invalid credentials')
def __init__(self, route, *routes, **params):
route = self.valid_route(route, params.pop('dir', None))
name = slugify(params.pop('name', None) or route or self.dir)
super(HtmlContent, self).__init__(route, *routes, name=name, **params)
# Add drafts index
if self.drafts:
self.add_child(
Drafts(self.drafts,
name=self.childname('drafts'),
index_template=self.drafts_template))
self.meta = copy(self.meta) if self.meta else {}
# Add children routes
meta = copy(self.meta)
if self.meta_children:
meta.update(self.meta_children)
file = self.HtmlFileRouter(self.child_url,
dir=self.dir,
name=self.childname('view'),
content=self.content,
html_body_template=self.html_body_template,
meta=meta,
uirouter=self.uirouter,
ngmodules=self.ngmodules)
self.add_child(file)
#
for url_path, file_path, ext in self.all_files(
include_subdirectories=False):
if url_path == 'index':
self.src = file_path
if self.src and self.index_template:
raise ImproperlyConfigured(
'Both index and index template specified')
def _iter_app(self, app_name_callables):
main = app_name_callables.pop('', None)
if not main:
raise ImproperlyConfigured('No main application in MultiApp')
yield main
for app in app_name_callables.values():
yield app
def middleware(self, app):
middleware = [self]
for backend in self.backends:
middleware.extend(backend.middleware(app) or ())
dotted_path = app.config['PAGINATION']
pagination = module_attribute(dotted_path)
if not pagination:
raise ImproperlyConfigured('Could not load paginator "%s"',
dotted_path)
app.pagination = pagination()
url = app.config['API_URL']
# If the api url is not absolute, add the api middleware
if url is not None:
if not is_absolute_uri(url):
# Add the preflight and token events
events = ('on_preflight', 'on_token')
app.add_events(events)
for backend in self.backends:
app.bind_events(backend, events)
api = RestRoot(url)
middleware.append(api)
for extension in app.extensions.values():
api_sections = getattr(extension, 'api_sections', None)
if api_sections:
for router in api_sections(app):
api.add_child(router)
app.api = ApiClient(app)
return middleware
def make_asynchronous():
try:
extensions.POLL_OK
except AttributeError: # pragma nocover
raise ImproperlyConfigured(
'Psycopg2 does not have support for asynchronous connections. '
'You need at least version 2.2.0 of Psycopg2.')
extensions.set_wait_callback(psycopg2_wait_callback)
def encode_token(self, request, user=None, expiry=None, **token):
'''Encode a JWT
'''
if not jwt: # pragma nocover
raise ImproperlyConfigured('JWT library not available')
if expiry:
token['exp'] = int(time.mktime(expiry.timetuple()))
request.app.fire('on_token', request, token, user)
return jwt.encode(token, request.config['SECRET_KEY'])
def template_engine(app, name):
cache = '_template_engine_%s' % name
engine = getattr(app, cache, None)
if engine is None:
engine = template_engines.get(name)
if engine is None:
raise ImproperlyConfigured('Template engine %s not available' %
name)
engine.configure(app)
setattr(app, cache, engine)
return engine
def csrf_token(self, request):
if not jwt: # pragma nocover
raise ImproperlyConfigured('JWT library not available')
session = request.cache.session
if session:
expiry = request.config['CSRF_EXPIRY']
secret_key = request.config['SECRET_KEY']
return jwt.encode(
{
'session': session.get_key(),
'exp': time.time() + expiry
}, secret_key)
def __call__(self, actor=None):
"""Register this application with the (optional) calling ``actor``.
If an ``actor`` is available (either via the function argument or via
the :func:`~pulsar.async.actor.get_actor` function) it must be
``arbiter``, otherwise this call is no-op.
If no actor is available, it means this application starts
pulsar engine by creating the ``arbiter`` with its
:ref:`global settings <setting-section-global-server-settings>`
copied to the arbiter :class:`.Config` container.
:return: the ``start`` one time event fired once this application
has fired it.
"""
if actor is None:
actor = get_actor()
monitor = None
if actor and actor.is_arbiter():
monitor = actor.get_actor(self.name)
if monitor is None and (not actor or actor.is_arbiter()):
self.cfg.on_start()
self.logger = self.cfg.configured_logger()
if not actor:
actor = pulsar.arbiter(cfg=self.cfg.clone())
else:
self.update_arbiter_params(actor)
if not self.cfg.exc_id:
self.cfg.set('exc_id', actor.cfg.exc_id)
if self.on_config(actor) is not False:
start = create_future(actor._loop)
actor.bind_event('start', partial(self._add_monitor, start))
return start
else:
return
elif monitor:
raise ImproperlyConfigured('%s already started ' % monitor.name)
else:
raise ImproperlyConfigured('Cannot start application from %s' %
actor)
def monitor_start(self, monitor):
'''Create the socket listening to the ``bind`` address.
If the platform does not support multiprocessing sockets set the
number of workers to 0.
'''
cfg = self.cfg
loop = monitor._loop
if (not pulsar.platform.has_multiProcessSocket
or cfg.concurrency == 'thread'):
cfg.set('workers', 0)
if not cfg.address:
raise ImproperlyConfigured('Could not open a socket. '
'No address to bind to')
ssl = None
if cfg.cert_file or cfg.key_file:
if cfg.cert_file and not os.path.exists(cfg.cert_file):
raise ImproperlyConfigured('cert_file "%s" does not exist' %
cfg.cert_file)
if cfg.key_file and not os.path.exists(cfg.key_file):
raise ImproperlyConfigured('key_file "%s" does not exist' %
cfg.key_file)
ssl = SSLContext(keyfile=cfg.key_file, certfile=cfg.cert_file)
address = parse_address(self.cfg.address)
# First create the sockets
try:
server = yield from loop.create_server(asyncio.Protocol, *address)
except socket.error as e:
raise ImproperlyConfigured(e)
else:
addresses = []
sockets = []
for sock in server.sockets:
addresses.append(sock.getsockname())
sockets.append(sock)
loop.remove_reader(sock.fileno())
monitor.sockets = sockets
monitor.ssl = ssl
cfg.addresses = addresses
def scheme(self):
'''Protocol scheme, one of ``http`` and ``https``
'''
HEADER = self.config['SECURE_PROXY_SSL_HEADER']
if HEADER:
try:
header, value = HEADER
except ValueError:
raise ImproperlyConfigured(
'The SECURE_PROXY_SSL_HEADER setting must be a tuple '
'containing two values.')
return 'https' if self.environ.get(header) == value else 'http'
return self.environ.get('HTTPS') == 'on'
async def run(self, name, config, options):
path = os.path.join(self.repo_path, 'docs')
if not os.path.isdir(path):
raise ImproperlyConfigured('path "%s" missing' % path)
os.chdir(path)
try:
text = await self.execute('make', self.cfg.docs)
finally:
os.chdir(self.repo_path)
self.logger.info(text)
if self.cfg.push:
await self.upload()
def _autodiscover(self):
datastore = self.binds
if not datastore:
datastore = {}
elif isinstance(datastore, str):
datastore = {'default': datastore}
if datastore and 'default' not in datastore:
raise ImproperlyConfigured('default datastore not specified')
self.binds = datastore
return register_applications(self.app,
copy(datastore),
self.app.config['EXTENSIONS'],
green=self.app.config['GREEN_WSGI'])
def __init__(self, app, binds):
self.app = app
super().__init__(binds)
models = OrderedDict()
for module in self.app.module_iterator('models'):
models.update(odm.get_models(module) or ())
models.update(
((table.key, table) for table in odm.module_tables(module)))
for model in models.values():
self.register(model)
if self.is_green and not app.config['GREEN_POOL']:
raise ImproperlyConfigured('ODM requires a greenlet pool but '
'GREEN_POOL is not set to a positive '
'integer.')
def build_sphinx(self, app, location):
if not LuxSphinx:
raise ImproperlyConfigured('Sphinx not installed')
path = self.html_router.path()[1:]
if path:
location = os.path.join(location, path)
if not os.path.isdir(location):
os.makedirs(location)
srcdir = os.path.abspath(self.dir)
doctreedir = os.path.join(location, '_doctrees')
app = LuxSphinx(app, srcdir, srcdir, location, doctreedir, 'lux')
force_all = False
app.build(force_all)
return app.data
def validate_csrf_token(self, request, token):
if not jwt: # pragma nocover
raise ImproperlyConfigured('JWT library not available')
if not token:
raise PermissionDenied(self.REASON_BAD_TOKEN)
try:
secret_key = request.config['SECRET_KEY']
token = jwt.decode(token, secret_key)
except jwt.ExpiredSignature:
raise PermissionDenied('Expired token')
except Exception:
raise PermissionDenied(self.REASON_BAD_TOKEN)
else:
if token['session'] != request.cache.session.get_key():
raise PermissionDenied(self.REASON_BAD_TOKEN)
def decode_token(self, request, token):
if not jwt: # pragma nocover
raise ImproperlyConfigured('JWT library not available')
try:
return jwt.decode(token, request.config['SECRET_KEY'])
except jwt.ExpiredSignature:
request.app.logger.warning('JWT token has expired')
# In this case we want the client to perform
# a new authentication. Raise 401
raise Http401('Token')
except Exception as exc:
request.app.logger.warning(str(exc))
# In this case we want the client to perform
# a new authentication. Raise 401
raise BadRequest
def middleware(self, app):
try:
html5 = app.config['ANGULAR_UI_ROUTER']
except KeyError:
raise ImproperlyConfigured(
'"lux.extensions.static" requires '
'"lux.extensions.angular" in EXTENSIONS')
path = app.config['MEDIA_URL']
api_url = app.config['STATIC_API'] or ''
if api_url.startswith('/'):
api_url = api_url[1:]
if api_url.endswith('/'):
api_url = api_url[:-1]
if not api_url and html5:
raise ImproperlyConfigured('STATIC_API url must be defined')
middleware = []
app.api = None
if api_url:
app.api = JsonRoot(api_url)
middleware.extend([app.api, JsonRedirect(api_url)])
if app.config['STATIC_MEDIA']:
middleware.append(
MediaBuilder(path, app.meta.media_dir, show_indexes=app.debug))
return middleware
def _buildurl(self, **kw):
pre = ''
if self._user:
if self._password:
pre = '%s:%s@' % (self._user, self._password)
else:
pre = '%s@' % self._user
elif self._password:
raise ImproperlyConfigured('password but not user')
assert self._password
host = self._host
if isinstance(host, tuple):
host = '%s:%s' % host
host = '%s%s' % (pre, host)
path = '/%s' % self._database if self._database else ''
kw.update(self._urlparams)
query = urlencode(kw)
scheme = self._name
if self._scheme:
scheme = '%s+%s' % (self._scheme, scheme)
return urlunparse((scheme, host, path, '', query, ''))
def _create_model(self, model):
model_name = model.__name__
meta = type(self._base_declarative)
if isinstance(model, meta):
raise ImproperlyConfigured('Cannot register declarative classes '
'only mixins allowed')
base = getattr(model, '__inherit_from__', None)
if base:
if base not in self._declarative_register:
models = self._bases.get(base)
if not models:
self._bases[base] = models = []
models.append(model)
return
else:
base = self._declarative_register[base]
else:
base = self._base_declarative
#
# Create SqlAlchemy Model
model = meta(model_name, (model, base), {})
create = getattr(model, '__create_sql__', None)
name = model_name.lower()
if create:
event.listen(self.metadata, 'after_create',
DDL(create.format({'name': name})))
drop = getattr(model, '__drop_sql__', None)
if not drop:
logger.warning(
'Model %s has create statement but not drop. '
'To mute this warning add a __drop_sql__ '
'statement in the model class', name)
else:
event.listen(self.metadata, 'before_drop',
DDL(drop.format({'name': name})))
return model, name
def request(self, request):
'''Check for ``HTTP_AUTHORIZATION`` header and if it is available
and the authentication type if ``bearer`` try to perform
authentication using JWT_.
'''
if not jwt: # pragma nocover
raise ImproperlyConfigured('JWT library not available')
auth = request.get('HTTP_AUTHORIZATION')
user = request.cache.user
if auth and user.is_anonymous():
auth_type, key = auth.split(None, 1)
auth_type = auth_type.lower()
if auth_type == 'bearer':
try:
token = self.decode_token(request, key)
except Http401:
raise
except Exception:
request.app.logger.exception('Could not load user')
else:
request.cache.session = token
user = self.get_user(request, **token)
if user:
request.cache.user = user
def _build(self):
for app in self.build():
if not isinstance(app, new_app):
raise ImproperlyConfigured(
'You must use new_app when building a MultiApp')
yield app
def __init__(self, choices):
if not choices:
raise ImproperlyConfigured(
'ChoiceType needs list of choices defined.'
)
self.choices_dict = dict(choices)