def set_config_types(config):
"""
convert raw config values to proper types
:param config:
:return:
"""
config = copy.deepcopy(config)
config["debug"] = asbool(config["debug"])
config["port"] = int(config["port"])
config["validate_requests"] = asbool(config["validate_requests"])
config["enforce_https"] = asbool(config["enforce_https"])
if not config["cookie_secret"]:
config["cookie_secret"] = str(uuid.uuid4())
if config["http_scheme"] not in ["http", "https"]:
config["http_scheme"] = ""
for key in ["allow_posting_from", "allow_cors"]:
if not config[key]:
continue
# if those keys are strings from ini convert to lists of individual values
if isinstance(config[key], str):
try:
listed = [ip.strip() for ip in config[key].split(",") if ip.strip()]
config[key] = listed
except ValueError:
pass
return config
def get_global_base_url(require_secure=None, override_port=None):
"""Get the base URL of this server
DO NOT USE directly, except for Linked data;
use Discussion.get_base_url()
"""
port = str(override_port or config.get('public_port'))
accept_secure_connection = asbool(
config.get('accept_secure_connection'))
require_secure_connection = accept_secure_connection and (require_secure or asbool(
config.get('require_secure_connection')))
service = 'http'
portString = ''
if accept_secure_connection or require_secure_connection:
if port is None or port == "443":
service += 's'
elif port == "80":
if require_secure_connection:
service += 's' # assume standard port upgrade
else:
if require_secure_connection:
service += 's'
portString = (':'+port)
else:
if port is not None and port != "80":
portString = (':'+port)
return '%s://%s%s' % (
service, config.get('public_hostname'), portString)
def __init__(self, request):
super(AuthController, self).__init__(request)
schema = request.registry.getUtility(ILoginSchema)
self.schema = schema().bind(request=self.request)
form = request.registry.getUtility(ILoginForm)
self.login_redirect_view = get_config_route(
request,
'horus.login_redirect'
)
self.logout_redirect_view = get_config_route(
request,
'horus.logout_redirect'
)
self.require_activation = asbool(
self.settings.get('horus.require_activation', True)
)
self.allow_inactive_login = asbool(
self.settings.get('horus.allow_inactive_login', False)
)
# XXX: Bootstrap classes leak into Deform here
login_button = deform.Button(name="login_email", title="Login with email", css_class="btn-lg btn-block")
self.form = form(self.schema, buttons=(login_button,))
# If the form is embedded on other pages force it go to right HTTP POST endpoint
self.form.action = request.route_url("login")
def __init__(self, request):
super(AuthController, self).__init__(request)
schema = request.registry.getUtility(ILoginSchema)
self.schema = schema().bind(request=self.request)
form = request.registry.getUtility(ILoginForm)
self.login_redirect_view = get_config_route(
request,
'horus.login_redirect'
)
self.logout_redirect_view = get_config_route(
request,
'horus.logout_redirect'
)
self.require_activation = asbool(
self.settings.get('horus.require_activation', True)
)
self.allow_inactive_login = asbool(
self.settings.get('horus.allow_inactive_login', False)
)
self.form = form(self.schema, buttons=(self.Str.login_button,))
def rewrite(self, session=None, skip_rollback=True, deferred=True,
start=False):
self.check_initialized()
session = session or Session.object_session(self)
if not session:
raise DetachedInstanceError()
self.log.info("Rewriting configuration for %s", self)
if asbool(self.settings.get('supervisor.enabled')):
pfx = self.settings.get('supervisor.command.prefix', 'aybu')
session.activity_log.add(render, 'supervisor.conf.mako',
self.paths.configs.supervisor_conf,
env=self, uwsgi=self.uwsgi_config,
program_prefix=pfx,
skip_rollback=skip_rollback,
deferred=deferred)
self.update_supervisor_conf(session)
elif asbool(self.settings.get('upstart.enabled')):
self.log.info("rendering %s", self.paths.configs.upstart_conf)
session.activity_log.add(render, 'upstart.conf.mako',
self.paths.configs.upstart_conf,
env=self, uwsgi=self.uwsgi_config,
skip_rollback=skip_rollback,
deferred=deferred)
action = "start" if start else "restart"
self.update_upstart(session, action)
def test_analytics_full_config(self):
import analytics
parser = self.get_parser()
self.setup_logging(parser)
settings = dict(parser.items('app:testapp'))
config = testing.setUp(settings=settings)
config.include('pyramid_analytics')
# Check config
self.assertEqual(analytics.default_client.secret,
settings['analytics.secret'])
self.assertEqual(analytics.default_client.flush_at,
int(settings['analytics.flush_at']))
self.assertEqual(analytics.default_client.flush_after.seconds,
int(settings['analytics.flush_after']))
self.assertEqual(analytics.default_client.async,
asbool(settings['analytics.async']))
self.assertEqual(analytics.default_client.send,
asbool(settings['analytics.send']))
self.assertEqual(analytics.default_client.max_queue_size,
int(settings['analytics.max_queue_size']))
# Check logging
log_config = dict(parser.items('logger_analytics'))
analytics_logger = logging.getLogger(log_config['qualname'])
self.assertEqual(analytics_logger.propagate,
int(log_config['propagate']))
self.assertEqual(analytics_logger.level,
getattr(logging, log_config['level']))
def includeme(config):
"""
RPC loader for Pyramid
"""
global _do_xmlrpc, _do_jsonrpc
cfg = config.registry.settings
_do_xmlrpc = asbool(cfg.get('netprofile.rpc.xmlrpc', True))
_do_jsonrpc = asbool(cfg.get('netprofile.rpc.jsonrpc', True))
if _do_xmlrpc:
config.include('pyramid_rpc.xmlrpc')
config.add_xmlrpc_endpoint('api.xmlrpc', '/api/xmlrpc')
if _do_jsonrpc:
config.include('pyramid_rpc.jsonrpc')
renderer = JSON()
def _json_datetime(obj, req):
if obj.tzinfo is None:
obj = obj.replace(tzinfo=tzlocal())
return obj.isoformat()
renderer.add_adapter(dt.datetime, _json_datetime)
renderer.add_adapter(dt.date, lambda obj, req: obj.isoformat())
renderer.add_adapter(dt.time, lambda obj, req: obj.isoformat())
renderer.add_adapter(ipaddress.IPv4Address, lambda obj, req: int(obj))
renderer.add_adapter(decimal.Decimal, lambda obj, req: str(obj))
config.add_renderer('jsonrpc', renderer)
config.add_jsonrpc_endpoint('api.jsonrpc', '/api/jsonrpc',
default_renderer='jsonrpc')
config.scan()
def get_cstruct(self, request: Request, params: dict=None):
"""Return cstruct data.
Bind `request` and `self.context` to colander schema
(self.schema). Get sheet appstruct data and serialize.
:param request: Bind to schema and get principals to filter elements
by 'view' permission.
:param params: Parameters to update the search query to find reference
data.
Automatically set params are: `only_visible` and `allows` view
permission.
"""
params = params or {}
filter_view_permission = asbool(self.registry.settings.get(
'adhocracy.filter_by_view_permission', True))
if filter_view_permission:
params['allows'] = (request.effective_principals, 'view')
filter_visible = asbool(self.registry.settings.get(
'adhocracy.filter_by_visible', True))
if filter_visible:
params['only_visible'] = True
schema = self._get_schema_for_cstruct(request, params)
appstruct = self.get(params=params)
cstruct = schema.serialize(appstruct)
return cstruct
def setup_configurator(settings):
"""
Takes care of some of the session initialisation code so that the app
using pyramidcms doesn't have to do so.
"""
# the secret key comes from the PasteDeploy ini file, it is required.
secret_key = settings["session.secret"]
cookie_httponly = asbool(settings.get("session.cookie_httponly", True))
cookie_secure = asbool(settings.get("session.cookie_secure", False))
# setup security policies
authn_policy = AuthTktAuthenticationPolicy(
secret_key, http_only=cookie_httponly, secure=cookie_secure, callback=groupfinder, hashalg="sha512"
)
# FIXME: allow user to choose their own session factory in the ini file
# the user might prefer to use pyramid_redis_sessions for example...
session_factory = SignedCookieSessionFactory(secret_key, httponly=cookie_httponly, secure=cookie_secure)
# setup the Configurator
return Configurator(
settings=settings,
authentication_policy=authn_policy,
authorization_policy=ACLAuthorizationPolicy(),
root_factory=RootFactory,
session_factory=session_factory,
)
def __init__(self, context, request):
self.context = context
self.request = request
self.version = __version__
self.home_url = request.application_url
self.help_url = request.registry.settings.get('help.url')
self.support_url = request.registry.settings.get('support.url') or "http://support.eucalyptus.com"
self.aws_enabled = asbool(request.registry.settings.get('aws.enabled'))
self.aws_regions = AWS_REGIONS
self.default_region = request.registry.settings.get('aws.default.region')
self.browser_password_save = 'true' if asbool(request.registry.settings.get('browser.password.save')) else 'false'
self.cloud_type = request.session.get('cloud_type')
self.selected_region = self.request.session.get('region', self.default_region)
self.selected_region_label = self.get_selected_region_label(self.selected_region)
self.username = self.request.session.get('username')
self.account = self.request.session.get('account')
self.username_label = self.request.session.get('username_label')
self.euca_logout_form = EucaLogoutForm(request=self.request)
self.date_format = _(u'%I:%M:%S %p %b %d %Y')
self.angular_date_format = _(u'hh:mm:ss a MMM d yyyy')
self.tag_pattern_key = '^(?!aws:).{0,128}$'
self.tag_pattern_value = '^(?!aws:).{0,256}$'
self.integer_gt_zero_pattern = '^[1-9]\d*$'
self.non_negative_pattern = '^[0-9]\d*$'
self.cidr_pattern = '{0}{1}'.format(
'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}',
'(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])(\/\d+)$'
)
self.querystring = self.get_query_string()
self.help_html_dir = 'eucaconsole:static/html/help/'
def setup_app(registry, ini_location):
loader = INILoader(celery_app, ini_file=ini_location)
celery_config = loader.read_configuration()
#: TODO: There might be other variables requiring special handling
boolify(celery_config, 'CELERY_ALWAYS_EAGER', 'CELERY_ENABLE_UTC', 'CELERY_RESULT_PERSISTENT')
if asbool(celery_config.get('USE_CELERYCONFIG', False)) is True:
config_path = 'celeryconfig'
celery_app.config_from_object(config_path)
else:
# TODO: Couldn't find a way with celery to do this
hijack_logger = asbool(
celery_config.get('CELERYD_HIJACK_ROOT_LOGGER', False)
)
celery_config['CELERYD_HIJACK_ROOT_LOGGER'] = hijack_logger
if hijack_logger is False:
global ini_file
ini_file = ini_location
signals.setup_logging.connect(configure_logging)
celery_app.config_from_object(celery_config)
celery_app.conf.update({'PYRAMID_REGISTRY': registry})
def sign_up(self, user_data: dict) -> Response:
"""Sign up a new user."""
user_registry = get_user_registry(self.request)
user = user_registry.sign_up(registration_source="email", user_data=user_data)
# Notify site creator to initialize the admin for the first user
self.request.registry.notify(UserCreated(self.request, user))
settings = self.request.registry.settings
require_activation = asbool(settings.get('websauna.require_activation', True))
autologin = asbool(settings.get('websauna.autologin', False))
if require_activation:
self.create_email_activation(user)
elif not autologin:
messages.add(self.request, msg_id="msg-sign-up-complete", msg="Sign up complete. Welcome!", kind="success")
self.request.registry.notify(NewRegistrationEvent(self.request, user, None, user_data))
self.request.dbsession.flush() # in order to get the id
if autologin:
login_service = get_login_service(self.request.registry)
return login_service.authenticate(self.request, user)
else: # not autologin: user must log in just after registering.
return render_to_response('login/waiting_for_activation.html', {"user": user}, request=self.request)
def set_status(self, new_status, request, parent=True):
root = find_root(self)
schema = self.type_info.schema
properties = self.upgrade_properties()
item_id = '{}/'.format(resource_path(self))
current_status = properties.get('status')
if not current_status:
raise ValidationFailure('body', ['status'], 'No property status')
if not self._valid_status(new_status, schema, parent):
return False
force_transition = asbool(request.params.get('force_transition'))
if not self._valid_transition(current_status, new_status, parent, force_transition):
return False
force_audit = asbool(request.params.get('force_audit'))
self._block_on_audits(item_id, force_audit, request, parent, new_status)
update = asbool(request.params.get('update'))
self._update_status(new_status, current_status, properties, schema, request, item_id, update)
request._set_status_considered_paths.add((item_id, current_status, new_status))
logging.warn(
'Considering {} from status {} to status {}'.format(item_id, current_status, new_status)
)
block_children = self._calculate_block_children(request, force_transition)
child_paths = self._get_child_paths(current_status, new_status, block_children)
embedded_properties = request.embed(item_id, '@@embedded')
related_objects = self._get_related_object(child_paths, embedded_properties, request)
self._set_status_on_related_objects(new_status, related_objects, root, request)
return True
def configure_error_views(self):
settings = self.settings
# Forbidden view overrides helpful auth debug error messages,
# so pull in only when really needed
if not asbool(settings["pyramid.debug_authorization"]):
from websauna.system.core.views import forbidden
self.config.scan(forbidden)
if not asbool(settings["pyramid.debug_notfound"]):
from websauna.system.core.views import notfound
self.config.scan(notfound)
# Internal server error must be only activated in the production mode, as it clashes with pyramid_debugtoolbar
has_debug_toolbar = "pyramid_debugtoolbar" in aslist(settings.get("pyramid.includes", []))
debug_toolbar_enabled = has_debug_toolbar and asbool(settings.get("debugtoolbar.enabled", True))
if not debug_toolbar_enabled:
from websauna.system.core.views import internalservererror
self.config.scan(internalservererror)
if settings.get("websauna.error_test_trigger", False):
from websauna.system.core.views import errortrigger
self.config.scan(errortrigger)
self.config.add_route('error_trigger', '/error-trigger')
def main(global_config, **settings):
settings = dict(global_config, **settings)
settings.setdefault('jinja2.i18n.domain', 'CheesePrism')
session_factory = UnencryptedCookieSessionFactoryConfig('cheeseprism')
config = Configurator(root_factory=App, settings=settings,
session_factory=session_factory,
authentication_policy=\
BasicAuthenticationPolicy(BasicAuthenticationPolicy.noop_check))
setup_workers(config.registry)
config.add_translation_dirs('locale/')
config.include('.request')
config.include('.views')
config.include('.index')
tempspec = settings.get('cheeseprism.index_templates', '')
config.registry['cp.index_templates'] = EnvFactory.from_str(tempspec)
if asbool(settings.get('cheeseprism.pipcache_mirror', False)):
config.include('.sync.pip')
if asbool(settings.get('cheeseprism.auto_sync', False)):
config.include('.sync.auto')
tempfile_limit = int(settings.get('cheeseprism.temp_file_limit', 10*1024))
config.add_request_method(lambda req: tempfile_limit,
name='request_body_tempfile_limit', reify=True)
config.add_request_method(lambda req: asbool(settings.get('cheeseprism.disable.regenerate', False)),
name='disable_regen', reify=True)
return config.make_wsgi_app()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
config = Configurator(settings=settings)
# add html renderer (chameleon based)
config.add_renderer(name='.html', factory='p_started.renderer.AppDistRendererFactory')
production_config = asbool(settings.get('PRODUCTION', 'false'))
production_config = asbool(settings.get('PRODUCTION', 'false'))
production = os.environ.get('PRODUCTION', production_config)
minify_config = settings.get('minify', 'app')
minify = os.environ.get('minify', minify_config)
config.include('pyramid_chameleon')
# static views
# See also config.override_asset if you want to avoid minify
config.add_static_view('scripts', 'p_started:webapp/%s/scripts' % minify, cache_max_age=3600)
config.add_static_view('styles', 'p_started:webapp/%s/styles' % minify, cache_max_age=3600)
config.add_static_view('images', 'p_started:webapp/%s/images' % minify, cache_max_age=3600)
if not production:
# we expose the bower_components dir just for development deployments, not good for production
config.add_static_view('bower_components', 'p_started:webapp/%s/bower_components' % minify, cache_max_age=3600)
else:
config.add_static_view('fonts', 'p_started:webapp/%s/fonts' % minify, cache_max_age=3600)
# routes
config.add_route('home', '/')
config.scan()
return config.make_wsgi_app()
def includeme(config):
"""Configure basic REST settings for a Pyramid application.
You should not call this function directly, but use
:py:func:`pyramid.config.Configurator.include` to initialise
the REST toolkit.
.. code-block:: python
:linenos:
config = Configurator()
config.include('rest_toolkit')
"""
settings = config.registry.settings
settings['rest_toolkit.debug'] = \
settings.get('debug_all') or \
settings.get('pyramid.debug_all') or \
settings.get('rest_toolkit.debug') or \
asbool(os.environ.get('PYRAMID_DEBUG_ALL')) or \
asbool(os.environ.get('REST_TOOLKIT_DEBUG'))
if not settings['rest_toolkit.debug']:
config.add_view('rest_toolkit.error.generic',
context=Exception, renderer='json',
permission=NO_PERMISSION_REQUIRED)
config.add_view('rest_toolkit.error.http_error', context=IExceptionResponse, renderer='json')
config.add_view('rest_toolkit.error.http_error', context=WSGIHTTPException, renderer='json')
config.add_notfound_view('rest_toolkit.error.notfound', renderer='json')
config.add_forbidden_view('rest_toolkit.error.forbidden', renderer='json')
def __init__(self, registry):
self.es = registry[ELASTIC_SEARCH]
self.esstorage = registry[STORAGE]
self.index = registry.settings['snovault.elasticsearch.index']
self.queue_server = None
self.queue_worker = None
self.worker_runs = []
available_queues = registry['available_queues']
queue_type = registry.settings.get('queue_type', None)
is_queue_server = asbool(registry.settings.get('queue_server'))
is_queue_worker = asbool(registry.settings.get('queue_worker'))
queue_worker_processes = int(
registry.settings.get('queue_worker_processes', 1)
)
queue_worker_chunk_size = int(
registry.settings.get('queue_worker_chunk_size', 1024)
)
queue_worker_batch_size = int(
registry.settings.get('queue_worker_batch_size', 1024)
)
queue_options = {
'processes': queue_worker_processes,
'chunk_size': queue_worker_chunk_size,
'batch_size': queue_worker_batch_size,
}
if is_queue_server and queue_type in available_queues:
if not queue_type or queue_type == 'Simple':
self.queue_server = SimpleUuidServer(queue_options)
else:
log.error('No queue available for Indexer')
if self.queue_server and is_queue_worker:
self.queue_worker = self.queue_server.get_worker()
def client_activate(request):
if authenticated_userid(request):
return HTTPSeeOther(location=request.route_url('access.cl.home'))
did_fail = True
cur_locale = locale_neg(request)
cfg = request.registry.settings
comb_js = asbool(cfg.get('netprofile.client.combine_js', False))
can_reg = asbool(cfg.get('netprofile.client.registration.enabled', False))
must_verify = asbool(cfg.get('netprofile.client.registration.verify_email', True))
link_id = int(cfg.get('netprofile.client.registration.link_id', 1))
rand_len = int(cfg.get('netprofile.client.registration.code_length', 20))
if (not can_reg) or (not must_verify):
return HTTPSeeOther(location=request.route_url('access.cl.login'))
code = request.GET.get('code', '').strip().upper()
login = request.GET.get('for', '')
if code and login and (len(code) == rand_len):
sess = DBSession()
for link in sess.query(AccessEntityLink)\
.options(joinedload(AccessEntityLink.entity))\
.filter(AccessEntityLink.type_id == link_id, AccessEntityLink.value == code):
# TODO: implement code timeouts
ent = link.entity
if (ent.access_state == AccessState.block_inactive.value) and (ent.nick == login):
ent.access_state = AccessState.ok.value
sess.delete(link)
did_fail = False
break
tpldef = {
'failed' : did_fail,
'comb_js' : comb_js,
'cur_loc' : cur_locale
}
request.run_hook('access.cl.tpldef.activate', tpldef, request)
return tpldef
def configure(cls, settings):
kwargs = super(LDAPAccessBackend, cls).configure(settings)
conn = LDAP(
admin_field=settings.get("auth.ldap.admin_field"),
admin_group_dn=settings.get("auth.ldap.admin_group_dn"),
admin_value=aslist(
settings.get("auth.ldap.admin_value", []), flatten=False
),
base_dn=settings.get("auth.ldap.base_dn"),
cache_time=settings.get("auth.ldap.cache_time"),
service_dn=settings.get("auth.ldap.service_dn"),
service_password=settings.get("auth.ldap.service_password", ""),
service_username=settings.get("auth.ldap.service_username"),
url=settings["auth.ldap.url"],
user_dn_format=settings.get("auth.ldap.user_dn_format"),
user_search_filter=settings.get("auth.ldap.user_search_filter"),
ignore_cert=asbool(settings.get("auth.ldap.ignore_cert")),
ignore_referrals=asbool(settings.get("auth.ldap.ignore_referrals", False)),
ignore_multiple_results=asbool(
settings.get("auth.ldap.ignore_multiple_results", False)
),
)
conn.connect()
kwargs["conn"] = conn
fallback = settings.get("auth.ldap.fallback")
if fallback == "config":
kw = ConfigAccessBackend.configure(settings)
kwargs["fallback_factory"] = lambda r: ConfigAccessBackend(r, **kw)
return kwargs
def includeme(config):
"""Bind to the db engine specifed in ``config.registry.settings``.
Setup::
>>> from mock import Mock
>>> import pyramid_basemodel
>>> _engine_from_config = pyramid_basemodel.engine_from_config
>>> _bind_engine = pyramid_basemodel.bind_engine
>>> pyramid_basemodel.engine_from_config = Mock()
>>> pyramid_basemodel.engine_from_config.return_value = 'engine'
>>> pyramid_basemodel.bind_engine = Mock()
>>> mock_config = Mock()
>>> mock_config.registry.settings = {}
Calls ``bind_engine`` with the configured ``engine``::
>>> includeme(mock_config)
>>> pyramid_basemodel.bind_engine.assert_called_with('engine',
... should_create=True, should_drop=False)
Teardown::
>>> pyramid_basemodel.engine_from_config = _engine_from_config
>>> pyramid_basemodel.bind_engine = _bind_engine
"""
# Bind the engine.
settings = config.registry.settings
engine = engine_from_config(settings, 'sqlalchemy.')
should_create = asbool(settings.get('basemodel.should_create_all', True))
should_drop = asbool(settings.get('basemodel.should_drop_all', False))
bind_engine(engine, should_create=should_create, should_drop=should_drop)
def authenticate_user(self, user: IUser, login_source:str, location: str=None):
"""Make the current session logged in session for this particular user."""
request = self.request
settings = request.registry.settings
require_activation = asbool(settings.get('horus.require_activation', True))
allow_inactive_login = asbool(settings.get('horus.allow_inactive_login', False))
if (not allow_inactive_login) and require_activation and (not user.is_activated()):
raise AuthenticationFailure('Your account is not active, please check your e-mail. If your account activation email as expired please request a password reset.')
if not user.can_login():
raise AuthenticationFailure('This user account cannot log in at the moment.')
user_registry = get_user_registry(request)
token = user_registry.get_session_token(user)
headers = remember(request, token)
# assert headers, "Authentication backend did not give us any session headers"
if not location:
location = get_config_route(request, 'horus.login_redirect')
self.greet_user(user)
self.update_login_data(user)
e = events.Login(request, user)
request.registry.notify(e)
return HTTPFound(location=location, headers=headers)
def _new_response(event):
request = event.request
settings = request.registry.settings
response = event.response
# TODO: add static URL if set
csp = 'default-src \'self\' www.google.com; style-src \'self\' www.google.com \'unsafe-inline\''
if request.debug_enabled:
csp += '; script-src \'self\' www.google.com \'unsafe-inline\''
response.headerlist.extend((
('Content-Security-Policy', csp),
('X-Content-Type-Options', 'nosniff')
))
if 'X-Frame-Options' not in response.headers:
response.headerlist.append(('X-Frame-Options', 'DENY'))
if asbool(settings.get('netprofile.http.sts.enabled', False)):
try:
max_age = int(settings.get('netprofile.http.sts.max_age', 604800))
except (TypeError, ValueError):
max_age = 604800
sts_chunks = [ 'max-age=' + str(max_age) ]
if asbool(settings.get('netprofile.http.sts.include_subdomains', False)):
sts_chunks.append('includeSubDomains')
if asbool(settings.get('netprofile.http.sts.preload', False)):
sts_chunks.append('preload')
response.headerlist.append(('Strict-Transport-Security', '; '.join(sts_chunks)))
def load(self, settings, prefix):
d = {key[len(prefix):]: value
for key, value in settings.items()
if key.startswith(prefix)}
self.sitename = d.get('sitename', 'PizzaVolus')
footer_file = d.get('footer_file')
if footer_file:
with open(footer_file) as f:
self.footer_text = f.read()
else:
self.footer_text = d.get('footer_text')
self.log_user_auth = asbool(d.get('log_user_auth', True))
self.log_address_auth = asbool(d.get('log_address_auth', False))
self.user_log_limit = int(d.get('user_log_limit', 10))
self.address_log_limit = int(d.get('address_log_limit', 5))
default_recaptcha_api = 'https://www.google.com/recaptcha/api/siteverify'
self.recaptcha_site = d.get('recaptcha_site')
self.recaptcha_secret = d.get('recaptcha_secret')
self.recaptcha_api = d.get('recaptcha_api', default_recaptcha_api)
self.use_recaptcha = self.recaptcha_secret and self.recaptcha_site
# Required level to see logs.
# "address" < "user" < other values (disable log access)
self.address_log_access = d.get('address_log_access', 'user')
self.user_log_access = d.get('user_log_access', 'user')
# Default local name regexp
self.lname_pattern = d.get('lname_pattern', '^[a-zA-Z0-9_.-]{3,64}$')
def includeme(config):
"""Bind to the db engine specifed in ``config.registry.settings``.
Setup::
>>> from mock import Mock
>>> import pyramid_basemodel
>>> _engine_from_config = pyramid_basemodel.engine_from_config
>>> _bind_engine = pyramid_basemodel.bind_engine
>>> pyramid_basemodel.engine_from_config = Mock()
>>> pyramid_basemodel.engine_from_config.return_value = 'engine'
>>> pyramid_basemodel.bind_engine = Mock()
>>> mock_config = Mock()
>>> mock_config.registry.settings = {}
Calls ``bind_engine`` with the configured ``engine``::
>>> includeme(mock_config)
>>>
>>> mock_config.action.assert_called_with(None,
... pyramid_basemodel.bind_engine,
... ('engine',),
... {'should_create': False, 'should_drop': False})
Unless told not to::
>>> pyramid_basemodel.bind_engine = Mock()
>>> mock_config = Mock()
>>> mock_config.registry.settings = {'basemodel.should_bind_engine': False}
>>> includeme(mock_config)
>>> mock_config.action.called
False
Teardown::
>>> pyramid_basemodel.engine_from_config = _engine_from_config
>>> pyramid_basemodel.bind_engine = _bind_engine
"""
# Bind the engine.
settings = config.get_settings()
engine_kwargs_factory = settings.pop('sqlalchemy.engine_kwargs_factory', None)
if engine_kwargs_factory:
kwargs_factory = config.maybe_dotted(engine_kwargs_factory)
engine_kwargs = kwargs_factory(config.registry)
else:
engine_kwargs = {}
pool_class = settings.pop('sqlalchemy.pool_class', None)
if pool_class:
dotted_name = DottedNameResolver()
engine_kwargs['poolclass'] = dotted_name.resolve(pool_class)
should_bind = asbool(settings.get('basemodel.should_bind_engine', True))
should_create = asbool(settings.get('basemodel.should_create_all', False))
should_drop = asbool(settings.get('basemodel.should_drop_all', False))
if should_bind:
engine = engine_from_config(settings, 'sqlalchemy.', **engine_kwargs)
config.action(None, bind_engine, (engine,), {
'should_create': should_create,
'should_drop': should_drop})
def includeme(config):
registry = config.registry
settings = registry.settings
config.add_renderer('json', JSON(indent=4))
config.include('pyramid_oauthlib')
config.add_token_type(Token)
# Configure the token policy
authn_debug = config.registry.settings.get('debug_authorization')
authn_policy = OAuthAuthenticationPolicy(
environ_key='HTTP_X_ANNOTATOR_AUTH_TOKEN',
debug=authn_debug,
)
config.set_authentication_policy(authn_policy)
# Configure ElasticSearch
store_from_settings(settings)
# Maybe initialize the models
if asbool(settings.get('basemodel.should_drop_all', False)):
delete_db()
if asbool(settings.get('basemodel.should_create_all', True)):
create_db()
config.scan(__name__)
def post_attachment_view(request, file_field):
settings = request.registry.settings
keep_old_files = asbool(settings.get('attachment.keep_old_files', False))
if not keep_old_files:
# Remove potential existing attachment.
utils.delete_attachment(request)
# Store file locally.
content = request.POST.get(file_field)
randomize = True
if 'randomize' in request.GET:
randomize = asbool(request.GET['randomize'])
attachment = utils.save_file(content, request, randomize=randomize)
# Update related record.
record = {k: v for k, v in request.POST.items() if k != file_field}
for k, v in record.items():
record[k] = json.loads(v)
record.setdefault('data', {})[file_field] = attachment
utils.patch_record(record, request)
# Return attachment data (with location header)
request.response.headers['Location'] = utils.record_uri(request,
prefix=True)
return attachment
def inject_globals(event):
"""Injects global variables into every template after the view
is processed but before the template is rendered."""
# request is available in every template, and we can just pull it
# in like so...
request = event['request']
settings = request.registry.settings
# ... and then set a couple global variables that will be available
# in every template, so we don't have to pass them through the
# dict returned by the view every time!
event['title'] = settings['tahrir.title']
event['base_url'] = settings['tahrir.base_url']
event['logged_in'] = authenticated_userid(request)
person = request.db.get_person(event['logged_in'])
event['logged_in_person'] = person
event['footer'] = tahrir.views.load_docs(request, 'footer')
event['twitter'] = asbool(settings.get('tahrir.social.twitter'))
event['twitter_user_text'] = settings.get(
'tahrir.social.twitter_user_text')
event['twitter_user_hash'] = settings.get(
'tahrir.social.twitter_user_hash')
event['gplus'] = asbool(settings.get('tahrir.social.gplus'))
# Cause jquery.js to be injected into the page.
tw2.jquery.jquery_js.display()
def __init__(self, request):
super(LoginView, self).__init__(request)
self.title_parts = [_(u'Login')]
self.euca_login_form = EucaLoginForm(self.request, formdata=self.request.params or None)
self.aws_login_form = AWSLoginForm(self.request, formdata=self.request.params or None)
self.aws_enabled = asbool(request.registry.settings.get('enable.aws'))
referrer = urlparse(self.request.url).path
login_url = self.request.route_path('login')
logout_url = self.request.route_path('logout')
if referrer in [login_url, logout_url]:
referrer = '/' # never use the login form (or logout view) itself as came_from
self.came_from = self.sanitize_url(self.request.params.get('came_from', referrer))
self.login_form_errors = []
self.duration = str(int(self.request.registry.settings.get('session.cookie_expires')) + 60)
self.login_refresh = str(int(self.request.registry.settings.get('session.timeout')) - 60)
self.secure_session = asbool(self.request.registry.settings.get('session.secure', False))
self.https_proxy = self.request.environ.get('HTTP_X_FORWARDED_PROTO') == 'https'
self.https_scheme = self.request.scheme == 'https'
options_json = BaseView.escape_json(json.dumps(dict(
account=request.params.get('account', default=''),
username=request.params.get('username', default=''),
)))
self.render_dict = dict(
https_required=self.show_https_warning(),
euca_login_form=self.euca_login_form,
aws_login_form=self.aws_login_form,
login_form_errors=self.login_form_errors,
aws_enabled=self.aws_enabled,
duration=self.duration,
login_refresh=self.login_refresh,
came_from=self.came_from,
controller_options_json=options_json,
)
def includeme(config):
DEFAULT = {
'versions': '3.0.3',
'use_min_file': True,
'use_cdn': False,
'static_path': {
'cdn': "//netdna.bootstrapcdn.com/bootstrap/",
'local': 'bootstrap/'
},
'cache_max_age': 3600,
}
settings = config.get_settings()
setting_prefix = "bootstrap."
def get_setting(attr, default=None):
return settings.get(setting_prefix + attr, default)
versions = get_setting('versions', DEFAULT['versions'])
use_min_file = asbool(get_setting("use_min_file", DEFAULT['use_min_file']))
bootstraps = BootstrapFactory.build_bootstraps(versions, use_min_file)
use_cdn = asbool(get_setting("use_cdn"))
if use_cdn:
static_path = DEFAULT['static_path']['cdn']
else:
static_path = get_setting('static_path',
DEFAULT['static_path']['local'])
cache_max_age = get_setting('cache_max_age', DEFAULT['cache_max_age'])
for version in bootstraps:
config.add_static_view(static_path + version,
"pyramid_bootstrap:static/{}".format(version),
cache_max_age=cache_max_age)
config.scan('pyramid_bootstrap.event_subscribers')
def opentracing_tween_factory(handler, registry):
'''
The factory method is called once, and we thus retrieve the settings as defined
on the global configuration.
We set the 'opentracing_tracer' in the settings to, for further reference and usage.
'''
base_tracer = registry.settings.get('ot.base_tracer', opentracing.Tracer())
traced_attrs = aslist(registry.settings.get('ot.traced_attributes', []))
trace_all = asbool(registry.settings.get('ot.trace_all', False))
operation_name_func = None
if 'ot.base_tracer_func' in registry.settings:
base_tracer_func = registry.settings.get('ot.base_tracer_func')
base_tracer = _call_base_tracer_func(base_tracer_func,
registry.settings)
if 'ot.operation_name_func' in registry.settings:
operation_name_func = registry.settings.get('ot.operation_name_func')
if not callable(operation_name_func):
operation_name_func = _get_function_from_name(operation_name_func)
tracer = PyramidTracer(base_tracer, trace_all, operation_name_func)
registry.settings['ot.tracer'] = tracer
def opentracing_tween(req):
# if tracing for all requests is disabled, continue with the
# normal handlers flow and return immediately.
if not trace_all:
return handler(req)
tracer._apply_tracing(req, traced_attrs)
try:
res = handler(req)
except:
tracer._finish_tracing(req, error=True)
raise
tracer._finish_tracing(req)
return res
return opentracing_tween
def wsgi_application_from_cwconfig(cwconfig,
profile=False,
profile_output=None,
profile_dump_every=None):
""" Build a WSGI application from a cubicweb configuration
:param cwconfig: A CubicWeb configuration
:param profile: Enable profiling. See :ref:`profiling`.
:param profile_output: Profiling output filename. See :ref:`profiling`.
:param profile_dump_every: Profiling number of requests before dumping the
stats. See :ref:`profiling`.
:returns: A fully operationnal WSGI application
"""
config = config_from_cwconfig(cwconfig)
profile = profile or asbool(
config.registry.settings.get('cubicweb.profile.enable', False))
if profile:
config.add_route('profile_ping', '_profile/ping')
config.add_route('profile_cnx', '_profile/cnx')
config.scan('cubicweb.pyramid.profile')
app = config.make_wsgi_app()
# This replaces completely web/cors.py, which is not used by
# cubicweb.pyramid anymore
app = wsgicors.CORS(
app,
origin=' '.join(cwconfig['access-control-allow-origin']),
headers=', '.join(cwconfig['access-control-allow-headers']),
expose_headers=', '.join(cwconfig['access-control-expose-headers']),
methods=', '.join(cwconfig['access-control-allow-methods']),
maxage=cwconfig['access-control-max-age'],
credentials='true')
if profile:
from cubicweb.pyramid.profile import wsgi_profile
filename = profile_output or config.registry.settings.get(
'cubicweb.profile.output', 'program.prof')
dump_every = profile_dump_every or config.registry.settings.get(
'cubicweb.profile.dump_every', 100)
app = wsgi_profile(app, filename=filename, dump_every=dump_every)
return app
def send(self, message_data=None):
"""Send the message with the given subject
body can be sent as part of send() or it can be set to the object as
msg.body = "xxx"
"""
self.body = self._get_message_body(self.message_file, message_data)
msg = MIMEMultipart('related')
msg['Subject'] = self.subject
msg['From'] = self.from_addr
msg['To'] = self.to
plain_text = MIMEText(self.body, 'plain', _charset="UTF-8")
msg.attach(plain_text)
is_live = asbool(self.settings.get('email.enable', False))
if not is_live:
print msg.as_string()
return MSG_STATUS['sent']
else:
try:
all_emails = msg['To']
smtp_server = self.settings.get('email.host')
if smtp_server == 'sendmail':
sendmail(msg['To'], msg['From'], msg['Subject'], self.body)
else:
mail_server = smtplib.SMTP(smtp_server)
mail_server.sendmail(msg['From'], all_emails,
msg.as_string())
mail_server.quit()
return MSG_STATUS['sent']
except smtplib.SMTPException, exc:
LOG.error("SMTP Error sending notice for: {0} ".format(
str(msg)))
return MSG_STATUS['error']
def validate_from_bucket_schema_or_400(data, resource_name, request, id_field,
ignore_fields):
"""Lookup in the parent objects if a schema was defined for this resource.
If the schema validation feature is enabled, if a schema is/are defined, and if the
data does not validate it/them, then it raises a 400 exception.
"""
settings = request.registry.settings
schema_validation = "experimental_collection_schema_validation"
# If disabled from settings, do nothing.
if not asbool(settings.get(schema_validation)):
return
bucket_id = request.matchdict["bucket_id"]
bucket_uri = utils.instance_uri(request, "bucket", id=bucket_id)
buckets = request.bound_data.setdefault("buckets", {})
if bucket_uri not in buckets:
# Unknown yet, fetch from storage.
bucket = object_exists_or_404(request,
resource_name="bucket",
parent_id="",
object_id=bucket_id)
buckets[bucket_uri] = bucket
# Let's see if the bucket defines a schema for this resource.
metadata_field = f"{resource_name}:schema"
bucket = buckets[bucket_uri]
if metadata_field not in bucket:
return
# Validate or fail with 400.
schema = bucket[metadata_field]
try:
validate_schema(data,
schema,
ignore_fields=ignore_fields,
id_field=id_field)
except ValidationError as e:
raise_invalid(request, name=e.field, description=e.message)
except RefResolutionError as e:
raise_invalid(request, name="schema", description=str(e))
def includeme(config):
settings = config.registry.settings
settings.setdefault('snovault.elasticsearch.index', 'snovault')
config.add_request_method(datastore, 'datastore', reify=True)
addresses = aslist(settings['elasticsearch.server'])
config.registry[ELASTIC_SEARCH] = Elasticsearch(
addresses,
serializer=PyramidJSONSerializer(json_renderer),
connection_class=TimedUrllib3HttpConnection,
retry_on_timeout=True,
maxsize=50)
config.include('.cached_views')
if not config.registry.settings.get('indexer'):
config.include('.esstorage')
config.include('.indexer')
if asbool(settings.get('indexer')) and not PY2:
config.include('.mpindexer')
def set_status(self, new_status, request, parent=True):
status_set = super(File, self).set_status(
new_status,
request,
parent=parent,
)
if not status_set or not asbool(request.params.get('update')):
return False
# Change permission in S3.
try:
if new_status in self.public_s3_statuses:
self.set_public_s3()
elif new_status in self.private_s3_statuses:
self.set_private_s3()
except ClientError as e:
# Demo trying to set ACL on production object?
if e.response['Error']['Code'] == 'AccessDenied':
logging.warn(e)
else:
raise e
return True
def replace(prefix, *args, **kwargs):
"""
Example:
from pyramid_auto_env import replace
settings = get_appsettings('development.ini')
update_settings = replace('AES', **settings)
"""
for k, v in kwargs.items():
transformed_key = "{}_{}".format(
prefix.upper(),
k.upper().replace('.', '_').replace('-', '_'))
envvar_value = os.environ.get(transformed_key)
if not envvar_value:
continue
kwargs[k] = asbool(envvar_value) if envvar_value.lower() in [
'true', 'false'
] else envvar_value
logger.info(
'Found settings replacement for "{}" at "{}". Setting it up.'.
format(k, transformed_key))
return kwargs
def main(global_config, **settings):
config = get_configurator(global_config, **settings)
# Use autocommit if we're in testing mode.
mock_browserid = asbool(os.path.expandvars(global_config.get('test', '')))
if mock_browserid:
config.autocommit = True
# Get all the default config for appsync.
config.include(includeme)
# Add testing views if we're in testing mode.
if mock_browserid:
config.scan("appsync.tests.views")
config.registry['mock_browserid'] = True
app = config.make_wsgi_app()
retry_after = config.settings.get('global.retry_after', '120')
errapp = CatchAuthError(app, retry_after=retry_after)
errapp.registry = app.registry
return errapp
def get_ssl_verify_option(method, url, settings, request_options=None):
# type: (str, str, AnySettingsContainer, Optional[SettingsType]) -> bool
"""
Obtains the SSL verification option from combined settings from ``weaver.ssl_verify`` and parsed
``weaver.request_options`` file for the corresponding request.
:param method: request method (GET, POST, etc.).
:param url: request URL.
:param settings: application setting container with pre-loaded *request options* specifications.
:param request_options: pre-processed *request options* for method/URL to avoid re-parsing the settings.
:returns: SSL ``verify`` option to be passed down to some ``request`` function.
"""
if not settings:
return True
settings = get_settings(settings)
if not asbool(settings.get("weaver.ssl_verify", True)):
return False
req_opts = request_options or get_request_options(method, url, settings)
if not req_opts.get("ssl_verify", req_opts.get("verify", True)):
return False
return True
def unregister_service_view(request):
"""
Unregister a service.
"""
service = ar.get_service_matchdict_checked(request)
service_push = asbool(ar.get_multiformat_body(request, "service_push", default=False))
svc_content = sf.format_service(service, show_private_url=True)
svc_res_id = service.resource_id
ax.evaluate_call(lambda: models.RESOURCE_TREE_SERVICE.delete_branch(resource_id=svc_res_id, db_session=request.db),
fallback=lambda: request.db.rollback(), http_error=HTTPForbidden,
msg_on_fail="Delete service from resource tree failed.", content=svc_content)
def remove_service_magpie_and_phoenix(svc, svc_push, db_session):
db_session.delete(svc)
if svc_push and svc.type in SERVICES_PHOENIX_ALLOWED:
sync_services_phoenix(db_session.query(models.Service))
ax.evaluate_call(lambda: remove_service_magpie_and_phoenix(service, service_push, request.db),
fallback=lambda: request.db.rollback(), http_error=HTTPForbidden,
msg_on_fail=s.Service_DELETE_ForbiddenResponseSchema.description, content=svc_content)
return ax.valid_http(http_success=HTTPOk, detail=s.Service_DELETE_OkResponseSchema.description)
def kotti_configure(settings):
""" Add a line like this to you .ini file::
kotti.configurators =
kotti_es.kotti_configure
to enable the ``kotti_es`` add-on.
:param settings: Kotti configuration dictionary.
:type settings: dict
"""
settings['pyramid.includes'] += ' kotti_es'
override_search = asbool(settings.get('kotti_es.override_search_content',
True))
if override_search:
settings['kotti.search_content'] = 'kotti_es.util.es_search_content'
if 'kotti_es.index_action' not in settings:
settings['kotti_es.index_action'] = 'kotti_es.sqla.default_index_action'
if 'kotti_es.is_blacklisted' not in settings:
settings['kotti_es.is_blacklisted'] = 'kotti_es.util.is_blacklisted'
def configure_engine(settings, test_setup=False):
from .renderers import json_renderer
engine_url = settings.get('sqlalchemy.url')
if not engine_url:
# Already setup by test fixture
return None
engine_opts = {}
if engine_url.startswith('postgresql'):
engine_opts['json_serializer'] = json_renderer.dumps
engine = engine_from_config(settings, 'sqlalchemy.', **engine_opts)
if engine.url.drivername == 'postgresql':
timeout = settings.get('postgresql.statement_timeout')
if timeout:
timeout = int(timeout) * 1000
set_postgresql_statement_timeout(engine, timeout)
if test_setup:
return engine
if asbool(settings.get('create_tables', False)):
Base.metadata.create_all(engine)
DBSession.configure(bind=engine)
return engine
def piwik_UsersManager_hasSuperUserAccess(piwik_url, piwik_api_token, userLogin):
params = {
"module": "API",
"format": "JSON",
"token_auth": piwik_api_token
}
params["method"] = "UsersManager.hasSuperUserAccess"
params["userLogin"] = userLogin
result = requests.get(piwik_url, params=params, timeout=15)
if result.status_code != 200:
raise requests.ConnectionError()
content = result.json() # Content should be like {"value": true}
# log.debug( "piwik_UsersManager_hasSuperUserAccess", content)
if not content:
raise requests.ConnectionError()
return asbool(content.get("value", False))
def __init__(self, secret, persistent, defaults={}, prefix='', **settings):
self.persistent = persistent
unset = object()
kw = {}
# load string settings
for name in ('cookie_name', 'path', 'domain', 'hashalg'):
value = settings.get(prefix + name, defaults.get(name, unset))
if value is not unset:
kw[name] = value
# load boolean settings
for name in ('secure', 'include_ip', 'http_only', 'wild_domain',
'parent_domain', 'debug'):
value = settings.get(prefix + name, defaults.get(name, unset))
if value is not unset:
kw[name] = asbool(value)
# load int settings
for name in ('timeout', 'reissue_time', 'max_age'):
value = settings.get(prefix + name, defaults.get(name, unset))
if value is not unset:
kw[name] = int(value)
super(CWAuthTktAuthenticationPolicy, self).__init__(secret, **kw)
def includeme(config):
auto_reload = asbool(config.registry.settings.get('h.reload_assets',
False))
config.add_view(route_name='assets', view=assets_view)
config.add_view(route_name='assets_client', view=assets_client_view)
assets_env = Environment('/assets',
'h/assets.ini',
'build/manifest.json',
auto_reload=auto_reload)
assets_client_env = Environment(
'/assets/client',
'h/assets_client.ini',
'node_modules/hypothesis/build/manifest.json',
auto_reload=auto_reload)
# We store the environment objects on the registry so that the Jinja2
# integration can be configured in app.py
config.registry['assets_env'] = assets_env
config.registry['assets_client_env'] = assets_client_env
def trace_tween_factory(handler, registry):
# configuration
settings = registry.settings
service = settings.get('datadog_trace_service') or 'pyramid'
tracer = settings.get('datadog_tracer') or ddtrace.tracer
enabled = asbool(settings.get('datadog_trace_enabled', tracer.enabled))
# set the service info
tracer.set_service_info(
service=service,
app="pyramid",
app_type=AppTypes.web)
if enabled:
# make a request tracing function
def trace_tween(request):
with tracer.trace('pyramid.request', service=service, resource='404') as span:
setattr(request, '_datadog_span', span) # used to find the tracer in templates
response = None
try:
response = handler(request)
except Exception:
span.set_tag(http.STATUS_CODE, 500)
raise
finally:
span.span_type = http.TYPE
# set request tags
span.set_tag(http.URL, request.path)
if request.matched_route:
span.resource = request.matched_route.name
# set response tags
if response:
span.set_tag(http.STATUS_CODE, response.status_code)
if 500 <= response.status_code < 600:
span.error = 1
return response
return trace_tween
# if timing support is not enabled, return the original handler
return handler
def internal_server_error(context, request):
"""Generate the default internal server error page when exception falls through from a view.
This view is marked as CSRF exempt, so that HTTP POST requests to API endpoints do not cause additional Bad CSRF exception when HTTP 500 Internal server error is raised.
Also see https://github.com/Pylons/pyramid_tm/issues/40
"""
if HAS_NEW_PYRAMID_TM:
if not can_access_transaction_in_excview(context, request):
# Kill some of db aware properties in the case templates
# might accidentally touch them
request.__dict__["user"] = None
else:
# We should have db and request.user available,
# we have just started a new transction for this view
reset_transaction_aware_properties(request)
else:
request.__dict__["user"] = None
# Tell Sentry handler to log this exception on sentry
request.registry.notify(InternalServerError(context, request))
if asbool(
request.registry.settings.get("websauna.log_internal_server_error",
True)):
logger.exception(context)
html = render('core/internalservererror.html', {}, request=request)
resp = Response(html)
resp.status_code = 500
# Hint pyramid_redis_session not to generate any session cookies for this response
resp.cache_control.public = True
# Make sure nothing is written or no transaction left open on 500
request.tm.abort()
return resp
def ops_published_data_search_handler(request):
"""Search for published-data at OPS"""
# Constituents: abstract, biblio and/or full-cycle
constituents = request.params.get('constituents', 'full-cycle')
# CQL query string
query = request.params.get('expression', '')
log.info('query raw: %s', query)
query_object, query = cql_prepare_query(query)
log.info('query cql: %s', query)
# range: x-y, maximum delta is 100, default is 25
range = request.params.get('range')
range = range or '1-25'
# Search options
family_swap_default = asbool(request.params.get('family_swap_default'))
try:
if family_swap_default:
result = ops_published_data_search_swap_family(
constituents, query, range)
else:
result = ops_published_data_search(constituents, query, range)
propagate_keywords(request, query_object)
return result
except NoResultsException as ex:
# Forward response to let the frontend recognize zero hits
request.response.status = HTTPNotFound.code
return ex.data
except Exception as ex:
message = handle_generic_exception(request, ex, 'ops-search', query)
request.errors.add('ops-search', 'search', message)
log.info('query finished')
def includeme(config):
"""Add routing and static view to deliver the frontend application."""
config.include('pyramid_cachebust')
config.include('pyramid_mako')
settings = config.registry.settings
cachebust_enabled = asbool(settings.get('cachebust.enabled', 'false'))
if cachebust_enabled:
cache_max_age = 30 * 24 * 60 * 60 # 30 days
else:
cache_max_age = 0
config.add_route('config_json', 'config.json')
config.add_view(config_view,
route_name='config_json',
renderer='json',
http_cache=(cache_max_age, {
'public': True
}))
add_frontend_route(config, 'embed', 'embed/{directive}')
add_frontend_route(config, 'register', 'register')
add_frontend_route(config, 'login', 'login')
add_frontend_route(config, 'password_reset', 'password_reset/')
add_frontend_route(config, 'create_password_reset',
'create_password_reset')
add_frontend_route(config, 'activate', 'activate/{key}')
add_frontend_route(config, 'activation_error', 'activation_error')
add_frontend_route(config, 'root', '')
add_frontend_route(config, 'resource', 'r/*path')
config.add_route('require_config', 'static/require-config.js')
config.add_view(require_config_view,
route_name='require_config',
http_cache=(cache_max_age, {
'public': True
}))
# AdhocracySDK shall not be cached the way other static files are cached
config.add_route('adhocracy_sdk', 'AdhocracySDK.js')
config.add_view(adhocracy_sdk_view, route_name='adhocracy_sdk')
config.add_static_view('static',
'adhocracy_frontend:build/',
cache_max_age=cache_max_age)
config.add_subscriber(add_cors_headers, NewResponse)
def includeme(config):
""" Initialize views and renderers at app start-up time. """
settings = config.get_settings()
config.add_renderer('json', json_renderer_factory)
config.include('.traversal')
default_discussion = settings.get('default_discussion', None)
if default_discussion:
config.add_route('discussion_list', '/discussions')
config.add_view(
lambda req: HTTPFound('/'+default_discussion),
route_name='default_disc_redirect')
config.add_route('default_disc_redirect', '/')
else:
config.add_route('discussion_list', '/')
config.include(backbone_include, route_prefix='/{discussion_slug}')
if asbool(config.get_settings().get('assembl_handle_exceptions', 'true')):
config.add_view(error_view, context=Exception)
# authentication
config.include('.auth')
config.include('.api')
config.include('.api2')
config.include('.home')
config.include('.admin')
config.add_route('home', '/{discussion_slug}')
config.add_route('home-auto', '/{discussion_slug}/')
def redirector(request):
return HTTPMovedPermanently(request.route_url('home', discussion_slug=request.matchdict.get('discussion_slug')))
config.add_view(redirector, route_name='home-auto')
default_context['cache_bust'] = \
config.registry.settings['requirejs.cache_bust']
def _subclass_specific_config(cls, settings, common_config):
"""Extract GCP-specific config settings: specifically, the path to
the service account key file, and the project id. Both are
optional.
"""
service_account_json_filename = settings.get(
"storage.gcp_service_account_json_filename") or os.getenv(
"GOOGLE_APPLICATION_CREDENTIALS")
if (service_account_json_filename
and not os.path.isfile(service_account_json_filename)
and not cls.test):
raise Exception(
"Service account json file not found at path {}".format(
service_account_json_filename))
bucket_name = settings.get("storage.bucket")
if bucket_name is None:
raise ValueError("You must specify the 'storage.bucket'")
iam_signer_service_account_email = settings.get(
"storage.iam_signer_service_account_email")
if iam_signer_service_account_email is None and service_account_json_filename:
with io.open(service_account_json_filename, "r",
encoding="utf-8") as ifile:
credentials = json.load(ifile)
iam_signer_service_account_email = credentials.get("client_email")
return {
"service_account_json_filename":
service_account_json_filename,
"project_id":
settings.get("storage.gcp_project_id"),
"use_iam_signer":
asbool(settings.get("storage.gcp_use_iam_signer", False)),
"iam_signer_service_account_email":
iam_signer_service_account_email,
"bucket_factory":
lambda: cls.get_bucket(bucket_name, settings),
}
def setup_version_redirection(config):
"""Add a view which redirects to the current version of the API."""
settings = config.get_settings()
redirect_enabled = settings["version_prefix_redirect_enabled"]
version_prefix_redirection_enabled = asbool(redirect_enabled)
cache_seconds = int(settings["version_prefix_redirect_ttl_seconds"])
route_prefix = config.route_prefix
config.registry.route_prefix = route_prefix
# Redirect to the current version of the API if the prefix isn't used.
# Do not redirect if kinto.version_prefix_redirect_enabled is set to
# False.
if not version_prefix_redirection_enabled:
return
def _redirect_to_version_view(request):
if request.method.lower() == "options":
# CORS responses should always have status 200.
return utils.reapply_cors(request, Response())
querystring = request.url[(request.url.rindex(request.path) + len(request.path)) :]
redirect = f"/{route_prefix}{request.path}{querystring}"
resp = HTTPTemporaryRedirect(redirect)
if cache_seconds >= 0:
resp.cache_expires(cache_seconds)
raise resp
# Disable the route prefix passed by the app.
config.route_prefix = None
config.add_route(name="redirect_to_version", pattern=r"/{path:(?!v[0-9]+)[^\r\n]*}")
config.add_view(
view=_redirect_to_version_view,
route_name="redirect_to_version",
permission=NO_PERMISSION_REQUIRED,
)
config.route_prefix = route_prefix
def includeme(config):
settings = config.registry.settings
if asbool(settings.get('phoenix.wizard', 'false')):
config.include('phoenix.twitcherclient')
config.include('phoenix.wizard.views.start')
config.include('phoenix.wizard.views.wps')
config.include('phoenix.wizard.views.wpsprocess')
config.include('phoenix.wizard.views.literalinputs')
config.include('phoenix.wizard.views.complexinputs')
config.include('phoenix.wizard.views.source')
config.include('phoenix.wizard.views.esgfsearch')
config.include('phoenix.wizard.views.threddsservice')
config.include('phoenix.wizard.views.solrsearch')
config.include('phoenix.wizard.views.done')
# add malleefowl wps
def get_wps(request):
settings = request.registry.settings
if 'wps' not in settings:
logger.debug('register malleefowl wps service')
try:
service_name = 'malleefowl'
registry = twitcher_service_factory(request.registry)
logger.debug("register: name=%s, url=%s", service_name, settings['wps.url'])
registry.register_service(url=settings['wps.url'], data={'name': service_name}, overwrite=True)
settings['wps'] = WebProcessingService(
url=request.route_url('owsproxy', service_name=service_name),
skip_caps=True, verify=False)
logger.debug('malleefowl proxy url: %s', settings['wps'].url)
except:
logger.exception('Could not connect malleefowl wps %s', settings['wps.url'])
return settings.get('wps')
config.add_request_method(get_wps, 'wps', reify=True)
# check if wizard is activated
def wizard_activated(request):
return asbool(settings.get('phoenix.wizard', 'false'))
config.add_request_method(wizard_activated, reify=True)
def get_queried_jobs(request):
"""
Retrieve the list of jobs which can be filtered, sorted, paged and categorized using query parameters.
"""
settings = get_settings(request)
service, process = validate_service_process(request)
detail = asbool(request.params.get("detail", False))
page = int(request.params.get("page", "0"))
limit = int(request.params.get("limit", "10"))
email = request.params.get("notification_email", None)
filters = {
"page": page,
"limit": limit,
# split by comma and filter empty stings
"tags": list(filter(lambda s: s, request.params.get("tags", "").split(","))),
"access": request.params.get("access", None),
"status": request.params.get("status", None),
"sort": request.params.get("sort", sort.SORT_CREATED),
"notification_email": encrypt_email(email, settings) if email else None,
# service and process can be specified by query (short route) or by path (full route)
"process": process,
"service": service,
}
groups = request.params.get("groups", "")
groups = groups.split(",") if groups else None
store = get_db(request).get_store(StoreJobs)
items, total = store.find_jobs(request=request, group_by=groups, **filters)
body = {"total": total}
def _job_list(jobs):
return [j.json(settings) if detail else j.id for j in jobs]
if groups:
for grouped_jobs in items:
grouped_jobs["jobs"] = _job_list(grouped_jobs["jobs"])
body.update({"groups": items})
else:
body.update({"jobs": _job_list(items), "page": page, "limit": limit})
body = sd.GetQueriedJobsSchema().deserialize(body)
return HTTPOk(json=body)
def get_file(request):
# TODO: Add a route that enables the call to have the filename
# appended to the end. This is so that gmail can read the services
# Read more here:
# http://stackoverflow.com/questions/20903967/gmails-new-image-caching-is-breaking-image-links-in-newsletter
if request.method == 'HEAD':
# GET view_config captures HEAD...
return get_file_header(request)
ctx = request.context
document = ctx._instance
f = File.get(document.id)
if f.infected:
raise HTTPNotAcceptable("Infected with a virus")
handoff_to_nginx = asbool(config.get('handoff_to_nginx', False))
if handoff_to_nginx:
kwargs = dict(body='')
else:
if 'Range' in request.headers:
raise HTTPRequestRangeNotSatisfiable()
fs = f.file_stream
app_iter = None
environ = request.environ
if 'wsgi.file_wrapper' in environ and f.path:
app_iter = environ['wsgi.file_wrapper'](fs, _BLOCK_SIZE)
if app_iter is None:
app_iter = FileIter(fs, _BLOCK_SIZE)
kwargs = dict(app_iter=app_iter)
r = Response(
content_length=f.file_size,
content_type=str(f.mime_type),
last_modified=f.creation_date,
expires=datetime.now() + timedelta(days=365),
accept_ranges="bytes" if handoff_to_nginx else "none",
content_disposition=disposition(f.title), # RFC 6266
**kwargs
)
if handoff_to_nginx:
r.headers[b'X-Accel-Redirect'] = f.handoff_url
return r
def includeme(config):
"""
For inclusion by Pyramid.
"""
config.add_request_method(get_user, str('user'), reify=True)
config.add_request_method(get_acls, str('acls'), reify=True)
config.add_request_method(get_settings, str('settings'), reify=True)
settings = config.registry.settings
authn_policy = PluginAuthenticationPolicy(
SessionAuthenticationPolicy(callback=find_princs), {
'/dav':
DigestAuthenticationPolicy(
settings.get('netprofile.auth.secret'),
find_princs_digest,
realm=settings.get('netprofile.auth.digest.realm',
'NetProfile UI'),
check_timestamp=asbool(
settings.get('netprofile.auth.digest.check_timestamp',
'true')),
timestamp_max_ahead=int(
settings.get('netprofile.auth.digest.timestamp_max_ahead',
5)),
timestamp_max_behind=int(
settings.get('netprofile.auth.digest.timestamp_max_behind',
120))),
'/api':
BasicAuthAuthenticationPolicy(
find_princs_basic,
settings.get('netprofile.auth.rpc_realm', 'NetProfile RPC'),
settings.get('netprofile.debug'))
})
authz_policy = ACLAuthorizationPolicy()
config.set_authorization_policy(authz_policy)
config.set_authentication_policy(authn_policy)
config.add_subscriber(_auth_to_db, ContextFound)
config.add_subscriber(_check_session, PluginPolicySelected)
def render(self):
settings = self.request.registry.settings
ldap = False
if 'pyvac.use_ldap' in settings:
ldap = asbool(settings.get('pyvac.use_ldap'))
if 'form.cancelled' in self.request.params:
return self.redirect()
log.debug('rendering %s' % self.__class__.__name__)
errors = []
model = self.get_model()
if self.user and not self.user.is_admin:
if model.id != self.user.id:
return self.redirect('home')
if 'form.submitted' in self.request.params:
self.validate(model, errors)
if not errors:
try:
self.update_model(model)
model.validate(self.session, ldap=ldap)
except ModelError as err:
errors.extend(err.errors)
if not errors:
self.save_model(model)
return self.redirect()
rv = {'errors': errors,
self.model.__tablename__: model,
'use_ldap': ldap,
'csrf_token': self.request.session.get_csrf_token()}
self.update_view(model, rv)
log.debug(repr(rv))
return rv
def get_connection(self, conn_type='ec2', cloud_type=None, region=None, access_key=None,
secret_key=None, security_token=None):
conn = None
if cloud_type is None:
cloud_type = self.cloud_type
if region is None:
region = self.region
if access_key is None:
access_key = self.access_key
if secret_key is None:
secret_key = self.secret_key
if security_token is None:
security_token = self.security_token
validate_certs = False
if self.request.registry.settings: # do this to pass tests
validate_certs = asbool(self.request.registry.settings.get('connection.ssl.validation', False))
certs_file = self.request.registry.settings.get('connection.ssl.certfile', None)
try:
if cloud_type == 'aws':
conn = ConnectionManager.aws_connection(
region, access_key, secret_key, security_token, conn_type, validate_certs)
elif cloud_type == 'euca':
host = self._get_ufs_host_setting_()
port = self._get_ufs_port_setting_()
dns_enabled = self.request.session.get('dns_enabled', True)
regions = RegionCache(None).regions()
if len(regions) > 0:
for region in regions:
if region['endpoints']['ec2'].find(host) > -1:
self.default_region = region['name']
conn = ConnectionManager.euca_connection(
host, port, region, access_key, secret_key, security_token,
conn_type, dns_enabled, validate_certs, certs_file
)
except socket.error as err:
BaseView.handle_error(err=BotoServerError(504, str(err)), request=self.request)
return conn
def collection_add(context, request, render=None):
if render is None:
render = request.params.get('render', True)
properties = request.validated
item = context.add(properties)
request.registry.notify(Created(item, request))
if render == 'uuid':
item_uri = '/%s' % item.uuid
else:
item_uri = request.resource_path(item)
if asbool(render) is True:
rendered = embed(request, item_uri + '?embed=false')
else:
rendered = item_uri
request.response.status = 201
request.response.location = item_uri
result = {
'status': 'success',
'@type': ['result'],
'@graph': [rendered],
}
return result
def __init__(self, dbsession, **settings):
"""Initializes Whoosh by setting up and loading indexes for lookup."""
self._dbsession = dbsession
self.schema = ChipSchema()
self.directory = settings.get(
'whoosh.store',
os.path.join(settings['config_path'], 'whoosh-data')
)
self.indexname = settings.get(
'whoosh.indexname',
'chips'
)
self.rebuild = asbool(settings.get('whoosh.rebuild', 'false'))
self.storage = FileStorage(self.directory)
self.setindex()
if self.rebuild:
self.setindex()
self.buildindex()
else:
self.setindex()
