def configure(self, config):
"""Load config settings for this extension from config file.
See IConfigurable.
"""
if "googleanalytics.id" not in config:
msg = "Missing googleanalytics.id in config"
raise GoogleAnalyticsException(msg)
self.googleanalytics_id = config["googleanalytics.id"]
self.googleanalytics_domain = config.get("googleanalytics.domain", "auto")
self.googleanalytics_javascript_url = h.url_for_static("/scripts/ckanext-googleanalytics.js")
# If resource_prefix is not in config file then write the default value
# to the config dict, otherwise templates seem to get 'true' when they
# try to read resource_prefix from config.
if "googleanalytics_resource_prefix" not in config:
config["googleanalytics_resource_prefix"] = commands.DEFAULT_RESOURCE_URL_TAG
self.googleanalytics_resource_prefix = config["googleanalytics_resource_prefix"]
self.show_downloads = converters.asbool(config.get("googleanalytics.show_downloads", True))
self.track_events = converters.asbool(config.get("googleanalytics.track_events", False))
if not converters.asbool(config.get("ckan.legacy_templates", "false")):
p.toolkit.add_resource("fanstatic_library", "ckanext-googleanalytics")
# spawn a pool of 5 threads, and pass them queue instance
for i in range(5):
t = AnalyticsPostThread(self.analytics_queue)
t.setDaemon(True)
t.start()
def configure(self, config):
'''Load config settings for this extension from config file.
See IConfigurable.
'''
if 'googleanalytics.id' not in config:
msg = "Missing googleanalytics.id in config"
raise GoogleAnalyticsException(msg)
self.googleanalytics_id = config['googleanalytics.id']
self.googleanalytics_domain = config.get(
'googleanalytics.domain', 'auto')
self.googleanalytics_javascript_url = h.url_for_static(
'/scripts/ckanext-googleanalytics.js')
# If resource_prefix is not in config file then write the default value
# to the config dict, otherwise templates seem to get 'true' when they
# try to read resource_prefix from config.
if 'googleanalytics_resource_prefix' not in config:
config['googleanalytics_resource_prefix'] = (
commands.DEFAULT_RESOURCE_URL_TAG)
self.googleanalytics_resource_prefix = config[
'googleanalytics_resource_prefix']
self.show_downloads = converters.asbool(
config.get('googleanalytics.show_downloads', True))
self.track_events = converters.asbool(
config.get('googleanalytics.track_events', False))
if not converters.asbool(config.get('ckan.legacy_templates', 'false')):
p.toolkit.add_resource('fanstatic_library', 'ckanext-googleanalytics')
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[os.path.join(root, 'templates')])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='nwrsc', paths=paths)
config['routes.map'] = make_map()
config['pylons.app_globals'] = app_globals.Globals()
config['pylons.h'] = nwrsc.lib.helpers
config['nwrsc.onsite'] = asbool(config.get('nwrsc.onsite', 'false'))
config['nwrsc.private'] = asbool(config.get('nwrsc.private', 'false'))
if 'archivedir' not in config:
config['archivedir'] = '/doesnotexist'
# Create the Mako TemplateLookup, with the default auto-escaping
config['pylons.app_globals'].mako_lookup = DatabaseLookup(
directories=paths['templates'],
error_handler=handle_mako_error,
module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
input_encoding='utf-8', output_encoding='utf-8',
imports=['from webhelpers.html import escape; from nwrsc.lib.helpers import oneline'],
default_filters=['escape'])
def make_open_id_middleware(
app,
global_conf,
# Should this default to something, or inherit something from global_conf?:
data_store_path,
auth_prefix="/oid",
login_redirect=None,
catch_401=False,
url_to_username=None,
apply_auth_tkt=False,
auth_tkt_logout_path=None,
):
from paste.deploy.converters import asbool
from paste.util import import_string
catch_401 = asbool(catch_401)
if url_to_username and isinstance(url_to_username, str):
url_to_username = import_string.eval_import(url_to_username)
apply_auth_tkt = asbool(apply_auth_tkt)
new_app = AuthOpenIDHandler(
app,
data_store_path=data_store_path,
auth_prefix=auth_prefix,
login_redirect=login_redirect,
catch_401=catch_401,
url_to_username=url_to_username or None,
)
if apply_auth_tkt:
from paste.auth import auth_tkt
new_app = auth_tkt.make_auth_tkt_middleware(new_app, global_conf, logout_path=auth_tkt_logout_path)
return new_app
def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``global_conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
config = load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp(config=config)
# Routing/Session Middleware
app = RoutesMiddleware(app, config['routes.map'])
app = SessionMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app, [417])
else:
app = StatusCodeRedirect(app, [400, 401, 403, 404, 417, 500])
# authenticator = OCSAuthenticator(config)
# app = AuthBasicHandler(app, "OCSManager", authenticator)
fqdn = "%(hostname)s.%(dnsdomain)s" % config["samba"]
app = NTLMAuthHandler(app, samba_host=fqdn)
# Establish the Registry for this application
app = RegistryManager(app)
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, app])
app.config = config
return app
def connect(self, config, mailbox, username, password):
self.mailbox = mailbox
self.server = IMAPClient(config.get('mail.host'),
use_uid=asbool(config.get('mail.uid', True)),
ssl=asbool(config.get('mail.ssl', True)))
self.server.login(username, password)
self.server.select_folder(mailbox)
def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``global_conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp()
# Routing/Session/Cache Middleware
app = RoutesMiddleware(app, config['routes.map'])
app = SessionMiddleware(app, config)
app = CacheMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
app = twa.make_middleware(app, {
'toscawidgets.framework': 'pylons',
'toscawidgets.framework.default_view': 'mako',
})
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app)
else:
app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
# Establish the Registry for this application
app = RegistryManager(app)
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, app])
return app
def before_index(self, pkg_dict):
# Resource type is multivalue field
types = []
for tag_string in pkg_dict.get('res_type', []):
if tag_string:
types += [tag.strip() for tag in tag_string.split(',')]
pkg_dict['res_type'] = types
# Resuse conditions
from paste.deploy.converters import asbool
# TODO prefix extras_ - create searchable text in PL/EN
pkg_dict['license_condition_modification'] = asbool(pkg_dict.get('license_condition_modification'))
pkg_dict['license_condition_original'] = asbool(pkg_dict.get('license_condition_original'))
pkg_dict['license_condition_source'] = asbool(pkg_dict.get('license_condition_source'))
pkg_dict['license_condition_timestamp'] = asbool(pkg_dict.get('license_condition_timestamp'))
pkg_dict['license_condition_responsibilities'] = bool(pkg_dict.get('license_condition_responsibilities','').strip())
pkg_dict['has_any_reuse_conditions'] = pkg_dict['license_condition_modification'] \
or pkg_dict['license_condition_original'] or pkg_dict['license_condition_source'] \
or pkg_dict['license_condition_timestamp'] or pkg_dict['license_condition_responsibilities']
restrictions = []
for restr in ['modification', 'original', 'source', 'timestamp', 'responsibilities']:
if pkg_dict['license_condition_' + restr]:
restrictions.append(restr)
if not restrictions:
restrictions = ['none']
pkg_dict['vocab_reuse_conditions'] = restrictions
return pkg_dict
def __init__(self):
self.debug = asbool(config.get('debug', False))
self.site_title = config.get(
'openspending.site_title',
'OpenSpending ETL'
)
self.site_slogan = config.get(
'openspending.site_slogan',
'Extract, Transform, Load'
)
self.site_logo = config.get(
'openspending.site_logo',
'/images/datastore-logo.png'
)
self.wiki_link = config.get(
'openspending.wiki_link',
'http://wiki.openspending.org'
)
self.api_link = config.get(
'openspending.api_link',
'http://wiki.openspending.org/API'
)
self.lists_link = config.get(
'openspending.lists_link',
'http://lists.okfn.org/mailman/listinfo/openspending-discuss'
)
self.sandbox_mode = asbool(config.get(
'openspending.sandbox_mode',
False
))
def __init__(self):
'''
:ivar to_addr: The default recipient email address
Will be set to the configuration variable "email_to",
if configuration is available
:ivar from_addr: The default sender email address
Will be set to the configuration variable "admin_email_from",
if configuration is available
'''
self.to_addr = config.get('email_to')
self.from_addr = config.get('admin_email_from')
smtp_server = config.get('smtp_server')
smtp_use_tls = asbool(config.get('smtp_use_tls'))
smtp_username = config.get('smtp_username')
smtp_password = config.get('smtp_password')
test = asbool(config.get('test'))
if test:
log.debug('Using DummyDelivery()')
self.delivery = DummyDelivery()
else: # pragma: no cover
if smtp_server:
mailer = SMTPMailer(hostname=smtp_server,
username=smtp_username, password=smtp_password,
force_tls=smtp_use_tls)
log.debug('Using SMTPMailer(hostname=%s, ...)', smtp_server)
else:
mailer = SendmailMailer()
log.debug('Using SendmailMailer()')
self.delivery = DirectMailDelivery(mailer)
def run_scgi_thread(wsgi_app, global_conf,
scriptName='', host=None, port=None, socket=None, umask=None,
allowedServers='127.0.0.1', logRequest=True,
debug=None):
import flup.server.scgi
if socket:
assert host is None and port is None
sock = socket
elif host:
assert host is not None and port is not None
sock = (host, int(port))
ensure_port_cleanup([sock])
else:
sock = None
if umask is not None:
umask = int(umask)
if debug is None:
debug = global_conf.get('debug', False)
s = flup.server.scgi.WSGIServer(
wsgi_app,
scriptName=scriptName,
bindAddress=sock,
umask=umask,
allowedServers=aslist(allowedServers),
debug=asbool(debug),
loggingLevel=asbool(logRequest) and logging.INFO or logging.WARNING
)
# Remove all the private handlers
for handler in s.logger.handlers:
s.logger.removeHandler(handler)
s.run()
def document(self):
resp = request.environ.get('pylons.original_response')
if resp is None:
raise abort(404)
response.status = resp.status
if resp.content_type == 'text/javascript':
response.content_type == resp.content_type
return resp.body
# YOU DO NOT SEE THIS. IF YOU DO, ITS NOT WHAT IT LOOKS LIKE
# I DID NOT HAVE REGEX RELATIONS WITH THAT HTML PAGE
for match in BODY_RE.finditer(resp.body):
c.error_message = match.group(1)
c.error_code = cgi.escape(request.GET.get('code',
str(resp.status_int)))
if not c.error_message:
c.error_message = _("Error %s") % c.error_code
if asbool(config.get('adhocracy.interactive_debugging', 'false')):
c.trace_url = request.environ['pylons.original_response']\
.headers.get('X-Debug-URL', None)
if c.trace_url is not None:
# this may only happen in debug mode
assert(asbool(config.get('debug', 'false')))
else:
c.trace_url = None
return render("/error/http.html")
def __init__(self,
app,
secret,
name='authkit',
params=None,
includeip=True,
signoutpath=None,
enforce=False,
ticket_class=AuthKitTicket,
nouserincookie=False,
session_middleware='beaker.session'
):
log.debug("Setting up the cookie middleware")
secure = False
if params.has_key('secure') and asbool(params['secure']) == True:
secure = True
# secure not needed!
AuthTKTMiddleware.__init__(self, app, secret, cookie_name=name,
secure=secure, include_ip=asbool(includeip),
logout_path=signoutpath)
self.ticket_class = ticket_class
self.cookie_params = params and params.copy() or {}
self.cookie_enforce = enforce
if self.cookie_enforce and not self.cookie_params.has_key('expires'):
raise AuthKitConfigError(
"Cannot enforce cookie expiration since no "
"cookie_params expires' has been set"
)
self.nouserincookie = nouserincookie
self.session_middleware = session_middleware
def deployservice(self, service):
""" activate manifest, if already active then skip """
LOG.info('deploy service for service(%s) with body: %s', service, request.body)
try:
appGlobal = config['pylons.app_globals']
# parse the body
if (request.body == ""):
return errorResult(request, response, Errors.INVALID_REQUEST,
'No body found in post command',
controller = self)
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
packages = requestjson['package']
skipProp = asbool(requestjson['skipProp']) if 'skipProp' in requestjson else configutil.getConfigAsBool('download_skip_prop')
skipActivation = asbool(requestjson['skipActivation']) if 'skipActivation' in requestjson else False
# activate manifest if not already activated
if manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
deployServiceThread = DeployService(appGlobal.threadMgr, service, manifest, packages, skipProp = skipProp, skipActivation = skipActivation)
self.injectJobCtx(deployServiceThread)
deployServiceThread.start()
deployServiceThread.threadMgrEvent.wait()
return statusResult(request, response, deployServiceThread, controller = self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def __init__(self):
"""One instance of Globals is created during application
initialization and is available during requests via the
'app_globals' variable
"""
self.map_key = config['map_key']
self.captcha_enabled = asbool(config['captcha_enabled'])
self.captcha_pubkey = config['captcha_pubkey']
self.captcha_privkey = config['captcha_privkey']
self.akismet_enabled = asbool(config['akismet_enabled'])
self.akismet_key = config['akismet_key']
self.akismet_url = config['akismet_url']
self.allow_tags = aslist(config['allow_tags'], sep=',')
host_whitelist = aslist(config['host_whitelist'], sep=',')
# prefix www subdomain to all hosts as a convenience
self.host_whitelist = set(host_whitelist)
for host in host_whitelist:
if not host.startswith('www.'):
self.host_whitelist.add('www.' + host)
self.support_sending_enabled = asbool(config['support_sending_enabled'])
self.support_email = config['support_email']
self.smtp_host = config['smtp_server']
self.smtp_port = config['smtp_port']
def _make_app(self, config, full_stack=True, static_files=True):
# The Pylons WSGI app
log.pcore.debug("Initializing middleware...")
app = PylonsApp(config=config)
# Routing/Session Middleware
app = RoutesMiddleware(app, config['routes.map'], singleton=False)
app = SessionMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
spells = Implementations(IMiddlewareSpell)
for spell in spells:
app = spell.add_middleware(app)
if asbool(full_stack):
# Handle Python exceptions
global_conf = config # I think that it's correct, config is slightly modified global_conf
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app)
else:
app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
# Establish the Registry for this application
app = RegistryManager(app)
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, app])
app.config = config
return app
def __init__(self, app, config):
# pylint: disable=F0401
from paste.deploy.converters import asbool
# pylint: enable=F0401
self.app = app
self.compact_html = asbool(config.get('compact.html', True))
self.compact_script = asbool(config.get('compact.script', True))
def init_config(self, global_conf, app_conf):
"""Initialize the config object.
tg.config is a proxy for pylons.configuration.config that allows
attribute style access, so it's automatically setup when we create
the pylons config.
Besides basic initialization, this method copies all the values
in base_config into the ``pylons.configuration.config`` and
``tg.config`` objects.
"""
pylons_config.init_app(global_conf, app_conf,
package=self.package.__name__,
paths=self.paths)
self.auto_reload_templates = asbool(config.get('auto_reload_templates', True))
pylons_config['application_root_module'] = self.get_root_module()
config.update(self)
# set up the response options to None. This allows
# you to set the proper content type within a controller method
# if you choose.
pylons_config['pylons.response_options']['headers']['Content-Type'] = None
#see http://trac.turbogears.org/ticket/2247
if asbool(config['debug']):
config['pylons.strict_tmpl_context'] = True
else:
config['pylons.strict_tmpl_context'] = False
self.after_init_config()
def run(wsgi_app, global_conf,
host='0.0.0.0', port='8080',
multiplayer=False,
testing=False):
port = int(port)
multiplayer = asbool(multiplayer)
testing = asbool(testing)
wsgi_app = DevserverWSGIContainer(wsgi_app)
handlers = []
if multiplayer:
handlers.append(('/multiplayer/(.*)/(.*)', MultiplayerHandler))
handlers.append(('/api/v1/multiplayer/status', MultiplayerStatusHandler))
handlers.append(('/api/v1/multiplayer/status/session/(.*)', SessionStatusHandler))
if testing:
raw_response_dir = path_join(dirname(__file__), 'raw-response')
handlers.append(('/raw-response/(.*)',
ResponseFromFileHandler, dict(path=raw_response_dir)))
handlers.append(('/local/v1/save/([^/]+)/(.*)', SaveFileHandler))
handlers.append(('.*', FallbackHandler, dict(fallback=wsgi_app)))
tornado_app = DevserverApplication(handlers, transforms=[])
handlers = None
server = HTTPServer(tornado_app)
server.listen(port, host)
print 'Serving on %s:%u view at http://127.0.0.1:%u' % (host, port, port)
IOLoop.instance().start()
def app_factory(global_config, **local_conf):
"""Create a WSGI application instance."""
global CONFIG
#initialize global config object
CONFIG.update_values(global_config.copy())
CONFIG.update_values(local_conf)
application = DuendeApplication(CONFIG)
application = AuthMiddleware(application, CONFIG)
application = ViewResolverMiddleware(application, CONFIG)
application = FlashMessageMiddleware(application)
application = RegistryManager(application)
application = SessionMiddleware(application, CONFIG)
application = ErrorMiddleware(application, CONFIG)
if asbool(CONFIG['debug']):
if asbool(CONFIG.get('debug.multiprocess')):
#This middleware supports multithread
application = CgitbMiddleware(application, {'debug': True})
else:
#display errors to client when debug is enabled
application = EvalException(application)
return application
def events(self, id, format=u'html'):
if not c.instance.display_category_pages:
abort(404)
category = get_entity_or_abort(model.CategoryBadge, id)
events = h.category.event_q(
category,
event_filter=request.params.getall('event_filter'),
count=min(int(request.params.get('count', 50)), 100),
).all()
enable_sorts = asbool(request.params.get('enable_sorts', 'true'))
enable_pages = asbool(request.params.get('enable_pages', 'true'))
row_type = request.params.get('row_type', 'row')
if row_type not in ['row', 'profile_row', 'sidebar_row', 'tiny_row']:
abort(400)
data = {
'event_pager': pager.events(events,
enable_sorts=enable_sorts,
enable_pages=enable_pages,
row_type=row_type),
}
return render('/category/events.html', data,
overlay=format == 'overlay',
overlay_size=OVERLAY_SMALL)
def pre(self, command, output_dir, vars):
"""Called before template is applied."""
package_logger = vars["package"]
if package_logger == "root":
# Rename the app logger in the rare case a project is named 'root'
package_logger = "app"
vars["package_logger"] = package_logger
template_engine = vars.setdefault("template_engine", pylons.configuration.default_template_engine)
if template_engine == "mako":
# Support a Babel extractor default for Mako
vars[
"babel_templates_extractor"
] = "('templates/**.mako', 'mako', {'input_encoding': 'utf-8'})" ",\n%s#%s" % (" " * 4, " " * 8)
else:
vars["babel_templates_extractor"] = ""
# Ensure these exist in the namespace
for name in self.ensure_names:
vars.setdefault(name, "")
vars["version"] = vars.get("version", "0.1")
vars["zip_safe"] = asbool(vars.get("zip_safe", "false"))
vars["sqlalchemy"] = asbool(vars.get("sqlalchemy", "false"))
def make_deliverance_middleware(app, global_conf,
rule_uri=None, rule_filename=None,
theme_uri=None,
debug=None,
execute_pyref=None):
assert sum([bool(x) for x in [rule_uri, rule_filename]]) == 1, (
"You must give one, and only one, of rule_uri or rule_filename")
if debug is None:
debug = asbool(global_conf.get('debug', False))
else:
debug = asbool(debug)
if rule_filename:
rule_getter = FileRuleGetter(rule_filename, always_reload=debug)
elif rule_uri.startswith('file://'):
rule_uri = rule_uri[len('file://'):]
rule_getter = FileRuleGetter(rule_uri, always_reload=debug)
else:
rule_getter = SubrequestRuleGetter(rule_uri)
execute_pyref = asbool(execute_pyref)
app = DeliveranceMiddleware(app, rule_getter, default_theme=theme_uri)
app = security.SecurityContext.middleware(
app,
display_local_files=debug, display_logging=debug,
execute_pyref=execute_pyref)
return app
def wrap_in_middleware( app, global_conf, **local_conf ):
conf = global_conf.copy()
conf.update(local_conf)
debug = asbool( conf.get( 'debug', False ) )
app = beaker.middleware.SessionMiddleware( app, conf )
app = GetSessionMiddleware( app, conf )
#from paste.translogger import TransLogger
#app = TransLogger( app, conf )
from paste import httpexceptions
app = httpexceptions.make_middleware( app, conf )
if debug:
from paste.debug import prints
app = prints.PrintDebugMiddleware( app, conf )
if debug and asbool( conf.get( 'use_interactive', False ) ):
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
from weberror import evalexception
app = evalexception.EvalException( app, conf )
LOG.debug( "Enabling 'eval exceptions' middleware" )
else:
# Not in interactive debug mode, just use the regular error middleware
from paste.exceptions import errormiddleware
app = errormiddleware.ErrorMiddleware( app, conf )
LOG.debug( "Enabling 'error' middleware" )
return app
def _setup(self):
def get_db_settings():
return meta.Session.query(model.Setting).all()
# set settings cache object
# Fails when I attempt to store this in type file
settings = app_globals.cache.get_cache(
'base._setup',
type='memory'
).get(key='settings', createfunc=get_db_settings)
pylons.c.settings = {}
for setting in settings:
pylons.c.settings[setting.key] = setting.value
pylons.c.display_tagline = asbool(pylons.c.settings.get('display_tagline', 'false'))
pylons.c.display_admin_email = asbool(pylons.c.settings.get('display_admin_email', 'false'))
pylons.c.enable_googlesearch = asbool(pylons.c.settings.get('enable_googlesearch', 'false'))
pylons.c.enable_googleanalytics = asbool(pylons.c.settings.get('enable_googleanalytics', 'false'))
pylons.c.enable_akismet = asbool(pylons.c.settings.get('enable_akismet', 'false'))
pylons.c.enable_twitter_display = asbool(pylons.c.settings.get('enable_twitter_display', 'false'))
pylons.c.enable_delicious_display = asbool(pylons.c.settings.get('enable_delicious_display', 'false'))
pylons.c.enable_flickr_display = asbool(pylons.c.settings.get('enable_flickr_display', 'false'))
# This is a temporary solution until I find the time to add these settings to the database
pylons.c.enable_github_display = True
pylons.c.settings['github_screenname'] = u'leveille'
pylons.c.enable_bitbucket_display = True
pylons.c.settings['bitbucket_screenname'] = u'leveille'
pylons.c.use_minified_assets = asbool(pylons.c.settings.get('use_minified_assets', 'false'))
pylons.c.use_externalposts_feed = asbool(pylons.c.settings.get('use_externalposts_feed', 'false'))
def setup(self):
self.model.__dict__['engine'] = engine_from_config(self.config, prefix="%s.sqlalchemy." % (self.prefix, ))
self.model.metadata.bind = self.model.engine
if self.soup:
from sqlalchemy.ext.sqlsoup import SqlSoup, Session
self.model.__dict__['soup'] = SqlSoup(self.model.metadata)
self._session = Session
else:
args = dict(
bind = self.model.engine,
autocommit = asbool(self.config.get('%s.autocommit' % (self.prefix, ), False)),
autoflush = asbool(self.config.get('%s.autoflush' % (self.prefix, ), True)),
twophase = asbool(self.config.get('%s.twophase' % (self.prefix, ), False)),
)
setup = getattr(self.model, 'setup', None)
if hasattr(setup, '__call__'):
args = setup(args)
self._session = sessionmaker(**args)
populate = getattr(self.model, 'populate', None)
if hasattr(populate, '__call__'):
for table in self.model.metadata.sorted_tables:
event.listen(table, 'after_create', self.populate_table)
def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``global_conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
config = load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp(config=config)
# Fixme: Decide where to place a reverse-proxy middleware!
if asbool(config.get('wordpresser.enable')):
wp_proxy_host = config.get('wordpresser.proxy_host', 'http://localhost:8080/')
wp_path_prefix = config.get('wordpresser.path_prefix', '')
app = WordpresserMiddleware(app, wp_proxy_host, wp_path_prefix)
# Routing/Session Middleware
app = RoutesMiddleware(app, config['routes.map'], singleton=False)
app = SessionMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401,403,404 status codes (and 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app)
else:
app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
# Establish the Registry for this application
app = RegistryManager(app)
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, app])
app.config = config
return app
def renderer_factory(info):
"""*info* contains:
::
name = Attribute('The value passed by the user as the renderer name')
package = Attribute('The "current package" when the renderer '
'configuration statement was found')
type = Attribute('The renderer type name')
registry = Attribute('The "current" application registry when the '
'renderer was created')
settings = Attribute('The ISettings dictionary related to the '
'current app')
"""
registry = info.registry
settings = info.settings
if not hasattr(registry, 'kajiki_loader'):
from kajiki import FileLoader
registry.kajiki_loader = FileLoader(
base=abspath_from_resource_spec(settings['kajiki.directory']),
reload=asbool(settings.get('reload_templates')),
force_mode=asbool(settings.get('kajiki.force_mode')),
autoescape_text=asbool(settings.get('kajiki.autoescape')),
)
return KajikiTemplateRenderer(info)
def pre(self, command, output_dir, vars):
"""Called before template is applied."""
package_logger = vars['package']
if package_logger == 'root':
# Rename the app logger in the rare case a project is named 'root'
package_logger = 'app'
vars['package_logger'] = package_logger
template_engine = \
vars.setdefault('template_engine',
pylons.configuration.default_template_engine)
if template_engine == 'mako':
# Support a Babel extractor default for Mako
vars['babel_templates_extractor'] = \
("('templates/**.mako', 'mako', {'input_encoding': 'utf-8'})"
",\n%s#%s" % (' ' * 4, ' ' * 8))
else:
vars['babel_templates_extractor'] = ''
# Ensure these exist in the namespace
for name in self.ensure_names:
vars.setdefault(name, '')
vars['version'] = vars.get('version', '0.1')
vars['zip_safe'] = asbool(vars.get('zip_safe', 'false'))
vars['sqlalchemy'] = asbool(vars.get('sqlalchemy', 'false'))
def make_cgitb_middleware(app, global_conf,
display=NoDefault,
logdir=None,
context=5,
format='html'):
"""
Wraps the application in the ``cgitb`` (standard library)
error catcher.
display:
If true (or debug is set in the global configuration)
then the traceback will be displayed in the browser
logdir:
Writes logs of all errors in that directory
context:
Number of lines of context to show around each line of
source code
"""
from paste.deploy.converters import asbool
if display is not NoDefault:
display = asbool(display)
if 'debug' in global_conf:
global_conf['debug'] = asbool(global_conf['debug'])
return CgitbMiddleware(
app, global_conf=global_conf,
display=display,
logdir=logdir,
context=context,
format=format)
def make_app(global_conf, full_stack=True, **app_conf):
"""Create a Pylons WSGI application and return it
`global_conf`
The inherited configuration for this application. Normally from the
[DEFAULT] section of the Paste ini file.
`full_stack`
Whether or not this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable full_stack
when this application is "managed" by another WSGI middleware.
`app_conf`
The application's local configuration. Normally specified in the
[app:<name>] section of the Paste ini file (where <name> defaults to
main).
"""
# Configure the Pylons environment
load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp(base_wsgi_app=RedditApp)
# CUSTOM MIDDLEWARE HERE (filtered by the error handling middlewares)
app = LimitUploadSize(app)
app = ProfilingMiddleware(app)
app = SourceViewMiddleware(app)
app = DomainListingMiddleware(app)
app = SubredditMiddleware(app)
app = ExtensionMiddleware(app)
app = DomainMiddleware(app)
log_path = global_conf.get('log_path')
if log_path:
process_iden = global_conf.get('scgi_port', 'default')
app = RequestLogMiddleware(log_path, process_iden, app)
#TODO: breaks on 404
#app = make_gzip_middleware(app, app_conf)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, error_template=error_template,
**config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and 500 when
# debug is disabled)
app = ErrorDocuments(app, global_conf, mapper=error_mapper, **app_conf)
# Establish the Registry for this application
app = RegistryManager(app)
# Static files
javascripts_app = StaticJavascripts()
# Set cache headers indicating the client should cache for 7 days
static_app = StaticURLParser(config['pylons.paths']['static_files'], cache_max_age=604800)
app = Cascade([static_app, javascripts_app, app])
app = AbsoluteRedirectMiddleware(app)
#add the rewrite rules
app = RewriteMiddleware(app)
app = CleanupMiddleware(app)
return app
def index_package(self, pkg_dict, defer_commit=False):
if pkg_dict is None:
return
# tracking summary values will be stale, never store them
tracking_summary = pkg_dict.pop('tracking_summary', None)
for r in pkg_dict.get('resources', []):
r.pop('tracking_summary', None)
data_dict_json = json.dumps(pkg_dict)
if config.get('ckan.cache_validated_datasets', True):
package_plugin = lib_plugins.lookup_package_plugin(
pkg_dict.get('type'))
schema = package_plugin.show_package_schema()
validated_pkg_dict, errors = lib_plugins.plugin_validate(
package_plugin, {
'model': model,
'session': model.Session
}, pkg_dict, schema, 'package_show')
pkg_dict['validated_data_dict'] = json.dumps(
validated_pkg_dict,
cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)
pkg_dict['data_dict'] = data_dict_json
# add to string field for sorting
title = pkg_dict.get('title')
if title:
pkg_dict['title_string'] = title
# delete the package if there is no state, or the state is `deleted`
if (not pkg_dict.get('state') or 'deleted' in pkg_dict.get('state')):
return self.delete_package(pkg_dict)
index_fields = RESERVED_FIELDS + pkg_dict.keys()
# include the extras in the main namespace
extras = pkg_dict.get('extras', [])
for extra in extras:
key, value = extra['key'], extra['value']
if isinstance(value, (tuple, list)):
value = " ".join(map(unicode, value))
key = ''.join([c for c in key if c in KEY_CHARS])
pkg_dict['extras_' + key] = value
if key not in index_fields:
pkg_dict[key] = value
pkg_dict.pop('extras', None)
# add tags, removing vocab tags from 'tags' list and adding them as
# vocab_<tag name> so that they can be used in facets
non_vocab_tag_names = []
tags = pkg_dict.pop('tags', [])
context = {'model': model}
for tag in tags:
if tag.get('vocabulary_id'):
data = {'id': tag['vocabulary_id']}
vocab = logic.get_action('vocabulary_show')(context, data)
key = u'vocab_%s' % vocab['name']
if key in pkg_dict:
pkg_dict[key].append(tag['name'])
else:
pkg_dict[key] = [tag['name']]
else:
non_vocab_tag_names.append(tag['name'])
pkg_dict['tags'] = non_vocab_tag_names
# add groups
groups = pkg_dict.pop('groups', [])
# we use the capacity to make things private in the search index
if pkg_dict['private']:
pkg_dict['capacity'] = 'private'
else:
pkg_dict['capacity'] = 'public'
pkg_dict['groups'] = [group['name'] for group in groups]
# if there is an owner_org we want to add this to groups for index
# purposes
if pkg_dict.get('organization'):
pkg_dict['organization'] = pkg_dict['organization']['name']
else:
pkg_dict['organization'] = None
# tracking
if not tracking_summary:
tracking_summary = model.TrackingSummary.get_for_package(
pkg_dict['id'])
pkg_dict['views_total'] = tracking_summary['total']
pkg_dict['views_recent'] = tracking_summary['recent']
resource_fields = [('name', 'res_name'),
('description', 'res_description'),
('format', 'res_format'), ('url', 'res_url'),
('resource_type', 'res_type')]
resource_extras = [(e, 'res_extras_' + e)
for e in model.Resource.get_extra_columns()]
# flatten the structure for indexing:
for resource in pkg_dict.get('resources', []):
for (okey, nkey) in resource_fields + resource_extras:
pkg_dict[nkey] = pkg_dict.get(nkey,
[]) + [resource.get(okey, u'')]
pkg_dict.pop('resources', None)
rel_dict = collections.defaultdict(list)
subjects = pkg_dict.pop("relationships_as_subject", [])
objects = pkg_dict.pop("relationships_as_object", [])
for rel in objects:
type = model.PackageRelationship.forward_to_reverse_type(
rel['type'])
rel_dict[type].append(
model.Package.get(rel['subject_package_id']).name)
for rel in subjects:
type = rel['type']
rel_dict[type].append(
model.Package.get(rel['object_package_id']).name)
for key, value in rel_dict.iteritems():
if key not in pkg_dict:
pkg_dict[key] = value
pkg_dict[TYPE_FIELD] = PACKAGE_TYPE
# Save dataset type
pkg_dict['dataset_type'] = pkg_dict['type']
# clean the dict fixing keys and dates
# FIXME where are we getting these dirty keys from? can we not just
# fix them in the correct place or is this something that always will
# be needed? For my data not changing the keys seems to not cause a
# problem.
new_dict = {}
bogus_date = datetime.datetime(1, 1, 1)
for key, value in pkg_dict.items():
key = key.encode('ascii', 'ignore')
if key.endswith('_date'):
try:
date = parse(value, default=bogus_date)
if date != bogus_date:
value = date.isoformat() + 'Z'
else:
# The date field was empty, so dateutil filled it with
# the default bogus date
value = None
except ValueError:
continue
new_dict[key] = value
pkg_dict = new_dict
for k in ('title', 'notes', 'title_string'):
if k in pkg_dict and pkg_dict[k]:
pkg_dict[k] = escape_xml_illegal_chars(pkg_dict[k])
# modify dates (SOLR is quite picky with dates, and only accepts ISO dates
# with UTC time (i.e trailing Z)
# See http://lucene.apache.org/solr/api/org/apache/solr/schema/DateField.html
# Re build index fix
pkg_dict['metadata_modified'] += 'Z'
if isinstance(pkg_dict['metadata_created'], str):
pkg_dict['metadata_created'] += 'Z'
else:
pkg_dict['metadata_created'] = pkg_dict['metadata_modified']
# mark this CKAN instance as data source:
pkg_dict['site_id'] = config.get('ckan.site_id')
# Strip a selection of the fields.
# These fields are possible candidates for sorting search results on,
# so we strip leading spaces because solr will sort " " before "a" or "A".
for field_name in ['title']:
try:
value = pkg_dict.get(field_name)
if value:
pkg_dict[field_name] = value.lstrip()
except KeyError:
pass
# add a unique index_id to avoid conflicts
import hashlib
pkg_dict['index_id'] = hashlib.md5(
'%s%s' % (pkg_dict['id'], config.get('ckan.site_id'))).hexdigest()
for item in PluginImplementations(IPackageController):
pkg_dict = item.before_index(pkg_dict)
assert pkg_dict, 'Plugin must return non empty package dict on index'
# send to solr:
try:
conn = make_connection()
commit = not defer_commit
if not asbool(config.get('ckan.search.solr_commit', 'true')):
commit = False
conn.add(docs=[pkg_dict], commit=commit)
except pysolr.SolrError, e:
msg = 'Solr returned an error: {0}'.format(
e[:1000] # limit huge responses
)
raise SearchIndexError(msg)
def checkout_url(self):
return dict(app=self.app,
merge_allowed=not asbool(config.get('scm.merge.{}.disabled'.format(self.app.config.tool_name))),
)
def _run_connector(self, environ, request, conn_name, op, *path,
**remote_params):
response = None
# check last part of path to see if it is json data
dispatch_params = dict()
if len(path) > 0:
p = urllib.unquote_plus(path[-1].lstrip())
if p.startswith('{'):
dp = json.loads(p)
f = dp.get('filters')
if isinstance(f, basestring):
dp['filters'] = json.loads(f)
path = path[:-1]
# scrub dispatch_params keys of unicode
# so we can pass as keywords
for (k, v) in dp.iteritems():
dispatch_params[str(k)] = v
# prevent trailing slash
if not p:
path = path[:-1]
path = '/'.join(path)
else:
path = ''
conn = self._connectors.get(conn_name)
# pretty print output
pretty_print = False
if '_pp' in remote_params:
del remote_params['_pp']
pretty_print = True
if conn:
conn_obj = conn['connector_class'](environ, request)
r = None
if asbool(config.get('profile.connectors')):
try:
import cProfile as profile
except ImportError:
import profile
directory = config.get('profile.dir', '')
# Make sure the id is unique for each thread
self.profile_id_counter_lock.acquire()
prof_id_counter = self.profile_id_counter
self.profile_id_counter += 1
self.profile_id_counter_lock.release()
ip = request.remote_addr
timestamp = time.time()
profile_id = "%s_%f_%s_%i" % (conn_name, timestamp, ip,
prof_id_counter)
self.outstanding_profile_ids[profile_id] = True
prof_file_name = "connector_%s.prof" % profile_id
info_file_name = "connector_%s.info" % profile_id
# output call info
file_name = os.path.join(directory, info_file_name)
f = open(file_name, 'w')
f.write('{"name": "%s", "op": "%s", "path": "%s", '
'"remote_params": %s, "ip": "%s", "timestamp": '
'%f, "id_counter": %i, "id": "%s"}' %
(conn_name, op, path, json.dumps(remote_params), ip,
timestamp, prof_id_counter, profile_id))
f.close()
# in order to get the results back we need to pass an object
# by refrence which will be populated with the actual results
result = {'r': None}
# profile call
file_name = os.path.join(directory, prof_file_name)
profile.runctx(
"result['r'] = conn_obj._dispatch(op, path, "
"remote_params, **dispatch_params)", None, {
'conn_obj': conn_obj,
'op': op,
'path': path,
'remote_params': remote_params,
'dispatch_params': dispatch_params,
'result': result
}, file_name)
r = result['r']
# add profile id to results
r['moksha_profile_id'] = profile_id
else:
try:
r = conn_obj._dispatch(op, path, remote_params,
**dispatch_params)
except Exception:
# Just print out the error so we can see it in /var/log/...
traceback.print_exc()
# But then re-raise the exception with its original context
raise
if pretty_print:
r = '<pre>' + pformat(r) + '</pre>'
elif not isinstance(r, basestring):
r = json.dumps(r, separators=(',', ':'))
if isinstance(r, unicode):
r = r.encode('utf-8', 'replace')
response = Response(r)
else:
response = Response(status='404 Not Found')
return response
def set_defaults(self, template_engine):
conf = self.current_conf()
# Load the MIMETypes with its default types
MIMETypes.init()
# Ensure all the keys from defaults are present, load them if not
for key, val in copy.deepcopy(PylonsConfig.defaults).iteritems():
conf.setdefault(key, val)
# Setup the prefix to override the routes if necessary.
prefix = conf.get('prefix')
if prefix:
warnings.warn(pylons.legacy.prefix_warning % prefix,
DeprecationWarning, 3)
map = conf.get('routes.map')
if map:
map.prefix = prefix
map._created_regs = False
# Load the errorware configuration from the Paste configuration file
# These all have defaults, and emails are only sent if configured and
# if this application is running in production mode
errorware = {}
errorware['debug'] = asbool(conf.get('debug'))
if not errorware['debug']:
errorware['debug'] = False
errorware['error_email'] = conf.get('email_to')
errorware['error_log'] = conf.get('error_log', None)
errorware['smtp_server'] = conf.get('smtp_server', 'localhost')
errorware['error_subject_prefix'] = conf.get(
'error_subject_prefix', 'WebApp Error: ')
errorware['from_address'] = conf.get(
'from_address',
conf.get('error_email_from', '*****@*****.**'))
errorware['error_message'] = conf.get(
'error_message', 'An internal server error occurred')
# Standard Pylons configuration directives for Myghty
myghty_defaults = {}
# Raise a complete error for the error middleware to catch
myghty_defaults['raise_error'] = True
myghty_defaults['output_encoding'] = \
conf['pylons.response_options']['charset']
myghty_defaults['component_root'] = [{os.path.basename(path): path} \
for path in conf['pylons.paths']['templates']]
# Merge additional globals
myghty_defaults.setdefault('allow_globals',
[]).extend(pylons.templating.PYLONS_VARS)
myghty_template_options = {}
if 'myghty_data_dir' in conf:
warnings.warn(
"Old config option found in ini file, replace "
"'myghty_data_dir' option with 'data_dir'", DeprecationWarning,
3)
myghty_defaults['data_dir'] = conf['myghty_data_dir']
elif 'cache_dir' in conf:
myghty_defaults['data_dir'] = os.path.join(conf['cache_dir'],
'templates')
# Copy in some defaults
if 'cache_dir' in conf:
conf.setdefault('beaker.session.data_dir',
os.path.join(conf['cache_dir'], 'sessions'))
conf.setdefault('beaker.cache.data_dir',
os.path.join(conf['cache_dir'], 'cache'))
# Copy Myghty defaults and options into template options
for k, v in myghty_defaults.iteritems():
myghty_template_options['myghty.' + k] = v
# Legacy copy of session and cache settings into conf
if k.startswith('session_') or k.startswith('cache_'):
conf[k] = v
# Copy old session/cache config to new keys for Beaker 0.7+
for key, val in conf.items():
if key.startswith('cache_'):
conf['cache.' + key[6:]] = val
elif key.startswith('session_'):
conf['session.' + key[8:]] = val
# Setup the main template options dict
conf['buffet.template_options'].update(myghty_template_options)
# Setup several defaults for various template languages
defaults = {}
# Rearrange template options as default for Mako
defaults['mako.directories'] = conf['pylons.paths']['templates']
defaults['mako.filesystem_checks'] = True
defaults['mako.output_encoding'] = \
conf['pylons.response_options']['charset']
if 'cache_dir' in conf:
defaults['mako.module_directory'] = \
os.path.join(conf['cache_dir'], 'templates')
# Setup kid defaults
defaults['kid.assume_encoding'] = 'utf-8'
defaults['kid.encoding'] = conf['pylons.response_options']['charset']
# Merge template options into defaults
defaults.update(conf['buffet.template_options'])
conf['buffet.template_options'] = defaults
# Prepare our default template engine
if template_engine == 'pylonsmyghty':
self.add_template_engine('pylonsmyghty', None,
myghty_template_options)
elif template_engine == 'mako':
self.add_template_engine('mako', '')
elif template_engine in ['genshi', 'kid']:
self.add_template_engine(template_engine,
conf['pylons.package'] + '.templates')
elif template_engine == 'cheetah':
self.add_template_engine(template_engine,
'%s.templates' % conf['pylons.package'])
log.debug(
"Loaded %s template engine as the default template "
"renderer", template_engine)
conf['pylons.cache_dir'] = conf.pop('cache_dir',
conf['app_conf'].get('cache_dir'))
# Save our errorware values
conf['pylons.errorware'] = errorware
def include_raw_solr_results(self, search_params):
if asbool(search_params.get('extras').get('ext_highlight')):
return True
else:
return False
from query import (TagSearchQuery, ResourceSearchQuery, PackageSearchQuery,
QueryOptions, convert_legacy_parameters_to_solr)
log = logging.getLogger(__name__)
def text_traceback():
with warnings.catch_warnings():
warnings.simplefilter("ignore")
res = 'the original traceback:'.join(
cgitb.text(
sys.exc_info()).split('the original traceback:')[1:]).strip()
return res
SIMPLE_SEARCH = asbool(config.get('ckan.simple_search', False))
SUPPORTED_SCHEMA_VERSIONS = ['2.0']
DEFAULT_OPTIONS = {
'limit': 20,
'offset': 0,
# about presenting the results
'order_by': 'rank',
'return_objects': False,
'ref_entity_with_attr': 'name',
'all_fields': False,
'search_tags': True,
'callback': None, # simply passed through
}
def components(self):
if not asbool(config.get('debug')):
raise ValueError('Not in debugging mode')
else:
return render('/debug/components.html')
def _group_or_org_update(context, data_dict, is_org=False):
model = context['model']
user = context['user']
session = context['session']
id = _get_or_bust(data_dict, 'id')
parent = context.get('parent', None)
group = model.Group.get(id)
context["group"] = group
if group is None:
raise NotFound('Group was not found.')
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(group.type)
try:
schema = group_plugin.form_to_db_schema_options({'type':'update',
'api':'api_version' in context,
'context': context})
except AttributeError:
schema = group_plugin.form_to_db_schema()
if is_org:
_check_access('organization_update', context, data_dict)
else:
_check_access('group_update', context, data_dict)
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = _validate(data_dict, schema, context)
log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'),
context.get('group').name if context.get('group') else '',
data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
# when editing an org we do not want to update the packages if using the
# new templates.
if ((not is_org)
and not converters.asbool(
config.get('ckan.legacy_templates', False))
and 'api_version' not in context):
context['prevent_packages_update'] = True
group = model_save.group_dict_save(data, context)
if parent:
parent_group = model.Group.get( parent )
if parent_group and not parent_group in group.get_groups(group.type):
# Delete all of this groups memberships
current = session.query(model.Member).\
filter(model.Member.table_id == group.id).\
filter(model.Member.table_name == "group").all()
if current:
log.debug('Parents of group %s deleted: %r', group.name,
[membership.group.name for membership in current])
for c in current:
session.delete(c)
member = model.Member(group=parent_group, table_id=group.id, table_name='group')
session.add(member)
log.debug('Group %s is made child of group %s',
group.name, parent_group.name)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.edit(group)
if is_org:
activity_type = 'changed organization'
else:
activity_type = 'changed group'
activity_dict = {
'user_id': model.User.by_name(user.decode('utf8')).id,
'object_id': group.id,
'activity_type': activity_type,
}
# Handle 'deleted' groups.
# When the user marks a group as deleted this comes through here as
# a 'changed' group activity. We detect this and change it to a 'deleted'
# activity.
if group.state == u'deleted':
if session.query(ckan.model.Activity).filter_by(
object_id=group.id, activity_type='deleted').all():
# A 'deleted group' activity for this group has already been
# emitted.
# FIXME: What if the group was deleted and then activated again?
activity_dict = None
else:
# We will emit a 'deleted group' activity.
activity_dict['activity_type'] = 'deleted group'
if activity_dict is not None:
activity_dict['data'] = {
'group': dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
_get_action('activity_create')(activity_create_context, activity_dict)
# TODO: Also create an activity detail recording what exactly changed
# in the group.
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.group_dictize(group, context)
def get_bool(self, key):
return asbool(self.get(key))
def make_flask_stack(conf, **app_conf):
""" This has to pass the flask app through all the same middleware that
Pylons used """
root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
debug = asbool(app_conf.get('debug', app_conf.get('DEBUG', False)))
testing = asbool(app_conf.get('testing', app_conf.get('TESTING', False)))
app = flask_app = CKANFlask(__name__)
app.debug = debug
app.testing = testing
app.template_folder = os.path.join(root, 'templates')
app.app_ctx_globals_class = CKAN_AppCtxGlobals
app.url_rule_class = CKAN_Rule
# Update Flask config with the CKAN values. We use the common config
# object as values might have been modified on `load_environment`
if config:
app.config.update(config)
else:
app.config.update(conf)
app.config.update(app_conf)
# Do all the Flask-specific stuff before adding other middlewares
# Secret key needed for flask-debug-toolbar and sessions
if not app.config.get('SECRET_KEY'):
app.config['SECRET_KEY'] = config.get('beaker.session.secret')
if not app.config.get('SECRET_KEY'):
raise RuntimeError(u'You must provide a value for the secret key'
' with the SECRET_KEY config option')
if debug:
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
DebugToolbarExtension(app)
# Add Jinja2 extensions and filters
extensions = [
'jinja2.ext.do', 'jinja2.ext.with_', jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
]
for extension in extensions:
app.jinja_env.add_extension(extension)
app.jinja_env.filters['empty_and_escape'] = \
jinja_extensions.empty_and_escape
app.jinja_env.filters['truncate'] = jinja_extensions.truncate
# Common handlers for all requests
app.before_request(ckan_before_request)
app.after_request(ckan_after_request)
# Template context processors
app.context_processor(helper_functions)
app.context_processor(c_object)
@app.route('/hello', methods=['GET'])
def hello_world():
return 'Hello World, this is served by Flask'
@app.route('/hello', methods=['POST'])
def hello_world_post():
return 'Hello World, this was posted to Flask'
# Auto-register all blueprints defined in the `views` folder
_register_core_blueprints(app)
# Set up each IBlueprint extension as a Flask Blueprint
for plugin in PluginImplementations(IBlueprint):
if hasattr(plugin, 'get_blueprint'):
app.register_extension_blueprint(plugin.get_blueprint())
# Start other middleware
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
# Fanstatic
if debug:
fanstatic_config = {
'versioning': True,
'recompute_hashes': True,
'minified': False,
'bottom': True,
'bundle': False,
}
else:
fanstatic_config = {
'versioning': True,
'recompute_hashes': False,
'minified': True,
'bottom': True,
'bundle': True,
}
app = Fanstatic(app, **fanstatic_config)
for plugin in PluginImplementations(IMiddleware):
try:
app = plugin.make_error_log_middleware(app, config)
except AttributeError:
log.critical('Middleware class {0} is missing the method'
'make_error_log_middleware.'.format(
plugin.__class__.__name__))
# Update the main CKAN config object with the Flask specific keys
# that were set here or autogenerated
flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
for key in flask_config_keys:
config[key] = flask_app.config[key]
# Add a reference to the actual Flask app so it's easier to access
app._wsgi_app = flask_app
return app
def edit(self, id=None, data=None, errors=None, error_summary=None):
context = {
'save': 'save' in request.params,
'schema': self._edit_form_to_db_schema(),
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj
}
if id is None:
if c.userobj:
id = c.userobj.id
else:
abort(400, _('No user specified'))
data_dict = {'id': id}
try:
check_access('user_update', context, data_dict)
except NotAuthorized:
abort(403, _('Unauthorized to edit a user.'))
if (context['save']) and not data:
return self._save_edit(id, context)
try:
old_data = get_action('user_show')(context, data_dict)
schema = self._db_to_edit_form_schema()
if schema:
old_data, errors = \
dictization_functions.validate(old_data, schema, context)
c.display_name = old_data.get('display_name')
c.user_name = old_data.get('name')
data = data or old_data
except NotAuthorized:
abort(403, _('Unauthorized to edit user %s') % '')
except NotFound:
abort(404, _('User not found'))
user_obj = context.get('user_obj')
if not (authz.is_sysadmin(c.user) or c.user == user_obj.name):
abort(403,
_('User %s not authorized to edit %s') % (str(c.user), id))
errors = errors or {}
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
self._setup_template_variables(
{
'model': model,
'session': model.Session,
'user': c.user
}, data_dict)
c.is_myself = True
c.show_email_notifications = asbool(
config.get('ckan.activity_streams_email_notifications'))
c.form = render(self.edit_user_form, extra_vars=vars)
return render('user/edit.html')
def _load_email_config(self) -> None:
"""
Load config for email related stuff
"""
# TODO - G.M - 27-03-2018 - [Email] Restore email config
###
# EMAIL related stuff (notification, reply)
##
self.EMAIL__NOTIFICATION__ENABLED_ON_INVITATION = asbool(
self.get_raw_config("email.notification.enabled_on_invitation",
"true"))
# TODO - G.M - 2019-04-05 - keep as parameters
# or set it as constant,
# see https://github.com/tracim/tracim/issues/1569
self.EMAIL__NOTIFICATION__NOTIFIED_EVENTS = [
ActionDescription.COMMENT,
ActionDescription.CREATION,
ActionDescription.EDITION,
ActionDescription.REVISION,
ActionDescription.STATUS_UPDATE,
]
# TODO - G.M - 2019-04-04 - need to be better handled:
# dynamic default value and allow user to set this value.
# see :https://github.com/tracim/tracim/issues/1555
self.EMAIL__NOTIFICATION__NOTIFIED_CONTENTS = [
"html-document",
"thread",
"file",
"comment",
# 'folder' --folder is skipped
]
self.EMAIL__NOTIFICATION__FROM__EMAIL = self.get_raw_config(
"email.notification.from.email", "noreply+{user_id}@trac.im")
self.EMAIL__NOTIFICATION__FROM = self.get_raw_config(
"email.notification.from")
if self.get_raw_config("email.notification.from"):
raise ConfigurationError(
"email.notification.from configuration is deprecated. "
"Use instead email.notification.from.email and "
"email.notification.from.default_label.")
self.EMAIL__NOTIFICATION__FROM__DEFAULT_LABEL = self.get_raw_config(
"email.notification.from.default_label", "Tracim Notifications")
self.EMAIL__NOTIFICATION__REPLY_TO__EMAIL = self.get_raw_config(
"email.notification.reply_to.email")
self.EMAIL__NOTIFICATION__REFERENCES__EMAIL = self.get_raw_config(
"email.notification.references.email")
# Content update notification
self.EMAIL__NOTIFICATION__CONTENT_UPDATE__TEMPLATE__HTML = self.get_raw_config(
"email.notification.content_update.template.html")
self.EMAIL__NOTIFICATION__CONTENT_UPDATE__SUBJECT = self.get_raw_config(
"email.notification.content_update.subject",
_("[{website_title}] [{workspace_label}] {content_label} ({content_status_label})"
),
)
# Created account notification
self.EMAIL__NOTIFICATION__CREATED_ACCOUNT__TEMPLATE__HTML = self.get_raw_config(
"email.notification.created_account.template.html")
self.EMAIL__NOTIFICATION__CREATED_ACCOUNT__SUBJECT = self.get_raw_config(
"email.notification.created_account.subject",
_("[{website_title}] Someone created an account for you"),
)
# Reset password notification
self.EMAIL__NOTIFICATION__RESET_PASSWORD_REQUEST__TEMPLATE__HTML = self.get_raw_config(
"email.notification.reset_password_request.template.html")
self.EMAIL__NOTIFICATION__RESET_PASSWORD_REQUEST__SUBJECT = self.get_raw_config(
"email.notification.reset_password_request.subject",
_("[{website_title}] A password reset has been requested"),
)
# TODO - G.M - 2019-01-22 - add feature to process notification email
# asynchronously see issue https://github.com/tracim/tracim/issues/1345
self.EMAIL__NOTIFICATION__PROCESSING_MODE = "sync"
self.EMAIL__NOTIFICATION__ACTIVATED = asbool(
self.get_raw_config("email.notification.activated"))
self.EMAIL__NOTIFICATION__SMTP__SERVER = self.get_raw_config(
"email.notification.smtp.server")
self.EMAIL__NOTIFICATION__SMTP__PORT = self.get_raw_config(
"email.notification.smtp.port")
self.EMAIL__NOTIFICATION__SMTP__USER = self.get_raw_config(
"email.notification.smtp.user")
self.EMAIL__NOTIFICATION__SMTP__PASSWORD = self.get_raw_config(
"email.notification.smtp.password", secret=True)
self.EMAIL__REPLY__ACTIVATED = asbool(
self.get_raw_config("email.reply.activated", "false"))
self.EMAIL__REPLY__IMAP__SERVER = self.get_raw_config(
"email.reply.imap.server")
self.EMAIL__REPLY__IMAP__PORT = self.get_raw_config(
"email.reply.imap.port")
self.EMAIL__REPLY__IMAP__USER = self.get_raw_config(
"email.reply.imap.user")
self.EMAIL__REPLY__IMAP__PASSWORD = self.get_raw_config(
"email.reply.imap.password", secret=True)
self.EMAIL__REPLY__IMAP__FOLDER = self.get_raw_config(
"email.reply.imap.folder")
self.EMAIL__REPLY__CHECK__HEARTBEAT = int(
self.get_raw_config("email.reply.check.heartbeat", "60"))
self.EMAIL__REPLY__IMAP__USE_SSL = asbool(
self.get_raw_config("email.reply.imap.use_ssl"))
self.EMAIL__REPLY__IMAP__USE_IDLE = asbool(
self.get_raw_config("email.reply.imap.use_idle", "true"))
self.EMAIL__REPLY__CONNECTION__MAX_LIFETIME = int(
self.get_raw_config("email.reply.connection.max_lifetime",
"600") # 10 minutes
)
self.EMAIL__REPLY__USE_HTML_PARSING = asbool(
self.get_raw_config("email.reply.use_html_parsing", "true"))
self.EMAIL__REPLY__USE_TXT_PARSING = asbool(
self.get_raw_config("email.reply.use_txt_parsing", "true"))
self.EMAIL__REPLY__LOCKFILE_PATH = self.get_raw_config(
"email.reply.lockfile_path", "")
self.EMAIL__PROCESSING_MODE = self.get_raw_config(
"email.processing_mode", "sync").upper()
self.EMAIL__ASYNC__REDIS__HOST = self.get_raw_config(
"email.async.redis.host", "localhost")
self.EMAIL__ASYNC__REDIS__PORT = int(
self.get_raw_config("email.async.redis.port", "6379"))
self.EMAIL__ASYNC__REDIS__DB = int(
self.get_raw_config("email.async.redis.db", "0"))
self.NEW_USER__INVITATION__DO_NOTIFY = asbool(
self.get_raw_config("new_user.invitation.do_notify", "True"))
self.NEW_USER__INVITATION__MINIMAL_PROFILE = self.get_raw_config(
"new_user.invitation.minimal_profile", Group.TIM_MANAGER_GROUPNAME)
def make_app(global_conf,
full_stack=True,
static_files=True,
gzip=True,
**app_conf):
"""Create a Pylons WSGI application and return it
``global_conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
config = load_environment(global_conf, app_conf)
# The Pylons WSGI app
#app = PylonsApp(config=config)
app = CronusPylonApp(config=config)
# Routing/Session/Cache Middleware
app = RoutesMiddleware(app, config['routes.map'], singleton=False)
app = SessionMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
# if asbool(config['debug']):
# app = StatusCodeRedirect(app)
# else:
# app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
app = StatusCodeRedirect(app, errors=())
# Establish the Registry for this application
app = RegistryManager(app)
apps = [app]
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
apps.insert(0, static_app)
#app = Cascade([static_app, app])
if asbool(gzip):
# serve gzip response
gzip_app = gzipmiddelware.gzipmiddleware(app, compress_level=6)
apps.insert(0, gzip_app)
app = Cascade(apps)
app.config = config
return app
def init_app(self,
global_conf,
app_conf,
package=None,
template_engine=default_template_engine,
paths=None):
"""Initialize configuration for the application
.. note
This *must* be called at least once, as soon as possible
tosetup all the configuration options.
``global_conf``
Several options are expected to be set for a Pylons web
application. They will be loaded from the global_config
which has the main Paste options. If ``debug`` is not
enabled as a global config option, the following option
*must* be set:
* error_to - The email address to send the debug error to
The optional config options in this case are:
* smtp_server - The SMTP server to use, defaults to
'localhost'
* error_log - A logfile to write the error to
* error_subject_prefix - The prefix of the error email
subject
* from_address - Whom the error email should be from
``app_conf``
Defaults supplied via the [app:main] section from the Paste
config file. ``load_config`` only cares about whether a
'prefix' option is set, if so it will update Routes to
ensure URL's take that into account.
``package``
The name of the application package, to be stored in the
app_conf.
.. versionchanged:: 0.9.7
``template_engine`` is no longer required, and can be set
to :data:`None` to avoid loading the default one.
``template_engine``
Declare the default template engine to setup. Choices are
kid, genshi, mako (the default), and pylonsmyghty.
"""
log.debug("Initializing configuration, package: '%s'", package)
conf = global_conf.copy()
conf.update(app_conf)
conf.update(dict(app_conf=app_conf, global_conf=global_conf))
conf.update(self.pop('environment_load', {}))
if paths:
conf['pylons.paths'] = paths
# XXX Legacy: More backwards compatibility locations for the package
# name
conf['pylons.package'] = conf['package'] = \
conf['app_conf']['package'] = package
if 'debug' in conf:
conf['debug'] = asbool(conf['debug'])
if paths and 'root_path' in paths:
warnings.warn(pylons.legacy.root_path, DeprecationWarning, 2)
paths['root'] = paths['root_path']
log.debug("Pushing process configuration")
self.push_process_config(conf)
self.set_defaults(template_engine)
def init_app(self, global_conf, app_conf, package=None, paths=None):
"""Initialize configuration for the application
.. note
This *must* be called at least once, as soon as possible
to setup all the configuration options.
``global_conf``
Several options are expected to be set for a Pylons web
application. They will be loaded from the global_config
which has the main Paste options. If ``debug`` is not
enabled as a global config option, the following option
*must* be set:
* error_to - The email address to send the debug error to
The optional config options in this case are:
* smtp_server - The SMTP server to use, defaults to
'localhost'
* error_log - A logfile to write the error to
* error_subject_prefix - The prefix of the error email
subject
* from_address - Whom the error email should be from
``app_conf``
Defaults supplied via the [app:main] section from the Paste
config file. ``load_config`` only cares about whether a
'prefix' option is set, if so it will update Routes to
ensure URL's take that into account.
``package``
The name of the application package, to be stored in the
app_conf.
.. versionchanged:: 1.0
``template_engine`` option is no longer supported.
"""
log.debug("Initializing configuration, package: '%s'", package)
conf = global_conf.copy()
conf.update(app_conf)
conf.update(dict(app_conf=app_conf, global_conf=global_conf))
conf.update(self.pop('environment_load', {}))
if paths:
conf['pylons.paths'] = paths
conf['pylons.package'] = package
conf['debug'] = asbool(conf.get('debug'))
# Load the MIMETypes with its default types
MIMETypes.init()
# Ensure all the keys from defaults are present, load them if not
for key, val in copy.deepcopy(PylonsConfig.defaults).iteritems():
conf.setdefault(key, val)
# Load the errorware configuration from the Paste configuration file
# These all have defaults, and emails are only sent if configured and
# if this application is running in production mode
errorware = {}
errorware['debug'] = conf['debug']
if not errorware['debug']:
errorware['debug'] = False
errorware['error_email'] = conf.get('email_to')
errorware['error_log'] = conf.get('error_log', None)
errorware['smtp_server'] = conf.get('smtp_server', 'localhost')
errorware['error_subject_prefix'] = conf.get(
'error_subject_prefix', 'WebApp Error: ')
errorware['from_address'] = conf.get(
'from_address',
conf.get('error_email_from', '*****@*****.**'))
errorware['error_message'] = conf.get(
'error_message', 'An internal server error occurred')
# Copy in some defaults
if 'cache_dir' in conf:
conf.setdefault('beaker.session.data_dir',
os.path.join(conf['cache_dir'], 'sessions'))
conf.setdefault('beaker.cache.data_dir',
os.path.join(conf['cache_dir'], 'cache'))
conf['pylons.cache_dir'] = conf.pop('cache_dir',
conf['app_conf'].get('cache_dir'))
# Save our errorware values
conf['pylons.errorware'] = errorware
# Load conf dict into self
self.update(conf)
def startAgentGlobals(startThreadMgr=True,
startPackageMgr=True,
startAgentMonitor=True):
"""
Create and start the global objects
"""
# cleanup incomplete manifests
from agent.controllers.service import ServiceController
ServiceController.cleanupServices()
# stop the existing agents
stopAgentGlobals()
appGlobal = pylons.config['pylons.app_globals']
# load config override from agent .metadata.json
configutil.loadPylonConfig(pylons.config)
configutil.loadConfigOverrides()
LOG.info("Complete loading config overrides")
# build in memory security token cache
appGlobal.authztoken = str(uuid.uuid4())
appGlobal.encryptedtokens = {}
agentauth.buildTokenCache(appGlobal.authztoken)
LOG.info("Complete building security token cache")
# agent health
appGlobal.agentHealth = 'True'
appGlobal.agentHealthFactor = None
from datetime import datetime
appGlobal.agentInfo = {
'version_wiri': agenthealth.loadVersion(),
'startup_time': str(datetime.now())
}
appGlobal.osInfo = agenthealth.getOsInfo()
appGlobal.diskOk = True
appGlobal.autoRestartEnabled = asbool(
pylons.config['auto_restart_enabled'])
LOG.info("Agent health initialized")
# start thread mgr
appGlobal.threadMgr = ThreadMgr(
garbageFreq=pylons.config['threadMgr_garbage_freq'],
maxThreadAge=pylons.config['threadMgr_thread_age'])
if startThreadMgr:
appGlobal.threadMgr.start()
LOG.info("Agent thread manager started")
# start package mgr
appGlobal.packageMgr = PackageMgr(
garbageFreq=pylons.config['packageMgr_garbage_freq'],
maxPackageAge=pylons.config['packageMgr_package_age'])
if startPackageMgr:
appGlobal.packageMgr.start()
LOG.info("Agent package manager started")
# now load saved .metadata for previously saved hwPath, serverAddress, and serverAuth
enrollMeta = manifestutil.readJsonServiceMeta('agent', ['hwPath'])
hwPath = enrollMeta.get('hwPath')
if hwPath is not None:
appGlobal.hwPath = hwPath
LOG.info("Agent hwPath loaded")
#start monitor manager
from agent.lib.monitors.monitor import AgentMonitor
appGlobal.agentMonitor = AgentMonitor()
if startAgentMonitor:
appGlobal.agentMonitor.start()
LOG.info("Agent monitor started")
# start graphite udp server
# from agent.lib.monitors.graphited import GraphiteDServer
# from multiprocessing import Pipe
# graphiteEnabled = configutil.getConfigAsBool('graphite_enabled')
# if startAgentMonitor and graphiteEnabled:
# output_p, input_p = Pipe()
# appGlobal.graphited = GraphiteDServer(input_p)
# appGlobal.graphited.start()
# udpMsgProcessThread = threading.Thread(target=GraphiteDServer.read_queue, args=(output_p,))
# udpMsgProcessThread.daemon = True
# udpMsgProcessThread.start()
# LOG.info("Agent graphited started")
# Declare dictionary for storing dynamic controllers
appGlobal.dynacontrollers = dict()
# metrix manager initialization
from agent.lib.monitors.metrix_manager import MetrixManager
from agent.lib.monitors.system_monitor import SystemMonitor
appGlobal.metrixManager = MetrixManager()
appGlobal.systemMonitor = SystemMonitor()
appGlobal.metrixManager.register('OS', appGlobal.systemMonitor.getOSinfo,
1)
appGlobal.metrixManager.register('Node Name',
appGlobal.systemMonitor.getNodeName, 1)
appGlobal.metrixManager.register('Free Memory(KB)',
appGlobal.systemMonitor.getFreeMemory, 2)
appGlobal.metrixManager.register('CPU Usage(%)',
appGlobal.systemMonitor.getCpuUsage, 2)
appGlobal.metrixManager.register('Version', agenthealth.loadVersion, 5)
appGlobal.metrixManager.register('Configuration',
validate_internals.getConfigFile, 6)
appGlobal.metrixManager.register('Configuration Overrides',
configutil.getConfigOverrides, 7)
appGlobal.metrixManager.register(
'hwPath', lambda: appGlobal.hwPath
if hasattr(appGlobal, 'hwPath') else None, 5)
appGlobal.metrixManager.register(AGENT_HEALTH_VI_KEY,
lambda: appGlobal.agentHealth, 1)
appGlobal.metrixManager.register(AGENT_HEALTH_FACTOR_VI_KEY,
lambda: appGlobal.agentHealthFactor, 1)
appGlobal.metrixManager.register(OS_INFO_KEY, lambda: appGlobal.osInfo, 2)
appGlobal.metrixManager.register(AGENT_INFO_KEY,
lambda: appGlobal.agentInfo, 2)
LOG.info("Agent health metrics registered")
# start all agent modules
modulestartthread = threading.Thread(
target=ModuleController.loadModuleOnAgentStartup)
modulestartthread.start()
LOG.info("Local modules started")
# start all services with active manifest, and load dynamic controllers
servicestartthread = threading.Thread(
target=ServiceController.startServicesOnAgentStartup)
servicestartthread.start()
LOG.info("Local services started")
appGlobal.sdutil = shutdownAgent
def handle_new_request_action(self, username, request_action):
'''Handles sending email to the person who created the request, as well
as updating the state of the request depending on the data sent.
:param username: The user's name.
:type username: string
:param request_action: The current action. Can be either reply or
reject
:type request_action: string
:rtype: json
'''
data = dict(toolkit.request.POST)
counters_data_dict = {'package_id': data['package_id'], 'flag': ''}
if 'rejected' in data:
data['rejected'] = asbool(data['rejected'])
counters_data_dict['flag'] = 'declined'
else:
counters_data_dict['flag'] = 'replied'
message_content = data.get('message_content')
if message_content is None or message_content == '':
payload = {
'success': False,
'error': {
'message_content': 'Missing value'
}
}
return json.dumps(payload)
try:
_get_action('requestdata_request_patch', data)
except NotAuthorized:
abort(403, _('Not authorized to use this action.'))
except ValidationError as e:
error = {'success': False, 'error': {'fields': e.error_dict}}
return json.dumps(error)
to = data['send_to']
subject = config.get('ckan.site_title') + ': Data request ' +\
request_action
file = data.get('file_upload')
response = send_email(message_content, to, subject, file=file)
if response['success'] is False:
error = {
'success': False,
'error': {
'fields': {
'email': response['message']
}
}
}
return json.dumps(error)
success = {'success': True, 'message': 'Message was sent successfully'}
get_action('requestdata_increment_request_data_counters')(
{}, counters_data_dict)
return json.dumps(success)
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
mycontroller = pkg_resources \
.EntryPoint \
.parse('x=%s' % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
p.load_all(config)
# Load the synchronous search plugin, unless already loaded or
# explicitly disabled
if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
asbool(config.get('ckan.search.automatic_indexing', True)):
log.debug('Loading the synchronous search plugin')
p.load('synchronous_search')
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
import ckan.lib.search as search
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
config['routes.map'] = routing.make_map()
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
#################################################################
# #
# HORRIBLE GENSHI HACK #
# #
#################################################################
# #
# Genshi does strange things to get stuff out of the template #
# variables. This stops it from handling properties in the #
# correct way as it returns the property rather than the actual #
# value of the property. #
# #
# By overriding lookup_attr() in the LookupBase class we are #
# able to get the required behaviour. Using @property allows #
# us to move functionality out of templates whilst maintaining #
# backwards compatability. #
# #
#################################################################
'''
This code is based on Genshi code
Copyright © 2006-2012 Edgewall Software
All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the following
conditions are met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
The name of the author may not be used to endorse or promote
products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
from genshi.template.eval import LookupBase
@classmethod
def genshi_lookup_attr(cls, obj, key):
__traceback_hide__ = True
try:
val = getattr(obj, key)
except AttributeError:
if hasattr(obj.__class__, key):
raise
else:
try:
val = obj[key]
except (KeyError, TypeError):
val = cls.undefined(key, owner=obj)
if isinstance(val, property):
val = val.fget()
return val
setattr(LookupBase, 'lookup_attr', genshi_lookup_attr)
del genshi_lookup_attr
del LookupBase
#################################################################
# #
# END OF GENSHI HACK #
# #
#################################################################
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
'^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except sqlalchemy.exc.ProgrammingError:
# The database is not initialised. This is a bit dirty.
pass
def index(self):
'''
This is the main function of the management web UI
'''
try:
c.debug = asbool(config.get('debug', False))
c.title = "LinOTP Management"
admin_user = getUserFromRequest(request)
if admin_user.has_key('login'):
# Get a list of their allowed admin realms.
# Set their current realm to the first one in the list.
c.login = admin_user['login']
realms = getAdminRealms(c.login)
if (realms):
c.realm = realms[0]
else:
c.realm = getDefaultRealm()
c.admin = "%s@%s" % (c.login, c.realm)
log.debug("[index] importers: %s" % IMPORT_TEXT)
c.importers = IMPORT_TEXT
c.help_url = config.get('help_url')
## add render info for token type config
confs = _getTokenTypeConfig('config')
token_config_tab = {}
token_config_div = {}
for conf in confs:
tab = ''
div = ''
try:
#loc = conf +'_token_settings'
tab = confs.get(conf).get('title')
#tab = '<li ><a href=#'+loc+'>'+tab+'</a></li>'
div = confs.get(conf).get('html')
#div = +div+'</div>'
except Exception as e:
log.debug(
'[index] no config info for token type %s (%r)' %
(conf, e))
if tab is not None and div is not None and len(
tab) > 0 and len(div) > 0:
token_config_tab[conf] = tab
token_config_div[conf] = div
c.token_config_tab = token_config_tab
c.token_config_div = token_config_div
## add the enrollment fragments from the token definition
## tab: <option value="ocra">${_("OCRA - challenge/response Token")}</option>
## div: "<div id='"+ tt + "'>"+enroll+"</div>"
enrolls = _getTokenTypeConfig('init')
token_enroll_tab = {}
token_enroll_div = {}
for conf in enrolls:
tab = ''
div = ''
try:
tab = enrolls.get(conf).get('title')
div = enrolls.get(conf).get('html')
except Exception as e:
log.debug(
'[index] no enrollment info for token type %s (%r)' %
(conf, e))
if tab is not None and div is not None and len(
tab) > 0 and len(div) > 0:
token_enroll_tab[conf] = tab
token_enroll_div[conf] = div
c.token_enroll_tab = token_enroll_tab
c.token_enroll_div = token_enroll_div
c.tokentypes = _getTokenTypes()
http_host = request.environ.get("HTTP_HOST")
url_scheme = request.environ.get("wsgi.url_scheme")
# Use WebAuth to logout instead of Apache.
c.logout_url = "%s://%s/account/logout" % (url_scheme, http_host)
Session.commit()
ren = render('/manage/manage-base.mako')
return ren
except PolicyException as pe:
log.error("[index] Error during checking policies: %r" % pe)
log.error("[index] %s" % traceback.format_exc())
Session.rollback()
return sendError(response, unicode(pe), 1)
except Exception as ex:
log.error("[index] failed! %r" % ex)
log.error("[index] %s" % traceback.format_exc())
Session.rollback()
return sendError(response, ex)
finally:
Session.close()
log.debug('[index] done')
try:
for item in plugins.PluginImplementations(plugins.IPackageController):
data_dict = item.before_search(data_dict)
except NotFound, e:
base_abort(404, 'Wrong parameter value in url')
# the extension may have decided that it is not necessary to perform
# the query
abort = data_dict.get('abort_search', False)
if data_dict.get('sort') in (None, 'rank'):
data_dict['sort'] = 'score desc, ' + DEFAULT_SORTING
results = []
if not abort:
if asbool(data_dict.get('use_default_schema')):
data_source = 'data_dict'
else:
data_source = 'validated_data_dict'
data_dict.pop('use_default_schema', None)
result_fl = data_dict.get('fl')
if not result_fl:
data_dict['fl'] = 'id {0}'.format(data_source)
else:
data_dict['fl'] = ' '.join(result_fl)
# Remove before these hit solr FIXME: whitelist instead
include_private = asbool(data_dict.pop('include_private', False))
include_drafts = asbool(data_dict.pop('include_drafts', False))
data_dict.setdefault('fq', '')
def make_middleware(app, global_conf, **local_conf):
"""
Based on the configuration wrap `app` in a set of common and useful
middleware.
"""
# Merge the global and local configurations
conf = global_conf.copy()
conf.update(local_conf)
debug = asbool(conf.get('debug', False))
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware(app, conf)
log.debug("Enabling 'httpexceptions' middleware")
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware(app, conf)
log.debug("Enabling 'recursive' middleware")
# Session middleware puts a session factory into the environment
if asbool(conf.get('use_session', True)):
store = flup_session.MemorySessionStore()
app = flup_session.SessionMiddleware(store, app)
log.debug("Enabling 'flup session' middleware")
# Beaker session middleware
if asbool(conf.get('use_beaker_session', False)):
pkg_resources.require("Beaker")
import beaker.session
app = beaker.session.SessionMiddleware(app, conf)
log.debug("Enabling 'beaker session' middleware")
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
if debug:
# Middleware to check for WSGI compliance
if asbool(conf.get('use_lint', True)):
from paste import lint
app = lint.make_middleware(app, conf)
log.debug("Enabling 'lint' middleware")
# Middleware to run the python profiler on each request
if asbool(conf.get('use_profile', False)):
from paste.debug import profile
app = profile.ProfileMiddleware(app, conf)
log.debug("Enabling 'profile' middleware")
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
if asbool(conf.get('use_interactive', False)):
from paste import evalexception
app = evalexception.EvalException(app, conf)
log.debug("Enabling 'eval exceptions' middleware")
else:
from paste.exceptions import errormiddleware
app = errormiddleware.ErrorMiddleware(app, conf)
log.debug("Enabling 'error' middleware")
# Middleware that intercepts print statements and shows them on the
# returned page
if asbool(conf.get('use_printdebug', True)):
from paste.debug import prints
app = prints.PrintDebugMiddleware(app, conf)
log.debug("Enabling 'print debug' middleware")
else:
# Not in debug mode, just use the regular error middleware
from paste.exceptions import errormiddleware
app = errormiddleware.ErrorMiddleware(app, conf)
log.debug("Enabling 'error' middleware")
# Transaction logging (apache access.log style)
if asbool(conf.get('use_translogger', True)):
from paste.translogger import TransLogger
app = TransLogger(app)
log.debug("Enabling 'trans logger' middleware")
# Config middleware just stores the paste config along with the request,
# not sure we need this but useful
from paste.deploy.config import ConfigMiddleware
app = ConfigMiddleware(app, conf)
log.debug("Enabling 'config' middleware")
# X-Forwarded-Host handling
app = XForwardedHostMiddleware(app)
log.debug("Enabling 'x-forwarded-host' middleware")
return app
def make_app(global_conf, full_stack=True, **app_conf):
"""Create a Pylons WSGI application and return it
`global_conf`
The inherited configuration for this application. Normally from the
[DEFAULT] section of the Paste ini file.
`full_stack`
Whether or not this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable full_stack
when this application is "managed" by another WSGI middleware.
`app_conf`
The application's local configuration. Normally specified in the
[app:<name>] section of the Paste ini file (where <name> defaults to
main).
"""
# Configure the Pylons environment
load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp(base_wsgi_app=RedditApp)
# CUSTOM MIDDLEWARE HERE (filtered by the error handling middlewares)
# last thing first from here down
app = CleanupMiddleware(app)
app = LimitUploadSize(app)
app = ProfileGraphMiddleware(app)
app = ProfilingMiddleware(app)
app = SourceViewMiddleware(app)
app = DomainListingMiddleware(app)
app = SubredditMiddleware(app)
app = ExtensionMiddleware(app)
app = DomainMiddleware(app)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app,
global_conf,
error_template=error_template,
**config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and 500 when
# debug is disabled)
app = ErrorDocuments(app, global_conf, mapper=error_mapper, **app_conf)
# Establish the Registry for this application
app = RegistryManager(app)
# Static files
javascripts_app = StaticJavascripts()
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, javascripts_app, app])
#add the rewrite rules
app = RewriteMiddleware(app)
return app
def can_hotcopy(self, source_url):
enabled = asbool(tg.config.get('scm.git.hotcopy', True))
is_local = os.path.exists(source_url)
requested = self._repo.app.config.options.get('hotcopy', False)
return enabled and is_local and requested
import re
import fileinput
import pylons
from paste.deploy import loadapp
from paste.deploy.converters import asbool, aslist
from popserver.lib import helpers
if len(sys.argv) != 2:
print "Usage: %s [path_to_ini_file]" % sys.argv[0]
exit(1)
loadapp('config:%s' % sys.argv[1], relative_to=os.getcwd())
#helpers.config = pylons.config
make_bundle = asbool(
pylons.config.get('popego.serve_bundled_stylesheets', False))
bundle_files = aslist(
pylons.config.get('popego.stylesheet_bundle_files', None), ',', True)
if make_bundle is None:
sys.stderr.write(
'``popego.serve_bundled_styesheets`` not defined or False. Exiting.')
exit(1)
if len(bundle_files) == 0:
sys.stderr.write(
'``popego.stylesheet_bundle_files`` not defined or empty. Exiting.')
exit(1)
url_rex = re.compile(r'url\((.*)\)')
relative_url_rex = re.compile(r'^(?:\.\./)*(.+)$')
def paged_diffs(self,
commit_id,
start=0,
end=None,
onlyChangedFiles=False):
result = {
'added': [],
'removed': [],
'changed': [],
'copied': [],
'renamed': []
}
cmd_args = [
'--no-commit-id',
'--name-status',
'--no-abbrev',
'--root',
# show tree entry itself as well as subtrees (Commit.added_paths relies on this)
'-t',
'-z' # don't escape filenames and use \x00 as fields delimiter
]
if onlyChangedFiles:
cmd_args[4] = '-r'
if asbool(tg.config.get('scm.commit.git.detect_copies', True)):
cmd_args += ['-M', '-C']
cmd_output = self._git.git.diff_tree(commit_id, *cmd_args).split(
'\x00'
)[:-1] # don't escape filenames and use \x00 as fields delimiter
''' cmd_output will be like:
[
'A',
'filename',
'D',
'another filename',
'M',
'po',
'R100', # <-- These next three lines would only show up with 'detect_copies' enabled
'po/sr.po',
'po/sr_Latn.po',
]
'''
x = 0
files = []
while x < len(cmd_output):
status = cmd_output[x][0]
if status in ('R', 'C'):
ratio = float(cmd_output[x][1:4]) / 100.0
files.append((status, {
'new': h.really_unicode(cmd_output[x + 2]),
'old': h.really_unicode(cmd_output[x + 1]),
'ratio': ratio,
}))
x += 3
else:
files.append((status, h.really_unicode(cmd_output[x + 1])))
x += 2
for status, name in files[start:end]:
change_list_types = {
'R': result['renamed'],
'C': result['copied'],
'A': result['added'],
'D': result['removed'],
'M': result['changed'],
'T': result['changed'],
}
if status in change_list_types:
change_list = change_list_types[status]
change_list.append(name)
else:
log.error(
'Unexpected git change status: "%s" on file %s commit %s repo %s',
status, name, commit_id, self._repo.full_fs_path)
result['total'] = len(files)
return result
def make_app(global_conf, full_stack=True, **app_conf):
"""Create a Pylons WSGI application and return it
`global_conf`
The inherited configuration for this application. Normally from the
[DEFAULT] section of the Paste ini file.
`full_stack`
Whether or not this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable full_stack
when this application is "managed" by another WSGI middleware.
`app_conf`
The application's local configuration. Normally specified in the
[app:<name>] section of the Paste ini file (where <name> defaults to
main).
"""
# Configure the Pylons environment
config = load_environment(global_conf, app_conf)
g = config['pylons.app_globals']
# The Pylons WSGI app
app = RedditApp(config=config)
app = RoutesMiddleware(app, config["routes.map"])
# CUSTOM MIDDLEWARE HERE (filtered by the error handling middlewares)
# last thing first from here down
app = CleanupMiddleware(app)
app = LimitUploadSize(app)
profile_directory = g.config.get('profile_directory')
if profile_directory:
app = ProfilingMiddleware(app, profile_directory)
app = DomainListingMiddleware(app)
app = SubredditMiddleware(app, config=config)
app = ExtensionMiddleware(app)
app = DomainMiddleware(app, config=config)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and 500 when
# debug is disabled)
app = ErrorDocuments(app, global_conf, error_mapper, **app_conf)
# Establish the Registry for this application
app = RegistryManager(app)
# Static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
static_cascade = [static_app, app]
if config['r2.plugins'] and g.config['uncompressedJS']:
plugin_static_apps = Cascade([
StaticURLParser(plugin.static_dir)
for plugin in config['r2.plugins']
])
static_cascade.insert(0, plugin_static_apps)
app = Cascade(static_cascade)
app = FullPathMiddleware(app)
if not g.config['uncompressedJS'] and g.config['debug']:
static_fallback = StaticTestMiddleware(static_app,
g.config['static_path'],
g.config['static_domain'])
app = Cascade([static_fallback, app])
app = SafetyMiddleware(app)
app.config = config
return app
def package_search(context, data_dict):
# sometimes context['schema'] is None
schema = (context.get('schema')
or logic.schema.default_package_search_schema())
data_dict, errors = _validate(data_dict, schema, context)
# put the extras back into the data_dict so that the search can
# report needless parameters
data_dict.update(data_dict.get('__extras', {}))
data_dict.pop('__extras', None)
if errors:
raise ValidationError(errors)
model = context['model']
session = context['session']
user = context.get('user')
_check_access('package_search', context, data_dict)
# Move ext_ params to extras and remove them from the root of the search
# params, so they don't cause and error
data_dict['extras'] = data_dict.get('extras', {})
for key in [key for key in data_dict.keys() if key.startswith('ext_')]:
data_dict['extras'][key] = data_dict.pop(key)
# check if some extension needs to modify the search params
for item in plugins.PluginImplementations(plugins.IPackageController):
data_dict = item.before_search(data_dict)
# the extension may have decided that it is not necessary to perform
# the query
abort = data_dict.get('abort_search', False)
if data_dict.get('sort') in (None, 'rank'):
data_dict['sort'] = 'score desc, metadata_modified desc'
results = []
if not abort:
if asbool(data_dict.get('use_default_schema')):
data_source = 'data_dict'
else:
data_source = 'validated_data_dict'
data_dict.pop('use_default_schema', None)
result_fl = data_dict.get('fl')
if not result_fl:
data_dict['fl'] = 'id {0}'.format(data_source)
else:
data_dict['fl'] = ' '.join(result_fl)
# Remove before these hit solr FIXME: whitelist instead
include_private = asbool(data_dict.pop('include_private', False))
include_drafts = asbool(data_dict.pop('include_drafts', False))
data_dict.setdefault('fq', '')
if not include_private:
data_dict['fq'] = '+capacity:public ' + data_dict['fq']
if include_drafts:
data_dict['fq'] += ' +state:(active OR draft)'
# Pop these ones as Solr does not need them
extras = data_dict.pop('extras', None)
# enforce permission filter based on user
if context.get('ignore_auth') or (user and authz.is_sysadmin(user)):
labels = None
else:
labels = lib_plugins.get_permission_labels(
).get_user_dataset_labels(context['auth_user_obj'])
query = PackageSearchQuery()
query.run(data_dict, permission_labels=labels)
# Add them back so extensions can use them on after_search
data_dict['extras'] = extras
if result_fl and not extras.get('fl_compatible', False):
for package in query.results:
if package.get('extras'):
package.update(package['extras'])
package.pop('extras')
results.append(package)
else:
for package in query.results:
# get the package object
package_dict = package.get(data_source)
## use data in search index if there
if package_dict:
# the package_dict still needs translating when being viewed
package_dict = json.loads(package_dict)
if context.get('for_view'):
for item in plugins.PluginImplementations(
plugins.IPackageController):
package_dict = item.before_view(package_dict)
results.append(package_dict)
else:
log.error(
'No package_dict is coming from solr for package '
'id %s', package['id'])
count = query.count
facets = query.facets
raw_solr_results = {
'results': query.results,
'highlighting': query.highlighting,
'count': query.count,
'facets': query.facets,
}
else:
count = 0
facets = {}
results = []
raw_solr_results = {}
search_results = {
'count': count,
'facets': facets,
'results': results,
'sort': data_dict['sort'],
}
include_raw_solr_results = False
for item in plugins.PluginImplementations(plugins.IPackageController):
if 'include_raw_solr_results' in dir(item):
include_raw_solr_results = include_raw_solr_results \
or item.include_raw_solr_results(data_dict)
if include_raw_solr_results:
search_results['raw_solr_results'] = raw_solr_results
# create a lookup table of group name to title for all the groups and
# organizations in the current search's facets.
group_names = []
for field_name in ('groups', 'organization'):
group_names.extend(facets.get(field_name, {}).keys())
groups = (session.query(model.Group.name, model.Group.title).filter(
model.Group.name.in_(group_names)).all() if group_names else [])
group_titles_by_name = dict(groups)
# Transform facets into a more useful data structure.
restructured_facets = {}
for key, value in facets.items():
restructured_facets[key] = {'title': key, 'items': []}
for key_, value_ in value.items():
new_facet_dict = {}
new_facet_dict['name'] = key_
if key in ('groups', 'organization'):
display_name = group_titles_by_name.get(key_, key_)
display_name = display_name if display_name and display_name.strip(
) else key_
new_facet_dict['display_name'] = display_name
elif key == 'license_id':
license = model.Package.get_license_register().get(key_)
if license:
new_facet_dict['display_name'] = license.title
else:
new_facet_dict['display_name'] = key_
else:
new_facet_dict['display_name'] = key_
new_facet_dict['count'] = value_
restructured_facets[key]['items'].append(new_facet_dict)
search_results['search_facets'] = restructured_facets
# check if some extension needs to modify the search results
for item in plugins.PluginImplementations(plugins.IPackageController):
search_results = item.after_search(search_results, data_dict)
# After extensions have had a chance to modify the facets, sort them by
# display name.
for facet in search_results['search_facets']:
search_results['search_facets'][facet]['items'] = sorted(
search_results['search_facets'][facet]['items'],
key=lambda facet: facet['display_name'],
reverse=True)
return search_results
def _make_core_app(root, global_conf, full_stack=True, **app_conf):
"""
Set allura up with the settings found in the PasteDeploy configuration
file used.
:param root: The controller module containing the TG root
:param global_conf: The global settings for allura (those
defined under the ``[DEFAULT]`` section).
:type global_conf: dict
:param full_stack: Should the whole TG2 stack be set up?
:type full_stack: str or bool
:return: The allura application with all the relevant middleware
loaded.
This is the PasteDeploy factory for the allura application.
``app_conf`` contains all the application-specific settings (those defined
under ``[app:main]``.
"""
# Run all the initialization code here
mimetypes.init([pkg_resources.resource_filename('allura', 'etc/mime.types')] + mimetypes.knownfiles)
# Configure MongoDB
ming.configure(**app_conf)
# Configure ActivityStream
if asbool(app_conf.get('activitystream.recording.enabled', False)):
activitystream.configure(**h.convert_bools(app_conf, prefix='activitystream.'))
# Configure EW variable provider
ew.render.TemplateEngine.register_variable_provider(get_tg_vars)
# Set FormEncode language to english, as we don't support any other locales
formencode.api.set_stdtranslation(domain='FormEncode', languages=['en'])
# Create base app
base_config = ForgeConfig(root)
load_environment = base_config.make_load_environment()
# Code adapted from tg.configuration, replacing the following lines:
# make_base_app = base_config.setup_tg_wsgi_app(load_environment)
# app = make_base_app(global_conf, full_stack=True, **app_conf)
# Configure the TG environment
load_environment(global_conf, app_conf)
app = tg.TGApp()
for mw_ep in h.iter_entry_points('allura.middleware'):
Middleware = mw_ep.load()
if getattr(Middleware, 'when', 'inner') == 'inner':
app = Middleware(app, config)
# Required for sessions
app = SessionMiddleware(app, config, data_serializer=BeakerPickleSerializerWithLatin1())
# Handle "Remember me" functionality
app = RememberLoginMiddleware(app, config)
# Redirect 401 to the login page
app = LoginRedirectMiddleware(app)
# Add instrumentation
app = AlluraTimerMiddleware(app, app_conf)
# Clear cookies when the CSRF field isn't posted
if not app_conf.get('disable_csrf_protection'):
app = CSRFMiddleware(app, '_session_id')
if asbool(config.get('cors.enabled', False)):
# Handle CORS requests
allowed_methods = aslist(config.get('cors.methods'))
allowed_headers = aslist(config.get('cors.headers'))
cache_duration = asint(config.get('cors.cache_duration', 0))
app = CORSMiddleware(app, allowed_methods, allowed_headers, cache_duration)
# Setup the allura SOPs
app = allura_globals_middleware(app)
# Ensure http and https used per config
if config.get('override_root') != 'task':
app = SSLMiddleware(app, app_conf.get('no_redirect.pattern'),
app_conf.get('force_ssl.pattern'),
app_conf.get('force_ssl.logged_in'))
# Setup resource manager, widget context SOP
app = ew.WidgetMiddleware(
app,
compress=True,
use_cache=not asbool(global_conf['debug']),
script_name=app_conf.get('ew.script_name', '/_ew_resources/'),
url_base=app_conf.get('ew.url_base', '/_ew_resources/'),
extra_headers=ast.literal_eval(app_conf.get('ew.extra_headers', '[]')),
cache_max_age=asint(app_conf.get('ew.cache_header_seconds', 60*60*24*365)),
# settings to pass through to jinja Environment for EW core widgets
# these are for the easywidgets' own [easy_widgets.engines] entry point
# (the Allura [easy_widgets.engines] entry point is named "jinja" (not jinja2) but it doesn't need
# any settings since it is a class that uses the same jinja env as the rest of allura)
**{
'jinja2.auto_reload': asbool(config['auto_reload_templates']),
'jinja2.bytecode_cache': AlluraJinjaRenderer._setup_bytecode_cache(),
'jinja2.cache_size': asint(config.get('jinja_cache_size', -1)),
}
)
# Handle static files (by tool)
app = StaticFilesMiddleware(app, app_conf.get('static.script_name'))
# Handle setup and flushing of Ming ORM sessions
app = MingMiddleware(app)
# Set up the registry for stacked object proxies (SOPs).
# streaming=true ensures they won't be cleaned up till
# the WSGI application's iterator is exhausted
app = RegistryManager(app, streaming=True)
# "task" wsgi would get a 2nd request to /error/document if we used this middleware
if config.get('override_root') not in ('task', 'basetest_project_root'):
if asbool(config['debug']):
# Converts exceptions to HTTP errors, shows traceback in debug mode
# don't use TG footer with extra CSS & images that take time to load
tg.error.footer_html = '<!-- %s %s -->'
app = tg.error.ErrorHandler(app, global_conf, **config['tg.errorware'])
else:
app = ErrorMiddleware(app, config, **config['tg.errorware'])
app = SetRequestHostFromConfig(app, config)
# Redirect some status codes to /error/document
if asbool(config['debug']):
app = StatusCodeRedirect(app, base_config.handle_status_codes)
else:
app = StatusCodeRedirect(
app, base_config.handle_status_codes + [500])
for mw_ep in h.iter_entry_points('allura.middleware'):
Middleware = mw_ep.load()
if getattr(Middleware, 'when', 'inner') == 'outer':
app = Middleware(app, config)
return app
def _load_global_config(self) -> None:
"""
Load generic config
"""
###
# General
###
self.SQLALCHEMY__URL = self.get_raw_config("sqlalchemy.url", "")
self.DEFAULT_LANG = self.get_raw_config("default_lang",
DEFAULT_FALLBACK_LANG)
backend_folder = os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))
tracim_v2_folder = os.path.dirname(backend_folder)
default_color_config_file_path = os.path.join(tracim_v2_folder,
"color.json")
self.COLOR__CONFIG_FILE_PATH = self.get_raw_config(
"color.config_file_path", default_color_config_file_path)
default_enabled_app = ("contents/thread,"
"contents/file,"
"contents/html-document,"
"contents/folder,"
"agenda")
self.APP__ENABLED = string_to_list(
self.get_raw_config("app.enabled", default_enabled_app),
separator=",",
cast_func=str,
do_strip=True,
)
self.DEPOT_STORAGE_DIR = self.get_raw_config("depot_storage_dir")
self.DEPOT_STORAGE_NAME = self.get_raw_config("depot_storage_name")
self.PREVIEW_CACHE_DIR = self.get_raw_config("preview_cache_dir")
self.AUTH_TYPES = string_to_list(
self.get_raw_config("auth_types", "internal"),
separator=",",
cast_func=AuthType,
do_strip=True,
)
self.REMOTE_USER_HEADER = self.get_raw_config("remote_user_header",
None)
# TODO - G.M - 2018-09-11 - Deprecated param
# self.DATA_UPDATE_ALLOWED_DURATION = int(self.get_raw_config(
# 'content.update.allowed.duration',
# 0,
# ))
self.API__KEY = self.get_raw_config("api.key", "", secret=True)
self.SESSION__REISSUE_TIME = int(
self.get_raw_config("session.reissue_time", "120"))
self.SESSION__DATA_DIR = self.get_raw_config("session.data_dir")
self.SESSION__LOCK_DIR = self.get_raw_config("session.lock_dir")
self.WEBSITE__TITLE = self.get_raw_config("website.title", "TRACIM")
# base url of the frontend
self.WEBSITE__BASE_URL = self.get_raw_config("website.base_url", "")
self.API__BASE_URL = self.get_raw_config("api.base_url",
self.WEBSITE__BASE_URL)
if self.API__BASE_URL != self.WEBSITE__BASE_URL:
default_cors_allowed_origin = "{},{}".format(
self.WEBSITE__BASE_URL, self.API__BASE_URL)
else:
default_cors_allowed_origin = self.WEBSITE__BASE_URL
self.CORS__ACCESS_CONTROL_ALLOWED_ORIGIN = string_to_list(
self.get_raw_config("cors.access-control-allowed-origin",
default_cors_allowed_origin),
separator=",",
cast_func=str,
do_strip=True,
)
self.USER__AUTH_TOKEN__VALIDITY = int(
self.get_raw_config("user.auth_token.validity", "604800"))
# TODO - G.M - 2019-03-14 - retrocompat code,
# will be deleted in the future (https://github.com/tracim/tracim/issues/1483)
defaut_reset_password_validity = "900"
self.USER__RESET_PASSWORD__VALIDITY = self.get_raw_config(
"user.reset_password.validity")
if self.USER__RESET_PASSWORD__VALIDITY:
logger.warning(
self,
"user.reset_password.validity parameter is deprecated ! "
"please use user.reset_password.token_lifetime instead.",
)
self.USER__RESET_PASSWORD__TOKEN_LIFETIME = self.USER__RESET_PASSWORD__VALIDITY
else:
self.USER__RESET_PASSWORD__TOKEN_LIFETIME = int(
self.get_raw_config("user.reset_password.token_lifetime",
defaut_reset_password_validity))
self.DEBUG = asbool(self.get_raw_config("debug", "false"))
self.PREVIEW__JPG__RESTRICTED_DIMS = asbool(
self.get_raw_config("preview.jpg.restricted_dims", "false"))
self.PREVIEW__JPG__ALLOWED_DIMS = string_to_list(
self.get_raw_config("preview.jpg.allowed_dims", "256x256"),
cast_func=PreviewDim.from_string,
separator=",",
)
self.FRONTEND__SERVE = asbool(
self.get_raw_config("frontend.serve", "false"))
# INFO - G.M - 2018-08-06 - we pretend that frontend_dist_folder
# is probably in frontend subfolder
# of tracim_v2 parent of both backend and frontend
backend_folder = os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))
tracim_v2_folder = os.path.dirname(backend_folder)
backend_i18n_folder = os.path.join(backend_folder, "tracim_backend",
"locale")
self.BACKEND__I18N_FOLDER_PATH = self.get_raw_config(
"backend.i18n_folder_path", backend_i18n_folder)
frontend_dist_folder = os.path.join(tracim_v2_folder, "frontend",
"dist")
self.FRONTEND__DIST_FOLDER_PATH = self.get_raw_config(
"frontend.dist_folder_path", frontend_dist_folder)