def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
self._target = None
self._startTime = None
self._endTime = None
self._doProcess = True
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
self._aw = AccessWrapper(
) # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._doProcess = True
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._doProcess = True # Flag which indicates whether the RH process
def __init__(self, req=None):
"""Constructor. Initialises the rh setting up basic attributes so it is
able to process the request.
Parameters:
req - OBSOLETE, MUST BE NONE
"""
RequestHandlerBase.__init__(self, req)
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper(
) # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True # Flag which indicates whether the RH process
def __init__(self):
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True # Flag which indicates whether the RH process
def __init__(self, req=None):
"""Constructor. Initialises the rh setting up basic attributes so it is
able to process the request.
Parameters:
req - OBSOLETE, MUST BE NONE
"""
RequestHandlerBase.__init__(self, req)
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() #Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True #Flag which indicates whether the RH process
class RH(RequestHandlerBase):
"""This class is the base for request handlers of the application. A request
handler will be instantiated when a web request arrives to mod_python;
the mp layer will forward the request to the corresponding request
handler which will know which action has to be performed (displaying a
web page or performing some operation and redirecting to another page).
Request handlers will be responsible for parsing the parameters coming
from a mod_python request, handle the errors which occurred during the
action to perform, managing the sessions, checking security for each
operation (thus they implement the access control system of the web
interface).
It is important to encapsulate all this here as in case of changing
the web application framework we'll just need to adapt this layer (the
rest of the system wouldn't need any change).
Attributes:
_uh - (URLHandler) Associated URLHandler which points to the
current rh.
_req - UNUSED/OBSOLETE, always None
_requestStarted - (bool) Flag which tells whether a DB transaction
has been started or not.
_aw - (AccessWrapper) Current access information for the rh.
_target - (Locable) Reference to an object which is the destination
of the operations needed to carry out the rh. If set it must
provide (through the standard Locable interface) the methods
to get the url parameters in order to reproduce the access to
the rh.
_reqParams - (dict) Dictionary containing the received HTTP
parameters (independently of the method) transformed into
python data types. The key is the parameter name while the
value should be the received paramter value (or values).
"""
_doNotSanitizeFields = []
CSRF_ENABLED = None # require a csrf_token when accessing the RH with anything but GET
EVENT_FEATURE = None # require a certain event feature when accessing the RH. See `EventFeature` for details
#: A dict specifying how the url should be normalized.
#: `args` is a dictionary mapping view args keys to callables
#: used to retrieve the expected value for those arguments if they
#: are present in the request's view args.
#: `locators` is a set of callables returning objects with locators.
#: `preserved_args` is a set of view arg names which will always
#: be copied from the current request if present.
#: The callables are always invoked with a single `self` argument
#: containing the RH instance.
#: `endpoint` may be used to specify the endpoint used to build
#: the URL in case of a redirect. Usually this should not be used
#: in favor of ``request.endpoint`` being used if no custom endpoint
#: is set.
#: Arguments specified in the `defaults` of any rule matching the
#: current endpoint are always excluded when checking if the args
#: match or when building a new URL.
#: If the view args built from the returned objects do not match
#: the request's view args, a redirect is issued automatically.
#: If the request is not using GET/HEAD, a 404 error is raised
#: instead of a redirect since such requests cannot be redirected
#: but executing them on the wrong URL may pose a security risk in
#: case and of the non-relevant URL segments is used for access
#: checks.
normalize_url_spec = {
'args': {},
'locators': set(),
'preserved_args': set(),
'endpoint': None
}
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._doProcess = True # Flag which indicates whether the RH process
# must be carried out; this is useful for
# the checkProtection methods when they
# detect that an immediate redirection is
# needed
# Methods =============================================================
def validate_json(self, schema, json=None):
"""Validates the request's JSON payload using a JSON schema.
:param schema: The JSON schema used for validation.
:param json: The JSON object (defaults to ``request.json``)
:raises BadRequest: if the JSON validation failed
"""
if json is None:
json = request.json
try:
jsonschema.validate(json, schema)
except jsonschema.ValidationError as e:
raise BadRequest('Invalid JSON payload: {}'.format(e.message))
def getTarget(self):
return self._target
def _setSessionUser(self):
self._aw.setUser(session.avatar)
@property
def csrf_token(self):
return session.csrf_token if session.csrf_protected else ''
def _getRequestParams(self):
return self._reqParams
def getRequestParams(self):
return self._getRequestParams()
def _redirect(self, targetURL, status=303):
if isinstance(targetURL, Response):
status = targetURL.status_code
targetURL = targetURL.headers['Location']
else:
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def _processError(self, e):
raise
def normalize_url(self):
"""Performs URL normalization.
This uses the :attr:`normalize_url_spec` to check if the URL
params are what they should be and redirects or fails depending
on the HTTP method used if it's not the case.
:return: ``None`` or a redirect response
"""
if current_app.debug and self.normalize_url_spec is RH.normalize_url_spec:
# in case of ``class SomeRH(RH, MixinWithNormalization)``
# the default value from `RH` overwrites the normalization
# rule from ``MixinWithNormalization``. this is never what
# the developer wants so we fail if it happens. the proper
# solution is ``class SomeRH(MixinWithNormalization, RH)``
cls = next((x
for x in inspect.getmro(self.__class__)
if (x is not RH and x is not self.__class__ and hasattr(x, 'normalize_url_spec') and
getattr(x, 'normalize_url_spec', None) is not RH.normalize_url_spec)),
None)
if cls is not None:
raise Exception('Normalization rule of {} in {} is overwritten by base RH. Put mixins with class-level '
'attributes on the left of the base class'.format(cls, self.__class__))
if not self.normalize_url_spec or not any(self.normalize_url_spec.itervalues()):
return
spec = {
'args': self.normalize_url_spec.get('args', {}),
'locators': self.normalize_url_spec.get('locators', set()),
'preserved_args': self.normalize_url_spec.get('preserved_args', set()),
'endpoint': self.normalize_url_spec.get('endpoint', None)
}
# Initialize the new view args with preserved arguments (since those would be lost otherwise)
new_view_args = {k: v for k, v in request.view_args.iteritems() if k in spec['preserved_args']}
# Retrieve the expected values for all simple arguments (if they are currently present)
for key, getter in spec['args'].iteritems():
if key in request.view_args:
new_view_args[key] = getter(self)
# Retrieve the expected values from locators
prev_locator_args = {}
for getter in spec['locators']:
value = getter(self)
if value is None:
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
locator_args = get_locator(value)
reused_keys = set(locator_args) & prev_locator_args.viewkeys()
if any(locator_args[k] != prev_locator_args[k] for k in reused_keys):
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
new_view_args.update(locator_args)
prev_locator_args.update(locator_args)
# Get all default values provided by the url map for the endpoint
defaults = set(itertools.chain.from_iterable(r.defaults
for r in current_app.url_map.iter_rules(request.endpoint)
if r.defaults))
def _convert(v):
# some legacy code has numeric ids in the locator data, but still takes
# string ids in the url rule (usually for confId)
return unicode(v) if isinstance(v, (int, long)) else v
provided = {k: _convert(v) for k, v in request.view_args.iteritems() if k not in defaults}
new_view_args = {k: _convert(v) for k, v in new_view_args.iteritems()}
if new_view_args != provided:
if request.method in {'GET', 'HEAD'}:
endpoint = spec['endpoint'] or request.endpoint
try:
return redirect(url_for(endpoint, **dict(request.args.to_dict(), **new_view_args)))
except BuildError as e:
if current_app.debug:
raise
logger.warn('BuildError during normalization: %s', e)
raise NotFound
else:
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
def _checkParams(self, params):
"""This method is called before _checkProtection and is a good place
to assign variables from request params to member variables.
Note that in any new code the params argument SHOULD be IGNORED.
Use the following objects provided by Flask instead:
from flask import request
request.view_args (URL route params)
request.args (GET params (from the query string))
request.form (POST params)
request.values (GET+POST params - use only if ABSOLUTELY NECESSARY)
If you only want to run some code for GET or POST requests, you can create
a method named e.g. _checkParams_POST which will be executed AFTER this one.
The method is called without any arguments (except self).
"""
pass
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [m for m in HTTP_VERBS if hasattr(self, '_process_' + m)]
raise MethodNotAllowed(valid_methods)
return method()
def _checkCSRF(self):
token = request.headers.get('X-CSRF-Token') or request.form.get('csrf_token')
if token is None:
# Might be a WTForm with a prefix. In that case the field name is '<prefix>-csrf_token'
token = next((v for k, v in request.form.iteritems() if k.endswith('-csrf_token')), None)
if self.CSRF_ENABLED and request.method != 'GET' and token != session.csrf_token:
msg = _(u"It looks like there was a problem with your current session. Please use your browser's back "
u"button, reload the page and try again.")
raise BadRequest(msg)
elif self.CSRF_ENABLED is None and current_app.debug and request.method != 'GET':
# Warn if CSRF is not enabled for a RH in new code
module = self.__class__.__module__
if module.startswith('indico.modules.') or module.startswith('indico.core.'):
msg = (u'{} request sent to {} which has no CSRF checks. Set `CSRF_ENABLED = True` in the class to '
u'enable them.').format(request.method, self.__class__.__name__)
warnings.warn(msg, RuntimeWarning)
# legacy csrf check (referer-based):
# Check referer for POST requests. We do it here so we can properly use indico's error handling
if Config.getInstance().getCSRFLevel() < 3 or request.method != 'POST':
return
referer = request.referrer
# allow empty - otherwise we might lock out paranoid users blocking referers
if not referer:
return
# valid http referer
if referer.startswith(Config.getInstance().getBaseURL()):
return
raise BadRefererError('This operation is not allowed from an external referer.')
def _check_event_feature(self):
from indico.modules.events.features.util import require_feature
event_id = request.view_args.get('confId') or request.view_args.get('event_id')
if event_id is not None:
require_feature(event_id, self.EVENT_FEATURE)
@jsonify_error
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH."""
if Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPGenericError(self).display()
@jsonify_error(status=500, logging_level='exception')
def _processUnexpectedError(self, e):
"""Unexpected errors"""
self._responseUtil.redirect = None
if Config.getInstance().getEmbeddedWebserver() or Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPUnexpectedError(self).display()
@jsonify_error(status=403)
def _processForbidden(self, e):
if session.user is None and not request.is_xhr and not e.response and request.blueprint != 'auth':
return redirect_to_login(reason=_("Please log in to access this page."))
message = _("Access Denied")
explanation = get_error_description(e)
return render_error(message, explanation)
@jsonify_error(status=400)
def _processBadRequest(self, e):
message = _("Bad Request")
return render_error(message, e.description)
@jsonify_error(status=401)
def _processUnauthorized(self, e):
message = _("Unauthorized")
return render_error(message, e.description)
@jsonify_error(status=400)
def _processBadData(self, e):
message = _("Invalid or expired token")
return render_error(message, e.message)
@jsonify_error(status=403)
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPAccessError(self).display()
@jsonify_error
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPKeyAccessError(self).display()
@jsonify_error
def _processModificationError(self, e):
"""Handles modification errors occured during the process of a RH."""
if not session.user:
return redirect_to_login(reason=_("Please log in to access this page. If you have a modification key, you "
"may enter it afterwards."))
return errors.WPModificationError(self).display()
@jsonify_error(status=400)
def _processBadRequestKeyError(self, e):
"""Request lacks a necessary key for processing"""
msg = _('Required argument missing: %s') % e.message
return errors.WPFormValuesError(self, msg).display()
@jsonify_error
def _processNoReportError(self, e):
"""Process errors without reporting"""
return errors.WPNoReportError(self, e).display()
@jsonify_error(status=400)
def _processUserValueError(self, e):
"""Process errors without reporting"""
return errors.WPNoReportError(self, e).display()
@jsonify_error(status=404)
def _processNotFoundError(self, e):
if isinstance(e, NotFound):
message = _("Page not found") # that's a bit nicer than "404: Not Found"
explanation = get_error_description(e)
else:
message = e.getMessage()
explanation = e.getExplanation()
return render_error(message, explanation)
@jsonify_error
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
return errors.WPFormValuesError(self, e).display()
@jsonify_error
def _processRestrictedHTML(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
@jsonify_error
def _processHtmlForbiddenTag(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
def _check_auth(self, params):
self._setSessionUser()
if session.user:
logger.info('Request authenticated: %r', session.user)
self._checkCSRF()
self._reqParams = copy.copy(params)
def _do_process(self, profile):
profile_name = res = ''
try:
# old code gets parameters from call
# new code utilizes of flask.request
if len(inspect.getargspec(self._checkParams).args) < 2:
cp_result = self._checkParams()
else:
cp_result = self._checkParams(self._reqParams)
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
func = getattr(self, '_checkParams_' + request.method, None)
if func:
cp_result = func()
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFoundError(_('The specified item could not be found.'), title=_('Item not found'))
rv = self.normalize_url()
if rv is not None:
return '', rv
self._checkProtection()
func = getattr(self, '_checkProtection_' + request.method, None)
if func:
func()
security.Sanitization.sanitizationCheck(self._target,
self._reqParams,
self._aw,
self._doNotSanitizeFields)
if self._doProcess:
if profile:
profile_name = os.path.join(Config.getInstance().getTempDir(), 'stone{}.prof'.format(random.random()))
result = [None]
profiler.runctx('result[0] = self._process()', globals(), locals(), profile_name)
res = result[0]
else:
res = self._process()
return profile_name, res
def process(self, params):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
cfg = Config.getInstance()
profile = cfg.getProfile()
profile_name, res, textLog = '', '', []
self._startTime = datetime.now()
g.rh = self
if self.EVENT_FEATURE is not None:
self._check_event_feature()
textLog.append("%s : Database request started" % (datetime.now() - self._startTime))
logger.info(u'Request started: %s %s [IP=%s] [PID=%s]',
request.method, request.relative_url, request.remote_addr, os.getpid())
is_error_response = False
try:
try:
fossilize.clearCache()
GenericMailer.flushQueue(False)
self._check_auth(params)
profile_name, res = self._do_process(profile)
signals.after_process.send()
if self.commit:
if GenericMailer.has_queue():
# ensure we fail early (before sending out e-mails)
# in case there are DB constraint violations, etc...
db.enforce_constraints()
GenericMailer.flushQueue(True)
db.session.commit()
else:
db.session.rollback()
except DatabaseError:
handle_sqlalchemy_database_error() # this will re-raise an exception
logger.info('Request successful')
except Exception as e:
db.session.rollback()
res = self._getMethodByExceptionName(e)(e)
if isinstance(e, HTTPException) and e.response is not None:
res = e.response
is_error_response = True
totalTime = (datetime.now() - self._startTime)
textLog.append('{} : Request ended'.format(totalTime))
# log request timing
if profile and os.path.isfile(profile_name):
rep = Config.getInstance().getTempDir()
stats = pstats.Stats(profile_name)
stats.sort_stats('cumulative', 'time', 'calls')
stats.dump_stats(os.path.join(rep, 'IndicoRequestProfile.log'))
output = StringIO.StringIO()
sys.stdout = output
stats.print_stats(100)
sys.stdout = sys.__stdout__
s = output.getvalue()
f = file(os.path.join(rep, 'IndicoRequest.log'), 'a+')
f.write('--------------------------------\n')
f.write('URL : {}\n'.format(request.url))
f.write('{} : start request\n'.format(self._startTime))
f.write('params:{}'.format(params))
f.write('\n'.join(textLog))
f.write(s)
f.write('--------------------------------\n\n')
f.close()
if profile and profile_name and os.path.exists(profile_name):
os.remove(profile_name)
if self._responseUtil.call:
return self._responseUtil.make_call()
if is_error_response and isinstance(res, (current_app.response_class, Response)):
# if we went through error handling code, responseUtil._status has been changed
# so make_response() would fail
return res
# In case of no process needed, we should return empty string to avoid erroneous output
# specially with getVars breaking the JS files.
if not self._doProcess or res is None:
return self._responseUtil.make_empty()
return self._responseUtil.make_response(res)
def _getMethodByExceptionName(self, e):
exception_name = {
'NotFound': 'NotFoundError',
'MaKaCError': 'GeneralError',
'IndicoError': 'GeneralError',
'ValueError': 'UnexpectedError',
'Exception': 'UnexpectedError',
'AccessControlError': 'AccessError'
}.get(type(e).__name__, type(e).__name__)
if isinstance(e, BadData): # we also want its subclasses
exception_name = 'BadData'
return getattr(self, '_process{}'.format(exception_name), self._processUnexpectedError)
class RH(RequestHandlerBase):
"""This class is the base for request handlers of the application. A request
handler will be instantiated when a web request arrives to mod_python;
the mp layer will forward the request to the corresponding request
handler which will know which action has to be performed (displaying a
web page or performing some operation and redirecting to another page).
Request handlers will be responsible for parsing the parameters coming
from a mod_python request, handle the errors which occurred during the
action to perform, managing the sessions, checking security for each
operation (thus they implement the access control system of the web
interface).
It is important to encapsulate all this here as in case of changing
the web application framework we'll just need to adapt this layer (the
rest of the system wouldn't need any change).
Attributes:
_uh - (URLHandler) Associated URLHandler which points to the
current rh.
_req - UNUSED/OBSOLETE, always None
_requestStarted - (bool) Flag which tells whether a DB transaction
has been started or not.
_aw - (AccessWrapper) Current access information for the rh.
_target - (Locable) Reference to an object which is the destination
of the operations needed to carry out the rh. If set it must
provide (through the standard Locable interface) the methods
to get the url parameters in order to reproduce the access to
the rh.
_reqParams - (dict) Dictionary containing the received HTTP
parameters (independently of the method) transformed into
python data types. The key is the parameter name while the
value should be the received paramter value (or values).
"""
_tohttps = False # set this value to True for the RH that must be HTTPS when there is a BaseSecureURL
_doNotSanitizeFields = []
_isMobile = True # this value means that the generated web page can be mobile
HTTP_VERBS = frozenset(('GET', 'POST', 'PUT', 'DELETE'))
def __init__(self):
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True # Flag which indicates whether the RH process
# must be carried out; this is useful for
# the checkProtection methods when they
# detect that an immediate redirection is
# needed
# Methods =============================================================
def getTarget(self):
return self._target
def isMobile(self):
return self._isMobile
def _setSessionUser(self):
self._aw.setUser(session.user)
@property
def csrf_token(self):
return session.csrf_token
def _getRequestParams(self):
return self._reqParams
def getRequestParams(self):
return self._getRequestParams()
def _disableCaching(self):
"""Disables caching"""
# IE doesn't seem to like 'no-cache' Cache-Control headers...
if request.user_agent.browser == 'msie':
# actually, the only way to safely disable caching seems to be this one
self._responseUtil.headers["Cache-Control"] = "private"
self._responseUtil.headers["Expires"] = "-1"
else:
self._responseUtil.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
self._responseUtil.headers["Pragma"] = "no-cache"
def _redirect(self, targetURL, status=303):
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def _changeRH(self, rh, params):
"""Calls the specified RH after processing this one"""
self._responseUtil.call = lambda: rh().process(params)
def _checkHttpsRedirect(self):
"""If HTTPS must be used but it is not, redirect!"""
if self.use_https() and not request.is_secure:
self._redirect(self.getRequestURL(secure=True))
return True
else:
return False
def _normaliseListParam(self, param):
if not isinstance(param, list):
return [param]
return param
def _processError(self, e):
raise
def _checkParams(self, params):
"""This method is called before _checkProtection and is a good place
to assign variables from request params to member variables.
Note that in any new code the params argument SHOULD be IGNORED.
Use the following objects provided by Flask instead:
from flask import request
request.view_args (URL route params)
request.args (GET params (from the query string))
request.form (POST params)
request.values (GET+POST params - use only if ABSOLUTELY NECESSARY)
If you only want to run some code for GET or POST requests, you can create
a method named e.g. _checkParams_POST which will be executed AFTER this one.
The method is called without any arguments (except self).
"""
pass
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [m for m in self.HTTP_VERBS if hasattr(self, '_process_' + m)]
raise MethodNotAllowed(valid_methods)
return method()
def _checkCSRF(self):
# Check referer for POST requests. We do it here so we can properly use indico's error handling
if Config.getInstance().getCSRFLevel() < 3 or request.method != 'POST':
return
referer = request.referrer
# allow empty - otherwise we might lock out paranoid users blocking referers
if not referer:
return
# valid http referer
if referer.startswith(Config.getInstance().getBaseURL()):
return
# valid https referer - if https is enabled
base_secure = Config.getInstance().getBaseSecureURL()
if base_secure and referer.startswith(base_secure):
return
raise BadRefererError('This operation is not allowed from an external referer.')
@jsonify_error
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH."""
if Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPGenericError(self).display()
@jsonify_error(status=500, logging_level='exception')
def _processUnexpectedError(self, e):
"""Unexpected errors"""
self._responseUtil.redirect = None
if Config.getInstance().getEmbeddedWebserver() or Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPUnexpectedError(self).display()
@jsonify_error
def _processHostnameResolveError(self, e):
"""Unexpected errors"""
return errors.WPHostnameResolveError(self).display()
@jsonify_error(status=403)
def _processForbidden(self, e):
message = _("Access Denied")
if e.description == Forbidden.description:
explanation = _("You are not allowed to access this page.")
else:
explanation = e.description
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error(status=403)
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPAccessError(self).display()
@jsonify_error
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
# We are going to redirect to the page asking for access key
# and so it must be https if there is a BaseSecureURL. And that's
# why we set _tohttps to True.
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPKeyAccessError(self).display()
@jsonify_error
def _processModificationError(self, e):
"""Handles modification errors occured during the process of a RH."""
# Redirect to HTTPS in case the user is logged in
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPModificationError(self).display()
@jsonify_error(status=400)
def _processBadRequestKeyError(self, e):
"""Request lacks a necessary key for processing"""
msg = _('Required argument missing: %s') % e.message
return errors.WPFormValuesError(self, msg).display()
# TODO: check this method to integrate with jsonify error
def _processOAuthError(self, e):
res = json.dumps(e.fossilize())
header = oauth.build_authenticate_header(realm=Config.getInstance().getBaseSecureURL())
self._responseUtil.headers.extend(header)
self._responseUtil.content_type = 'application/json'
self._responseUtil.status = e.code
return res
@jsonify_error
def _processConferenceClosedError(self, e):
"""Treats access to modification pages for conferences when they are closed."""
return WPConferenceModificationClosed(self, e._conf).display()
@jsonify_error
def _processTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPTimingError(self, e).display()
@jsonify_error
def _processNoReportError(self, e):
"""Process errors without reporting"""
return errors.WPNoReportError(self, e).display()
@jsonify_error(status=404)
def _processNotFoundError(self, e):
if isinstance(e, NotFound):
message = _("Page not found") # that's a bit nicer than "404: Not Found"
if e.description == NotFound.description:
explanation = _("The page you are looking for doesn't exist.")
else:
explanation = e.description
else:
message = e.getMessage()
explanation = e.getExplanation()
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error
def _processParentTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPParentTimingError(self, e).display()
@jsonify_error
def _processEntryTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPEntryTimingError(self, e).display()
@jsonify_error
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
return errors.WPFormValuesError(self, e).display()
@jsonify_error
def _processLaTeXError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPLaTeXError(self, e).display()
@jsonify_error
def _processRestrictedHTML(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
@jsonify_error
def _processHtmlScriptError(self, e):
""" TODO """
return errors.WPHtmlScriptError(self, escape(str(e))).display()
@jsonify_error
def _processHtmlForbiddenTag(self, e):
""" TODO """
return errors.WPRestrictedHTML(self, escape(str(e))).display()
def _process_retry_setup(self):
# clear the fossile cache at the start of each request
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
# clear the existing redis pipeline
if self._redisPipeline:
self._redisPipeline.reset()
def _process_retry_auth_check(self, params):
# keep a link to the web session in the access wrapper
# this is used for checking access/modification key existence
# in the user session
self._aw.setIP(request.remote_addr)
self._setSessionUser()
if self._getAuth():
if self._getUser():
Logger.get('requestHandler').info('Request %s identified with user %s (%s)' % (
request, self._getUser().getFullName(), self._getUser().getId()))
if not self._tohttps and Config.getInstance().getAuthenticatedEnforceSecure():
self._tohttps = True
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
self._checkCSRF()
self._reqParams = copy.copy(params)
def _process_retry_do(self, profile):
profile_name, res = '', ''
try:
# old code gets parameters from call
# new code utilizes of flask.request
if len(inspect.getargspec(self._checkParams).args) < 2:
cp_result = self._checkParams()
else:
cp_result = self._checkParams(self._reqParams)
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
func = getattr(self, '_checkParams_' + request.method, None)
if func:
cp_result = func()
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFoundError(_('The specified item could not be found.'), title=_('Item not found'))
self._checkProtection()
func = getattr(self, '_checkProtection_' + request.method, None)
if func:
func()
security.Sanitization.sanitizationCheck(self._target,
self._reqParams,
self._aw,
self._doNotSanitizeFields)
if self._doProcess:
if profile:
profile_name = os.path.join(Config.getInstance().getTempDir(), 'stone{}.prof'.format(random.random()))
result = [None]
profiler.runctx('result[0] = self._process()', globals(), locals(), profile_name)
res = result[0]
else:
res = self._process()
return profile_name, res
def _process_retry(self, params, retry, profile, forced_conflicts):
self._process_retry_setup()
self._process_retry_auth_check(params)
DBMgr.getInstance().sync()
return self._process_retry_do(profile)
def _process_success(self):
Logger.get('requestHandler').info('Request {} successful'.format(request))
# request is succesfull, now, doing tasks that must be done only once
try:
flush_after_commit_queue(True)
GenericMailer.flushQueue(True) # send emails
self._deleteTempFiles()
except:
Logger.get('mail').exception('Mail sending operation failed')
# execute redis pipeline if we have one
if self._redisPipeline:
try:
self._redisPipeline.execute()
except RedisError:
Logger.get('redis').exception('Could not execute pipeline')
def process(self, params):
if request.method not in self.HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
cfg = Config.getInstance()
forced_conflicts, max_retries, profile = cfg.getForceConflicts(), cfg.getMaxRetries(), cfg.getProfile()
profile_name, res, textLog = '', '', []
self._startTime = datetime.now()
# clear the context
ContextManager.destroy()
ContextManager.set('currentRH', self)
g.rh = self
#redirect to https if necessary
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
DBMgr.getInstance().startRequest()
textLog.append("%s : Database request started" % (datetime.now() - self._startTime))
Logger.get('requestHandler').info('[pid=%s] Request %s started' % (
os.getpid(), request))
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
signals.before_retry.send()
try:
Logger.get('requestHandler').info('\t[pid=%s] from host %s' % (os.getpid(), request.remote_addr))
profile_name, res = self._process_retry(params, i, profile, forced_conflicts)
signals.after_process.send()
if i < forced_conflicts: # raise conflict error if enabled to easily handle conflict error case
raise ConflictError
transaction.commit()
DBMgr.getInstance().endRequest(commit=False)
break
except (ConflictError, POSKeyError):
transaction.abort()
import traceback
# only log conflict if it wasn't forced
if i >= forced_conflicts:
Logger.get('requestHandler').warning('Conflict in Database! (Request %s)\n%s' % (request, traceback.format_exc()))
except ClientDisconnected:
transaction.abort()
Logger.get('requestHandler').warning('Client Disconnected! (Request {})'.format(request))
time.sleep(i)
self._process_success()
except Exception as e:
transaction.abort()
res = self._getMethodByExceptionName(e)(e)
totalTime = (datetime.now() - self._startTime)
textLog.append('{} : Request ended'.format(totalTime))
# log request timing
if profile and totalTime > timedelta(0, 1) and os.path.isfile(profile_name):
rep = Config.getInstance().getTempDir()
stats = pstats.Stats(profile_name)
stats.strip_dirs()
stats.sort_stats('cumulative', 'time', 'calls')
stats.dump_stats(os.path.join(rep, 'IndicoRequestProfile.log'))
output = StringIO.StringIO()
sys.stdout = output
stats.print_stats(100)
sys.stdout = sys.__stdout__
s = output.getvalue()
f = file(os.path.join(rep, 'IndicoRequest.log'), 'a+')
f.write('--------------------------------\n')
f.write('URL : {}\n'.format(request.url))
f.write('{} : start request\n'.format(self._startTime))
f.write('params:{}'.format(params))
f.write('\n'.join(textLog))
f.write('\n')
f.write('retried : {}\n'.format(10-retry))
f.write(s)
f.write('--------------------------------\n\n')
f.close()
if profile and profile_name and os.path.exists(profile_name):
os.remove(profile_name)
if self._responseUtil.call:
return self._responseUtil.make_call()
# In case of no process needed, we should return empty string to avoid erroneous output
# specially with getVars breaking the JS files.
if not self._doProcess or res is None:
return self._responseUtil.make_empty()
return self._responseUtil.make_response(res)
def _getMethodByExceptionName(self, e):
exception_name = {
'NotFound': 'NotFoundError',
'MaKaCError': 'GeneralError',
'IndicoError': 'GeneralError',
'ValueError': 'UnexpectedError',
'Exception': 'UnexpectedError',
'AccessControlError': 'AccessError'
}.get(type(e).__name__, type(e).__name__)
return getattr(self, '_process{}'.format(exception_name), self._processUnexpectedError)
def _deleteTempFiles(self):
if len(self._tempFilesToDelete) > 0:
for f in self._tempFilesToDelete:
if f is not None:
os.remove(f)
relativeURL = None
def handler(prefix, path):
path = posixpath.join('/', prefix, path)
ContextManager.destroy()
clearCache() # init fossil cache
logger = Logger.get('httpapi')
if request.method == 'POST':
# Convert POST data to a query string
queryParams = [(key, [x.encode('utf-8') for x in values])
for key, values in request.form.iterlists()]
query = urllib.urlencode(queryParams, doseq=1)
# we only need/keep multiple values so we can properly validate the signature.
# the legacy code below expects a dict with just the first value.
# if you write a new api endpoint that needs multiple values get them from
# ``request.values.getlist()`` directly
queryParams = {key: values[0] for key, values in queryParams}
else:
# Parse the actual query string
queryParams = dict((key, value.encode('utf-8'))
for key, value in request.args.iteritems())
query = request.query_string
dbi = DBMgr.getInstance()
dbi.startRequest()
apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None)
cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'],
'no') == 'yes'
signature = get_query_parameter(queryParams, ['signature'])
timestamp = get_query_parameter(queryParams, ['timestamp'],
0,
integer=True)
noCache = get_query_parameter(queryParams, ['nc', 'nocache'],
'no') == 'yes'
pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes'
onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'],
'no') == 'yes'
onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'],
'no') == 'yes'
scope = 'read:legacy_api' if request.method == 'GET' else 'write:legacy_api'
try:
oauth_valid, oauth_request = oauth.verify_request([scope])
if not oauth_valid and oauth_request and oauth_request.error_message != 'Bearer token not found.':
raise BadRequest('OAuth error: {}'.format(
oauth_request.error_message))
elif g.get(
'received_oauth_token'
) and oauth_request.error_message == 'Bearer token not found.':
raise BadRequest('OAuth error: Invalid token')
except ValueError:
# XXX: Dirty hack to workaround a bug in flask-oauthlib that causes it
# not to properly urlencode request query strings
# Related issue (https://github.com/lepture/flask-oauthlib/issues/213)
oauth_valid = False
# Get our handler function and its argument and response type
hook, dformat = HTTPAPIHook.parseRequest(path, queryParams)
if hook is None or dformat is None:
raise NotFound
# Disable caching if we are not just retrieving data (or the hook requires it)
if request.method == 'POST' or hook.NO_CACHE:
noCache = True
ak = error = result = None
ts = int(time.time())
typeMap = {}
responseUtil = ResponseUtil()
is_response = False
try:
used_session = None
if cookieAuth:
used_session = session
if not used_session.user: # ignore guest sessions
used_session = None
if apiKey or oauth_valid or not used_session:
if not oauth_valid:
# Validate the API key (and its signature)
ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp,
path, query)
if enforceOnlyPublic:
onlyPublic = True
# Create an access wrapper for the API key's user
aw = buildAW(ak, onlyPublic)
else: # Access Token (OAuth)
at = load_token(oauth_request.access_token.access_token)
aw = buildAW(at, onlyPublic)
# Get rid of API key in cache key if we did not impersonate a user
if ak and aw.getUser() is None:
cacheKey = normalizeQuery(
path,
query,
remove=('_', 'ak', 'apiKey', 'signature', 'timestamp',
'nc', 'nocache', 'oa', 'onlyauthed'))
else:
cacheKey = normalizeQuery(path,
query,
remove=('_', 'signature',
'timestamp', 'nc', 'nocache',
'oa', 'onlyauthed'))
if signature:
# in case the request was signed, store the result under a different key
cacheKey = 'signed_' + cacheKey
else:
# We authenticated using a session cookie.
if Config.getInstance().getCSRFLevel() >= 2:
token = request.headers.get(
'X-CSRF-Token',
get_query_parameter(queryParams, ['csrftoken']))
if used_session.csrf_protected and used_session.csrf_token != token:
raise HTTPAPIError('Invalid CSRF token', 403)
aw = AccessWrapper()
if not onlyPublic:
aw.setUser(used_session.avatar)
userPrefix = 'user-{}_'.format(used_session.user.id)
cacheKey = userPrefix + normalizeQuery(
path,
query,
remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa',
'onlyauthed', 'csrftoken'))
# Bail out if the user requires authentication but is not authenticated
if onlyAuthed and not aw.getUser():
raise HTTPAPIError('Not authenticated', 403)
addToCache = not hook.NO_CACHE
cache = GenericCache('HTTPAPI')
cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey)
if not noCache:
obj = cache.get(cacheKey)
if obj is not None:
result, extra, ts, complete, typeMap = obj
addToCache = False
if result is None:
ContextManager.set("currentAW", aw)
# Perform the actual exporting
res = hook(aw)
if isinstance(res, current_app.response_class):
addToCache = False
is_response = True
result, extra, complete, typeMap = res, {}, True, {}
elif isinstance(res, tuple) and len(res) == 4:
result, extra, complete, typeMap = res
else:
result, extra, complete, typeMap = res, {}, True, {}
if result is not None and addToCache:
ttl = api_settings.get('cache_ttl')
if ttl > 0:
cache.set(cacheKey, (result, extra, ts, complete, typeMap),
ttl)
except HTTPAPIError, e:
error = e
if e.getCode():
responseUtil.status = e.getCode()
if responseUtil.status == 405:
responseUtil.headers[
'Allow'] = 'GET' if request.method == 'POST' else 'POST'
def handler(prefix, path):
path = posixpath.join('/', prefix, path)
ContextManager.destroy()
clearCache() # init fossil cache
logger = Logger.get('httpapi')
if request.method == 'POST':
# Convert POST data to a query string
queryParams = dict((key, value.encode('utf-8')) for key, value in request.form.iteritems())
query = urllib.urlencode(queryParams)
else:
# Parse the actual query string
queryParams = dict((key, value.encode('utf-8')) for key, value in request.args.iteritems())
query = request.query_string
dbi = DBMgr.getInstance()
dbi.startRequest()
minfo = HelperMaKaCInfo.getMaKaCInfoInstance()
if minfo.getRoomBookingModuleActive():
Factory.getDALManager().connect()
apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None)
cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'], 'no') == 'yes'
signature = get_query_parameter(queryParams, ['signature'])
timestamp = get_query_parameter(queryParams, ['timestamp'], 0, integer=True)
noCache = get_query_parameter(queryParams, ['nc', 'nocache'], 'no') == 'yes'
pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes'
onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'], 'no') == 'yes'
onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'], 'no') == 'yes'
oauthToken = 'oauth_token' in queryParams
# Get our handler function and its argument and response type
hook, dformat = HTTPAPIHook.parseRequest(path, queryParams)
if hook is None or dformat is None:
raise NotFound
# Disable caching if we are not just retrieving data (or the hook requires it)
if request.method == 'POST' or hook.NO_CACHE:
noCache = True
ak = error = result = None
ts = int(time.time())
typeMap = {}
responseUtil = ResponseUtil()
try:
used_session = None
if cookieAuth:
used_session = session
if not used_session.user: # ignore guest sessions
used_session = None
if apiKey or oauthToken or not used_session:
if not oauthToken:
# Validate the API key (and its signature)
ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp, path, query)
if enforceOnlyPublic:
onlyPublic = True
# Create an access wrapper for the API key's user
aw = buildAW(ak, onlyPublic)
else: # Access Token (OAuth)
at = OAuthUtils.OAuthCheckAccessResource()
aw = buildAW(at, onlyPublic)
# Get rid of API key in cache key if we did not impersonate a user
if ak and aw.getUser() is None:
cacheKey = normalizeQuery(path, query,
remove=('_', 'ak', 'apiKey', 'signature', 'timestamp', 'nc', 'nocache',
'oa', 'onlyauthed'))
else:
cacheKey = normalizeQuery(path, query,
remove=('_', 'signature', 'timestamp', 'nc', 'nocache', 'oa', 'onlyauthed'))
if signature:
# in case the request was signed, store the result under a different key
cacheKey = 'signed_' + cacheKey
else:
# We authenticated using a session cookie.
if Config.getInstance().getCSRFLevel() >= 2:
token = request.headers.get('X-CSRF-Token', get_query_parameter(queryParams, ['csrftoken']))
if used_session.csrf_protected and used_session.csrf_token != token:
raise HTTPAPIError('Invalid CSRF token', 403)
aw = AccessWrapper()
if not onlyPublic:
aw.setUser(used_session.user)
userPrefix = 'user-' + used_session.user.getId() + '_'
cacheKey = userPrefix + normalizeQuery(path, query,
remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa', 'onlyauthed',
'csrftoken'))
# Bail out if the user requires authentication but is not authenticated
if onlyAuthed and not aw.getUser():
raise HTTPAPIError('Not authenticated', 403)
addToCache = not hook.NO_CACHE
cache = GenericCache('HTTPAPI')
cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey)
if not noCache:
obj = cache.get(cacheKey)
if obj is not None:
result, extra, ts, complete, typeMap = obj
addToCache = False
if result is None:
# Perform the actual exporting
res = hook(aw)
if isinstance(res, tuple) and len(res) == 4:
result, extra, complete, typeMap = res
else:
result, extra, complete, typeMap = res, {}, True, {}
if result is not None and addToCache:
ttl = HelperMaKaCInfo.getMaKaCInfoInstance().getAPICacheTTL()
cache.set(cacheKey, (result, extra, ts, complete, typeMap), ttl)
except HTTPAPIError, e:
error = e
if e.getCode():
responseUtil.status = e.getCode()
if responseUtil.status == 405:
responseUtil.headers['Allow'] = 'GET' if request.method == 'POST' else 'POST'
class RH(RequestHandlerBase):
"""This class is the base for request handlers of the application. A request
handler will be instantiated when a web request arrives to mod_python;
the mp layer will forward the request to the corresponding request
handler which will know which action has to be performed (displaying a
web page or performing some operation and redirecting to another page).
Request handlers will be responsible for parsing the parameters coming
from a mod_python request, handle the errors which occurred during the
action to perform, managing the sessions, checking security for each
operation (thus they implement the access control system of the web
interface).
It is important to encapsulate all this here as in case of changing
the web application framework we'll just need to adapt this layer (the
rest of the system wouldn't need any change).
Attributes:
_uh - (URLHandler) Associated URLHandler which points to the
current rh.
_req - UNUSED/OBSOLETE, always None
_requestStarted - (bool) Flag which tells whether a DB transaction
has been started or not.
_aw - (AccessWrapper) Current access information for the rh.
_target - (Locable) Reference to an object which is the destination
of the operations needed to carry out the rh. If set it must
provide (through the standard Locable interface) the methods
to get the url parameters in order to reproduce the access to
the rh.
_reqParams - (dict) Dictionary containing the received HTTP
parameters (independently of the method) transformed into
python data types. The key is the parameter name while the
value should be the received paramter value (or values).
"""
_tohttps = False # set this value to True for the RH that must be HTTPS when there is a BaseSecureURL
_doNotSanitizeFields = []
_isMobile = True # this value means that the generated web page can be mobile
HTTP_VERBS = frozenset(('GET', 'POST', 'PUT', 'DELETE'))
def __init__(self):
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper(
) # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True # Flag which indicates whether the RH process
# must be carried out; this is useful for
# the checkProtection methods when they
# detect that an immediate redirection is
# needed
# Methods =============================================================
def getTarget(self):
return self._target
def isMobile(self):
return self._isMobile
def _setSessionUser(self):
self._aw.setUser(session.avatar)
@property
def csrf_token(self):
return session.csrf_token
def _getRequestParams(self):
return self._reqParams
def getRequestParams(self):
return self._getRequestParams()
def _disableCaching(self):
"""Disables caching"""
# IE doesn't seem to like 'no-cache' Cache-Control headers...
if request.user_agent.browser == 'msie':
# actually, the only way to safely disable caching seems to be this one
self._responseUtil.headers["Cache-Control"] = "private"
self._responseUtil.headers["Expires"] = "-1"
else:
self._responseUtil.headers[
"Cache-Control"] = "no-store, no-cache, must-revalidate"
self._responseUtil.headers["Pragma"] = "no-cache"
def _redirect(self, targetURL, status=303):
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def _changeRH(self, rh, params):
"""Calls the specified RH after processing this one"""
self._responseUtil.call = lambda: rh().process(params)
def _checkHttpsRedirect(self):
"""If HTTPS must be used but it is not, redirect!"""
if self.use_https() and not request.is_secure:
self._redirect(self.getRequestURL(secure=True))
return True
else:
return False
def _normaliseListParam(self, param):
if not isinstance(param, list):
return [param]
return param
def _processError(self, e):
raise
def _checkParams(self, params):
"""This method is called before _checkProtection and is a good place
to assign variables from request params to member variables.
Note that in any new code the params argument SHOULD be IGNORED.
Use the following objects provided by Flask instead:
from flask import request
request.view_args (URL route params)
request.args (GET params (from the query string))
request.form (POST params)
request.values (GET+POST params - use only if ABSOLUTELY NECESSARY)
If you only want to run some code for GET or POST requests, you can create
a method named e.g. _checkParams_POST which will be executed AFTER this one.
The method is called without any arguments (except self).
"""
pass
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [
m for m in self.HTTP_VERBS if hasattr(self, '_process_' + m)
]
raise MethodNotAllowed(valid_methods)
return method()
def _checkCSRF(self):
# Check referer for POST requests. We do it here so we can properly use indico's error handling
if Config.getInstance().getCSRFLevel() < 3 or request.method != 'POST':
return
referer = request.referrer
# allow empty - otherwise we might lock out paranoid users blocking referers
if not referer:
return
# valid http referer
if referer.startswith(Config.getInstance().getBaseURL()):
return
# valid https referer - if https is enabled
base_secure = Config.getInstance().getBaseSecureURL()
if base_secure and referer.startswith(base_secure):
return
raise BadRefererError(
'This operation is not allowed from an external referer.')
@jsonify_error
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH."""
if Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPGenericError(self).display()
@jsonify_error(status=500, logging_level='exception')
def _processUnexpectedError(self, e):
"""Unexpected errors"""
self._responseUtil.redirect = None
if Config.getInstance().getEmbeddedWebserver() or Config.getInstance(
).getPropagateAllExceptions():
raise
return errors.WPUnexpectedError(self).display()
@jsonify_error
def _processHostnameResolveError(self, e):
"""Unexpected errors"""
return errors.WPHostnameResolveError(self).display()
@jsonify_error(status=403)
def _processForbidden(self, e):
message = _("Access Denied")
if e.description == Forbidden.description:
explanation = _("You are not allowed to access this page.")
else:
explanation = e.description
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error(status=400)
def _processBadRequest(self, e):
message = _("Bad Request")
return WErrorWSGI((message, e.description)).getHTML()
@jsonify_error(status=400)
def _processBadData(self, e):
message = _("Invalid or expired token")
return WErrorWSGI((message, e.message)).getHTML()
@jsonify_error(status=403)
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPAccessError(self).display()
@jsonify_error
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
# We are going to redirect to the page asking for access key
# and so it must be https if there is a BaseSecureURL. And that's
# why we set _tohttps to True.
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPKeyAccessError(self).display()
@jsonify_error
def _processModificationError(self, e):
"""Handles modification errors occured during the process of a RH."""
# Redirect to HTTPS in case the user is logged in
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPModificationError(self).display()
@jsonify_error(status=400)
def _processBadRequestKeyError(self, e):
"""Request lacks a necessary key for processing"""
msg = _('Required argument missing: %s') % e.message
return errors.WPFormValuesError(self, msg).display()
# TODO: check this method to integrate with jsonify error
def _processOAuthError(self, e):
res = json.dumps(e.fossilize())
header = oauth.build_authenticate_header(
realm=Config.getInstance().getBaseSecureURL())
self._responseUtil.headers.extend(header)
self._responseUtil.content_type = 'application/json'
self._responseUtil.status = e.code
return res
@jsonify_error
def _processConferenceClosedError(self, e):
"""Treats access to modification pages for conferences when they are closed."""
return WPConferenceModificationClosed(self, e._conf).display()
@jsonify_error
def _processTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPTimingError(self, e).display()
@jsonify_error
def _processNoReportError(self, e):
"""Process errors without reporting"""
return errors.WPNoReportError(self, e).display()
@jsonify_error(status=404)
def _processNotFoundError(self, e):
if isinstance(e, NotFound):
message = _(
"Page not found") # that's a bit nicer than "404: Not Found"
if e.description == NotFound.description:
explanation = _("The page you are looking for doesn't exist.")
else:
explanation = e.description
else:
message = e.getMessage()
explanation = e.getExplanation()
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error
def _processParentTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPParentTimingError(self, e).display()
@jsonify_error
def _processEntryTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPEntryTimingError(self, e).display()
@jsonify_error
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
return errors.WPFormValuesError(self, e).display()
@jsonify_error
def _processLaTeXError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPLaTeXError(self, e).display()
@jsonify_error
def _processRestrictedHTML(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
@jsonify_error
def _processHtmlScriptError(self, e):
""" TODO """
return errors.WPHtmlScriptError(self, escape(str(e))).display()
@jsonify_error
def _processHtmlForbiddenTag(self, e):
""" TODO """
return errors.WPRestrictedHTML(self, escape(str(e))).display()
def _process_retry_setup(self):
# clear the fossile cache at the start of each request
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
# clear the existing redis pipeline
if self._redisPipeline:
self._redisPipeline.reset()
def _process_retry_auth_check(self, params):
# keep a link to the web session in the access wrapper
# this is used for checking access/modification key existence
# in the user session
self._aw.setIP(request.remote_addr)
self._setSessionUser()
if self._getAuth():
if self._getUser():
Logger.get('requestHandler').info(
'Request %s identified with user %s (%s)' %
(request, self._getUser().getFullName(),
self._getUser().getId()))
if not self._tohttps and Config.getInstance(
).getAuthenticatedEnforceSecure():
self._tohttps = True
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
self._checkCSRF()
self._reqParams = copy.copy(params)
def _process_retry_do(self, profile):
profile_name, res = '', ''
try:
# old code gets parameters from call
# new code utilizes of flask.request
if len(inspect.getargspec(self._checkParams).args) < 2:
cp_result = self._checkParams()
else:
cp_result = self._checkParams(self._reqParams)
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
func = getattr(self, '_checkParams_' + request.method, None)
if func:
cp_result = func()
if isinstance(cp_result,
(current_app.response_class, Response)):
return '', cp_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFoundError(_('The specified item could not be found.'),
title=_('Item not found'))
self._checkProtection()
func = getattr(self, '_checkProtection_' + request.method, None)
if func:
func()
security.Sanitization.sanitizationCheck(self._target, self._reqParams,
self._aw,
self._doNotSanitizeFields)
if self._doProcess:
if profile:
profile_name = os.path.join(
Config.getInstance().getTempDir(),
'stone{}.prof'.format(random.random()))
result = [None]
profiler.runctx('result[0] = self._process()', globals(),
locals(), profile_name)
res = result[0]
else:
res = self._process()
return profile_name, res
def _process_retry(self, params, retry, profile, forced_conflicts):
self._process_retry_setup()
self._process_retry_auth_check(params)
DBMgr.getInstance().sync()
return self._process_retry_do(profile)
def _process_success(self):
Logger.get('requestHandler').info(
'Request {} successful'.format(request))
# request is succesfull, now, doing tasks that must be done only once
try:
flush_after_commit_queue(True)
GenericMailer.flushQueue(True) # send emails
self._deleteTempFiles()
except:
Logger.get('mail').exception('Mail sending operation failed')
# execute redis pipeline if we have one
if self._redisPipeline:
try:
self._redisPipeline.execute()
except RedisError:
Logger.get('redis').exception('Could not execute pipeline')
def process(self, params):
if request.method not in self.HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
cfg = Config.getInstance()
forced_conflicts, max_retries, profile = cfg.getForceConflicts(
), cfg.getMaxRetries(), cfg.getProfile()
profile_name, res, textLog = '', '', []
self._startTime = datetime.now()
# clear the context
ContextManager.destroy()
ContextManager.set('currentRH', self)
g.rh = self
#redirect to https if necessary
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
DBMgr.getInstance().startRequest()
textLog.append("%s : Database request started" %
(datetime.now() - self._startTime))
Logger.get('requestHandler').info('[pid=%s] Request %s started' %
(os.getpid(), request))
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
signals.before_retry.send()
try:
Logger.get('requestHandler').info(
'\t[pid=%s] from host %s' %
(os.getpid(), request.remote_addr))
profile_name, res = self._process_retry(
params, i, profile, forced_conflicts)
signals.after_process.send()
if i < forced_conflicts: # raise conflict error if enabled to easily handle conflict error case
raise ConflictError
transaction.commit()
DBMgr.getInstance().endRequest(commit=False)
break
except (ConflictError, POSKeyError):
transaction.abort()
import traceback
# only log conflict if it wasn't forced
if i >= forced_conflicts:
Logger.get('requestHandler').warning(
'Conflict in Database! (Request %s)\n%s' %
(request, traceback.format_exc()))
except ClientDisconnected:
transaction.abort()
Logger.get('requestHandler').warning(
'Client Disconnected! (Request {})'.format(
request))
time.sleep(i)
self._process_success()
except Exception as e:
transaction.abort()
res = self._getMethodByExceptionName(e)(e)
totalTime = (datetime.now() - self._startTime)
textLog.append('{} : Request ended'.format(totalTime))
# log request timing
if profile and totalTime > timedelta(
0, 1) and os.path.isfile(profile_name):
rep = Config.getInstance().getTempDir()
stats = pstats.Stats(profile_name)
stats.strip_dirs()
stats.sort_stats('cumulative', 'time', 'calls')
stats.dump_stats(os.path.join(rep, 'IndicoRequestProfile.log'))
output = StringIO.StringIO()
sys.stdout = output
stats.print_stats(100)
sys.stdout = sys.__stdout__
s = output.getvalue()
f = file(os.path.join(rep, 'IndicoRequest.log'), 'a+')
f.write('--------------------------------\n')
f.write('URL : {}\n'.format(request.url))
f.write('{} : start request\n'.format(self._startTime))
f.write('params:{}'.format(params))
f.write('\n'.join(textLog))
f.write('\n')
f.write('retried : {}\n'.format(10 - retry))
f.write(s)
f.write('--------------------------------\n\n')
f.close()
if profile and profile_name and os.path.exists(profile_name):
os.remove(profile_name)
if self._responseUtil.call:
return self._responseUtil.make_call()
# In case of no process needed, we should return empty string to avoid erroneous output
# specially with getVars breaking the JS files.
if not self._doProcess or res is None:
return self._responseUtil.make_empty()
return self._responseUtil.make_response(res)
def _getMethodByExceptionName(self, e):
exception_name = {
'NotFound': 'NotFoundError',
'MaKaCError': 'GeneralError',
'IndicoError': 'GeneralError',
'ValueError': 'UnexpectedError',
'Exception': 'UnexpectedError',
'AccessControlError': 'AccessError'
}.get(type(e).__name__,
type(e).__name__)
if isinstance(e, BadData): # we also want its subclasses
exception_name = 'BadData'
return getattr(self, '_process{}'.format(exception_name),
self._processUnexpectedError)
def _deleteTempFiles(self):
if len(self._tempFilesToDelete) > 0:
for f in self._tempFilesToDelete:
if f is not None:
os.remove(f)
relativeURL = None
class RH(RequestHandlerBase):
"""This class is the base for request handlers of the application. A request
handler will be instantiated when a web request arrives to mod_python;
the mp layer will forward the request to the corresponding request
handler which will know which action has to be performed (displaying a
web page or performing some operation and redirecting to another page).
Request handlers will be responsible for parsing the parameters coming
from a mod_python request, handle the errors which occurred during the
action to perform, managing the sessions, checking security for each
operation (thus they implement the access control system of the web
interface).
It is important to encapsulate all this here as in case of changing
the web application framework we'll just need to adapt this layer (the
rest of the system wouldn't need any change).
Attributes:
_uh - (URLHandler) Associated URLHandler which points to the
current rh.
_req - UNUSED/OBSOLETE, always None
_requestStarted - (bool) Flag which tells whether a DB transaction
has been started or not.
_aw - (AccessWrapper) Current access information for the rh.
_target - (Locable) Reference to an object which is the destination
of the operations needed to carry out the rh. If set it must
provide (through the standard Locable interface) the methods
to get the url parameters in order to reproduce the access to
the rh.
_reqParams - (dict) Dictionary containing the received HTTP
parameters (independently of the method) transformed into
python data types. The key is the parameter name while the
value should be the received paramter value (or values).
"""
_tohttps = False # set this value to True for the RH that must be HTTPS when there is a BaseSecureURL
_doNotSanitizeFields = []
_isMobile = True # this value means that the generated web page can be mobile
CSRF_ENABLED = False # require a csrf_token when accessing the RH with anything but GET
EVENT_FEATURE = None # require a certain event feature when accessing the RH. See `EventFeature` for details
#: A dict specifying how the url should be normalized.
#: `args` is a dictionary mapping view args keys to callables
#: used to retrieve the expected value for those arguments if they
#: are present in the request's view args.
#: `locators` is a set of callables returning objects with locators.
#: `preserved_args` is a set of view arg names which will always
#: be copied from the current request if present.
#: The callables are always invoked with a single `self` argument
#: containing the RH instance.
#: Arguments specified in the `defaults` of any rule matching the
#: current endpoint are always excluded when checking if the args
#: match or when building a new URL.
#: If the view args built from the returned objects do not match
#: the request's view args, a redirect is issued automatically.
#: If the request is not using GET/HEAD, a 404 error is raised
#: instead of a redirect since such requests cannot be redirected
#: but executing them on the wrong URL may pose a security risk in
#: case and of the non-relevant URL segments is used for access
#: checks.
normalize_url_spec = {
'args': {},
'locators': set(),
'preserved_args': set()
}
def __init__(self):
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True # Flag which indicates whether the RH process
# must be carried out; this is useful for
# the checkProtection methods when they
# detect that an immediate redirection is
# needed
# Methods =============================================================
def getTarget(self):
return self._target
def isMobile(self):
return self._isMobile
def _setSessionUser(self):
self._aw.setUser(session.avatar)
@property
def csrf_token(self):
return session.csrf_token if session.csrf_protected else ''
def _getRequestParams(self):
return self._reqParams
def getRequestParams(self):
return self._getRequestParams()
def _disableCaching(self):
"""Disables caching"""
# IE doesn't seem to like 'no-cache' Cache-Control headers...
if request.user_agent.browser == 'msie':
# actually, the only way to safely disable caching seems to be this one
self._responseUtil.headers["Cache-Control"] = "private"
self._responseUtil.headers["Expires"] = "-1"
else:
self._responseUtil.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
self._responseUtil.headers["Pragma"] = "no-cache"
def _redirect(self, targetURL, status=303):
if isinstance(targetURL, Response):
status = targetURL.status_code
targetURL = targetURL.headers['Location']
else:
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def _changeRH(self, rh, params):
"""Calls the specified RH after processing this one"""
self._responseUtil.call = lambda: rh().process(params)
def _checkHttpsRedirect(self):
"""If HTTPS must be used but it is not, redirect!"""
if self.use_https() and not request.is_secure:
self._redirect(self.getRequestURL(secure=True))
return True
else:
return False
def _normaliseListParam(self, param):
if not isinstance(param, list):
return [param]
return param
def _processError(self, e):
raise
def normalize_url(self):
"""Performs URL normalization.
This uses the :attr:`normalize_url_spec` to check if the URL
params are what they should be and redirects or fails depending
on the HTTP method used if it's not the case.
:return: ``None`` or a redirect response
"""
if not self.normalize_url_spec or not any(self.normalize_url_spec.itervalues()):
return
spec = {
'args': self.normalize_url_spec.get('args', {}),
'locators': self.normalize_url_spec.get('locators', set()),
'preserved_args': self.normalize_url_spec.get('preserved_args', set()),
}
# Initialize the new view args with preserved arguments (since those would be lost otherwise)
new_view_args = {k: v for k, v in request.view_args.iteritems() if k in spec['preserved_args']}
# Retrieve the expected values for all simple arguments (if they are currently present)
for key, getter in spec['args'].iteritems():
if key in request.view_args:
new_view_args[key] = getter(self)
# Retrieve the expected values from locators
for getter in spec['locators']:
value = getter(self)
if value is None:
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
try:
expected = value.locator
except AttributeError:
try:
expected = value.getLocator()
except AttributeError:
raise AttributeError("'{}' object has neither 'locator' nor 'getLocator'".format(type(value)))
new_view_args.update(expected)
# Get all default values provided by the url map for the endpoint
defaults = set(itertools.chain.from_iterable(r.defaults
for r in current_app.url_map.iter_rules(request.endpoint)
if r.defaults))
def _convert(v):
# some legacy code has numeric ids in the locator data, but still takes
# string ids in the url rule (usually for confId)
return unicode(v) if isinstance(v, (int, long)) else v
provided = {k: _convert(v) for k, v in request.view_args.iteritems() if k not in defaults}
new_view_args = {k: _convert(v) for k, v in new_view_args.iteritems()}
if new_view_args != provided:
if request.method in {'GET', 'HEAD'}:
return redirect(url_for(request.endpoint, **dict(request.args.to_dict(), **new_view_args)))
else:
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
def _checkParams(self, params):
"""This method is called before _checkProtection and is a good place
to assign variables from request params to member variables.
Note that in any new code the params argument SHOULD be IGNORED.
Use the following objects provided by Flask instead:
from flask import request
request.view_args (URL route params)
request.args (GET params (from the query string))
request.form (POST params)
request.values (GET+POST params - use only if ABSOLUTELY NECESSARY)
If you only want to run some code for GET or POST requests, you can create
a method named e.g. _checkParams_POST which will be executed AFTER this one.
The method is called without any arguments (except self).
"""
pass
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [m for m in HTTP_VERBS if hasattr(self, '_process_' + m)]
raise MethodNotAllowed(valid_methods)
return method()
def _checkCSRF(self):
token = request.headers.get('X-CSRF-Token', request.form.get('csrf_token'))
if self.CSRF_ENABLED and request.method != 'GET' and token != session.csrf_token:
msg = _(u"It looks like there was a problem with your current session. Please use your browser's back "
u"button, reload the page and try again.")
raise BadRequest(msg)
elif not self.CSRF_ENABLED and current_app.debug and request.method != 'GET':
# Warn if CSRF is not enabled for a RH in new code
module = self.__class__.__module__
if module.startswith('indico.modules.') or module.startswith('indico.core.'):
msg = (u'{} request sent to {} which has no CSRF checks. Set `CSRF_ENABLED = True` in the class to '
u'enable them.').format(request.method, self.__class__.__name__)
warnings.warn(msg, RuntimeWarning)
# legacy csrf check (referer-based):
# Check referer for POST requests. We do it here so we can properly use indico's error handling
if Config.getInstance().getCSRFLevel() < 3 or request.method != 'POST':
return
referer = request.referrer
# allow empty - otherwise we might lock out paranoid users blocking referers
if not referer:
return
# valid http referer
if referer.startswith(Config.getInstance().getBaseURL()):
return
# valid https referer - if https is enabled
base_secure = Config.getInstance().getBaseSecureURL()
if base_secure and referer.startswith(base_secure):
return
raise BadRefererError('This operation is not allowed from an external referer.')
def _check_event_feature(self):
from indico.modules.events.features.util import require_feature
event_id = request.view_args.get('confId') or request.view_args.get('event_id')
if event_id is not None:
require_feature(event_id, self.EVENT_FEATURE)
@jsonify_error
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH."""
if Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPGenericError(self).display()
@jsonify_error(status=500, logging_level='exception')
def _processUnexpectedError(self, e):
"""Unexpected errors"""
self._responseUtil.redirect = None
if Config.getInstance().getEmbeddedWebserver() or Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPUnexpectedError(self).display()
@jsonify_error(status=403)
def _processForbidden(self, e):
if session.user is None and not request.is_xhr and not e.response:
return redirect_to_login(reason=_("Please log in to access this page."))
message = _("Access Denied")
explanation = get_error_description(e)
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error(status=400)
def _processBadRequest(self, e):
message = _("Bad Request")
return WErrorWSGI((message, e.description)).getHTML()
@jsonify_error(status=400)
def _processBadData(self, e):
message = _("Invalid or expired token")
return WErrorWSGI((message, e.message)).getHTML()
@jsonify_error(status=403)
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPAccessError(self).display()
@jsonify_error
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
# We are going to redirect to the page asking for access key
# and so it must be https if there is a BaseSecureURL. And that's
# why we set _tohttps to True.
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPKeyAccessError(self).display()
@jsonify_error
def _processModificationError(self, e):
"""Handles modification errors occured during the process of a RH."""
# Redirect to HTTPS in case the user is logged in
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPModificationError(self).display()
@jsonify_error(status=400)
def _processBadRequestKeyError(self, e):
"""Request lacks a necessary key for processing"""
msg = _('Required argument missing: %s') % e.message
return errors.WPFormValuesError(self, msg).display()
@jsonify_error
def _processConferenceClosedError(self, e):
"""Treats access to modification pages for conferences when they are closed."""
return WPConferenceModificationClosed(self, e._conf).display()
@jsonify_error
def _processTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPTimingError(self, e).display()
@jsonify_error
def _processNoReportError(self, e):
"""Process errors without reporting"""
return errors.WPNoReportError(self, e).display()
@jsonify_error(status=404)
def _processNotFoundError(self, e):
if isinstance(e, NotFound):
message = _("Page not found") # that's a bit nicer than "404: Not Found"
explanation = get_error_description(e)
else:
message = e.getMessage()
explanation = e.getExplanation()
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error
def _processParentTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPParentTimingError(self, e).display()
@jsonify_error
def _processEntryTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPEntryTimingError(self, e).display()
@jsonify_error
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
return errors.WPFormValuesError(self, e).display()
@jsonify_error
def _processLaTeXError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPLaTeXError(self, e).display()
@jsonify_error
def _processRestrictedHTML(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
@jsonify_error
def _processHtmlForbiddenTag(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
def _process_retry_setup(self):
# clear the fossile cache at the start of each request
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
# clear the existing redis pipeline
if self._redisPipeline:
self._redisPipeline.reset()
def _process_retry_auth_check(self, params):
# keep a link to the web session in the access wrapper
# this is used for checking access/modification key existence
# in the user session
self._setSessionUser()
if self._getAuth():
if self._getUser():
Logger.get('requestHandler').info('Request %s identified with user %s (%s)' % (
request, self._getUser().getFullName(), self._getUser().getId()))
if not self._tohttps and Config.getInstance().getAuthenticatedEnforceSecure():
self._tohttps = True
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
self._checkCSRF()
self._reqParams = copy.copy(params)
def _process_retry_do(self, profile):
profile_name, res = '', ''
try:
# old code gets parameters from call
# new code utilizes of flask.request
if len(inspect.getargspec(self._checkParams).args) < 2:
cp_result = self._checkParams()
else:
cp_result = self._checkParams(self._reqParams)
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
func = getattr(self, '_checkParams_' + request.method, None)
if func:
cp_result = func()
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFoundError(_('The specified item could not be found.'), title=_('Item not found'))
rv = self.normalize_url()
if rv is not None:
return '', rv
self._checkProtection()
func = getattr(self, '_checkProtection_' + request.method, None)
if func:
func()
security.Sanitization.sanitizationCheck(self._target,
self._reqParams,
self._aw,
self._doNotSanitizeFields)
if self._doProcess:
if profile:
profile_name = os.path.join(Config.getInstance().getTempDir(), 'stone{}.prof'.format(random.random()))
result = [None]
profiler.runctx('result[0] = self._process()', globals(), locals(), profile_name)
res = result[0]
else:
res = self._process()
return profile_name, res
def _process_retry(self, params, retry, profile, forced_conflicts):
self._process_retry_setup()
self._process_retry_auth_check(params)
DBMgr.getInstance().sync()
return self._process_retry_do(profile)
def _process_success(self):
Logger.get('requestHandler').info('Request {} successful'.format(request))
# request is succesfull, now, doing tasks that must be done only once
try:
flush_after_commit_queue(True)
GenericMailer.flushQueue(True) # send emails
self._deleteTempFiles()
except:
Logger.get('mail').exception('Mail sending operation failed')
# execute redis pipeline if we have one
if self._redisPipeline:
try:
self._redisPipeline.execute()
except RedisError:
Logger.get('redis').exception('Could not execute pipeline')
def process(self, params):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
cfg = Config.getInstance()
forced_conflicts, max_retries, profile = cfg.getForceConflicts(), cfg.getMaxRetries(), cfg.getProfile()
profile_name, res, textLog = '', '', []
self._startTime = datetime.now()
# clear the context
ContextManager.destroy()
ContextManager.set('currentRH', self)
g.rh = self
#redirect to https if necessary
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
if self.EVENT_FEATURE is not None:
self._check_event_feature()
DBMgr.getInstance().startRequest()
textLog.append("%s : Database request started" % (datetime.now() - self._startTime))
Logger.get('requestHandler').info('[pid=%s] Request %s started' % (
os.getpid(), request))
is_error_response = False
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
signals.before_retry.send()
try:
Logger.get('requestHandler').info('\t[pid=%s] from host %s' % (os.getpid(), request.remote_addr))
profile_name, res = self._process_retry(params, i, profile, forced_conflicts)
signals.after_process.send()
if i < forced_conflicts: # raise conflict error if enabled to easily handle conflict error case
raise ConflictError
transaction.commit()
DBMgr.getInstance().endRequest(commit=False)
break
except (ConflictError, POSKeyError):
transaction.abort()
import traceback
# only log conflict if it wasn't forced
if i >= forced_conflicts:
Logger.get('requestHandler').warning('Conflict in Database! (Request %s)\n%s' % (request, traceback.format_exc()))
except ClientDisconnected:
transaction.abort()
Logger.get('requestHandler').warning('Client Disconnected! (Request {})'.format(request))
time.sleep(i)
self._process_success()
except Exception as e:
transaction.abort()
res = self._getMethodByExceptionName(e)(e)
if isinstance(e, HTTPException) and e.response is not None:
res = e.response
is_error_response = True
totalTime = (datetime.now() - self._startTime)
textLog.append('{} : Request ended'.format(totalTime))
# log request timing
if profile and totalTime > timedelta(0, 1) and os.path.isfile(profile_name):
rep = Config.getInstance().getTempDir()
stats = pstats.Stats(profile_name)
stats.strip_dirs()
stats.sort_stats('cumulative', 'time', 'calls')
stats.dump_stats(os.path.join(rep, 'IndicoRequestProfile.log'))
output = StringIO.StringIO()
sys.stdout = output
stats.print_stats(100)
sys.stdout = sys.__stdout__
s = output.getvalue()
f = file(os.path.join(rep, 'IndicoRequest.log'), 'a+')
f.write('--------------------------------\n')
f.write('URL : {}\n'.format(request.url))
f.write('{} : start request\n'.format(self._startTime))
f.write('params:{}'.format(params))
f.write('\n'.join(textLog))
f.write('\n')
f.write('retried : {}\n'.format(10-retry))
f.write(s)
f.write('--------------------------------\n\n')
f.close()
if profile and profile_name and os.path.exists(profile_name):
os.remove(profile_name)
if self._responseUtil.call:
return self._responseUtil.make_call()
if is_error_response and isinstance(res, (current_app.response_class, Response)):
# if we went through error handling code, responseUtil._status has been changed
# so make_response() would fail
return res
# In case of no process needed, we should return empty string to avoid erroneous output
# specially with getVars breaking the JS files.
if not self._doProcess or res is None:
return self._responseUtil.make_empty()
return self._responseUtil.make_response(res)
def _getMethodByExceptionName(self, e):
exception_name = {
'NotFound': 'NotFoundError',
'MaKaCError': 'GeneralError',
'IndicoError': 'GeneralError',
'ValueError': 'UnexpectedError',
'Exception': 'UnexpectedError',
'AccessControlError': 'AccessError'
}.get(type(e).__name__, type(e).__name__)
if isinstance(e, BadData): # we also want its subclasses
exception_name = 'BadData'
return getattr(self, '_process{}'.format(exception_name), self._processUnexpectedError)
def _deleteTempFiles(self):
if len(self._tempFilesToDelete) > 0:
for f in self._tempFilesToDelete:
if f is not None:
os.remove(f)
relativeURL = None
def handler(prefix, path):
path = posixpath.join('/', prefix, path)
ContextManager.destroy()
clearCache() # init fossil cache
logger = Logger.get('httpapi')
if request.method == 'POST':
# Convert POST data to a query string
queryParams = dict((key, value.encode('utf-8'))
for key, value in request.form.iteritems())
query = urllib.urlencode(queryParams)
else:
# Parse the actual query string
queryParams = dict((key, value.encode('utf-8'))
for key, value in request.args.iteritems())
query = request.query_string
dbi = DBMgr.getInstance()
dbi.startRequest()
apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None)
cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'],
'no') == 'yes'
signature = get_query_parameter(queryParams, ['signature'])
timestamp = get_query_parameter(queryParams, ['timestamp'],
0,
integer=True)
noCache = get_query_parameter(queryParams, ['nc', 'nocache'],
'no') == 'yes'
pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes'
onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'],
'no') == 'yes'
onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'],
'no') == 'yes'
oauthToken = 'oauth_token' in queryParams
# Check if OAuth data is supplied in the Authorization header
if not oauthToken and request.headers.get('Authorization') is not None:
oauthToken = 'oauth_token' in request.headers.get('Authorization')
# Get our handler function and its argument and response type
hook, dformat = HTTPAPIHook.parseRequest(path, queryParams)
if hook is None or dformat is None:
raise NotFound
# Disable caching if we are not just retrieving data (or the hook requires it)
if request.method == 'POST' or hook.NO_CACHE:
noCache = True
ak = error = result = None
ts = int(time.time())
typeMap = {}
responseUtil = ResponseUtil()
try:
used_session = None
if cookieAuth:
used_session = session
if not used_session.avatar: # ignore guest sessions
used_session = None
if apiKey or oauthToken or not used_session:
if not oauthToken:
# Validate the API key (and its signature)
ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp,
path, query)
if enforceOnlyPublic:
onlyPublic = True
# Create an access wrapper for the API key's user
aw = buildAW(ak, onlyPublic)
else: # Access Token (OAuth)
at = OAuthUtils.OAuthCheckAccessResource()
aw = buildAW(at, onlyPublic)
# Get rid of API key in cache key if we did not impersonate a user
if ak and aw.getUser() is None:
cacheKey = normalizeQuery(
path,
query,
remove=('_', 'ak', 'apiKey', 'signature', 'timestamp',
'nc', 'nocache', 'oa', 'onlyauthed'))
else:
cacheKey = normalizeQuery(path,
query,
remove=('_', 'signature',
'timestamp', 'nc', 'nocache',
'oa', 'onlyauthed'))
if signature:
# in case the request was signed, store the result under a different key
cacheKey = 'signed_' + cacheKey
else:
# We authenticated using a session cookie.
if Config.getInstance().getCSRFLevel() >= 2:
token = request.headers.get(
'X-CSRF-Token',
get_query_parameter(queryParams, ['csrftoken']))
if used_session.csrf_protected and used_session.csrf_token != token:
raise HTTPAPIError('Invalid CSRF token', 403)
aw = AccessWrapper()
if not onlyPublic:
aw.setUser(used_session.avatar)
userPrefix = 'user-' + used_session.avatar.getId() + '_'
cacheKey = userPrefix + normalizeQuery(
path,
query,
remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa',
'onlyauthed', 'csrftoken'))
# Bail out if the user requires authentication but is not authenticated
if onlyAuthed and not aw.getUser():
raise HTTPAPIError('Not authenticated', 403)
addToCache = not hook.NO_CACHE
cache = GenericCache('HTTPAPI')
cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey)
if not noCache:
obj = cache.get(cacheKey)
if obj is not None:
result, extra, ts, complete, typeMap = obj
addToCache = False
if result is None:
ContextManager.set("currentAW", aw)
# Perform the actual exporting
res = hook(aw)
if isinstance(res, tuple) and len(res) == 4:
result, extra, complete, typeMap = res
else:
result, extra, complete, typeMap = res, {}, True, {}
if result is not None and addToCache:
ttl = api_settings.get('cache_ttl')
cache.set(cacheKey, (result, extra, ts, complete, typeMap), ttl)
except HTTPAPIError, e:
error = e
if e.getCode():
responseUtil.status = e.getCode()
if responseUtil.status == 405:
responseUtil.headers[
'Allow'] = 'GET' if request.method == 'POST' else 'POST'
class RH(object):
NOT_SANITIZED_FIELDS = frozenset()
CSRF_ENABLED = True # require a csrf_token when accessing the RH with anything but GET
EVENT_FEATURE = None # require a certain event feature when accessing the RH. See `EventFeature` for details
DENY_FRAMES = False # whether to send an X-Frame-Options:DENY header
CHECK_HTML = False # whether to run the legacy HTML sanitizer
#: A dict specifying how the url should be normalized.
#: `args` is a dictionary mapping view args keys to callables
#: used to retrieve the expected value for those arguments if they
#: are present in the request's view args.
#: `locators` is a set of callables returning objects with locators.
#: `preserved_args` is a set of view arg names which will always
#: be copied from the current request if present.
#: The callables are always invoked with a single `self` argument
#: containing the RH instance.
#: `endpoint` may be used to specify the endpoint used to build
#: the URL in case of a redirect. Usually this should not be used
#: in favor of ``request.endpoint`` being used if no custom endpoint
#: is set.
#: Arguments specified in the `defaults` of any rule matching the
#: current endpoint are always excluded when checking if the args
#: match or when building a new URL.
#: If the view args built from the returned objects do not match
#: the request's view args, a redirect is issued automatically.
#: If the request is not using GET/HEAD, a 404 error is raised
#: instead of a redirect since such requests cannot be redirected
#: but executing them on the wrong URL may pose a security risk in
#: case and of the non-relevant URL segments is used for access
#: checks.
normalize_url_spec = {
'args': {},
'locators': set(),
'preserved_args': set(),
'endpoint': None
}
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
# Methods =============================================================
def validate_json(self, schema, json=None):
"""Validates the request's JSON payload using a JSON schema.
:param schema: The JSON schema used for validation.
:param json: The JSON object (defaults to ``request.json``)
:raises BadRequest: if the JSON validation failed
"""
if json is None:
json = request.json
try:
jsonschema.validate(json, schema)
except jsonschema.ValidationError as e:
raise BadRequest('Invalid JSON payload: {}'.format(e.message))
@property
def csrf_token(self):
return session.csrf_token if session.csrf_protected else ''
def normalize_url(self):
"""Performs URL normalization.
This uses the :attr:`normalize_url_spec` to check if the URL
params are what they should be and redirects or fails depending
on the HTTP method used if it's not the case.
:return: ``None`` or a redirect response
"""
if current_app.debug and self.normalize_url_spec is RH.normalize_url_spec:
# in case of ``class SomeRH(RH, MixinWithNormalization)``
# the default value from `RH` overwrites the normalization
# rule from ``MixinWithNormalization``. this is never what
# the developer wants so we fail if it happens. the proper
# solution is ``class SomeRH(MixinWithNormalization, RH)``
cls = next((x
for x in inspect.getmro(self.__class__)
if (x is not RH and x is not self.__class__ and hasattr(x, 'normalize_url_spec') and
getattr(x, 'normalize_url_spec', None) is not RH.normalize_url_spec)),
None)
if cls is not None:
raise Exception('Normalization rule of {} in {} is overwritten by base RH. Put mixins with class-level '
'attributes on the left of the base class'.format(cls, self.__class__))
if not self.normalize_url_spec or not any(self.normalize_url_spec.itervalues()):
return
spec = {
'args': self.normalize_url_spec.get('args', {}),
'locators': self.normalize_url_spec.get('locators', set()),
'preserved_args': self.normalize_url_spec.get('preserved_args', set()),
'endpoint': self.normalize_url_spec.get('endpoint', None)
}
# Initialize the new view args with preserved arguments (since those would be lost otherwise)
new_view_args = {k: v for k, v in request.view_args.iteritems() if k in spec['preserved_args']}
# Retrieve the expected values for all simple arguments (if they are currently present)
for key, getter in spec['args'].iteritems():
if key in request.view_args:
new_view_args[key] = getter(self)
# Retrieve the expected values from locators
prev_locator_args = {}
for getter in spec['locators']:
value = getter(self)
if value is None:
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
locator_args = get_locator(value)
reused_keys = set(locator_args) & prev_locator_args.viewkeys()
if any(locator_args[k] != prev_locator_args[k] for k in reused_keys):
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
new_view_args.update(locator_args)
prev_locator_args.update(locator_args)
# Get all default values provided by the url map for the endpoint
defaults = set(itertools.chain.from_iterable(r.defaults
for r in current_app.url_map.iter_rules(request.endpoint)
if r.defaults))
def _convert(v):
# some legacy code has numeric ids in the locator data, but still takes
# string ids in the url rule (usually for confId)
return unicode(v) if isinstance(v, (int, long)) else v
provided = {k: _convert(v) for k, v in request.view_args.iteritems() if k not in defaults}
new_view_args = {k: _convert(v) for k, v in new_view_args.iteritems()}
if new_view_args != provided:
if request.method in {'GET', 'HEAD'}:
endpoint = spec['endpoint'] or request.endpoint
try:
return redirect(url_for(endpoint, **dict(request.args.to_dict(), **new_view_args)))
except BuildError as e:
if current_app.debug:
raise
logger.warn('BuildError during normalization: %s', e)
raise NotFound
else:
raise NotFound('The URL contains invalid data. Please go to the previous page and refresh it.')
def _process_args(self):
"""
This method is called before _check_access and url normalization
and is a good place to fetch objects from the database based on
variables from request params.
"""
def _check_access(self):
"""
This method is called after _process_args and is a good place
to check if the user is permitted to perform some actions.
"""
def _process(self):
"""Dispatch to a method named ``_process_<verb>``.
Except for RESTful endpoints it is usually best to just
override this method, especially when using WTForms.
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [m for m in HTTP_VERBS if hasattr(self, '_process_' + m)]
raise MethodNotAllowed(valid_methods)
return method()
def _check_csrf(self):
token = request.headers.get('X-CSRF-Token') or request.form.get('csrf_token')
if token is None:
# Might be a WTForm with a prefix. In that case the field name is '<prefix>-csrf_token'
token = next((v for k, v in request.form.iteritems() if k.endswith('-csrf_token')), None)
if self.CSRF_ENABLED and request.method != 'GET' and token != session.csrf_token:
msg = _("It looks like there was a problem with your current session. Please use your browser's back "
"button, reload the page and try again.")
raise BadRequest(msg)
def _check_event_feature(self):
from indico.modules.events.features.util import require_feature
event_id = request.view_args.get('confId') or request.view_args.get('event_id')
if event_id is not None:
require_feature(event_id, self.EVENT_FEATURE)
def _do_process(self):
try:
args_result = self._process_args()
if isinstance(args_result, (current_app.response_class, Response)):
return args_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFound(_('The specified item could not be found.'))
rv = self.normalize_url()
if rv is not None:
return rv
self._check_access()
if self.CHECK_HTML:
Sanitization.sanitizationCheck(create_flat_args(), self.NOT_SANITIZED_FIELDS)
if config.PROFILE:
result = [None]
profile_path = os.path.join(config.TEMP_DIR, '{}-{}.prof'.format(type(self).__name__, time.time()))
cProfile.runctx('result[0] = self._process()', globals(), locals(), profile_path)
return result[0]
else:
return self._process()
def process(self):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
res = ''
g.rh = self
sentry_set_tags({'rh': self.__class__.__name__})
if self.EVENT_FEATURE is not None:
self._check_event_feature()
logger.info('%s %s [IP=%s] [PID=%s] [UID=%r]',
request.method, request.relative_url, request.remote_addr, os.getpid(), session.get('_user_id'))
try:
fossilize.clearCache()
GenericMailer.flushQueue(False)
self._check_csrf()
res = self._do_process()
signals.after_process.send()
if self.commit:
if GenericMailer.has_queue():
# ensure we fail early (before sending out e-mails)
# in case there are DB constraint violations, etc...
db.enforce_constraints()
GenericMailer.flushQueue(True)
db.session.commit()
else:
db.session.rollback()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error() # this will re-raise an exception
logger.debug('Request successful')
if res is None:
return self._responseUtil.make_empty()
response = self._responseUtil.make_response(res)
if self.DENY_FRAMES:
response.headers['X-Frame-Options'] = 'DENY'
return response
class RH(RequestHandlerBase):
"""This class is the base for request handlers of the application. A request
handler will be instantiated when a web request arrives to mod_python;
the mp layer will forward the request to the corresponding request
handler which will know which action has to be performed (displaying a
web page or performing some operation and redirecting to another page).
Request handlers will be responsible for parsing the parameters coming
from a mod_python request, handle the errors which occurred during the
action to perform, managing the sessions, checking security for each
operation (thus they implement the access control system of the web
interface).
It is important to encapsulate all this here as in case of changing
the web application framework we'll just need to adapt this layer (the
rest of the system wouldn't need any change).
Attributes:
_uh - (URLHandler) Associated URLHandler which points to the
current rh.
_req - UNUSED/OBSOLETE, always None
_requestStarted - (bool) Flag which tells whether a DB transaction
has been started or not.
_aw - (AccessWrapper) Current access information for the rh.
_target - (Locable) Reference to an object which is the destination
of the operations needed to carry out the rh. If set it must
provide (through the standard Locable interface) the methods
to get the url parameters in order to reproduce the access to
the rh.
_reqParams - (dict) Dictionary containing the received HTTP
parameters (independently of the method) transformed into
python data types. The key is the parameter name while the
value should be the received paramter value (or values).
"""
_tohttps = False # set this value to True for the RH that must be HTTPS when there is a BaseSecureURL
_doNotSanitizeFields = []
_isMobile = True # this value means that the generated web page can be mobile
CSRF_ENABLED = False # require a csrf_token when accessing the RH with anything but GET
EVENT_FEATURE = None # require a certain event feature when accessing the RH. See `EventFeature` for details
#: A dict specifying how the url should be normalized.
#: `args` is a dictionary mapping view args keys to callables
#: used to retrieve the expected value for those arguments if they
#: are present in the request's view args.
#: `locators` is a set of callables returning objects with locators.
#: `preserved_args` is a set of view arg names which will always
#: be copied from the current request if present.
#: The callables are always invoked with a single `self` argument
#: containing the RH instance.
#: Arguments specified in the `defaults` of any rule matching the
#: current endpoint are always excluded when checking if the args
#: match or when building a new URL.
#: If the view args built from the returned objects do not match
#: the request's view args, a redirect is issued automatically.
#: If the request is not using GET/HEAD, a 404 error is raised
#: instead of a redirect since such requests cannot be redirected
#: but executing them on the wrong URL may pose a security risk in
#: case and of the non-relevant URL segments is used for access
#: checks.
normalize_url_spec = {
'args': {},
'locators': set(),
'preserved_args': set()
}
def __init__(self):
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper(
) # Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True # Flag which indicates whether the RH process
# must be carried out; this is useful for
# the checkProtection methods when they
# detect that an immediate redirection is
# needed
# Methods =============================================================
def getTarget(self):
return self._target
def isMobile(self):
return self._isMobile
def _setSessionUser(self):
self._aw.setUser(session.avatar)
@property
def csrf_token(self):
return session.csrf_token if session.csrf_protected else ''
def _getRequestParams(self):
return self._reqParams
def getRequestParams(self):
return self._getRequestParams()
def _disableCaching(self):
"""Disables caching"""
# IE doesn't seem to like 'no-cache' Cache-Control headers...
if request.user_agent.browser == 'msie':
# actually, the only way to safely disable caching seems to be this one
self._responseUtil.headers["Cache-Control"] = "private"
self._responseUtil.headers["Expires"] = "-1"
else:
self._responseUtil.headers[
"Cache-Control"] = "no-store, no-cache, must-revalidate"
self._responseUtil.headers["Pragma"] = "no-cache"
def _redirect(self, targetURL, status=303):
if isinstance(targetURL, Response):
status = targetURL.status_code
targetURL = targetURL.headers['Location']
else:
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def _changeRH(self, rh, params):
"""Calls the specified RH after processing this one"""
self._responseUtil.call = lambda: rh().process(params)
def _checkHttpsRedirect(self):
"""If HTTPS must be used but it is not, redirect!"""
if self.use_https() and not request.is_secure:
self._redirect(self.getRequestURL(secure=True))
return True
else:
return False
def _normaliseListParam(self, param):
if not isinstance(param, list):
return [param]
return param
def _processError(self, e):
raise
def normalize_url(self):
"""Performs URL normalization.
This uses the :attr:`normalize_url_spec` to check if the URL
params are what they should be and redirects or fails depending
on the HTTP method used if it's not the case.
:return: ``None`` or a redirect response
"""
if current_app.debug and self.normalize_url_spec is RH.normalize_url_spec:
# in case of ``class SomeRH(RH, MixinWithNormalization)``
# the default value from `RH` overwrites the normalization
# rule from ``MixinWithNormalization``. this is never what
# the developer wants so we fail if it happens. the proper
# solution is ``class SomeRH(MixinWithNormalization, RH)``
cls = next((x for x in inspect.getmro(self.__class__)
if (x is not RH and x is not self.__class__
and hasattr(x, 'normalize_url_spec')
and getattr(x, 'normalize_url_spec',
None) is not RH.normalize_url_spec)),
None)
if cls is not None:
raise Exception(
'Normalization rule of {} in {} is overwritten by base RH. Put mixins with class-level '
'attributes on the left of the base class'.format(
cls, self.__class__))
if not self.normalize_url_spec or not any(
self.normalize_url_spec.itervalues()):
return
spec = {
'args': self.normalize_url_spec.get('args', {}),
'locators': self.normalize_url_spec.get('locators', set()),
'preserved_args':
self.normalize_url_spec.get('preserved_args', set()),
}
# Initialize the new view args with preserved arguments (since those would be lost otherwise)
new_view_args = {
k: v
for k, v in request.view_args.iteritems()
if k in spec['preserved_args']
}
# Retrieve the expected values for all simple arguments (if they are currently present)
for key, getter in spec['args'].iteritems():
if key in request.view_args:
new_view_args[key] = getter(self)
# Retrieve the expected values from locators
for getter in spec['locators']:
value = getter(self)
if value is None:
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
new_view_args.update(get_locator(value))
# Get all default values provided by the url map for the endpoint
defaults = set(
itertools.chain.from_iterable(
r.defaults
for r in current_app.url_map.iter_rules(request.endpoint)
if r.defaults))
def _convert(v):
# some legacy code has numeric ids in the locator data, but still takes
# string ids in the url rule (usually for confId)
return unicode(v) if isinstance(v, (int, long)) else v
provided = {
k: _convert(v)
for k, v in request.view_args.iteritems() if k not in defaults
}
new_view_args = {k: _convert(v) for k, v in new_view_args.iteritems()}
if new_view_args != provided:
if request.method in {'GET', 'HEAD'}:
try:
return redirect(
url_for(
request.endpoint,
**dict(request.args.to_dict(), **new_view_args)))
except BuildError as e:
if current_app.debug:
raise
Logger.get('requestHandler').warn(
'BuildError during normalization: %s', e)
raise NotFound
else:
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
def _checkParams(self, params):
"""This method is called before _checkProtection and is a good place
to assign variables from request params to member variables.
Note that in any new code the params argument SHOULD be IGNORED.
Use the following objects provided by Flask instead:
from flask import request
request.view_args (URL route params)
request.args (GET params (from the query string))
request.form (POST params)
request.values (GET+POST params - use only if ABSOLUTELY NECESSARY)
If you only want to run some code for GET or POST requests, you can create
a method named e.g. _checkParams_POST which will be executed AFTER this one.
The method is called without any arguments (except self).
"""
pass
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [
m for m in HTTP_VERBS if hasattr(self, '_process_' + m)
]
raise MethodNotAllowed(valid_methods)
return method()
def _checkCSRF(self):
token = request.headers.get('X-CSRF-Token') or request.form.get(
'csrf_token')
if token is None:
# Might be a WTForm with a prefix. In that case the field name is '<prefix>-csrf_token'
token = next((v for k, v in request.form.iteritems()
if k.endswith('-csrf_token')), None)
if self.CSRF_ENABLED and request.method != 'GET' and token != session.csrf_token:
msg = _(
u"It looks like there was a problem with your current session. Please use your browser's back "
u"button, reload the page and try again.")
raise BadRequest(msg)
elif not self.CSRF_ENABLED and current_app.debug and request.method != 'GET':
# Warn if CSRF is not enabled for a RH in new code
module = self.__class__.__module__
if module.startswith('indico.modules.') or module.startswith(
'indico.core.'):
msg = (
u'{} request sent to {} which has no CSRF checks. Set `CSRF_ENABLED = True` in the class to '
u'enable them.').format(request.method,
self.__class__.__name__)
warnings.warn(msg, RuntimeWarning)
# legacy csrf check (referer-based):
# Check referer for POST requests. We do it here so we can properly use indico's error handling
if Config.getInstance().getCSRFLevel() < 3 or request.method != 'POST':
return
referer = request.referrer
# allow empty - otherwise we might lock out paranoid users blocking referers
if not referer:
return
# valid http referer
if referer.startswith(Config.getInstance().getBaseURL()):
return
# valid https referer - if https is enabled
base_secure = Config.getInstance().getBaseSecureURL()
if base_secure and referer.startswith(base_secure):
return
raise BadRefererError(
'This operation is not allowed from an external referer.')
def _check_event_feature(self):
from indico.modules.events.features.util import require_feature
event_id = request.view_args.get('confId') or request.view_args.get(
'event_id')
if event_id is not None:
require_feature(event_id, self.EVENT_FEATURE)
@jsonify_error
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH."""
if Config.getInstance().getPropagateAllExceptions():
raise
return errors.WPGenericError(self).display()
@jsonify_error(status=500, logging_level='exception')
def _processUnexpectedError(self, e):
"""Unexpected errors"""
self._responseUtil.redirect = None
if Config.getInstance().getEmbeddedWebserver() or Config.getInstance(
).getPropagateAllExceptions():
raise
return errors.WPUnexpectedError(self).display()
@jsonify_error(status=403)
def _processForbidden(self, e):
if session.user is None and not request.is_xhr and not e.response:
return redirect_to_login(
reason=_("Please log in to access this page."))
message = _("Access Denied")
explanation = get_error_description(e)
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error(status=400)
def _processBadRequest(self, e):
message = _("Bad Request")
return WErrorWSGI((message, e.description)).getHTML()
@jsonify_error(status=400)
def _processBadData(self, e):
message = _("Invalid or expired token")
return WErrorWSGI((message, e.message)).getHTML()
@jsonify_error(status=403)
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPAccessError(self).display()
@jsonify_error
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
# We are going to redirect to the page asking for access key
# and so it must be https if there is a BaseSecureURL. And that's
# why we set _tohttps to True.
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPKeyAccessError(self).display()
@jsonify_error
def _processModificationError(self, e):
"""Handles modification errors occured during the process of a RH."""
# Redirect to HTTPS in case the user is logged in
self._tohttps = True
if self._checkHttpsRedirect():
return
return errors.WPModificationError(self).display()
@jsonify_error(status=400)
def _processBadRequestKeyError(self, e):
"""Request lacks a necessary key for processing"""
msg = _('Required argument missing: %s') % e.message
return errors.WPFormValuesError(self, msg).display()
@jsonify_error
def _processConferenceClosedError(self, e):
"""Treats access to modification pages for conferences when they are closed."""
return WPConferenceModificationClosed(self, e._conf).display()
@jsonify_error
def _processTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPTimingError(self, e).display()
@jsonify_error
def _processNoReportError(self, e):
"""Process errors without reporting"""
return errors.WPNoReportError(self, e).display()
@jsonify_error(status=404)
def _processNotFoundError(self, e):
if isinstance(e, NotFound):
message = _(
"Page not found") # that's a bit nicer than "404: Not Found"
explanation = get_error_description(e)
else:
message = e.getMessage()
explanation = e.getExplanation()
return WErrorWSGI((message, explanation)).getHTML()
@jsonify_error
def _processParentTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPParentTimingError(self, e).display()
@jsonify_error
def _processEntryTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
return errors.WPEntryTimingError(self, e).display()
@jsonify_error
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
return errors.WPFormValuesError(self, e).display()
@jsonify_error
def _processLaTeXError(self, e):
"""Treats access errors occured during the process of a RH."""
return errors.WPLaTeXError(self, e).display()
@jsonify_error
def _processRestrictedHTML(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
@jsonify_error
def _processHtmlForbiddenTag(self, e):
return errors.WPRestrictedHTML(self, escape(str(e))).display()
def _process_retry_setup(self):
# clear the fossile cache at the start of each request
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
# clear the existing redis pipeline
if self._redisPipeline:
self._redisPipeline.reset()
def _process_retry_auth_check(self, params):
# keep a link to the web session in the access wrapper
# this is used for checking access/modification key existence
# in the user session
self._setSessionUser()
if self._getAuth():
if self._getUser():
Logger.get('requestHandler').info(
'Request %s identified with user %s (%s)' %
(request, self._getUser().getFullName(),
self._getUser().getId()))
if not self._tohttps and Config.getInstance(
).getAuthenticatedEnforceSecure():
self._tohttps = True
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
self._checkCSRF()
self._reqParams = copy.copy(params)
def _process_retry_do(self, profile):
profile_name, res = '', ''
try:
# old code gets parameters from call
# new code utilizes of flask.request
if len(inspect.getargspec(self._checkParams).args) < 2:
cp_result = self._checkParams()
else:
cp_result = self._checkParams(self._reqParams)
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
func = getattr(self, '_checkParams_' + request.method, None)
if func:
cp_result = func()
if isinstance(cp_result,
(current_app.response_class, Response)):
return '', cp_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFoundError(_('The specified item could not be found.'),
title=_('Item not found'))
rv = self.normalize_url()
if rv is not None:
return '', rv
self._checkProtection()
func = getattr(self, '_checkProtection_' + request.method, None)
if func:
func()
security.Sanitization.sanitizationCheck(self._target, self._reqParams,
self._aw,
self._doNotSanitizeFields)
if self._doProcess:
if profile:
profile_name = os.path.join(
Config.getInstance().getTempDir(),
'stone{}.prof'.format(random.random()))
result = [None]
profiler.runctx('result[0] = self._process()', globals(),
locals(), profile_name)
res = result[0]
else:
res = self._process()
return profile_name, res
def _process_retry(self, params, retry, profile, forced_conflicts):
self._process_retry_setup()
self._process_retry_auth_check(params)
DBMgr.getInstance().sync()
return self._process_retry_do(profile)
def _process_success(self):
Logger.get('requestHandler').info(
'Request {} successful'.format(request))
# request is succesfull, now, doing tasks that must be done only once
try:
flush_after_commit_queue(True)
GenericMailer.flushQueue(True) # send emails
self._deleteTempFiles()
except:
Logger.get('mail').exception('Mail sending operation failed')
# execute redis pipeline if we have one
if self._redisPipeline:
try:
self._redisPipeline.execute()
except RedisError:
Logger.get('redis').exception('Could not execute pipeline')
def process(self, params):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
cfg = Config.getInstance()
forced_conflicts, max_retries, profile = cfg.getForceConflicts(
), cfg.getMaxRetries(), cfg.getProfile()
profile_name, res, textLog = '', '', []
self._startTime = datetime.now()
# clear the context
ContextManager.destroy()
ContextManager.set('currentRH', self)
g.rh = self
#redirect to https if necessary
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
if self.EVENT_FEATURE is not None:
self._check_event_feature()
DBMgr.getInstance().startRequest()
textLog.append("%s : Database request started" %
(datetime.now() - self._startTime))
Logger.get('requestHandler').info('[pid=%s] Request %s started' %
(os.getpid(), request))
is_error_response = False
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
signals.before_retry.send()
try:
Logger.get('requestHandler').info(
'\t[pid=%s] from host %s' %
(os.getpid(), request.remote_addr))
profile_name, res = self._process_retry(
params, i, profile, forced_conflicts)
signals.after_process.send()
if i < forced_conflicts: # raise conflict error if enabled to easily handle conflict error case
raise ConflictError
transaction.commit()
DBMgr.getInstance().endRequest(commit=False)
break
except (ConflictError, POSKeyError):
transaction.abort()
import traceback
# only log conflict if it wasn't forced
if i >= forced_conflicts:
Logger.get('requestHandler').warning(
'Conflict in Database! (Request %s)\n%s' %
(request, traceback.format_exc()))
except ClientDisconnected:
transaction.abort()
Logger.get('requestHandler').warning(
'Client Disconnected! (Request {})'.format(
request))
time.sleep(i)
self._process_success()
except Exception as e:
transaction.abort()
res = self._getMethodByExceptionName(e)(e)
if isinstance(e, HTTPException) and e.response is not None:
res = e.response
is_error_response = True
totalTime = (datetime.now() - self._startTime)
textLog.append('{} : Request ended'.format(totalTime))
# log request timing
if profile and totalTime > timedelta(
0, 1) and os.path.isfile(profile_name):
rep = Config.getInstance().getTempDir()
stats = pstats.Stats(profile_name)
stats.strip_dirs()
stats.sort_stats('cumulative', 'time', 'calls')
stats.dump_stats(os.path.join(rep, 'IndicoRequestProfile.log'))
output = StringIO.StringIO()
sys.stdout = output
stats.print_stats(100)
sys.stdout = sys.__stdout__
s = output.getvalue()
f = file(os.path.join(rep, 'IndicoRequest.log'), 'a+')
f.write('--------------------------------\n')
f.write('URL : {}\n'.format(request.url))
f.write('{} : start request\n'.format(self._startTime))
f.write('params:{}'.format(params))
f.write('\n'.join(textLog))
f.write(s)
f.write('--------------------------------\n\n')
f.close()
if profile and profile_name and os.path.exists(profile_name):
os.remove(profile_name)
if self._responseUtil.call:
return self._responseUtil.make_call()
if is_error_response and isinstance(
res, (current_app.response_class, Response)):
# if we went through error handling code, responseUtil._status has been changed
# so make_response() would fail
return res
# In case of no process needed, we should return empty string to avoid erroneous output
# specially with getVars breaking the JS files.
if not self._doProcess or res is None:
return self._responseUtil.make_empty()
return self._responseUtil.make_response(res)
def _getMethodByExceptionName(self, e):
exception_name = {
'NotFound': 'NotFoundError',
'MaKaCError': 'GeneralError',
'IndicoError': 'GeneralError',
'ValueError': 'UnexpectedError',
'Exception': 'UnexpectedError',
'AccessControlError': 'AccessError'
}.get(type(e).__name__,
type(e).__name__)
if isinstance(e, BadData): # we also want its subclasses
exception_name = 'BadData'
return getattr(self, '_process{}'.format(exception_name),
self._processUnexpectedError)
def _deleteTempFiles(self):
if len(self._tempFilesToDelete) > 0:
for f in self._tempFilesToDelete:
if f is not None:
os.remove(f)
relativeURL = None
class RH(object):
CSRF_ENABLED = True # require a csrf_token when accessing the RH with anything but GET
EVENT_FEATURE = None # require a certain event feature when accessing the RH. See `EventFeature` for details
DENY_FRAMES = False # whether to send an X-Frame-Options:DENY header
#: A dict specifying how the url should be normalized.
#: `args` is a dictionary mapping view args keys to callables
#: used to retrieve the expected value for those arguments if they
#: are present in the request's view args.
#: `locators` is a set of callables returning objects with locators.
#: `preserved_args` is a set of view arg names which will always
#: be copied from the current request if present.
#: The callables are always invoked with a single `self` argument
#: containing the RH instance.
#: `endpoint` may be used to specify the endpoint used to build
#: the URL in case of a redirect. Usually this should not be used
#: in favor of ``request.endpoint`` being used if no custom endpoint
#: is set.
#: Arguments specified in the `defaults` of any rule matching the
#: current endpoint are always excluded when checking if the args
#: match or when building a new URL.
#: If the view args built from the returned objects do not match
#: the request's view args, a redirect is issued automatically.
#: If the request is not using GET/HEAD, a 404 error is raised
#: instead of a redirect since such requests cannot be redirected
#: but executing them on the wrong URL may pose a security risk in
#: case and of the non-relevant URL segments is used for access
#: checks.
normalize_url_spec = {
'args': {},
'locators': set(),
'preserved_args': set(),
'endpoint': None
}
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
# Methods =============================================================
def validate_json(self, schema, json=None):
"""Validates the request's JSON payload using a JSON schema.
:param schema: The JSON schema used for validation.
:param json: The JSON object (defaults to ``request.json``)
:raises BadRequest: if the JSON validation failed
"""
if json is None:
json = request.json
try:
jsonschema.validate(json, schema)
except jsonschema.ValidationError as e:
raise BadRequest('Invalid JSON payload: {}'.format(e.message))
@property
def csrf_token(self):
return session.csrf_token if session.csrf_protected else ''
def normalize_url(self):
"""Performs URL normalization.
This uses the :attr:`normalize_url_spec` to check if the URL
params are what they should be and redirects or fails depending
on the HTTP method used if it's not the case.
:return: ``None`` or a redirect response
"""
if current_app.debug and self.normalize_url_spec is RH.normalize_url_spec:
# in case of ``class SomeRH(RH, MixinWithNormalization)``
# the default value from `RH` overwrites the normalization
# rule from ``MixinWithNormalization``. this is never what
# the developer wants so we fail if it happens. the proper
# solution is ``class SomeRH(MixinWithNormalization, RH)``
cls = next((x for x in inspect.getmro(self.__class__)
if (x is not RH and x is not self.__class__
and hasattr(x, 'normalize_url_spec')
and getattr(x, 'normalize_url_spec',
None) is not RH.normalize_url_spec)),
None)
if cls is not None:
raise Exception(
'Normalization rule of {} in {} is overwritten by base RH. Put mixins with class-level '
'attributes on the left of the base class'.format(
cls, self.__class__))
if not self.normalize_url_spec or not any(
self.normalize_url_spec.itervalues()):
return
spec = {
'args': self.normalize_url_spec.get('args', {}),
'locators': self.normalize_url_spec.get('locators', set()),
'preserved_args':
self.normalize_url_spec.get('preserved_args', set()),
'endpoint': self.normalize_url_spec.get('endpoint', None)
}
# Initialize the new view args with preserved arguments (since those would be lost otherwise)
new_view_args = {
k: v
for k, v in request.view_args.iteritems()
if k in spec['preserved_args']
}
# Retrieve the expected values for all simple arguments (if they are currently present)
for key, getter in spec['args'].iteritems():
if key in request.view_args:
new_view_args[key] = getter(self)
# Retrieve the expected values from locators
prev_locator_args = {}
for getter in spec['locators']:
value = getter(self)
if value is None:
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
locator_args = get_locator(value)
reused_keys = set(locator_args) & prev_locator_args.viewkeys()
if any(locator_args[k] != prev_locator_args[k]
for k in reused_keys):
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
new_view_args.update(locator_args)
prev_locator_args.update(locator_args)
# Get all default values provided by the url map for the endpoint
defaults = set(
itertools.chain.from_iterable(
r.defaults
for r in current_app.url_map.iter_rules(request.endpoint)
if r.defaults))
def _convert(v):
# some legacy code has numeric ids in the locator data, but still takes
# string ids in the url rule (usually for confId)
return unicode(v) if isinstance(v, (int, long)) else v
provided = {
k: _convert(v)
for k, v in request.view_args.iteritems() if k not in defaults
}
new_view_args = {k: _convert(v) for k, v in new_view_args.iteritems()}
if new_view_args != provided:
if request.method in {'GET', 'HEAD'}:
endpoint = spec['endpoint'] or request.endpoint
try:
return redirect(
url_for(
endpoint,
**dict(request.args.to_dict(), **new_view_args)))
except BuildError as e:
if current_app.debug:
raise
logger.warn('BuildError during normalization: %s', e)
raise NotFound
else:
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
def _process_args(self):
"""
This method is called before _check_access and url normalization
and is a good place to fetch objects from the database based on
variables from request params.
"""
def _check_access(self):
"""
This method is called after _process_args and is a good place
to check if the user is permitted to perform some actions.
"""
def _process(self):
"""Dispatch to a method named ``_process_<verb>``.
Except for RESTful endpoints it is usually best to just
override this method, especially when using WTForms.
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [
m for m in HTTP_VERBS if hasattr(self, '_process_' + m)
]
raise MethodNotAllowed(valid_methods)
return method()
def _check_csrf(self):
token = request.headers.get('X-CSRF-Token') or request.form.get(
'csrf_token')
if token is None:
# Might be a WTForm with a prefix. In that case the field name is '<prefix>-csrf_token'
token = next((v for k, v in request.form.iteritems()
if k.endswith('-csrf_token')), None)
if self.CSRF_ENABLED and request.method != 'GET' and token != session.csrf_token:
msg = _(
"It looks like there was a problem with your current session. Please use your browser's back "
"button, reload the page and try again.")
raise BadRequest(msg)
def _check_event_feature(self):
from indico.modules.events.features.util import require_feature
event_id = request.view_args.get('confId') or request.view_args.get(
'event_id')
if event_id is not None:
require_feature(event_id, self.EVENT_FEATURE)
def _do_process(self):
try:
args_result = self._process_args()
signals.rh.process_args.send(type(self),
rh=self,
result=args_result)
if isinstance(args_result, (current_app.response_class, Response)):
return args_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFound(_('The specified item could not be found.'))
rv = self.normalize_url()
if rv is not None:
return rv
self._check_access()
signals.rh.check_access.send(type(self), rh=self)
signal_rv = values_from_signal(signals.rh.before_process.send(
type(self), rh=self),
single_value=True,
as_list=True)
if signal_rv and len(signal_rv) != 1:
raise Exception(
'More than one signal handler returned custom RH result')
elif signal_rv:
return signal_rv[0]
if config.PROFILE:
result = [None]
profile_path = os.path.join(
config.TEMP_DIR,
'{}-{}.prof'.format(type(self).__name__, time.time()))
cProfile.runctx('result[0] = self._process()', globals(), locals(),
profile_path)
rv = result[0]
else:
rv = self._process()
signal_rv = values_from_signal(signals.rh.process.send(type(self),
rh=self,
result=rv),
single_value=True,
as_list=True)
if signal_rv and len(signal_rv) != 1:
raise Exception(
'More than one signal handler returned new RH result')
elif signal_rv:
return signal_rv[0]
else:
return rv
def process(self):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
res = ''
g.rh = self
sentry_set_tags({'rh': self.__class__.__name__})
if self.EVENT_FEATURE is not None:
self._check_event_feature()
logger.info('%s %s [IP=%s] [PID=%s] [UID=%r]', request.method,
request.relative_url, request.remote_addr, os.getpid(),
session.get('_user_id'))
try:
fossilize.clearCache()
init_email_queue()
self._check_csrf()
res = self._do_process()
signals.after_process.send()
if self.commit:
db.session.commit()
flush_email_queue()
else:
db.session.rollback()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error(
) # this will re-raise an exception
except Exception:
# rollback to avoid errors as rendering the error page
# within the indico layout may trigger an auto-flush
db.session.rollback()
raise
logger.debug('Request successful')
if res is None:
return self._responseUtil.make_empty()
response = self._responseUtil.make_response(res)
if self.DENY_FRAMES:
response.headers['X-Frame-Options'] = 'DENY'
return response
class RH(RequestHandlerBase):
NOT_SANITIZED_FIELDS = frozenset()
CSRF_ENABLED = True # require a csrf_token when accessing the RH with anything but GET
EVENT_FEATURE = None # require a certain event feature when accessing the RH. See `EventFeature` for details
DENY_FRAMES = False # whether to send an X-Frame-Options:DENY header
#: A dict specifying how the url should be normalized.
#: `args` is a dictionary mapping view args keys to callables
#: used to retrieve the expected value for those arguments if they
#: are present in the request's view args.
#: `locators` is a set of callables returning objects with locators.
#: `preserved_args` is a set of view arg names which will always
#: be copied from the current request if present.
#: The callables are always invoked with a single `self` argument
#: containing the RH instance.
#: `endpoint` may be used to specify the endpoint used to build
#: the URL in case of a redirect. Usually this should not be used
#: in favor of ``request.endpoint`` being used if no custom endpoint
#: is set.
#: Arguments specified in the `defaults` of any rule matching the
#: current endpoint are always excluded when checking if the args
#: match or when building a new URL.
#: If the view args built from the returned objects do not match
#: the request's view args, a redirect is issued automatically.
#: If the request is not using GET/HEAD, a 404 error is raised
#: instead of a redirect since such requests cannot be redirected
#: but executing them on the wrong URL may pose a security risk in
#: case and of the non-relevant URL segments is used for access
#: checks.
normalize_url_spec = {
'args': {},
'locators': set(),
'preserved_args': set(),
'endpoint': None
}
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
self._target = None
self._startTime = None
self._endTime = None
self._doProcess = True
# Methods =============================================================
def validate_json(self, schema, json=None):
"""Validates the request's JSON payload using a JSON schema.
:param schema: The JSON schema used for validation.
:param json: The JSON object (defaults to ``request.json``)
:raises BadRequest: if the JSON validation failed
"""
if json is None:
json = request.json
try:
jsonschema.validate(json, schema)
except jsonschema.ValidationError as e:
raise BadRequest('Invalid JSON payload: {}'.format(e.message))
@property
def csrf_token(self):
return session.csrf_token if session.csrf_protected else ''
def _redirect(self, targetURL, status=303):
if isinstance(targetURL, Response):
status = targetURL.status_code
targetURL = targetURL.headers['Location']
else:
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def normalize_url(self):
"""Performs URL normalization.
This uses the :attr:`normalize_url_spec` to check if the URL
params are what they should be and redirects or fails depending
on the HTTP method used if it's not the case.
:return: ``None`` or a redirect response
"""
if current_app.debug and self.normalize_url_spec is RH.normalize_url_spec:
# in case of ``class SomeRH(RH, MixinWithNormalization)``
# the default value from `RH` overwrites the normalization
# rule from ``MixinWithNormalization``. this is never what
# the developer wants so we fail if it happens. the proper
# solution is ``class SomeRH(MixinWithNormalization, RH)``
cls = next((x for x in inspect.getmro(self.__class__)
if (x is not RH and x is not self.__class__
and hasattr(x, 'normalize_url_spec')
and getattr(x, 'normalize_url_spec',
None) is not RH.normalize_url_spec)),
None)
if cls is not None:
raise Exception(
'Normalization rule of {} in {} is overwritten by base RH. Put mixins with class-level '
'attributes on the left of the base class'.format(
cls, self.__class__))
if not self.normalize_url_spec or not any(
self.normalize_url_spec.itervalues()):
return
spec = {
'args': self.normalize_url_spec.get('args', {}),
'locators': self.normalize_url_spec.get('locators', set()),
'preserved_args':
self.normalize_url_spec.get('preserved_args', set()),
'endpoint': self.normalize_url_spec.get('endpoint', None)
}
# Initialize the new view args with preserved arguments (since those would be lost otherwise)
new_view_args = {
k: v
for k, v in request.view_args.iteritems()
if k in spec['preserved_args']
}
# Retrieve the expected values for all simple arguments (if they are currently present)
for key, getter in spec['args'].iteritems():
if key in request.view_args:
new_view_args[key] = getter(self)
# Retrieve the expected values from locators
prev_locator_args = {}
for getter in spec['locators']:
value = getter(self)
if value is None:
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
locator_args = get_locator(value)
reused_keys = set(locator_args) & prev_locator_args.viewkeys()
if any(locator_args[k] != prev_locator_args[k]
for k in reused_keys):
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
new_view_args.update(locator_args)
prev_locator_args.update(locator_args)
# Get all default values provided by the url map for the endpoint
defaults = set(
itertools.chain.from_iterable(
r.defaults
for r in current_app.url_map.iter_rules(request.endpoint)
if r.defaults))
def _convert(v):
# some legacy code has numeric ids in the locator data, but still takes
# string ids in the url rule (usually for confId)
return unicode(v) if isinstance(v, (int, long)) else v
provided = {
k: _convert(v)
for k, v in request.view_args.iteritems() if k not in defaults
}
new_view_args = {k: _convert(v) for k, v in new_view_args.iteritems()}
if new_view_args != provided:
if request.method in {'GET', 'HEAD'}:
endpoint = spec['endpoint'] or request.endpoint
try:
return redirect(
url_for(
endpoint,
**dict(request.args.to_dict(), **new_view_args)))
except BuildError as e:
if current_app.debug:
raise
logger.warn('BuildError during normalization: %s', e)
raise NotFound
else:
raise NotFound(
'The URL contains invalid data. Please go to the previous page and refresh it.'
)
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [
m for m in HTTP_VERBS if hasattr(self, '_process_' + m)
]
raise MethodNotAllowed(valid_methods)
return method()
def _check_csrf(self):
token = request.headers.get('X-CSRF-Token') or request.form.get(
'csrf_token')
if token is None:
# Might be a WTForm with a prefix. In that case the field name is '<prefix>-csrf_token'
token = next((v for k, v in request.form.iteritems()
if k.endswith('-csrf_token')), None)
if self.CSRF_ENABLED and request.method != 'GET' and token != session.csrf_token:
msg = _(
u"It looks like there was a problem with your current session. Please use your browser's back "
u"button, reload the page and try again.")
raise BadRequest(msg)
def _check_event_feature(self):
from indico.modules.events.features.util import require_feature
event_id = request.view_args.get('confId') or request.view_args.get(
'event_id')
if event_id is not None:
require_feature(event_id, self.EVENT_FEATURE)
@jsonify_error
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH."""
if config.PROPAGATE_ALL_EXCEPTIONS:
raise
return WPGenericError(self).display()
@jsonify_error(status=500, logging_level='exception', log_sentry=True)
def _processUnexpectedError(self, e):
"""Unexpected errors"""
self._responseUtil.redirect = None
if config.PROPAGATE_ALL_EXCEPTIONS:
raise
sentry_log_exception()
return WPUnexpectedError(self).display()
@jsonify_error(status=403)
def _processForbidden(self, e):
if session.user is None and not request.is_xhr and not e.response and request.blueprint != 'auth':
return redirect_to_login(
reason=_("Please log in to access this page."))
message = _("Access Denied")
explanation = get_error_description(e)
return render_error(message, explanation)
@jsonify_error(status=400)
def _processBadRequest(self, e):
message = _("Bad Request")
return render_error(message, e.description)
@jsonify_error(status=401)
def _processUnauthorized(self, e):
message = _("Unauthorized")
return render_error(message, e.description)
@jsonify_error(status=400)
def _processBadData(self, e):
message = _("Invalid or expired token")
return render_error(message, e.message)
@jsonify_error(status=403)
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return WPAccessError(self).display()
@jsonify_error
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
return WPKeyAccessError(self).display()
@jsonify_error
def _processModificationError(self, e):
"""Handles modification errors occured during the process of a RH."""
if not session.user:
return redirect_to_login(reason=_(
"Please log in to access this page. If you have a modification key, you "
"may enter it afterwards."))
return WPModificationError(self).display()
@jsonify_error(status=400)
def _processBadRequestKeyError(self, e):
"""Request lacks a necessary key for processing"""
msg = _(u'Required argument missing: {}').format(to_unicode(e.message))
return WPFormValuesError(self, msg).display()
@jsonify_error
def _processNoReportError(self, e):
"""Process errors without reporting"""
return WPNoReportError(self, e).display()
@jsonify_error(status=400)
def _processUserValueError(self, e):
"""Process errors without reporting"""
return WPNoReportError(self, e).display()
@jsonify_error(status=404)
def _processNotFoundError(self, e):
if isinstance(e, NotFound):
message = _(
"Page not found") # that's a bit nicer than "404: Not Found"
explanation = get_error_description(e)
else:
message = e.getMessage()
explanation = e.getExplanation()
return render_error(message, explanation)
@jsonify_error(status=400)
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
return WPFormValuesError(self, e).display()
@jsonify_error(status=400)
def _processRestrictedHTML(self, e):
return WPRestrictedHTML(self, escape(str(e))).display()
@jsonify_error(status=400)
def _processHtmlForbiddenTag(self, e):
return WPRestrictedHTML(self, escape(str(e))).display()
def _check_auth(self):
if session.user:
logger.info('Request authenticated: %r', session.user)
self._check_csrf()
def _do_process(self, profile):
profile_name = res = ''
try:
cp_result = self._process_args()
if isinstance(cp_result, (current_app.response_class, Response)):
return '', cp_result
except NoResultFound: # sqlalchemy .one() not finding anything
raise NotFoundError(_('The specified item could not be found.'),
title=_('Item not found'))
rv = self.normalize_url()
if rv is not None:
return '', rv
self._check_access()
Sanitization.sanitizationCheck(create_flat_args(),
self.NOT_SANITIZED_FIELDS)
if self._doProcess:
if profile:
profile_name = os.path.join(
config.TEMP_DIR, 'stone{}.prof'.format(random.random()))
result = [None]
cProfile.runctx('result[0] = self._process()', globals(),
locals(), profile_name)
res = result[0]
else:
res = self._process()
return profile_name, res
def process(self):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
profile = config.PROFILE
profile_name, res, textLog = '', '', []
self._startTime = datetime.now()
g.rh = self
sentry_set_tags({'rh': self.__class__.__name__})
if self.EVENT_FEATURE is not None:
self._check_event_feature()
textLog.append("%s : Database request started" %
(datetime.now() - self._startTime))
logger.info(u'Request started: %s %s [IP=%s] [PID=%s]', request.method,
request.relative_url, request.remote_addr, os.getpid())
is_error_response = False
try:
try:
fossilize.clearCache()
GenericMailer.flushQueue(False)
self._check_auth()
profile_name, res = self._do_process(profile)
signals.after_process.send()
if self.commit:
if GenericMailer.has_queue():
# ensure we fail early (before sending out e-mails)
# in case there are DB constraint violations, etc...
db.enforce_constraints()
GenericMailer.flushQueue(True)
db.session.commit()
else:
db.session.rollback()
except DatabaseError:
handle_sqlalchemy_database_error(
) # this will re-raise an exception
logger.info('Request successful')
except Exception as e:
db.session.rollback()
res = self._get_error_handler(e)(e)
if isinstance(e, HTTPException) and e.response is not None:
res = e.response
is_error_response = True
totalTime = (datetime.now() - self._startTime)
textLog.append('{} : Request ended'.format(totalTime))
# log request timing
if profile and os.path.isfile(profile_name):
rep = config.TEMP_DIR
stats = pstats.Stats(profile_name)
stats.sort_stats('cumulative', 'time', 'calls')
stats.dump_stats(os.path.join(rep, 'IndicoRequestProfile.log'))
os.remove(profile_name)
if is_error_response and isinstance(
res, (current_app.response_class, Response)):
# if we went through error handling code, responseUtil._status has been changed
# so make_response() would fail
return res
# In case of no process needed, we should return empty string to avoid erroneous output
# specially with getVars breaking the JS files.
if not self._doProcess or res is None:
return self._responseUtil.make_empty()
response = self._responseUtil.make_response(res)
if self.DENY_FRAMES:
response.headers['X-Frame-Options'] = 'DENY'
return response
def _get_error_handler(self, e):
exception_name = {
'NotFound': 'NotFoundError',
'MaKaCError': 'GeneralError',
'IndicoError': 'GeneralError',
'ValueError': 'UnexpectedError',
'Exception': 'UnexpectedError',
'AccessControlError': 'AccessError'
}.get(type(e).__name__,
type(e).__name__)
if isinstance(e, BadData): # we also want its subclasses
exception_name = 'BadData'
return getattr(self, '_process{}'.format(exception_name),
self._processUnexpectedError)
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
def handler(prefix, path):
path = posixpath.join('/', prefix, path)
ContextManager.destroy()
clearCache() # init fossil cache
logger = Logger.get('httpapi')
if request.method == 'POST':
# Convert POST data to a query string
queryParams = dict((key, value.encode('utf-8')) for key, value in request.form.iteritems())
query = urllib.urlencode(queryParams)
else:
# Parse the actual query string
queryParams = dict((key, value.encode('utf-8')) for key, value in request.args.iteritems())
query = request.query_string
dbi = DBMgr.getInstance()
dbi.startRequest()
apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None)
cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'], 'no') == 'yes'
signature = get_query_parameter(queryParams, ['signature'])
timestamp = get_query_parameter(queryParams, ['timestamp'], 0, integer=True)
noCache = get_query_parameter(queryParams, ['nc', 'nocache'], 'no') == 'yes'
pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes'
onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'], 'no') == 'yes'
onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'], 'no') == 'yes'
scope = 'read:legacy_api' if request.method == 'GET' else 'write:legacy_api'
try:
oauth_valid, oauth_request = oauth.verify_request([scope])
if not oauth_valid and oauth_request and oauth_request.error_message != 'Bearer token not found.':
raise BadRequest('OAuth error: {}'.format(oauth_request.error_message))
elif g.get('received_oauth_token') and oauth_request.error_message == 'Bearer token not found.':
raise BadRequest('OAuth error: Invalid token')
except ValueError:
# XXX: Dirty hack to workaround a bug in flask-oauthlib that causes it
# not to properly urlencode request query strings
# Related issue (https://github.com/lepture/flask-oauthlib/issues/213)
oauth_valid = False
# Get our handler function and its argument and response type
hook, dformat = HTTPAPIHook.parseRequest(path, queryParams)
if hook is None or dformat is None:
raise NotFound
# Disable caching if we are not just retrieving data (or the hook requires it)
if request.method == 'POST' or hook.NO_CACHE:
noCache = True
ak = error = result = None
ts = int(time.time())
typeMap = {}
responseUtil = ResponseUtil()
try:
used_session = None
if cookieAuth:
used_session = session
if not used_session.user: # ignore guest sessions
used_session = None
if apiKey or oauth_valid or not used_session:
if not oauth_valid:
# Validate the API key (and its signature)
ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp, path, query)
if enforceOnlyPublic:
onlyPublic = True
# Create an access wrapper for the API key's user
aw = buildAW(ak, onlyPublic)
else: # Access Token (OAuth)
at = load_token(oauth_request.access_token.access_token)
aw = buildAW(at, onlyPublic)
# Get rid of API key in cache key if we did not impersonate a user
if ak and aw.getUser() is None:
cacheKey = normalizeQuery(path, query,
remove=('_', 'ak', 'apiKey', 'signature', 'timestamp', 'nc', 'nocache',
'oa', 'onlyauthed'))
else:
cacheKey = normalizeQuery(path, query,
remove=('_', 'signature', 'timestamp', 'nc', 'nocache', 'oa', 'onlyauthed'))
if signature:
# in case the request was signed, store the result under a different key
cacheKey = 'signed_' + cacheKey
else:
# We authenticated using a session cookie.
if Config.getInstance().getCSRFLevel() >= 2:
token = request.headers.get('X-CSRF-Token', get_query_parameter(queryParams, ['csrftoken']))
if used_session.csrf_protected and used_session.csrf_token != token:
raise HTTPAPIError('Invalid CSRF token', 403)
aw = AccessWrapper()
if not onlyPublic:
aw.setUser(used_session.avatar)
userPrefix = 'user-{}_'.format(used_session.user.id)
cacheKey = userPrefix + normalizeQuery(path, query,
remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa', 'onlyauthed',
'csrftoken'))
# Bail out if the user requires authentication but is not authenticated
if onlyAuthed and not aw.getUser():
raise HTTPAPIError('Not authenticated', 403)
addToCache = not hook.NO_CACHE
cache = GenericCache('HTTPAPI')
cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey)
if not noCache:
obj = cache.get(cacheKey)
if obj is not None:
result, extra, ts, complete, typeMap = obj
addToCache = False
if result is None:
ContextManager.set("currentAW", aw)
# Perform the actual exporting
res = hook(aw)
if isinstance(res, tuple) and len(res) == 4:
result, extra, complete, typeMap = res
else:
result, extra, complete, typeMap = res, {}, True, {}
if result is not None and addToCache:
ttl = api_settings.get('cache_ttl')
cache.set(cacheKey, (result, extra, ts, complete, typeMap), ttl)
except HTTPAPIError, e:
error = e
if e.getCode():
responseUtil.status = e.getCode()
if responseUtil.status == 405:
responseUtil.headers['Allow'] = 'GET' if request.method == 'POST' else 'POST'
class RH(RequestHandlerBase):
"""This class is the base for request handlers of the application. A request
handler will be instantiated when a web request arrives to mod_python;
the mp layer will forward the request to the corresponding request
handler which will know which action has to be performed (displaying a
web page or performing some operation and redirecting to another page).
Request handlers will be responsible for parsing the parameters coming
from a mod_python request, handle the errors which occurred during the
action to perform, managing the sessions, checking security for each
operation (thus they implement the access control system of the web
interface).
It is important to encapsulate all this here as in case of changing
the web application framework we'll just need to adapt this layer (the
rest of the system wouldn't need any change).
Attributes:
_uh - (URLHandler) Associated URLHandler which points to the
current rh.
_req - UNUSED/OBSOLETE, always None
_requestStarted - (bool) Flag which tells whether a DB transaction
has been started or not.
_aw - (AccessWrapper) Current access information for the rh.
_target - (Locable) Reference to an object which is the destination
of the operations needed to carry out the rh. If set it must
provide (through the standard Locable interface) the methods
to get the url parameters in order to reproduce the access to
the rh.
_reqParams - (dict) Dictionary containing the received HTTP
parameters (independently of the method) transformed into
python data types. The key is the parameter name while the
value should be the received paramter value (or values).
"""
_tohttps = False # set this value to True for the RH that must be HTTPS when there is a BaseSecureURL
_doNotSanitizeFields = []
_isMobile = True # this value means that the generated web page can be mobile
HTTP_VERBS = frozenset(('GET', 'POST', 'PUT', 'DELETE'))
def __init__(self, req=None):
"""Constructor. Initialises the rh setting up basic attributes so it is
able to process the request.
Parameters:
req - OBSOLETE, MUST BE NONE
"""
RequestHandlerBase.__init__(self, req)
self._responseUtil = ResponseUtil()
self._requestStarted = False
self._aw = AccessWrapper() #Fill in the aw instance with the current information
self._target = None
self._reqParams = {}
self._startTime = None
self._endTime = None
self._tempFilesToDelete = []
self._redisPipeline = None
self._doProcess = True #Flag which indicates whether the RH process
# must be carried out; this is useful for
# the checkProtection methods when they
# detect that an inmediate redirection is
# needed
# Methods =============================================================
def getTarget(self):
return self._target
def isMobile(self):
return self._isMobile
def _setSessionUser(self):
self._aw.setUser(session.user)
@property
def csrf_token(self):
return session.csrf_token
def _getRequestParams(self):
return self._reqParams
def getRequestParams(self):
return self._getRequestParams()
def _disableCaching(self):
"""Disables caching"""
# IE doesn't seem to like 'no-cache' Cache-Control headers...
if request.user_agent.browser == 'msie':
# actually, the only way to safely disable caching seems to be this one
self._responseUtil.headers["Cache-Control"] = "private"
self._responseUtil.headers["Expires"] = "-1"
else:
self._responseUtil.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
self._responseUtil.headers["Pragma"] = "no-cache"
def _redirect(self, targetURL, status=303):
targetURL = str(targetURL)
if "\r" in targetURL or "\n" in targetURL:
raise MaKaCError(_("http header CRLF injection detected"))
self._responseUtil.redirect = (targetURL, status)
def _changeRH(self, rh, params):
"""Calls the specified RH after processing this one"""
self._responseUtil.call = lambda: rh(None).process(params)
def _checkHttpsRedirect(self):
"""If HTTPS must be used but it is not, redirect!"""
if self.use_https() and not request.is_secure:
self._redirect(self.getRequestURL(secure=True))
return True
else:
return False
def _normaliseListParam(self, param):
if not isinstance(param, list):
return [param]
return param
def _processError(self, ex):
raise
def _checkParams(self, params):
"""This method is called before _checkProtection and is a good place
to assign variables from request params to member variables.
Note that in any new code the params argument SHOULD be IGNORED.
Use the following objects provided by Flask instead:
from flask import request
request.view_args (URL route params)
request.args (GET params (from the query string))
request.form (POST params)
request.values (GET+POST params - use only if ABSOLUTELY NECESSARY)
If you only want to run some code for GET or POST requests, you can create
a method named e.g. _checkParams_POST which will be executed AFTER this one.
The method is called without any arguments (except self).
"""
pass
def _process(self):
"""The default process method dispatches to a method containing
the HTTP verb used for the current request, e.g. _process_POST.
When implementing this please consider that you most likely want/need
only GET and POST - the other verbs are not supported everywhere!
"""
method = getattr(self, '_process_' + request.method, None)
if method is None:
valid_methods = [m for m in self.HTTP_VERBS if hasattr(self, '_process_' + m)]
raise MethodNotAllowed(valid_methods)
return method()
def _checkCSRF(self):
# Check referer for POST requests. We do it here so we can properly use indico's error handling
if Config.getInstance().getCSRFLevel() < 3 or request.method != 'POST':
return
referer = request.referrer
# allow empty - otherwise we might lock out paranoid users blocking referers
if not referer:
return
# valid http referer
if referer.startswith(Config.getInstance().getBaseURL()):
return
# valid https referer - if https is enabled
base_secure = Config.getInstance().getBaseSecureURL()
if base_secure and referer.startswith(base_secure):
return
raise BadRefererError('This operation is not allowed from an external referer.')
def _processGeneralError(self, e):
"""Treats general errors occured during the process of a RH. """
Logger.get('requestHandler').info('Request %s finished with: "%s"' % (request, e))
p=errors.WPGenericError(self)
return p.display()
def _processUnexpectedError(self, e):
"""Unexpected errors"""
Logger.get('requestHandler').exception('Request %s failed: "%s"' % (request, e))
p=errors.WPUnexpectedError(self)
return p.display()
def _processHostnameResolveError(self, e):
"""Unexpected errors"""
Logger.get('requestHandler').exception('Request %s failed: "%s"' % (request, e))
p=errors.WPHostnameResolveError(self)
return p.display()
def _processAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with AccessError: "%s"' % (request, e))
self._responseUtil.status = 403
p=errors.WPAccessError(self)
return p.display()
def _processKeyAccessError(self, e):
"""Treats access errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with KeyAccessError: "%s"' % (request, e))
# We are going to redirect to the page asking for access key
# and so it must be https if there is a BaseSecureURL. And that's
# why we set _tohttps to True.
self._tohttps = True
if self._checkHttpsRedirect():
return
p=errors.WPKeyAccessError(self)
return p.display()
def _processModificationError(self, e):
"""Treats modification errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with ModificationError: "%s"' % (request, e))
p=errors.WPModificationError(self)
return p.display()
def _processConferenceClosedError(self, e):
"""Treats access to modification pages for conferences when they are closed."""
p = WPConferenceModificationClosed( self, e._conf )
return p.display()
def _processTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with TimingError: "%s"' % (request, e))
p = errors.WPTimingError(self, e)
return p.display()
def _processNoReportError(self, e):
"""Process errors without reporting"""
Logger.get('requestHandler').info('Request %s finished with NoReportError: "%s"' % (request, e))
p=errors.WPNoReportError(self,e)
return p.display()
def _processNotFoundError(self, e):
"""Process not found error; uses NoReportError template"""
Logger.get('requestHandler').info('Request %s finished with NotFoundError: "%s"' % (request, e))
self._responseUtil.status = 404
p=errors.WPNoReportError(self,e)
return p.display()
def _processParentTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with ParentTimingError: "%s"' % (request, e))
p=errors.WPParentTimingError(self,e)
return p.display()
def _processEntryTimingError(self, e):
"""Treats timing errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with EntryTimingError: "%s"' % (request, e))
p=errors.WPEntryTimingError(self,e)
return p.display()
def _processFormValuesError(self, e):
"""Treats user input related errors occured during the process of a RH."""
Logger.get('requestHandler').info('Request %s finished with FormValuesError: "%s"' % (request, e))
p=errors.WPFormValuesError(self,e)
return p.display()
def _processHtmlScriptError(self, e):
Logger.get('requestHandler').info('Request %s finished with ProcessHtmlScriptError: "%s"' % (request, e))
p=errors.WPHtmlScriptError(self, escape(str(e)))
return p.display()
def _processRestrictedHTML(self, e):
Logger.get('requestHandler').info('Request %s finished with ProcessRestrictedHTMLError: "%s"' % (request, e))
p=errors.WPRestrictedHTML(self, escape(str(e)))
return p.display()
def process(self, params):
if request.method not in self.HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
profile = Config.getInstance().getProfile()
proffilename = ""
res = ""
MAX_RETRIES = 10
retry = MAX_RETRIES
textLog = []
self._startTime = datetime.now()
# clear the context
ContextManager.destroy()
ContextManager.set('currentRH', self)
#redirect to https if necessary
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
DBMgr.getInstance().startRequest()
self._startRequestSpecific2RH() # I.e. implemented by Room Booking request handlers
textLog.append("%s : Database request started" % (datetime.now() - self._startTime))
Logger.get('requestHandler').info('[pid=%s] Request %s started' % (
os.getpid(), request))
# notify components that the request has started
self._notify('requestStarted')
forcedConflicts = Config.getInstance().getForceConflicts()
try:
while retry>0:
if retry < MAX_RETRIES:
# notify components that the request is being retried
self._notify('requestRetry', MAX_RETRIES - retry)
try:
Logger.get('requestHandler').info('\t[pid=%s] from host %s' % (os.getpid(), request.remote_addr))
try:
# clear the fossile cache at the start of each request
fossilize.clearCache()
# delete all queued emails
GenericMailer.flushQueue(False)
# clear the existing redis pipeline
if self._redisPipeline:
self._redisPipeline.reset()
DBMgr.getInstance().sync()
# keep a link to the web session in the access wrapper
# this is used for checking access/modification key existence
# in the user session
self._aw.setIP(request.remote_addr)
self._setSessionUser()
self._setLang(params)
if self._getAuth():
if self._getUser():
Logger.get('requestHandler').info('Request %s identified with user %s (%s)' % (
request, self._getUser().getFullName(), self._getUser().getId()))
if not self._tohttps and Config.getInstance().getAuthenticatedEnforceSecure():
self._tohttps = True
if self._checkHttpsRedirect():
return self._responseUtil.make_redirect()
self._checkCSRF()
self._reqParams = copy.copy(params)
self._checkParams(self._reqParams)
func = getattr(self, '_checkParams_' + request.method, None)
if func:
func()
self._checkProtection()
func = getattr(self, '_checkProtection_' + request.method, None)
if func:
func()
security.Sanitization.sanitizationCheck(self._target,
self._reqParams,
self._aw, self._doNotSanitizeFields)
if self._doProcess:
if profile:
import profile, pstats
proffilename = os.path.join(Config.getInstance().getTempDir(), "stone%s.prof" % str(random.random()))
result = [None]
profile.runctx("result[0] = self._process()", globals(), locals(), proffilename)
res = result[0]
else:
res = self._process()
# notify components that the request has finished
self._notify('requestFinished')
# Raise a conflict error if enabled. This allows detecting conflict-related issues easily.
if retry > (MAX_RETRIES - forcedConflicts):
raise ConflictError
self._endRequestSpecific2RH( True ) # I.e. implemented by Room Booking request handlers
DBMgr.getInstance().endRequest( True )
Logger.get('requestHandler').info('Request %s successful' % request)
#request succesfull, now, doing tas that must be done only once
try:
GenericMailer.flushQueue(True) # send emails
self._deleteTempFiles()
except:
Logger.get('mail').exception('Mail sending operation failed')
pass
# execute redis pipeline if we have one
if self._redisPipeline:
try:
self._redisPipeline.execute()
except RedisError:
Logger.get('redis').exception('Could not execute pipeline')
break
except MaKaCError, e:
#DBMgr.getInstance().endRequest(False)
res = self._processError(e)
except (ConflictError, POSKeyError):
import traceback
# only log conflict if it wasn't forced
if retry <= (MAX_RETRIES - forcedConflicts):
Logger.get('requestHandler').warning('Conflict in Database! (Request %s)\n%s' % (request, traceback.format_exc()))
self._abortSpecific2RH()
DBMgr.getInstance().abort()
retry -= 1
continue
except ClientDisconnected:
Logger.get('requestHandler').warning('Client Disconnected! (Request %s)' % request)
self._abortSpecific2RH()
DBMgr.getInstance().abort()
retry -= 1
time.sleep(10-retry)
continue
except KeyAccessError, e:
#Key Access error treatment
res = self._processKeyAccessError( e )
self._endRequestSpecific2RH( False )
DBMgr.getInstance().endRequest(False)
except AccessError, e:
#Access error treatment
res = self._processAccessError( e )
self._endRequestSpecific2RH( False )
DBMgr.getInstance().endRequest(False)
def __init__(self):
self.commit = True
self._responseUtil = ResponseUtil()
def handler(prefix, path):
path = posixpath.join('/', prefix, path)
clearCache() # init fossil cache
logger = Logger.get('httpapi')
if request.method == 'POST':
# Convert POST data to a query string
queryParams = [(key, [x.encode('utf-8') for x in values])
for key, values in request.form.iterlists()]
query = urllib.urlencode(queryParams, doseq=1)
# we only need/keep multiple values so we can properly validate the signature.
# the legacy code below expects a dict with just the first value.
# if you write a new api endpoint that needs multiple values get them from
# ``request.values.getlist()`` directly
queryParams = {key: values[0] for key, values in queryParams}
else:
# Parse the actual query string
queryParams = dict((key, value.encode('utf-8'))
for key, value in request.args.iteritems())
query = request.query_string
apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None)
cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'],
'no') == 'yes'
signature = get_query_parameter(queryParams, ['signature'])
timestamp = get_query_parameter(queryParams, ['timestamp'],
0,
integer=True)
noCache = get_query_parameter(queryParams, ['nc', 'nocache'],
'no') == 'yes'
pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes'
onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'],
'no') == 'yes'
onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'],
'no') == 'yes'
scope = 'read:legacy_api' if request.method == 'GET' else 'write:legacy_api'
try:
oauth_valid, oauth_request = oauth.verify_request([scope])
if not oauth_valid and oauth_request and oauth_request.error_message != 'Bearer token not found.':
raise BadRequest('OAuth error: {}'.format(
oauth_request.error_message))
elif g.get(
'received_oauth_token'
) and oauth_request.error_message == 'Bearer token not found.':
raise BadRequest('OAuth error: Invalid token')
except ValueError:
# XXX: Dirty hack to workaround a bug in flask-oauthlib that causes it
# not to properly urlencode request query strings
# Related issue (https://github.com/lepture/flask-oauthlib/issues/213)
oauth_valid = False
# Get our handler function and its argument and response type
hook, dformat = HTTPAPIHook.parseRequest(path, queryParams)
if hook is None or dformat is None:
raise NotFound
# Disable caching if we are not just retrieving data (or the hook requires it)
if request.method == 'POST' or hook.NO_CACHE:
noCache = True
ak = error = result = None
ts = int(time.time())
typeMap = {}
responseUtil = ResponseUtil()
is_response = False
try:
used_session = None
if cookieAuth:
used_session = session
if not used_session.user: # ignore guest sessions
used_session = None
if apiKey or oauth_valid or not used_session:
if not oauth_valid:
# Validate the API key (and its signature)
ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp,
path, query)
if enforceOnlyPublic:
onlyPublic = True
# Create an access wrapper for the API key's user
user = ak.user if ak and not onlyPublic else None
else: # Access Token (OAuth)
at = load_token(oauth_request.access_token.access_token)
user = at.user if at and not onlyPublic else None
# Get rid of API key in cache key if we did not impersonate a user
if ak and user is None:
cacheKey = normalizeQuery(
path,
query,
remove=('_', 'ak', 'apiKey', 'signature', 'timestamp',
'nc', 'nocache', 'oa', 'onlyauthed'))
else:
cacheKey = normalizeQuery(path,
query,
remove=('_', 'signature',
'timestamp', 'nc', 'nocache',
'oa', 'onlyauthed'))
if signature:
# in case the request was signed, store the result under a different key
cacheKey = 'signed_' + cacheKey
else:
# We authenticated using a session cookie.
token = request.headers.get(
'X-CSRF-Token', get_query_parameter(queryParams,
['csrftoken']))
if used_session.csrf_protected and used_session.csrf_token != token:
raise HTTPAPIError('Invalid CSRF token', 403)
user = used_session.user if not onlyPublic else None
userPrefix = 'user-{}_'.format(used_session.user.id)
cacheKey = userPrefix + normalizeQuery(
path,
query,
remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa',
'onlyauthed', 'csrftoken'))
# Bail out if the user requires authentication but is not authenticated
if onlyAuthed and not user:
raise HTTPAPIError('Not authenticated', 403)
addToCache = not hook.NO_CACHE
cache = GenericCache('HTTPAPI')
cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey)
if not noCache:
obj = cache.get(cacheKey)
if obj is not None:
result, extra, ts, complete, typeMap = obj
addToCache = False
if result is None:
g.current_api_user = user
# Perform the actual exporting
res = hook(user)
if isinstance(res, current_app.response_class):
addToCache = False
is_response = True
result, extra, complete, typeMap = res, {}, True, {}
elif isinstance(res, tuple) and len(res) == 4:
result, extra, complete, typeMap = res
else:
result, extra, complete, typeMap = res, {}, True, {}
if result is not None and addToCache:
ttl = api_settings.get('cache_ttl')
if ttl > 0:
cache.set(cacheKey, (result, extra, ts, complete, typeMap),
ttl)
except HTTPAPIError as e:
error = e
if e.getCode():
responseUtil.status = e.getCode()
if responseUtil.status == 405:
responseUtil.headers[
'Allow'] = 'GET' if request.method == 'POST' else 'POST'
if result is None and error is None:
# TODO: usage page
raise NotFound
else:
if ak and error is None:
# Commit only if there was an API key and no error
norm_path, norm_query = normalizeQuery(path,
query,
remove=('signature',
'timestamp'),
separate=True)
uri = to_unicode('?'.join(filter(None, (norm_path, norm_query))))
ak.register_used(request.remote_addr, uri, not onlyPublic)
db.session.commit()
else:
# No need to commit stuff if we didn't use an API key (nothing was written)
# XXX do we even need this?
db.session.rollback()
# Log successful POST api requests
if error is None and request.method == 'POST':
logger.info('API request: %s?%s', path, query)
if is_response:
return result
serializer = Serializer.create(dformat,
query_params=queryParams,
pretty=pretty,
typeMap=typeMap,
**hook.serializer_args)
if error:
if not serializer.schemaless:
# if our serializer has a specific schema (HTML, ICAL, etc...)
# use JSON, since it is universal
serializer = Serializer.create('json')
result = fossilize(error)
else:
if serializer.encapsulate:
result = fossilize(
HTTPAPIResult(result, path, query, ts, complete, extra),
IHTTPAPIExportResultFossil)
del result['_fossil']
try:
data = serializer(result)
serializer.set_headers(responseUtil)
return responseUtil.make_response(data)
except:
logger.exception('Serialization error in request %s?%s', path,
query)
raise