def shell_cmd(verbose, with_req_context):
try:
from IPython.terminal.ipapp import TerminalIPythonApp
except ImportError:
click.echo(
cformat(
'%{red!}You need to `pip install ipython` to use the Indico shell'
))
sys.exit(1)
current_app.config['REPL'] = True # disables e.g. memoize_request
request_stats_request_started()
context, info = _make_shell_context()
banner = cformat('%{yellow!}Indico v{} is ready for your commands').format(
indico.__version__)
if verbose:
banner = '\n'.join(info + ['', banner])
ctx = current_app.make_shell_context()
ctx.update(context)
clearCache()
stack = ExitStack()
if with_req_context:
stack.enter_context(
current_app.test_request_context(base_url=config.BASE_URL))
with stack:
ipython_app = TerminalIPythonApp.instance(user_ns=ctx,
display_banner=False)
ipython_app.initialize(argv=[])
ipython_app.shell.show_banner(banner)
ipython_app.start()
def invokeMethod(self, method, params):
result = None
try:
fossilize.clearCache()
GenericMailer.flushQueue(False)
try:
try:
result = processRequest(method, copy.deepcopy(params))
signals.after_process.send()
except NoReportError as e:
raise ServiceNoReportError(e.getMessage())
except NoReportIndicoError as e:
raise ServiceNoReportError(e.getMessage(),
title=_("Error"))
db.session.commit()
except DatabaseError:
handle_sqlalchemy_database_error()
GenericMailer.flushQueue(True)
except Exception as e:
db.session.rollback()
if isinstance(e, CausedError):
raise
elif isinstance(e, ConstraintViolated):
raise ProcessError, ('ERR-P0', e.message), sys.exc_info()[2]
else:
raise ProcessError('ERR-P0', 'Error processing method.')
return result
def _invokeMethodRetryBefore(self):
# clear/init fossil cache
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
DBMgr.getInstance().sync()
def _invokeMethodRetryBefore(self):
# clear/init fossil cache
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
DBMgr.getInstance().sync()
def process():
clearCache()
payload = request.json
Logger.get('rpc').info('json rpc request. request: %r', payload)
rv = {}
if 'id' in payload:
rv['id'] = payload['id']
rv['result'] = invoke_method(str(payload['method']),
payload.get('params', []))
return jsonify(rv)
def run(self, no_ipython, use_bpython, quiet):
context = self.get_context()
if not quiet:
self.banner = '\n'.join(self._info + ['', self.banner])
if use_bpython:
# bpython does not support escape sequences :(
# https://github.com/bpython/bpython/issues/396
self.banner = strip_ansi(self.banner)
clearCache()
with context['dbi'].global_connection():
super(IndicoShell, self).run(no_ipython or use_bpython, not use_bpython)
def setUp(self):
if os.environ.get('INDICO_SKIP_SLOW_TESTS') == '1':
if self._slow:
self.skipTest('Slow tests disabled')
testMethod = getattr(self, self._testMethodName)
if getattr(testMethod, '_slow', False):
self.skipTest('Slow tests disabled')
setLocale('en_GB')
Logger.removeHandler('smtp')
clearCache() # init/clear fossil cache
self._configFeatures(self)
self._benchmark.start()
def run(self, no_ipython, use_bpython, quiet):
current_app.config['REPL'] = True # disables e.g. memoize_request
context = self.get_context()
if not quiet:
self.banner = '\n'.join(self._info + ['', self.banner])
if use_bpython:
# bpython does not support escape sequences :(
# https://github.com/bpython/bpython/issues/396
self.banner = strip_ansi(self.banner)
clearCache()
with context['dbi'].global_connection():
self.run_shell(no_ipython or use_bpython, not use_bpython, quiet)
def run(self, no_ipython, use_bpython, quiet):
context = self.get_context()
if not quiet:
self.banner = '\n'.join(self._info + ['', self.banner])
if use_bpython:
# bpython does not support escape sequences :(
# https://github.com/bpython/bpython/issues/396
self.banner = strip_ansi(self.banner)
clearCache()
with context['dbi'].global_connection():
super(IndicoShell, self).run(no_ipython or use_bpython,
not use_bpython)
def setUp(self):
if os.environ.get('INDICO_SKIP_SLOW_TESTS') == '1':
if self._slow:
self.skipTest('Slow tests disabled')
testMethod = getattr(self, self._testMethodName)
if getattr(testMethod, '_slow', False):
self.skipTest('Slow tests disabled')
setLocale('en_GB')
Logger.removeHandler('smtp')
clearCache() # init/clear fossil cache
self._configFeatures(self)
self._benchmark.start()
def invoke_method(method, params):
result = None
fossilize.clearCache()
GenericMailer.flushQueue(False)
try:
result = _process_request(method, copy.deepcopy(params))
signals.after_process.send()
db.session.commit()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error()
GenericMailer.flushQueue(True)
return result
def invoke_method(method, params):
result = None
fossilize.clearCache()
GenericMailer.flushQueue(False)
try:
result = _process_request(method, copy.deepcopy(params))
signals.after_process.send()
db.session.commit()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error()
GenericMailer.flushQueue(True)
return result
def invoke_method(method, params):
result = None
fossilize.clearCache()
init_email_queue()
try:
result = _process_request(method, copy.deepcopy(params))
signals.after_process.send()
db.session.commit()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error()
except Exception:
db.session.rollback()
raise
flush_email_queue()
return result
def invoke_method(method, params):
result = None
fossilize.clearCache()
init_email_queue()
try:
result = _process_request(method, copy.deepcopy(params))
signals.after_process.send()
db.session.commit()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error()
except Exception:
db.session.rollback()
raise
flush_email_queue()
return result
def process(self):
if request.method not in HTTP_VERBS:
# Just to be sure that we don't get some crappy http verb we don't expect
raise BadRequest
res = ''
g.rh = self
sentry_set_tags({'rh': self.__class__.__name__})
if self.EVENT_FEATURE is not None:
self._check_event_feature()
logger.info('%s %s [IP=%s] [PID=%s] [UID=%r]', request.method,
request.relative_url, request.remote_addr, os.getpid(),
session.get('_user_id'))
try:
fossilize.clearCache()
init_email_queue()
self._check_csrf()
res = self._do_process()
signals.after_process.send()
if self.commit:
db.session.commit()
flush_email_queue()
else:
db.session.rollback()
except DatabaseError:
db.session.rollback()
handle_sqlalchemy_database_error(
) # this will re-raise an exception
except Exception:
# rollback to avoid errors as rendering the error page
# within the indico layout may trigger an auto-flush
db.session.rollback()
raise
logger.debug('Request successful')
if res is None:
# flask doesn't accept None but we might be returning it in some places...
res = ''
response = current_app.make_response(res)
if self.DENY_FRAMES:
response.headers['X-Frame-Options'] = 'DENY'
return response
def __call__(s, *args, **kwargs):
stack = ExitStack()
stack.enter_context(self.flask_app.app_context())
if getattr(s, 'request_context', False):
stack.enter_context(self.flask_app.test_request_context(base_url=config.BASE_URL))
args = _CelerySAWrapper.unwrap_args(args)
kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs)
plugin = getattr(s, 'plugin', s.request.get('indico_plugin'))
if isinstance(plugin, basestring):
plugin_name = plugin
plugin = plugin_engine.get_plugin(plugin)
if plugin is None:
stack.close()
raise ValueError('Plugin not active: ' + plugin_name)
stack.enter_context(plugin_context(plugin))
clearCache()
with stack:
request_stats_request_started()
return super(IndicoTask, s).__call__(*args, **kwargs)
def __call__(s, *args, **kwargs):
stack = ExitStack()
stack.enter_context(self.flask_app.app_context())
stack.enter_context(DBMgr.getInstance().global_connection())
if getattr(s, 'request_context', False):
stack.enter_context(self.flask_app.test_request_context())
args = _CelerySAWrapper.unwrap_args(args)
kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs)
plugin = getattr(s, 'plugin', kwargs.pop('__current_plugin__', None))
if isinstance(plugin, basestring):
plugin_name = plugin
plugin = plugin_engine.get_plugin(plugin)
if plugin is None:
stack.close()
raise ValueError('Plugin not active: ' + plugin_name)
stack.enter_context(plugin_context(plugin))
clearCache()
with stack:
return super(IndicoTask, s).__call__(*args, **kwargs)
def __call__(s, *args, **kwargs):
stack = ExitStack()
stack.enter_context(self.flask_app.app_context())
if getattr(s, 'request_context', False):
stack.enter_context(
self.flask_app.test_request_context(
base_url=config.BASE_URL))
args = _CelerySAWrapper.unwrap_args(args)
kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs)
plugin = getattr(s, 'plugin', s.request.get('indico_plugin'))
if isinstance(plugin, basestring):
plugin_name = plugin
plugin = plugin_engine.get_plugin(plugin)
if plugin is None:
stack.close()
raise ValueError('Plugin not active: ' + plugin_name)
stack.enter_context(plugin_context(plugin))
clearCache()
with stack:
request_stats_request_started()
return super(IndicoTask, s).__call__(*args, **kwargs)
def __call__(s, *args, **kwargs):
stack = ExitStack()
stack.enter_context(self.flask_app.app_context())
stack.enter_context(DBMgr.getInstance().global_connection())
if getattr(s, 'request_context', False):
stack.enter_context(
self.flask_app.test_request_context(
base_url=Config.getInstance().getBaseURL()))
args = _CelerySAWrapper.unwrap_args(args)
kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs)
plugin = getattr(s, 'plugin',
kwargs.pop('__current_plugin__', None))
if isinstance(plugin, basestring):
plugin_name = plugin
plugin = plugin_engine.get_plugin(plugin)
if plugin is None:
stack.close()
raise ValueError('Plugin not active: ' + plugin_name)
stack.enter_context(plugin_context(plugin))
clearCache()
with stack:
return super(IndicoTask, s).__call__(*args, **kwargs)
def shell_cmd(verbose, with_req_context):
try:
from IPython.terminal.ipapp import TerminalIPythonApp
except ImportError:
click.echo(cformat('%{red!}You need to `pip install ipython` to use the Indico shell'))
sys.exit(1)
current_app.config['REPL'] = True # disables e.g. memoize_request
request_stats_request_started()
context, info = _make_shell_context()
banner = cformat('%{yellow!}Indico v{} is ready for your commands').format(indico.__version__)
if verbose:
banner = '\n'.join(info + ['', banner])
ctx = current_app.make_shell_context()
ctx.update(context)
clearCache()
stack = ExitStack()
if with_req_context:
stack.enter_context(current_app.test_request_context(base_url=config.BASE_URL))
with stack:
ipython_app = TerminalIPythonApp.instance(user_ns=ctx, display_banner=False)
ipython_app.initialize(argv=[])
ipython_app.shell.show_banner(banner)
ipython_app.start()
def _process_task(self):
with self._dbi.transaction():
with self._app.app_context():
fossilize.clearCache()
self._task.start(self._executionDelay)
transaction.commit()
def _process_task(self):
with self._dbi.transaction():
with self._app.app_context():
fossilize.clearCache()
self._task.start(self._executionDelay)
transaction.commit()
def run(self):
self._prepare()
self._logger.info('Running task %s.. (delay: %s)' % (self._task.id, self._executionDelay))
# We will try to run the task TASK_MAX_RETRIES
# times and if it continues failing we abort it
i = 0
# RoomBooking forces us to connect to its own DB if needed
# Maybe we should add some extension point here that lets plugins
# define their own actions on DB connect/disconnect/commit/abort
# potentially conflict-prone (!)
with self._dbi.transaction(sync=True):
with self._rbdbi.transaction():
self._task.prepare()
delayed = False
while i < self._config.task_max_tries:
# Otherwise objects modified in indico itself are not updated here
if hasattr(self._rbdbi, 'sync'):
self._rbdbi.sync()
try:
if i > 0:
self._dbi.abort()
# delete all queued emails
GenericMailer.flushQueue(False)
# restore logger
self._task.plugLogger(self._logger)
with self._dbi.transaction():
with self._rbdbi.transaction():
self._logger.info('Task cycle %d' % i)
i = i + 1
# clear the fossile cache at the start of each task
fossilize.clearCache()
self._task.start(self._executionDelay)
break
except TaskDelayed, e:
nextRunIn = e.delaySeconds
self._executionDelay = 0
delayed = True
self._logger.info("%s delayed by %d seconds" % (self._task, e.delaySeconds))
base.TimeSource.get().sleep(nextRunIn)
except Exception, e:
self._logger.exception("%s failed with exception '%s'. " % \
(self._task, e))
if i < self._config.task_max_tries:
nextRunIn = i * 10 # secs
self._logger.warning("Retrying for the %dth time in %d secs.." % \
(i + 1, nextRunIn))
# if i is still low enough, we sleep progressively more
# so that if the error is caused by concurrency we don't make
# the problem worse by hammering the server.
base.TimeSource.get().sleep(nextRunIn)
def invokeMethod(self, method, params):
MAX_RETRIES = 10
# clear the context
ContextManager.destroy()
DBMgr.getInstance().startRequest()
# room booking database
_startRequestSpecific2RH()
# notify components that the request has started
self._notify("requestStarted")
forcedConflicts = Config.getInstance().getForceConflicts()
retry = MAX_RETRIES
try:
while retry > 0:
if retry < MAX_RETRIES:
# notify components that the request is being retried
self._notify("requestRetry", MAX_RETRIES - retry)
# clear/init fossil cache
fossilize.clearCache()
try:
# delete all queued emails
GenericMailer.flushQueue(False)
DBMgr.getInstance().sync()
try:
result = processRequest(method, copy.deepcopy(params))
except MaKaC.errors.NoReportError, e:
raise NoReportError(e.getMessage())
rh = ContextManager.get("currentRH")
# notify components that the request has ended
self._notify("requestFinished")
# Raise a conflict error if enabled. This allows detecting conflict-related issues easily.
if retry > (MAX_RETRIES - forcedConflicts):
raise ConflictError
_endRequestSpecific2RH(True)
DBMgr.getInstance().endRequest(True)
GenericMailer.flushQueue(True) # send emails
if rh._redisPipeline:
try:
rh._redisPipeline.execute()
except RedisError:
Logger.get("redis").exception("Could not execute pipeline")
break
except ConflictError:
_abortSpecific2RH()
DBMgr.getInstance().abort()
retry -= 1
continue
except ClientDisconnected:
_abortSpecific2RH()
DBMgr.getInstance().abort()
retry -= 1
time.sleep(MAX_RETRIES - retry)
continue
def handler(prefix, path):
path = posixpath.join('/', prefix, path)
clearCache() # init fossil cache
logger = Logger.get('httpapi')
if request.method == 'POST':
# Convert POST data to a query string
queryParams = [(key, [x.encode('utf-8') for x in values])
for key, values in request.form.iterlists()]
query = urllib.urlencode(queryParams, doseq=1)
# we only need/keep multiple values so we can properly validate the signature.
# the legacy code below expects a dict with just the first value.
# if you write a new api endpoint that needs multiple values get them from
# ``request.values.getlist()`` directly
queryParams = {key: values[0] for key, values in queryParams}
else:
# Parse the actual query string
queryParams = dict((key, value.encode('utf-8'))
for key, value in request.args.iteritems())
query = request.query_string
apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None)
cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'],
'no') == 'yes'
signature = get_query_parameter(queryParams, ['signature'])
timestamp = get_query_parameter(queryParams, ['timestamp'],
0,
integer=True)
noCache = get_query_parameter(queryParams, ['nc', 'nocache'],
'no') == 'yes'
pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes'
onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'],
'no') == 'yes'
onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'],
'no') == 'yes'
scope = 'read:legacy_api' if request.method == 'GET' else 'write:legacy_api'
if not request.headers.get('Authorization',
'').lower().startswith('basic '):
try:
oauth_valid, oauth_request = oauth.verify_request([scope])
if not oauth_valid and oauth_request and oauth_request.error_message != 'Bearer token not found.':
raise BadRequest('OAuth error: {}'.format(
oauth_request.error_message))
elif g.get(
'received_oauth_token'
) and oauth_request.error_message == 'Bearer token not found.':
raise BadRequest('OAuth error: Invalid token')
except ValueError:
# XXX: Dirty hack to workaround a bug in flask-oauthlib that causes it
# not to properly urlencode request query strings
# Related issue (https://github.com/lepture/flask-oauthlib/issues/213)
oauth_valid = False
else:
oauth_valid = False
# Get our handler function and its argument and response type
hook, dformat = HTTPAPIHook.parseRequest(path, queryParams)
if hook is None or dformat is None:
raise NotFound
# Disable caching if we are not just retrieving data (or the hook requires it)
if request.method == 'POST' or hook.NO_CACHE:
noCache = True
ak = error = result = None
ts = int(time.time())
typeMap = {}
status_code = None
is_response = False
try:
used_session = None
if cookieAuth:
used_session = session
if not used_session.user: # ignore guest sessions
used_session = None
if apiKey or oauth_valid or not used_session:
if not oauth_valid:
# Validate the API key (and its signature)
ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp,
path, query)
if enforceOnlyPublic:
onlyPublic = True
# Create an access wrapper for the API key's user
user = ak.user if ak and not onlyPublic else None
else: # Access Token (OAuth)
at = load_token(oauth_request.access_token.access_token)
user = at.user if at and not onlyPublic else None
# Get rid of API key in cache key if we did not impersonate a user
if ak and user is None:
cacheKey = normalizeQuery(
path,
query,
remove=('_', 'ak', 'apiKey', 'signature', 'timestamp',
'nc', 'nocache', 'oa', 'onlyauthed'))
else:
cacheKey = normalizeQuery(path,
query,
remove=('_', 'signature',
'timestamp', 'nc', 'nocache',
'oa', 'onlyauthed'))
if signature:
# in case the request was signed, store the result under a different key
cacheKey = 'signed_' + cacheKey
else:
# We authenticated using a session cookie.
token = request.headers.get(
'X-CSRF-Token', get_query_parameter(queryParams,
['csrftoken']))
if used_session.csrf_protected and used_session.csrf_token != token:
raise HTTPAPIError('Invalid CSRF token', 403)
user = used_session.user if not onlyPublic else None
userPrefix = 'user-{}_'.format(used_session.user.id)
cacheKey = userPrefix + normalizeQuery(
path,
query,
remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa',
'onlyauthed', 'csrftoken'))
# Bail out if the user requires authentication but is not authenticated
if onlyAuthed and not user:
raise HTTPAPIError('Not authenticated', 403)
addToCache = not hook.NO_CACHE
cache = GenericCache('HTTPAPI')
cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey)
if not noCache:
obj = cache.get(cacheKey)
if obj is not None:
result, extra, ts, complete, typeMap = obj
addToCache = False
if result is None:
g.current_api_user = user
# Perform the actual exporting
res = hook(user)
if isinstance(res, current_app.response_class):
addToCache = False
is_response = True
result, extra, complete, typeMap = res, {}, True, {}
elif isinstance(res, tuple) and len(res) == 4:
result, extra, complete, typeMap = res
else:
result, extra, complete, typeMap = res, {}, True, {}
if result is not None and addToCache:
ttl = api_settings.get('cache_ttl')
if ttl > 0:
cache.set(cacheKey, (result, extra, ts, complete, typeMap),
ttl)
except HTTPAPIError as e:
error = e
if e.getCode():
status_code = e.getCode()
if result is None and error is None:
# TODO: usage page
raise NotFound
else:
if ak and error is None:
# Commit only if there was an API key and no error
norm_path, norm_query = normalizeQuery(path,
query,
remove=('signature',
'timestamp'),
separate=True)
uri = to_unicode('?'.join(filter(None, (norm_path, norm_query))))
ak.register_used(request.remote_addr, uri, not onlyPublic)
db.session.commit()
else:
# No need to commit stuff if we didn't use an API key (nothing was written)
# XXX do we even need this?
db.session.rollback()
# Log successful POST api requests
if error is None and request.method == 'POST':
logger.info('API request: %s?%s', path, query)
if is_response:
return result
serializer = Serializer.create(dformat,
query_params=queryParams,
pretty=pretty,
typeMap=typeMap,
**hook.serializer_args)
if error:
if not serializer.schemaless:
# if our serializer has a specific schema (HTML, ICAL, etc...)
# use JSON, since it is universal
serializer = Serializer.create('json')
result = fossilize(error)
else:
if serializer.encapsulate:
result = fossilize(
HTTPAPIResult(result, path, query, ts, complete, extra),
IHTTPAPIExportResultFossil)
del result['_fossil']
try:
data = serializer(result)
response = current_app.make_response(data)
content_type = serializer.get_response_content_type()
if content_type:
response.content_type = content_type
if status_code:
response.status_code = status_code
return response
except Exception:
logger.exception('Serialization error in request %s?%s', path,
query)
raise
def setUp(self):
setLocale('en_GB')
Logger.removeHandler('smtp')
clearCache() # init/clear fossil cache
self._configFeatures(self)
