def shell_cmd(verbose, with_req_context): try: from IPython.terminal.ipapp import TerminalIPythonApp except ImportError: click.echo( cformat( '%{red!}You need to `pip install ipython` to use the Indico shell' )) sys.exit(1) current_app.config['REPL'] = True # disables e.g. memoize_request request_stats_request_started() context, info = _make_shell_context() banner = cformat('%{yellow!}Indico v{} is ready for your commands').format( indico.__version__) if verbose: banner = '\n'.join(info + ['', banner]) ctx = current_app.make_shell_context() ctx.update(context) clearCache() stack = ExitStack() if with_req_context: stack.enter_context( current_app.test_request_context(base_url=config.BASE_URL)) with stack: ipython_app = TerminalIPythonApp.instance(user_ns=ctx, display_banner=False) ipython_app.initialize(argv=[]) ipython_app.shell.show_banner(banner) ipython_app.start()
def invokeMethod(self, method, params): result = None try: fossilize.clearCache() GenericMailer.flushQueue(False) try: try: result = processRequest(method, copy.deepcopy(params)) signals.after_process.send() except NoReportError as e: raise ServiceNoReportError(e.getMessage()) except NoReportIndicoError as e: raise ServiceNoReportError(e.getMessage(), title=_("Error")) db.session.commit() except DatabaseError: handle_sqlalchemy_database_error() GenericMailer.flushQueue(True) except Exception as e: db.session.rollback() if isinstance(e, CausedError): raise elif isinstance(e, ConstraintViolated): raise ProcessError, ('ERR-P0', e.message), sys.exc_info()[2] else: raise ProcessError('ERR-P0', 'Error processing method.') return result
def _invokeMethodRetryBefore(self): # clear/init fossil cache fossilize.clearCache() # clear after-commit queue flush_after_commit_queue(False) # delete all queued emails GenericMailer.flushQueue(False) DBMgr.getInstance().sync()
def process(): clearCache() payload = request.json Logger.get('rpc').info('json rpc request. request: %r', payload) rv = {} if 'id' in payload: rv['id'] = payload['id'] rv['result'] = invoke_method(str(payload['method']), payload.get('params', [])) return jsonify(rv)
def run(self, no_ipython, use_bpython, quiet): context = self.get_context() if not quiet: self.banner = '\n'.join(self._info + ['', self.banner]) if use_bpython: # bpython does not support escape sequences :( # https://github.com/bpython/bpython/issues/396 self.banner = strip_ansi(self.banner) clearCache() with context['dbi'].global_connection(): super(IndicoShell, self).run(no_ipython or use_bpython, not use_bpython)
def setUp(self): if os.environ.get('INDICO_SKIP_SLOW_TESTS') == '1': if self._slow: self.skipTest('Slow tests disabled') testMethod = getattr(self, self._testMethodName) if getattr(testMethod, '_slow', False): self.skipTest('Slow tests disabled') setLocale('en_GB') Logger.removeHandler('smtp') clearCache() # init/clear fossil cache self._configFeatures(self) self._benchmark.start()
def run(self, no_ipython, use_bpython, quiet): current_app.config['REPL'] = True # disables e.g. memoize_request context = self.get_context() if not quiet: self.banner = '\n'.join(self._info + ['', self.banner]) if use_bpython: # bpython does not support escape sequences :( # https://github.com/bpython/bpython/issues/396 self.banner = strip_ansi(self.banner) clearCache() with context['dbi'].global_connection(): self.run_shell(no_ipython or use_bpython, not use_bpython, quiet)
def invoke_method(method, params): result = None fossilize.clearCache() GenericMailer.flushQueue(False) try: result = _process_request(method, copy.deepcopy(params)) signals.after_process.send() db.session.commit() except DatabaseError: db.session.rollback() handle_sqlalchemy_database_error() GenericMailer.flushQueue(True) return result
def invoke_method(method, params): result = None fossilize.clearCache() init_email_queue() try: result = _process_request(method, copy.deepcopy(params)) signals.after_process.send() db.session.commit() except DatabaseError: db.session.rollback() handle_sqlalchemy_database_error() except Exception: db.session.rollback() raise flush_email_queue() return result
def process(self): if request.method not in HTTP_VERBS: # Just to be sure that we don't get some crappy http verb we don't expect raise BadRequest res = '' g.rh = self sentry_set_tags({'rh': self.__class__.__name__}) if self.EVENT_FEATURE is not None: self._check_event_feature() logger.info('%s %s [IP=%s] [PID=%s] [UID=%r]', request.method, request.relative_url, request.remote_addr, os.getpid(), session.get('_user_id')) try: fossilize.clearCache() init_email_queue() self._check_csrf() res = self._do_process() signals.after_process.send() if self.commit: db.session.commit() flush_email_queue() else: db.session.rollback() except DatabaseError: db.session.rollback() handle_sqlalchemy_database_error( ) # this will re-raise an exception except Exception: # rollback to avoid errors as rendering the error page # within the indico layout may trigger an auto-flush db.session.rollback() raise logger.debug('Request successful') if res is None: # flask doesn't accept None but we might be returning it in some places... res = '' response = current_app.make_response(res) if self.DENY_FRAMES: response.headers['X-Frame-Options'] = 'DENY' return response
def __call__(s, *args, **kwargs): stack = ExitStack() stack.enter_context(self.flask_app.app_context()) if getattr(s, 'request_context', False): stack.enter_context(self.flask_app.test_request_context(base_url=config.BASE_URL)) args = _CelerySAWrapper.unwrap_args(args) kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs) plugin = getattr(s, 'plugin', s.request.get('indico_plugin')) if isinstance(plugin, basestring): plugin_name = plugin plugin = plugin_engine.get_plugin(plugin) if plugin is None: stack.close() raise ValueError('Plugin not active: ' + plugin_name) stack.enter_context(plugin_context(plugin)) clearCache() with stack: request_stats_request_started() return super(IndicoTask, s).__call__(*args, **kwargs)
def __call__(s, *args, **kwargs): stack = ExitStack() stack.enter_context(self.flask_app.app_context()) stack.enter_context(DBMgr.getInstance().global_connection()) if getattr(s, 'request_context', False): stack.enter_context(self.flask_app.test_request_context()) args = _CelerySAWrapper.unwrap_args(args) kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs) plugin = getattr(s, 'plugin', kwargs.pop('__current_plugin__', None)) if isinstance(plugin, basestring): plugin_name = plugin plugin = plugin_engine.get_plugin(plugin) if plugin is None: stack.close() raise ValueError('Plugin not active: ' + plugin_name) stack.enter_context(plugin_context(plugin)) clearCache() with stack: return super(IndicoTask, s).__call__(*args, **kwargs)
def __call__(s, *args, **kwargs): stack = ExitStack() stack.enter_context(self.flask_app.app_context()) if getattr(s, 'request_context', False): stack.enter_context( self.flask_app.test_request_context( base_url=config.BASE_URL)) args = _CelerySAWrapper.unwrap_args(args) kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs) plugin = getattr(s, 'plugin', s.request.get('indico_plugin')) if isinstance(plugin, basestring): plugin_name = plugin plugin = plugin_engine.get_plugin(plugin) if plugin is None: stack.close() raise ValueError('Plugin not active: ' + plugin_name) stack.enter_context(plugin_context(plugin)) clearCache() with stack: request_stats_request_started() return super(IndicoTask, s).__call__(*args, **kwargs)
def __call__(s, *args, **kwargs): stack = ExitStack() stack.enter_context(self.flask_app.app_context()) stack.enter_context(DBMgr.getInstance().global_connection()) if getattr(s, 'request_context', False): stack.enter_context( self.flask_app.test_request_context( base_url=Config.getInstance().getBaseURL())) args = _CelerySAWrapper.unwrap_args(args) kwargs = _CelerySAWrapper.unwrap_kwargs(kwargs) plugin = getattr(s, 'plugin', kwargs.pop('__current_plugin__', None)) if isinstance(plugin, basestring): plugin_name = plugin plugin = plugin_engine.get_plugin(plugin) if plugin is None: stack.close() raise ValueError('Plugin not active: ' + plugin_name) stack.enter_context(plugin_context(plugin)) clearCache() with stack: return super(IndicoTask, s).__call__(*args, **kwargs)
def shell_cmd(verbose, with_req_context): try: from IPython.terminal.ipapp import TerminalIPythonApp except ImportError: click.echo(cformat('%{red!}You need to `pip install ipython` to use the Indico shell')) sys.exit(1) current_app.config['REPL'] = True # disables e.g. memoize_request request_stats_request_started() context, info = _make_shell_context() banner = cformat('%{yellow!}Indico v{} is ready for your commands').format(indico.__version__) if verbose: banner = '\n'.join(info + ['', banner]) ctx = current_app.make_shell_context() ctx.update(context) clearCache() stack = ExitStack() if with_req_context: stack.enter_context(current_app.test_request_context(base_url=config.BASE_URL)) with stack: ipython_app = TerminalIPythonApp.instance(user_ns=ctx, display_banner=False) ipython_app.initialize(argv=[]) ipython_app.shell.show_banner(banner) ipython_app.start()
def _process_task(self): with self._dbi.transaction(): with self._app.app_context(): fossilize.clearCache() self._task.start(self._executionDelay) transaction.commit()
def run(self): self._prepare() self._logger.info('Running task %s.. (delay: %s)' % (self._task.id, self._executionDelay)) # We will try to run the task TASK_MAX_RETRIES # times and if it continues failing we abort it i = 0 # RoomBooking forces us to connect to its own DB if needed # Maybe we should add some extension point here that lets plugins # define their own actions on DB connect/disconnect/commit/abort # potentially conflict-prone (!) with self._dbi.transaction(sync=True): with self._rbdbi.transaction(): self._task.prepare() delayed = False while i < self._config.task_max_tries: # Otherwise objects modified in indico itself are not updated here if hasattr(self._rbdbi, 'sync'): self._rbdbi.sync() try: if i > 0: self._dbi.abort() # delete all queued emails GenericMailer.flushQueue(False) # restore logger self._task.plugLogger(self._logger) with self._dbi.transaction(): with self._rbdbi.transaction(): self._logger.info('Task cycle %d' % i) i = i + 1 # clear the fossile cache at the start of each task fossilize.clearCache() self._task.start(self._executionDelay) break except TaskDelayed, e: nextRunIn = e.delaySeconds self._executionDelay = 0 delayed = True self._logger.info("%s delayed by %d seconds" % (self._task, e.delaySeconds)) base.TimeSource.get().sleep(nextRunIn) except Exception, e: self._logger.exception("%s failed with exception '%s'. " % \ (self._task, e)) if i < self._config.task_max_tries: nextRunIn = i * 10 # secs self._logger.warning("Retrying for the %dth time in %d secs.." % \ (i + 1, nextRunIn)) # if i is still low enough, we sleep progressively more # so that if the error is caused by concurrency we don't make # the problem worse by hammering the server. base.TimeSource.get().sleep(nextRunIn)
def invokeMethod(self, method, params): MAX_RETRIES = 10 # clear the context ContextManager.destroy() DBMgr.getInstance().startRequest() # room booking database _startRequestSpecific2RH() # notify components that the request has started self._notify("requestStarted") forcedConflicts = Config.getInstance().getForceConflicts() retry = MAX_RETRIES try: while retry > 0: if retry < MAX_RETRIES: # notify components that the request is being retried self._notify("requestRetry", MAX_RETRIES - retry) # clear/init fossil cache fossilize.clearCache() try: # delete all queued emails GenericMailer.flushQueue(False) DBMgr.getInstance().sync() try: result = processRequest(method, copy.deepcopy(params)) except MaKaC.errors.NoReportError, e: raise NoReportError(e.getMessage()) rh = ContextManager.get("currentRH") # notify components that the request has ended self._notify("requestFinished") # Raise a conflict error if enabled. This allows detecting conflict-related issues easily. if retry > (MAX_RETRIES - forcedConflicts): raise ConflictError _endRequestSpecific2RH(True) DBMgr.getInstance().endRequest(True) GenericMailer.flushQueue(True) # send emails if rh._redisPipeline: try: rh._redisPipeline.execute() except RedisError: Logger.get("redis").exception("Could not execute pipeline") break except ConflictError: _abortSpecific2RH() DBMgr.getInstance().abort() retry -= 1 continue except ClientDisconnected: _abortSpecific2RH() DBMgr.getInstance().abort() retry -= 1 time.sleep(MAX_RETRIES - retry) continue
def handler(prefix, path): path = posixpath.join('/', prefix, path) clearCache() # init fossil cache logger = Logger.get('httpapi') if request.method == 'POST': # Convert POST data to a query string queryParams = [(key, [x.encode('utf-8') for x in values]) for key, values in request.form.iterlists()] query = urllib.urlencode(queryParams, doseq=1) # we only need/keep multiple values so we can properly validate the signature. # the legacy code below expects a dict with just the first value. # if you write a new api endpoint that needs multiple values get them from # ``request.values.getlist()`` directly queryParams = {key: values[0] for key, values in queryParams} else: # Parse the actual query string queryParams = dict((key, value.encode('utf-8')) for key, value in request.args.iteritems()) query = request.query_string apiKey = get_query_parameter(queryParams, ['ak', 'apikey'], None) cookieAuth = get_query_parameter(queryParams, ['ca', 'cookieauth'], 'no') == 'yes' signature = get_query_parameter(queryParams, ['signature']) timestamp = get_query_parameter(queryParams, ['timestamp'], 0, integer=True) noCache = get_query_parameter(queryParams, ['nc', 'nocache'], 'no') == 'yes' pretty = get_query_parameter(queryParams, ['p', 'pretty'], 'no') == 'yes' onlyPublic = get_query_parameter(queryParams, ['op', 'onlypublic'], 'no') == 'yes' onlyAuthed = get_query_parameter(queryParams, ['oa', 'onlyauthed'], 'no') == 'yes' scope = 'read:legacy_api' if request.method == 'GET' else 'write:legacy_api' if not request.headers.get('Authorization', '').lower().startswith('basic '): try: oauth_valid, oauth_request = oauth.verify_request([scope]) if not oauth_valid and oauth_request and oauth_request.error_message != 'Bearer token not found.': raise BadRequest('OAuth error: {}'.format( oauth_request.error_message)) elif g.get( 'received_oauth_token' ) and oauth_request.error_message == 'Bearer token not found.': raise BadRequest('OAuth error: Invalid token') except ValueError: # XXX: Dirty hack to workaround a bug in flask-oauthlib that causes it # not to properly urlencode request query strings # Related issue (https://github.com/lepture/flask-oauthlib/issues/213) oauth_valid = False else: oauth_valid = False # Get our handler function and its argument and response type hook, dformat = HTTPAPIHook.parseRequest(path, queryParams) if hook is None or dformat is None: raise NotFound # Disable caching if we are not just retrieving data (or the hook requires it) if request.method == 'POST' or hook.NO_CACHE: noCache = True ak = error = result = None ts = int(time.time()) typeMap = {} status_code = None is_response = False try: used_session = None if cookieAuth: used_session = session if not used_session.user: # ignore guest sessions used_session = None if apiKey or oauth_valid or not used_session: if not oauth_valid: # Validate the API key (and its signature) ak, enforceOnlyPublic = checkAK(apiKey, signature, timestamp, path, query) if enforceOnlyPublic: onlyPublic = True # Create an access wrapper for the API key's user user = ak.user if ak and not onlyPublic else None else: # Access Token (OAuth) at = load_token(oauth_request.access_token.access_token) user = at.user if at and not onlyPublic else None # Get rid of API key in cache key if we did not impersonate a user if ak and user is None: cacheKey = normalizeQuery( path, query, remove=('_', 'ak', 'apiKey', 'signature', 'timestamp', 'nc', 'nocache', 'oa', 'onlyauthed')) else: cacheKey = normalizeQuery(path, query, remove=('_', 'signature', 'timestamp', 'nc', 'nocache', 'oa', 'onlyauthed')) if signature: # in case the request was signed, store the result under a different key cacheKey = 'signed_' + cacheKey else: # We authenticated using a session cookie. token = request.headers.get( 'X-CSRF-Token', get_query_parameter(queryParams, ['csrftoken'])) if used_session.csrf_protected and used_session.csrf_token != token: raise HTTPAPIError('Invalid CSRF token', 403) user = used_session.user if not onlyPublic else None userPrefix = 'user-{}_'.format(used_session.user.id) cacheKey = userPrefix + normalizeQuery( path, query, remove=('_', 'nc', 'nocache', 'ca', 'cookieauth', 'oa', 'onlyauthed', 'csrftoken')) # Bail out if the user requires authentication but is not authenticated if onlyAuthed and not user: raise HTTPAPIError('Not authenticated', 403) addToCache = not hook.NO_CACHE cache = GenericCache('HTTPAPI') cacheKey = RE_REMOVE_EXTENSION.sub('', cacheKey) if not noCache: obj = cache.get(cacheKey) if obj is not None: result, extra, ts, complete, typeMap = obj addToCache = False if result is None: g.current_api_user = user # Perform the actual exporting res = hook(user) if isinstance(res, current_app.response_class): addToCache = False is_response = True result, extra, complete, typeMap = res, {}, True, {} elif isinstance(res, tuple) and len(res) == 4: result, extra, complete, typeMap = res else: result, extra, complete, typeMap = res, {}, True, {} if result is not None and addToCache: ttl = api_settings.get('cache_ttl') if ttl > 0: cache.set(cacheKey, (result, extra, ts, complete, typeMap), ttl) except HTTPAPIError as e: error = e if e.getCode(): status_code = e.getCode() if result is None and error is None: # TODO: usage page raise NotFound else: if ak and error is None: # Commit only if there was an API key and no error norm_path, norm_query = normalizeQuery(path, query, remove=('signature', 'timestamp'), separate=True) uri = to_unicode('?'.join(filter(None, (norm_path, norm_query)))) ak.register_used(request.remote_addr, uri, not onlyPublic) db.session.commit() else: # No need to commit stuff if we didn't use an API key (nothing was written) # XXX do we even need this? db.session.rollback() # Log successful POST api requests if error is None and request.method == 'POST': logger.info('API request: %s?%s', path, query) if is_response: return result serializer = Serializer.create(dformat, query_params=queryParams, pretty=pretty, typeMap=typeMap, **hook.serializer_args) if error: if not serializer.schemaless: # if our serializer has a specific schema (HTML, ICAL, etc...) # use JSON, since it is universal serializer = Serializer.create('json') result = fossilize(error) else: if serializer.encapsulate: result = fossilize( HTTPAPIResult(result, path, query, ts, complete, extra), IHTTPAPIExportResultFossil) del result['_fossil'] try: data = serializer(result) response = current_app.make_response(data) content_type = serializer.get_response_content_type() if content_type: response.content_type = content_type if status_code: response.status_code = status_code return response except Exception: logger.exception('Serialization error in request %s?%s', path, query) raise
def setUp(self): setLocale('en_GB') Logger.removeHandler('smtp') clearCache() # init/clear fossil cache self._configFeatures(self)