def run(self): self._prepare() self._logger.info('Running task {}.. (delay: {})'.format(self._task.id, self._executionDelay)) try: for i, retry in enumerate(transaction.attempts(self._config.task_max_tries)): with retry: self._logger.info('Task attempt #{}'.format(i)) if i > 0: self._prepare_retry() try: self._process_task() break except ConflictError: transaction.abort() except ClientDisconnected: self._logger.warning("Retrying for the {}th time in {} secs..".format(i + 1, seconds)) transaction.abort() time.sleep(i * 10) except TaskDelayed, e: self._logger.info("{} delayed by {} seconds".format(self._task, e.delaySeconds)) self._delayed = True self._executionDelay = 0 time.sleep(e.delaySeconds) flush_after_commit_queue(True) GenericMailer.flushQueue(True)
def invokeMethod(self, method, params, req): MAX_RETRIES = 10 # clear the context ContextManager.destroy() DBMgr.getInstance().startRequest() # room booking database _startRequestSpecific2RH() # notify components that the request has started self._notify('requestStarted', req) forcedConflicts = Config.getInstance().getForceConflicts() retry = MAX_RETRIES try: while retry > 0: if retry < MAX_RETRIES: # notify components that the request is being retried self._notify('requestRetry', req, MAX_RETRIES - retry) try: # delete all queued emails GenericMailer.flushQueue(False) DBMgr.getInstance().sync() try: result = processRequest(method, copy.deepcopy(params), req) except MaKaC.errors.NoReportError, e: raise NoReportError(e.getMsg()) rh = ContextManager.get('currentRH') # notify components that the request has ended self._notify('requestFinished', req) # Raise a conflict error if enabled. This allows detecting conflict-related issues easily. if retry > (MAX_RETRIES - forcedConflicts): raise ConflictError _endRequestSpecific2RH( True ) DBMgr.getInstance().endRequest(True) GenericMailer.flushQueue(True) # send emails if rh._redisPipeline: try: rh._redisPipeline.execute() except RedisError: Logger.get('redis').exception('Could not execute pipeline') break except ConflictError: _abortSpecific2RH() DBMgr.getInstance().abort() retry -= 1 continue except ClientDisconnected: _abortSpecific2RH() DBMgr.getInstance().abort() retry -= 1 time.sleep(MAX_RETRIES - retry) continue
def _invokeMethodRetryBefore(self): # clear/init fossil cache fossilize.clearCache() # clear after-commit queue flush_after_commit_queue(False) # delete all queued emails GenericMailer.flushQueue(False) DBMgr.getInstance().sync()
def _invokeMethodSuccess(self): rh = ContextManager.get('currentRH') flush_after_commit_queue(True) # run after-commit functions GenericMailer.flushQueue(True) # send emails if rh._redisPipeline: try: rh._redisPipeline.execute() except RedisError: Logger.get('redis').exception('Could not execute pipeline')
def invokeMethod(self, method, params, req): MAX_RETRIES = 10 # clear the context ContextManager.destroy() DBMgr.getInstance().startRequest() # room booking database _startRequestSpecific2RH() # notify components that the request has started self._notify('requestStarted', req) forcedConflicts = Config.getInstance().getForceConflicts() retry = MAX_RETRIES try: while retry > 0: if retry < MAX_RETRIES: # notify components that the request is being retried self._notify('requestRetry', req, MAX_RETRIES - retry) try: # delete all queued emails GenericMailer.flushQueue(False) DBMgr.getInstance().sync() try: result = processRequest(method, copy.deepcopy(params), req) except MaKaC.errors.NoReportError, e: raise NoReportError(e.getMsg()) # notify components that the request has ended self._notify('requestFinished', req) # Raise a conflict error if enabled. This allows detecting conflict-related issues easily. if retry > (MAX_RETRIES - forcedConflicts): raise ConflictError _endRequestSpecific2RH(True) DBMgr.getInstance().endRequest(True) GenericMailer.flushQueue(True) # send emails break except ConflictError: _abortSpecific2RH() DBMgr.getInstance().abort() retry -= 1 continue except ClientDisconnected: _abortSpecific2RH() DBMgr.getInstance().abort() retry -= 1 time.sleep(MAX_RETRIES - retry) continue
def __call__(self, aw): """Perform the actual exporting""" if self.HTTP_POST != (request.method == 'POST'): raise HTTPAPIError( 'This action requires %s' % ('POST' if self.HTTP_POST else 'GET'), 405) if not self.GUEST_ALLOWED and not aw.getUser(): raise HTTPAPIError('Guest access to this resource is forbidden.', 403) method_name = self._getMethodName() func = getattr(self, method_name, None) extra_func = getattr(self, method_name + '_extra', None) if not func: raise NotImplementedError(method_name) if not self.COMMIT: is_response, resultList, complete, extra = self._perform( aw, func, extra_func) else: dbi = DBMgr.getInstance() try: for i, retry in enumerate(transaction.attempts(10)): with retry: if i > 0: dbi.abort() flush_after_commit_queue(False) GenericMailer.flushQueue(False) dbi.sync() try: is_response, resultList, complete, extra = self._perform( aw, func, extra_func) transaction.commit() flush_after_commit_queue(True) GenericMailer.flushQueue(True) break except ConflictError: transaction.abort() except ClientDisconnected: transaction.abort() time.sleep(i * 5) else: raise HTTPAPIError( 'An unresolvable database conflict has occured', 500) except Exception: transaction.abort() raise if is_response: return resultList return resultList, extra, complete, self.SERIALIZER_TYPE_MAP
def __call__(self, aw): """Perform the actual exporting""" if self.HTTP_POST != (request.method == 'POST'): raise HTTPAPIError('This action requires %s' % ('POST' if self.HTTP_POST else 'GET'), 405) if not self.GUEST_ALLOWED and not aw.getUser(): raise HTTPAPIError('Guest access to this resource is forbidden.', 403) method_name = self._getMethodName() func = getattr(self, method_name, None) extra_func = getattr(self, method_name + '_extra', None) if not func: raise NotImplementedError(method_name) if not self.COMMIT: is_response, resultList, complete, extra = self._perform(aw, func, extra_func) else: dbi = DBMgr.getInstance() try: for i, retry in enumerate(transaction.attempts(10)): with retry: if i > 0: dbi.abort() flush_after_commit_queue(False) GenericMailer.flushQueue(False) dbi.sync() try: is_response, resultList, complete, extra = self._perform(aw, func, extra_func) transaction.commit() flush_after_commit_queue(True) GenericMailer.flushQueue(True) break except ConflictError: transaction.abort() except ClientDisconnected: transaction.abort() time.sleep(i * 5) else: raise HTTPAPIError('An unresolvable database conflict has occured', 500) except Exception: transaction.abort() raise if is_response: return resultList return resultList, extra, complete, self.SERIALIZER_TYPE_MAP
def _prepare_retry(self): self._dbi.abort() self._dbi.sync() flush_after_commit_queue(False) GenericMailer.flushQueue(False) self._task.plugLogger(self._logger)
def run(self): self._prepare() self._logger.info('Running task %s.. (delay: %s)' % (self._task.id, self._executionDelay)) # We will try to run the task TASK_MAX_RETRIES # times and if it continues failing we abort it i = 0 # RoomBooking forces us to connect to its own DB if needed # Maybe we should add some extension point here that lets plugins # define their own actions on DB connect/disconnect/commit/abort # potentially conflict-prone (!) with self._dbi.transaction(sync=True): with self._rbdbi.transaction(): self._task.prepare() delayed = False while i < self._config.task_max_tries: # Otherwise objects modified in indico itself are not updated here if hasattr(self._rbdbi, 'sync'): self._rbdbi.sync() try: if i > 0: self._dbi.abort() # delete all queued emails GenericMailer.flushQueue(False) # restore logger self._task.plugLogger(self._logger) with self._dbi.transaction(): with self._rbdbi.transaction(): self._logger.info('Task cycle %d' % i) i = i + 1 # clear the fossile cache at the start of each task fossilize.clearCache() self._task.start(self._executionDelay) break except TaskDelayed, e: nextRunIn = e.delaySeconds self._executionDelay = 0 delayed = True self._logger.info("%s delayed by %d seconds" % (self._task, e.delaySeconds)) base.TimeSource.get().sleep(nextRunIn) except Exception, e: self._logger.exception("%s failed with exception '%s'. " % \ (self._task, e)) if i < self._config.task_max_tries: nextRunIn = i * 10 # secs self._logger.warning("Retrying for the %dth time in %d secs.." % \ (i + 1, nextRunIn)) # if i is still low enough, we sleep progressively more # so that if the error is caused by concurrency we don't make # the problem worse by hammering the server. base.TimeSource.get().sleep(nextRunIn)
except Exception, e: self._logger.exception("%s failed with exception '%s'. " % \ (self._task, e)) if i < self._config.task_max_tries: nextRunIn = i * 10 # secs self._logger.warning("Retrying for the %dth time in %d secs.." % \ (i + 1, nextRunIn)) # if i is still low enough, we sleep progressively more # so that if the error is caused by concurrency we don't make # the problem worse by hammering the server. base.TimeSource.get().sleep(nextRunIn) GenericMailer.flushQueue(True) self._logger.info('Ended on: %s' % self._task.endedOn) # task successfully finished if self._task.endedOn: with self._dbi.transaction(): self._setResult(True) if i > (1 + int(delayed)): self._logger.warning("%s failed %d times before " "finishing correctly" % (self._task, i - int(delayed) - 1)) else: with self._dbi.transaction(): self._setResult(False) self._logger.error("%s failed too many (%d) times. " "Aborting its execution.." % (self._task, i))
def run(self): self._prepare() self._logger.info('Running task %s.. (delay: %s)' % (self._task.id, self._executionDelay)) # We will try to run the task TASK_MAX_RETRIES # times and if it continues failing we abort it i = 0 # RoomBooking forces us to connect to its own DB if needed # Maybe we should add some extension point here that lets plugins # define their own actions on DB connect/disconnect/commit/abort # potentially conflict-prone (!) with self._dbi.transaction(sync=True): with self._rbdbi.transaction(): self._task.prepare() delayed = False while i < self._config.task_max_tries: # Otherwise objects modified in indico itself are not updated here if hasattr(self._rbdbi, 'sync'): self._rbdbi.sync() try: if i > 0: self._dbi.abort() GenericMailer.flushQueue(False) # restore logger self._task.plugLogger(self._logger) with self._dbi.transaction(): with self._rbdbi.transaction(): self._logger.info('Task cycle %d' % i) i = i + 1 self._task.start(self._executionDelay) break except TaskDelayed, e: nextRunIn = e.delaySeconds self._executionDelay = 0 delayed = True self._logger.info("%s delayed by %d seconds" % (self._task, e.delaySeconds)) base.TimeSource.get().sleep(nextRunIn) except Exception, e: self._logger.exception("%s failed with exception '%s'. " % \ (self._task, e)) if i < self._config.task_max_tries: nextRunIn = i * 10 # secs self._logger.warning("Retrying for the %dth time in %d secs.." % \ (i + 1, nextRunIn)) # if i is still low enough, we sleep progressively more # so that if the error is caused by concurrency we don't make # the problem worse by hammering the server. base.TimeSource.get().sleep(nextRunIn)
except Exception, e: self._logger.exception("%s failed with exception '%s'. " % \ (self._task, e)) if i < self._config.task_max_tries: nextRunIn = i * 10 # secs self._logger.warning("Retrying for the %dth time in %d secs.." % \ (i + 1, nextRunIn)) # if i is still low enough, we sleep progressively more # so that if the error is caused by concurrency we don't make # the problem worse by hammering the server. base.TimeSource.get().sleep(nextRunIn) GenericMailer.flushQueue(True) self._logger.info('Ended on: %s' % self._task.endedOn) # task successfully finished if self._task.endedOn: with self._dbi.transaction(): self._setResult(True) if i > (1 + int(delayed)): self._logger.warning("%s failed %d times before " "finishing correctly" % (self._task, i - int(delayed) - 1)) else: with self._dbi.transaction(): self._setResult(False) self._logger.error("%s failed too many (%d) times. "