def _addTaskError(task, error=None): if error == None: task.errors.append(log.getFailureMessage(Failure())) elif isinstance(error, Failure): task.errors.append(log.getFailureMessage(error)) elif isinstance(error, Exception): task.errors.append(log.getFailureMessage(Failure(error))) else: task.errors.append(error)
def _buildBody(sender, recipients, subject, msg, info=None, debug=None, failure=None, exception=None, documents=None): body = [msg] if info: body.append("Information:\n\n%s" % info) if debug: body.append("Additional Debug Info:\n\n%s" % debug) if exception: body.append("Exception Message: %s\n\nException Traceback:\n\n%s" % (log.getExceptionMessage(exception), log.getExceptionTraceback(exception))) if failure: body.append("Failure Message: %s\n\nFailure Traceback:\n%s" % (log.getFailureMessage(failure), log.getFailureTraceback(failure))) msg = MIMEMultipart() msg['Subject'] = subject msg['From'] = sender msg['To'] = recipients txt = MIMEText("\n\n\n".join(body)) msg.attach(txt) if documents: for doc in documents: mimeType = doc.getMimeType() mainType, subType = mimeType.split('/', 1) data = MIMEBase(mainType, subType) data.set_payload(doc.asString()) email.Encoders.encode_base64(data) data.add_header('Content-Disposition', 'attachment', filename=doc.label) msg.attach(data) return str(msg)
def __isOperationTerminated(self, failure, status, resultDef): if not self.isValid(): # Assume the objectives is fulfilled, resultDef.callback(self) return True if failure: if failure.check(DeadReferenceError): msg = ("Forced Stop/Delete of component '%s' aborted " "because the PB reference is dead" % self.getLabel()) self.warning("%s", msg) error = errors.OperationAbortedError(msg, cause=failure) resultDef.errback(error) return True if failure.check(ferrors.UnknownComponentError): self.debug("Forced Stop/Delete of component '%s' aborted " "because the component is unknown (already deleted ?)", self.label) resultDef.callback(self) return True if failure.check(PBConnectionLost): msg = ("Forced Stop/Delete of component '%s' aborted " "because the remote connection was lost" % self.label) self.warning("%s", msg) error = admerrs.OperationAbortedError(msg, cause=failure) resultDef.errback(error) return True status['last-message'] = log.getFailureMessage(failure) return False
def __ebInitializationFailed(self, failure): #FIXME: Better Error Handling self.log("%s initialization failed: %s", self.__class__.__name__, log.getFailureMessage(failure)) self._onInitFailed(failure) #Propagate failures return failure
def moveToDoneFailure(failure, otherDest): context.warning("Failed to move input file: %s", log.getFailureMessage(failure)) context.reporter.addError(failure) context.reporter.setFatalError(failure.getErrorMessage()) self._fireError(context, failure.getErrorMessage()) d = moveSource(otherDest, failure) d.addErrback(totalFailure, failure) return d
def __asyncForceKillFailed(self, failure, status, label, resultDef): if self.__isOperationTerminated(failure, status, resultDef): return if failure.check(admerrs.OrphanComponentError): # The component don't have worker self.__stopOrDelete(None, status, label, resultDef) return self.warning("Component '%s' killing failed: %s", label, log.getFailureMessage(failure)) self.__asyncRetryKillIfNeeded(False, status, label, resultDef)
def _unexpected_error(self, failure=None, task=None): if not failure: failure = Failure() log.notifyFailure(self, failure, "Unexpected error%s", (task and " during %s" % task) or "", cleanTraceback=True) m = messages.Error(_(failure.getErrorMessage()), debug=log.getFailureMessage(failure)) self.addMessage(m) return failure
def __ebParentAborted(self, failure): self._failure = failure self._active = False self.log("%s activation failed: %s", self.__class__.__name__, log.getFailureMessage(failure)) self._activeWaiters.fireErrbacks(failure) self._onAborted(failure) self._activeChildWaiters.fireErrbacks(failure) if self.parent: self.parent._childElementAborted()
def _abort(self, failure): """ Called when the element couldn't be added due to error. """ assert not self._triggered self._triggered = True if not isinstance(failure, Failure): failure = Failure(failure) self.log("Start %s abortion for %s", self.__class__.__name__, log.getFailureMessage(failure)) #Don't wait for parent when aborting self.__ebParentAborted(failure)
def __unexpectedError(self, failure=None, task=None): self._fireStatusChanged(TranscoderStatusEnum.unexpected_error) if not failure: failure = Failure() self.onJobError(failure.getErrorMessage()) log.notifyFailure(self, failure, "Unexpected error%s", (task and " during %s" % task) or "", cleanTraceback=True) m = messages.Error(T_(failure.getErrorMessage()), debug=log.getFailureMessage(failure)) self.addMessage(m) return failure
def __asyncForceDeleteFailed(self, failure, status, label, resultDef): if self.__isOperationTerminated(failure, status, resultDef): return status["delete-retries"] = status.setdefault("delete-retries", 0) + 1 if status["delete-retries"] > adminconsts.FORCED_DELETION_MAX_RETRY: # if deletion fail, theres nothing we can do, do we ? self.__deletionFailed(status, label, resultDef) return if failure.check(ferrors.BusyComponentError): # The component is buzy changing mood, # so wait mood change (with a larger timeout) d = self.waitMoodChange(adminconsts.FORCED_DELETION_BUZY_TIMEOUT) d.addBoth(self.__stopOrDelete, status, label, resultDef) return self.warning("Fail to delete component '%s': %s", label, log.getFailureMessage(failure)) self.__asyncForceDelete(None, status, label, resultDef)
def __ebJobFailed(self, failure): try: report = self._report if not failure.check(FlumotionError): m = messages.Error(T_(failure.getErrorMessage()), debug=log.getFailureMessage(failure)) self.addMessage(m) # FIXME: Very ugly, should not ask the job for this self._reportDefaultPath = self._job.getFailedReportPath() self.__syncReport(report) self.__deleteTempReport() self._fireStatusChanged(TranscoderStatusEnum.failed) self.__finalize(report, False) except Exception, e: log.notifyException(self, e, "Unexpected exception", cleanTraceback=True) self.__unexpectedError()
def _notifyDebug(self, msg, info=None, debug=None, failure=None, exception=None, documents=None): infoMsg = ["File Monitor Debug Notification: %s" % msg] debugMsg = [] if info: infoMsg.append("Information:\n\n%s" % info) if debug: debugMsg.append("Additional Debug Info:\n\n%s" % debug) if failure: debugMsg.append("Failure Message: %s\nFailure Traceback:\n%s" % (log.getFailureMessage(failure), log.getFailureTraceback(failure))) if exception: debugMsg.append("Exception Message: %s\n\nException Traceback:\n%s" % (log.getExceptionMessage(exception), log.getExceptionTraceback(exception))) m = messages.Warning(_("\n\n".join(infoMsg)), debug="\n\n".join(debugMsg)) self.addMessage(m)
def __asyncForceStopFailed(self, failure, status, label, resultDef): if self.__isOperationTerminated(failure, status, resultDef): return status["stop-retries"] = status.setdefault("stop-retries", 0) + 1 if status["stop-retries"] > adminconsts.FORCED_DELETION_MAX_RETRY: if (status.get("already_killed", False) or (not status.get("can_kill", True))): # if already killed or we are not allowed to kill, # theres nothing we can do, do we ? self.__deletionFailed(status, label, resultDef) return # If we already tried too much, kill the component status["kill-retries"] = status.setdefault("kill-retries", 0) + 1 if status["kill-retries"] > adminconsts.FORCED_DELETION_MAX_RETRY: # if kill fail, theres nothing we can do, do we ? self.__deletionFailed(status, label, resultDef) return d = self.kill() args = (status, label, resultDef) d.addCallbacks(self.__asyncRetryKillIfNeeded, self.__asyncForceKillFailed, callbackArgs=args, errbackArgs=args) return if failure.check(ferrors.BusyComponentError): # The component is buzy changing mood, # so wait mood change (with a larger timeout) d = self.waitMoodChange(adminconsts.FORCED_DELETION_BUZY_TIMEOUT) d.addBoth(self.__stopOrDelete, status, label, resultDef) return # FIXME: make flumotion raise a specific exception # when there is mood conflicts if failure.check(ferrors.ComponentError): #Maybe the component was already stopped ? if self.getMood() == moods.sleeping: self.__asyncForceDelete(None, status, label, resultDef) return # The component raised an error # so just log the error and try again self.warning("Fail to stop component '%s': %s", label, log.getFailureMessage(failure)) self.__asyncForceStop(None, status, label, resultDef)
def __ebAcknowledgeError(self, failure): try: self.warning("Transcoding acknowledge Error: %s", log.getFailureMessage(failure)) self._fireStatusChanged(TranscoderStatusEnum.failed) self.setMood(moods.sad) # FIXME: Very ugly, should not ask the job for this newReportPath = self._job.getFailedReportPath() if newReportPath != self._reportDefaultPath: self._reportDefaultPath = newReportPath self._fireTranscodingReport(self._reportDefaultPath) self.__syncReport(self._report) self.__terminate(self._report, self._status) return failure except Exception, e: log.notifyException(self, e, "Unexpected exception", cleanTraceback=True) self.__unexpectedError() # Reraise for the do_acknowledge call to return the failure raise e
def __ebComponentNotHappy(self, failure, compPxy, workerName): self.warning("Admin task '%s' component '%s' " "fail to become happy on worker '%s': %s", self.label, compPxy.getName(), workerName, log.getFailureMessage(failure)) self.__abortComponentStartup(compPxy)
def _ebNotificationFailed(failure, kind): log.warning("%s notification failed: %s", kind, log.getFailureMessage(failure), category=adminconsts.NOTIFIER_LOG_CATEGORY)
def __ebPostMailFailed(self, failure, activCtx): retryLeftDesc = self.__getRetriesLeftDesc(activCtx) self.debug("Mail post '%s' failed (%s): %s", activCtx.label, retryLeftDesc, log.getFailureMessage(failure)) self.__retryNotification(activCtx)
def __ebUIStateRetrievalFailed(self, failure): self._retrievingUIState = False self.warning("Component '%s' fail to retrieve its UI state: %s", self.label, log.getFailureMessage(failure)) self._uiState.fail(failure)
def __ebSQLExecFailed(self, failure, activity): retryLeftDesc = self.__getRetriesLeftDesc(activity) self.debug("SQL execution '%s' failed (%s): %s", activity.label, retryLeftDesc, log.getFailureMessage(failure)) self.__retryNotification(activity)
def move_failed(failure, src, dest): msg = ("Fail to move file '%s' to '%s': %s" % (src, dest, log.getFailureMessage(failure))) self.warning("%s", msg) raise TranscoderError(msg, cause=failure)
def totalFailure(failure, oldFailure): context.warning("Failed to move input file: %s", log.getFailureMessage(failure)) context.reporter.addError(failure) self._fireError(context, failure.getErrorMessage()) return oldFailure or failure