def _addTaskError(task, error=None):
if error == None:
task.errors.append(log.getFailureMessage(Failure()))
elif isinstance(error, Failure):
task.errors.append(log.getFailureMessage(error))
elif isinstance(error, Exception):
task.errors.append(log.getFailureMessage(Failure(error)))
else:
task.errors.append(error)
def _buildBody(sender, recipients, subject, msg, info=None, debug=None,
failure=None, exception=None, documents=None):
body = [msg]
if info:
body.append("Information:\n\n%s" % info)
if debug:
body.append("Additional Debug Info:\n\n%s" % debug)
if exception:
body.append("Exception Message: %s\n\nException Traceback:\n\n%s"
% (log.getExceptionMessage(exception),
log.getExceptionTraceback(exception)))
if failure:
body.append("Failure Message: %s\n\nFailure Traceback:\n%s"
% (log.getFailureMessage(failure),
log.getFailureTraceback(failure)))
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = recipients
txt = MIMEText("\n\n\n".join(body))
msg.attach(txt)
if documents:
for doc in documents:
mimeType = doc.getMimeType()
mainType, subType = mimeType.split('/', 1)
data = MIMEBase(mainType, subType)
data.set_payload(doc.asString())
email.Encoders.encode_base64(data)
data.add_header('Content-Disposition', 'attachment',
filename=doc.label)
msg.attach(data)
return str(msg)
def __isOperationTerminated(self, failure, status, resultDef):
if not self.isValid():
# Assume the objectives is fulfilled,
resultDef.callback(self)
return True
if failure:
if failure.check(DeadReferenceError):
msg = ("Forced Stop/Delete of component '%s' aborted "
"because the PB reference is dead" % self.getLabel())
self.warning("%s", msg)
error = errors.OperationAbortedError(msg, cause=failure)
resultDef.errback(error)
return True
if failure.check(ferrors.UnknownComponentError):
self.debug("Forced Stop/Delete of component '%s' aborted "
"because the component is unknown (already deleted ?)",
self.label)
resultDef.callback(self)
return True
if failure.check(PBConnectionLost):
msg = ("Forced Stop/Delete of component '%s' aborted "
"because the remote connection was lost" % self.label)
self.warning("%s", msg)
error = admerrs.OperationAbortedError(msg, cause=failure)
resultDef.errback(error)
return True
status['last-message'] = log.getFailureMessage(failure)
return False
def __ebInitializationFailed(self, failure):
#FIXME: Better Error Handling
self.log("%s initialization failed: %s",
self.__class__.__name__,
log.getFailureMessage(failure))
self._onInitFailed(failure)
#Propagate failures
return failure
def moveToDoneFailure(failure, otherDest):
context.warning("Failed to move input file: %s",
log.getFailureMessage(failure))
context.reporter.addError(failure)
context.reporter.setFatalError(failure.getErrorMessage())
self._fireError(context, failure.getErrorMessage())
d = moveSource(otherDest, failure)
d.addErrback(totalFailure, failure)
return d
def __asyncForceKillFailed(self, failure, status, label, resultDef):
if self.__isOperationTerminated(failure, status, resultDef): return
if failure.check(admerrs.OrphanComponentError):
# The component don't have worker
self.__stopOrDelete(None, status,
label, resultDef)
return
self.warning("Component '%s' killing failed: %s",
label, log.getFailureMessage(failure))
self.__asyncRetryKillIfNeeded(False, status, label, resultDef)
def _unexpected_error(self, failure=None, task=None):
if not failure:
failure = Failure()
log.notifyFailure(self, failure,
"Unexpected error%s",
(task and " during %s" % task) or "",
cleanTraceback=True)
m = messages.Error(_(failure.getErrorMessage()),
debug=log.getFailureMessage(failure))
self.addMessage(m)
return failure
def __ebParentAborted(self, failure):
self._failure = failure
self._active = False
self.log("%s activation failed: %s",
self.__class__.__name__,
log.getFailureMessage(failure))
self._activeWaiters.fireErrbacks(failure)
self._onAborted(failure)
self._activeChildWaiters.fireErrbacks(failure)
if self.parent:
self.parent._childElementAborted()
def _abort(self, failure):
"""
Called when the element couldn't be added due to error.
"""
assert not self._triggered
self._triggered = True
if not isinstance(failure, Failure):
failure = Failure(failure)
self.log("Start %s abortion for %s",
self.__class__.__name__,
log.getFailureMessage(failure))
#Don't wait for parent when aborting
self.__ebParentAborted(failure)
def __unexpectedError(self, failure=None, task=None):
self._fireStatusChanged(TranscoderStatusEnum.unexpected_error)
if not failure:
failure = Failure()
self.onJobError(failure.getErrorMessage())
log.notifyFailure(self, failure,
"Unexpected error%s",
(task and " during %s" % task) or "",
cleanTraceback=True)
m = messages.Error(T_(failure.getErrorMessage()),
debug=log.getFailureMessage(failure))
self.addMessage(m)
return failure
def __asyncForceDeleteFailed(self, failure, status, label, resultDef):
if self.__isOperationTerminated(failure, status, resultDef): return
status["delete-retries"] = status.setdefault("delete-retries", 0) + 1
if status["delete-retries"] > adminconsts.FORCED_DELETION_MAX_RETRY:
# if deletion fail, theres nothing we can do, do we ?
self.__deletionFailed(status, label, resultDef)
return
if failure.check(ferrors.BusyComponentError):
# The component is buzy changing mood,
# so wait mood change (with a larger timeout)
d = self.waitMoodChange(adminconsts.FORCED_DELETION_BUZY_TIMEOUT)
d.addBoth(self.__stopOrDelete, status, label, resultDef)
return
self.warning("Fail to delete component '%s': %s",
label, log.getFailureMessage(failure))
self.__asyncForceDelete(None, status, label, resultDef)
def __ebJobFailed(self, failure):
try:
report = self._report
if not failure.check(FlumotionError):
m = messages.Error(T_(failure.getErrorMessage()),
debug=log.getFailureMessage(failure))
self.addMessage(m)
# FIXME: Very ugly, should not ask the job for this
self._reportDefaultPath = self._job.getFailedReportPath()
self.__syncReport(report)
self.__deleteTempReport()
self._fireStatusChanged(TranscoderStatusEnum.failed)
self.__finalize(report, False)
except Exception, e:
log.notifyException(self, e,
"Unexpected exception",
cleanTraceback=True)
self.__unexpectedError()
def _notifyDebug(self, msg, info=None, debug=None, failure=None,
exception=None, documents=None):
infoMsg = ["File Monitor Debug Notification: %s" % msg]
debugMsg = []
if info:
infoMsg.append("Information:\n\n%s" % info)
if debug:
debugMsg.append("Additional Debug Info:\n\n%s" % debug)
if failure:
debugMsg.append("Failure Message: %s\nFailure Traceback:\n%s"
% (log.getFailureMessage(failure),
log.getFailureTraceback(failure)))
if exception:
debugMsg.append("Exception Message: %s\n\nException Traceback:\n%s"
% (log.getExceptionMessage(exception),
log.getExceptionTraceback(exception)))
m = messages.Warning(_("\n\n".join(infoMsg)),
debug="\n\n".join(debugMsg))
self.addMessage(m)
def __asyncForceStopFailed(self, failure, status, label, resultDef):
if self.__isOperationTerminated(failure, status, resultDef): return
status["stop-retries"] = status.setdefault("stop-retries", 0) + 1
if status["stop-retries"] > adminconsts.FORCED_DELETION_MAX_RETRY:
if (status.get("already_killed", False)
or (not status.get("can_kill", True))):
# if already killed or we are not allowed to kill,
# theres nothing we can do, do we ?
self.__deletionFailed(status, label, resultDef)
return
# If we already tried too much, kill the component
status["kill-retries"] = status.setdefault("kill-retries", 0) + 1
if status["kill-retries"] > adminconsts.FORCED_DELETION_MAX_RETRY:
# if kill fail, theres nothing we can do, do we ?
self.__deletionFailed(status, label, resultDef)
return
d = self.kill()
args = (status, label, resultDef)
d.addCallbacks(self.__asyncRetryKillIfNeeded,
self.__asyncForceKillFailed,
callbackArgs=args, errbackArgs=args)
return
if failure.check(ferrors.BusyComponentError):
# The component is buzy changing mood,
# so wait mood change (with a larger timeout)
d = self.waitMoodChange(adminconsts.FORCED_DELETION_BUZY_TIMEOUT)
d.addBoth(self.__stopOrDelete, status, label, resultDef)
return
# FIXME: make flumotion raise a specific exception
# when there is mood conflicts
if failure.check(ferrors.ComponentError):
#Maybe the component was already stopped ?
if self.getMood() == moods.sleeping:
self.__asyncForceDelete(None, status,
label, resultDef)
return
# The component raised an error
# so just log the error and try again
self.warning("Fail to stop component '%s': %s",
label, log.getFailureMessage(failure))
self.__asyncForceStop(None, status, label, resultDef)
def __ebAcknowledgeError(self, failure):
try:
self.warning("Transcoding acknowledge Error: %s",
log.getFailureMessage(failure))
self._fireStatusChanged(TranscoderStatusEnum.failed)
self.setMood(moods.sad)
# FIXME: Very ugly, should not ask the job for this
newReportPath = self._job.getFailedReportPath()
if newReportPath != self._reportDefaultPath:
self._reportDefaultPath = newReportPath
self._fireTranscodingReport(self._reportDefaultPath)
self.__syncReport(self._report)
self.__terminate(self._report, self._status)
return failure
except Exception, e:
log.notifyException(self, e,
"Unexpected exception",
cleanTraceback=True)
self.__unexpectedError()
# Reraise for the do_acknowledge call to return the failure
raise e
def __ebComponentNotHappy(self, failure, compPxy, workerName):
self.warning("Admin task '%s' component '%s' "
"fail to become happy on worker '%s': %s",
self.label, compPxy.getName(), workerName,
log.getFailureMessage(failure))
self.__abortComponentStartup(compPxy)
def _ebNotificationFailed(failure, kind):
log.warning("%s notification failed: %s", kind,
log.getFailureMessage(failure),
category=adminconsts.NOTIFIER_LOG_CATEGORY)
def __ebPostMailFailed(self, failure, activCtx):
retryLeftDesc = self.__getRetriesLeftDesc(activCtx)
self.debug("Mail post '%s' failed (%s): %s",
activCtx.label, retryLeftDesc,
log.getFailureMessage(failure))
self.__retryNotification(activCtx)
def __ebUIStateRetrievalFailed(self, failure):
self._retrievingUIState = False
self.warning("Component '%s' fail to retrieve its UI state: %s",
self.label, log.getFailureMessage(failure))
self._uiState.fail(failure)
def __ebSQLExecFailed(self, failure, activity):
retryLeftDesc = self.__getRetriesLeftDesc(activity)
self.debug("SQL execution '%s' failed (%s): %s",
activity.label, retryLeftDesc,
log.getFailureMessage(failure))
self.__retryNotification(activity)
def move_failed(failure, src, dest):
msg = ("Fail to move file '%s' to '%s': %s"
% (src, dest, log.getFailureMessage(failure)))
self.warning("%s", msg)
raise TranscoderError(msg, cause=failure)
def totalFailure(failure, oldFailure):
context.warning("Failed to move input file: %s",
log.getFailureMessage(failure))
context.reporter.addError(failure)
self._fireError(context, failure.getErrorMessage())
return oldFailure or failure
