def __retrieveReportPath(self, virtPath, timeout, retry):
assert timeout is None, "Timeout not supported yet"
localPath = self.__localizeReportPath(virtPath)
if not localPath: return defer.succeed(None)
if not os.path.exists(localPath):
# Maybe the file has been moved, so we retry
if retry > 0:
d = self._callRemote("getReportPath")
d.addCallback(self.__updateReportPath)
d.addCallback(self.__retrieveReportPath, timeout, retry - 1)
return d
return defer.succeed(virtPath)
def pause(self, timeout=None):
if self._state == TaskStateEnum.terminated:
# If terminated, a task can be paused and resume silently
return defer.succeed(self)
if not (self._state in [TaskStateEnum.started]):
return defer.fail(admerrs.TranscoderError("Cannot pause %s task '%s'"
% (self._state.name,
self.label)))
self.log("Pausing admin task '%s'", self.label)
self._state = TaskStateEnum.paused
# No longer have associated worker
self._workerPxy = None
# No longer started
self._startWaiters.reset()
return defer.succeed(self)
def start(self, timeout=None):
for activCtx in self._activities:
left = activCtx.getTimeLeftBeforeRetry()
d = self.__startupNotification(activCtx, left)
d.addErrback(defer.resolveFailure)
del self._activities[:]
return defer.succeed(self)
def retrieveProfileNotifications(self, profData):
try:
assert isinstance(profData, ImmutableDataWrapper)
d = profData._getData()
assert isinstance(d, dataprops.ProfileData)
result = []
for req in d.notifyFailedRequests:
if req:
result.append(_createReqNotif(profData, False, req))
for req in d.notifyDoneRequests:
if req:
result.append(_createReqNotif(profData, True, req))
params = d.notifyParams
for sql in d.notifyFailedSQL:
if sql:
result.append(_createSQLNotif(profData, False, params, sql))
for sql in d.notifyDoneSQL:
if sql:
result.append(_createSQLNotif(profData, True, params, sql))
recipientsLine = d.notifyFailedMailRecipients
if recipientsLine:
recipients = utils.splitMailRecipients(recipientsLine)
notification = _createMailNotif(profData, False, recipients)
result.append(notification)
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve profile notifications data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def stop(self):
self._context.info("Stopping transcoding job")
self._setJobState(JobStateEnum.stopping)
self._stopping = True
# If already tell to stop, do nothing
if self._stoppingDefer:
return self._stoppingDefer
# If the job has been acknowledged, wait until completion
if self._runningState == RunningState.acknowledged:
self._stoppingDefer = defer.Deferred()
self._ackList.addBoth(self._stoppingDefer.callback)
else:
abortDefs = []
# if there is a transcoder, try to abort it
if self._transcoder:
abortDefs.append(self._transcoder.abort())
# if there is pending analysis, try to abort them
abortDefs.append(self._analyst.abort())
# if there is running process, try to abort them
for process in self._processes:
abortDefs.append(process.abort())
if len(abortDefs) > 0:
self._stoppingDefer = defer.DeferredList(abortDefs,
fireOnOneCallback=False,
fireOnOneErrback=False,
consumeErrors=True)
else:
self._stoppingDefer = defer.succeed(self)
#ensure the deferred result value is self
self._stoppingDefer.addCallback(lambda r, v: v, self)
return self._stoppingDefer
def __cbInitiateTargetsProcessing(self, sourceAnalysis, context):
# If stopping don't do anything
if self._isStopping(): return
assert isinstance(sourceAnalysis, analyst.MediaAnalysis)
sourceCtx = context.getSourceContext()
context.debug("Transcoding source file '%s'",
sourceCtx.getInputPath())
self._setJobState(JobStateEnum.transcoding)
transcoder = MediaTranscoder(self,
preparedCB=self._transcoderPreparedCallback,
playingCB=self._transcoderPlayingCallback,
progressCB=self._transcoderProgressCallback)
d = defer.succeed(None)
targets = []
for targetCtx in context.getTargetContexts():
target = self._targetsLookup[targetCtx.config.type](targetCtx)
if isinstance(target, basetargets.TranscodingTarget):
transcoder.addProducer(target)
elif isinstance(target, basetargets.TargetProcessing):
# The processing targets like IdentityTarget
# will be processed before starting the transcoder.
d.addCallback(defer.dropResult, target.process)
else:
self.warning("Unknown target-processing class '%s'",
target.__class__.__name__)
continue
targets.append(target)
d.addCallback(self.__cbStartupTranscoder, context, transcoder, sourceAnalysis)
d.addCallback(defer.overrideResult, targets)
return d
def initialize(self):
self._compPxySet.connectListener("component-added", self,
self.__onComponentAddedToSet)
self._compPxySet.connectListener("component-removed", self,
self.__onComponentRemovedFromSet)
self._compPxySet.refreshListener(self)
return defer.succeed(self)
def waitPotentialWorker(self, timeout=None):
compPxy = self.getActiveComponent()
if compPxy:
return defer.succeed(compPxy.getWorkerProxy())
d = self.__waitPotentialComponent(timeout)
d.addCallbacks(self.__cbGetValidWorker,
self.__ebNoValidWorker)
return d
def waitReport(self, timeout=None):
# Prevent blocking if not running and no report path has been received
if self.isRunning():
d = self._reportPath.wait(timeout)
else:
d = defer.succeed(self.__getLocalReportPath())
d.addCallback(self.__loadReport)
return d
def adapt(value):
if value is None:
return None
if isinstance(value, (list, tuple)):
d = defer.succeed(list())
for v in value:
d.addCallback(_adaptListValue, v)
return d
if isinstance(value, dict):
d = defer.succeed(dict())
for k, v in value.items():
d.addCallback(_adaptDictValue, k, v)
return d
if defer.isDeferred(value):
return value.addCallback(adapt)
if admifaces.IAdminInterface.providedBy(value):
return mediums.IServerMedium(value)
return value
def retrieveReport(self, timeout=None, retry=1):
virtPath = self._reportPath.getValue()
if self.isRunning():
d = self.__retrieveReportPath(virtPath, timeout, retry)
else:
d = defer.succeed(virtPath)
d.addCallback(self.__localizeReportPath)
d.addCallback(self.__loadReport)
return d
def waitActive(self, timeout=None):
"""
Gives a deferred that will be called when the element
has been activated (added)
"""
if self._active:
return defer.succeed(self)
if self._failure:
return defer.fail(self._failure)
return self._activeWaiters.wait(timeout)
def _waitDeleteComponent(self, compPxy):
self.debug("Admin task '%s' is deleting component '%s'",
self.label, compPxy.getName())
# Don't delete sad component
if compPxy.getMood() != moods.sad:
d = compPxy.forceDelete()
d.addErrback(self.__ebComponentDeleteFailed, compPxy.getName())
return d
# If sad, act like if the component was successfully deleted
return defer.succeed(compPxy)
def _waitStopComponent(self, compPxy):
self.debug("Admin task '%s' is stopping component '%s'",
self.label, compPxy.getName())
# Don't stop sad component
if compPxy.getMood() not in (moods.sad, moods.sleeping):
d = compPxy.forceStop()
d.addErrback(self.__ebComponentStopFailed, compPxy.getName())
return d
# If sad, act like if the component was successfully stopped
return defer.succeed(compPxy)
def waitIdle(self, timeout=None):
compPxy = self.getActiveComponent()
if compPxy:
# Wait UI State to be sure the file events are fired
d = compPxy.waitUIState(timeout)
d.addBoth(defer.overrideResult, self)
else:
d = defer.succeed(self)
self._doChainWaitIdle(d)
return d
def delete(self, *data):
try:
for mutable in data:
if not isinstance(mutable, MutableDataWrapper):
raise NotImplementedError()
mutable._delete()
return defer.succeed(self)
except Exception, e:
error = datasource.DeletionError(cause=e)
return defer.fail(error)
def retrieveCustomerInfo(self, custData):
try:
assert isinstance(custData, ImmutableDataWrapper)
result = ImmutableWrapper((custData.identifier, "info"),
CUST_INFO_TMPL)
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve customer info data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def retrieveCustomers(self):
try:
result = [ImmutableDataWrapper(c, k, hidden=['profiles'])
for k, c in self._customersData.items()
if c != None]
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve customers data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def _childWaitActive(self, timeout=None):
"""
Diffrent than waitActive to ensure calling order.
First the deffered added with waitActive and then
the ones added by _childWaitActive.
Should not be used by child classes, it's realy a private method.
"""
if self._active:
return defer.succeed(self)
if self._failure:
return defer.fail(self._failure)
return self._activeChildWaiters.wait(timeout)
def retrieveProfiles(self, custData):
try:
assert isinstance(custData, ImmutableDataWrapper)
result = [ImmutableDataWrapper(p, k, hidden=['targets'])
for k, p in custData._getData().profiles.items()
if p != None]
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve profiles data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def retrieveTargets(self, profData):
try:
assert isinstance(profData, ImmutableDataWrapper)
result = [ImmutableDataWrapper(t, k, hidden=['config', 'type'])
for k, t in profData._getData().targets.items()
if t != None]
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve targets data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def diagnoseTranscodingFailure(self, task, transPxy):
diagnostic = []
d = defer.succeed(diagnostic)
# Ensure we have the last report path
d.addCallback(self.__waitReportPath, transPxy)
d.addCallback(self.__componentDiagnostic, transPxy)
d.addCallback(self.__sourceFileDiagnostic, transPxy)
d.addCallback(self.__transcoderDiagnostic, transPxy)
d.addCallback(self.__pipelineDiagnostic, transPxy)
d.addCallback(self.__finishTranscodingFailureDiagnostic)
return d
def retrieveDefaults(self):
try:
result = ImmutableDataWrapper(self._adminData, "defaults",
hidden=['customersDir',
'activitiesDir',
'name'])
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve default data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def process(self):
try:
result = self._doProcessing()
if defer.isDeferred(result):
result.addCallbacks(defer.dropResult, self._targetFailed, callbackArgs=(self._targetDone,))
return result
else:
self._targetDone()
return defer.succeed(self)
except:
f = Failure()
self._targetFailed(f)
return defer.fail(f)
def retrieveActivities(self, type, states=None):
try:
assert type in ActivityTypeEnum
states = states and set(states)
result = []
tmpl = _activityTemplateLookup.get(type)
result = [MutableDataWrapper(self, tmpl, p, f, a)
for f, (p, a) in self._activitiesData.items()
if a.type == type]
return defer.succeed(result)
except Exception, e:
msg = "Failed to retrieve activities data"
ex = datasource.RetrievalError(msg, cause=e)
f = failure.Failure(ex)
return defer.fail(f)
def resume(self, timeout=None):
if self._state == TaskStateEnum.terminated:
# If terminated, a task can be rpaused and resume silently
return defer.succeed(self)
if not (self._state in [TaskStateEnum.paused,
TaskStateEnum.resuming]):
return defer.fail(admerrs.TranscoderError("Cannot resume %s task '%s'"
% (self._state.name,
self.label)))
if self._state == TaskStateEnum.paused:
self.log("Ready to resume admin task '%s'", self.label)
self._state = TaskStateEnum.resuming
self.__startup()
# Resuming and starting is the same for now
return self._startWaiters.wait(timeout)
def waitIdle(self, timeout=None):
"""
Wait for all pending elements to be activated or aborted,
and then all child elements to become idle too.
"""
if self.isIdle():
d = defer.succeed(self)
else:
d = self._idleWaiters.wait(timeout)
_idleLogger.log("%s #%s: Wait idle state (%s); Value: %d, Target: %d",
self.__class__.__name__, id(self), id(d),
self._idleWaiters.getValue(),
self._idleWaiters.getTarget())
d.addCallback(self.__cbWaitChildIdle, timeout, id(d))
return d
def do_stop(self, *args, **kwargs):
def component_stop(result):
return component.BaseComponent.do_stop(self)
try:
if self._job:
d = self._job.stop()
else:
d = defer.succeed(None)
d.addCallback(component_stop)
d.addErrback(self.__ebStopErrorFilter)
return d
except:
self.__unexpectedError(task="component stopping")
def forceStop(self):
"""
Use with caution, this method will try all it can and
more than one time to stop the component.
It will stop it, and kill it if neccessary.
"""
assert self._compState, "Component has been removed"
self.log("Stopping (Forced) component '%s'", self.label)
mood = self.getMood()
if mood == moods.sleeping:
return defer.succeed(self)
else:
d = defer.Deferred()
status = {"can_delete": False}
self.__asyncForceStop(None, status, self.label, d)
return d
def __cbMoveOutputFiles(self, succeed, context):
# If stopping don't do anything
if self._isStopping(): return
if succeed:
context.debug("Moving output files")
self._setJobState(JobStateEnum.output_file_moving)
d = defer.succeed(context)
for targetCtx in context.getTargetContexts():
for src, dest in targetCtx.reporter.getFiles():
d.addCallback(self.__asyncMove, src, dest, self._pathAttr)
d.addCallback(defer.overrideResult, succeed)
return d
else:
context.debug("Skipping moving output files, "
"because transcoding fail")
return succeed
