def eb(f):
self.debug(
'parse: eb: failure %s' % log.getFailureMessage(f))
# Nagios exceptions have already got their feedback covered
if not f.check(util.NagiosException):
util.unknown(log.getFailureMessage(f))
reactor.callLater(0, reactor.stop)
def eb(failure):
self.debug(
'parse: eb: failure %s' % log.getFailureMessage(failure))
# Nagios exceptions have already got their feedback covered
if not failure.check(util.NagiosException):
util.unknown(log.getFailureMessage(failure))
reactor.callLater(0, reactor.stop)
def error(failure):
if self.extraTenacious:
self.debug('connection problem to %s: %s',
self._connector.getDestination(),
log.getFailureMessage(failure))
self.debug('we are tenacious, so trying again later')
self.disconnect()
elif failure.check(errors.ConnectionFailedError):
self.debug("emitting connection-failed")
self.medium.emit('connection-failed', "I failed my master")
self.debug("emitted connection-failed")
elif failure.check(errors.ConnectionRefusedError):
self.debug("emitting connection-refused")
self.medium.emit('connection-refused')
self.debug("emitted connection-refused")
elif failure.check(errors.NotAuthenticatedError):
# FIXME: unauthorized login emit !
self.debug("emitting connection-refused")
self.medium.emit('connection-refused')
self.debug("emitted connection-refused")
else:
self.medium.emit('connection-error', failure)
self.warning('connection error to %s:: %s',
self._connector.getDestination(),
log.getFailureMessage(failure))
def error(failure):
if self.extraTenacious:
self.debug('connection problem to %s: %s',
self._connector.getDestination(),
log.getFailureMessage(failure))
self.debug('we are tenacious, so trying again later')
self.disconnect()
elif failure.check(errors.ConnectionFailedError):
self.debug("emitting connection-failed")
self.medium.emit('connection-failed', "I failed my master")
self.debug("emitted connection-failed")
elif failure.check(errors.ConnectionRefusedError):
self.debug("emitting connection-refused")
self.medium.emit('connection-refused')
self.debug("emitted connection-refused")
elif failure.check(errors.NotAuthenticatedError):
# FIXME: unauthorized login emit !
self.debug("emitting connection-refused")
self.medium.emit('connection-refused')
self.debug("emitted connection-refused")
else:
self.medium.emit('connection-error', failure)
self.warning('connection error to %s:: %s',
self._connector.getDestination(),
log.getFailureMessage(failure))
def eb(failure):
# FIXME
if failure.check(errors.NoMethodError) \
or failure.check(flavors.NoSuchMethod):
common.errorRaise("No method '%s' on manager." % methodName)
elif failure.check(errors.RemoteRunError):
common.errorRaise(log.getFailureMessage(failure))
else:
common.errorRaise(log.getFailureMessage(failure))
def eb(failure):
if failure.check(errors.NoMethodError):
common.errorRaise("No method '%s' on worker '%s'." % (methodName, workerName))
elif failure.check(errors.SleepingComponentError):
common.errorRaise("Component '%s' is sleeping." % p.componentId)
elif failure.check(errors.RemoteRunError):
common.errorRaise(log.getFailureMessage(failure))
else:
common.errorRaise(log.getFailureMessage(failure))
def eb(failure):
# FIXME
if failure.check(errors.NoMethodError) \
or failure.check(flavors.NoSuchMethod):
common.errorRaise("No method '%s' on manager." % methodName)
elif failure.check(errors.RemoteRunError):
common.errorRaise(log.getFailureMessage(failure))
else:
common.errorRaise(log.getFailureMessage(failure))
def eb(failure):
if failure.check(errors.NoMethodError):
common.errorRaise("No method '%s' on worker '%s'." %
(methodName, workerName))
elif failure.check(errors.SleepingComponentError):
common.errorRaise("Component '%s' is sleeping." %
p.componentId)
elif failure.check(errors.RemoteRunError):
common.errorRaise(log.getFailureMessage(failure))
else:
common.errorRaise(log.getFailureMessage(failure))
def eb(failure):
self.debug("parse: eb: failure %s" % log.getFailureMessage(failure))
if failure.check(common.Exited):
sys.stderr.write(failure.value.msg + "\n")
reactor.exitStatus = failure.value.code
else:
sys.stderr.write(log.getFailureMessage(failure) + "\n")
reactor.exitStatus = 1
reactor.callLater(0, reactor.stop)
return
def eb(failure):
self.debug('parse: eb: failure %s' %
log.getFailureMessage(failure))
if failure.check(common.Exited):
sys.stderr.write(failure.value.msg + '\n')
reactor.exitStatus = failure.value.code
else:
sys.stderr.write(log.getFailureMessage(failure) + '\n')
reactor.exitStatus = 1
reactor.callLater(0, reactor.stop)
return
def seekingFailed(failure):
# swallow the failure and serve the file from the beginning
self.warning("Seeking in MP4 file %s failed: %s", provider,
log.getFailureMessage(failure))
provider.seek(0)
request.setHeader('Content-Length', str(length))
return ret
def _terminateRequest(self, body, request):
if body == server.NOT_DONE_YET:
# _renderRequest will return NOT_DONE_YET if it started serving the
# file. This means the callback chain started by _renderRequest has
# finished and we're currently serving the file.
return
if isinstance(body, Failure):
# Something went wrong, log it
self.warning("Failure during request rendering: %s",
log.getFailureMessage(body))
if body.check(weberror.Error):
err = body.value
page = weberror.ErrorPage(err.status, err.message,
err.response)
elif body.check(fileprovider.UnavailableError):
page = self.serviceUnavailable
elif body.check(fileprovider.AccessError):
page = self.forbiddenResource
elif body.check(fileprovider.NotFoundError):
page = self.childNotFound
else:
page = self.internalServerError
body = page.render(request)
if body:
# the callback chain from _renderRequest chose to return a string
# body, write it out to the client
request.write(body)
self.debug('[fd %5d] Terminate request %r',
request.transport.fileno(), request)
request.finish()
def _terminateRequest(self, body, request):
if body == server.NOT_DONE_YET:
# _renderRequest will return NOT_DONE_YET if it started serving the
# file. This means the callback chain started by _renderRequest has
# finished and we're currently serving the file.
return
if isinstance(body, Failure):
# Something went wrong, log it
self.warning("Failure during request rendering: %s",
log.getFailureMessage(body))
if body.check(weberror.Error):
err = body.value
page = weberror.ErrorPage(err.status, err.message,
err.response)
elif body.check(fileprovider.UnavailableError):
page = self.serviceUnavailable
elif body.check(fileprovider.AccessError):
page = self.forbiddenResource
elif body.check(fileprovider.NotFoundError):
page = self.childNotFound
else:
page = self.internalServerError
body = page.render(request)
if body:
# the callback chain from _renderRequest chose to return a string
# body, write it out to the client
request.write(body)
self.debug('[fd %5d] Terminate request %r',
request.transport.fileno(), request)
request.finish()
def eb(failure):
if failure.check(errors.ComponentMoodError,
errors.BusyComponentError):
common.errorRaise("Component '%s' is in the wrong mood." %
self.parentCommand.componentId)
else:
common.errorRaise(log.getFailureMessage(failure))
def error(failure):
self.warning('failed to retrieve %s from %s:%d', fullFeedId, host,
port)
self.debug('failure: %s', log.getFailureMessage(failure))
self.debug('closing connection')
self.stopConnecting()
return failure
def clientConnectionFailed(self, connector, reason):
"""
@type connector: implementation of
L{twisted.internet.interfaces.IConnector}
@param reason: L{twisted.spread.pb.failure.Failure}
"""
if reason.check(error.DNSLookupError):
self.debug('DNS lookup error')
if not self.extraTenacious:
self.medium.connectionFailed(reason)
return
elif (reason.check(error.ConnectionRefusedError)
or reason.check(error.ConnectError)):
# If we're logging in for the first time, we want to make this a
# real error; we present a dialog, etc.
# However, if we fail later on (e.g. manager shut down, and
# hasn't yet been restarted), we want to keep trying to reconnect,
# so we just log a message.
self.debug("Error connecting to %s: %s",
connector.getDestination(),
log.getFailureMessage(reason))
if self.hasBeenConnected:
self.log("we've been connected before though, so going "
"to retry")
# fall through
elif self.extraTenacious:
self.log("trying again due to +100 tenacity")
# fall through
else:
self.log("telling medium about connection failure")
self.medium.connectionFailed(reason)
return
fpb.ReconnectingFPBClientFactory.clientConnectionFailed(
self, connector, reason)
def eb(failure):
if failure.check(errors.ComponentMoodError,
errors.BusyComponentError):
common.errorRaise("Component '%s' is in the wrong mood." %
self.parentCommand.componentId)
else:
common.errorRaise(log.getFailureMessage(failure))
def _cbRequestAvatarId(self, validKey, credentials):
if not validKey:
return failure.Failure(UnauthorizedLogin())
if not credentials.signature:
return failure.Failure(error.ValidPublicKey())
else:
try:
if conch.version.major < 10:
pubKey = keys.getPublicKeyObject(data=credentials.blob)
if keys.verifySignature(pubKey, credentials.signature,
credentials.sigData):
return credentials.username
else:
pubKey = keys.Key.fromString(credentials.blob)
if pubKey.verify(credentials.signature,
credentials.sigData):
return credentials.username
except: # any error should be treated as a failed login
f = failure.Failure()
log.warning('manhole',
'error checking signature on creds %r: %r',
credentials, log.getFailureMessage(f))
return f
return failure.Failure(UnauthorizedLogin())
def clientConnectionFailed(self, connector, reason):
"""
@type connector: implementation of
L{twisted.internet.interfaces.IConnector}
@param reason: L{twisted.spread.pb.failure.Failure}
"""
if reason.check(error.DNSLookupError):
self.debug('DNS lookup error')
if not self.extraTenacious:
self.medium.connectionFailed(reason)
return
elif (reason.check(error.ConnectionRefusedError)
or reason.check(error.ConnectError)):
# If we're logging in for the first time, we want to make this a
# real error; we present a dialog, etc.
# However, if we fail later on (e.g. manager shut down, and
# hasn't yet been restarted), we want to keep trying to reconnect,
# so we just log a message.
self.debug("Error connecting to %s: %s",
connector.getDestination(),
log.getFailureMessage(reason))
if self.hasBeenConnected:
self.log("we've been connected before though, so going "
"to retry")
# fall through
elif self.extraTenacious:
self.log("trying again due to +100 tenacity")
# fall through
else:
self.log("telling medium about connection failure")
self.medium.connectionFailed(reason)
return
fpb.ReconnectingFPBClientFactory.clientConnectionFailed(self,
connector, reason)
def _cbRequestAvatarId(self, validKey, credentials):
if not validKey:
return failure.Failure(UnauthorizedLogin())
if not credentials.signature:
return failure.Failure(error.ValidPublicKey())
else:
try:
if conch.version.major < 10:
pubKey = keys.getPublicKeyObject(data=credentials.blob)
if keys.verifySignature(pubKey, credentials.signature,
credentials.sigData):
return credentials.username
else:
pubKey = keys.Key.fromString(credentials.blob)
if pubKey.verify(credentials.signature,
credentials.sigData):
return credentials.username
except: # any error should be treated as a failed login
f = failure.Failure()
log.warning('manhole',
'error checking signature on creds %r: %r',
credentials, log.getFailureMessage(f))
return f
return failure.Failure(UnauthorizedLogin())
def error(failure):
self.warning('failed to retrieve %s from %s:%d', fullFeedId,
host, port)
self.debug('failure: %s', log.getFailureMessage(failure))
self.debug('closing connection')
self.stopConnecting()
return failure
def seekingFailed(failure):
# swallow the failure and serve the file from the beginning
self.warning("Seeking in MP4 file %s failed: %s", provider,
log.getFailureMessage(failure))
provider.seek(0)
request.setHeader('Content-Length', str(length))
return ret
def _ebReadFailed(self, failure):
self._pending = None
if self._finished:
return
self.warning("Failure during file %s reading: %s", self.provider, log.getFailureMessage(failure))
self._terminate()
def errback(failure):
log.debug('launch', log.getFailureMessage(failure))
print "Error occurred: %s" % failure.getErrorMessage()
failure.printDetailedTraceback()
reactor.failure = True
if reactor.running:
print "Stopping reactor."
reactor.stop()
def _ebReadFailed(self, failure):
self._pending = None
if self._finished:
return
self.warning('Failure during file %s reading: %s',
self.provider, log.getFailureMessage(failure))
self._terminate()
def error(failure, job):
msg = log.getFailureMessage(failure)
if failure.check(errors.ComponentCreateError):
self.warning('could not create component %s of type %s:'
' %s', job.avatarId, job.type, msg)
else:
self.warning('unhandled error creating component %s: %s',
job.avatarId, msg)
# FIXME: drills down too much?
self._heaven._startSet.createFailed(job.avatarId, failure)
def failed(failure):
if failure.check(errors.ConnectionRefusedError):
print "Manager refused connection. Check your user and password."
elif failure.check(errors.ConnectionFailedError):
message = "".join(failure.value.args)
print "Connection to manager failed: %s" % message
else:
print ("Exception while connecting to manager: %s"
% log.getFailureMessage(failure))
return failure
def error(failure, job):
msg = log.getFailureMessage(failure)
if failure.check(errors.ComponentCreateError):
self.warning('could not create component %s of type %s:'
' %s', job.avatarId, job.type, msg)
else:
self.warning('unhandled error creating component %s: %s',
job.avatarId, msg)
# FIXME: drills down too much?
self._heaven._startSet.createFailed(job.avatarId, failure)
def failed(failure):
if failure.check(errors.ConnectionRefusedError):
print "Manager refused connection. Check your user and password."
elif failure.check(errors.ConnectionFailedError):
message = "".join(failure.value.args)
print "Connection to manager failed: %s" % message
else:
print ("Exception while connecting to manager: %s"
% log.getFailureMessage(failure))
return failure
def _defaultErrback(self, failure, request):
if failure.check(errors.NotAuthenticatedError) is None:
# If something else went wrong, we want to either disconnect the
# client and give them a 500 Internal Server Error or just allow
# them, depending on the configuration.
self.debug("Authorization request failed: %s",
log.getFailureMessage(failure))
if self._allowDefault:
self.debug("Authorization failed, but allowing anyway")
return None
self._handleUnauthorized(request, http.INTERNAL_SERVER_ERROR)
return failure
def _defaultErrback(self, failure, request):
if failure.check(errors.NotAuthenticatedError) is None:
# If something else went wrong, we want to either disconnect the
# client and give them a 500 Internal Server Error or just allow
# them, depending on the configuration.
self.debug("Authorization request failed: %s",
log.getFailureMessage(failure))
if self._allowDefault:
self.debug("Authorization failed, but allowing anyway")
return None
self._handleUnauthorized(request, http.INTERNAL_SERVER_ERROR)
return failure
def got_error(failure):
txt = log.getFailureMessage(failure)
self.debug('got_error: %s', txt)
if not failure.check(errors.ComponentSetupHandledError):
self.warning('Setup failed: %s', txt)
m = messages.Error(T_(N_("Could not setup component.")),
debug=txt,
mid="component-setup-%s" % self.name)
# will call setMood(moods.sad)
self.addMessage(m)
# swallow
return None
def got_error(failure):
txt = log.getFailureMessage(failure)
self.debug('got_error: %s', txt)
if not failure.check(errors.ComponentSetupHandledError):
self.warning('Setup failed: %s', txt)
m = messages.Error(T_(N_("Could not setup component.")),
debug=txt,
mid="component-setup-%s" % self.name)
# will call setMood(moods.sad)
self.addMessage(m)
# swallow
return None
def onErrorCloseConnection(failure):
try:
host = mind.broker.transport.getHost()
remote = '%s:%d' % (host.host, host.port)
except:
remote = '(unknown)'
self.warning('failed login -- closing connection to %s', remote)
self.debug('failure: %s', log.getFailureMessage(failure))
try:
mind.broker.transport.loseConnection()
except Exception, e:
self.info('loseConnection failed: %s',
log.getExceptionMessage(e))
def _createErrback(self, failure, state):
# FIXME: make ConfigError copyable so we can .check() it here
# and print a nicer warning
self.warning('failed to create component %s: %s',
state.get('name'), log.getFailureMessage(failure))
if failure.check(errors.ComponentAlreadyRunningError):
if self._componentMappers[state].jobState:
self.info('component appears to have logged in in the '
'meantime')
else:
self.info('component appears to be running already; '
'treating it as lost until it logs in')
state.setMood(moods.lost.value)
else:
message = messages.Error(T_(
N_("The component could not be started.")),
debug=log.getFailureMessage(failure))
state.setMood(moods.sad.value)
state.append('messages', message)
return None
def connectionFailed(self, failure):
# called by client factory
if failure.check(error.DNSLookupError):
message = ("Could not look up host '%s'." %
self.connectionInfo.host)
elif failure.check(error.ConnectionRefusedError):
message = ("Could not connect to host '%s' on port %d." %
(self.connectionInfo.host, self.connectionInfo.port))
else:
message = ("Unexpected failure.\nDebug information: %s" %
log.getFailureMessage(failure))
self.debug('emitting connection-failed')
self.emit('connection-failed', message)
self.debug('emitted connection-failed')
def _terminateRequest(self, body, request):
if body == server.NOT_DONE_YET:
# Currently serving the file
return
if isinstance(body, Failure):
# Something goes wrong, log it
self.warning("Failure during request rendering: %s",
log.getFailureMessage(body))
body = self.internalServerError.render(request)
if body:
# render result/error page
request.write(body)
self.debug('[fd %5d] Terminate request %r',
request.transport.fileno(), request)
request.finish()
def onErrorCloseConnection(failure):
try:
host = mind.broker.transport.getHost()
remote = '%s:%d' % (host.host, host.port)
except:
remote = '(unknown)'
self.warning('failed login -- closing connection to %s',
remote)
self.debug('failure: %s', log.getFailureMessage(failure))
try:
mind.broker.transport.loseConnection()
except Exception, e:
self.info('loseConnection failed: %s',
log.getExceptionMessage(e))
def connectionFailed(self, failure):
# called by client factory
if failure.check(error.DNSLookupError):
message = ("Could not look up host '%s'."
% self.connectionInfo.host)
elif failure.check(error.ConnectionRefusedError):
message = ("Could not connect to host '%s' on port %d."
% (self.connectionInfo.host,
self.connectionInfo.port))
else:
message = ("Unexpected failure.\nDebug information: %s"
% log.getFailureMessage(failure))
self.debug('emitting connection-failed')
self.emit('connection-failed', message)
self.debug('emitted connection-failed')
def errback(failure):
msg = None
if failure.check(errors.NoMethodError):
msg = "Remote method '%s' does not exist." % methodName
msg += "\n" + failure.value
else:
msg = log.getFailureMessage(failure)
# FIXME: we probably need a nicer way of getting component
# messages shown from the admin model, but this allows us to
# make sure every type of admin has these messages
self.warning(msg)
m = messages.Warning(T_(N_("Internal error in component.")),
debug=msg)
componentState.observe_append('messages', m)
return failure
def errback(failure):
msg = None
if failure.check(errors.NoMethodError):
msg = "Remote method '%s' does not exist." % methodName
msg += "\n" + failure.value
else:
msg = log.getFailureMessage(failure)
# FIXME: we probably need a nicer way of getting component
# messages shown from the admin model, but this allows us to
# make sure every type of admin has these messages
self.warning(msg)
m = messages.Warning(T_(N_("Internal error in component.")),
debug=msg)
componentState.observe_append('messages', m)
return failure
def clientConnectionLost(self, connector, reason):
"""
@type connector: implementation of
L{twisted.internet.interfaces.IConnector}
@param reason: L{twisted.spread.pb.failure.Failure}
"""
self.debug("Lost connection to %s: %s",
connector.getDestination(), log.getFailureMessage(reason))
if self.hasBeenAuthenticated:
self.log("Have been authenticated before. Trying again.")
elif self.extraTenacious:
self.log("We are extra tenacious, trying again")
else:
self.log("Telling medium about connection failure")
self.medium.connectionFailed(reason)
return
RFC = fpb.ReconnectingFPBClientFactory
RFC.clientConnectionLost(self, connector, reason)
def clientConnectionLost(self, connector, reason):
"""
@type connector: implementation of
L{twisted.internet.interfaces.IConnector}
@param reason: L{twisted.spread.pb.failure.Failure}
"""
self.debug("Lost connection to %s: %s", connector.getDestination(),
log.getFailureMessage(reason))
if self.hasBeenAuthenticated:
self.log("Have been authenticated before. Trying again.")
elif self.extraTenacious:
self.log("We are extra tenacious, trying again")
else:
self.log("Telling medium about connection failure")
self.medium.connectionFailed(reason)
return
RFC = fpb.ReconnectingFPBClientFactory
RFC.clientConnectionLost(self, connector, reason)
def loginFailed(failure):
def gotRoot(root):
# an idempotent method, should return a network failure if
# the remote side disconnects as it should
return root.callRemote("getKeycardClasses")
def gotError(failure):
self.assertAdditionalFDsOpen(1, "feedSent (socket)")
self.info("success")
def gotKeycardClasses(classes):
raise AssertionError("should not get here")
self.info("loginFailed: %s", log.getFailureMessage(failure))
failure.trap(errors.NotAuthenticatedError)
d = factory.getRootObject() # should fire immediately
d.addCallback(gotRoot)
d.addCallbacks(gotKeycardClasses, gotError)
return d
def loginFailed(failure):
def gotRoot(root):
# an idempotent method, should return a network failure if
# the remote side disconnects as it should
return root.callRemote('getKeycardClasses')
def gotError(failure):
self.assertAdditionalFDsOpen(1, 'feedSent (socket)')
self.info('success')
def gotKeycardClasses(classes):
raise AssertionError('should not get here')
self.info('loginFailed: %s', log.getFailureMessage(failure))
failure.trap(errors.NotAuthenticatedError)
d = factory.getRootObject() # should fire immediately
d.addCallback(gotRoot)
d.addCallbacks(gotKeycardClasses, gotError)
return d
def error(failure):
self.addMessage(messages.WARNING,
'provide-master-clock',
N_('Failed to provide the master clock'),
debug=log.getFailureMessage(failure))
self.vishnu.releasePortsOnWorker(self.getWorkerName(), [port])
def invocationError(failure):
self.warning(
'Exception raised while calling '
'%s.%s(*args=%r, **kwargs=%r): %s', module, function, args,
kwargs, log.getFailureMessage(failure))
return failure
def renderFinishedErrback(failure):
return error(log.getFailureMessage(failure))
def renderFinishedErrback(failure):
return error(log.getFailureMessage(failure))
def _ebRequestAvatarId(self, f):
if not f.check(UnauthorizedLogin, error.ValidPublicKey):
log.warning('manhole', 'failed login: %r',
log.getFailureMessage(f))
return failure.Failure(UnauthorizedLogin())
return f
def error(failure):
self.addMessage(messages.WARNING, 'provide-master-clock',
N_('Failed to provide the master clock'),
debug=log.getFailureMessage(failure))
self.vishnu.releasePortsOnWorker(self.getWorkerName(), [port])
def _ebReadFailed(self, failure):
self._pending = None
self.warning('Failure during file %s reading: %s',
self.provider, log.getFailureMessage(failure))
self._terminate()
def _ebRequestAvatarId(self, f):
if not f.check(UnauthorizedLogin, error.ValidPublicKey):
log.warning('manhole', 'failed login: %r',
log.getFailureMessage(f))
return failure.Failure(UnauthorizedLogin())
return f
def invocationError(failure):
self.warning('Exception raised while calling '
'%s.%s(*args=%r, **kwargs=%r): %s',
module, function, args, kwargs,
log.getFailureMessage(failure))
return failure
def _addErrback(self, err):
self.warning("Could not resolve host %s", log.getFailureMessage(err))
return
def eb(failure):
common.errorRaise(log.getFailureMessage(failure))
def pingFailed(failure):
# ignoring the connection failures so they don't end up in
# the logs - we'll notice the lack of pingback eventually
failure.trap(pb.PBConnectionLost)
self.log('ping failed: %s' % log.getFailureMessage(failure))
def pingFailed(failure):
# ignoring the connection failures so they don't end up in
# the logs - we'll notice the lack of pingback eventually
failure.trap(pb.PBConnectionLost)
self.log('ping failed: %s' % log.getFailureMessage(failure))
def errback(failure):
self.warning('not polling %s%s:%s: failed to get ui ' 'state')
self.debug('reason: %s', log.getFailureMessage(failure))
