def test_unauthenticated(self):
"""
Test the behavior of locateChild when SACLs are enabled and the request
is unauthenticated
should return a 401 UnauthorizedResponse
"""
self.root.resource.useSacls = True
request = SimpleRequest(
self.site,
"GET",
"/principals/"
)
resrc, segments = (yield maybeDeferred(
self.root.locateChild, request, ["principals"]
))
try:
resrc, segments = (yield maybeDeferred(
resrc.locateChild, request, ["principals"]
))
raise AssertionError(
"RootResource.locateChild did not return an error"
)
except HTTPError, e:
self.assertEquals(e.response.code, 401)
def _call_method(self, request):
"""Calls given method with given params and returns it value."""
method = self.method_data[request['method']]['method']
params = request['params']
result = None
try:
if isinstance(params, list):
# Does it have enough arguments?
if len(params) < self._man_args(method):
raise InvalidParamsError('not enough arguments')
# Does it have too many arguments?
if not self._vargs(method) \
and len(params) > self._max_args(method):
raise InvalidParamsError('too many arguments')
result = yield defer.maybeDeferred(method, *params)
elif isinstance(params, dict):
# Do not accept keyword arguments if the jsonrpc version is
# not >=1.1.
if request['jsonrpc'] < 11:
raise KeywordError
result = yield defer.maybeDeferred(method, **params)
else: # No params
result = yield defer.maybeDeferred(method)
except JSONRPCError:
raise
except Exception:
# Exception was raised inside the method.
log.msg('Exception raised while invoking RPC method "{}".'.format(
request['method']))
log.err()
raise ServerError
defer.returnValue(result)
def process_result(result, total=None, found_item=None):
if result == None:
result = []
l = []
def process_items(result, tm):
if result is None:
result = []
for i in result:
if i[0]:
didl.addItem(i[1])
return build_response(tm)
for i in result:
d = defer.maybeDeferred(i.get_item)
l.append(d)
if found_item is not None:
def got_child_count(count):
dl = defer.DeferredList(l)
dl.addCallback(process_items, count)
return dl
d = defer.maybeDeferred(found_item.get_child_count)
d.addCallback(got_child_count)
return d
elif total is None:
total = item.get_child_count()
dl = defer.DeferredList(l)
dl.addCallback(process_items, total)
return dl
def test_badCredentials(self):
"""
Test the behavior of locateChild when SACLS are enabled, and
incorrect credentials are given.
should return a 401 UnauthorizedResponse
"""
self.root.resource.useSacls = True
request = SimpleRequest(
self.site,
"GET",
"/principals/",
headers=http_headers.Headers({
"Authorization": ["basic", "%s" % (
"dreid:dreid".encode("base64"),)]}))
resrc, segments = (yield maybeDeferred(
self.root.locateChild, request, ["principals"]
))
try:
resrc, segments = (yield maybeDeferred(
resrc.locateChild, request, ["principals"]
))
except HTTPError, e:
self.assertEquals(e.response.code, 401)
def render(self, request):
"""This is a modified version of render() from XMLRPC that will
pass keyword arguments with extra state information if it can.
Currently this just consists of the current request.
"""
request.content.seek(0, 0)
args, functionPath = xmlrpclib.loads(request.content.read())
try:
function = self._getFunction(functionPath)
except xmlrpc.NoSuchFunction:
self._cbRender(
xmlrpc.Fault(self.NOT_FOUND, "no such function %s" % functionPath),
request
)
else:
request.setHeader("content-type", "text/xml")
# Pass as many keyword args to the function as we can.
# Note that this currently doesn't work on protected functions.
kwargs = {}
try:
if 'request' in function.func_code.co_varnames:
kwargs['request'] = request
except AttributeError:
pass
defer.maybeDeferred(function, *args, **kwargs).addErrback(
self._ebRender
).addCallback(
self._cbRender, request
)
return server.NOT_DONE_YET
def validQuery(self, portOnServer, portOnClient):
serverAddr = self.transport.getHost()[1], portOnServer
clientAddr = self.transport.getPeer()[1], portOnClient
defer.maybeDeferred(self.lookup, serverAddr, clientAddr
).addCallback(self._cbLookup, portOnServer, portOnClient
).addErrback(self._ebLookup, portOnServer, portOnClient
)
def children(self, request, tag):
"""
Renderer which yields all child object tags as table rows.
"""
whenChildren = (
maybeDeferred(self.resource.listChildren)
.addCallback(sorted)
.addCallback(
lambda names: gatherResults(
[maybeDeferred(self.resource.getChild, x) for x in names]
)
.addCallback(lambda children: zip(children, names))
)
)
@whenChildren.addCallback
def gotChildren(children):
for even, [child, name] in zip(cycle(["odd", "even"]), children):
[url, name, size, lastModified, contentType] = map(
str, self.resource.getChildDirectoryEntry(
child, name, request)
)
yield tag.clone().fillSlots(
url=url, name=name, size=str(size),
lastModified=lastModified, even=even, type=contentType,
)
return whenChildren
def start(self):
yield defer.maybeDeferred(self.configure)
for spider, batches in self._scheduled.iteritems():
yield self._schedule(spider, batches)
yield defer.maybeDeferred(self.engine.start)
def dataPhase(self):
while len(self.buf) >= NBDRequestHeader.size:
header = NBDRequestHeader(self.buf[:NBDRequestHeader.size])
log.msg("Received %r" % header)
if header.requestType == NBD_CMD_WRITE:
if len(self.buf) < header.size + header.length:
log.msg("Incomplete write request received")
return
data = self.buf[header.size:header.size+header.length]
self.buf = self.buf[header.size+header.length:]
d = defer.maybeDeferred(self.bdev.write, header.offset, data)
d.addCallback(self.reply, header.handle)
d.addErrback(self.errorReply, header.handle)
elif header.requestType == NBD_CMD_READ:
self.buf = self.buf[header.size:]
d = defer.maybeDeferred(self.bdev.read, header.offset, header.length)
d.addCallback(self.reply, header.handle)
d.addErrback(self.errorReply, header.handle)
elif header.requestType == NBD_CMD_DISC:
self.transport.loseConnection()
return
else:
self.buf = self.buf[header.size:]
log.msg("Unsupported request type %d" % header.requestType)
self.transport.write(
struct.pack(">II8s",
NBD_REPLY_MAGIC,
1, # Operation not permitted
header.handle
)
)
def testPortforward(self):
serverProtocol = wire.Echo()
realServerFactory = protocol.ServerFactory()
realServerFactory.protocol = lambda: serverProtocol
realServerPort = reactor.listenTCP(0, realServerFactory,
interface='127.0.0.1')
proxyServerFactory = portforward.ProxyFactory('127.0.0.1',
realServerPort.getHost().port)
proxyServerPort = reactor.listenTCP(0, proxyServerFactory,
interface='127.0.0.1')
nBytes = 1000
received = []
clientProtocol = protocol.Protocol()
clientProtocol.dataReceived = received.extend
clientProtocol.connectionMade = lambda: clientProtocol.transport.write('x' * nBytes)
clientFactory = protocol.ClientFactory()
clientFactory.protocol = lambda: clientProtocol
reactor.connectTCP('127.0.0.1', proxyServerPort.getHost().port,
clientFactory)
c = 0
while len(received) < nBytes and c < 100:
reactor.iterate(0.01)
c += 1
self.assertEquals(''.join(received), 'x' * nBytes)
clientProtocol.transport.loseConnection()
serverProtocol.transport.loseConnection()
return defer.gatherResults([
defer.maybeDeferred(realServerPort.stopListening),
defer.maybeDeferred(proxyServerPort.stopListening)])
def render(self, request):
"""Handle a SOAP command."""
data = request.content.read()
p, header, body, attrs = SOAPpy.parseSOAPRPC(data, 1, 1, 1)
methodName, args, kwargs, ns = p._name, p._aslist, p._asdict, p._ns
# deal with changes in SOAPpy 0.11
if callable(args):
args = args()
if callable(kwargs):
kwargs = kwargs()
function = self.lookupFunction(methodName)
if not function:
self._methodNotFound(request, methodName)
return server.NOT_DONE_YET
else:
if hasattr(function, "useKeywords"):
keywords = {}
for k, v in kwargs.items():
keywords[str(k)] = v
d = defer.maybeDeferred(function, **keywords)
else:
d = defer.maybeDeferred(function, *args)
d.addCallback(self._gotResult, request, methodName)
d.addErrback(self._gotError, request, methodName)
return server.NOT_DONE_YET
def stopService(self):
# This will stop plugin Processors
D2 = defer.maybeDeferred(service.MultiService.stopService, self)
U = defer.maybeDeferred(self.udp.stopListening)
T = defer.maybeDeferred(self.tcp.stopListening)
return defer.DeferredList([U,T,D2], consumeErrors=True)
def getPackagesLoop(self, result = None):
if result:
if not isinstance(result, failure.Failure):
self.tmppackages.append(result)
else:
logging.getLogger().error("Error: %s", str(result))
if not self.p_apis and self.tmppackages:
# Merge temporary results
lp = self.tmppackages[0]
map(lambda p: _merge_list(lp, p), self.tmppackages)
self.packages.extend(map(lambda m: [m, self.index, self.p_api_first], lp))
self.tmppackages = []
if self.plists and not self.p_apis:
# Fill self.p_apis if empty
self.p_apis = self.plists.pop()
self.p_api_first = self.p_apis[0][0]
self.index = self.index + 1
if self.p_apis:
p_api = self.p_apis.pop()
if 'pending' in self.filt:
d = defer.maybeDeferred(PackageGetA(p_api[0]).getAllPendingPackages, p_api[1])
else:
d = defer.maybeDeferred(PackageGetA(p_api[0]).getAllPackages, p_api[1])
d.addCallbacks(self.getPackagesLoop)
else:
# No more remote call to do, we are done
self.sendResult(self.packages)
def render_POST(self, request):
request.content.seek(0, 0)
request.setHeader(b"content-type", b"text/xml; charset=utf-8")
try:
args, functionPath = xmlrpclib.loads(request.content.read(), use_datetime=self.useDateTime)
except Exception as e:
f = Fault(self.FAILURE, "Can't deserialize input: %s" % (e,))
self._cbRender(f, request)
else:
try:
function = self.lookupProcedure(functionPath)
except Fault as f:
self._cbRender(f, request)
else:
# Use this list to track whether the response has failed or not.
# This will be used later on to decide if the result of the
# Deferred should be written out and Request.finish called.
responseFailed = []
request.notifyFinish().addErrback(responseFailed.append)
if getattr(function, "withRequest", False):
d = defer.maybeDeferred(function, request, *args)
else:
d = defer.maybeDeferred(function, *args)
d.addErrback(self._ebRender)
d.addCallback(self._cbRender, request, responseFailed)
return server.NOT_DONE_YET
def privmsg(self, user, channel, message):
nick, _, host = user.partition('!')
message = message.strip()
if not message.startswith('!'): # not a trigger command
return # do nothing
command, sep, rest = message.lstrip('!').partition(' ')
# Get the function corresponding to the command given.
func = getattr(self, 'command_' + command, None)
# Or, if there was no function, ignore the message.
if func is None:
return
# maybeDeferred will always return a Deferred. It calls func(rest), and
# if that returned a Deferred, return that. Otherwise, return the return
# value of the function wrapped in twisted.internet.defer.succeed. If
# an exception was raised, wrap the traceback in
# twisted.internet.defer.fail and return that.
if command == 'items':
rest = (channel, nick, rest)
d = defer.maybeDeferred(func, rest)
else:
d = defer.maybeDeferred(func, rest)
# Add callbacks to deal with whatever the command results are.
# If the command gives error, the _show_error callback will turn the
# error into a terse message first:
d.addErrback(self._show_error)
# Whatever is returned is sent back as a reply:
if channel == self.nickname:
# When channel == self.nickname, the message was sent to the bot
# directly and not to a channel. So we will answer directly too:
d.addCallback(self._send_message, nick)
else:
# Otherwise, send the answer to the channel, and use the nick
# as addressing in the message itself:
d.addCallback(self._send_message, channel, nick)
def run(self):
"""
Выполнить команду.
В процессе выполнения команды или проверки условий её выполнения (наличие всех параметров)
и т.п., могут возникать исключения, которые будут переданы через errback к deferred, или
в случае успешного выполнения вызовется callback.
@return: deferred, которая будет вызвана после завершения выполнения команды
и формирования её результата
@rtype: L{defer.Deferred}
"""
def checkResult(result):
self.checkResult()
return result
def finalize(res):
try:
self.finalize()
except:
if not sys.modules.has_key('twisted.trial.runner'):
log.err(failure.Failure())
return res
try:
self.checkParams()
d = defer.maybeDeferred(self.init).addCallback(lambda _: defer.maybeDeferred(self.perform))
except errors.BaseCommandException:
return defer.fail()
return d.addCallback(checkResult).addBoth(finalize)
def dispatch_message(payload, msg, dispatchIn, conv=None):
"""
Dispatches a message by operation in a given class.
Expected message content:
body = {
"op": "operation name here",
"content": # Any valid type here (str, list, dict)
}
"""
try:
log_message(msg)
if "op" in payload:
op = payload['op']
content = payload.get('content','')
opname = 'op_' + str(op)
# dynamically invoke the operation in the given class
if hasattr(dispatchIn, opname):
opf = getattr(dispatchIn, opname)
return defer.maybeDeferred(opf, content, payload, msg)
elif hasattr(dispatchIn,'op_none'):
return defer.maybeDeferred(dispatchIn.op_none, content, payload, msg)
else:
logging.error("Receive() failed. Cannot dispatch to catch")
else:
logging.error("Invalid message. No 'op' in header", payload)
except StandardError, e:
log_exception('Exception while dispatching: ',e)
def _component_stop(self):
def on_stop(result):
self._component_state = "Stopped"
if self._component_timer and self._component_timer.running:
self._component_timer.stop()
return True
def on_stop_fail(result):
self._component_state = "Started"
self._component_stopping_deferred = None
log.error(result)
return result
if self._component_state != "Stopped" and self._component_state != "Stopping":
if hasattr(self, "stop"):
self._component_state = "Stopping"
d = maybeDeferred(self.stop)
d.addCallback(on_stop)
d.addErrback(on_stop_fail)
self._component_stopping_deferred = d
else:
d = maybeDeferred(on_stop, None)
if self._component_state == "Stopping":
return self._component_stopping_deferred
return succeed(None)
def initialize(self):
for test_class, _ in self.testCases:
# Initialize Input Processor
test_class.inputs = yield defer.maybeDeferred(test_class().getInputProcessor)
# Run the setupClass method
yield defer.maybeDeferred(test_class.setUpClass)
def is_user_admin(self, user):
"""Check if an user is admin - returns a deferred!"""
user = user.lower()
d = defer.Deferred()
def _cb_userinfo(userinfo):
if not userinfo:
d.callback(False)
else:
if userinfo[2] in self.get_adminlist():
d.callback(True)
else:
d.callback(False)
def _cb_auth(authinfo):
if not authinfo:
d.callback(False)
else:
if authinfo[1] in self.get_adminlist():
d.callback(True)
else:
d.callback(False)
if self.config["Connection"].get("adminbyhost", False):
maybe_def = defer.maybeDeferred(self.user_info, user)
maybe_def.addCallback(_cb_userinfo)
else:
maybe_def = defer.maybeDeferred(self.get_auth, user)
maybe_def.addCallback(_cb_auth)
return d
def _component_start(self):
def on_start(result):
self._component_state = "Started"
self._component_starting_deferred = None
self._component_start_timer()
return True
def on_start_fail(result):
self._component_state = "Stopped"
self._component_starting_deferred = None
log.error(result)
return result
if self._component_state == "Stopped":
if hasattr(self, "start"):
self._component_state = "Starting"
d = maybeDeferred(self.start)
d.addCallback(on_start)
d.addErrback(on_start_fail)
self._component_starting_deferred = d
else:
d = maybeDeferred(on_start, None)
elif self._component_state == "Starting":
return self._component_starting_deferred
elif self._component_state == "Started":
d = succeed(True)
else:
d = fail("Cannot start a component not in a Stopped state!")
return d
def checkid_setup(registry, requestData, user=None):
"""
This method will validate and redirect a successful request to its
return_to param. If the user isn't logged in, or doesn't have an account,
we'll redirect to an internal page.
@param registry: the current OpenID registry
@type registry: L{OpenIDRegistry}
@param requestData: the current request data
@type requestData: L{OpenIDRequest}
@param user: the current user
@type user: L{txopenid.user.User}
@return: association response
@rtype: L{nevow.url.URL}
"""
if(user is not None):
def _identity_state():
return user.hasIdentity(requestData['openid.identity'])
def _trust_state():
return user.trustsRoot(requestData['openid.trust_root'])
if not(yield maybeDeferred(_identity_state)):
return_to = util.appendQuery(OPENID_IDENTITY_URL, requestData)
elif not(yield maybeDeferred(_trust_state)):
return_to = util.appendQuery(OPENID_TRUST_URL, requestData)
else:
return_to = get_login_response(registry, requestData)
else:
return_to = util.appendQuery(OPENID_LOGIN_URL, requestData)
returnValue(URL.fromString(return_to))
def poll(self):
if self.dq.pending:
return
c = yield maybeDeferred(self.q.count)
if c:
msg = yield maybeDeferred(self.q.pop)
returnValue(self.dq.put(self._message(msg)))
def testUNIX(self):
if not interfaces.IReactorUNIX(reactor, None):
raise unittest.SkipTest, "This reactor does not support UNIX domain sockets"
s = service.MultiService()
s.startService()
factory = protocol.ServerFactory()
factory.protocol = TestEcho
TestEcho.d = defer.Deferred()
t = internet.UNIXServer("echo.skt", factory)
t.setServiceParent(s)
class Foo(basic.LineReceiver):
def connectionMade(self):
self.transport.write("lalala\r\n")
def lineReceived(self, line):
self.factory.line = line
self.transport.loseConnection()
factory = protocol.ClientFactory()
factory.protocol = Foo
factory.line = None
internet.UNIXClient("echo.skt", factory).setServiceParent(s)
util.spinWhile(lambda: factory.line is None)
self.assertEqual(factory.line, "lalala")
util.wait(defer.maybeDeferred(s.stopService))
util.wait(TestEcho.d)
TestEcho.d = defer.Deferred()
factory.line = None
s.startService()
util.spinWhile(lambda: factory.line is None)
self.assertEqual(factory.line, "lalala")
util.wait(defer.maybeDeferred(s.stopService))
util.wait(TestEcho.d)
def _callMethod(self, request_dict):
"""
Here we actually find and call the method.
@type request_dict: dict
@param request_dict: Dict with details about the method
@rtype: Deferred
@return: Deferred, that will eventually fire with the method's result.
@raise JSONRPCError: When method not found.
"""
function = getattr(self, 'jsonrpc_%s' % request_dict['method'], None)
if callable(function):
if 'params' in request_dict:
if isinstance(request_dict['params'], dict):
d = maybeDeferred(function, **request_dict['params'])
else:
d = maybeDeferred(function, *request_dict['params'])
else:
d = maybeDeferred(function)
return d
else:
msg = 'Method %s not found' % request_dict['method']
raise jsonrpc.JSONRPCError(msg, jsonrpc.METHOD_NOT_FOUND,
id_=request_dict['id'],
version=request_dict['jsonrpc'])
def doProvisioning(self, authtoken):
"""
Loops on all the provisioners to perform the provisioning.
@param authtoken: AuthenticationToken containing user informations
@type authtoken: AuthenticationToken
@return; Deferred resulting to authtoken
"""
def _cbError(failure):
self.logger.exception(failure.getTraceback())
raise ProvisioningError
d = None
if authtoken.isAuthenticated():
login = authtoken.getLogin()
for name, klass in self.components:
self.logger.debug("Provisioning user with %s / %s" % (name, str(klass)))
instance = klass()
if login.lower() in instance.config.exclude:
self.logger.debug("User %s is in the exclude list of this provisioner, so skipping it" % login)
continue
if not d:
d = defer.maybeDeferred(instance.doProvisioning, authtoken)
else:
d.addCallback(lambda x: defer.maybeDeferred(instance.doProvisioning, authtoken))
if d:
ret = d.addCallback(lambda x:authtoken).addErrback(_cbError)
else:
ret = defer.succeed(authtoken)
return ret
def testPrivileged(self):
factory = protocol.ServerFactory()
factory.protocol = TestEcho
TestEcho.d = defer.Deferred()
t = internet.TCPServer(0, factory)
t.privileged = 1
t.privilegedStartService()
num = t._port.getHost().port
class Foo(basic.LineReceiver):
def connectionMade(self):
self.transport.write("lalala\r\n")
def lineReceived(self, line):
self.factory.line = line
self.transport.loseConnection()
factory = protocol.ClientFactory()
factory.protocol = Foo
factory.line = None
c = internet.TCPClient("127.0.0.1", num, factory)
c.startService()
util.spinWhile(lambda: factory.line is None)
self.assertEqual(factory.line, "lalala")
util.wait(defer.maybeDeferred(c.stopService))
util.wait(defer.maybeDeferred(t.stopService))
util.wait(TestEcho.d)
def twisted_collect(self, S3Connection, TwistedS3Connection):
key = Mock()
key.size = 3
bucket = Mock()
bucket.get_all_keys.return_value = maybeDeferred(lambda: [key] * 2)
bucket.name = sentinel.component
connection = Mock()
connection.get_all_buckets.return_value = maybeDeferred(lambda: [bucket])
TwistedS3Connection.adapt.return_value = connection
S3Connection.return_value = None
config = Mock()
config.datasources = [MagicMock()]
config.datasources[0].component = sentinel.component
config.datasources[0].ec2accesskey = "ACCESSKEY"
config.datasources[0].ec2secretkey = "SECRETKEY"
from ZenPacks.zenoss.AWS.dsplugins import S3BucketPlugin
plugin = S3BucketPlugin()
data = yield plugin.collect(config)
self.assertIn(sentinel.component, data["values"])
d = data["values"][sentinel.component]
self.assertEquals(d["keys_count"][0], 2)
self.assertEquals(d["total_size"][0], 6)
def privmsg(self, user, channel, message, tasks=None):
try:
message = message.decode('utf-8')
except UnicodeDecodeError:
try:
message = message.decode('iso-8859-1')
except UnicodeDecodeError:
message = message.decode('cp1252')
message = cleanblanks(message)
nick, user = self.log(message, user, channel)
d = None
if channel == "#gazouilleur" and not message.startswith("%schans" % config.COMMAND_CHARACTER):
return
if not message.startswith(config.COMMAND_CHARACTER):
if self.nickname.lower() in message.lower():
d = defer.maybeDeferred(self.command_test)
else:
return
message = message.encode('utf-8')
if config.DEBUG:
loggvar("COMMAND: %s: %s" % (user, message), channel)
command, _, rest = message.lstrip(config.COMMAND_CHARACTER).partition(' ')
func = self._find_command_function(command)
if func is None and d is None:
d = defer.maybeDeferred(self.command_help, command, channel, nick)
target = nick if channel == self.nickname else channel
if d is None:
if self._can_user_do(nick, channel, func):
d = defer.maybeDeferred(func, rest, channel, nick)
else:
return self._send_message("Sorry, you don't have the rights to use this command in this channel.", target, nick)
d.addCallback(self._send_message, target, nick, tasks=tasks)
d.addErrback(self._show_error, target, nick)
def check_password(self, username, password):
if username == "":
log.info("Rejected worker for blank username")
defer.returnValue(False)
allowed_chars = Set('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.')
if Set(username).issubset(allowed_chars) != True:
log.info("Username contains bad arguments")
defer.returnValue(False)
if username.count('.') > 1:
log.info("Username contains multiple . ")
defer.returnValue(False)
# Force username and password to be strings
username = str(username)
password = str(password)
if not settings.USERS_CHECK_PASSWORD and (yield self.user_exists(username)):
defer.returnValue(True)
elif self.cache.get(username) == password:
defer.returnValue(True)
elif (yield defer.maybeDeferred(self.dbi.check_password, username, password)):
self.cache.set(username, password)
defer.returnValue(True)
elif settings.USERS_AUTOADD == True:
uid = yield defer.maybeDeferred(self.dbi.get_uid, username)
if uid != False:
self.dbi.insert_worker(uid, username, password)
self.cache.set(username, password)
defer.returnValue(True)
log.info("Authentication for %s failed" % username)
defer.returnValue(False)
def test_no_userpass(self):
d = defer.maybeDeferred(manual.CommandlineUserManager)
return self.assertFailure(d, AssertionError)
def requestAvatarId(self, credentials):
d = defer.maybeDeferred(self.checkKey, credentials)
d.addCallback(self._cbRequestAvatarId, credentials)
d.addErrback(self._ebRequestAvatarId)
return d
def wrapped(*args: Any, **kwargs: Any) -> "defer.Deferred[Dict]":
# If we're passed a cache_context then we'll want to call its
# invalidate() whenever we are invalidated
invalidate_callback = kwargs.pop("on_invalidate", None)
arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
list_args = arg_dict[self.list_name]
results = {}
def update_results_dict(res: Any, arg: Hashable) -> None:
results[arg] = res
# list of deferreds to wait for
cached_defers = []
missing = set()
# If the cache takes a single arg then that is used as the key,
# otherwise a tuple is used.
if num_args == 1:
def arg_to_cache_key(arg: Hashable) -> Hashable:
return arg
else:
keylist = list(keyargs)
def arg_to_cache_key(arg: Hashable) -> Hashable:
keylist[self.list_pos] = arg
return tuple(keylist)
for arg in list_args:
try:
res = cache.get(arg_to_cache_key(arg),
callback=invalidate_callback)
if not res.called:
res.addCallback(update_results_dict, arg)
cached_defers.append(res)
else:
results[arg] = res.result
except KeyError:
missing.add(arg)
if missing:
# we need a deferred for each entry in the list,
# which we put in the cache. Each deferred resolves with the
# relevant result for that key.
deferreds_map = {}
for arg in missing:
deferred: "defer.Deferred[Any]" = defer.Deferred()
deferreds_map[arg] = deferred
key = arg_to_cache_key(arg)
cached_defers.append(
cache.set(key, deferred, callback=invalidate_callback))
def complete_all(res: Dict[Hashable, Any]) -> None:
# the wrapped function has completed. It returns a dict.
# We can now update our own result map, and then resolve the
# observable deferreds in the cache.
for e, d1 in deferreds_map.items():
val = res.get(e, None)
# make sure we update the results map before running the
# deferreds, because as soon as we run the last deferred, the
# gatherResults() below will complete and return the result
# dict to our caller.
results[e] = val
d1.callback(val)
def errback_all(f: Failure) -> None:
# the wrapped function has failed. Propagate the failure into
# the cache, which will invalidate the entry, and cause the
# relevant cached_deferreds to fail, which will propagate the
# failure to our caller.
for d1 in deferreds_map.values():
d1.errback(f)
args_to_call = dict(arg_dict)
args_to_call[self.list_name] = missing
# dispatch the call, and attach the two handlers
defer.maybeDeferred(preserve_fn(self.orig),
**args_to_call).addCallbacks(
complete_all, errback_all)
if cached_defers:
d = defer.gatherResults(cached_defers,
consumeErrors=True).addCallbacks(
lambda _: results,
unwrapFirstError)
if missing:
# We started a new call to `self.orig`, so we must always wait for it to
# complete. Otherwise we might mark our current logging context as
# finished while `self.orig` is still using it in the background.
d = delay_cancellation(d)
return make_deferred_yieldable(d)
else:
return defer.succeed(results)
def read(self):
d = maybeDeferred(self.interface.scrape_all)
d.addCallbacks(self.read_callback, self.read_errback)
def reconfigServiceWithBuildbotConfig(self, new_config):
www = new_config.www
self.authz = www.get('authz')
if self.authz is not None:
self.authz.setMaster(self.master)
need_new_site = False
if self.site:
# if config params have changed, set need_new_site to True.
# There are none right now.
need_new_site = False
else:
if www['port']:
need_new_site = True
if need_new_site:
self.setupSite(new_config)
if self.site:
self.reconfigSite(new_config)
if www['port'] != self.port:
if self.port_service:
yield defer.maybeDeferred(
lambda: self.port_service.disownServiceParent())
self.port_service = None
self.port = www['port']
if self.port:
port = self.port
if isinstance(port, int):
port = "tcp:%d" % port
self.port_service = strports.service(port, self.site)
# monkey-patch in some code to get the actual Port object
# returned by endpoint.listen(). But only for tests.
if port == "tcp:0:interface=127.0.0.1":
if hasattr(self.port_service, 'endpoint'):
old_listen = self.port_service.endpoint.listen
def listen(factory):
d = old_listen(factory)
@d.addCallback
def keep(port):
self._getPort = lambda: port
return port
return d
self.port_service.endpoint.listen = listen
else:
# older twisted's just have the port sitting there
# as an instance attribute
self._getPort = lambda: self.port_service._port
yield self.port_service.setServiceParent(self)
if not self.port_service:
log.msg("No web server configured on this master")
yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
self, new_config)
def run_function():
d = defer.maybeDeferred(function, *args, **kwargs)
d.addCallbacks(self._got_success, self._got_failure)
d.addBoth(self._stop_reactor)
def test_no_port(self):
d = defer.maybeDeferred(manual.CommandlineUserManager,
username="******",
passwd="y")
return self.assertFailure(d, AssertionError)
def tearDown(self):
if self.endpoint:
return defer.maybeDeferred(self.endpoint.stopListening)
def _callFromThread(): result = defer.maybeDeferred(f, *a, **kw) result.addBoth(queue.put)
def stopService(self):
res = defer.maybeDeferred(AbstractBuildSlave.stopService, self)
if self.slave is not None:
d = self._soft_disconnect()
res = defer.DeferredList([res, d])
return res
def _stop(self):
result = yield defer.maybeDeferred(self.stop)
self._running = False
defer.returnValue(result)
def startService(self, _reactor=reactor):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
_reactor.callWhenRunning(d.callback, None)
yield d
try:
# load the configuration file, treating errors as fatal
try:
self.config = config.MasterConfig.loadConfig(self.basedir,
self.configFileName)
except config.ConfigErrors, e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
_reactor.stop()
return
except:
klog.err_json(failure.Failure(), 'while starting BuildMaster')
_reactor.stop()
return
# set up services that need access to the config before everything else
# gets told to reconfig
try:
yield self.db.setup()
except connector.DatabaseNotReadyError:
# (message was already logged)
_reactor.stop()
return
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
_reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# call the parent method
yield defer.maybeDeferred(lambda :
service.MultiService.startService(self))
# give all services a chance to load the new configuration, rather than
# the base configuration
yield self.reconfigService(self.config)
def handleMessage(self, message_container):
# IHL7Receiver allows implementations to return a Deferred or the
# result, so ensure we return a Deferred here
return defer.maybeDeferred(self.receiver.handleMessage, message_container)
def renderHTTP(self, ctx):
return defer.maybeDeferred(_File.renderHTTP, self,
ctx).addBoth(self._cleanup)
def create_nodes(self, reactor, names, distribution, metadata={}):
"""
Create nodes with the given names.
:param reactor: The reactor.
:param name: The names of the nodes.
:type name: list of str
:param str distribution: The name of the distribution to
install on the nodes.
:param dict metadata: Metadata to associate with the nodes.
:return: A list of ``Deferred``s each firing with an INode
when the corresponding node is created. The list has
the same order as :param:`names`.
"""
size = self._default_size
image_name = self._image_names[distribution]
create_node_arguments = self._create_node_arguments()
def handle_create_error(failure, name):
# XXX This could be dangerous... What about a pre-existing
# node with the same name (or even multiple nodes if the name
# does not have to be unique)?
Message.log(
message_type="flocker:provision:libcloud:create_node:failed",
node_name=name,
)
write_failure(failure)
d = self._async_cleanup_node_named(reactor, name)
d.addCallback(lambda _: failure)
return d
def make_node(node):
public_address = _filter_ipv4(node.public_ips)[0]
if isinstance(public_address, unicode):
public_address = public_address.encode("ascii")
if self._use_private_addresses:
private_address = _filter_ipv4(node.private_ips)[0]
else:
private_address = None
if isinstance(private_address, unicode):
private_address = private_address.encode("ascii")
Message.log(
message_type="flocker:provision:libcloud:node_created",
name=node.name,
id=node.id,
public_address=public_address,
private_address=private_address,
)
return LibcloudNode(provisioner=self,
node=node,
address=public_address,
private_address=private_address,
distribution=distribution)
action = start_action(
action_type=u"flocker:provision:libcloud:create_nodes",
instance_count=len(names),
distribution=distribution,
size=size,
metadata=metadata,
)
with action.context():
results = []
for name in names:
Message.log(
message_type=u"flocker:provision:libcloud:creating_node",
node_name=name,
)
d = maybeDeferred(self._driver.create_node,
name=name,
image=get_image(self._driver, image_name),
size=get_size(self._driver, size),
ex_keyname=self._keyname,
ex_metadata=metadata,
**create_node_arguments)
d = DeferredContext(d)
d.addCallbacks(
lambda node: self._wait_until_running(reactor, node),
errback=handle_create_error,
errbackArgs=(name, ),
)
d.addCallback(make_node)
results.append(d.result)
action_completion = DeferredContext(DeferredList(results))
action_completion.addActionFinish()
# Individual results and errors should be consumed by the caller,
# so we can leave action_completion alone now.
return results
def _setup(self):
result = yield defer.maybeDeferred(self.start)
self._running = True
defer.returnValue(result)
def render_POST(self, request):
request.content.seek(0, 0)
request.setHeader("content-type", "text/xml")
try:
args, functionPath = xmlrpclib.loads(request.content.read())
except Exception, e:
f = Fault(self.FAILURE, "Can't deserialize input: %s" % (e, ))
self._cbRender(f, request)
else:
try:
function = self._getFunction(functionPath)
except Fault, f:
self._cbRender(f, request)
else:
d = defer.maybeDeferred(function, *args)
d.addErrback(self._ebRender)
d.addCallback(self._cbRender, request)
return server.NOT_DONE_YET
def _cbRender(self, result, request):
if isinstance(result, Handler):
result = result.result
if not isinstance(result, Fault):
result = (result, )
try:
try:
content = xmlrpclib.dumps(result,
methodresponse=True,
allow_none=self.allowNone)
except Exception, e:
def make_call():
d = maybeDeferred(f)
d = DeferredContext(d)
d.addCallbacks(handle_success, errback=handle_failure)
return d.result
def stop(self):
"""Starts a graceful stop of the crawler and returns a deferred that is
fired when the crawler is stopped."""
if self.crawling:
self.crawling = False
yield defer.maybeDeferred(self.engine.stop)
def _handleGotRevision(self, res):
d = defer.maybeDeferred(self.parseGotRevision)
d.addCallback(lambda got_revision: self.sendStatus(
{'got_revision': got_revision}))
return d
def wrapped(*args, **kwargs):
# If we're passed a cache_context then we'll want to call its
# invalidate() whenever we are invalidated
invalidate_callback = kwargs.pop("on_invalidate", None)
arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
list_args = arg_dict[self.list_name]
results = {}
def update_results_dict(res, arg):
results[arg] = res
# list of deferreds to wait for
cached_defers = []
missing = set()
# If the cache takes a single arg then that is used as the key,
# otherwise a tuple is used.
if num_args == 1:
def arg_to_cache_key(arg):
return arg
else:
keylist = list(keyargs)
def arg_to_cache_key(arg):
keylist[self.list_pos] = arg
return tuple(keylist)
for arg in list_args:
try:
res = cache.get(arg_to_cache_key(arg), callback=invalidate_callback)
if not res.called:
res.addCallback(update_results_dict, arg)
cached_defers.append(res)
else:
results[arg] = res.result
except KeyError:
missing.add(arg)
if missing:
# we need a deferred for each entry in the list,
# which we put in the cache. Each deferred resolves with the
# relevant result for that key.
deferreds_map = {}
for arg in missing:
deferred = defer.Deferred()
deferreds_map[arg] = deferred
key = arg_to_cache_key(arg)
cache.set(key, deferred, callback=invalidate_callback)
def complete_all(res):
# the wrapped function has completed. It returns a
# a dict. We can now resolve the observable deferreds in
# the cache and update our own result map.
for e in missing:
val = res.get(e, None)
deferreds_map[e].callback(val)
results[e] = val
def errback(f):
# the wrapped function has failed. Invalidate any cache
# entries we're supposed to be populating, and fail
# their deferreds.
for e in missing:
key = arg_to_cache_key(e)
cache.invalidate(key)
deferreds_map[e].errback(f)
# return the failure, to propagate to our caller.
return f
args_to_call = dict(arg_dict)
# copy the missing set before sending it to the callee, to guard against
# modification.
args_to_call[self.list_name] = tuple(missing)
cached_defers.append(
defer.maybeDeferred(
preserve_fn(self.orig), **args_to_call
).addCallbacks(complete_all, errback)
)
if cached_defers:
d = defer.gatherResults(cached_defers, consumeErrors=True).addCallbacks(
lambda _: results, unwrapFirstError
)
return make_deferred_yieldable(d)
else:
return defer.succeed(results)
def allUploadsDone(result):
d = defer.maybeDeferred(self.allUploadsDone, result, sources, masterdest)
d.addCallback(lambda _: result)
return d
def _do_authenticate(self, protoinfo):
"""
Callback on PROTOCOLINFO to actually authenticate once we know
what's supported.
"""
methods = None
cookie_auth = False
for line in protoinfo.split('\n'):
if line[:5] == 'AUTH ':
kw = parse_keywords(line[5:].replace(' ', '\n'))
methods = kw['METHODS'].split(',')
if not methods:
raise RuntimeError(
"Didn't find AUTH line in PROTOCOLINFO response.")
if 'SAFECOOKIE' in methods or 'COOKIE' in methods:
cookiefile_match = re.search(r'COOKIEFILE=("(?:[^"\\]|\\.)*")',
protoinfo)
if cookiefile_match:
cookiefile = cookiefile_match.group(1)
cookiefile = unescape_quoted_string(cookiefile)
try:
self._read_cookie(cookiefile)
cookie_auth = True
except IOError as why:
txtorlog.msg("Reading COOKIEFILE failed: " + str(why))
if self.password_function and 'HASHEDPASSWORD' in methods:
txtorlog.msg("Falling back to password")
else:
raise RuntimeError(
"Failed to read COOKIEFILE '{fname}': {msg}\n".
format(
fname=cookiefile,
msg=str(why),
)
# "On Debian, join the debian-tor group"
)
else:
txtorlog.msg("Didn't get COOKIEFILE")
raise RuntimeError(
"Got 'COOKIE' or 'SAFECOOKIE' method, but no 'COOKIEFILE'")
if cookie_auth:
if 'SAFECOOKIE' in methods:
txtorlog.msg("Using SAFECOOKIE authentication", cookiefile,
len(self._cookie_data), "bytes")
self.client_nonce = os.urandom(32)
cmd = b'AUTHCHALLENGE SAFECOOKIE ' + \
hexlify(self.client_nonce)
d = self.queue_command(cmd)
d.addCallback(self._safecookie_authchallenge)
d.addCallback(self._bootstrap)
d.addErrback(self._auth_failed)
return
elif 'COOKIE' in methods:
txtorlog.msg("Using COOKIE authentication", cookiefile,
len(self._cookie_data), "bytes")
d = self.authenticate(self._cookie_data)
d.addCallback(self._bootstrap)
d.addErrback(self._auth_failed)
return
if self.password_function and 'HASHEDPASSWORD' in methods:
d = defer.maybeDeferred(self.password_function)
d.addCallback(self._do_password_authentication)
d.addErrback(self._auth_failed)
return
if 'NULL' in methods:
d = self.queue_command('AUTHENTICATE')
d.addCallback(self._bootstrap)
d.addErrback(self._auth_failed)
return
raise RuntimeError(
"The Tor I connected to doesn't support SAFECOOKIE nor COOKIE"
" authentication (or we can't read the cookie files) and I have"
" no password_function specified.")
def tearDown(self):
d3 = Deferred()
self._serverProtocol.notifyOnDisconnect(lambda: d3.callback(None))
return DeferredList([
maybeDeferred(self.serverPort.stopListening),
maybeDeferred(self.clientConn.disconnect), d3])
def startConnecting(self):
d = defer.maybeDeferred(self.sf.resolveAddress)
d.addCallback(self._cbResolveDone)
d.addErrback(self._ebResolveErr)
def runs_in_reactor(args, kwargs):
d = maybeDeferred(function, *args, **kwargs)
return EventualResult(d)
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name)
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = {s.name: s for s in new_config_attr}
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError("config.%s should be a list or dictionary" %
(self.config_attr))
new_set = set(new_by_name)
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
# While we're at it find any service that don't know how to reconfig,
# and, if they have changed, add them to both removed and added, so that we
# run the new version
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
if old != new:
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
# disownServiceParent calls stopService after removing the relationship
# as child might use self.master.data to stop itself, its better to stop it first
# (this is related to the fact that self.master is found by recursively looking at self.parent
# for a master)
yield child.stopService()
# it has already called, so do not call it again
child.stopService = lambda: None
yield child.disownServiceParent()
# HACK: we still keep a reference to the master for some cleanup tasks which are not waited by
# to stopService (like the complex worker disconnection mechanism)
# http://trac.buildbot.net/ticket/3583
child.parent = self.master
for n in added_names:
child = new_by_name[n]
# setup service's objectid
if hasattr(child, 'objectid'):
class_name = '%s.%s' % (child.__class__.__module__,
child.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
child.name, class_name)
child.objectid = objectid
yield defer.maybeDeferred(child.setServiceParent, self)
# As the services that were just added got
# reconfigServiceWithSibling called by
# setServiceParent->startService,
# we avoid calling it again by selecting
# in reconfigurable_services, services
# that were not added just now
reconfigurable_services = [
svc for svc in self if svc.name not in added_names
]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError(
"{}: child {} should have a defined name attribute".format(
self, svc))
config_sibling = new_by_name.get(svc.name)
try:
yield svc.reconfigServiceWithSibling(config_sibling)
except NotImplementedError:
# legacy support. Its too painful to transition old code to new Service life cycle
# so we implement switch of child when the service raises NotImplementedError
# Note this means that self will stop, and sibling will take ownership
# means that we have a small time where the service is unavailable.
yield svc.disownServiceParent()
config_sibling.objectid = svc.objectid
yield config_sibling.setServiceParent(self)
def submit(self, job, test=False):
""" Submit the job to the device. If the device is currently running
a job it will be queued and run when this is finished.
This handles iteration over the path model defined by the job and
sending commands to the actual device using roughly the procedure is
as follows:
device.connect()
model = device.init(job)
for cmd in device.process(model):
device.handle(cmd)
device.finish()
device.disconnect()
Subclasses provided by your own DeviceDriver may reimplement this
to handle path interpolation however needed. The return value is
ignored.
The live plot view will update whenever the device.position object
is updated. On devices with lower cpu/gpu capabilities this should
be updated sparingly (ie the raspberry pi).
Parameters
-----------
job: Instance of `inkcut.job.models.Job`
The job to execute on the device
test: bool
Do a test run. This specifies whether the commands should be
sent to the actual device or not. If True, the connection will
be replaced with a virtual connection that captures all the
command output.
"""
log.debug("device | submit {}".format(job))
try:
#: Only allow one job at a time
if self.busy:
queue = self.queue[:]
queue.append(job)
self.queue = queue #: Copy and reassign so the UI updates
log.info("Job {} put in device queue".format(job))
return
with self.device_busy():
#: Set the current the job
self.job = job
self.status = "Initializing job"
#: Get the time to sleep based for each unit of movement
config = self.config
#: Rate px/ms
if config.custom_rate >= 0:
rate = config.custom_rate
elif self.connection.always_spools or config.spooled:
rate = 0
elif config.interpolate:
if config.step_time > 0:
rate = config.step_size/float(config.step_time)
else:
rate = 0 # Undefined
else:
rate = from_unit(
config.speed, # in/s or cm/s
config.speed_units.split("/")[0])/1000.0
# Device model is updated in real time
model = yield defer.maybeDeferred(self.init, job)
#: Local references are faster
info = job.info
#: Determine the length for tracking progress
whole_path = QtGui.QPainterPath()
#: Some versions of Qt seem to require a value in
#: toSubpathPolygons
m = QtGui.QTransform.fromScale(1, 1)
for path in model.toSubpathPolygons(m):
for i, p in enumerate(path):
whole_path.lineTo(p)
total_length = whole_path.length()
total_moved = 0
log.debug("device | Path length: {}".format(total_length))
#: So a estimate of the duration can be determined
info.length = total_length
info.speed = rate*1000 #: Convert to px/s
#: Waiting for approval
info.status = 'waiting'
#: If marked for auto approve start now
if info.auto_approve:
info.status = 'approved'
else:
#: Check for approval before starting
yield defer.maybeDeferred(info.request_approval)
if info.status != 'approved':
self.status = "Job cancelled"
return
#: Update stats
info.status = 'running'
info.started = datetime.now()
self.status = "Connecting to device"
with self.device_connection(
test or config.test_mode) as connection:
self.status = "Processing job"
try:
yield defer.maybeDeferred(self.connect)
protocol = connection.protocol
#: Write startup command
if config.commands_before:
yield defer.maybeDeferred(connection.write,
config.commands_before)
self.status = "Working..."
if config.force_enabled:
yield defer.maybeDeferred(
protocol.set_force, config.force)
if config.speed_enabled:
yield defer.maybeDeferred(
protocol.set_velocity, config.speed)
#: For point in the path
for (d, cmd, args, kwargs) in self.process(model):
#: Check if we paused
if info.paused:
self.status = "Job paused"
#: Sleep until resumed, cancelled, or the
#: connection drops
while (info.paused and not info.cancelled
and connection.connected):
yield async_sleep(300) # ms
#: Check for cancel for non interpolated jobs
if info.cancelled:
self.status = "Job cancelled"
info.status = 'cancelled'
break
elif not connection.connected:
self.status = "connection error"
info.status = 'error'
break
#: Invoke the command
#: If you want to let the device handle more complex
#: commands such as curves do it in process and handle
yield defer.maybeDeferred(cmd, *args, **kwargs)
total_moved += d
#: d should be the device must move in px
#: so wait a proportional amount of time for the device
#: to catch up. This avoids buffer errors from dumping
#: everything at once.
#: Since sending is way faster than cutting
#: we must delay (without blocking the UI) before
#: sending the next command or the device's buffer
#: quickly gets filled and crappy china piece cutters
#: get all jacked up. If the transport sends to a spooled
#: output (such as a printer) this can be set to 0
if rate > 0:
# log.debug("d={}, delay={} t={}".format(
# d, delay, d/delay
# ))
yield async_sleep(d/rate)
#: TODO: Check if we need to update the ui
#: Set the job progress based on how far we've gone
if total_length > 0:
info.progress = int(max(0, min(100,
100*total_moved/total_length)))
if info.status != 'error':
#: We're done, send any finalization commands
yield defer.maybeDeferred(self.finish)
#: Write finalize command
if config.commands_after:
yield defer.maybeDeferred(connection.write,
config.commands_after)
#: Update stats
info.ended = datetime.now()
#: If not cancelled or errored
if info.status == 'running':
info.done = True
info.status = 'complete'
except Exception as e:
log.error(traceback.format_exc())
raise
finally:
if connection.connected:
yield defer.maybeDeferred(self.disconnect)
#: Set the origin
if job.feed_to_end and job.info.status == 'complete':
self.origin = self.position
#: If the user didn't cancel, set the origin and
#: Process any jobs that entered the queue while this was running
if self.queue and not job.info.cancelled:
queue = self.queue[:]
job = queue.pop(0) #: Pull the first job off the queue
log.info("Rescheduling {} from queue".format(job))
self.queue = queue #: Copy and reassign so the UI updates
#: Call a minute later
timed_call(60000, self.submit, job)
except Exception as e:
log.error(' device | Execution error {}'.format(
traceback.format_exc()))
raise
def storeResource(
request,
source=None, source_uri=None, data=None,
destination=None, destination_uri=None,
deletesource=False,
depth="0"
):
"""
Function that does common PUT/COPY/MOVE behaviour.
@param request: the L{txweb2.server.Request} for the current HTTP request.
@param source: the L{DAVFile} for the source resource to copy from, or None if source data
is to be read from the request.
@param source_uri: the URI for the source resource.
@param data: a C{str} to copy data from instead of the request stream.
@param destination: the L{DAVFile} for the destination resource to copy into.
@param destination_uri: the URI for the destination resource.
@param deletesource: True if the source resource is to be deleted on successful completion, False otherwise.
@param depth: a C{str} containing the COPY/MOVE Depth header value.
@return: status response.
"""
try:
assert request is not None and destination is not None and destination_uri is not None
assert (source is None) or (source is not None and source_uri is not None)
assert not deletesource or (deletesource and source is not None)
except AssertionError:
log.error("Invalid arguments to storeResource():")
log.error("request=%s\n" % (request,))
log.error("source=%s\n" % (source,))
log.error("source_uri=%s\n" % (source_uri,))
log.error("data=%s\n" % (data,))
log.error("destination=%s\n" % (destination,))
log.error("destination_uri=%s\n" % (destination_uri,))
log.error("deletesource=%s\n" % (deletesource,))
log.error("depth=%s\n" % (depth,))
raise
class RollbackState(object):
"""
This class encapsulates the state needed to rollback the entire PUT/COPY/MOVE
transaction, leaving the server state the same as it was before the request was
processed. The DoRollback method will actually execute the rollback operations.
"""
def __init__(self):
self.active = True
self.source_copy = None
self.destination_copy = None
self.destination_created = False
self.source_deleted = False
def Rollback(self):
"""
Rollback the server state. Do not allow this to raise another exception. If
rollback fails then we are going to be left in an awkward state that will need
to be cleaned up eventually.
"""
if self.active:
self.active = False
log.error("Rollback: rollback")
try:
if self.source_copy and self.source_deleted:
self.source_copy.moveTo(source.fp)
log.error("Rollback: source restored %s to %s" % (self.source_copy.path, source.fp.path))
self.source_copy = None
self.source_deleted = False
if self.destination_copy:
destination.fp.remove()
log.error("Rollback: destination restored %s to %s" % (self.destination_copy.path, destination.fp.path))
self.destination_copy.moveTo(destination.fp)
self.destination_copy = None
elif self.destination_created:
destination.fp.remove()
log.error("Rollback: destination removed %s" % (destination.fp.path,))
self.destination_created = False
except:
log.error("Rollback: exception caught and not handled: %s" % Failure())
def Commit(self):
"""
Commit the resource changes by wiping the rollback state.
"""
if self.active:
log.error("Rollback: commit")
self.active = False
if self.source_copy:
self.source_copy.remove()
log.error("Rollback: removed source backup %s" % (self.source_copy.path,))
self.source_copy = None
if self.destination_copy:
self.destination_copy.remove()
log.error("Rollback: removed destination backup %s" % (self.destination_copy.path,))
self.destination_copy = None
self.destination_created = False
self.source_deleted = False
rollback = RollbackState()
try:
"""
Handle validation operations here.
"""
"""
Handle rollback setup here.
"""
# Do quota checks on destination and source before we start messing with adding other files
destquota = waitForDeferred(destination.quota(request))
yield destquota
destquota = destquota.getResult()
if destquota is not None and destination.exists():
old_dest_size = waitForDeferred(destination.quotaSize(request))
yield old_dest_size
old_dest_size = old_dest_size.getResult()
else:
old_dest_size = 0
if source is not None:
sourcequota = waitForDeferred(source.quota(request))
yield sourcequota
sourcequota = sourcequota.getResult()
if sourcequota is not None and source.exists():
old_source_size = waitForDeferred(source.quotaSize(request))
yield old_source_size
old_source_size = old_source_size.getResult()
else:
old_source_size = 0
else:
sourcequota = None
old_source_size = 0
# We may need to restore the original resource data if the PUT/COPY/MOVE fails,
# so rename the original file in case we need to rollback.
overwrite = destination.exists()
if overwrite:
rollback.destination_copy = FilePath(destination.fp.path)
rollback.destination_copy.path += ".rollback"
destination.fp.copyTo(rollback.destination_copy)
else:
rollback.destination_created = True
if deletesource:
rollback.source_copy = FilePath(source.fp.path)
rollback.source_copy.path += ".rollback"
source.fp.copyTo(rollback.source_copy)
"""
Handle actual store operations here.
"""
# Do put or copy based on whether source exists
if source is not None:
response = maybeDeferred(copy, source.fp, destination.fp, destination_uri, depth)
else:
datastream = request.stream
if data is not None:
datastream = MemoryStream(data)
md5 = MD5Stream(datastream)
response = maybeDeferred(put, md5, destination.fp)
response = waitForDeferred(response)
yield response
response = response.getResult()
# Update the MD5 value on the resource
if source is not None:
# Copy MD5 value from source to destination
if source.hasDeadProperty(TwistedGETContentMD5):
md5 = source.readDeadProperty(TwistedGETContentMD5)
destination.writeDeadProperty(md5)
else:
# Finish MD5 calc and write dead property
md5.close()
md5 = md5.getMD5()
destination.writeDeadProperty(TwistedGETContentMD5.fromString(md5))
# Update the content-type value on the resource if it is not been copied or moved
if source is None:
content_type = request.headers.getHeader("content-type")
if content_type is not None:
destination.writeDeadProperty(davxml.GETContentType.fromString(generateContentType(content_type)))
response = IResponse(response)
# Do quota check on destination
if destquota is not None:
# Get size of new/old resources
new_dest_size = waitForDeferred(destination.quotaSize(request))
yield new_dest_size
new_dest_size = new_dest_size.getResult()
diff_size = new_dest_size - old_dest_size
if diff_size >= destquota[0]:
log.error("Over quota: available %d, need %d" % (destquota[0], diff_size))
raise HTTPError(ErrorResponse(
responsecode.INSUFFICIENT_STORAGE_SPACE,
(dav_namespace, "quota-not-exceeded")
))
d = waitForDeferred(destination.quotaSizeAdjust(request, diff_size))
yield d
d.getResult()
if deletesource:
# Delete the source resource
if sourcequota is not None:
delete_size = 0 - old_source_size
d = waitForDeferred(source.quotaSizeAdjust(request, delete_size))
yield d
d.getResult()
delete(source_uri, source.fp, depth)
rollback.source_deleted = True
# Can now commit changes and forget the rollback details
rollback.Commit()
yield response
return
except:
# Roll back changes to original server state. Note this may do nothing
# if the rollback has already ocurred or changes already committed.
rollback.Rollback()
raise
def _on_authorized(self, is_authorized, request, worker_name):
data = json.loads(request.content.read())
if not is_authorized:
request.write(
self.json_error(data.get('id', 0), -1,
"Bad worker credentials"))
request.finish()
return
if not self.job_registry.last_job:
log.warning('Getworkmaker is waiting for a job...')
request.write(
self.json_error(data.get('id', 0), -1,
"Getworkmake is waiting for a job..."))
request.finish()
return
if data['method'] == 'getwork':
if 'params' not in data or not len(data['params']):
# getwork request
log.info("Worker '%s' asks for new work" % worker_name)
extensions = request.getHeader('x-mining-extensions')
no_midstate = extensions and 'midstate' in extensions
request.write(
self.json_response(
data.get('id', 0),
self.job_registry.getwork(no_midstate=no_midstate)))
request.finish()
return
else:
# submit
d = defer.maybeDeferred(self.job_registry.submit,
data['params'][0], worker_name)
start_time = time.time()
d.addCallback(self._on_submit, request, data.get('id', 0),
data['params'][0][:160], worker_name, start_time)
d.addErrback(self._on_submit_failure, request,
data.get('id', 0), data['params'][0][:160],
worker_name, start_time)
return
elif data['method'] == 'eth_getWork':
#log.info("Worker '%s' eth_getWork" % worker_name)
request.write(
self.json_response(data.get('id', 0),
self.job_registry.getwork()))
request.finish()
return
elif data['method'] == 'eth_submitWork':
d = defer.maybeDeferred(self.job_registry.submit,
data['params'][0], data['params'][1],
data['params'][2], worker_name)
start_time = time.time()
d.addCallback(self._on_submit, request, data.get('id', 0),
data['params'], worker_name, start_time)
d.addErrback(self._on_submit_failure, request, data.get('id', 0),
data['params'], worker_name, start_time)
return
elif data['method'] == 'eth_submitHashrate':
#log.info("Worker '%s' eth_submitHashrate" % worker_name)
request.write(self.json_response(data.get('id', 0), True))
request.finish()
return
request.write(
self.json_error(1, -1, "Unsupported method '%s'" % data['method']))
request.finish()
