def test_unauthenticated(self): """ Test the behavior of locateChild when SACLs are enabled and the request is unauthenticated should return a 401 UnauthorizedResponse """ self.root.resource.useSacls = True request = SimpleRequest( self.site, "GET", "/principals/" ) resrc, segments = (yield maybeDeferred( self.root.locateChild, request, ["principals"] )) try: resrc, segments = (yield maybeDeferred( resrc.locateChild, request, ["principals"] )) raise AssertionError( "RootResource.locateChild did not return an error" ) except HTTPError, e: self.assertEquals(e.response.code, 401)
def _call_method(self, request): """Calls given method with given params and returns it value.""" method = self.method_data[request['method']]['method'] params = request['params'] result = None try: if isinstance(params, list): # Does it have enough arguments? if len(params) < self._man_args(method): raise InvalidParamsError('not enough arguments') # Does it have too many arguments? if not self._vargs(method) \ and len(params) > self._max_args(method): raise InvalidParamsError('too many arguments') result = yield defer.maybeDeferred(method, *params) elif isinstance(params, dict): # Do not accept keyword arguments if the jsonrpc version is # not >=1.1. if request['jsonrpc'] < 11: raise KeywordError result = yield defer.maybeDeferred(method, **params) else: # No params result = yield defer.maybeDeferred(method) except JSONRPCError: raise except Exception: # Exception was raised inside the method. log.msg('Exception raised while invoking RPC method "{}".'.format( request['method'])) log.err() raise ServerError defer.returnValue(result)
def process_result(result, total=None, found_item=None): if result == None: result = [] l = [] def process_items(result, tm): if result is None: result = [] for i in result: if i[0]: didl.addItem(i[1]) return build_response(tm) for i in result: d = defer.maybeDeferred(i.get_item) l.append(d) if found_item is not None: def got_child_count(count): dl = defer.DeferredList(l) dl.addCallback(process_items, count) return dl d = defer.maybeDeferred(found_item.get_child_count) d.addCallback(got_child_count) return d elif total is None: total = item.get_child_count() dl = defer.DeferredList(l) dl.addCallback(process_items, total) return dl
def test_badCredentials(self): """ Test the behavior of locateChild when SACLS are enabled, and incorrect credentials are given. should return a 401 UnauthorizedResponse """ self.root.resource.useSacls = True request = SimpleRequest( self.site, "GET", "/principals/", headers=http_headers.Headers({ "Authorization": ["basic", "%s" % ( "dreid:dreid".encode("base64"),)]})) resrc, segments = (yield maybeDeferred( self.root.locateChild, request, ["principals"] )) try: resrc, segments = (yield maybeDeferred( resrc.locateChild, request, ["principals"] )) except HTTPError, e: self.assertEquals(e.response.code, 401)
def render(self, request): """This is a modified version of render() from XMLRPC that will pass keyword arguments with extra state information if it can. Currently this just consists of the current request. """ request.content.seek(0, 0) args, functionPath = xmlrpclib.loads(request.content.read()) try: function = self._getFunction(functionPath) except xmlrpc.NoSuchFunction: self._cbRender( xmlrpc.Fault(self.NOT_FOUND, "no such function %s" % functionPath), request ) else: request.setHeader("content-type", "text/xml") # Pass as many keyword args to the function as we can. # Note that this currently doesn't work on protected functions. kwargs = {} try: if 'request' in function.func_code.co_varnames: kwargs['request'] = request except AttributeError: pass defer.maybeDeferred(function, *args, **kwargs).addErrback( self._ebRender ).addCallback( self._cbRender, request ) return server.NOT_DONE_YET
def validQuery(self, portOnServer, portOnClient): serverAddr = self.transport.getHost()[1], portOnServer clientAddr = self.transport.getPeer()[1], portOnClient defer.maybeDeferred(self.lookup, serverAddr, clientAddr ).addCallback(self._cbLookup, portOnServer, portOnClient ).addErrback(self._ebLookup, portOnServer, portOnClient )
def children(self, request, tag): """ Renderer which yields all child object tags as table rows. """ whenChildren = ( maybeDeferred(self.resource.listChildren) .addCallback(sorted) .addCallback( lambda names: gatherResults( [maybeDeferred(self.resource.getChild, x) for x in names] ) .addCallback(lambda children: zip(children, names)) ) ) @whenChildren.addCallback def gotChildren(children): for even, [child, name] in zip(cycle(["odd", "even"]), children): [url, name, size, lastModified, contentType] = map( str, self.resource.getChildDirectoryEntry( child, name, request) ) yield tag.clone().fillSlots( url=url, name=name, size=str(size), lastModified=lastModified, even=even, type=contentType, ) return whenChildren
def start(self): yield defer.maybeDeferred(self.configure) for spider, batches in self._scheduled.iteritems(): yield self._schedule(spider, batches) yield defer.maybeDeferred(self.engine.start)
def dataPhase(self): while len(self.buf) >= NBDRequestHeader.size: header = NBDRequestHeader(self.buf[:NBDRequestHeader.size]) log.msg("Received %r" % header) if header.requestType == NBD_CMD_WRITE: if len(self.buf) < header.size + header.length: log.msg("Incomplete write request received") return data = self.buf[header.size:header.size+header.length] self.buf = self.buf[header.size+header.length:] d = defer.maybeDeferred(self.bdev.write, header.offset, data) d.addCallback(self.reply, header.handle) d.addErrback(self.errorReply, header.handle) elif header.requestType == NBD_CMD_READ: self.buf = self.buf[header.size:] d = defer.maybeDeferred(self.bdev.read, header.offset, header.length) d.addCallback(self.reply, header.handle) d.addErrback(self.errorReply, header.handle) elif header.requestType == NBD_CMD_DISC: self.transport.loseConnection() return else: self.buf = self.buf[header.size:] log.msg("Unsupported request type %d" % header.requestType) self.transport.write( struct.pack(">II8s", NBD_REPLY_MAGIC, 1, # Operation not permitted header.handle ) )
def testPortforward(self): serverProtocol = wire.Echo() realServerFactory = protocol.ServerFactory() realServerFactory.protocol = lambda: serverProtocol realServerPort = reactor.listenTCP(0, realServerFactory, interface='127.0.0.1') proxyServerFactory = portforward.ProxyFactory('127.0.0.1', realServerPort.getHost().port) proxyServerPort = reactor.listenTCP(0, proxyServerFactory, interface='127.0.0.1') nBytes = 1000 received = [] clientProtocol = protocol.Protocol() clientProtocol.dataReceived = received.extend clientProtocol.connectionMade = lambda: clientProtocol.transport.write('x' * nBytes) clientFactory = protocol.ClientFactory() clientFactory.protocol = lambda: clientProtocol reactor.connectTCP('127.0.0.1', proxyServerPort.getHost().port, clientFactory) c = 0 while len(received) < nBytes and c < 100: reactor.iterate(0.01) c += 1 self.assertEquals(''.join(received), 'x' * nBytes) clientProtocol.transport.loseConnection() serverProtocol.transport.loseConnection() return defer.gatherResults([ defer.maybeDeferred(realServerPort.stopListening), defer.maybeDeferred(proxyServerPort.stopListening)])
def render(self, request): """Handle a SOAP command.""" data = request.content.read() p, header, body, attrs = SOAPpy.parseSOAPRPC(data, 1, 1, 1) methodName, args, kwargs, ns = p._name, p._aslist, p._asdict, p._ns # deal with changes in SOAPpy 0.11 if callable(args): args = args() if callable(kwargs): kwargs = kwargs() function = self.lookupFunction(methodName) if not function: self._methodNotFound(request, methodName) return server.NOT_DONE_YET else: if hasattr(function, "useKeywords"): keywords = {} for k, v in kwargs.items(): keywords[str(k)] = v d = defer.maybeDeferred(function, **keywords) else: d = defer.maybeDeferred(function, *args) d.addCallback(self._gotResult, request, methodName) d.addErrback(self._gotError, request, methodName) return server.NOT_DONE_YET
def stopService(self): # This will stop plugin Processors D2 = defer.maybeDeferred(service.MultiService.stopService, self) U = defer.maybeDeferred(self.udp.stopListening) T = defer.maybeDeferred(self.tcp.stopListening) return defer.DeferredList([U,T,D2], consumeErrors=True)
def getPackagesLoop(self, result = None): if result: if not isinstance(result, failure.Failure): self.tmppackages.append(result) else: logging.getLogger().error("Error: %s", str(result)) if not self.p_apis and self.tmppackages: # Merge temporary results lp = self.tmppackages[0] map(lambda p: _merge_list(lp, p), self.tmppackages) self.packages.extend(map(lambda m: [m, self.index, self.p_api_first], lp)) self.tmppackages = [] if self.plists and not self.p_apis: # Fill self.p_apis if empty self.p_apis = self.plists.pop() self.p_api_first = self.p_apis[0][0] self.index = self.index + 1 if self.p_apis: p_api = self.p_apis.pop() if 'pending' in self.filt: d = defer.maybeDeferred(PackageGetA(p_api[0]).getAllPendingPackages, p_api[1]) else: d = defer.maybeDeferred(PackageGetA(p_api[0]).getAllPackages, p_api[1]) d.addCallbacks(self.getPackagesLoop) else: # No more remote call to do, we are done self.sendResult(self.packages)
def render_POST(self, request): request.content.seek(0, 0) request.setHeader(b"content-type", b"text/xml; charset=utf-8") try: args, functionPath = xmlrpclib.loads(request.content.read(), use_datetime=self.useDateTime) except Exception as e: f = Fault(self.FAILURE, "Can't deserialize input: %s" % (e,)) self._cbRender(f, request) else: try: function = self.lookupProcedure(functionPath) except Fault as f: self._cbRender(f, request) else: # Use this list to track whether the response has failed or not. # This will be used later on to decide if the result of the # Deferred should be written out and Request.finish called. responseFailed = [] request.notifyFinish().addErrback(responseFailed.append) if getattr(function, "withRequest", False): d = defer.maybeDeferred(function, request, *args) else: d = defer.maybeDeferred(function, *args) d.addErrback(self._ebRender) d.addCallback(self._cbRender, request, responseFailed) return server.NOT_DONE_YET
def privmsg(self, user, channel, message): nick, _, host = user.partition('!') message = message.strip() if not message.startswith('!'): # not a trigger command return # do nothing command, sep, rest = message.lstrip('!').partition(' ') # Get the function corresponding to the command given. func = getattr(self, 'command_' + command, None) # Or, if there was no function, ignore the message. if func is None: return # maybeDeferred will always return a Deferred. It calls func(rest), and # if that returned a Deferred, return that. Otherwise, return the return # value of the function wrapped in twisted.internet.defer.succeed. If # an exception was raised, wrap the traceback in # twisted.internet.defer.fail and return that. if command == 'items': rest = (channel, nick, rest) d = defer.maybeDeferred(func, rest) else: d = defer.maybeDeferred(func, rest) # Add callbacks to deal with whatever the command results are. # If the command gives error, the _show_error callback will turn the # error into a terse message first: d.addErrback(self._show_error) # Whatever is returned is sent back as a reply: if channel == self.nickname: # When channel == self.nickname, the message was sent to the bot # directly and not to a channel. So we will answer directly too: d.addCallback(self._send_message, nick) else: # Otherwise, send the answer to the channel, and use the nick # as addressing in the message itself: d.addCallback(self._send_message, channel, nick)
def run(self): """ Выполнить команду. В процессе выполнения команды или проверки условий её выполнения (наличие всех параметров) и т.п., могут возникать исключения, которые будут переданы через errback к deferred, или в случае успешного выполнения вызовется callback. @return: deferred, которая будет вызвана после завершения выполнения команды и формирования её результата @rtype: L{defer.Deferred} """ def checkResult(result): self.checkResult() return result def finalize(res): try: self.finalize() except: if not sys.modules.has_key('twisted.trial.runner'): log.err(failure.Failure()) return res try: self.checkParams() d = defer.maybeDeferred(self.init).addCallback(lambda _: defer.maybeDeferred(self.perform)) except errors.BaseCommandException: return defer.fail() return d.addCallback(checkResult).addBoth(finalize)
def dispatch_message(payload, msg, dispatchIn, conv=None): """ Dispatches a message by operation in a given class. Expected message content: body = { "op": "operation name here", "content": # Any valid type here (str, list, dict) } """ try: log_message(msg) if "op" in payload: op = payload['op'] content = payload.get('content','') opname = 'op_' + str(op) # dynamically invoke the operation in the given class if hasattr(dispatchIn, opname): opf = getattr(dispatchIn, opname) return defer.maybeDeferred(opf, content, payload, msg) elif hasattr(dispatchIn,'op_none'): return defer.maybeDeferred(dispatchIn.op_none, content, payload, msg) else: logging.error("Receive() failed. Cannot dispatch to catch") else: logging.error("Invalid message. No 'op' in header", payload) except StandardError, e: log_exception('Exception while dispatching: ',e)
def _component_stop(self): def on_stop(result): self._component_state = "Stopped" if self._component_timer and self._component_timer.running: self._component_timer.stop() return True def on_stop_fail(result): self._component_state = "Started" self._component_stopping_deferred = None log.error(result) return result if self._component_state != "Stopped" and self._component_state != "Stopping": if hasattr(self, "stop"): self._component_state = "Stopping" d = maybeDeferred(self.stop) d.addCallback(on_stop) d.addErrback(on_stop_fail) self._component_stopping_deferred = d else: d = maybeDeferred(on_stop, None) if self._component_state == "Stopping": return self._component_stopping_deferred return succeed(None)
def initialize(self): for test_class, _ in self.testCases: # Initialize Input Processor test_class.inputs = yield defer.maybeDeferred(test_class().getInputProcessor) # Run the setupClass method yield defer.maybeDeferred(test_class.setUpClass)
def is_user_admin(self, user): """Check if an user is admin - returns a deferred!""" user = user.lower() d = defer.Deferred() def _cb_userinfo(userinfo): if not userinfo: d.callback(False) else: if userinfo[2] in self.get_adminlist(): d.callback(True) else: d.callback(False) def _cb_auth(authinfo): if not authinfo: d.callback(False) else: if authinfo[1] in self.get_adminlist(): d.callback(True) else: d.callback(False) if self.config["Connection"].get("adminbyhost", False): maybe_def = defer.maybeDeferred(self.user_info, user) maybe_def.addCallback(_cb_userinfo) else: maybe_def = defer.maybeDeferred(self.get_auth, user) maybe_def.addCallback(_cb_auth) return d
def _component_start(self): def on_start(result): self._component_state = "Started" self._component_starting_deferred = None self._component_start_timer() return True def on_start_fail(result): self._component_state = "Stopped" self._component_starting_deferred = None log.error(result) return result if self._component_state == "Stopped": if hasattr(self, "start"): self._component_state = "Starting" d = maybeDeferred(self.start) d.addCallback(on_start) d.addErrback(on_start_fail) self._component_starting_deferred = d else: d = maybeDeferred(on_start, None) elif self._component_state == "Starting": return self._component_starting_deferred elif self._component_state == "Started": d = succeed(True) else: d = fail("Cannot start a component not in a Stopped state!") return d
def checkid_setup(registry, requestData, user=None): """ This method will validate and redirect a successful request to its return_to param. If the user isn't logged in, or doesn't have an account, we'll redirect to an internal page. @param registry: the current OpenID registry @type registry: L{OpenIDRegistry} @param requestData: the current request data @type requestData: L{OpenIDRequest} @param user: the current user @type user: L{txopenid.user.User} @return: association response @rtype: L{nevow.url.URL} """ if(user is not None): def _identity_state(): return user.hasIdentity(requestData['openid.identity']) def _trust_state(): return user.trustsRoot(requestData['openid.trust_root']) if not(yield maybeDeferred(_identity_state)): return_to = util.appendQuery(OPENID_IDENTITY_URL, requestData) elif not(yield maybeDeferred(_trust_state)): return_to = util.appendQuery(OPENID_TRUST_URL, requestData) else: return_to = get_login_response(registry, requestData) else: return_to = util.appendQuery(OPENID_LOGIN_URL, requestData) returnValue(URL.fromString(return_to))
def poll(self): if self.dq.pending: return c = yield maybeDeferred(self.q.count) if c: msg = yield maybeDeferred(self.q.pop) returnValue(self.dq.put(self._message(msg)))
def testUNIX(self): if not interfaces.IReactorUNIX(reactor, None): raise unittest.SkipTest, "This reactor does not support UNIX domain sockets" s = service.MultiService() s.startService() factory = protocol.ServerFactory() factory.protocol = TestEcho TestEcho.d = defer.Deferred() t = internet.UNIXServer("echo.skt", factory) t.setServiceParent(s) class Foo(basic.LineReceiver): def connectionMade(self): self.transport.write("lalala\r\n") def lineReceived(self, line): self.factory.line = line self.transport.loseConnection() factory = protocol.ClientFactory() factory.protocol = Foo factory.line = None internet.UNIXClient("echo.skt", factory).setServiceParent(s) util.spinWhile(lambda: factory.line is None) self.assertEqual(factory.line, "lalala") util.wait(defer.maybeDeferred(s.stopService)) util.wait(TestEcho.d) TestEcho.d = defer.Deferred() factory.line = None s.startService() util.spinWhile(lambda: factory.line is None) self.assertEqual(factory.line, "lalala") util.wait(defer.maybeDeferred(s.stopService)) util.wait(TestEcho.d)
def _callMethod(self, request_dict): """ Here we actually find and call the method. @type request_dict: dict @param request_dict: Dict with details about the method @rtype: Deferred @return: Deferred, that will eventually fire with the method's result. @raise JSONRPCError: When method not found. """ function = getattr(self, 'jsonrpc_%s' % request_dict['method'], None) if callable(function): if 'params' in request_dict: if isinstance(request_dict['params'], dict): d = maybeDeferred(function, **request_dict['params']) else: d = maybeDeferred(function, *request_dict['params']) else: d = maybeDeferred(function) return d else: msg = 'Method %s not found' % request_dict['method'] raise jsonrpc.JSONRPCError(msg, jsonrpc.METHOD_NOT_FOUND, id_=request_dict['id'], version=request_dict['jsonrpc'])
def doProvisioning(self, authtoken): """ Loops on all the provisioners to perform the provisioning. @param authtoken: AuthenticationToken containing user informations @type authtoken: AuthenticationToken @return; Deferred resulting to authtoken """ def _cbError(failure): self.logger.exception(failure.getTraceback()) raise ProvisioningError d = None if authtoken.isAuthenticated(): login = authtoken.getLogin() for name, klass in self.components: self.logger.debug("Provisioning user with %s / %s" % (name, str(klass))) instance = klass() if login.lower() in instance.config.exclude: self.logger.debug("User %s is in the exclude list of this provisioner, so skipping it" % login) continue if not d: d = defer.maybeDeferred(instance.doProvisioning, authtoken) else: d.addCallback(lambda x: defer.maybeDeferred(instance.doProvisioning, authtoken)) if d: ret = d.addCallback(lambda x:authtoken).addErrback(_cbError) else: ret = defer.succeed(authtoken) return ret
def testPrivileged(self): factory = protocol.ServerFactory() factory.protocol = TestEcho TestEcho.d = defer.Deferred() t = internet.TCPServer(0, factory) t.privileged = 1 t.privilegedStartService() num = t._port.getHost().port class Foo(basic.LineReceiver): def connectionMade(self): self.transport.write("lalala\r\n") def lineReceived(self, line): self.factory.line = line self.transport.loseConnection() factory = protocol.ClientFactory() factory.protocol = Foo factory.line = None c = internet.TCPClient("127.0.0.1", num, factory) c.startService() util.spinWhile(lambda: factory.line is None) self.assertEqual(factory.line, "lalala") util.wait(defer.maybeDeferred(c.stopService)) util.wait(defer.maybeDeferred(t.stopService)) util.wait(TestEcho.d)
def twisted_collect(self, S3Connection, TwistedS3Connection): key = Mock() key.size = 3 bucket = Mock() bucket.get_all_keys.return_value = maybeDeferred(lambda: [key] * 2) bucket.name = sentinel.component connection = Mock() connection.get_all_buckets.return_value = maybeDeferred(lambda: [bucket]) TwistedS3Connection.adapt.return_value = connection S3Connection.return_value = None config = Mock() config.datasources = [MagicMock()] config.datasources[0].component = sentinel.component config.datasources[0].ec2accesskey = "ACCESSKEY" config.datasources[0].ec2secretkey = "SECRETKEY" from ZenPacks.zenoss.AWS.dsplugins import S3BucketPlugin plugin = S3BucketPlugin() data = yield plugin.collect(config) self.assertIn(sentinel.component, data["values"]) d = data["values"][sentinel.component] self.assertEquals(d["keys_count"][0], 2) self.assertEquals(d["total_size"][0], 6)
def privmsg(self, user, channel, message, tasks=None): try: message = message.decode('utf-8') except UnicodeDecodeError: try: message = message.decode('iso-8859-1') except UnicodeDecodeError: message = message.decode('cp1252') message = cleanblanks(message) nick, user = self.log(message, user, channel) d = None if channel == "#gazouilleur" and not message.startswith("%schans" % config.COMMAND_CHARACTER): return if not message.startswith(config.COMMAND_CHARACTER): if self.nickname.lower() in message.lower(): d = defer.maybeDeferred(self.command_test) else: return message = message.encode('utf-8') if config.DEBUG: loggvar("COMMAND: %s: %s" % (user, message), channel) command, _, rest = message.lstrip(config.COMMAND_CHARACTER).partition(' ') func = self._find_command_function(command) if func is None and d is None: d = defer.maybeDeferred(self.command_help, command, channel, nick) target = nick if channel == self.nickname else channel if d is None: if self._can_user_do(nick, channel, func): d = defer.maybeDeferred(func, rest, channel, nick) else: return self._send_message("Sorry, you don't have the rights to use this command in this channel.", target, nick) d.addCallback(self._send_message, target, nick, tasks=tasks) d.addErrback(self._show_error, target, nick)
def check_password(self, username, password): if username == "": log.info("Rejected worker for blank username") defer.returnValue(False) allowed_chars = Set('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.') if Set(username).issubset(allowed_chars) != True: log.info("Username contains bad arguments") defer.returnValue(False) if username.count('.') > 1: log.info("Username contains multiple . ") defer.returnValue(False) # Force username and password to be strings username = str(username) password = str(password) if not settings.USERS_CHECK_PASSWORD and (yield self.user_exists(username)): defer.returnValue(True) elif self.cache.get(username) == password: defer.returnValue(True) elif (yield defer.maybeDeferred(self.dbi.check_password, username, password)): self.cache.set(username, password) defer.returnValue(True) elif settings.USERS_AUTOADD == True: uid = yield defer.maybeDeferred(self.dbi.get_uid, username) if uid != False: self.dbi.insert_worker(uid, username, password) self.cache.set(username, password) defer.returnValue(True) log.info("Authentication for %s failed" % username) defer.returnValue(False)
def test_no_userpass(self): d = defer.maybeDeferred(manual.CommandlineUserManager) return self.assertFailure(d, AssertionError)
def requestAvatarId(self, credentials): d = defer.maybeDeferred(self.checkKey, credentials) d.addCallback(self._cbRequestAvatarId, credentials) d.addErrback(self._ebRequestAvatarId) return d
def wrapped(*args: Any, **kwargs: Any) -> "defer.Deferred[Dict]": # If we're passed a cache_context then we'll want to call its # invalidate() whenever we are invalidated invalidate_callback = kwargs.pop("on_invalidate", None) arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names] list_args = arg_dict[self.list_name] results = {} def update_results_dict(res: Any, arg: Hashable) -> None: results[arg] = res # list of deferreds to wait for cached_defers = [] missing = set() # If the cache takes a single arg then that is used as the key, # otherwise a tuple is used. if num_args == 1: def arg_to_cache_key(arg: Hashable) -> Hashable: return arg else: keylist = list(keyargs) def arg_to_cache_key(arg: Hashable) -> Hashable: keylist[self.list_pos] = arg return tuple(keylist) for arg in list_args: try: res = cache.get(arg_to_cache_key(arg), callback=invalidate_callback) if not res.called: res.addCallback(update_results_dict, arg) cached_defers.append(res) else: results[arg] = res.result except KeyError: missing.add(arg) if missing: # we need a deferred for each entry in the list, # which we put in the cache. Each deferred resolves with the # relevant result for that key. deferreds_map = {} for arg in missing: deferred: "defer.Deferred[Any]" = defer.Deferred() deferreds_map[arg] = deferred key = arg_to_cache_key(arg) cached_defers.append( cache.set(key, deferred, callback=invalidate_callback)) def complete_all(res: Dict[Hashable, Any]) -> None: # the wrapped function has completed. It returns a dict. # We can now update our own result map, and then resolve the # observable deferreds in the cache. for e, d1 in deferreds_map.items(): val = res.get(e, None) # make sure we update the results map before running the # deferreds, because as soon as we run the last deferred, the # gatherResults() below will complete and return the result # dict to our caller. results[e] = val d1.callback(val) def errback_all(f: Failure) -> None: # the wrapped function has failed. Propagate the failure into # the cache, which will invalidate the entry, and cause the # relevant cached_deferreds to fail, which will propagate the # failure to our caller. for d1 in deferreds_map.values(): d1.errback(f) args_to_call = dict(arg_dict) args_to_call[self.list_name] = missing # dispatch the call, and attach the two handlers defer.maybeDeferred(preserve_fn(self.orig), **args_to_call).addCallbacks( complete_all, errback_all) if cached_defers: d = defer.gatherResults(cached_defers, consumeErrors=True).addCallbacks( lambda _: results, unwrapFirstError) if missing: # We started a new call to `self.orig`, so we must always wait for it to # complete. Otherwise we might mark our current logging context as # finished while `self.orig` is still using it in the background. d = delay_cancellation(d) return make_deferred_yieldable(d) else: return defer.succeed(results)
def read(self): d = maybeDeferred(self.interface.scrape_all) d.addCallbacks(self.read_callback, self.read_errback)
def reconfigServiceWithBuildbotConfig(self, new_config): www = new_config.www self.authz = www.get('authz') if self.authz is not None: self.authz.setMaster(self.master) need_new_site = False if self.site: # if config params have changed, set need_new_site to True. # There are none right now. need_new_site = False else: if www['port']: need_new_site = True if need_new_site: self.setupSite(new_config) if self.site: self.reconfigSite(new_config) if www['port'] != self.port: if self.port_service: yield defer.maybeDeferred( lambda: self.port_service.disownServiceParent()) self.port_service = None self.port = www['port'] if self.port: port = self.port if isinstance(port, int): port = "tcp:%d" % port self.port_service = strports.service(port, self.site) # monkey-patch in some code to get the actual Port object # returned by endpoint.listen(). But only for tests. if port == "tcp:0:interface=127.0.0.1": if hasattr(self.port_service, 'endpoint'): old_listen = self.port_service.endpoint.listen def listen(factory): d = old_listen(factory) @d.addCallback def keep(port): self._getPort = lambda: port return port return d self.port_service.endpoint.listen = listen else: # older twisted's just have the port sitting there # as an instance attribute self._getPort = lambda: self.port_service._port yield self.port_service.setServiceParent(self) if not self.port_service: log.msg("No web server configured on this master") yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig( self, new_config)
def run_function(): d = defer.maybeDeferred(function, *args, **kwargs) d.addCallbacks(self._got_success, self._got_failure) d.addBoth(self._stop_reactor)
def test_no_port(self): d = defer.maybeDeferred(manual.CommandlineUserManager, username="******", passwd="y") return self.assertFailure(d, AssertionError)
def tearDown(self): if self.endpoint: return defer.maybeDeferred(self.endpoint.stopListening)
def _callFromThread(): result = defer.maybeDeferred(f, *a, **kw) result.addBoth(queue.put)
def stopService(self): res = defer.maybeDeferred(AbstractBuildSlave.stopService, self) if self.slave is not None: d = self._soft_disconnect() res = defer.DeferredList([res, d]) return res
def _stop(self): result = yield defer.maybeDeferred(self.stop) self._running = False defer.returnValue(result)
def startService(self, _reactor=reactor): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() _reactor.callWhenRunning(d.callback, None) yield d try: # load the configuration file, treating errors as fatal try: self.config = config.MasterConfig.loadConfig(self.basedir, self.configFileName) except config.ConfigErrors, e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") _reactor.stop() return except: klog.err_json(failure.Failure(), 'while starting BuildMaster') _reactor.stop() return # set up services that need access to the config before everything else # gets told to reconfig try: yield self.db.setup() except connector.DatabaseNotReadyError: # (message was already logged) _reactor.stop() return if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) if hasattr(signal, "SIGUSR1"): def sigusr1(*args): _reactor.callLater(0, self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # call the parent method yield defer.maybeDeferred(lambda : service.MultiService.startService(self)) # give all services a chance to load the new configuration, rather than # the base configuration yield self.reconfigService(self.config)
def handleMessage(self, message_container): # IHL7Receiver allows implementations to return a Deferred or the # result, so ensure we return a Deferred here return defer.maybeDeferred(self.receiver.handleMessage, message_container)
def renderHTTP(self, ctx): return defer.maybeDeferred(_File.renderHTTP, self, ctx).addBoth(self._cleanup)
def create_nodes(self, reactor, names, distribution, metadata={}): """ Create nodes with the given names. :param reactor: The reactor. :param name: The names of the nodes. :type name: list of str :param str distribution: The name of the distribution to install on the nodes. :param dict metadata: Metadata to associate with the nodes. :return: A list of ``Deferred``s each firing with an INode when the corresponding node is created. The list has the same order as :param:`names`. """ size = self._default_size image_name = self._image_names[distribution] create_node_arguments = self._create_node_arguments() def handle_create_error(failure, name): # XXX This could be dangerous... What about a pre-existing # node with the same name (or even multiple nodes if the name # does not have to be unique)? Message.log( message_type="flocker:provision:libcloud:create_node:failed", node_name=name, ) write_failure(failure) d = self._async_cleanup_node_named(reactor, name) d.addCallback(lambda _: failure) return d def make_node(node): public_address = _filter_ipv4(node.public_ips)[0] if isinstance(public_address, unicode): public_address = public_address.encode("ascii") if self._use_private_addresses: private_address = _filter_ipv4(node.private_ips)[0] else: private_address = None if isinstance(private_address, unicode): private_address = private_address.encode("ascii") Message.log( message_type="flocker:provision:libcloud:node_created", name=node.name, id=node.id, public_address=public_address, private_address=private_address, ) return LibcloudNode(provisioner=self, node=node, address=public_address, private_address=private_address, distribution=distribution) action = start_action( action_type=u"flocker:provision:libcloud:create_nodes", instance_count=len(names), distribution=distribution, size=size, metadata=metadata, ) with action.context(): results = [] for name in names: Message.log( message_type=u"flocker:provision:libcloud:creating_node", node_name=name, ) d = maybeDeferred(self._driver.create_node, name=name, image=get_image(self._driver, image_name), size=get_size(self._driver, size), ex_keyname=self._keyname, ex_metadata=metadata, **create_node_arguments) d = DeferredContext(d) d.addCallbacks( lambda node: self._wait_until_running(reactor, node), errback=handle_create_error, errbackArgs=(name, ), ) d.addCallback(make_node) results.append(d.result) action_completion = DeferredContext(DeferredList(results)) action_completion.addActionFinish() # Individual results and errors should be consumed by the caller, # so we can leave action_completion alone now. return results
def _setup(self): result = yield defer.maybeDeferred(self.start) self._running = True defer.returnValue(result)
def render_POST(self, request): request.content.seek(0, 0) request.setHeader("content-type", "text/xml") try: args, functionPath = xmlrpclib.loads(request.content.read()) except Exception, e: f = Fault(self.FAILURE, "Can't deserialize input: %s" % (e, )) self._cbRender(f, request) else: try: function = self._getFunction(functionPath) except Fault, f: self._cbRender(f, request) else: d = defer.maybeDeferred(function, *args) d.addErrback(self._ebRender) d.addCallback(self._cbRender, request) return server.NOT_DONE_YET def _cbRender(self, result, request): if isinstance(result, Handler): result = result.result if not isinstance(result, Fault): result = (result, ) try: try: content = xmlrpclib.dumps(result, methodresponse=True, allow_none=self.allowNone) except Exception, e:
def make_call(): d = maybeDeferred(f) d = DeferredContext(d) d.addCallbacks(handle_success, errback=handle_failure) return d.result
def stop(self): """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False yield defer.maybeDeferred(self.engine.stop)
def _handleGotRevision(self, res): d = defer.maybeDeferred(self.parseGotRevision) d.addCallback(lambda got_revision: self.sendStatus( {'got_revision': got_revision})) return d
def wrapped(*args, **kwargs): # If we're passed a cache_context then we'll want to call its # invalidate() whenever we are invalidated invalidate_callback = kwargs.pop("on_invalidate", None) arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names] list_args = arg_dict[self.list_name] results = {} def update_results_dict(res, arg): results[arg] = res # list of deferreds to wait for cached_defers = [] missing = set() # If the cache takes a single arg then that is used as the key, # otherwise a tuple is used. if num_args == 1: def arg_to_cache_key(arg): return arg else: keylist = list(keyargs) def arg_to_cache_key(arg): keylist[self.list_pos] = arg return tuple(keylist) for arg in list_args: try: res = cache.get(arg_to_cache_key(arg), callback=invalidate_callback) if not res.called: res.addCallback(update_results_dict, arg) cached_defers.append(res) else: results[arg] = res.result except KeyError: missing.add(arg) if missing: # we need a deferred for each entry in the list, # which we put in the cache. Each deferred resolves with the # relevant result for that key. deferreds_map = {} for arg in missing: deferred = defer.Deferred() deferreds_map[arg] = deferred key = arg_to_cache_key(arg) cache.set(key, deferred, callback=invalidate_callback) def complete_all(res): # the wrapped function has completed. It returns a # a dict. We can now resolve the observable deferreds in # the cache and update our own result map. for e in missing: val = res.get(e, None) deferreds_map[e].callback(val) results[e] = val def errback(f): # the wrapped function has failed. Invalidate any cache # entries we're supposed to be populating, and fail # their deferreds. for e in missing: key = arg_to_cache_key(e) cache.invalidate(key) deferreds_map[e].errback(f) # return the failure, to propagate to our caller. return f args_to_call = dict(arg_dict) # copy the missing set before sending it to the callee, to guard against # modification. args_to_call[self.list_name] = tuple(missing) cached_defers.append( defer.maybeDeferred( preserve_fn(self.orig), **args_to_call ).addCallbacks(complete_all, errback) ) if cached_defers: d = defer.gatherResults(cached_defers, consumeErrors=True).addCallbacks( lambda _: results, unwrapFirstError ) return make_deferred_yieldable(d) else: return defer.succeed(results)
def allUploadsDone(result): d = defer.maybeDeferred(self.allUploadsDone, result, sources, masterdest) d.addCallback(lambda _: result) return d
def _do_authenticate(self, protoinfo): """ Callback on PROTOCOLINFO to actually authenticate once we know what's supported. """ methods = None cookie_auth = False for line in protoinfo.split('\n'): if line[:5] == 'AUTH ': kw = parse_keywords(line[5:].replace(' ', '\n')) methods = kw['METHODS'].split(',') if not methods: raise RuntimeError( "Didn't find AUTH line in PROTOCOLINFO response.") if 'SAFECOOKIE' in methods or 'COOKIE' in methods: cookiefile_match = re.search(r'COOKIEFILE=("(?:[^"\\]|\\.)*")', protoinfo) if cookiefile_match: cookiefile = cookiefile_match.group(1) cookiefile = unescape_quoted_string(cookiefile) try: self._read_cookie(cookiefile) cookie_auth = True except IOError as why: txtorlog.msg("Reading COOKIEFILE failed: " + str(why)) if self.password_function and 'HASHEDPASSWORD' in methods: txtorlog.msg("Falling back to password") else: raise RuntimeError( "Failed to read COOKIEFILE '{fname}': {msg}\n". format( fname=cookiefile, msg=str(why), ) # "On Debian, join the debian-tor group" ) else: txtorlog.msg("Didn't get COOKIEFILE") raise RuntimeError( "Got 'COOKIE' or 'SAFECOOKIE' method, but no 'COOKIEFILE'") if cookie_auth: if 'SAFECOOKIE' in methods: txtorlog.msg("Using SAFECOOKIE authentication", cookiefile, len(self._cookie_data), "bytes") self.client_nonce = os.urandom(32) cmd = b'AUTHCHALLENGE SAFECOOKIE ' + \ hexlify(self.client_nonce) d = self.queue_command(cmd) d.addCallback(self._safecookie_authchallenge) d.addCallback(self._bootstrap) d.addErrback(self._auth_failed) return elif 'COOKIE' in methods: txtorlog.msg("Using COOKIE authentication", cookiefile, len(self._cookie_data), "bytes") d = self.authenticate(self._cookie_data) d.addCallback(self._bootstrap) d.addErrback(self._auth_failed) return if self.password_function and 'HASHEDPASSWORD' in methods: d = defer.maybeDeferred(self.password_function) d.addCallback(self._do_password_authentication) d.addErrback(self._auth_failed) return if 'NULL' in methods: d = self.queue_command('AUTHENTICATE') d.addCallback(self._bootstrap) d.addErrback(self._auth_failed) return raise RuntimeError( "The Tor I connected to doesn't support SAFECOOKIE nor COOKIE" " authentication (or we can't read the cookie files) and I have" " no password_function specified.")
def tearDown(self): d3 = Deferred() self._serverProtocol.notifyOnDisconnect(lambda: d3.callback(None)) return DeferredList([ maybeDeferred(self.serverPort.stopListening), maybeDeferred(self.clientConn.disconnect), d3])
def startConnecting(self): d = defer.maybeDeferred(self.sf.resolveAddress) d.addCallback(self._cbResolveDone) d.addErrback(self._ebResolveErr)
def runs_in_reactor(args, kwargs): d = maybeDeferred(function, *args, **kwargs) return EventualResult(d)
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = {s.name: s for s in new_config_attr} elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError("config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove # While we're at it find any service that don't know how to reconfig, # and, if they have changed, add them to both removed and added, so that we # run the new version for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'): if old != new: removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] # disownServiceParent calls stopService after removing the relationship # as child might use self.master.data to stop itself, its better to stop it first # (this is related to the fact that self.master is found by recursively looking at self.parent # for a master) yield child.stopService() # it has already called, so do not call it again child.stopService = lambda: None yield child.disownServiceParent() # HACK: we still keep a reference to the master for some cleanup tasks which are not waited by # to stopService (like the complex worker disconnection mechanism) # http://trac.buildbot.net/ticket/3583 child.parent = self.master for n in added_names: child = new_by_name[n] # setup service's objectid if hasattr(child, 'objectid'): class_name = '%s.%s' % (child.__class__.__module__, child.__class__.__name__) objectid = yield self.master.db.state.getObjectId( child.name, class_name) child.objectid = objectid yield defer.maybeDeferred(child.setServiceParent, self) # As the services that were just added got # reconfigServiceWithSibling called by # setServiceParent->startService, # we avoid calling it again by selecting # in reconfigurable_services, services # that were not added just now reconfigurable_services = [ svc for svc in self if svc.name not in added_names ] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError( "{}: child {} should have a defined name attribute".format( self, svc)) config_sibling = new_by_name.get(svc.name) try: yield svc.reconfigServiceWithSibling(config_sibling) except NotImplementedError: # legacy support. Its too painful to transition old code to new Service life cycle # so we implement switch of child when the service raises NotImplementedError # Note this means that self will stop, and sibling will take ownership # means that we have a small time where the service is unavailable. yield svc.disownServiceParent() config_sibling.objectid = svc.objectid yield config_sibling.setServiceParent(self)
def submit(self, job, test=False): """ Submit the job to the device. If the device is currently running a job it will be queued and run when this is finished. This handles iteration over the path model defined by the job and sending commands to the actual device using roughly the procedure is as follows: device.connect() model = device.init(job) for cmd in device.process(model): device.handle(cmd) device.finish() device.disconnect() Subclasses provided by your own DeviceDriver may reimplement this to handle path interpolation however needed. The return value is ignored. The live plot view will update whenever the device.position object is updated. On devices with lower cpu/gpu capabilities this should be updated sparingly (ie the raspberry pi). Parameters ----------- job: Instance of `inkcut.job.models.Job` The job to execute on the device test: bool Do a test run. This specifies whether the commands should be sent to the actual device or not. If True, the connection will be replaced with a virtual connection that captures all the command output. """ log.debug("device | submit {}".format(job)) try: #: Only allow one job at a time if self.busy: queue = self.queue[:] queue.append(job) self.queue = queue #: Copy and reassign so the UI updates log.info("Job {} put in device queue".format(job)) return with self.device_busy(): #: Set the current the job self.job = job self.status = "Initializing job" #: Get the time to sleep based for each unit of movement config = self.config #: Rate px/ms if config.custom_rate >= 0: rate = config.custom_rate elif self.connection.always_spools or config.spooled: rate = 0 elif config.interpolate: if config.step_time > 0: rate = config.step_size/float(config.step_time) else: rate = 0 # Undefined else: rate = from_unit( config.speed, # in/s or cm/s config.speed_units.split("/")[0])/1000.0 # Device model is updated in real time model = yield defer.maybeDeferred(self.init, job) #: Local references are faster info = job.info #: Determine the length for tracking progress whole_path = QtGui.QPainterPath() #: Some versions of Qt seem to require a value in #: toSubpathPolygons m = QtGui.QTransform.fromScale(1, 1) for path in model.toSubpathPolygons(m): for i, p in enumerate(path): whole_path.lineTo(p) total_length = whole_path.length() total_moved = 0 log.debug("device | Path length: {}".format(total_length)) #: So a estimate of the duration can be determined info.length = total_length info.speed = rate*1000 #: Convert to px/s #: Waiting for approval info.status = 'waiting' #: If marked for auto approve start now if info.auto_approve: info.status = 'approved' else: #: Check for approval before starting yield defer.maybeDeferred(info.request_approval) if info.status != 'approved': self.status = "Job cancelled" return #: Update stats info.status = 'running' info.started = datetime.now() self.status = "Connecting to device" with self.device_connection( test or config.test_mode) as connection: self.status = "Processing job" try: yield defer.maybeDeferred(self.connect) protocol = connection.protocol #: Write startup command if config.commands_before: yield defer.maybeDeferred(connection.write, config.commands_before) self.status = "Working..." if config.force_enabled: yield defer.maybeDeferred( protocol.set_force, config.force) if config.speed_enabled: yield defer.maybeDeferred( protocol.set_velocity, config.speed) #: For point in the path for (d, cmd, args, kwargs) in self.process(model): #: Check if we paused if info.paused: self.status = "Job paused" #: Sleep until resumed, cancelled, or the #: connection drops while (info.paused and not info.cancelled and connection.connected): yield async_sleep(300) # ms #: Check for cancel for non interpolated jobs if info.cancelled: self.status = "Job cancelled" info.status = 'cancelled' break elif not connection.connected: self.status = "connection error" info.status = 'error' break #: Invoke the command #: If you want to let the device handle more complex #: commands such as curves do it in process and handle yield defer.maybeDeferred(cmd, *args, **kwargs) total_moved += d #: d should be the device must move in px #: so wait a proportional amount of time for the device #: to catch up. This avoids buffer errors from dumping #: everything at once. #: Since sending is way faster than cutting #: we must delay (without blocking the UI) before #: sending the next command or the device's buffer #: quickly gets filled and crappy china piece cutters #: get all jacked up. If the transport sends to a spooled #: output (such as a printer) this can be set to 0 if rate > 0: # log.debug("d={}, delay={} t={}".format( # d, delay, d/delay # )) yield async_sleep(d/rate) #: TODO: Check if we need to update the ui #: Set the job progress based on how far we've gone if total_length > 0: info.progress = int(max(0, min(100, 100*total_moved/total_length))) if info.status != 'error': #: We're done, send any finalization commands yield defer.maybeDeferred(self.finish) #: Write finalize command if config.commands_after: yield defer.maybeDeferred(connection.write, config.commands_after) #: Update stats info.ended = datetime.now() #: If not cancelled or errored if info.status == 'running': info.done = True info.status = 'complete' except Exception as e: log.error(traceback.format_exc()) raise finally: if connection.connected: yield defer.maybeDeferred(self.disconnect) #: Set the origin if job.feed_to_end and job.info.status == 'complete': self.origin = self.position #: If the user didn't cancel, set the origin and #: Process any jobs that entered the queue while this was running if self.queue and not job.info.cancelled: queue = self.queue[:] job = queue.pop(0) #: Pull the first job off the queue log.info("Rescheduling {} from queue".format(job)) self.queue = queue #: Copy and reassign so the UI updates #: Call a minute later timed_call(60000, self.submit, job) except Exception as e: log.error(' device | Execution error {}'.format( traceback.format_exc())) raise
def storeResource( request, source=None, source_uri=None, data=None, destination=None, destination_uri=None, deletesource=False, depth="0" ): """ Function that does common PUT/COPY/MOVE behaviour. @param request: the L{txweb2.server.Request} for the current HTTP request. @param source: the L{DAVFile} for the source resource to copy from, or None if source data is to be read from the request. @param source_uri: the URI for the source resource. @param data: a C{str} to copy data from instead of the request stream. @param destination: the L{DAVFile} for the destination resource to copy into. @param destination_uri: the URI for the destination resource. @param deletesource: True if the source resource is to be deleted on successful completion, False otherwise. @param depth: a C{str} containing the COPY/MOVE Depth header value. @return: status response. """ try: assert request is not None and destination is not None and destination_uri is not None assert (source is None) or (source is not None and source_uri is not None) assert not deletesource or (deletesource and source is not None) except AssertionError: log.error("Invalid arguments to storeResource():") log.error("request=%s\n" % (request,)) log.error("source=%s\n" % (source,)) log.error("source_uri=%s\n" % (source_uri,)) log.error("data=%s\n" % (data,)) log.error("destination=%s\n" % (destination,)) log.error("destination_uri=%s\n" % (destination_uri,)) log.error("deletesource=%s\n" % (deletesource,)) log.error("depth=%s\n" % (depth,)) raise class RollbackState(object): """ This class encapsulates the state needed to rollback the entire PUT/COPY/MOVE transaction, leaving the server state the same as it was before the request was processed. The DoRollback method will actually execute the rollback operations. """ def __init__(self): self.active = True self.source_copy = None self.destination_copy = None self.destination_created = False self.source_deleted = False def Rollback(self): """ Rollback the server state. Do not allow this to raise another exception. If rollback fails then we are going to be left in an awkward state that will need to be cleaned up eventually. """ if self.active: self.active = False log.error("Rollback: rollback") try: if self.source_copy and self.source_deleted: self.source_copy.moveTo(source.fp) log.error("Rollback: source restored %s to %s" % (self.source_copy.path, source.fp.path)) self.source_copy = None self.source_deleted = False if self.destination_copy: destination.fp.remove() log.error("Rollback: destination restored %s to %s" % (self.destination_copy.path, destination.fp.path)) self.destination_copy.moveTo(destination.fp) self.destination_copy = None elif self.destination_created: destination.fp.remove() log.error("Rollback: destination removed %s" % (destination.fp.path,)) self.destination_created = False except: log.error("Rollback: exception caught and not handled: %s" % Failure()) def Commit(self): """ Commit the resource changes by wiping the rollback state. """ if self.active: log.error("Rollback: commit") self.active = False if self.source_copy: self.source_copy.remove() log.error("Rollback: removed source backup %s" % (self.source_copy.path,)) self.source_copy = None if self.destination_copy: self.destination_copy.remove() log.error("Rollback: removed destination backup %s" % (self.destination_copy.path,)) self.destination_copy = None self.destination_created = False self.source_deleted = False rollback = RollbackState() try: """ Handle validation operations here. """ """ Handle rollback setup here. """ # Do quota checks on destination and source before we start messing with adding other files destquota = waitForDeferred(destination.quota(request)) yield destquota destquota = destquota.getResult() if destquota is not None and destination.exists(): old_dest_size = waitForDeferred(destination.quotaSize(request)) yield old_dest_size old_dest_size = old_dest_size.getResult() else: old_dest_size = 0 if source is not None: sourcequota = waitForDeferred(source.quota(request)) yield sourcequota sourcequota = sourcequota.getResult() if sourcequota is not None and source.exists(): old_source_size = waitForDeferred(source.quotaSize(request)) yield old_source_size old_source_size = old_source_size.getResult() else: old_source_size = 0 else: sourcequota = None old_source_size = 0 # We may need to restore the original resource data if the PUT/COPY/MOVE fails, # so rename the original file in case we need to rollback. overwrite = destination.exists() if overwrite: rollback.destination_copy = FilePath(destination.fp.path) rollback.destination_copy.path += ".rollback" destination.fp.copyTo(rollback.destination_copy) else: rollback.destination_created = True if deletesource: rollback.source_copy = FilePath(source.fp.path) rollback.source_copy.path += ".rollback" source.fp.copyTo(rollback.source_copy) """ Handle actual store operations here. """ # Do put or copy based on whether source exists if source is not None: response = maybeDeferred(copy, source.fp, destination.fp, destination_uri, depth) else: datastream = request.stream if data is not None: datastream = MemoryStream(data) md5 = MD5Stream(datastream) response = maybeDeferred(put, md5, destination.fp) response = waitForDeferred(response) yield response response = response.getResult() # Update the MD5 value on the resource if source is not None: # Copy MD5 value from source to destination if source.hasDeadProperty(TwistedGETContentMD5): md5 = source.readDeadProperty(TwistedGETContentMD5) destination.writeDeadProperty(md5) else: # Finish MD5 calc and write dead property md5.close() md5 = md5.getMD5() destination.writeDeadProperty(TwistedGETContentMD5.fromString(md5)) # Update the content-type value on the resource if it is not been copied or moved if source is None: content_type = request.headers.getHeader("content-type") if content_type is not None: destination.writeDeadProperty(davxml.GETContentType.fromString(generateContentType(content_type))) response = IResponse(response) # Do quota check on destination if destquota is not None: # Get size of new/old resources new_dest_size = waitForDeferred(destination.quotaSize(request)) yield new_dest_size new_dest_size = new_dest_size.getResult() diff_size = new_dest_size - old_dest_size if diff_size >= destquota[0]: log.error("Over quota: available %d, need %d" % (destquota[0], diff_size)) raise HTTPError(ErrorResponse( responsecode.INSUFFICIENT_STORAGE_SPACE, (dav_namespace, "quota-not-exceeded") )) d = waitForDeferred(destination.quotaSizeAdjust(request, diff_size)) yield d d.getResult() if deletesource: # Delete the source resource if sourcequota is not None: delete_size = 0 - old_source_size d = waitForDeferred(source.quotaSizeAdjust(request, delete_size)) yield d d.getResult() delete(source_uri, source.fp, depth) rollback.source_deleted = True # Can now commit changes and forget the rollback details rollback.Commit() yield response return except: # Roll back changes to original server state. Note this may do nothing # if the rollback has already ocurred or changes already committed. rollback.Rollback() raise
def _on_authorized(self, is_authorized, request, worker_name): data = json.loads(request.content.read()) if not is_authorized: request.write( self.json_error(data.get('id', 0), -1, "Bad worker credentials")) request.finish() return if not self.job_registry.last_job: log.warning('Getworkmaker is waiting for a job...') request.write( self.json_error(data.get('id', 0), -1, "Getworkmake is waiting for a job...")) request.finish() return if data['method'] == 'getwork': if 'params' not in data or not len(data['params']): # getwork request log.info("Worker '%s' asks for new work" % worker_name) extensions = request.getHeader('x-mining-extensions') no_midstate = extensions and 'midstate' in extensions request.write( self.json_response( data.get('id', 0), self.job_registry.getwork(no_midstate=no_midstate))) request.finish() return else: # submit d = defer.maybeDeferred(self.job_registry.submit, data['params'][0], worker_name) start_time = time.time() d.addCallback(self._on_submit, request, data.get('id', 0), data['params'][0][:160], worker_name, start_time) d.addErrback(self._on_submit_failure, request, data.get('id', 0), data['params'][0][:160], worker_name, start_time) return elif data['method'] == 'eth_getWork': #log.info("Worker '%s' eth_getWork" % worker_name) request.write( self.json_response(data.get('id', 0), self.job_registry.getwork())) request.finish() return elif data['method'] == 'eth_submitWork': d = defer.maybeDeferred(self.job_registry.submit, data['params'][0], data['params'][1], data['params'][2], worker_name) start_time = time.time() d.addCallback(self._on_submit, request, data.get('id', 0), data['params'], worker_name, start_time) d.addErrback(self._on_submit_failure, request, data.get('id', 0), data['params'], worker_name, start_time) return elif data['method'] == 'eth_submitHashrate': #log.info("Worker '%s' eth_submitHashrate" % worker_name) request.write(self.json_response(data.get('id', 0), True)) request.finish() return request.write( self.json_error(1, -1, "Unsupported method '%s'" % data['method'])) request.finish()