def test_complex2(self, runtime): def check(ls): if (2 == runtime.id) or (1 == runtime.id): self.assertEquals(ls[0][1], "V1") self.assertEquals(ls[1][1], "V1") self.assertEquals(ls[2][1], "V1") self.assertEquals(ls[3][1], "V2") else: self.assertEquals(ls[0][1], "V1") self.assertEquals(ls[1][1], "V1") self.assertEquals(ls[2][1], "V1") self.assertEquals(ls[3][1], "V2") self.assertEquals(ls[4][1], "V2") field = self.Zp results = [] results += runtime.broadcast(runtime.players.keys(), runtime.players.keys(), "V1") if runtime.id in [1, 2]: v = runtime.broadcast([1, 2], [3], "V2") if isinstance(v, list): results += v else: results.append(v) else: results += runtime.broadcast([1, 2], [3]) if 3 == runtime.id: results += [runtime.broadcast([3], runtime.players.keys(), str(7))] else: results += [runtime.broadcast([3], runtime.players.keys())] dls = DeferredList(results) runtime.schedule_callback(dls, check) dls.addErrback(runtime.error_handler) return dls
def _configure_ssh(self, deployment): """ :return: A ``Deferred`` which fires when all nodes have been configured with ssh keys. """ self.ssh_configuration.create_keypair() results = [] for node in deployment.nodes: results.append( deferToThread( self.ssh_configuration.configure_ssh, node.hostname, self.ssh_port ) ) d = DeferredList(results, fireOnOneErrback=True, consumeErrors=True) # Exit with ssh's output if it failed for some reason: def got_failure(failure): if failure.value.subFailure.check(CalledProcessError): raise SystemExit( b"Error connecting to cluster node: " + failure.value.subFailure.value.output) else: return failure d.addErrback(got_failure) return d
def _observe_nodes(self, nodes): if self.stopped: lg.warn('DiscoveryTask[%r] : discovery process already stopped' % self.id) return if _Debug: lg.out( _DebugLevel, 'lookup.DiscoveryTask[%r]._observe_nodes started for %d items layer_id=%d' % ( self.id, len(nodes), self.layer_id, )) observe_list = [] for node in nodes: d = self.observe_method(node, layer_id=self.layer_id) d.addCallback(self._on_node_observed, node) d.addErrback(self._on_node_observe_failed, node) observe_list.append(d) self.observed_count = len(nodes) dl = DeferredList(observe_list, consumeErrors=False) dl.addCallback(self._on_all_nodes_observed) if _Debug: dl.addErrback(lg.errback, debug=_Debug, debug_level=_DebugLevel, method='DiscoveryTask._observe_nodes')
def doBackupKeys(self, *args, **kwargs): """ Action method. """ for old_key_id in list(self.keys_to_rename.keys()): new_key_id, is_private = self.keys_to_rename[old_key_id] if old_key_id in self.stored_keys and new_key_id not in self.stored_keys: self.keys_to_upload.add(new_key_id) if new_key_id in self.stored_keys and old_key_id in self.stored_keys: self.keys_to_erase[old_key_id] = is_private for key_id in my_keys.known_keys().keys(): if key_id not in self.stored_keys or key_id in self.not_stored_keys: self.keys_to_upload.add(key_id) keys_saved = [] for key_id in self.keys_to_upload: res = key_ring.do_backup_key(key_id, wait_result=True) keys_saved.append(res) self.saved_count += 1 if _Debug: lg.args(_DebugLevel, keys_saved=len(keys_saved)) wait_all_saved = DeferredList(keys_saved, fireOnOneErrback=False, consumeErrors=True) wait_all_saved.addCallback(lambda ok: self.automat('backup-ok', ok)) wait_all_saved.addErrback(lambda err: self.automat('error', err))
def doRequestMyIdentity(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'id_rotator.doRequestMyIdentity') def _cb(results): if _Debug: lg.args(_DebugLevel, results=results) # TODO: validate my identity in all responses self.automat('my-id-exist', results) return results def _eb(err): lg.err(err) self.automat('my-id-not-exist', err) if err.type == CancelledError: return None return err dl = [] for idurl in my_id.getLocalIdentity().getSources(as_originals=True): dl.append(net_misc.getPageTwisted(idurl, timeout=10)) d = DeferredList(dl, consumeErrors=True) d.addCallback(_cb) d.addErrback(_eb)
def _do_read_brokers(): all_brokers_results = [] for position in positions: one_broker_result = Deferred() all_brokers_results.append(one_broker_result) d = dht_records.get_message_broker( customer_idurl=customer_idurl, position=position, return_details=return_details, use_cache=use_cache, ) d.addCallback(_do_verify, position, one_broker_result) if _Debug: d.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='read_customer_message_brokers._do_read_brokers') d.addErrback(_on_error, position, one_broker_result) join_all_brokers = DeferredList(all_brokers_results, consumeErrors=True) join_all_brokers.addCallback(_do_collect_results) if _Debug: join_all_brokers.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='read_customer_message_brokers._do_read_brokers') join_all_brokers.addErrback(result.errback) return None
def setup_local(myName, virtualNet, classicalNet, lNode, func): """ Sets up - local classical communication server (if desired according to the configuration file) - client connection to the local virtual node quantum backend - client connections to all other classical communication servers Arguments myName name of this node (string) virtualNet servers of the virtual nodes (dictionary of host objects) classicalNet servers on the classical communication network (dictionary of host objects) lNode Twisted PB root to use as local server (if applicable) func function to run if all connections are set up """ # Initialize Twisted callback framework dList = [] # If we are listed as a server node for the classical network, start this server if myName in classicalNet.hostDict: try: logging.debug( "LOCAL %s: Starting local classical communication server.", myName) nb = classicalNet.hostDict[myName] nb.root = lNode nb.factory = pb.PBServerFactory(nb.root) reactor.listenTCP(nb.port, nb.factory) except Exception as e: logging.error( "LOCAL %s: Cannot start classical communication servers.", myName, e.strerror) return # Give the server some time to start up time.sleep(1) # Connect to the local virtual node simulating the "local" qubits logging.debug("LOCAL %s: Connecting to local virtual node.", myName) node = virtualNet.hostDict[myName] factory = pb.PBClientFactory() reactor.connectTCP(node.hostname, node.port, factory) deferVirtual = factory.getRootObject() dList.append(deferVirtual) # Set up a connection to all the other nodes in the classical network for node in classicalNet.hostDict: nb = classicalNet.hostDict[node] if nb.name != myName: logging.debug("LOCAL %s: Making classical connection to %s.", myName, nb.name) nb.factory = pb.PBClientFactory() reactor.connectTCP(nb.hostname, nb.port, nb.factory) dList.append(nb.factory.getRootObject()) deferList = DeferredList(dList, consumeErrors=True) deferList.addCallback(init_register, myName, virtualNet, classicalNet, lNode, func) deferList.addErrback(localError) reactor.run()
def run(nodes): def callback(*args, **kwargs): print('callback', args) l = args[0] assert len(l) > 0 def errback(*args, **kwargs): import traceback traceback.print_exc() def callback_dfl(*args): print('callback_dfl', args) reactor.stop() errback_dfl = errback time.sleep(3) try: list_of_deffered_set_value = [] for i in range(args.start, args.end): j = {'key'+str(i): 'value'+str(i), } d = dht_service.set_json_value(str(i), j, 60 * 60) d.addBoth(callback, j) d.addErrback(errback) list_of_deffered_set_value.append(d) dfl = DeferredList(list_of_deffered_set_value) dfl.addCallback(callback_dfl) dfl.addErrback(errback_dfl) except: print('ERRRORO!!') reactor.stop()
def _storeData(self, data, request_hash, confirm_cache_write, http_history=None): if len(data["response"]) == 0: return self._storeDataErrback(Failure(exc_value=Exception("Response data is of length 0")), data, request_hash) #data["content-sha1"] = sha1(data["response"]).hexdigest() if http_history is None: http_history = {} if "content-sha1" not in http_history: http_history["content-sha1"] = data["content-sha1"] if "content-changes" not in http_history: http_history["content-changes"] = [] if data["content-sha1"] != http_history["content-sha1"]: http_history["content-changes"].append(str(int(self.time_offset + time.time()))) http_history["content-changes"] = http_history["content-changes"][-10:] headers = {} http_history["content-changes"] = filter(lambda x:len(x) > 0, http_history["content-changes"]) headers["content-changes"] = ",".join(http_history["content-changes"]) headers["content-sha1"] = data["content-sha1"] if "cache-control" in data["headers"]: if isinstance(data["headers"]["cache-control"], (list, tuple)): if "no-cache" in data["headers"]["cache-control"][0]: return data else: if "no-cache" in data["headers"]["cache-control"]: return data if "expires" in data["headers"]: if isinstance(data["headers"]["expires"], (list, tuple)): headers["cache-expires"] = data["headers"]["expires"][0] else: headers["cache-expires"] = data["headers"]["expires"] if "etag" in data["headers"]: if isinstance(data["headers"]["etag"], (list, tuple)): headers["cache-etag"] = data["headers"]["etag"][0] else: headers["cache-etag"] = data["headers"]["etag"] if "last-modified" in data["headers"]: if isinstance(data["headers"]["last-modified"], (list, tuple)): headers["cache-last-modified"] = data["headers"]["last-modified"][0] else: headers["cache-last-modified"] = data["headers"]["last-modified"] if "content-type" in data["headers"]: if isinstance(data["headers"]["content-type"], (list, tuple)): headers["content_type"] = data["headers"]["content-type"][0] else: headers["content_type"] = data["headers"]["content-type"] headers_key = 'headers:%s' % request_hash http_key = 'http:%s' % request_hash logger.debug("Writing data for request %s to redis." % request_hash) deferreds = [] deferreds.append(self.redis_client.set(headers_key, compress(json.dumps(headers), 1))) deferreds.append(self.redis_client.set(http_key, compress(json.dumps(data["response"]), 1))) d = DeferredList(deferreds, consumeErrors=True) if confirm_cache_write: d.addCallback(self._storeDataCallback, data) d.addErrback(self._storeDataErrback, data, request_hash) return d return data
def send_payment_request(self, readTokens, writeTokens): """Called by a Circuit object when it wants to actually make a payment @param readTokens: the number of read tokens to pay for at each hop in the circuit @type readTokens: int @param writeTokens: the number of read tokens to pay for at each hop in the circuit @type writeTokens: int""" assert (readTokens + writeTokens) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells" #make sure our setup is done: if not self.setupDone: #have we even started? if not self.setupStarted: self.send_setup_message() self.queuedReadTokens += readTokens self.queuedWriteTokens += writeTokens return #dont bother trying to send payments for circuits that are already closed if self.circ.is_done(): return #send the payments deferreds = [] for paymentStream in self.paymentStreams.values(): deferreds.append(paymentStream.send_payment(readTokens, writeTokens)) paymentsDoneDeferred = DeferredList(deferreds) paymentsDoneDeferred.addErrback(self.generic_error_handler) addTokensDeferred = Deferred() self.inflightReadTokens += readTokens self.inflightWriteTokens += writeTokens #timeout in case the payment fails. We will close the circuit in this case. event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT, self.all_receipts_received, None, addTokensDeferred, readTokens, writeTokens, None) paymentsDoneDeferred.addCallback(self.all_receipts_received, addTokensDeferred, readTokens, writeTokens, event) addTokensDeferred.addCallback(self._add_tokens_callback, readTokens, writeTokens) addTokensDeferred.addErrback(self.generic_error_handler)
def _on_connected(self, ok): from twisted.internet.defer import DeferredList from logs import lg from dht import dht_service from dht import known_nodes from main.config import conf lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok)) dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store) dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request) known_seeds = known_nodes.nodes() dl = [] attached_layers = conf().getData('services/entangled-dht/attached-layers', default='') if attached_layers: lg.info('more DHT layers to be attached: %r' % attached_layers) for layer_id in attached_layers.split(','): if layer_id.strip(): dl.append(dht_service.open_layer( layer_id=int(layer_id.strip()), seed_nodes=known_seeds, connect_now=True, attach=True, )) if dl: d = DeferredList(dl) d.addCallback(self._on_layers_attached) d.addErrback(self._on_connect_failed) else: if self.starting_deferred and not self.starting_deferred.called: self.starting_deferred.callback(True) return ok
def send_deferred(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.send but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. If one signal handler fails, the deffered errback is called. """ spider = named.get('spider', None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) d.addCallback(lambda result: (receiver, result)) dfds.append(d) d = DeferredList(dfds, consumeErrors=True, fireOnOneErrback=True) def unwrap_firsterror(err): failure = err.trap(FirstError) return err.value.subFailure d.addErrback(unwrap_firsterror) d.addCallback(lambda out: [x[1] for x in out]) return d
def stop(services_list=[]): """ """ global _StopingDeferred global _StartingDeferred if _Debug: lg.args(_DebugLevel, services_list=services_list, starting=bool(_StartingDeferred), stoping=bool(_StopingDeferred)) if _StopingDeferred: lg.warn('driver.stop already called') return _StopingDeferred if _StartingDeferred and not _StartingDeferred.called: d = Deferred() d.errback(Exception('currently another service is starting')) return d if not services_list: services_list.extend(reversed(boot_up_order())) if _Debug: lg.out(_DebugLevel, 'driver.stop with %d services' % len(services_list)) dl = [] for name in services_list: svc = services().get(name, None) if not svc: raise ServiceNotFound(name) d = Deferred() dl.append(d) svc.automat('stop', d) _StopingDeferred = DeferredList(dl) _StopingDeferred.addCallback(on_stopped_all_services) _StopingDeferred.addErrback(on_services_failed_to_stop, services_list) return _StopingDeferred
def run(nodes): def callback(*args, **kwargs): print('callback', args) d = args[0] assert len(d) == 1 k, v = d.popitem() assert k.replace('key', '') == v.replace('value', '') def errback(*args, **kwargs): import traceback traceback.print_exc() def callback_dfl(*args): print('callback_dfl', args) reactor.stop() errback_dfl = errback time.sleep(3) try: list_of_deffered_set_value = [] for i in range(args.start, args.end): d = dht_service.get_json_value(str(i)) d.addBoth(callback) d.addErrback(errback) list_of_deffered_set_value.append(d) dfl = DeferredList(list_of_deffered_set_value) dfl.addCallback(callback_dfl) dfl.addErrback(errback_dfl) except Exception as exc: print('ERRRORO!!', exc) reactor.stop()
def render(self, request): request.setHeader('Content-type', 'text/javascript; charset=UTF-8') if len(request.postpath) > 0: if request.postpath[0] == "server": return simplejson.dumps(self.spider.getServerData()) elif request.postpath[0] == "exposed_function_details": return simplejson.dumps(self.spider.getExposedFunctionDetails()) elif request.postpath[0] == "get": if "uuid" in request.args: get_deferreds = [] for uuid in request.args["uuid"]: get_deferreds.append(self.spider.get(uuid)) d = DeferredList(get_deferreds, consumeErrors = True) d.addCallback(self._getCallback, request.args["uuid"]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required."))) elif request.postpath[0] == "show_reservation": if "uuid" in request.args: d = self.spider.showReservation(request.args["uuid"][0]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required."))) elif request.postpath[0] == "delete_reservation": if "uuid" in request.args: d = self.spider.deleteReservation(request.args["uuid"][0]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required."))) elif request.postpath[0] == "delete_function_reservations": if "function_name" in request.args: d = self.spider.deleteFunctionReservations(request.args["function_name"][0]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter function_name is required."))) elif request.postpath[0] == "execute_reservation": if "uuid" in request.args: d = self.spider.queryByUUID(request.args["uuid"][0]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
def _do_verify_my_sources(self): my_sources = my_id.getLocalIdentity().getSources(as_originals=True) dl = [] for idurl_bin in my_sources: d = net_misc.getPageTwisted(idurl_bin, timeout=5) dl.append(d) d = DeferredList(dl, consumeErrors=True) d.addCallback(self._do_check_ping_results) d.addErrback(lambda err: self.automat('ping-failed', err))
def deleteReservation( self, uuid, function_name="Unknown" ): logger.info("Deleting reservation %s, %s." % (function_name, uuid)) deferreds = [] deferreds.append(self.sdb.delete(self.aws_sdb_reservation_domain, uuid)) deferreds.append(self.s3.deleteObject(self.aws_s3_storage_bucket, uuid)) d = DeferredList(deferreds) d.addCallback(self._deleteReservationCallback, function_name, uuid) d.addErrback(self._deleteReservationErrback, function_name, uuid) return d
def create_all_or_nothing(deferred_list): def restore_failure(failure): return failure.value.subFailure instance = DeferredList(deferred_list, fireOnOneErrback=True, consumeErrors=True) instance.addErrback(restore_failure) return instance
def createReservation( self, function_name, **kwargs ): if not isinstance( function_name, str ): for key in self.functions: if self.functions[key]["function"] == function_name: function_name = key break if function_name not in self.functions: raise Exception("Function %s does not exist." % function_name ) function = self.functions[ function_name ] filtered_kwargs = {} for key in function["required_arguments"]: if key in kwargs: filtered_kwargs[key] = convertToUTF8( kwargs[key] ) else: raise Exception("Required parameter '%s' not found. Required parameters are %s. Optional parameters are %s." % (key, function["required_arguments"], function["optional_arguments"] )) for key in function["optional_arguments"]: if key in kwargs: filtered_kwargs[key] = convertToUTF8( kwargs[key] ) if function["interval"] > 0: reserved_arguments = {} reserved_arguments["reservation_function_name"] = function_name reserved_arguments["reservation_created"] = sdb_now(offset=self.time_offset) reserved_arguments["reservation_next_request"] = reserved_arguments["reservation_created"] reserved_arguments["reservation_error"] = "0" all_arguments = {} all_arguments.update( reserved_arguments ) all_arguments.update( filtered_kwargs ) uuid = uuid4().hex logger.debug( "Creating reservation on SimpleDB for %s, %s." % (function_name, uuid)) a = self.sdb.putAttributes( self.aws_sdb_reservation_domain, uuid, all_arguments ) a.addCallback( self._createReservationCallback, function_name, uuid ) a.addErrback( self._createReservationErrback, function_name, uuid ) if "call_immediately" in kwargs and not evaluateBoolean( kwargs["call_immediately"] ): d = DeferredList([a], consumeErrors=True) else: logger.debug( "Calling %s immediately with arguments:\n%s" % (function_name, PrettyPrinter.pformat(filtered_kwargs) ) ) b = self.callExposedFunctionImmediately( function["function"], filtered_kwargs, function_name ) d = DeferredList([a,b], consumeErrors=True) d.addCallback( self._createReservationCallback2, function_name, uuid ) d.addErrback( self._createReservationErrback2, function_name, uuid ) return d else: logger.debug( "Calling %s immediately with arguments:\n%s" % (function_name, PrettyPrinter.pformat(filtered_kwargs) ) ) d = self.callExposedFunctionImmediately( function["function"], filtered_kwargs, function_name ) return d
def testStoredReturn(self): deferreds = [] for filename in self.filenames: d = self.spider.rq.getPage("http://127.0.0.1:5000/function/load_data_stored", method="POST", postdata={"filename":filename}) d.addCallback( self._processStoredRequestCallback, filename ) deferreds.append(d) d = DeferredList(deferreds, consumeErrors=True ) d.addCallback(self._testStoredReturnCallback) d.addErrback( self._testStoredReturnErrback ) return d
def deleteReservation(self, uuid, function_name="Unknown"): LOGGER.info("Deleting reservation %s, %s." % (function_name, uuid)) parameters = {'uuid': uuid} query_string = urllib.urlencode(parameters) url = 'http://%s:%s/function/schedulerserver/remoteremovefromheap?%s' % (self.scheduler_server, self.schedulerserver_port, query_string) deferreds = [] deferreds.append(self.getPage(url=url)) deferreds.append(self.s3.deleteObject(self.aws_s3_storage_bucket, uuid)) d = DeferredList(deferreds) d.addCallback(self._deleteReservationCallback, function_name, uuid) d.addErrback(self._deleteReservationErrback, function_name, uuid) return d
def main(myName): # This file defines the network of virtual quantum nodes virtualFile = os.environ.get('NETSIM') + "/config/virtualNodes.cfg" # This file defines the network of CQC servers interfacing to virtual quantum nodes cqcFile = os.environ.get('NETSIM') + "/config/cqcNodes.cfg" # Read configuration files for the virtual quantum, as well as the classical network virtualNet = networkConfig(virtualFile) cqcNet = networkConfig(cqcFile) # Check if we should run a local classical server. If so, initialize the code # to handle remote connections on the classical communication network if myName in cqcNet.hostDict: myHost = cqcNet.hostDict[myName] cqc_factory = CQCFactory(myHost, myName, cqcNet) else: logging.error( "LOCAL %s: Cannot start classical communication servers.", myName, e.strerror) # Initialize Twisted callback framework dList = [] try: logging.debug( "LOCAL %s: Starting local classical communication server.", myName) myHost.root = cqc_factory myHost.factory = cqc_factory reactor.listenTCP(myHost.port, myHost.factory) except Exception as e: logging.error("LOCAL %s: Cannot start CQC server.", myName, e.strerror) return # Connect to the local virtual node simulating the "local" qubits try: logging.debug("LOCAL %s: Connecting to local virtual node.", myName) virtual_node = virtualNet.hostDict[myName] factory = pb.PBClientFactory() reactor.connectTCP(virtual_node.hostname, virtual_node.port, factory) deferVirtual = factory.getRootObject() dList.append(deferVirtual) deferList = DeferredList(dList, consumeErrors=True) deferList.addCallback(init_register, myName, cqc_factory) deferList.addErrback(localError) reactor.run() except Exception as e: logging.error("LOCAL %s: Cannot connect to SimulaQron backend.", myName) return
def _do_populate_tags(clients): """Send RPC calls to each rack controller, requesting evaluation of tags. :param clients: List of connected rack controllers that EvaluateTag will be called. """ def call_client(client_info): client = client_info["client"] return client( EvaluateTag, system_id=client_info["system_id"], tag_name=client_info["tag_name"], tag_definition=client_info["tag_definition"], tag_nsmap=client_info["tag_nsmap"], credentials=client_info["credentials"], nodes=client_info["nodes"], ) def check_results(results): for client_info, (success, result) in zip(clients, results): if success: maaslog.info( "Tag %s (%s) evaluated on rack controller %s (%s)", client_info["tag_name"], client_info["tag_definition"], client_info["hostname"], client_info["system_id"], ) else: maaslog.error( "Tag %s (%s) could not be evaluated on rack controller " "%s (%s): %s", client_info["tag_name"], client_info["tag_definition"], client_info["hostname"], client_info["system_id"], result.getErrorMessage(), ) d = DeferredList( (call_client(client_info) for client_info in clients), consumeErrors=True, ) d.addCallback(check_results) d.addErrback(log.err) # Do *not* return a Deferred; the caller is not meant to wait around for # this to finish. However, for the sake of testing, we return the Deferred # wrapped up in a list so that crochet does not block. return [d]
def publish(self): """Publish all added charms to provider storage and zookeeper. Returns the charm_state of all scheduled charms. """ publish_deferreds = [] for charm_id, charm in self._charm_add_queue: publish_deferreds.append(self._publish_one(charm_id, charm)) publish_deferred = DeferredList(publish_deferreds, fireOnOneErrback=1, consumeErrors=1) # callbacks and deferreds to unwind the dlist publish_deferred.addCallback(self._cb_extract_charm_state) publish_deferred.addErrback(self._eb_extract_error) return publish_deferred
def multi_set_attr(self, owner_userid, image_ids, attr_dict): """ Sets attributes on a list of images to be whatever is in the attr_dict. attr_dict must have keys that are valid in images.set_attr @param owner_username: Username @type owner_username: String @param media_id_list: set of media_ids to change @type media_id_list: List @param attr_dict: dictionary of attribute->values to apply to the list of images @type attr_dict: Dictionary @return: image attributes @rtype: List """ owner_userid = validation.cast_integer(owner_userid, 'owner_userid') if not isinstance(image_ids, (list, tuple)): raise errors.ValidationError, 'media_id_list must be a list or tuple' if not isinstance(attr_dict, dict): raise errors.ValidationError, 'attr_dict must be a dictionary' for x in image_ids: self.log.debug("validating image_id [%s]" % x) validation.cast_integer(x, 'x') for x in attr_dict.keys(): if x not in self.attr_fields: raise errors.ValidationError, 'attr_dict key [%s] is not valid!' % x self.log.debug("all args look OK in multi_set_attr") dl = [] for image_id in image_ids: image_id = validation.cast_integer(image_id, 'image_id') d_temp = Deferred() self.log.debug("adding deferreds for image_id [%s]" % image_id) for key, value in attr_dict.items(): self.log.debug("adding defferred for image_id [%s] key [%s]" % (image_id, key)) d_temp.addCallback(self.set_attr, image_id, key, value) d_temp.addCallback(lambda _: owner_userid) d_temp.callback(owner_userid) dl.append(d_temp) d_list = DeferredList(dl, fireOnOneErrback=True) d_list.addCallback(lambda _: 0); def handle_it(failure): self.log.warning("failure on multi_set_attr [%s]" % failure.getErrorMessage()) d_list.addErrback(handle_it) return d_list
def _fallback_and_ping_my_servers(self): """ Just ping all my id servers by sending a HTTP request to the "main page". """ my_sources = my_id.getLocalIdentity().getSources(as_originals=True) id_servers = [] for idurl_bin in my_sources: proto, host, port, _ = nameurl.UrlParse(idurl_bin) id_servers.append(nameurl.UrlMake(proto, host, port, '')) if _Debug: lg.args(_DebugLevel, id_servers=id_servers) dl = [] for url in id_servers: d = net_misc.getPageTwisted(url, timeout=5) dl.append(d) d = DeferredList(dl, consumeErrors=True) d.addCallback(lambda result: self.automat('ping-done', [])) d.addErrback(self._fallback_check_network_connected)
def test__deferredDHCPRequestErrback_cancels_all_on_FirstError(self): mock_cancelAll = self.patch(DHCPRequestMonitor, 'cancelAll') def raise_ioerror(): raise IOError() a = deferLater(reactor, 0.0, raise_ioerror) b = deferLater(reactor, 6, lambda: 'b') monitor = DHCPRequestMonitor('lo') monitor.deferredDHCPRequests = [a, b] deferredList = DeferredList(monitor.deferredDHCPRequests, consumeErrors=True, fireOnOneErrback=True) deferredList.addErrback(monitor.deferredDHCPRequestErrback) yield deferredList # Still have one call left in the reactor, since we mocked cancelAll(). b.cancel() self.assertThat(mock_cancelAll, MockCallsMatch(call([a, b])))
def xmlrpc_del_albums(self, info, set_ids, album_ids): @stack def process_set(set_id): dl2 = [] for album_id in album_ids: dl2.append(process_album(set_id, album_id)) dList2 = DeferredList(dl2, fireOnOneErrback=True) return dList2 @stack def process_album(set_id, album_id): return self.del_album(info['userid'], set_id, album_id) dl = [] for set_id in set_ids: dl.append(process_set(set_id)) dList = DeferredList(dl, fireOnOneErrback=True) dList.addCallback(lambda _: (0, "success")) dList.addErrback(lambda _: (-1, _.getErrorMessage)) return dList
def notifyOfUpdate(self, gridKeys: List[str]): """ Notify of Grid Updates This method is called by the client.GridCacheController when it receives updates from the server. """ self._filterOutOfflineVortexes() payloadsByVortexUuid = defaultdict(Payload) for gridKey in gridKeys: gridTuple = self._cacheController.encodedChunk(gridKey) if not gridTuple: gridTuple = EncodedGridTuple() gridTuple.gridKey = gridKeys vortexUuids = self._observedVortexUuidsByGridKey.get(gridKey, []) # Queue up the required client notifications for vortexUuid in vortexUuids: logger.debug("Sending unsolicited grid %s to vortex %s", gridKey, vortexUuid) payloadsByVortexUuid[vortexUuid].tuples.append(gridTuple) # Send the updates to the clients dl = [] for vortexUuid, payload in list(payloadsByVortexUuid.items()): payload.filt = clientGridWatchUpdateFromDeviceFilt # Serliase in thread, and then send. d = payload.makePayloadEnvelopeDefer() d.addCallback( lambda payloadEnvelope: payloadEnvelope.toVortexMsgDefer()) d.addCallback(VortexFactory.sendVortexMsg, destVortexUuid=vortexUuid) dl.append(d) # Log the errors, otherwise we don't care about them dl = DeferredList(dl, fireOnOneErrback=True) dl.addErrback(vortexLogFailure, logger, consumeError=True)
def _do_identity_cache(ret): all_stories = [] for _supplier_idurl in ret['suppliers']: if _supplier_idurl: _supplier_idurl = id_url.to_bin(_supplier_idurl) if not id_url.is_cached( _supplier_idurl) or not identitycache.HasFile( _supplier_idurl): one_supplier_story = identitycache.immediatelyCaching( _supplier_idurl) if _Debug: one_supplier_story.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='read_customer_suppliers._do_identity_cache' ) all_stories.append(one_supplier_story) _customer_idurl = id_url.to_bin(ret['customer_idurl']) if _customer_idurl and (not id_url.is_cached(_customer_idurl) or not identitycache.HasFile(_customer_idurl)): one_customer_story = identitycache.immediatelyCaching( _customer_idurl) if _Debug: one_customer_story.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='read_customer_suppliers._do_identity_cache') all_stories.append(one_customer_story) if _Debug: lg.args(_DebugLevel, all_stories=len(all_stories), ret=ret) id_cache_story = DeferredList(all_stories, consumeErrors=True) id_cache_story.addCallback(_do_save_customer_suppliers, ret) if _Debug: id_cache_story.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='read_customer_suppliers._do_identity_cache') id_cache_story.addErrback(result.errback) return id_cache_story
def send_payment_request(self, readTokens, writeTokens): """Called by a Circuit object when it wants to actually make a payment @param readTokens: the number of read tokens to pay for at each hop in the circuit @type readTokens: int @param writeTokens: the number of read tokens to pay for at each hop in the circuit @type writeTokens: int""" assert ( readTokens + writeTokens ) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells" #make sure our setup is done: if not self.setupDone: #have we even started? if not self.setupStarted: self.send_setup_message() self.queuedReadTokens += readTokens self.queuedWriteTokens += writeTokens return #dont bother trying to send payments for circuits that are already closed if self.circ.is_done(): return #send the payments deferreds = [] for paymentStream in self.paymentStreams.values(): deferreds.append( paymentStream.send_payment(readTokens, writeTokens)) paymentsDoneDeferred = DeferredList(deferreds) paymentsDoneDeferred.addErrback(self.generic_error_handler) addTokensDeferred = Deferred() self.inflightReadTokens += readTokens self.inflightWriteTokens += writeTokens #timeout in case the payment fails. We will close the circuit in this case. event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT, self.all_receipts_received, None, addTokensDeferred, readTokens, writeTokens, None) paymentsDoneDeferred.addCallback(self.all_receipts_received, addTokensDeferred, readTokens, writeTokens, event) addTokensDeferred.addCallback(self._add_tokens_callback, readTokens, writeTokens) addTokensDeferred.addErrback(self.generic_error_handler)
def moveFiles(self, virtSrcBase, virtDestBase, relFiles): self.debug("MOVING: %r, %r, %r", virtSrcBase, virtDestBase, relFiles) if not self._local: raise TranscoderError("Component not properly setup yet") def move_failed(failure, src, dest): msg = ("Fail to move file '%s' to '%s': %s" % (src, dest, log.getFailureMessage(failure))) self.warning("%s", msg) raise TranscoderError(msg, cause=failure) def move_file(src, dest, attr=None): self.debug("Moving file '%s' to '%s'", src, dest) dest_dir = os.path.dirname(dest) safe_mkdirs(dest_dir, "input file destination", attr) d = deferToThread(shutil.move, src, dest) d.addErrback(move_failed, src, dest) return d def move_files_failed(results): first_failure = None for ok, result in results: if not ok: if not first_failure: first_failure = result return first_failure sem = DeferredSemaphore(1) move_tasks = [] for file in relFiles: source_path = virtSrcBase.append(file).localize(self._local) dest_path = virtDestBase.append(file).localize(self._local) source_path = os.path.realpath(source_path) dest_path = os.path.realpath(dest_path) d = sem.run(move_file, source_path, dest_path, self._pathAttr) move_tasks.append(d) dl = DeferredList(move_tasks, consumeErrors=True) dl.addErrback(move_files_failed) return d
def run(): print('run') def callback(*args, **kwargs): print('callback', args) d = args[0] if not isinstance(d, dict): print('NOT FOUND') return assert isinstance( d, dict), 'result value is not dictionary, key-value not found' assert len(d) == 1 k, v = d.popitem() assert k.replace('key', '') == v.replace('value', '') def errback(*args, **kwargs): import traceback traceback.print_exc() def callback_dfl(*args): print('callback_dfl', args) reactor.stop() # @UndefinedVariable errback_dfl = errback time.sleep(3) try: list_of_deffered_set_value = [] for i in range(options.start, options.end + 1): d = dht_service.get_json_value(str(i), layer_id=options.layer) d.addBoth(callback) d.addErrback(errback) list_of_deffered_set_value.append(d) dfl = DeferredList(list_of_deffered_set_value) dfl.addCallback(callback_dfl) dfl.addErrback(errback_dfl) except Exception as exc: print('ERROR in run()', exc) reactor.stop() # @UndefinedVariable
def main(): """Main command line entry point.""" parser = argparse.ArgumentParser( description='Make requests to one or more HTTP or HTTPS URIs, ' 'and record the interactions in a cassette.', epilog='If no URIs are passed on the command line, they are ' 'read from standard input, one per line.') parser.add_argument( 'uris', metavar='URI', nargs='*', help='URI to fetch') parser.add_argument( 'cassette_path', metavar='CASSETTE', help='path to output cassette') args = parser.parse_args() uris = args.uris or imap(lambda x: x.strip(), sys.stdin) cassette_agent = CassetteAgent(Agent(reactor), args.cassette_path) agent = ContentDecoderAgent( RedirectAgent(cassette_agent), [('gzip', GzipDecoder)]) finished = DeferredList([agent.request('GET', uri) for uri in uris]) finished.addCallback(save_and_exit, cassette_agent) finished.addErrback(fail_and_exit) reactor.run()
def oneBatch(prev, ipList): done = 0 deferreds = [] for toDo in ipList: deferreds.append(makeRequest(toDo)) done += 1 if done == concurrentRequests: break else: print "\n%d requests ended in empty match" % prev d = getPage("%s/stats/avg" % (baseUrl, )) @d.addCallback def c2(time): print "Average matching time in server: %s" % time reactor.stop() return d = DeferredList(deferreds) @d.addCallback def c(results): return prev + len([x for x in results if x[1] in ("[]", "ok\n")]) d.addErrback(log.err) d.addBoth(oneBatch, ipList)
def run(): print('run') def callback(*args, **kwargs): print('callback', args, kwargs) l = args[0] assert len(l) > 0 def errback(*args, **kwargs): import traceback traceback.print_exc() def callback_dfl(*args): print('callback_dfl', args) reactor.stop() # @UndefinedVariable errback_dfl = errback time.sleep(3) try: list_of_deffered_set_value = [] for i in range(options.start, options.end + 1): j = { 'key' + str(i): 'value' + str(i), } d = dht_service.set_json_value(str(i), json_data=j, age=60 * 60, layer_id=options.layer) d.addBoth(callback, j) d.addErrback(errback) list_of_deffered_set_value.append(d) dfl = DeferredList(list_of_deffered_set_value) dfl.addCallback(callback_dfl) dfl.addErrback(errback_dfl) except Exception as exc: print('ERROR in run()', exc) reactor.stop() # @UndefinedVariable
def _on_connected(self, ok): from twisted.internet.defer import DeferredList from logs import lg from dht import dht_service from dht import known_nodes from main.config import conf from services import driver lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok)) dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store) dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request) known_seeds = known_nodes.nodes() dl = [] attached_layers = conf().getData('services/entangled-dht/attached-layers', default='') if attached_layers: attached_layers = list(filter(None, map(lambda v: int(str(v).strip()), attached_layers.split(',')))) else: attached_layers = [] lg.info('reading attached DHT layers from configuration: %r' % attached_layers) all_services_attached_layers = driver.get_attached_dht_layers().values() combined_services_attached_layers = set() count_combined = len(list(map(combined_services_attached_layers.update, all_services_attached_layers))) services_attached_layers = list(combined_services_attached_layers) lg.info('combined attached DHT layers from %d services: %r' % (count_combined, services_attached_layers, )) attached_layers = list(set(attached_layers + services_attached_layers)) lg.info('DHT layers to be attached at startup: %r' % attached_layers) for layer_id in attached_layers: dl.append(dht_service.open_layer( layer_id=layer_id, seed_nodes=known_seeds, connect_now=True, attach=True, )) if dl: d = DeferredList(dl) d.addCallback(self._on_layers_attached) d.addErrback(self._on_connect_failed) else: if self.starting_deferred and not self.starting_deferred.called: self.starting_deferred.callback(True) return ok
def render(self, request): request.setHeader('Content-type', 'text/javascript; charset=UTF-8') if len(request.postpath) > 0: if request.postpath[0] == "show_reservation": if "uuid" in request.args: deferreds = [] for uuid in request.args["uuid"]: deferreds.append(self.interfaceserver.showReservation(uuid)) d = DeferredList(deferreds, consumeErrors=True) d.addCallback(self._showReservationCallback, request.args["uuid"]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required."))) elif request.postpath[0] == "execute_reservation": if "uuid" in request.args: d = self.interfaceserver.executeReservation(request.args["uuid"][0]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required."))) elif request.postpath[0] == "delete_reservation": if "uuid" in request.args: d = self.interfaceserver.deleteReservation(request.args["uuid"][0]) d.addCallback(self._successResponse) d.addErrback(self._errorResponse) d.addCallback(self._immediateResponse, request) return server.NOT_DONE_YET else: return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
def receive_item(self, item, queue, auto_ack, incoming_callbacks): """ This function is called with EVERY incoming message. We don't have logic here, just send to factory. """ (channel, deliver, properties, message) = item queue_deferred = queue.get() # get the queue again, so we can add another callback to get the next message. queue_deferred.addCallback(self.receive_item, queue, auto_ack, incoming_callbacks) queue_deferred.addErrback(self.receive_item_err) # todo: use supplied error callback... which one? received_callbacks = [] for callback in incoming_callbacks: received_callbacks.append(maybeDeferred(callback, channel=channel, deliver=deliver, properties=properties, message=message) ) if auto_ack is False: received = DeferredList(received_callbacks, fireOnOneErrback=True, consumeErrors=True) received.addCallback(self._basic_ack, channel, deliver.delivery_tag) received.addErrback(self._basic_nack, channel, deliver.delivery_tag)
def _configure_ssh(self, deployment): """ :return: A ``Deferred`` which fires when all nodes have been configured with ssh keys. """ self.ssh_configuration.create_keypair() results = [] for node in deployment.nodes: results.append( deferToThread(self.ssh_configuration.configure_ssh, node.hostname, self.ssh_port)) d = DeferredList(results, fireOnOneErrback=True, consumeErrors=True) # Exit with ssh's output if it failed for some reason: def got_failure(failure): if failure.value.subFailure.check(CalledProcessError): raise SystemExit(b"Error connecting to cluster node: " + failure.value.subFailure.value.output) else: return failure d.addErrback(got_failure) return d
def _process_incoming_email(self, otherself, filepath, mask): """ Callback that processes incoming email @param otherself: Watch object for the current callback from inotify @type otherself: twisted.internet.inotify._Watch @param filepath: Path of the file that changed @type filepath: twisted.python.filepath.FilePath @param mask: identifier for the type of change that triggered this callback @type mask: int """ if os.path.split(filepath.dirname())[-1] == "new": log.msg("Processing new mail at %s" % (filepath.path,)) with filepath.open("r") as f: mail_data = f.read() mail = message_from_string(mail_data) owner = mail["To"] if owner is None: # default to Delivered-To owner = mail["Delivered-To"] if owner is None: log.err("Malformed mail, neither To: nor " "Delivered-To: field") log.msg("Mail owner: %s" % (owner,)) log.msg("%s received a new mail" % (owner,)) dpubk = self._users_cdb.getPubKey(owner) duuid = self._users_cdb.queryByAddress(owner) d = DeferredList([dpubk, duuid]) d.addCallbacks(self._gather_uuid_pubkey, log.err) d.addCallbacks(self._encrypt_message, log.err, (owner, mail_data)) d.addCallbacks(self._export_message, log.err) d.addCallbacks(self._conditional_remove, log.err, (filepath,)) d.addErrback(log.err)
def __init__(self, endpointA, endpointB): """ Initialize the connection between the two endpoints. The connection will be scheduled to be created here. @param endpointX: Endpoint which is part of the new connection. @type endpointX: rce.core.network.Endpoint """ assert endpointA != endpointB # Register the endpoints and create Protocol proxies self._serverEndpoint = endpointA self._clientEndpoint = endpointB self._serverProtocol = Protocol(endpointA) self._clientProtocol = Protocol(endpointB) serverStatus = Status(self._serverProtocol) clientStatus = Status(self._clientProtocol) # Create the keys used to validate the connection connectionID = uuid4().bytes serverKey = uuid4().bytes clientKey = uuid4().bytes # Create Validators and get result Deferred authServer = _ConnectionValidator(serverKey) authClient = _ConnectionValidator(clientKey) authServerResult = authServer.result authClientResult = authClient.result authenticator = DeferredList([authServerResult, authClientResult]) authenticator.addCallback(self._validate) authenticator.addErrback(self._logError) readyServer = endpointA.prepareConnection(connectionID, serverKey, authClient, serverStatus) readyClient = endpointB.prepareConnection(connectionID, clientKey, authServer, clientStatus) ready = DeferredList([readyServer, readyClient]) ready.addCallback(self._getAddress) ready.addCallback(self._connect, connectionID) ready.addErrback(self._connectPrepError, authenticator) ready.addErrback(self._logError)
def __init__(self, endpointA, endpointB): """ Initialize the connection between the two endpoints. The connection will be scheduled to be created here. @param endpointX: Endpoint which is part of the new connection. @type endpointX: rce.core.network.Endpoint """ assert endpointA != endpointB # Register the endpoints and create Protocol proxies self._serverEndpoint = endpointA self._clientEndpoint = endpointB self._serverProtocol = Protocol(endpointA) self._clientProtocol = Protocol(endpointB) # Create the keys used to validate the connection connectionID = uuid4().bytes serverKey = uuid4().bytes clientKey = uuid4().bytes # Create Validators and get result Deferred authServer = _ConnectionValidator(serverKey) authClient = _ConnectionValidator(clientKey) authServerResult = authServer.result authClientResult = authClient.result authenticator = DeferredList([authServerResult, authClientResult]) authenticator.addCallback(self._validate) authenticator.addErrback(self._logError) readyServer = endpointA.prepareConnection(connectionID, serverKey, authClient) readyClient = endpointB.prepareConnection(connectionID, clientKey, authServer) ready = DeferredList([readyServer, readyClient]) ready.addCallback(self._getAddress) ready.addCallback(self._connect, connectionID) ready.addErrback(self._connectPrepError, authenticator) ready.addErrback(self._logError)
def gather_results(deferreds, consume_errors=True): d = DeferredList(deferreds, fireOnOneErrback=1, consumeErrors=consume_errors) d.addCallback(lambda r: [x[1] for x in r]) d.addErrback(lambda f: f.value.subFailure) return d
media_id = result[1]["media_id"] owner_username = result[1]["owner_username"] d3 = self._locate_media(media_id) d3.addCallback(delete_image, media_id, owner_username) return d3 dl = [] for id in id_list: d = self.app.api.images.get_media_owner_id(id) d.addCallback(handle_media_info) dl.append(d) d_list = DeferredList(dl) d_list.addCallback(lambda _: (0, _)) d_list.addErrback(lambda _: (-1, _.getErrorMessage())) return d_list @stack def get_render(self, image_id, width, height, crop): """ Gets a render for an image. @param owner_username: User who owns the image @type owner_username: String @param media_id: ID of the image @type media_id: String @param width: Width of the render @type width: Integer
class DHCPRequestMonitor: def __init__(self, ifname: str, clock: IReactorThreads = None): if clock is None: clock = reactor self.clock = clock # type: IReactorThreads self.ifname = ifname # type: str self.servers = None # type: set self.dhcpRequestsDeferredList = None # type: DeferredList self.deferredDHCPRequests = [] # type: List[Deferred] self.transaction_id = make_dhcp_transaction_id() # type: bytes def send_requests_and_await_replies(self): """Sends out DHCP requests and waits for their replies. This method is intended to run under `deferToThread()`. Calls the reactor using `blockingCallFromThread` to queue the request packets. Blocks for ~10 seconds while checking for DHCP offers. :returns: `set` of `DHCPServer` objects. """ # Since deferToThread() might be delayed until the next thread is # available, it's better to kick off the DHCP requests from the # spawned thread rather than hoping the thread starts running after # we kick off the requests. blockingCallFromThread(self.clock, self.deferDHCPRequests) servers = set() # Convert the transaction_id to an integer so we can test it against # what the parsed DHCP packet will return. xid = int.from_bytes(self.transaction_id, byteorder="big") with udp_socket() as sock: sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) # Note: the empty string is the equivalent of INADDR_ANY. sock.bind(("", BOOTP_CLIENT_PORT)) # The timeout has to be relatively small, since we wake up every # timeout interval to check the elapsed time. sock.settimeout(0.5) runtime = 0 # Use a monotonic clock to ensure leaping backward in time won't # cause an infinite loop. start_time = time.monotonic() while runtime < REPLY_TIMEOUT: try: # Use recvfrom() to check the source IP for the request. # It could be interesting in cases where DHCP relay is in # use. data, (address, port) = sock.recvfrom(2048) except socket.timeout: continue else: offer = DHCP(data) if not offer.is_valid(): log.info( "Invalid DHCP response received from {address} " "on '{ifname}': {reason}", address=address, ifname=self.ifname, reason=offer.invalid_reason, ) elif offer.packet.xid == xid: # Offer matches our transaction ID, so check if it has # a Server Identifier option. server = offer.server_identifier if server is not None: servers.add(DHCPServer(server, address)) finally: runtime = time.monotonic() - start_time return servers @staticmethod def cancelAll(deferreds: List[Deferred]): for deferred in deferreds: deferred.cancel() def deferredDHCPRequestErrback(self, failure: Failure) -> Optional[Failure]: if failure.check(FirstError): # If an error occurred, cancel any other pending requests. # (The error is likely to occur for those requests, too.) # Unfortunately we can't cancel using the DeferredList, since # the DeferredList considers itself "called" the moment the first # errback is invoked. self.cancelAll(self.deferredDHCPRequests) # Suppress further error handling. The original Deferred's errback # has already been called. return None elif failure.check(DHCPProbeException): log.msg("DHCP probe failed. %s" % failure.getErrorMessage()) elif failure.check(CancelledError): # Intentionally cancelled; no need to spam the log. pass else: log.err( failure, "DHCP probe on '%s' failed with an unknown error." % (self.ifname), ) # Make sure the error is propagated to the DeferredList. # We need this so that the DeferredList knows to call us with # FirstError, which is our indicator to cancel the remaining calls. # (It's set to consumeErrors, so it won't spam the log.) return failure def deferDHCPRequests(self) -> None: """Queues some DHCP requests to fire off later. Delays calls slightly so we have a chance to open a listen socket. Uses the `clock`, `ifname`, and `transaction_id` ivars. Stores a `DeferredList` of periodic DHCP requests calls in the `dhcpRequestsDeferredList` ivar, and the list of `Deferred` objects in the `deferredDHCPRequests` ivar. """ self.deferredDHCPRequests = [] for seconds in DHCP_REQUEST_TIMING: packet = DHCPDiscoverPacket(transaction_id=self.transaction_id, seconds=seconds) # Wait 0.1 seconds before sending the request, so we have a chance # to open a listen socket. seconds += 0.1 deferred = deferLater( self.clock, seconds, send_dhcp_request_packet, packet, self.ifname, ) deferred.addErrback(self.deferredDHCPRequestErrback) self.deferredDHCPRequests.append(deferred) # Use fireOnOneErrback so that we know to cancel the remaining attempts # to send requests if one of them fails. self.dhcpRequestsDeferredList = DeferredList( self.deferredDHCPRequests, fireOnOneErrback=True, consumeErrors=True, ) self.dhcpRequestsDeferredList.addErrback( self.deferredDHCPRequestErrback) @inlineCallbacks def run(self) -> Deferred: """Queues DHCP requests to be sent, then waits (in a separate thread) for replies. Requests will be sent using an exponential back-off algorithm, to mimic a real DHCP client. But we'll just pretend we didn't see any of the replies, and hope for more servers to respond. The set of `DHCPServer`s that responded to the request(s) is stored in the `servers` ivar, which in turn makes the `dhcp_servers` and `dhcp_addresses` properties useful. """ servers = yield deferToThread(self.send_requests_and_await_replies) if len(servers) > 0: log.info( "External DHCP server(s) discovered on interface '{ifname}': " "{servers}", ifname=self.ifname, servers=", ".join( str(server) for server in sorted(list(servers))), ) self.servers = servers @property def dhcp_servers(self): return set(str(server.server) for server in self.servers) @property def dhcp_addresses(self): return set(str(server.address) for server in self.servers)
def connectTCP(self): c = [MultiClientProxy(self.servers,urlencode=True,debug=True) for i in range(self.count)] dl = [client.connectTCP() for client in c] dl = DeferredList(dl) dl.addCallback(self._connected, c) return dl.addErrback(self.err_back)
def send_direct_tor_message(self, msg, msgType, forward=True, numHops=1, sendOverCircuit=False): """Tunnel a message through Tor. There are two ways to send data: sendOverCircuit=True: These messages are visible (plain-text) to the hops that they pass through! Callers are responsible for any necessary secrecy and intergrity. sendOverCircuit=False: These messages are encrypted like normal, relayed Tor cells. They are thus encrypted and authenticated, but messages may not be sent between two relays (only between the origin and relays) In either case, messages that are too long will be sent in multiple cells. @param msg: the message to send @type msg: str @param msgType: the type of message. Must be one of MESSAGE_CODES. @type msgType: str @param forward: whether to send towards the exit (True) or towards the origin (False) @type forward: bool @param numHops: how many relays to traverse before the message is delivered. MUST NOT BE 0--in that case, call the handler directly yourself. @type numHops: int @param sendOverCircuit: whether to send over the circuit (True) or simply send over OR connections to adjacent hops (False) @type sendOverCircuit: bool""" if not self.torApp.is_ready(): raise TorCtl.TorCtlClosed if self.isClosed: log_msg("Cannot send Tor message, circuit was closed (%s)" % (msgType)) return # if numHops is 0, you should handle the message yourself, not send it assert numHops != 0, "cannot send a zero hop message" msg = Basic.write_byte(MESSAGE_CODES[msgType]) + msg # put the length in front of the message: msgLen = len(msg) msg = Basic.write_short(msgLen) + msg # who to send it to: nextHexId = self.nextHexId nextCircId = self.nextCircId if not forward: nextCircId = self.prevCircId nextHexId = self.prevHexId dList = [] # different message lengths depending on if sendOverCircuit if True or False: if sendOverCircuit: # are sent as normal relayed messages, so they should be this long WRITABLE_BYTES = 498 else: # since the Tor cell is 512 bytes, but we need 2 for circid, and 1 for the cell command WRITABLE_BYTES = 507 while len(msg) > 0: dataToSend = msg[:WRITABLE_BYTES] msg = msg[WRITABLE_BYTES:] def add_padding(tmp, desiredLen): extraChars = desiredLen - len(tmp) return tmp + (" " * extraChars) dataToSend = add_padding(dataToSend, WRITABLE_BYTES) dataToSend = dataToSend.encode("base64") dataToSend = dataToSend.replace("\n", "").replace("=", "") # convert sendOverCircuit to "1" or "0" for the control connection: if sendOverCircuit: sendOverCircuitToken = "1" else: sendOverCircuitToken = "0" dataToSend = "SENDPAYMENT %s %s %s %s %s\r\n" % ( nextHexId, nextCircId, dataToSend, numHops, sendOverCircuitToken, ) d = self.torApp.conn.sendAndRecv(dataToSend) dList.append(d) d = DeferredList(dList) def response(result): for x in result: if not x[0]: raise Exception(str(x)) if x[1][0][0] != "250": raise Exception(str(result)) read, write = x[1][0][1].split(" ") read = int(read) write = int(write) return (read, write) d.addCallback(response) def error(failure): # this happens occasionally when the circuit is closed at approximately the same time that we send a payment # it can be safely ignored because the circuit is closed and we already learned about it if "552 Cannot find circuit with id" in str(failure): log_msg("A circuit that we tried to send a payment message to was closed. Oops.", 4) self.close() return # otherwise, log an error because this is unexpected log_ex(failure, "SENDPAYMENT failed for circuit=%s" % (nextCircId), [TorCtl.ErrorReply]) d.addErrback(error) return d