def test_complex2(self, runtime):
def check(ls):
if (2 == runtime.id) or (1 == runtime.id):
self.assertEquals(ls[0][1], "V1")
self.assertEquals(ls[1][1], "V1")
self.assertEquals(ls[2][1], "V1")
self.assertEquals(ls[3][1], "V2")
else:
self.assertEquals(ls[0][1], "V1")
self.assertEquals(ls[1][1], "V1")
self.assertEquals(ls[2][1], "V1")
self.assertEquals(ls[3][1], "V2")
self.assertEquals(ls[4][1], "V2")
field = self.Zp
results = []
results += runtime.broadcast(runtime.players.keys(), runtime.players.keys(), "V1")
if runtime.id in [1, 2]:
v = runtime.broadcast([1, 2], [3], "V2")
if isinstance(v, list):
results += v
else:
results.append(v)
else:
results += runtime.broadcast([1, 2], [3])
if 3 == runtime.id:
results += [runtime.broadcast([3], runtime.players.keys(), str(7))]
else:
results += [runtime.broadcast([3], runtime.players.keys())]
dls = DeferredList(results)
runtime.schedule_callback(dls, check)
dls.addErrback(runtime.error_handler)
return dls
def _configure_ssh(self, deployment):
"""
:return: A ``Deferred`` which fires when all nodes have been configured
with ssh keys.
"""
self.ssh_configuration.create_keypair()
results = []
for node in deployment.nodes:
results.append(
deferToThread(
self.ssh_configuration.configure_ssh,
node.hostname, self.ssh_port
)
)
d = DeferredList(results, fireOnOneErrback=True, consumeErrors=True)
# Exit with ssh's output if it failed for some reason:
def got_failure(failure):
if failure.value.subFailure.check(CalledProcessError):
raise SystemExit(
b"Error connecting to cluster node: " +
failure.value.subFailure.value.output)
else:
return failure
d.addErrback(got_failure)
return d
def _observe_nodes(self, nodes):
if self.stopped:
lg.warn('DiscoveryTask[%r] : discovery process already stopped' %
self.id)
return
if _Debug:
lg.out(
_DebugLevel,
'lookup.DiscoveryTask[%r]._observe_nodes started for %d items layer_id=%d'
% (
self.id,
len(nodes),
self.layer_id,
))
observe_list = []
for node in nodes:
d = self.observe_method(node, layer_id=self.layer_id)
d.addCallback(self._on_node_observed, node)
d.addErrback(self._on_node_observe_failed, node)
observe_list.append(d)
self.observed_count = len(nodes)
dl = DeferredList(observe_list, consumeErrors=False)
dl.addCallback(self._on_all_nodes_observed)
if _Debug:
dl.addErrback(lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='DiscoveryTask._observe_nodes')
def doBackupKeys(self, *args, **kwargs):
"""
Action method.
"""
for old_key_id in list(self.keys_to_rename.keys()):
new_key_id, is_private = self.keys_to_rename[old_key_id]
if old_key_id in self.stored_keys and new_key_id not in self.stored_keys:
self.keys_to_upload.add(new_key_id)
if new_key_id in self.stored_keys and old_key_id in self.stored_keys:
self.keys_to_erase[old_key_id] = is_private
for key_id in my_keys.known_keys().keys():
if key_id not in self.stored_keys or key_id in self.not_stored_keys:
self.keys_to_upload.add(key_id)
keys_saved = []
for key_id in self.keys_to_upload:
res = key_ring.do_backup_key(key_id, wait_result=True)
keys_saved.append(res)
self.saved_count += 1
if _Debug:
lg.args(_DebugLevel, keys_saved=len(keys_saved))
wait_all_saved = DeferredList(keys_saved,
fireOnOneErrback=False,
consumeErrors=True)
wait_all_saved.addCallback(lambda ok: self.automat('backup-ok', ok))
wait_all_saved.addErrback(lambda err: self.automat('error', err))
def doRequestMyIdentity(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'id_rotator.doRequestMyIdentity')
def _cb(results):
if _Debug:
lg.args(_DebugLevel, results=results)
# TODO: validate my identity in all responses
self.automat('my-id-exist', results)
return results
def _eb(err):
lg.err(err)
self.automat('my-id-not-exist', err)
if err.type == CancelledError:
return None
return err
dl = []
for idurl in my_id.getLocalIdentity().getSources(as_originals=True):
dl.append(net_misc.getPageTwisted(idurl, timeout=10))
d = DeferredList(dl, consumeErrors=True)
d.addCallback(_cb)
d.addErrback(_eb)
def _do_read_brokers():
all_brokers_results = []
for position in positions:
one_broker_result = Deferred()
all_brokers_results.append(one_broker_result)
d = dht_records.get_message_broker(
customer_idurl=customer_idurl,
position=position,
return_details=return_details,
use_cache=use_cache,
)
d.addCallback(_do_verify, position, one_broker_result)
if _Debug:
d.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='read_customer_message_brokers._do_read_brokers')
d.addErrback(_on_error, position, one_broker_result)
join_all_brokers = DeferredList(all_brokers_results,
consumeErrors=True)
join_all_brokers.addCallback(_do_collect_results)
if _Debug:
join_all_brokers.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='read_customer_message_brokers._do_read_brokers')
join_all_brokers.addErrback(result.errback)
return None
def setup_local(myName, virtualNet, classicalNet, lNode, func):
"""
Sets up
- local classical communication server (if desired according to the configuration file)
- client connection to the local virtual node quantum backend
- client connections to all other classical communication servers
Arguments
myName name of this node (string)
virtualNet servers of the virtual nodes (dictionary of host objects)
classicalNet servers on the classical communication network (dictionary of host objects)
lNode Twisted PB root to use as local server (if applicable)
func function to run if all connections are set up
"""
# Initialize Twisted callback framework
dList = []
# If we are listed as a server node for the classical network, start this server
if myName in classicalNet.hostDict:
try:
logging.debug(
"LOCAL %s: Starting local classical communication server.",
myName)
nb = classicalNet.hostDict[myName]
nb.root = lNode
nb.factory = pb.PBServerFactory(nb.root)
reactor.listenTCP(nb.port, nb.factory)
except Exception as e:
logging.error(
"LOCAL %s: Cannot start classical communication servers.",
myName, e.strerror)
return
# Give the server some time to start up
time.sleep(1)
# Connect to the local virtual node simulating the "local" qubits
logging.debug("LOCAL %s: Connecting to local virtual node.", myName)
node = virtualNet.hostDict[myName]
factory = pb.PBClientFactory()
reactor.connectTCP(node.hostname, node.port, factory)
deferVirtual = factory.getRootObject()
dList.append(deferVirtual)
# Set up a connection to all the other nodes in the classical network
for node in classicalNet.hostDict:
nb = classicalNet.hostDict[node]
if nb.name != myName:
logging.debug("LOCAL %s: Making classical connection to %s.",
myName, nb.name)
nb.factory = pb.PBClientFactory()
reactor.connectTCP(nb.hostname, nb.port, nb.factory)
dList.append(nb.factory.getRootObject())
deferList = DeferredList(dList, consumeErrors=True)
deferList.addCallback(init_register, myName, virtualNet, classicalNet,
lNode, func)
deferList.addErrback(localError)
reactor.run()
def run(nodes):
def callback(*args, **kwargs):
print('callback', args)
l = args[0]
assert len(l) > 0
def errback(*args, **kwargs):
import traceback
traceback.print_exc()
def callback_dfl(*args):
print('callback_dfl', args)
reactor.stop()
errback_dfl = errback
time.sleep(3)
try:
list_of_deffered_set_value = []
for i in range(args.start, args.end):
j = {'key'+str(i): 'value'+str(i), }
d = dht_service.set_json_value(str(i), j, 60 * 60)
d.addBoth(callback, j)
d.addErrback(errback)
list_of_deffered_set_value.append(d)
dfl = DeferredList(list_of_deffered_set_value)
dfl.addCallback(callback_dfl)
dfl.addErrback(errback_dfl)
except:
print('ERRRORO!!')
reactor.stop()
def _storeData(self,
data,
request_hash,
confirm_cache_write,
http_history=None):
if len(data["response"]) == 0:
return self._storeDataErrback(Failure(exc_value=Exception("Response data is of length 0")), data, request_hash)
#data["content-sha1"] = sha1(data["response"]).hexdigest()
if http_history is None:
http_history = {}
if "content-sha1" not in http_history:
http_history["content-sha1"] = data["content-sha1"]
if "content-changes" not in http_history:
http_history["content-changes"] = []
if data["content-sha1"] != http_history["content-sha1"]:
http_history["content-changes"].append(str(int(self.time_offset + time.time())))
http_history["content-changes"] = http_history["content-changes"][-10:]
headers = {}
http_history["content-changes"] = filter(lambda x:len(x) > 0, http_history["content-changes"])
headers["content-changes"] = ",".join(http_history["content-changes"])
headers["content-sha1"] = data["content-sha1"]
if "cache-control" in data["headers"]:
if isinstance(data["headers"]["cache-control"], (list, tuple)):
if "no-cache" in data["headers"]["cache-control"][0]:
return data
else:
if "no-cache" in data["headers"]["cache-control"]:
return data
if "expires" in data["headers"]:
if isinstance(data["headers"]["expires"], (list, tuple)):
headers["cache-expires"] = data["headers"]["expires"][0]
else:
headers["cache-expires"] = data["headers"]["expires"]
if "etag" in data["headers"]:
if isinstance(data["headers"]["etag"], (list, tuple)):
headers["cache-etag"] = data["headers"]["etag"][0]
else:
headers["cache-etag"] = data["headers"]["etag"]
if "last-modified" in data["headers"]:
if isinstance(data["headers"]["last-modified"], (list, tuple)):
headers["cache-last-modified"] = data["headers"]["last-modified"][0]
else:
headers["cache-last-modified"] = data["headers"]["last-modified"]
if "content-type" in data["headers"]:
if isinstance(data["headers"]["content-type"], (list, tuple)):
headers["content_type"] = data["headers"]["content-type"][0]
else:
headers["content_type"] = data["headers"]["content-type"]
headers_key = 'headers:%s' % request_hash
http_key = 'http:%s' % request_hash
logger.debug("Writing data for request %s to redis." % request_hash)
deferreds = []
deferreds.append(self.redis_client.set(headers_key, compress(json.dumps(headers), 1)))
deferreds.append(self.redis_client.set(http_key, compress(json.dumps(data["response"]), 1)))
d = DeferredList(deferreds, consumeErrors=True)
if confirm_cache_write:
d.addCallback(self._storeDataCallback, data)
d.addErrback(self._storeDataErrback, data, request_hash)
return d
return data
def send_payment_request(self, readTokens, writeTokens):
"""Called by a Circuit object when it wants to actually make a payment
@param readTokens: the number of read tokens to pay for at each hop in the circuit
@type readTokens: int
@param writeTokens: the number of read tokens to pay for at each hop in the circuit
@type writeTokens: int"""
assert (readTokens + writeTokens) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells"
#make sure our setup is done:
if not self.setupDone:
#have we even started?
if not self.setupStarted:
self.send_setup_message()
self.queuedReadTokens += readTokens
self.queuedWriteTokens += writeTokens
return
#dont bother trying to send payments for circuits that are already closed
if self.circ.is_done():
return
#send the payments
deferreds = []
for paymentStream in self.paymentStreams.values():
deferreds.append(paymentStream.send_payment(readTokens, writeTokens))
paymentsDoneDeferred = DeferredList(deferreds)
paymentsDoneDeferred.addErrback(self.generic_error_handler)
addTokensDeferred = Deferred()
self.inflightReadTokens += readTokens
self.inflightWriteTokens += writeTokens
#timeout in case the payment fails. We will close the circuit in this case.
event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT, self.all_receipts_received, None, addTokensDeferred, readTokens, writeTokens, None)
paymentsDoneDeferred.addCallback(self.all_receipts_received, addTokensDeferred, readTokens, writeTokens, event)
addTokensDeferred.addCallback(self._add_tokens_callback, readTokens, writeTokens)
addTokensDeferred.addErrback(self.generic_error_handler)
def _on_connected(self, ok):
from twisted.internet.defer import DeferredList
from logs import lg
from dht import dht_service
from dht import known_nodes
from main.config import conf
lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok))
dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store)
dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request)
known_seeds = known_nodes.nodes()
dl = []
attached_layers = conf().getData('services/entangled-dht/attached-layers', default='')
if attached_layers:
lg.info('more DHT layers to be attached: %r' % attached_layers)
for layer_id in attached_layers.split(','):
if layer_id.strip():
dl.append(dht_service.open_layer(
layer_id=int(layer_id.strip()),
seed_nodes=known_seeds,
connect_now=True,
attach=True,
))
if dl:
d = DeferredList(dl)
d.addCallback(self._on_layers_attached)
d.addErrback(self._on_connect_failed)
else:
if self.starting_deferred and not self.starting_deferred.called:
self.starting_deferred.callback(True)
return ok
def send_deferred(signal=Any, sender=Anonymous, *arguments, **named):
"""Like pydispatcher.send but supports returning deferreds on signal handlers.
Returns a deferred that gets fired once all signal handlers deferreds were
fired.
If one signal handler fails, the deffered errback is called.
"""
spider = named.get('spider', None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d = maybeDeferred(robustApply,
receiver,
signal=signal,
sender=sender,
*arguments,
**named)
d.addCallback(lambda result: (receiver, result))
dfds.append(d)
d = DeferredList(dfds, consumeErrors=True, fireOnOneErrback=True)
def unwrap_firsterror(err):
failure = err.trap(FirstError)
return err.value.subFailure
d.addErrback(unwrap_firsterror)
d.addCallback(lambda out: [x[1] for x in out])
return d
def stop(services_list=[]):
"""
"""
global _StopingDeferred
global _StartingDeferred
if _Debug:
lg.args(_DebugLevel,
services_list=services_list,
starting=bool(_StartingDeferred),
stoping=bool(_StopingDeferred))
if _StopingDeferred:
lg.warn('driver.stop already called')
return _StopingDeferred
if _StartingDeferred and not _StartingDeferred.called:
d = Deferred()
d.errback(Exception('currently another service is starting'))
return d
if not services_list:
services_list.extend(reversed(boot_up_order()))
if _Debug:
lg.out(_DebugLevel,
'driver.stop with %d services' % len(services_list))
dl = []
for name in services_list:
svc = services().get(name, None)
if not svc:
raise ServiceNotFound(name)
d = Deferred()
dl.append(d)
svc.automat('stop', d)
_StopingDeferred = DeferredList(dl)
_StopingDeferred.addCallback(on_stopped_all_services)
_StopingDeferred.addErrback(on_services_failed_to_stop, services_list)
return _StopingDeferred
def test_complex2(self, runtime):
def check(ls):
if (2 == runtime.id) or (1 == runtime.id):
self.assertEquals(ls[0][1], "V1")
self.assertEquals(ls[1][1], "V1")
self.assertEquals(ls[2][1], "V1")
self.assertEquals(ls[3][1], "V2")
else:
self.assertEquals(ls[0][1], "V1")
self.assertEquals(ls[1][1], "V1")
self.assertEquals(ls[2][1], "V1")
self.assertEquals(ls[3][1], "V2")
self.assertEquals(ls[4][1], "V2")
field = self.Zp
results = []
results += runtime.broadcast(runtime.players.keys(), runtime.players.keys(), "V1")
if runtime.id in [1, 2]:
v = runtime.broadcast([1, 2], [3], "V2")
if isinstance(v, list):
results += v
else:
results.append(v)
else:
results += runtime.broadcast([1, 2], [3])
if 3 == runtime.id:
results += [runtime.broadcast([3], runtime.players.keys(), str(7))]
else:
results += [runtime.broadcast([3], runtime.players.keys())]
dls = DeferredList(results)
runtime.schedule_callback(dls, check)
dls.addErrback(runtime.error_handler)
return dls
def run(nodes):
def callback(*args, **kwargs):
print('callback', args)
d = args[0]
assert len(d) == 1
k, v = d.popitem()
assert k.replace('key', '') == v.replace('value', '')
def errback(*args, **kwargs):
import traceback
traceback.print_exc()
def callback_dfl(*args):
print('callback_dfl', args)
reactor.stop()
errback_dfl = errback
time.sleep(3)
try:
list_of_deffered_set_value = []
for i in range(args.start, args.end):
d = dht_service.get_json_value(str(i))
d.addBoth(callback)
d.addErrback(errback)
list_of_deffered_set_value.append(d)
dfl = DeferredList(list_of_deffered_set_value)
dfl.addCallback(callback_dfl)
dfl.addErrback(errback_dfl)
except Exception as exc:
print('ERRRORO!!', exc)
reactor.stop()
def render(self, request):
request.setHeader('Content-type', 'text/javascript; charset=UTF-8')
if len(request.postpath) > 0:
if request.postpath[0] == "server":
return simplejson.dumps(self.spider.getServerData())
elif request.postpath[0] == "exposed_function_details":
return simplejson.dumps(self.spider.getExposedFunctionDetails())
elif request.postpath[0] == "get":
if "uuid" in request.args:
get_deferreds = []
for uuid in request.args["uuid"]:
get_deferreds.append(self.spider.get(uuid))
d = DeferredList(get_deferreds, consumeErrors = True)
d.addCallback(self._getCallback, request.args["uuid"])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
elif request.postpath[0] == "show_reservation":
if "uuid" in request.args:
d = self.spider.showReservation(request.args["uuid"][0])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
elif request.postpath[0] == "delete_reservation":
if "uuid" in request.args:
d = self.spider.deleteReservation(request.args["uuid"][0])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
elif request.postpath[0] == "delete_function_reservations":
if "function_name" in request.args:
d = self.spider.deleteFunctionReservations(request.args["function_name"][0])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter function_name is required.")))
elif request.postpath[0] == "execute_reservation":
if "uuid" in request.args:
d = self.spider.queryByUUID(request.args["uuid"][0])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
def _do_verify_my_sources(self):
my_sources = my_id.getLocalIdentity().getSources(as_originals=True)
dl = []
for idurl_bin in my_sources:
d = net_misc.getPageTwisted(idurl_bin, timeout=5)
dl.append(d)
d = DeferredList(dl, consumeErrors=True)
d.addCallback(self._do_check_ping_results)
d.addErrback(lambda err: self.automat('ping-failed', err))
def deleteReservation( self, uuid, function_name="Unknown" ):
logger.info("Deleting reservation %s, %s." % (function_name, uuid))
deferreds = []
deferreds.append(self.sdb.delete(self.aws_sdb_reservation_domain, uuid))
deferreds.append(self.s3.deleteObject(self.aws_s3_storage_bucket, uuid))
d = DeferredList(deferreds)
d.addCallback(self._deleteReservationCallback, function_name, uuid)
d.addErrback(self._deleteReservationErrback, function_name, uuid)
return d
def create_all_or_nothing(deferred_list):
def restore_failure(failure):
return failure.value.subFailure
instance = DeferredList(deferred_list,
fireOnOneErrback=True,
consumeErrors=True)
instance.addErrback(restore_failure)
return instance
def createReservation( self, function_name, **kwargs ):
if not isinstance( function_name, str ):
for key in self.functions:
if self.functions[key]["function"] == function_name:
function_name = key
break
if function_name not in self.functions:
raise Exception("Function %s does not exist." % function_name )
function = self.functions[ function_name ]
filtered_kwargs = {}
for key in function["required_arguments"]:
if key in kwargs:
filtered_kwargs[key] = convertToUTF8( kwargs[key] )
else:
raise Exception("Required parameter '%s' not found. Required parameters are %s. Optional parameters are %s." % (key, function["required_arguments"], function["optional_arguments"] ))
for key in function["optional_arguments"]:
if key in kwargs:
filtered_kwargs[key] = convertToUTF8( kwargs[key] )
if function["interval"] > 0:
reserved_arguments = {}
reserved_arguments["reservation_function_name"] = function_name
reserved_arguments["reservation_created"] = sdb_now(offset=self.time_offset)
reserved_arguments["reservation_next_request"] = reserved_arguments["reservation_created"]
reserved_arguments["reservation_error"] = "0"
all_arguments = {}
all_arguments.update( reserved_arguments )
all_arguments.update( filtered_kwargs )
uuid = uuid4().hex
logger.debug( "Creating reservation on SimpleDB for %s, %s." % (function_name, uuid))
a = self.sdb.putAttributes( self.aws_sdb_reservation_domain, uuid, all_arguments )
a.addCallback( self._createReservationCallback, function_name, uuid )
a.addErrback( self._createReservationErrback, function_name, uuid )
if "call_immediately" in kwargs and not evaluateBoolean( kwargs["call_immediately"] ):
d = DeferredList([a], consumeErrors=True)
else:
logger.debug( "Calling %s immediately with arguments:\n%s" % (function_name, PrettyPrinter.pformat(filtered_kwargs) ) )
b = self.callExposedFunctionImmediately( function["function"], filtered_kwargs, function_name )
d = DeferredList([a,b], consumeErrors=True)
d.addCallback( self._createReservationCallback2, function_name, uuid )
d.addErrback( self._createReservationErrback2, function_name, uuid )
return d
else:
logger.debug( "Calling %s immediately with arguments:\n%s" % (function_name, PrettyPrinter.pformat(filtered_kwargs) ) )
d = self.callExposedFunctionImmediately( function["function"], filtered_kwargs, function_name )
return d
def testStoredReturn(self):
deferreds = []
for filename in self.filenames:
d = self.spider.rq.getPage("http://127.0.0.1:5000/function/load_data_stored", method="POST", postdata={"filename":filename})
d.addCallback( self._processStoredRequestCallback, filename )
deferreds.append(d)
d = DeferredList(deferreds, consumeErrors=True )
d.addCallback(self._testStoredReturnCallback)
d.addErrback( self._testStoredReturnErrback )
return d
def deleteReservation(self, uuid, function_name="Unknown"):
LOGGER.info("Deleting reservation %s, %s." % (function_name, uuid))
parameters = {'uuid': uuid}
query_string = urllib.urlencode(parameters)
url = 'http://%s:%s/function/schedulerserver/remoteremovefromheap?%s' % (self.scheduler_server, self.schedulerserver_port, query_string)
deferreds = []
deferreds.append(self.getPage(url=url))
deferreds.append(self.s3.deleteObject(self.aws_s3_storage_bucket, uuid))
d = DeferredList(deferreds)
d.addCallback(self._deleteReservationCallback, function_name, uuid)
d.addErrback(self._deleteReservationErrback, function_name, uuid)
return d
def main(myName):
# This file defines the network of virtual quantum nodes
virtualFile = os.environ.get('NETSIM') + "/config/virtualNodes.cfg"
# This file defines the network of CQC servers interfacing to virtual quantum nodes
cqcFile = os.environ.get('NETSIM') + "/config/cqcNodes.cfg"
# Read configuration files for the virtual quantum, as well as the classical network
virtualNet = networkConfig(virtualFile)
cqcNet = networkConfig(cqcFile)
# Check if we should run a local classical server. If so, initialize the code
# to handle remote connections on the classical communication network
if myName in cqcNet.hostDict:
myHost = cqcNet.hostDict[myName]
cqc_factory = CQCFactory(myHost, myName, cqcNet)
else:
logging.error(
"LOCAL %s: Cannot start classical communication servers.", myName,
e.strerror)
# Initialize Twisted callback framework
dList = []
try:
logging.debug(
"LOCAL %s: Starting local classical communication server.", myName)
myHost.root = cqc_factory
myHost.factory = cqc_factory
reactor.listenTCP(myHost.port, myHost.factory)
except Exception as e:
logging.error("LOCAL %s: Cannot start CQC server.", myName, e.strerror)
return
# Connect to the local virtual node simulating the "local" qubits
try:
logging.debug("LOCAL %s: Connecting to local virtual node.", myName)
virtual_node = virtualNet.hostDict[myName]
factory = pb.PBClientFactory()
reactor.connectTCP(virtual_node.hostname, virtual_node.port, factory)
deferVirtual = factory.getRootObject()
dList.append(deferVirtual)
deferList = DeferredList(dList, consumeErrors=True)
deferList.addCallback(init_register, myName, cqc_factory)
deferList.addErrback(localError)
reactor.run()
except Exception as e:
logging.error("LOCAL %s: Cannot connect to SimulaQron backend.",
myName)
return
def _do_populate_tags(clients):
"""Send RPC calls to each rack controller, requesting evaluation of tags.
:param clients: List of connected rack controllers that EvaluateTag
will be called.
"""
def call_client(client_info):
client = client_info["client"]
return client(
EvaluateTag,
system_id=client_info["system_id"],
tag_name=client_info["tag_name"],
tag_definition=client_info["tag_definition"],
tag_nsmap=client_info["tag_nsmap"],
credentials=client_info["credentials"],
nodes=client_info["nodes"],
)
def check_results(results):
for client_info, (success, result) in zip(clients, results):
if success:
maaslog.info(
"Tag %s (%s) evaluated on rack controller %s (%s)",
client_info["tag_name"],
client_info["tag_definition"],
client_info["hostname"],
client_info["system_id"],
)
else:
maaslog.error(
"Tag %s (%s) could not be evaluated on rack controller "
"%s (%s): %s",
client_info["tag_name"],
client_info["tag_definition"],
client_info["hostname"],
client_info["system_id"],
result.getErrorMessage(),
)
d = DeferredList(
(call_client(client_info) for client_info in clients),
consumeErrors=True,
)
d.addCallback(check_results)
d.addErrback(log.err)
# Do *not* return a Deferred; the caller is not meant to wait around for
# this to finish. However, for the sake of testing, we return the Deferred
# wrapped up in a list so that crochet does not block.
return [d]
def publish(self):
"""Publish all added charms to provider storage and zookeeper.
Returns the charm_state of all scheduled charms.
"""
publish_deferreds = []
for charm_id, charm in self._charm_add_queue:
publish_deferreds.append(self._publish_one(charm_id, charm))
publish_deferred = DeferredList(publish_deferreds,
fireOnOneErrback=1,
consumeErrors=1)
# callbacks and deferreds to unwind the dlist
publish_deferred.addCallback(self._cb_extract_charm_state)
publish_deferred.addErrback(self._eb_extract_error)
return publish_deferred
def publish(self):
"""Publish all added charms to provider storage and zookeeper.
Returns the charm_state of all scheduled charms.
"""
publish_deferreds = []
for charm_id, charm in self._charm_add_queue:
publish_deferreds.append(self._publish_one(charm_id, charm))
publish_deferred = DeferredList(publish_deferreds,
fireOnOneErrback=1,
consumeErrors=1)
# callbacks and deferreds to unwind the dlist
publish_deferred.addCallback(self._cb_extract_charm_state)
publish_deferred.addErrback(self._eb_extract_error)
return publish_deferred
def multi_set_attr(self, owner_userid, image_ids, attr_dict):
"""
Sets attributes on a list of images to be whatever is in the attr_dict.
attr_dict must have keys that are valid in images.set_attr
@param owner_username: Username
@type owner_username: String
@param media_id_list: set of media_ids to change
@type media_id_list: List
@param attr_dict: dictionary of attribute->values to apply to the list of images
@type attr_dict: Dictionary
@return: image attributes
@rtype: List
"""
owner_userid = validation.cast_integer(owner_userid, 'owner_userid')
if not isinstance(image_ids, (list, tuple)):
raise errors.ValidationError, 'media_id_list must be a list or tuple'
if not isinstance(attr_dict, dict):
raise errors.ValidationError, 'attr_dict must be a dictionary'
for x in image_ids:
self.log.debug("validating image_id [%s]" % x)
validation.cast_integer(x, 'x')
for x in attr_dict.keys():
if x not in self.attr_fields:
raise errors.ValidationError, 'attr_dict key [%s] is not valid!' % x
self.log.debug("all args look OK in multi_set_attr")
dl = []
for image_id in image_ids:
image_id = validation.cast_integer(image_id, 'image_id')
d_temp = Deferred()
self.log.debug("adding deferreds for image_id [%s]" % image_id)
for key, value in attr_dict.items():
self.log.debug("adding defferred for image_id [%s] key [%s]" % (image_id, key))
d_temp.addCallback(self.set_attr, image_id, key, value)
d_temp.addCallback(lambda _: owner_userid)
d_temp.callback(owner_userid)
dl.append(d_temp)
d_list = DeferredList(dl, fireOnOneErrback=True)
d_list.addCallback(lambda _: 0);
def handle_it(failure):
self.log.warning("failure on multi_set_attr [%s]" % failure.getErrorMessage())
d_list.addErrback(handle_it)
return d_list
def _fallback_and_ping_my_servers(self):
"""
Just ping all my id servers by sending a HTTP request to the "main page".
"""
my_sources = my_id.getLocalIdentity().getSources(as_originals=True)
id_servers = []
for idurl_bin in my_sources:
proto, host, port, _ = nameurl.UrlParse(idurl_bin)
id_servers.append(nameurl.UrlMake(proto, host, port, ''))
if _Debug:
lg.args(_DebugLevel, id_servers=id_servers)
dl = []
for url in id_servers:
d = net_misc.getPageTwisted(url, timeout=5)
dl.append(d)
d = DeferredList(dl, consumeErrors=True)
d.addCallback(lambda result: self.automat('ping-done', []))
d.addErrback(self._fallback_check_network_connected)
def test__deferredDHCPRequestErrback_cancels_all_on_FirstError(self):
mock_cancelAll = self.patch(DHCPRequestMonitor, 'cancelAll')
def raise_ioerror():
raise IOError()
a = deferLater(reactor, 0.0, raise_ioerror)
b = deferLater(reactor, 6, lambda: 'b')
monitor = DHCPRequestMonitor('lo')
monitor.deferredDHCPRequests = [a, b]
deferredList = DeferredList(monitor.deferredDHCPRequests,
consumeErrors=True,
fireOnOneErrback=True)
deferredList.addErrback(monitor.deferredDHCPRequestErrback)
yield deferredList
# Still have one call left in the reactor, since we mocked cancelAll().
b.cancel()
self.assertThat(mock_cancelAll, MockCallsMatch(call([a, b])))
def xmlrpc_del_albums(self, info, set_ids, album_ids): @stack def process_set(set_id): dl2 = [] for album_id in album_ids: dl2.append(process_album(set_id, album_id)) dList2 = DeferredList(dl2, fireOnOneErrback=True) return dList2 @stack def process_album(set_id, album_id): return self.del_album(info['userid'], set_id, album_id) dl = [] for set_id in set_ids: dl.append(process_set(set_id)) dList = DeferredList(dl, fireOnOneErrback=True) dList.addCallback(lambda _: (0, "success")) dList.addErrback(lambda _: (-1, _.getErrorMessage)) return dList
def notifyOfUpdate(self, gridKeys: List[str]):
""" Notify of Grid Updates
This method is called by the client.GridCacheController when it receives updates
from the server.
"""
self._filterOutOfflineVortexes()
payloadsByVortexUuid = defaultdict(Payload)
for gridKey in gridKeys:
gridTuple = self._cacheController.encodedChunk(gridKey)
if not gridTuple:
gridTuple = EncodedGridTuple()
gridTuple.gridKey = gridKeys
vortexUuids = self._observedVortexUuidsByGridKey.get(gridKey, [])
# Queue up the required client notifications
for vortexUuid in vortexUuids:
logger.debug("Sending unsolicited grid %s to vortex %s",
gridKey, vortexUuid)
payloadsByVortexUuid[vortexUuid].tuples.append(gridTuple)
# Send the updates to the clients
dl = []
for vortexUuid, payload in list(payloadsByVortexUuid.items()):
payload.filt = clientGridWatchUpdateFromDeviceFilt
# Serliase in thread, and then send.
d = payload.makePayloadEnvelopeDefer()
d.addCallback(
lambda payloadEnvelope: payloadEnvelope.toVortexMsgDefer())
d.addCallback(VortexFactory.sendVortexMsg,
destVortexUuid=vortexUuid)
dl.append(d)
# Log the errors, otherwise we don't care about them
dl = DeferredList(dl, fireOnOneErrback=True)
dl.addErrback(vortexLogFailure, logger, consumeError=True)
def _do_identity_cache(ret):
all_stories = []
for _supplier_idurl in ret['suppliers']:
if _supplier_idurl:
_supplier_idurl = id_url.to_bin(_supplier_idurl)
if not id_url.is_cached(
_supplier_idurl) or not identitycache.HasFile(
_supplier_idurl):
one_supplier_story = identitycache.immediatelyCaching(
_supplier_idurl)
if _Debug:
one_supplier_story.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='read_customer_suppliers._do_identity_cache'
)
all_stories.append(one_supplier_story)
_customer_idurl = id_url.to_bin(ret['customer_idurl'])
if _customer_idurl and (not id_url.is_cached(_customer_idurl)
or not identitycache.HasFile(_customer_idurl)):
one_customer_story = identitycache.immediatelyCaching(
_customer_idurl)
if _Debug:
one_customer_story.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='read_customer_suppliers._do_identity_cache')
all_stories.append(one_customer_story)
if _Debug:
lg.args(_DebugLevel, all_stories=len(all_stories), ret=ret)
id_cache_story = DeferredList(all_stories, consumeErrors=True)
id_cache_story.addCallback(_do_save_customer_suppliers, ret)
if _Debug:
id_cache_story.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='read_customer_suppliers._do_identity_cache')
id_cache_story.addErrback(result.errback)
return id_cache_story
def xmlrpc_del_albums(self, info, set_ids, album_ids):
@stack
def process_set(set_id):
dl2 = []
for album_id in album_ids:
dl2.append(process_album(set_id, album_id))
dList2 = DeferredList(dl2, fireOnOneErrback=True)
return dList2
@stack
def process_album(set_id, album_id):
return self.del_album(info['userid'], set_id, album_id)
dl = []
for set_id in set_ids:
dl.append(process_set(set_id))
dList = DeferredList(dl, fireOnOneErrback=True)
dList.addCallback(lambda _: (0, "success"))
dList.addErrback(lambda _: (-1, _.getErrorMessage))
return dList
def send_payment_request(self, readTokens, writeTokens):
"""Called by a Circuit object when it wants to actually make a payment
@param readTokens: the number of read tokens to pay for at each hop in the circuit
@type readTokens: int
@param writeTokens: the number of read tokens to pay for at each hop in the circuit
@type writeTokens: int"""
assert (
readTokens + writeTokens
) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells"
#make sure our setup is done:
if not self.setupDone:
#have we even started?
if not self.setupStarted:
self.send_setup_message()
self.queuedReadTokens += readTokens
self.queuedWriteTokens += writeTokens
return
#dont bother trying to send payments for circuits that are already closed
if self.circ.is_done():
return
#send the payments
deferreds = []
for paymentStream in self.paymentStreams.values():
deferreds.append(
paymentStream.send_payment(readTokens, writeTokens))
paymentsDoneDeferred = DeferredList(deferreds)
paymentsDoneDeferred.addErrback(self.generic_error_handler)
addTokensDeferred = Deferred()
self.inflightReadTokens += readTokens
self.inflightWriteTokens += writeTokens
#timeout in case the payment fails. We will close the circuit in this case.
event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT,
self.all_receipts_received, None,
addTokensDeferred, readTokens,
writeTokens, None)
paymentsDoneDeferred.addCallback(self.all_receipts_received,
addTokensDeferred, readTokens,
writeTokens, event)
addTokensDeferred.addCallback(self._add_tokens_callback, readTokens,
writeTokens)
addTokensDeferred.addErrback(self.generic_error_handler)
def moveFiles(self, virtSrcBase, virtDestBase, relFiles):
self.debug("MOVING: %r, %r, %r", virtSrcBase, virtDestBase, relFiles)
if not self._local:
raise TranscoderError("Component not properly setup yet")
def move_failed(failure, src, dest):
msg = ("Fail to move file '%s' to '%s': %s"
% (src, dest, log.getFailureMessage(failure)))
self.warning("%s", msg)
raise TranscoderError(msg, cause=failure)
def move_file(src, dest, attr=None):
self.debug("Moving file '%s' to '%s'", src, dest)
dest_dir = os.path.dirname(dest)
safe_mkdirs(dest_dir, "input file destination", attr)
d = deferToThread(shutil.move, src, dest)
d.addErrback(move_failed, src, dest)
return d
def move_files_failed(results):
first_failure = None
for ok, result in results:
if not ok:
if not first_failure:
first_failure = result
return first_failure
sem = DeferredSemaphore(1)
move_tasks = []
for file in relFiles:
source_path = virtSrcBase.append(file).localize(self._local)
dest_path = virtDestBase.append(file).localize(self._local)
source_path = os.path.realpath(source_path)
dest_path = os.path.realpath(dest_path)
d = sem.run(move_file, source_path, dest_path, self._pathAttr)
move_tasks.append(d)
dl = DeferredList(move_tasks, consumeErrors=True)
dl.addErrback(move_files_failed)
return d
def run():
print('run')
def callback(*args, **kwargs):
print('callback', args)
d = args[0]
if not isinstance(d, dict):
print('NOT FOUND')
return
assert isinstance(
d, dict), 'result value is not dictionary, key-value not found'
assert len(d) == 1
k, v = d.popitem()
assert k.replace('key', '') == v.replace('value', '')
def errback(*args, **kwargs):
import traceback
traceback.print_exc()
def callback_dfl(*args):
print('callback_dfl', args)
reactor.stop() # @UndefinedVariable
errback_dfl = errback
time.sleep(3)
try:
list_of_deffered_set_value = []
for i in range(options.start, options.end + 1):
d = dht_service.get_json_value(str(i), layer_id=options.layer)
d.addBoth(callback)
d.addErrback(errback)
list_of_deffered_set_value.append(d)
dfl = DeferredList(list_of_deffered_set_value)
dfl.addCallback(callback_dfl)
dfl.addErrback(errback_dfl)
except Exception as exc:
print('ERROR in run()', exc)
reactor.stop() # @UndefinedVariable
def main():
"""Main command line entry point."""
parser = argparse.ArgumentParser(
description='Make requests to one or more HTTP or HTTPS URIs, '
'and record the interactions in a cassette.',
epilog='If no URIs are passed on the command line, they are '
'read from standard input, one per line.')
parser.add_argument(
'uris', metavar='URI', nargs='*', help='URI to fetch')
parser.add_argument(
'cassette_path', metavar='CASSETTE',
help='path to output cassette')
args = parser.parse_args()
uris = args.uris or imap(lambda x: x.strip(), sys.stdin)
cassette_agent = CassetteAgent(Agent(reactor), args.cassette_path)
agent = ContentDecoderAgent(
RedirectAgent(cassette_agent), [('gzip', GzipDecoder)])
finished = DeferredList([agent.request('GET', uri) for uri in uris])
finished.addCallback(save_and_exit, cassette_agent)
finished.addErrback(fail_and_exit)
reactor.run()
def oneBatch(prev, ipList):
done = 0
deferreds = []
for toDo in ipList:
deferreds.append(makeRequest(toDo))
done += 1
if done == concurrentRequests: break
else:
print "\n%d requests ended in empty match" % prev
d = getPage("%s/stats/avg" % (baseUrl, ))
@d.addCallback
def c2(time):
print "Average matching time in server: %s" % time
reactor.stop()
return
d = DeferredList(deferreds)
@d.addCallback
def c(results):
return prev + len([x for x in results if x[1] in ("[]", "ok\n")])
d.addErrback(log.err)
d.addBoth(oneBatch, ipList)
def run():
print('run')
def callback(*args, **kwargs):
print('callback', args, kwargs)
l = args[0]
assert len(l) > 0
def errback(*args, **kwargs):
import traceback
traceback.print_exc()
def callback_dfl(*args):
print('callback_dfl', args)
reactor.stop() # @UndefinedVariable
errback_dfl = errback
time.sleep(3)
try:
list_of_deffered_set_value = []
for i in range(options.start, options.end + 1):
j = {
'key' + str(i): 'value' + str(i),
}
d = dht_service.set_json_value(str(i),
json_data=j,
age=60 * 60,
layer_id=options.layer)
d.addBoth(callback, j)
d.addErrback(errback)
list_of_deffered_set_value.append(d)
dfl = DeferredList(list_of_deffered_set_value)
dfl.addCallback(callback_dfl)
dfl.addErrback(errback_dfl)
except Exception as exc:
print('ERROR in run()', exc)
reactor.stop() # @UndefinedVariable
def _on_connected(self, ok):
from twisted.internet.defer import DeferredList
from logs import lg
from dht import dht_service
from dht import known_nodes
from main.config import conf
from services import driver
lg.info('DHT node connected ID0=[%s] : %r' % (dht_service.node().layers[0], ok))
dht_service.node().add_rpc_callback('store', self._on_dht_rpc_store)
dht_service.node().add_rpc_callback('request', self._on_dht_rpc_request)
known_seeds = known_nodes.nodes()
dl = []
attached_layers = conf().getData('services/entangled-dht/attached-layers', default='')
if attached_layers:
attached_layers = list(filter(None, map(lambda v: int(str(v).strip()), attached_layers.split(','))))
else:
attached_layers = []
lg.info('reading attached DHT layers from configuration: %r' % attached_layers)
all_services_attached_layers = driver.get_attached_dht_layers().values()
combined_services_attached_layers = set()
count_combined = len(list(map(combined_services_attached_layers.update, all_services_attached_layers)))
services_attached_layers = list(combined_services_attached_layers)
lg.info('combined attached DHT layers from %d services: %r' % (count_combined, services_attached_layers, ))
attached_layers = list(set(attached_layers + services_attached_layers))
lg.info('DHT layers to be attached at startup: %r' % attached_layers)
for layer_id in attached_layers:
dl.append(dht_service.open_layer(
layer_id=layer_id,
seed_nodes=known_seeds,
connect_now=True,
attach=True,
))
if dl:
d = DeferredList(dl)
d.addCallback(self._on_layers_attached)
d.addErrback(self._on_connect_failed)
else:
if self.starting_deferred and not self.starting_deferred.called:
self.starting_deferred.callback(True)
return ok
def render(self, request):
request.setHeader('Content-type', 'text/javascript; charset=UTF-8')
if len(request.postpath) > 0:
if request.postpath[0] == "show_reservation":
if "uuid" in request.args:
deferreds = []
for uuid in request.args["uuid"]:
deferreds.append(self.interfaceserver.showReservation(uuid))
d = DeferredList(deferreds, consumeErrors=True)
d.addCallback(self._showReservationCallback, request.args["uuid"])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
elif request.postpath[0] == "execute_reservation":
if "uuid" in request.args:
d = self.interfaceserver.executeReservation(request.args["uuid"][0])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
elif request.postpath[0] == "delete_reservation":
if "uuid" in request.args:
d = self.interfaceserver.deleteReservation(request.args["uuid"][0])
d.addCallback(self._successResponse)
d.addErrback(self._errorResponse)
d.addCallback(self._immediateResponse, request)
return server.NOT_DONE_YET
else:
return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
def receive_item(self, item, queue, auto_ack, incoming_callbacks):
"""
This function is called with EVERY incoming message. We don't have logic here, just send to factory.
"""
(channel, deliver, properties, message) = item
queue_deferred = queue.get() # get the queue again, so we can add another callback to get the next message.
queue_deferred.addCallback(self.receive_item, queue, auto_ack, incoming_callbacks)
queue_deferred.addErrback(self.receive_item_err) # todo: use supplied error callback... which one?
received_callbacks = []
for callback in incoming_callbacks:
received_callbacks.append(maybeDeferred(callback,
channel=channel,
deliver=deliver,
properties=properties,
message=message)
)
if auto_ack is False:
received = DeferredList(received_callbacks, fireOnOneErrback=True, consumeErrors=True)
received.addCallback(self._basic_ack, channel, deliver.delivery_tag)
received.addErrback(self._basic_nack, channel, deliver.delivery_tag)
def _configure_ssh(self, deployment):
"""
:return: A ``Deferred`` which fires when all nodes have been configured
with ssh keys.
"""
self.ssh_configuration.create_keypair()
results = []
for node in deployment.nodes:
results.append(
deferToThread(self.ssh_configuration.configure_ssh,
node.hostname, self.ssh_port))
d = DeferredList(results, fireOnOneErrback=True, consumeErrors=True)
# Exit with ssh's output if it failed for some reason:
def got_failure(failure):
if failure.value.subFailure.check(CalledProcessError):
raise SystemExit(b"Error connecting to cluster node: " +
failure.value.subFailure.value.output)
else:
return failure
d.addErrback(got_failure)
return d
def _process_incoming_email(self, otherself, filepath, mask):
"""
Callback that processes incoming email
@param otherself: Watch object for the current callback from
inotify
@type otherself: twisted.internet.inotify._Watch
@param filepath: Path of the file that changed
@type filepath: twisted.python.filepath.FilePath
@param mask: identifier for the type of change that triggered
this callback
@type mask: int
"""
if os.path.split(filepath.dirname())[-1] == "new":
log.msg("Processing new mail at %s" % (filepath.path,))
with filepath.open("r") as f:
mail_data = f.read()
mail = message_from_string(mail_data)
owner = mail["To"]
if owner is None: # default to Delivered-To
owner = mail["Delivered-To"]
if owner is None:
log.err("Malformed mail, neither To: nor "
"Delivered-To: field")
log.msg("Mail owner: %s" % (owner,))
log.msg("%s received a new mail" % (owner,))
dpubk = self._users_cdb.getPubKey(owner)
duuid = self._users_cdb.queryByAddress(owner)
d = DeferredList([dpubk, duuid])
d.addCallbacks(self._gather_uuid_pubkey, log.err)
d.addCallbacks(self._encrypt_message, log.err,
(owner, mail_data))
d.addCallbacks(self._export_message, log.err)
d.addCallbacks(self._conditional_remove, log.err,
(filepath,))
d.addErrback(log.err)
def __init__(self, endpointA, endpointB):
""" Initialize the connection between the two endpoints.
The connection will be scheduled to be created here.
@param endpointX: Endpoint which is part of the new connection.
@type endpointX: rce.core.network.Endpoint
"""
assert endpointA != endpointB
# Register the endpoints and create Protocol proxies
self._serverEndpoint = endpointA
self._clientEndpoint = endpointB
self._serverProtocol = Protocol(endpointA)
self._clientProtocol = Protocol(endpointB)
serverStatus = Status(self._serverProtocol)
clientStatus = Status(self._clientProtocol)
# Create the keys used to validate the connection
connectionID = uuid4().bytes
serverKey = uuid4().bytes
clientKey = uuid4().bytes
# Create Validators and get result Deferred
authServer = _ConnectionValidator(serverKey)
authClient = _ConnectionValidator(clientKey)
authServerResult = authServer.result
authClientResult = authClient.result
authenticator = DeferredList([authServerResult, authClientResult])
authenticator.addCallback(self._validate)
authenticator.addErrback(self._logError)
readyServer = endpointA.prepareConnection(connectionID, serverKey,
authClient, serverStatus)
readyClient = endpointB.prepareConnection(connectionID, clientKey,
authServer, clientStatus)
ready = DeferredList([readyServer, readyClient])
ready.addCallback(self._getAddress)
ready.addCallback(self._connect, connectionID)
ready.addErrback(self._connectPrepError, authenticator)
ready.addErrback(self._logError)
def __init__(self, endpointA, endpointB):
""" Initialize the connection between the two endpoints.
The connection will be scheduled to be created here.
@param endpointX: Endpoint which is part of the new connection.
@type endpointX: rce.core.network.Endpoint
"""
assert endpointA != endpointB
# Register the endpoints and create Protocol proxies
self._serverEndpoint = endpointA
self._clientEndpoint = endpointB
self._serverProtocol = Protocol(endpointA)
self._clientProtocol = Protocol(endpointB)
# Create the keys used to validate the connection
connectionID = uuid4().bytes
serverKey = uuid4().bytes
clientKey = uuid4().bytes
# Create Validators and get result Deferred
authServer = _ConnectionValidator(serverKey)
authClient = _ConnectionValidator(clientKey)
authServerResult = authServer.result
authClientResult = authClient.result
authenticator = DeferredList([authServerResult, authClientResult])
authenticator.addCallback(self._validate)
authenticator.addErrback(self._logError)
readyServer = endpointA.prepareConnection(connectionID, serverKey,
authClient)
readyClient = endpointB.prepareConnection(connectionID, clientKey,
authServer)
ready = DeferredList([readyServer, readyClient])
ready.addCallback(self._getAddress)
ready.addCallback(self._connect, connectionID)
ready.addErrback(self._connectPrepError, authenticator)
ready.addErrback(self._logError)
def gather_results(deferreds, consume_errors=True):
d = DeferredList(deferreds, fireOnOneErrback=1,
consumeErrors=consume_errors)
d.addCallback(lambda r: [x[1] for x in r])
d.addErrback(lambda f: f.value.subFailure)
return d
media_id = result[1]["media_id"]
owner_username = result[1]["owner_username"]
d3 = self._locate_media(media_id)
d3.addCallback(delete_image, media_id, owner_username)
return d3
dl = []
for id in id_list:
d = self.app.api.images.get_media_owner_id(id)
d.addCallback(handle_media_info)
dl.append(d)
d_list = DeferredList(dl)
d_list.addCallback(lambda _: (0, _))
d_list.addErrback(lambda _: (-1, _.getErrorMessage()))
return d_list
@stack
def get_render(self, image_id, width, height, crop):
"""
Gets a render for an image.
@param owner_username: User who owns the image
@type owner_username: String
@param media_id: ID of the image
@type media_id: String
@param width: Width of the render
@type width: Integer
class DHCPRequestMonitor:
def __init__(self, ifname: str, clock: IReactorThreads = None):
if clock is None:
clock = reactor
self.clock = clock # type: IReactorThreads
self.ifname = ifname # type: str
self.servers = None # type: set
self.dhcpRequestsDeferredList = None # type: DeferredList
self.deferredDHCPRequests = [] # type: List[Deferred]
self.transaction_id = make_dhcp_transaction_id() # type: bytes
def send_requests_and_await_replies(self):
"""Sends out DHCP requests and waits for their replies.
This method is intended to run under `deferToThread()`.
Calls the reactor using `blockingCallFromThread` to queue the request
packets.
Blocks for ~10 seconds while checking for DHCP offers.
:returns: `set` of `DHCPServer` objects.
"""
# Since deferToThread() might be delayed until the next thread is
# available, it's better to kick off the DHCP requests from the
# spawned thread rather than hoping the thread starts running after
# we kick off the requests.
blockingCallFromThread(self.clock, self.deferDHCPRequests)
servers = set()
# Convert the transaction_id to an integer so we can test it against
# what the parsed DHCP packet will return.
xid = int.from_bytes(self.transaction_id, byteorder="big")
with udp_socket() as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Note: the empty string is the equivalent of INADDR_ANY.
sock.bind(("", BOOTP_CLIENT_PORT))
# The timeout has to be relatively small, since we wake up every
# timeout interval to check the elapsed time.
sock.settimeout(0.5)
runtime = 0
# Use a monotonic clock to ensure leaping backward in time won't
# cause an infinite loop.
start_time = time.monotonic()
while runtime < REPLY_TIMEOUT:
try:
# Use recvfrom() to check the source IP for the request.
# It could be interesting in cases where DHCP relay is in
# use.
data, (address, port) = sock.recvfrom(2048)
except socket.timeout:
continue
else:
offer = DHCP(data)
if not offer.is_valid():
log.info(
"Invalid DHCP response received from {address} "
"on '{ifname}': {reason}",
address=address,
ifname=self.ifname,
reason=offer.invalid_reason,
)
elif offer.packet.xid == xid:
# Offer matches our transaction ID, so check if it has
# a Server Identifier option.
server = offer.server_identifier
if server is not None:
servers.add(DHCPServer(server, address))
finally:
runtime = time.monotonic() - start_time
return servers
@staticmethod
def cancelAll(deferreds: List[Deferred]):
for deferred in deferreds:
deferred.cancel()
def deferredDHCPRequestErrback(self,
failure: Failure) -> Optional[Failure]:
if failure.check(FirstError):
# If an error occurred, cancel any other pending requests.
# (The error is likely to occur for those requests, too.)
# Unfortunately we can't cancel using the DeferredList, since
# the DeferredList considers itself "called" the moment the first
# errback is invoked.
self.cancelAll(self.deferredDHCPRequests)
# Suppress further error handling. The original Deferred's errback
# has already been called.
return None
elif failure.check(DHCPProbeException):
log.msg("DHCP probe failed. %s" % failure.getErrorMessage())
elif failure.check(CancelledError):
# Intentionally cancelled; no need to spam the log.
pass
else:
log.err(
failure,
"DHCP probe on '%s' failed with an unknown error." %
(self.ifname),
)
# Make sure the error is propagated to the DeferredList.
# We need this so that the DeferredList knows to call us with
# FirstError, which is our indicator to cancel the remaining calls.
# (It's set to consumeErrors, so it won't spam the log.)
return failure
def deferDHCPRequests(self) -> None:
"""Queues some DHCP requests to fire off later.
Delays calls slightly so we have a chance to open a listen socket.
Uses the `clock`, `ifname`, and `transaction_id` ivars.
Stores a `DeferredList` of periodic DHCP requests calls in the
`dhcpRequestsDeferredList` ivar, and the list of `Deferred` objects in
the `deferredDHCPRequests` ivar.
"""
self.deferredDHCPRequests = []
for seconds in DHCP_REQUEST_TIMING:
packet = DHCPDiscoverPacket(transaction_id=self.transaction_id,
seconds=seconds)
# Wait 0.1 seconds before sending the request, so we have a chance
# to open a listen socket.
seconds += 0.1
deferred = deferLater(
self.clock,
seconds,
send_dhcp_request_packet,
packet,
self.ifname,
)
deferred.addErrback(self.deferredDHCPRequestErrback)
self.deferredDHCPRequests.append(deferred)
# Use fireOnOneErrback so that we know to cancel the remaining attempts
# to send requests if one of them fails.
self.dhcpRequestsDeferredList = DeferredList(
self.deferredDHCPRequests,
fireOnOneErrback=True,
consumeErrors=True,
)
self.dhcpRequestsDeferredList.addErrback(
self.deferredDHCPRequestErrback)
@inlineCallbacks
def run(self) -> Deferred:
"""Queues DHCP requests to be sent, then waits (in a separate thread)
for replies.
Requests will be sent using an exponential back-off algorithm, to mimic
a real DHCP client. But we'll just pretend we didn't see any of the
replies, and hope for more servers to respond.
The set of `DHCPServer`s that responded to the request(s) is stored
in the `servers` ivar, which in turn makes the `dhcp_servers` and
`dhcp_addresses` properties useful.
"""
servers = yield deferToThread(self.send_requests_and_await_replies)
if len(servers) > 0:
log.info(
"External DHCP server(s) discovered on interface '{ifname}': "
"{servers}",
ifname=self.ifname,
servers=", ".join(
str(server) for server in sorted(list(servers))),
)
self.servers = servers
@property
def dhcp_servers(self):
return set(str(server.server) for server in self.servers)
@property
def dhcp_addresses(self):
return set(str(server.address) for server in self.servers)
def connectTCP(self):
c = [MultiClientProxy(self.servers,urlencode=True,debug=True) for i in range(self.count)]
dl = [client.connectTCP() for client in c]
dl = DeferredList(dl)
dl.addCallback(self._connected, c)
return dl.addErrback(self.err_back)
def send_direct_tor_message(self, msg, msgType, forward=True, numHops=1, sendOverCircuit=False):
"""Tunnel a message through Tor. There are two ways to send data:
sendOverCircuit=True: These messages are visible (plain-text) to the hops
that they pass through! Callers are responsible for any necessary secrecy
and intergrity.
sendOverCircuit=False: These messages are encrypted like normal, relayed
Tor cells. They are thus encrypted and authenticated, but messages may not
be sent between two relays (only between the origin and relays)
In either case, messages that are too long will be sent in multiple cells.
@param msg: the message to send
@type msg: str
@param msgType: the type of message. Must be one of MESSAGE_CODES.
@type msgType: str
@param forward: whether to send towards the exit (True) or towards the origin (False)
@type forward: bool
@param numHops: how many relays to traverse before the message is delivered.
MUST NOT BE 0--in that case, call the handler directly yourself.
@type numHops: int
@param sendOverCircuit: whether to send over the circuit (True) or simply send over OR connections to adjacent hops (False)
@type sendOverCircuit: bool"""
if not self.torApp.is_ready():
raise TorCtl.TorCtlClosed
if self.isClosed:
log_msg("Cannot send Tor message, circuit was closed (%s)" % (msgType))
return
# if numHops is 0, you should handle the message yourself, not send it
assert numHops != 0, "cannot send a zero hop message"
msg = Basic.write_byte(MESSAGE_CODES[msgType]) + msg
# put the length in front of the message:
msgLen = len(msg)
msg = Basic.write_short(msgLen) + msg
# who to send it to:
nextHexId = self.nextHexId
nextCircId = self.nextCircId
if not forward:
nextCircId = self.prevCircId
nextHexId = self.prevHexId
dList = []
# different message lengths depending on if sendOverCircuit if True or False:
if sendOverCircuit:
# are sent as normal relayed messages, so they should be this long
WRITABLE_BYTES = 498
else:
# since the Tor cell is 512 bytes, but we need 2 for circid, and 1 for the cell command
WRITABLE_BYTES = 507
while len(msg) > 0:
dataToSend = msg[:WRITABLE_BYTES]
msg = msg[WRITABLE_BYTES:]
def add_padding(tmp, desiredLen):
extraChars = desiredLen - len(tmp)
return tmp + (" " * extraChars)
dataToSend = add_padding(dataToSend, WRITABLE_BYTES)
dataToSend = dataToSend.encode("base64")
dataToSend = dataToSend.replace("\n", "").replace("=", "")
# convert sendOverCircuit to "1" or "0" for the control connection:
if sendOverCircuit:
sendOverCircuitToken = "1"
else:
sendOverCircuitToken = "0"
dataToSend = "SENDPAYMENT %s %s %s %s %s\r\n" % (
nextHexId,
nextCircId,
dataToSend,
numHops,
sendOverCircuitToken,
)
d = self.torApp.conn.sendAndRecv(dataToSend)
dList.append(d)
d = DeferredList(dList)
def response(result):
for x in result:
if not x[0]:
raise Exception(str(x))
if x[1][0][0] != "250":
raise Exception(str(result))
read, write = x[1][0][1].split(" ")
read = int(read)
write = int(write)
return (read, write)
d.addCallback(response)
def error(failure):
# this happens occasionally when the circuit is closed at approximately the same time that we send a payment
# it can be safely ignored because the circuit is closed and we already learned about it
if "552 Cannot find circuit with id" in str(failure):
log_msg("A circuit that we tried to send a payment message to was closed. Oops.", 4)
self.close()
return
# otherwise, log an error because this is unexpected
log_ex(failure, "SENDPAYMENT failed for circuit=%s" % (nextCircId), [TorCtl.ErrorReply])
d.addErrback(error)
return d
