def connect(self, host=None, port=None, spec=None, user=None, password=None, vhost=None,
heartbeat=None, clientClass=None):
host = host or self.host
port = port or self.port
spec = spec or self.spec
user = user or self.user
password = password or self.password
vhost = vhost or self.vhost
heartbeat = heartbeat or self.heartbeat
clientClass = clientClass or self.clientClass
delegate = TwistedDelegate()
onConn = Deferred()
p = clientClass(delegate, vhost, txamqp.spec.load(spec), heartbeat=heartbeat)
f = protocol._InstanceFactory(reactor, p, onConn)
c = reactor.connectTCP(host, port, f)
def errb(thefailure):
thefailure.trap(error.ConnectionRefusedError)
print "failed to connect to host: %s, port: %s; These tests are designed to run against a running instance" \
" of the %s AMQP broker on the given host and port. failure: %r" % (host, port, self.broker, thefailure,)
thefailure.raiseException()
onConn.addErrback(errb)
self.connectors.append(c)
client = yield onConn
yield client.authenticate(user, password)
returnValue(client)
def test_http(self):
"""
When connecting to a HTTP server, a PB connection times
out.
"""
result = Deferred()
site = Site(Data("", "text/plain"))
client = HangCheckFactory(
PBClientFactory(), lambda: result.callback(None))
self.patch(HangCheckProtocol, '_HUNG_CONNECTION_TIMEOUT', 0.1)
connected_server_and_client(
self, site, client,
)
timer = reactor.callLater(2, result.cancel)
result.addCallback(lambda _: timer.cancel())
def check_for_timeout(reason):
reason.trap(CancelledError)
raise Exception("Didn't not hangup")
result.addErrback(check_for_timeout)
return result
def test_upload_with_throughput_counter_and_error(self):
client = FakeS3Client()
parts_generator = DummyPartsGenerator()
part_handler = ErroringPartHandler(100)
counter = PartsTransferredCounter('?')
stats = SlidingStats(self.clock.seconds(), size=100)
throughput_counter = ThroughputCounter(clock=self.clock, stats=stats)
d = Deferred()
amz_headers = {'acl': 'public-read'}
upload = MultipartUpload(client, None, parts_generator, part_handler,
counter, d, self.log)
upload.retry_strategy.clock = self.clock
upload.throughput_counter = throughput_counter
received = []
upload.on_part_generated = received.append
def eb(why):
self.flushLoggedErrors()
self.assertEquals(len(self.clock.calls), 110)
self.assertEquals(throughput_counter.read()[-1], (0, 0))
return why
upload.upload('mybucket', 'mykey', '', {}, amz_headers)
d.addErrback(eb)
return self.assertFailure(d, ValueError)
def get_response(cmd_data):
for cmd in cmd_data :
def __command_done(result, command) :
res = result
if not isinstance(result, CommandResult) :
res = CommandResult(status = result)
self.factory.next_state(ProtocolState.Report, ReportData(command, res), self.transport.connector)
ok_func = partial(__command_done, command = cmd)
def __command_error(reason, command):
res = CommandResult('Failed', str(reason))
self.factory.next_state(ProtocolState.Report, ReportData(command, res), self.transport.connector)
err_func = partial(__command_error, command = cmd)
# Obtain only new commands next time
if self.factory.timestamp is not None :
self.factory.timestamp = max(self.factory.timestamp, self._parse_date(cmd['timestamp']))
else :
self.factory.timestamp = self._parse_date(cmd['timestamp'])
# DeviceDelegate has to use this deferred object to notify us that command processing finished.
cmd_defer = Deferred()
cmd_defer.addCallback(ok_func)
cmd_defer.addErrback(err_func)
# Actual run of command
try :
self.factory.device_delegate.do_command(cmd, cmd_defer)
except Exception, err :
log.err('Failed to execute device-delegate do_command. Reason: <{0}>.'.format(err))
err_func(err)
def testTriggerSystemEvent3(self):
# make sure reactor can survive the loss of an event type while
# waiting for a before-trigger's Deferred to fire
l = []
d = Deferred()
d2 = Deferred()
def _returnDeferred(d=d):
return d
def _appendToList(l=l):
l.append(1)
def _ignore(failure):
return None
r = reactor
b1 = self.addTrigger("before", "defer3", _returnDeferred)
b2 = self.addTrigger("after", "defer3", _appendToList)
r.fireSystemEvent("defer3")
self.assertEquals(len(l), 0, "Event should not have fired yet.")
self.removeTrigger(b1)
self.removeTrigger(b2)
try:
d.callback(None) # cReactor gives errback to deferred
except ValueError:
pass
self.assertEquals(len(l), 0)
d.addErrback(_ignore)
def test_addActionFinishFailure(self):
"""
When the L{Deferred} referred to in L{DeferredContext.addActionFinish}
fires with an exception, a finish message is logged.
"""
d = Deferred()
logger = MemoryLogger()
action = Action(logger, "uuid", TaskLevel(level=[1]), "sys:me")
with action.context():
DeferredContext(d).addActionFinish()
exception = RuntimeError("because")
d.errback(exception)
assertContainsFields(
self,
logger.messages[0],
{
"task_uuid": "uuid",
"task_level": [1, 1],
"action_type": "sys:me",
"action_status": "failed",
"reason": "because",
"exception": "%s.RuntimeError" % (RuntimeError.__module__,),
},
)
d.addErrback(lambda _: None) # don't let Failure go to Twisted logs
def callRemote(self, _name, *args, **kw):
""" Make a call to the RemoteReference and return the result as a
Deferred. It exists to allow queuing of calls to remote reference
before the remote reference has arrived.
For more information refer to twisted.spread.pb.RemoteReference.
@param _name: Name of the method which should be called.
The prefix 'remote_' will be added to the name
in the remote object to select the method which
should be called.
@type _name: str
@param *args: Positional arguments which will be passed to
the remote method.
@param **kw: Keyworded arguments which will be passed to the
remote method.
@return: Deferred which will fire with the result of the
call or a Failure if there was a problem.
@rtype: twisted.internet.defer.Deferred
"""
if self.__failure is not None:
d = fail(self.__failure)
elif self.__pending is not None:
d = Deferred()
self.__pending.append(d)
else:
d = succeed(self.__obj)
d.addCallback(lambda ref: ref.callRemote(_name, *args, **kw))
d.addErrback(self.__filter, _name)
return d
def execute_config(self, log, transaction_id, scaling_group, launch_config):
"""
see :meth:`ISupervisor.execute_config`
"""
job_id = generate_job_id(scaling_group.uuid)
completion_d = Deferred()
log = log.bind(job_id=job_id,
worker=launch_config['type'],
tenant_id=scaling_group.tenant_id)
assert launch_config['type'] == 'launch_server'
undo = InMemoryUndoStack(self.coiterate)
def when_fails(result):
log.msg("Encountered an error, rewinding {worker!r} job undo stack.",
exc=result.value)
ud = undo.rewind()
ud.addCallback(lambda _: result)
return ud
completion_d.addErrback(when_fails)
log.msg("Authenticating for tenant")
d = self.auth_function(scaling_group.tenant_id, log=log)
def when_authenticated((auth_token, service_catalog)):
log.msg("Executing launch config.")
return launch_server_v1.launch_server(
log,
self.region,
scaling_group,
service_catalog,
auth_token,
launch_config['args'], undo)
d.addCallback(when_authenticated)
def when_launch_server_completed(result):
# XXX: Something should be done with this data. Currently only enough
# to pass to the controller to store in the active state is returned
server_details, lb_info = result
log.msg("Done executing launch config.",
server_id=server_details['server']['id'])
return {
'id': server_details['server']['id'],
'links': server_details['server']['links'],
'name': server_details['server']['name'],
'lb_info': lb_info
}
d.addCallback(when_launch_server_completed)
self.deferred_pool.add(d)
d.chainDeferred(completion_d)
return succeed((job_id, completion_d))
def put(self, uaid=""):
"""HTTP PUT
Update router type/data for a UAID.
"""
self.start_time = time.time()
if not self._validate_auth(uaid):
return self._write_response(
401, 109, message="Invalid Authentication")
params = self._load_params()
self.uaid = uaid
router_type = params.get("type")
router_data = params.get("data")
if router_type not in self.ap_settings.routers or not router_data:
log.msg("Invalid parameters", **self._client_info())
return self._write_response(
400, 108, message="Invalid arguments")
self.add_header("Content-Type", "application/json")
router = self.ap_settings.routers[router_type]
d = Deferred()
d.addCallback(router.register, router_data)
d.addCallback(self._save_router_data, router_type)
d.addCallback(self._success)
d.addErrback(self._router_fail_err)
d.addErrback(self._response_err)
d.callback(uaid)
def url_upload_data_async(url, files={}, data={}, progressCallback=None, doneCallback=None):
def produce_finished(data):
print 'produce finished',data
def produce_error(error):
print 'produce error', error
producerDeferred = Deferred()
producerDeferred.addCallback(produce_finished)
producerDeferred.addErrback(produce_error)
def receive_finished(data):
print 'recieve finished', data
doneCallback(data)
def receive_error(error):
Logger.error('Problem uploading file')
print 'recieve error', error
receiverDeferred = Deferred()
receiverDeferred.addCallback(receive_finished)
receiverDeferred.addErrback(receive_error)
producer = MultiPartProducer(files, data, progressCallback, producerDeferred)
receiver = StringReceiver(receiverDeferred)
agent = Agent(reactor)
headers = Headers({'User-Agent': ['Twisted Web Client Example']})
headers.addRawHeader("Content-Type", "multipart/form-data; boundary=%s" % producer.boundary)
requestDeffered = agent.request('POST', url, headers, producer)
requestDeffered.addCallback(lambda response: response.deliverBody(receiver))
def put(self, router_type="", router_token="", uaid="", chid=""):
"""HTTP PUT
Update router type/data for a UAID.
"""
self.start_time = time.time()
if not self._validate_auth(uaid):
return self._write_response(
401, 109, message="Invalid Authentication")
params = self._load_params()
self.uaid = uaid
router_data = params
if router_type not in self.ap_settings.routers or not router_data:
log.msg("Invalid parameters", **self._client_info())
return self._write_response(
400, 108, message="Invalid arguments")
router = self.ap_settings.routers[router_type]
valid, router_token = router.check_token(router_token)
if not valid:
newUrl = self._relocate(router_type, router_token, uaid, chid)
return self._write_response(
301, 0, "Location: %s" % newUrl,
headers={"Location": newUrl})
self.add_header("Content-Type", "application/json")
d = Deferred()
d.addCallback(router.register, router_data)
d.addCallback(self._save_router_data, router_type)
d.addCallback(self._success)
d.addErrback(self._router_fail_err)
d.addErrback(self._response_err)
d.callback(uaid)
def stopRemoteControlSession(self, authToken, sourceLogonId, targetLogonId):
logger.debug("stopRemoteControlSession(auth=%s, sessionId=%s targetSessionId=%s)" % (authToken, sourceLogonId, targetLogonId))
session = self.server.retrieveSession(sourceLogonId)
if not session:
logger.error("source session %s not present here" % sourceLogonId)
return False
targetSession = self.server.retrieveSession(targetLogonId)
if not targetSession:
logger.error("target session %s not present here" % targetLogonId)
return False
icpFactory = self.server.icpFactory
req = FdsApiStopRemoteControlRequest()
req.connectionId = session.connectionId
d = Deferred()
def onError(err):
logger.error('an error occured')
return ttypes.TException('Internal error in server')
def onSuccess(response):
logger.debug('response=%s' % response)
return response.success
d.addErrback(onError)
d.addCallback(onSuccess)
icpFactory.doQuery(FdsApiStopRemoteControl, req, Thrift2IcpHandler(self, d, FdsApiStopRemoteControlResponse))
return d
def __call__(self, sender, recipients, msg):
print "Hello!"
deferred = Deferred()
msg['From'] = sender
msg['To'] = ', '.join(recipients)
factory = smtp.ESMTPSenderFactory(
self.config.username,
self.config.password,
sender,
recipients,
cStringIO.StringIO(msg.as_string()),
deferred,
contextFactory=self.context,
requireTransportSecurity=False,
retries=0,
)
deferred.addErrback(self._on_error)
if self.config.use_ssl:
reactor.connectSSL(
self.config.host,
self.config.port,
factory,
self.context
)
else:
reactor.connectTCP(
self.config.host,
self.config.port,
factory
)
def start(self):
# We need to run 4 commands in a row, and each of them can fail
d = Deferred()
def chain(args, description, failMessage):
d.addCallback(self._spawnProcess, args, description, failMessage)
for serviceName in ['manager', 'worker']:
chain(["flumotion",
"-C", self._confDir,
"-L", self._logDir,
"-R", self._runDir,
"create", serviceName,
"admin", str(self._port)],
_('Creating %s ...') % serviceName,
_("Could not create %s." % serviceName))
chain(["flumotion",
"-C", self._confDir,
"-L", self._logDir,
"-R", self._runDir,
"start", serviceName, "admin"],
_('Starting %s ...' % serviceName),
_("Could not start %s." % serviceName))
d.addErrback(lambda f: None)
def done(result):
self._finished()
d.addCallback(done)
# start chain
d.callback(None)
return d
def testExchange(self):
def success(result):
offset = 2.063
offsetToReturn = float(offset) / 1000
liAlarm = False
offsetResult = 0
syncSource = True
self.assertEqual(result["offset"], offsetToReturn)
self.assertEqual(result["offsetResult"], offsetResult)
self.assertEqual(result["liAlarm"], liAlarm)
self.assertEqual(result["syncSource"], syncSource)
def failure(err):
self.fail(err.getErrorMessage())
timeout = 60.0
self.protocol = NtpProtocol(host="127.0.0.1", timeout=timeout)
self.protocol.transport = TestableDatagramTransport()
d = Deferred()
d.addCallback(success)
d.addErrback(failure)
self.protocol.d = d
# normally invoked by reactor.listenUDP method
self.protocol.startProtocol()
readstat = '\x16\x81\x00\x01\x06\x18\x00\x00\x00\x00\x00\x04g\xf3\x96Z'
self.protocol.datagramReceived(data=readstat, addr=None)
readvar = '\x16\x82\x00\x02\x96Zg\xf3\x00\x00\x00\x0eoffset=2.063\r\n\x00\x00'
self.protocol.datagramReceived(data=readvar, addr=None)
return d
def wrap_back(self, msg):
'''
wrap message, to support back messages
we get some info from message headers (tid and route_back key)
and send reply with this data
'''
d = Deferred()
def _push_message(reply):
if reply == NO_REPLY:
d1 = Deferred()
d1.callback(True)
if not self.no_ack:
self.client.basic_ack(msg)
return d1
route = msg.content['headers'].get(self.rb_name)
tid = msg.content['headers'].get(self.tid_name)
d1 = self.send_message(self.rq_exchange, route, reply,
tid=tid)
d1.addErrback(self._error)
if not self.no_ack:
self.client.basic_ack(msg)
return d1
def _read_new(_none):
if not self.parallel and not self._stopping:
reactor.callLater(0, self.read_message_loop)
d.addCallback(_push_message)
if not self.parallel:
d.addCallback(_read_new)
d.addErrback(self._error)
return d
def callRemote(self, method, *args, **kwargs):
"""
Call method on remote API. Method is a string and may include a package
name, such as 'admin.uptime'. Any additional arguments and keyword arguments
are passed to that method as arguments and keyword arguments.
"""
if not self.rootobj:
d = Deferred()
self.pending_calls.append((d, method, args, kwargs))
self.connect()
return d
api, method_name = method.split('.')
api = api.lower()
if self.local:
return maybeDeferred(getattr(getattr(self.app.api, api), method_name), *args, **kwargs)
else:
try:
d = self.rootobj.callRemote('api', method, *args, **kwargs)
d.addErrback(self._error_back, method, args, kwargs)
return d
except pb.DeadReferenceError:
self.rootobj = None
d = Deferred()
self.pending_calls.append((d, method, args, kwargs))
self.connect()
return d
def join(self, _):
def send(_, node):
factory = NodeClientFactory(OverlayService(self), {"command" : \
"join","node":self.my_node})
reactor.connectTCP(node["host"], node["tcp_port"], factory)
factory.deferred.addCallback(lambda _: node)
def sendcallback(_):
return node
def senderrback(_):
raise Exception()
factory.deferred.addCallbacks(sendcallback,senderrback)
return factory.deferred
def success(node):
coordinator = node
def error(e):
self.is_coordinator = True
self.my_node["id"] = 0
self.nextid = 1
self.members[self.my_node["host"]] = self.my_node
send_log("Notice", "I am coordinator")
return e
# search for running loadbalancers and join the overlay network
initialized = False
d = Deferred()
for node in self.config["nodes"]:
d.addErrback(send, node)
d.addCallbacks(success, error)
d.errback(0)
return d
def cbPostAuth(self, response, cookie):
#print 'cbPostAuth()'
finished = Deferred()
response.deliverBody(BeginningPrinter(finished))
finished.addCallback(self.cbPostAuthSuccess, cookie)
finished.addErrback(self.cbShutdown)
return finished
def _executeCommand(self, connection, protocolFactory):
"""
Given a secured SSH connection, try to execute a command in a new
channel created on it and associate the result with a protocol from the
given factory.
@param connection: See L{SSHCommandClientEndpoint.existingConnection}'s
C{connection} parameter.
@param protocolFactory: See L{SSHCommandClientEndpoint.connect}'s
C{protocolFactory} parameter.
@return: See L{SSHCommandClientEndpoint.connect}'s return value.
"""
commandConnected = Deferred()
def disconnectOnFailure(passthrough):
# Close the connection immediately in case of cancellation, since
# that implies user wants it gone immediately (e.g. a timeout):
immediate = passthrough.check(CancelledError)
self._creator.cleanupConnection(connection, immediate)
return passthrough
commandConnected.addErrback(disconnectOnFailure)
channel = _CommandChannel(
self._creator, self._command, protocolFactory, commandConnected)
connection.openChannel(channel)
return commandConnected
def send_payment_request(self, readTokens, writeTokens):
"""Called by a Circuit object when it wants to actually make a payment
@param readTokens: the number of read tokens to pay for at each hop in the circuit
@type readTokens: int
@param writeTokens: the number of read tokens to pay for at each hop in the circuit
@type writeTokens: int"""
assert (readTokens + writeTokens) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells"
#make sure our setup is done:
if not self.setupDone:
#have we even started?
if not self.setupStarted:
self.send_setup_message()
self.queuedReadTokens += readTokens
self.queuedWriteTokens += writeTokens
return
#dont bother trying to send payments for circuits that are already closed
if self.circ.is_done():
return
#send the payments
deferreds = []
for paymentStream in self.paymentStreams.values():
deferreds.append(paymentStream.send_payment(readTokens, writeTokens))
paymentsDoneDeferred = DeferredList(deferreds)
paymentsDoneDeferred.addErrback(self.generic_error_handler)
addTokensDeferred = Deferred()
self.inflightReadTokens += readTokens
self.inflightWriteTokens += writeTokens
#timeout in case the payment fails. We will close the circuit in this case.
event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT, self.all_receipts_received, None, addTokensDeferred, readTokens, writeTokens, None)
paymentsDoneDeferred.addCallback(self.all_receipts_received, addTokensDeferred, readTokens, writeTokens, event)
addTokensDeferred.addCallback(self._add_tokens_callback, readTokens, writeTokens)
addTokensDeferred.addErrback(self.generic_error_handler)
def hack(_, peer_id):
# Assume the parties are given a random share [r] by a
# trusted dealer.
share_r = self.random_share(field)
# 1. r = OpenTo(P_i, [r])
open_r = self.open(share_r, [peer_id])
def subtract_delta(delta, share_r):
delta = field(long(delta))
x = self.sub(share_r, delta)
return x
if peer_id == self.id:
def g(r, x):
delta = r - x
delta = self.broadcast([peer_id], self.players.keys(),
str(delta.value))
self.schedule_callback(delta, subtract_delta, share_r)
delta.addErrback(self.error_handler)
return delta
self.schedule_callback(open_r, g, number)
open_r.addErrback(self.error_handler)
return open_r
else:
d = Deferred()
def g(_, peer_id, share_r):
delta = self.broadcast([peer_id], self.players.keys())
self.schedule_callback(delta, subtract_delta, share_r)
delta.addErrback(self.error_handler)
return delta
self.schedule_callback(d, g, peer_id, share_r)
d.addErrback(self.error_handler)
d.callback(None)
return d
def jsonrpc_call(agent, server, data , bitHopper):
global i
try:
request = json.dumps({'method':'getwork', 'params':data, 'id':i}, ensure_ascii = True)
i = i +1
header = {'Authorization':["Basic " +base64.b64encode(server['user']+ ":" + server['pass'])], 'User-Agent': ['poclbm/20110709'],'Content-Type': ['application/json'] }
d = agent.request('POST', "http://" + server['mine_address'], Headers(header), StringProducer(request))
d.addErrback(bitHopper.log_dbg)
response = yield d
if response == None:
raise Exception("Response is none")
header = response.headers
#Check for long polling header
lp = bitHopper.lp
if lp.check_lp(server['pool_index']):
#bitHopper.log_msg('Inside LP check')
for k,v in header.getAllRawHeaders():
if k.lower() == 'x-long-polling':
lp.set_lp(v[0],server['pool_index'])
break
finish = Deferred()
finish.addErrback(bitHopper.log_dbg)
response.deliverBody(WorkProtocol(finish))
body = yield finish
except Exception, e:
bitHopper.log_dbg('Caught, jsonrpc_call insides')
bitHopper.log_dbg(e)
#traceback.print_exc
defer.returnValue(None)
def cachedZipResource(self, request, name, url, segments):
archivePath = self.config.CachedResources.child("{0}.zip".format(name))
if archivePath.exists():
d = Deferred()
d.callback(None)
else:
d = http_download(archivePath, url)
def readFromArchive(_):
filePath = ZipArchive(archivePath.path)
for segment in segments:
filePath = filePath.child(segment)
return filePath.getContent()
def notFoundHandler(f):
f.trap(KeyError)
request.setResponseCode(http.NOT_FOUND)
set_response_header(
request, HeaderName.contentType, ContentType.plain
)
return b"Not found."
d.addCallback(readFromArchive)
d.addErrback(notFoundHandler)
return d
def handle_images(result):
if result[0] == 0:
image_list = result[1]
##
## Find the image we're reindexing, and bump it's index by .1
##
for image in image_list:
if image['image_id'] == image_id:
if image['media_idx'] > new_idx:
image['media_idx'] = float(new_idx)-.1
else:
image['media_idx'] = float(new_idx)+.1
##
## Now, resort the image list.
##
self.log.debug("before sort: %s" % pformat(image_list))
image_list.sort(lambda a, b: cmp(a['media_idx'], b['media_idx']))
self.log.debug("after sort: %s" % pformat(image_list))
##
## The images are now in the proper order. This loop starts at
## 0 and updates each image's index, if necessary. If the new index
## matches the old index, nothing happens.
##
d3 = Deferred()
for image in image_list:
d3.addCallback(act_order, image)
d3.callback(0)
d3.addCallback(lambda _: (0, "success"))
d3.addErrback(lambda _: (-1, _.getErrorMessage()))
return d3
else:
return result
def cbGetFirstCookie(self, response):
onet_ubi_cookie = response.headers.getRawHeaders("Set-Cookie")[0]
onetzuo_ticket_cookie = response.headers.getRawHeaders("Set-Cookie")[1]
onet_cid_cookie = response.headers.getRawHeaders("Set-Cookie")[2]
# onet_sgn_cookie = response.headers.getRawHeaders('Set-Cookie')[3]
onet_ubi_match = re.search("onet_ubi=(.*?);", onet_ubi_cookie)
if onet_ubi_match:
onet_ubi_result = onet_ubi_match.group()
else:
onet_ubi_result = None
onetzuo_ticket_match = re.search("onetzuo_ticket=(.*?);", onetzuo_ticket_cookie)
if onetzuo_ticket_match:
onetzuo_ticket_result = onetzuo_ticket_match.group()
else:
onetzuo_ticket_result = None
onet_cid_match = re.search("onet_cid=(.*?);", onet_cid_cookie)
if onet_cid_match:
onet_cid_result = onet_cid_match.group()
else:
onet_cid_result = None
if onet_ubi_result != None and onetzuo_ticket_result != None and onet_cid_result != None:
finished = Deferred()
response.deliverBody(BeginningPrinter(finished))
finished.addCallback(self.cbGetFirstCookieSuccess, onet_ubi_result, onetzuo_ticket_result, onet_cid_result)
finished.addErrback(self.cbShutdown)
return finished
else:
self.authorise()
def cbRequest(response):
finished = Deferred()
finished.addCallback(cb)
finished.addErrback(err)
response.deliverBody(BufferedReception(finished))
cli.inflight -= 1
return finished
def _internal_connect(self, server, timeout, share=False):
"""Connect to the remote host provided via an ssh port forward.
An SSH process is fired with port forwarding established on localhost
22181, which the zookeeper client connects to.
:param server: Remote host to connect to, specified as hostname:port
:type string
:param timeout: An timeout interval in seconds.
:type float
Returns a connected client or error.
"""
hostname, port = self._parse_servers(server or self._servers)
start_time = time.time()
# Determine which port we'll be using.
local_port = get_open_port()
port_watcher = PortWatcher("localhost", local_port, timeout)
tunnel_error = Deferred()
# On a tunnel error, stop the port watch early and bail with error.
tunnel_error.addErrback(port_watcher.stop)
# If a tunnel error happens now or later, close the connection.
tunnel_error.addErrback(lambda x: self.close())
# Setup tunnel via an ssh process for port forwarding.
protocol = ClientTunnelProtocol(self, tunnel_error)
self._process = forward_port(
self.remote_user, local_port, hostname, int(port),
process_protocol=protocol, share=share)
# Wait for the tunneled port to open.
try:
yield port_watcher.async_wait()
except socket.error:
self.close() # Stop the tunnel process.
raise ConnectionTimeoutException("could not connect")
else:
# If we stopped because of a tunnel error, raise it.
if protocol.error:
yield tunnel_error
# Check timeout
new_timeout = timeout - (time.time() - start_time)
if new_timeout <= 0:
self.close()
raise ConnectionTimeoutException(
"could not connect before timeout")
# Connect the client
try:
yield super(SSHClient, self).connect(
"localhost:%d" % local_port, new_timeout)
except:
self.close() # Stop the tunnel
raise
returnValue(self)
def test_json(self):
deferred = Deferred()
deferred.addCallback(lambda _: self.callbacks.add("success"))
deferred.addErrback(lambda _: self.callbacks.add("failure"))
twisted_headers = Headers({"Content-Type": ["application/json"]})
twisted_response = TWResponse(
("HTTP", 1, 1), 200, "OK", twisted_headers, None)
request = Request(
method="GET", url="/",
kwargs={
"headers": {
"Content-Type": "application/json"}, "data": None})
data = {
os.urandom(6).encode("hex"): os.urandom(6).encode("hex"),
os.urandom(6).encode("hex"): os.urandom(6).encode("hex")
}
r = Response(deferred, twisted_response, request)
map(r.dataReceived, json.dumps(data))
r.connectionLost(responseDone)
self.assertEqual(self.callbacks, set(["success"]))
self.assertTrue(r._done)
self.assertEqual(r.json(), data)
return deferred
def ask(self, move, answers):
"""server sends move. We ask the user. answers is a list with possible answers,
the default answer being the first in the list."""
if not Options.gui:
return Client.ask(self, move, answers)
self.game.myself.computeSayable(move, answers)
deferred = Deferred()
deferred.addCallback(self.__askAnswered)
deferred.addErrback(self.__answerError, move, answers)
iAmActive = self.game.myself == self.game.activePlayer
self.game.myself.handBoard.setEnabled(iAmActive)
scene = Internal.scene
oldDialog = scene.clientDialog
if oldDialog and not oldDialog.answered:
raise Exception('old dialog %s:%s is unanswered, new Dialog: %s/%s' % (
str(oldDialog.move),
str([x.message.name for x in oldDialog.buttons]),
str(move), str(answers)))
if not oldDialog or not oldDialog.isVisible():
# always build a new dialog because if we change its layout before
# reshowing it, sometimes the old buttons are still visible in which
# case the next dialog will appear at a lower position than it
# should
scene.clientDialog = ClientDialog(
self,
scene.mainWindow.centralWidget())
assert scene.clientDialog.client is self
scene.clientDialog.askHuman(move, answers, deferred)
return deferred
def test_childConnectionLost(self):
"""
L{IProcessProtocol.childConnectionLost} is called each time a file
descriptor associated with a child process is closed.
"""
connected = Deferred()
lost = {0: Deferred(), 1: Deferred(), 2: Deferred()}
class Closer(ProcessProtocol):
def makeConnection(self, transport):
connected.callback(transport)
def childConnectionLost(self, childFD):
lost[childFD].callback(None)
target = b"twisted.internet.test.process_loseconnection"
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Closer(),
pyExe, [pyExe, b"-m", target],
env=properEnv,
usePTY=self.usePTY)
def cbConnected(transport):
transport.write(b'2\n')
return lost[2].addCallback(lambda ign: transport)
connected.addCallback(cbConnected)
def lostSecond(transport):
transport.write(b'1\n')
return lost[1].addCallback(lambda ign: transport)
connected.addCallback(lostSecond)
def lostFirst(transport):
transport.write(b'\n')
connected.addCallback(lostFirst)
connected.addErrback(err)
def cbEnded(ignored):
reactor.stop()
connected.addCallback(cbEnded)
self.runReactor(reactor)
def test_childConnectionLost(self):
"""
L{IProcessProtocol.childConnectionLost} is called each time a file
descriptor associated with a child process is closed.
"""
connected = Deferred()
lost = {0: Deferred(), 1: Deferred(), 2: Deferred()}
class Closer(ProcessProtocol):
def makeConnection(self, transport):
connected.callback(transport)
def childConnectionLost(self, childFD):
lost[childFD].callback(None)
source = (
"import os, sys\n"
"while 1:\n"
" line = sys.stdin.readline().strip()\n"
" if not line:\n"
" break\n"
" os.close(int(line))\n")
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, Closer(), sys.executable,
[sys.executable, "-c", source], usePTY=self.usePTY)
def cbConnected(transport):
transport.write('2\n')
return lost[2].addCallback(lambda ign: transport)
connected.addCallback(cbConnected)
def lostSecond(transport):
transport.write('1\n')
return lost[1].addCallback(lambda ign: transport)
connected.addCallback(lostSecond)
def lostFirst(transport):
transport.write('\n')
connected.addCallback(lostFirst)
connected.addErrback(err)
def cbEnded(ignored):
reactor.stop()
connected.addCallback(cbEnded)
self.runReactor(reactor)
def test_processExited(self):
"""
L{IProcessProtocol.processExited} is called when the child process
exits, even if file descriptors associated with the child are still
open.
"""
exited = Deferred()
allLost = Deferred()
lost = []
class Waiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD,))
lost.append(childFD)
if len(lost) == 3:
allLost.callback(None)
def processExited(self, reason):
msg('processExited(%r)' % (reason,))
# See test_processExitedWithSignal
exited.callback([reason])
self.transport.loseConnection()
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, Waiter(), sys.executable,
[sys.executable, self.keepStdioOpenProgram, "child",
self.keepStdioOpenArg],
usePTY=self.usePTY)
def cbExited((failure,)):
failure.trap(ProcessDone)
msg('cbExited; lost = %s' % (lost,))
self.assertEqual(lost, [])
return allLost
exited.addCallback(cbExited)
def cbAllLost(ignored):
self.assertEqual(set(lost), set([0, 1, 2]))
exited.addCallback(cbAllLost)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
class AMQPReconnectingFactory(ReconnectingClientFactory):
protocol = AMQPProtocol
def __init__(self, spec_path=None, vhost='/', username='******', \
password='******', host='localhost', port=5672, delegate=None):
self.username = username
self.password = password
self.delegate = delegate or TwistedDelegate()
self.vhost = vhost
self.spec = txamqp.spec.load(spec_path)
self.is_connected = False
self.host = host
self.port = port
def buildProtocol(self, addr):
p = self.protocol(self.delegate, self.vhost, self.spec)
p.factory = self
self.deferred = Deferred()
self.addCallbacksOnProtocol(self.deferred)
self.deferred.addErrback(log.err)
return p
@inlineCallbacks
def gotConnection(self, connection):
log.msg(' Got connection, authenticating.')
self.resetDelay()
yield connection.start({
'LOGIN': self.username,
'PASSWORD': self.password
})
channel = yield connection.channel(1)
yield channel.channel_open()
self.is_connected = True
self.deferred.callback((connection, channel))
def lostConnection(self, failure):
self.is_connected = False
def addCallbacksOnProtocol(self, deferred):
"""
This deferrred will fire on new connection, channel.
"""
self.deferred.addCallback(lambda *a, **kw: log.msg('Got new channel'))
def startAMQPConnection(self):
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, self)
def receivedAddrBroadcast(params):
(addresses, sender_rank) = params
def errorDummy(f):
log.critical('Error occurred: ' + str(f.getErrorMessage()))
log.critical(
'I SHOULD go into error reversion phase, but I won\'t.')
return f
def fireShufflingDeferred(v):
state.shuffle.fireShufflingDeferred()
return v
def checksumRoutine(_):
deferred = recombineChecksum(state.input.getAssignedEscrows(),
sender_rank)
reference_checksum = computeReferenceChecksum(addresses)
deferred.addBoth(compareChecksums,
reference_checksum=reference_checksum)
return deferred
def decideAction(checksum):
if (sender_rank == state.mixnet.getRank() - 1): # It's my turn
shuffled_addresses = decryptionMixnet(addresses)
deferred = broadcastShufflingResult(shuffled_addresses)
deferred.addCallback(_fire_shuffle_broadcast,
shuffled_addresses,
state.mixnet.getRank())
return deferred
if (state.mixnet.isLastMixpeer(sender_rank)
): # I have received the plaintext addresses => finalize!
ordered_addresses = orderLexicographically(addresses)
shuffled_addresses = computeFinalPermutation(
ordered_addresses, checksum)
state.input.assignOutputAddresses(shuffled_addresses)
state.shuffle.fireShufflingDeferred()
return
deferred = Deferred()
# We're at an intermediate peer, not the one starting the mixing => We have a predecessor to check
if (sender_rank >= 0):
deferred.addCallback(checksumRoutine)
deferred.addCallback(decideAction)
deferred.addErrback(errorDummy)
deferred.callback(None)
return deferred
def _yombo_universal_hook_(self, hook_name=None, **kwargs):
"""
Implements the universal hook.
:param kwargs:
:return:
"""
if hook_name in self.hook_listeners:
for a_callback in self.hook_listeners[hook_name]:
d = Deferred()
d.addCallback(lambda ignored: maybeDeferred(
a_callback, self, hook_name=hook_name, **kwargs))
d.addErrback(self.yombo_universal_hook_failure, hook_name,
a_callback)
d.callback(1)
yield d
def _route_notification(self, version, result, data, ttl=None):
self.version = self._client_info['message_id'] = version
notification = Notification(version=version,
data=data,
channel_id=self.chid,
headers=self.request.headers,
ttl=ttl)
d = Deferred()
d.addCallback(self.router.route_notification, result)
d.addCallback(self._router_completed, result)
d.addErrback(self._router_fail_err)
d.addErrback(self._response_err)
# Call the prepared router
d.callback(notification)
def test_unregisterProducerAfterDisconnect(self):
"""
If a producer is unregistered from a L{ITCPTransport} provider after
the transport has been disconnected (by the peer) and after
L{ITCPTransport.loseConnection} has been called, the transport is not
re-added to the reactor as a writer as would be necessary if the
transport were still connected.
"""
reactor = self.buildReactor()
port = reactor.listenTCP(0,
serverFactoryFor(ClosingProtocol),
interface=self.interface)
finished = Deferred()
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
writing = []
class ClientProtocol(Protocol):
"""
Protocol to connect, register a producer, try to lose the
connection, wait for the server to disconnect from us, and then
unregister the producer.
"""
def connectionMade(self):
log.msg("ClientProtocol.connectionMade")
self.transport.registerProducer(
_SimplePullProducer(self.transport), False)
self.transport.loseConnection()
def connectionLost(self, reason):
log.msg("ClientProtocol.connectionLost")
self.unregister()
writing.append(self.transport in _getWriters(reactor))
finished.callback(None)
def unregister(self):
log.msg("ClientProtocol unregister")
self.transport.unregisterProducer()
clientFactory = ClientFactory()
clientFactory.protocol = ClientProtocol
reactor.connectTCP(self.interface, port.getHost().port, clientFactory)
self.runReactor(reactor)
self.assertFalse(writing[0],
"Transport was writing after unregisterProducer.")
def authenticate_tenant(self, tenant_id):
"""
see :meth:`IAuthenticator.authenticate_tenant`
"""
log = self._log.bind(tenant_id=tenant_id)
if tenant_id in self._cache:
(created, data) = self._cache[tenant_id]
now = self._reactor.seconds()
if now - created <= self._ttl:
log.msg('otter.auth.cache.hit', age=now - created)
return succeed(data)
log.msg('otter.auth.cache.expired', age=now - created)
if tenant_id in self._waiters:
d = Deferred()
self._waiters[tenant_id].append(d)
log.msg('otter.auth.cache.waiting',
waiters=len(self._waiters[tenant_id]))
return d
def when_authenticated(result):
log.msg('otter.auth.cache.populate')
self._cache[tenant_id] = (self._reactor.seconds(), result)
waiters = self._waiters.pop(tenant_id, [])
for waiter in waiters:
waiter.callback(result)
return result
def when_auth_fails(failure):
waiters = self._waiters.pop(tenant_id, [])
for waiter in waiters:
waiter.errback(failure)
return failure
log.msg('otter.auth.cache.miss')
self._waiters[tenant_id] = []
d = self._authenticator.authenticate_tenant(tenant_id)
d.addCallback(when_authenticated)
d.addErrback(when_auth_fails)
return d
def start_all(self, ids, truncate=False):
"""
A starting point of the workflow.
@param ids: list of ids (commands_on_host) to start.
@type ids: list
"""
d1 = Deferred()
banned = self.bundles.get_banned_cohs()
ids = [id for id in ids if id not in banned]
already_initialized_circuits = [
c for c in self._circuits if c.id in ids
]
# already initialized circuits will be started
aicd = maybeDeferred(self._circuits_activate,
already_initialized_circuits)
aicd.addCallback(self._run_all)
aicd.addErrback(self._start_failed)
already_initialized_ids = [c.id for c in already_initialized_circuits]
new_ids = [id for id in ids if not id in already_initialized_ids]
if truncate and len(new_ids) > self.free_slots:
self.logger.info(
"Number of new circuits was truncated from %d to %d" %
(len(new_ids), self.free_slots))
new_ids = new_ids[:self.free_slots]
self.logger.info("Remaining circuits will be requested soon")
if len(new_ids) > 0:
d1 = self._setup_all(new_ids)
d1.addCallback(self._assign_launcher)
d1.addCallback(self._class_bundles)
d1.addCallback(self._class_all)
d1.addCallback(self._revolve_all)
d1.addCallback(self._run_all)
d1.addErrback(self._start_failed)
return d1
else:
succeed(True)
def __call__(self, **kwargs):
d = Deferred()
d.addCallback(self._callbackResponse)
d.addErrback(self._callbackError)
kwargs = self.method.serializeArgs(kwargs)
self.agent.sendCommand(d, self.interface.client,
self.interface.agentId,
self.name, kwargs)
# Asyncronously wait until response or error arrive
# Then TSAgent will call callback or errback in our Deferred
# and yield activates
result = yield d
returnValue(self.method.deserializeResult(result))
def test_processEnded(self):
"""
L{IProcessProtocol.processEnded} is called after the child process
exits and L{IProcessProtocol.childConnectionLost} is called for each of
its file descriptors.
"""
ended = Deferred()
lost = []
class Ender(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
self.transport.loseConnection()
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD, ))
lost.append(childFD)
def processExited(self, reason):
msg('processExited(%r)' % (reason, ))
def processEnded(self, reason):
msg('processEnded(%r)' % (reason, ))
ended.callback([reason])
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Ender(),
pyExe, [
pyExe, b"-m", self.keepStdioOpenProgram,
b"child", self.keepStdioOpenArg
],
env=properEnv,
usePTY=self.usePTY)
def cbEnded(args):
failure, = args
failure.trap(ProcessDone)
self.assertEqual(set(lost), set([0, 1, 2]))
ended.addCallback(cbEnded)
ended.addErrback(err)
ended.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def ping(idurl,
channel=None,
ack_timeout=15,
ping_retries=0,
keep_alive=False):
"""
Doing handshake with remote node only if it is currently not connected.
Returns Deferred object.
"""
idurl = strng.to_bin(idurl)
if _Debug:
lg.args(_DebugLevel,
idurl=idurl,
keep_alive=keep_alive,
channel=channel)
result = Deferred()
result.addErrback(on_ping_failed, idurl=idurl, channel=channel)
if id_url.is_empty(idurl):
result.errback(Exception('empty idurl provided'))
return result
if not id_url.is_cached(idurl):
if _Debug:
lg.dbg(
_DebugLevel,
'user identity %r not cached yet, executing clean handshake' %
idurl)
return handshaker.ping(
idurl=idurl,
ack_timeout=ack_timeout,
ping_retries=ping_retries,
channel=channel or 'clean_ping',
keep_alive=keep_alive,
)
idurl = id_url.field(idurl)
if not isKnown(idurl):
if not check_create(idurl, keep_alive=keep_alive):
raise Exception('can not create instance')
A(idurl,
'ping-now',
result,
channel=channel,
ack_timeout=ack_timeout,
ping_retries=ping_retries,
original_idurl=idurl.to_original())
return result
def virtualChannelClose(self, authToken, sessionId, virtualName, instance):
"""
Parameters:
- authToken
- sessionId
- virtualName
- instance
"""
logger.debug(
"virtualChannelClose(auth=%s, sessionId=%s virtualName=%s)" %
(authToken, sessionId, virtualName))
session = self.server.retrieveSession(sessionId)
if not session:
logger.error("session %s not present here" % sessionId)
return False
if session.token != authToken:
logger.error("authToken doesn't match for session %s" % sessionId)
return False
icpFactory = self.server.icpFactory
req = FdsApiVirtualChannelCloseRequest()
req.connectionId = session.connectionId
req.virtualName = virtualName
req.instance = instance
d = Deferred()
def onError(err):
logger.error('an error occured')
return False
def onSuccess(response):
logger.debug('returning %s' % (response.success))
return response.success
d.addErrback(onError)
d.addCallback(onSuccess)
icpFactory.doQuery(
FdsApiVirtualChannelClose, req,
Thrift2IcpHandler(self, d, FdsApiVirtualChannelCloseResponse))
return d
return True
def request_helo(self, encrypted_output_address):
address_deferred = Deferred()
address_deferred.addErrback(self.request_new_addresses)
address_deferred.addCallback(
self.request_helo_callback,
encrypted_output_address=encrypted_output_address)
# Assign new address
input_peer = self.state.input.addInputPeer()
self.state.commit.increasePeerCount()
if (input_peer is not None):
log.debug('I\'m going to request new addresses...')
address_deferred.errback(5)
else:
log.debug('Immediately going to real callback.')
address_deferred.callback(input_peer)
return address_deferred
def get_position_rank(self, user_id, request):
"""获取史实城官职榜（内部接口）
"""
timer = Timer(user_id)
req = internal_pb2.GetLegendCityPositionRankReq()
req.ParseFromString(request)
res = internal_pb2.GetLegendCityPositionRankRes()
res.status = 0
defer = Deferred()
for city_id in data_loader.LegendCityBasicInfo_dict:
defer.addCallback(self._calc_get_position_rank, city_id, res,
timer)
defer.addCallback(self._get_position_rank_succeed, req, res, timer)
defer.addErrback(self._get_position_rank_failed, req, timer)
defer.callback(True)
return defer
def do_launch(self, ident, scan, priority, pause_running_not_a_function):
print "pause_running: {}", pause_running_not_a_function, "for scan ident:", ident
d = Deferred()
scan.update_params(self.parent.all_parameters())
status = script_semaphore(ident, self.signals)
self._add_to_running(ident, scan, d, status, priority)
if pause_running_not_a_function:
d.addCallback(self.pause_running, scan, ident)
d.addCallback(self.launch_in_thread, scan, ident)
if pause_running_not_a_function:
d.addCallback(self.unpause_on_finish)
d.addCallback(self.remove_from_running, running_id=ident)
d.addCallback(self.launch_scripts)
d.addErrback(self.failure)
d.callback(True)
self.signals.on_running_new_script((ident, scan.name))
def test_robotstxt_error(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
err = error.DNSLookupError('Robotstxt address not found')
def return_failure(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.errback, failure.Failure(err))
return deferred
self.crawler.engine.download.side_effect = return_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock()
middleware.process_request(Request('http://site.local'), None)
deferred = Deferred()
deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.
assert_any_call()))
reactor.callFromThread(deferred.callback, None)
return deferred
def _check_retry_payloads(failed_payloads_with_errs):
"""Check our retry count and retry after a delay or errback
If we have more retries to try, create a deferred that will fire
with the result of delayed retry. If not, errback the remaining
deferreds with failure
Params:
failed_payloads - list of (payload, failure) tuples
"""
# Do we have retries left?
if self._req_attempts >= self._max_attempts:
# No, no retries left, fail each failed_payload with its
# associated failure
for p, f in failed_payloads_with_errs:
t_and_p = TopicAndPartition(p.topic, p.partition)
_deliver_result(deferredsByTopicPart[t_and_p], f)
return
# Retries remain! Schedule one...
d = Deferred()
dc = self.client.reactor.callLater(self._retry_interval,
d.callback,
[p for p, f in failed_payloads])
self._retry_interval *= self.RETRY_INTERVAL_FACTOR
# Cancel the callLater when request is cancelled before it fires
d.addErrback(_cancel_retry, dc)
# Reset the topic metadata for all topics which had failed_requests
# where the failures were of the kind UnknownTopicOrPartitionError
# or NotLeaderForPartitionError, since those indicate our client's
# metadata is out of date.
reset_topics = []
def _check_for_meta_error(tup):
payload, failure = tup
if (isinstance(failure, NotLeaderForPartitionError)
or isinstance(failure, UnknownTopicOrPartitionError)):
reset_topics.append(payload.topic)
map(_check_for_meta_error, failed_payloads)
if reset_topics:
self.client.reset_topic_metadata(*reset_topics)
d.addCallback(_do_retry)
return d
def _do_synchronize_keys(self):
"""
Make sure all my keys are stored on my suppliers nodes (encrypted with my master key).
If some key I do not have locally, but I know remote copy exists - download it.
If some key was not stored - make a remote copy on supplier machine.
When key was renamed (after identity rotate) make sure to store the latest copy and remove older one.
"""
from logs import lg
from userid import global_id
from userid import my_id
from interface import api
from storage import backup_control
from storage import index_synchronizer
from storage import keys_synchronizer
from twisted.internet.defer import Deferred
result = Deferred()
result.addCallback(self._on_keys_synchronized)
result.addErrback(self._on_keys_synchronize_failed)
is_in_sync = index_synchronizer.is_synchronized(
) and backup_control.revision() > 0
if not is_in_sync:
lg.warn('backup index database is not synchronized yet')
result.errback(
Exception('backup index database is not synchronized yet'))
return None
global_keys_folder_path = global_id.MakeGlobalID(
key_alias='master', customer=my_id.getGlobalID(), path='.keys')
res = api.file_exists(global_keys_folder_path)
if res['status'] != 'OK' or not res['result'] or not res['result'].get(
'exist'):
res = api.file_create(global_keys_folder_path, as_folder=True)
if res['status'] != 'OK':
lg.err(
'failed to create ".keys" folder "%s" in the catalog: %r' %
(global_keys_folder_path, res))
result.errback(
Exception(
'failed to create keys folder "%s" in the catalog: %r'
% (global_keys_folder_path, res)))
return
lg.info('created new remote folder ".keys" in the catalog: %r' %
global_keys_folder_path)
keys_synchronizer.A('sync', result)
def sendMail(self, view, message):
account = view[self.account].smtp
authRequired = account.useAuth
heloFallback = not authRequired
securityRequired = account.connectionSecurity == 'TLS'
if authRequired:
username = account.username
password = account.password
else:
username = None
password = None
retries = account.numRetries
timeout = account.timeout
deferred = Deferred()
deferred.addCallback(self._sendSuccess)
deferred.addErrback(self._sendFailure)
class _protocol(_TwistedESMTPSender):
def connectionMade(_self):
self.output("smtp: connected to %s" % (account.host))
_TwistedESMTPSender.connectionMade(_self)
# the code below comes from osaf.mail.smtp
factory = ESMTPSenderFactory(username, password,
message['From'], message['To'],
StringIO(str(message)), deferred, retries,
timeout, 1, heloFallback, authRequired,
securityRequired)
factory.protocol = _protocol
factory.testing = False
if account.connectionSecurity == 'SSL':
reactor.callFromThread(ssl.connectSSL, account.host, account.port,
factory, view)
else:
reactor.callFromThread(ssl.connectTCP, account.host, account.port,
factory, view)
def test_oldFlattenableError(self):
"""
If the old flattener encounters an asynchronous Failure while
flattening an L{IRenderable}, the returned L{Deferred} fires with the
failure.
"""
result = Deferred()
element = Element()
element.docFactory = stan(result)
request = FakeRequest()
context = WovenContext()
context.remember(request)
accumulator = []
finished = oldFlatten(element, context, accumulator.append, lambda ign: ign)
result.addErrback(lambda err: err.trap(RuntimeError))
finished = self.assertFailure(finished, RuntimeError)
result.errback(RuntimeError("test error"))
return finished
def add_task(
self,
pkg_path,
pkg_sha1,
task_id,
pkg_size, # noqa pylint:disable=too-many-arguments
client_options=None) -> Deferred:
_result = Deferred()
_result.addErrback(self._add_task_error)
def callback(r):
value = r, pkg_path, pkg_sha1, pkg_size
_result.callback(value)
_deferred = self.resource_manager.add_task(
[pkg_path], task_id, client_options=client_options)
_deferred.addCallback(callback)
_deferred.addErrback(_result.errback)
return _result
def subscribe(self):
def _logFailure(failure):
log.debug("subscriber reported {message}",
message=failure.getErrorMessage())
return failure
def _logGrantedQoS(value):
log.debug("subscriber response {value!r}", value=value)
return True
def _subscribe_topic(response, topic, qos):
log.info("subscriber response {value!r}", value=response)
return self.protocol.subscribe(topic, qos)
d = Deferred()
for topic, qos in self.subscriptions:
d.addCallback(_subscribe_topic, topic, qos)
d.addErrback(_logFailure)
d.callback("Start")
return d
def filter_data(tracedata, uuid, modality):
"""Filter & process, and forward trace for format conversion."""
module_ = importlib.import_module(
"contextmonkey.tracelayer.processingfilters.%s" %
(modality.traceprocessing))
class_ = getattr(module_, "%s" % modality.traceprocessing)
ostm = class_()
d = Deferred()
d.addCallback(ostm.filterTrace, uuid)
d.addErrback(ostm.handlefailure)
d.addCallback(ostm.processTrace, uuid)
d.addCallback(DataSourceManager.traceformatmanager.toProtobuf,
modality)
d.addCallback(receiveValue, uuid, modality)
d.callback(tracedata)
tracelayer.log(
"DataSourceManager-filterData-timestamp:",
str(modality.name) + " " + str("%0.20f" % time.time()) + " " +
str(uuid))
return d
def put(self, api_ver="v1", token=None):
sub = self.valid_input["subscription"]
user_data = sub["user_data"]
router = self.ap_settings.routers[user_data["router_type"]]
self._client_info["message_id"] = self.valid_input["version"]
notification = Notification(version=self.valid_input["version"],
data=self.valid_input["data"],
channel_id=str(sub["chid"]),
headers=self.request.headers,
ttl=None)
d = Deferred()
d.addCallback(router.route_notification, user_data)
d.addCallback(self._router_completed, user_data, "")
d.addErrback(self._router_fail_err)
d.addErrback(self._response_err)
# Call the prepared router
d.callback(notification)
def safe_stun(udp_port=None, dht_port=None, result_defer=None):
from twisted.internet.defer import Deferred
result = result_defer or Deferred()
def _check_response(response):
if response.get('result') == 'stun-success':
result.callback(response)
return None
http_stun(result_defer=result)
return None
def _fallback(err):
http_stun(result_defer=result)
return None
first_result = Deferred()
first_result.addCallback(_check_response)
first_result.addErrback(_fallback)
udp_dht_stun(udp_port, dht_port, result_defer=first_result)
return result
def send_payment_request(self, readTokens, writeTokens):
"""Called by a Circuit object when it wants to actually make a payment
@param readTokens: the number of read tokens to pay for at each hop in the circuit
@type readTokens: int
@param writeTokens: the number of read tokens to pay for at each hop in the circuit
@type writeTokens: int"""
assert (
readTokens + writeTokens
) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells"
#make sure our setup is done:
if not self.setupDone:
#have we even started?
if not self.setupStarted:
self.send_setup_message()
self.queuedReadTokens += readTokens
self.queuedWriteTokens += writeTokens
return
#dont bother trying to send payments for circuits that are already closed
if self.circ.is_done():
return
#send the payments
deferreds = []
for paymentStream in self.paymentStreams.values():
deferreds.append(
paymentStream.send_payment(readTokens, writeTokens))
paymentsDoneDeferred = DeferredList(deferreds)
paymentsDoneDeferred.addErrback(self.generic_error_handler)
addTokensDeferred = Deferred()
self.inflightReadTokens += readTokens
self.inflightWriteTokens += writeTokens
#timeout in case the payment fails. We will close the circuit in this case.
event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT,
self.all_receipts_received, None,
addTokensDeferred, readTokens,
writeTokens, None)
paymentsDoneDeferred.addCallback(self.all_receipts_received,
addTokensDeferred, readTokens,
writeTokens, event)
addTokensDeferred.addCallback(self._add_tokens_callback, readTokens,
writeTokens)
addTokensDeferred.addErrback(self.generic_error_handler)
def test_robotstxt_garbage(self):
# garbage response should be discarded, equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
middleware._logerror = mock.MagicMock()
middleware.process_request(Request('http://site.local'), None)
self.assertNotIgnored(Request('http://site.local'), middleware)
def test(r):
self.assertNotIgnored(Request('http://site.local/allowed'),
middleware)
self.assertNotIgnored(Request('http://site.local/admin/main'),
middleware)
self.assertNotIgnored(Request('http://site.local/static/'),
middleware)
deferred = Deferred()
deferred.addCallback(test)
deferred.addErrback(lambda _: self.assertIsNone(middleware._logerror.
assert_any_call()))
reactor.callFromThread(deferred.callback, None)
return deferred
def successCallback(response):
'''此时还没有开始响应体，但已经收到响应头的信息，如下：'''
# print('Response version:', response.version)
# print('Response code:', response.code)
# print('Response phrase:', response.phrase)
# print('Response headers:')
# print(pformat(list(response.headers.getAllRawHeaders())))
finished = Deferred()
# deliverBody方法接收一个Protocol对象，用来异步地接收数据
response.deliverBody(BodyCollector(finished, response.request.absoluteURI))
def handleBody(body):
print("twisted: ", len(body))
finished.addCallback(handleBody) # 当完成后，回调会在Protocol类中调用
def errorCallback(error):
print(response.request.absoluteURI, error)
finished.addErrback(errorCallback)
return finished
