def search_items(self, attributes):
"""Find items in any collection."""
d = Deferred()
result = []
def prompt_handle(unlocked):
"""Merge the items that were just unlocked."""
result.extend(unlocked)
return result
def unlock_handler(unlocked, prompt):
"""The items were unlocked, or a prompt should be shown first."""
result.extend(unlocked)
if prompt != "/":
d2 = self.do_prompt(prompt)
d2.addCallback(prompt_handle)
d2.chainDeferred(d)
else:
d.callback(result)
def items_found(unlocked, locked):
"""Called with two lists of found items."""
result.extend(unlocked)
if len(locked) > 0:
self.service.Unlock(locked,
reply_handler=unlock_handler,
error_handler=d.errback)
else:
d.callback(result)
self.service.SearchItems(attributes,
reply_handler=items_found,
error_handler=d.errback)
d.addCallback(self.make_item_list)
return d
def content(self):
def _add_content(data):
self._content = data
waiting_for_content = self._waiting_for_content
self._waiting_for_content = []
for d in waiting_for_content:
d.callback(self._content)
return self._content
if self._method == 'HEAD':
return succeed('')
if self._content is not None:
return succeed(self._content)
if self._content_d is not None:
d = Deferred()
d.addCallback(lambda _: self._content)
self._waiting_for_content.append(d)
return d
d = Deferred()
d.addCallback(_add_content)
self._content_d = d
self._response.deliverBody(_BodyCollector(d))
return d
def test_processTransportInterface(self):
"""
L{IReactorProcess.spawnProcess} connects the protocol passed to it
to a transport which provides L{IProcessTransport}.
"""
ended = Deferred()
protocol = _ShutdownCallbackProcessProtocol(ended)
reactor = self.buildReactor()
transport = reactor.spawnProcess(
protocol, sys.executable, [sys.executable, "-c", ""],
usePTY=self.usePTY)
# The transport is available synchronously, so we can check it right
# away (unlike many transport-based tests). This is convenient even
# though it's probably not how the spawnProcess interface should really
# work.
# We're not using verifyObject here because part of
# IProcessTransport is a lie - there are no getHost or getPeer
# methods. See #1124.
self.assertTrue(IProcessTransport.providedBy(transport))
# Let the process run and exit so we don't leave a zombie around.
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
def create_collection(self, label, alias=''):
"""Create a new collection with the specified properties."""
d = Deferred()
def createcollection_handler(collection, prompt):
"""A collection was created, or a prompt should be shown first."""
if prompt != "/":
self.do_prompt(prompt).chainDeferred(d)
else:
d.callback(collection)
def error_fallback(error):
"""Fall back to using the old property name."""
properties = {CLXN_LABEL_PROPERTY_OLD: dbus.String(
label,
variant_level=1)}
self.service.CreateCollection(
properties,
reply_handler=createcollection_handler,
error_handler=d.errback)
properties = {CLXN_LABEL_PROPERTY: dbus.String(label,
variant_level=1)}
try:
self.service.CreateCollection(
properties, alias,
reply_handler=createcollection_handler,
error_handler=error_fallback)
except TypeError:
error_fallback(None)
d.addCallback(lambda p: Collection(self, p))
return d
def _call(self, msg, cb, *args):
""" Internally used method which should be used to call the service.
@param msg: Request message which should be sent.
@param cb: Callback which should be called to process
response. The response will be the first
argument and additional arguments passed to the
_call method will be passed to the callback.
@param *args: Additional arguments which will be passed to the
callback.
"""
if not self._status:
raise InterfaceDisabledError('A disabled interface should not be '
'called.')
if not callable(cb):
raise TypeError('Callback has to be callable.')
uid = uuid4().hex
deferred = Deferred()
deferred.addCallback(cb, *args)
self._responses[uid] = deferred
self._conn.sendMessage(self._iTag, self._clsName, msg, uid)
def test(self):
crawler = mock.MagicMock()
crawler.settings = CrawlerSettings()
crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
crawler.engine.download = mock.MagicMock()
ROBOTS = re.sub(r'^\s+(?m)', '', '''
User-Agent: *
Disallow: /admin/
Disallow: /static/
''')
response = Response('http://site.local/robots.txt', body=ROBOTS)
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
return deferred
crawler.engine.download.side_effect = return_response
middleware = RobotsTxtMiddleware(crawler)
spider = None # not actually used
# There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
# and it is actually fetched only *after* first process_request completes.
# So, first process_request will always succeed.
# We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
self.assertIsNone(middleware.process_request(Request('http://site.local'), spider)) # not affected by robots.txt
def test(r):
self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider))
self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider)
self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider)
deferred = Deferred()
deferred.addCallback(test)
reactor.callFromThread(deferred.callback, None)
return deferred
def becomeGridHost(self, *args):
if self.clientObject.getLocalUser().gridHost:
for uuid in self.clientObject.users:
if self.clientObject.users[uuid].gridHostActive:
#TODO: Allow moderators to take gridhost from others.
showModalDialog(
self.window,
Gtk.MessageType.ERROR,
'The grid is already being hosted.'
)
return
#TODO: Show error dialogs on failures
self.setStatus('Loading OpenSim distribution...')
distribution = Distribution(self.clientObject.projectRoot, self.clientObject.externalhost, parent=self.window)
d = Deferred()
d.addCallback(self.startRobust)
distribution.load(d)
#TODO: Don't hardcode this
else:
showModalDialog(
self.window,
Gtk.MessageType.ERROR,
'You do not have permission to become the grid host.'
)
def _writeTest(self, write):
"""
Helper for testing L{IProcessTransport} write functionality. This
method spawns a child process and gives C{write} a chance to write some
bytes to it. It then verifies that the bytes were actually written to
it (by relying on the child process to echo them back).
@param write: A two-argument callable. This is invoked with a process
transport and some bytes to write to it.
"""
reactor = self.buildReactor()
ended = Deferred()
protocol = _ShutdownCallbackProcessProtocol(ended)
bytes = "hello, world" + os.linesep
program = (
"import sys\n"
"sys.stdout.write(sys.stdin.readline())\n"
)
def startup():
transport = reactor.spawnProcess(
protocol, sys.executable, [sys.executable, "-c", program])
try:
write(transport, bytes)
except:
err(None, "Unhandled exception while writing")
transport.signalProcess('KILL')
reactor.callWhenRunning(startup)
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
self.assertEqual(bytes, "".join(protocol.received[1]))
def test_http(self):
"""
When connecting to a HTTP server, a PB connection times
out.
"""
result = Deferred()
site = Site(Data("", "text/plain"))
client = HangCheckFactory(
PBClientFactory(), lambda: result.callback(None))
self.patch(HangCheckProtocol, '_HUNG_CONNECTION_TIMEOUT', 0.1)
connected_server_and_client(
self, site, client,
)
timer = reactor.callLater(2, result.cancel)
result.addCallback(lambda _: timer.cancel())
def check_for_timeout(reason):
reason.trap(CancelledError)
raise Exception("Didn't not hangup")
result.addErrback(check_for_timeout)
return result
def scale_down(self):
global overlay
n_workers = len(overlay.aws.workers)
d = Deferred()
for i in range(1, 1 + n_workers / 2):
d.addCallback(overlay.aws.term_worker(overlay.aws.workers[i]))
d.callback(0);
def do_host_count(self, request, params = {}):
"""
Because process_host_count above is meant to handle its
own finalization, do_host_count always returns NOT_DONE_YET. Its
much simpler to make a tiny hack then hack process_host count to straddle
to different scenarios ( return string and process itself )
"""
#if this a polling request and it's not the first one
if 'ts' in params and params['ts'] != 0:
#if the caller is up to date or from the future
if params['ts'] > self.myStore.lastChange:
#hold the connection open
d = Deferred()
def deferred_host_count(self, request):
request.write(self.process_host_count)
request.finish()
#and notify them when something changes
d.addCallback(self.deferred_host_count, request)
self.myStore.onChange.observe(d)
return NOT_DONE_YET
#If no TS or TS is out of date, process NOW
return self.process_host_count(request)
def test_full_run(self):
"""Verify a functional agent start via the 'run' method.
This test requires Zookeeper running on the default port of localhost.
The mocked portions are to prevent the daemon start from altering the
test environment (sys.stdout/sys.stderr, and reactor start).
"""
zookeeper.set_debug_level(0)
started = Deferred()
class DummyAgent(BaseAgent):
started = False
def start(self):
started.callback(self)
def validate_started(agent):
self.assertTrue(agent.client.connected)
started.addCallback(validate_started)
pid_file = self.makeFile()
self.change_args("es-agent", "--zookeeper-servers", get_test_zookeeper_address(), "--pidfile", pid_file)
runner = self.mocker.patch(AgentRunner)
logger = self.mocker.patch(AppLogger)
logger.start(MATCH_APP)
runner.startReactor(None, sys.stdout, sys.stderr)
logger.stop()
self.mocker.replay()
DummyAgent.run()
return started
def run(self, regionEnd):
regionStart = 9000
log.msg("[NAT] Check Start")
d = Deferred()
d.addCallback(self.allEstablished)
self.ports2 = None
if regionStart != regionEnd and regionEnd >= 9000:
self.ports2 = [8002, 8003, 8004, regionStart, regionEnd]
else:
self.ports2 = [8002, 8003, 8004, regionStart]
self.ports = []
self.processports = []
for port in self.ports2:
hasProcessRunning = False
for name in self.clientObject.processes:
process = self.clientObject.processes[name]
if process.consolePort == port + 10000:
hasProcessRunning = True
if process.consolePort == 18000:
if port == 8002 or port == 8003 or port == 8004:
hasProcessRunning = True
if not hasProcessRunning:
self.ports += [port]
else:
if port == 8002:
self.processports += [18000]
elif port != 8003 and port != 8004: # We only want ROBUST once.
self.processports += [port + 10000]
self.count = len(self.ports)
self.service.start(d, self.count, self.ports)
def send_command(self, command, expect='OK'):
self.log('Sending: %r' % (command,))
resp = Deferred()
resp.addCallback(self.debug)
self.deferreds.append((expect, resp))
self.sendLine(command)
return resp
def testChildResolve(self):
# I've seen problems with reactor.run under gtk2reactor. Spawn a
# child which just does reactor.resolve after the reactor has
# started, fail if it does not complete in a timely fashion.
helperPath = os.path.abspath(self.mktemp())
helperFile = open(helperPath, 'w')
# Eeueuuggg
reactorName = reactor.__module__
helperFile.write(resolve_helper % {'reactor': reactorName})
helperFile.close()
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
helperDeferred = Deferred()
helperProto = ChildResolveProtocol(helperDeferred)
reactor.spawnProcess(helperProto, sys.executable, ("python", "-u", helperPath), env)
def cbFinished((reason, output, error)):
# If the output is "done 127.0.0.1\n" we don't really care what
# else happened.
output = ''.join(output)
if output != 'done 127.0.0.1\n':
self.fail((
"The child process failed to produce the desired results:\n"
" Reason for termination was: %r\n"
" Output stream was: %r\n"
" Error stream was: %r\n") % (reason.getErrorMessage(), output, ''.join(error)))
helperDeferred.addCallback(cbFinished)
return helperDeferred
def start(self):
# We need to run 4 commands in a row, and each of them can fail
d = Deferred()
def chain(args, description, failMessage):
d.addCallback(self._spawnProcess, args, description, failMessage)
for serviceName in ['manager', 'worker']:
chain(["flumotion",
"-C", self._confDir,
"-L", self._logDir,
"-R", self._runDir,
"create", serviceName,
"admin", str(self._port)],
_('Creating %s ...') % serviceName,
_("Could not create %s." % serviceName))
chain(["flumotion",
"-C", self._confDir,
"-L", self._logDir,
"-R", self._runDir,
"start", serviceName, "admin"],
_('Starting %s ...' % serviceName),
_("Could not start %s." % serviceName))
d.addErrback(lambda f: None)
def done(result):
self._finished()
d.addCallback(done)
# start chain
d.callback(None)
return d
def write(res):
request.code = res.code
old_headers = request.responseHeaders
request.responseHeaders = res.headers
request.responseHeaders.setRawHeaders(
'content-encoding',
old_headers.getRawHeaders('content-encoding', []))
if not self._anonymous:
request.responseHeaders.addRawHeader("X-Proxied-By",
__version__.package + " " + __version__.base())
if request.isSecure() and host["sendhsts"]:
request.responseHeaders.setRawHeaders("Strict-Transport-Security",
["max-age=31536000"])
if self._clacks:
request.responseHeaders.addRawHeader("X-Clacks-Overhead",
"GNU Terry Pratchett")
for name, values in self._extraHeaders:
request.responseHeaders.setRawHeaders(name, values)
f = Deferred()
res.deliverBody(Downloader(f, request.write))
f.addCallback(lambda _: request.finish())
return f
def performRequest(self, command, *args, **kwargs):
"""
Select an available client and perform the given request on it.
@param command: A C{str} representing an attribute of
L{MemCacheProtocol}.
@parma args: Any positional arguments that should be passed to
C{command}.
@param kwargs: Any keyword arguments that should be passed to
C{command}.
@return: A L{Deferred} that fires with the result of the given command.
"""
if len(self._freeClients) > 0:
client = self._freeClients.pop()
d = self._performRequestOnClient(
client, command, *args, **kwargs)
elif len(self._busyClients) + self._pendingConnects >= self._maxClients:
d = Deferred()
self._commands.append((d, command, args, kwargs))
self.log_debug("Command queued: %s, %r, %r" % (
command, args, kwargs))
self._logClientStats()
else:
d = self._newClientConnection()
d.addCallback(self._performRequestOnClient,
command, *args, **kwargs)
return d
def cachedZipResource(self, request, name, url, segments):
archivePath = self.config.CachedResources.child("{0}.zip".format(name))
if archivePath.exists():
d = Deferred()
d.callback(None)
else:
d = http_download(archivePath, url)
def readFromArchive(_):
filePath = ZipArchive(archivePath.path)
for segment in segments:
filePath = filePath.child(segment)
return filePath.getContent()
def notFoundHandler(f):
f.trap(KeyError)
request.setResponseCode(http.NOT_FOUND)
set_response_header(
request, HeaderName.contentType, ContentType.plain
)
return b"Not found."
d.addCallback(readFromArchive)
d.addErrback(notFoundHandler)
return d
def test_noCompatibilityLayer(self):
"""
If no compatiblity layer is present, imports of gobject and friends
are disallowed.
We do this by running a process where we make sure gi.pygtkcompat
isn't present.
"""
from twisted.internet import reactor
if not IReactorProcess.providedBy(reactor):
raise SkipTest("No process support available in this reactor.")
result = Deferred()
class Stdout(ProcessProtocol):
data = b""
def errReceived(self, err):
print(err)
def outReceived(self, data):
self.data += data
def processExited(self, reason):
result.callback(self.data)
path = FilePath(__file__.encode("utf-8")).sibling(
b"process_gireactornocompat.py").path
reactor.spawnProcess(Stdout(), sys.executable, [sys.executable, path],
env=os.environ)
result.addCallback(self.assertEqual, b"success")
return result
def listening(port):
msg("Listening on %r" % (port.getHost(),))
endpoint = self.clientEndpoint(reactor, port.getHost())
lostConnectionDeferred = Deferred()
protocol = lambda: ClosingLaterProtocol(
lostConnectionDeferred)
client = endpoint.connect(factoryFor(protocol))
def write(proto):
msg("About to write to %r" % (proto,))
proto.transport.write('x')
client.addCallbacks(
write, lostConnectionDeferred.errback)
def disconnected(proto):
msg("%r disconnected" % (proto,))
proto.transport.write("some bytes to get lost")
proto.transport.writeSequence(["some", "more"])
finished.append(True)
lostConnectionDeferred.addCallback(disconnected)
serverConnectionLostDeferred.addCallback(disconnected)
return gatherResults([
lostConnectionDeferred,
serverConnectionLostDeferred])
def cbPostAuth(self, response, cookie):
#print 'cbPostAuth()'
finished = Deferred()
response.deliverBody(BeginningPrinter(finished))
finished.addCallback(self.cbPostAuthSuccess, cookie)
finished.addErrback(self.cbShutdown)
return finished
def test_json(self):
deferred = Deferred()
deferred.addCallback(lambda _: self.callbacks.add("success"))
deferred.addErrback(lambda _: self.callbacks.add("failure"))
twisted_headers = Headers({"Content-Type": ["application/json"]})
twisted_response = TWResponse(
("HTTP", 1, 1), 200, "OK", twisted_headers, None)
request = Request(
method="GET", url="/",
kwargs={
"headers": {
"Content-Type": "application/json"}, "data": None})
data = {
os.urandom(6).encode("hex"): os.urandom(6).encode("hex"),
os.urandom(6).encode("hex"): os.urandom(6).encode("hex")
}
r = Response(deferred, twisted_response, request)
map(r.dataReceived, json.dumps(data))
r.connectionLost(responseDone)
self.assertEqual(self.callbacks, set(["success"]))
self.assertTrue(r._done)
self.assertEqual(r.json(), data)
return deferred
def get_response(cmd_data):
for cmd in cmd_data :
def __command_done(result, command) :
res = result
if not isinstance(result, CommandResult) :
res = CommandResult(status = result)
self.factory.next_state(ProtocolState.Report, ReportData(command, res), self.transport.connector)
ok_func = partial(__command_done, command = cmd)
def __command_error(reason, command):
res = CommandResult('Failed', str(reason))
self.factory.next_state(ProtocolState.Report, ReportData(command, res), self.transport.connector)
err_func = partial(__command_error, command = cmd)
# Obtain only new commands next time
if self.factory.timestamp is not None :
self.factory.timestamp = max(self.factory.timestamp, self._parse_date(cmd['timestamp']))
else :
self.factory.timestamp = self._parse_date(cmd['timestamp'])
# DeviceDelegate has to use this deferred object to notify us that command processing finished.
cmd_defer = Deferred()
cmd_defer.addCallback(ok_func)
cmd_defer.addErrback(err_func)
# Actual run of command
try :
self.factory.device_delegate.do_command(cmd, cmd_defer)
except Exception, err :
log.err('Failed to execute device-delegate do_command. Reason: <{0}>.'.format(err))
err_func(err)
def outer_callback_1(res):
print 'In outer_callback_1'
d = Deferred()
d.addCallback(inner_callback_1)
d.addCallback(inner_callback_2)
d.callback(0)
return d
def cbRequest(response):
finished = Deferred()
finished.addCallback(cb)
finished.addErrback(err)
response.deliverBody(BufferedReception(finished))
cli.inflight -= 1
return finished
def callRemote(self, _name, *args, **kw):
""" Make a call to the RemoteReference and return the result as a
Deferred. It exists to allow queuing of calls to remote reference
before the remote reference has arrived.
For more information refer to twisted.spread.pb.RemoteReference.
@param _name: Name of the method which should be called.
The prefix 'remote_' will be added to the name
in the remote object to select the method which
should be called.
@type _name: str
@param *args: Positional arguments which will be passed to
the remote method.
@param **kw: Keyworded arguments which will be passed to the
remote method.
@return: Deferred which will fire with the result of the
call or a Failure if there was a problem.
@rtype: twisted.internet.defer.Deferred
"""
if self.__failure is not None:
d = fail(self.__failure)
elif self.__pending is not None:
d = Deferred()
self.__pending.append(d)
else:
d = succeed(self.__obj)
d.addCallback(lambda ref: ref.callRemote(_name, *args, **kw))
d.addErrback(self.__filter, _name)
return d
def test_disorderlyShutdown(self):
"""
If a L{TLSMemoryBIOProtocol} loses its connection unexpectedly, this is
reported to the application.
"""
clientConnectionLost = Deferred()
clientFactory = ClientFactory()
clientFactory.protocol = (
lambda: ConnectionLostNotifyingProtocol(
clientConnectionLost))
clientContextFactory = HandshakeCallbackContextFactory()
wrapperFactory = TLSMemoryBIOFactory(
clientContextFactory, True, clientFactory)
sslClientProtocol = wrapperFactory.buildProtocol(None)
# Client speaks first, so the server can be dumb.
serverProtocol = Protocol()
connectionDeferred = loopbackAsync(serverProtocol, sslClientProtocol)
# Now destroy the connection.
serverProtocol.transport.loseConnection()
# And when the connection completely dies, check the reason.
def cbDisconnected(clientProtocol):
clientProtocol.lostConnectionReason.trap(Error)
clientConnectionLost.addCallback(cbDisconnected)
return clientConnectionLost
def cbGetFirstCookie(self, response):
onet_ubi_cookie = response.headers.getRawHeaders("Set-Cookie")[0]
onetzuo_ticket_cookie = response.headers.getRawHeaders("Set-Cookie")[1]
onet_cid_cookie = response.headers.getRawHeaders("Set-Cookie")[2]
# onet_sgn_cookie = response.headers.getRawHeaders('Set-Cookie')[3]
onet_ubi_match = re.search("onet_ubi=(.*?);", onet_ubi_cookie)
if onet_ubi_match:
onet_ubi_result = onet_ubi_match.group()
else:
onet_ubi_result = None
onetzuo_ticket_match = re.search("onetzuo_ticket=(.*?);", onetzuo_ticket_cookie)
if onetzuo_ticket_match:
onetzuo_ticket_result = onetzuo_ticket_match.group()
else:
onetzuo_ticket_result = None
onet_cid_match = re.search("onet_cid=(.*?);", onet_cid_cookie)
if onet_cid_match:
onet_cid_result = onet_cid_match.group()
else:
onet_cid_result = None
if onet_ubi_result != None and onetzuo_ticket_result != None and onet_cid_result != None:
finished = Deferred()
response.deliverBody(BeginningPrinter(finished))
finished.addCallback(self.cbGetFirstCookieSuccess, onet_ubi_result, onetzuo_ticket_result, onet_cid_result)
finished.addErrback(self.cbShutdown)
return finished
else:
self.authorise()
def test_singleThread(self):
"""
The submission of a new job to a thread pool in response to the
C{onResult} callback does not cause a new thread to be added to the
thread pool.
This requires that the thread which calls C{onResult} to have first
marked itself as available so that when the new job is queued, that
thread may be considered to run it. This is desirable so that when
only N jobs are ever being executed in the thread pool at once only
N threads will ever be created.
"""
# Ensure no threads running
self.assertEquals(self.threadpool.workers, 0)
loopDeferred = Deferred()
def onResult(success, counter):
reactor.callFromThread(submit, counter)
def submit(counter):
if counter:
self.threadpool.callInThreadWithCallback(
onResult, lambda: counter - 1)
else:
loopDeferred.callback(None)
def cbLoop(ignored):
# Ensure there is only one thread running.
self.assertEqual(self.threadpool.workers, 1)
loopDeferred.addCallback(cbLoop)
submit(10)
return loopDeferred
def test_processExitedWithSignal(self):
"""
The C{reason} argument passed to L{IProcessProtocol.processExited} is a
L{ProcessTerminated} instance if the child process exits with a signal.
"""
sigName = 'TERM'
sigNum = getattr(signal, 'SIG' + sigName)
exited = Deferred()
source = (
"import sys\n"
# Talk so the parent process knows the process is running. This is
# necessary because ProcessProtocol.makeConnection may be called
# before this process is exec'd. It would be unfortunate if we
# SIGTERM'd the Twisted process while it was on its way to doing
# the exec.
"sys.stdout.write('x')\n"
"sys.stdout.flush()\n"
"sys.stdin.read()\n")
class Exiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
self.transport.signalProcess(sigName)
def childConnectionLost(self, fd):
msg('childConnectionLost(%d)' % (fd, ))
def processExited(self, reason):
msg('processExited(%r)' % (reason, ))
# Protect the Deferred from the failure so that it follows
# the callback chain. This doesn't use the errback chain
# because it wants to make sure reason is a Failure. An
# Exception would also make an errback-based test pass, and
# that would be wrong.
exited.callback([reason])
def processEnded(self, reason):
msg('processEnded(%r)' % (reason, ))
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Exiter(),
sys.executable, [sys.executable, "-c", source],
usePTY=self.usePTY)
def cbExited((failure, )):
# Trapping implicitly verifies that it's a Failure (rather than
# an exception) and explicitly makes sure it's the right type.
failure.trap(ProcessTerminated)
err = failure.value
if platform.isWindows():
# Windows can't really /have/ signals, so it certainly can't
# report them as the reason for termination. Maybe there's
# something better we could be doing here, anyway? Hard to
# say. Anyway, this inconsistency between different platforms
# is extremely unfortunate and I would remove it if I
# could. -exarkun
self.assertIdentical(err.signal, None)
self.assertEqual(err.exitCode, 1)
else:
self.assertEqual(err.signal, sigNum)
self.assertIdentical(err.exitCode, None)
exited.addCallback(cbExited)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def sendmail(authentication_username, authentication_password, from_address,
to_address, message_file, smtp_host, smtp_port, security, event=None):
"""
Sends an email using SMTPS/SMTP+TLS and torify the connection
@param authentication_username: account username
@param authentication_password: account password
@param from_address: the from address field of the email
@param to_address: the to address field of the email
@param message_file: the message content its a StringIO
@param smtp_host: the smtp host
@param smtp_port: the smtp port
@param security: may need to be STRING, here is converted at start
@param event: the event description, needed to keep track of failure/success
"""
notif_retries = 2
notif_timeout = 10
def printError(method, reason, event):
if event:
log.err("** failed notification within event %s" % event.type)
if isinstance(reason, Failure):
log.err("Failed to connect to %s:%d (Sock Error: %s) (Method: %s)" %
(smtp_host, smtp_port, reason.value, method))
log.debug(reason)
def esmtp_errback(reason, *args, **kwargs):
printError("ESMTP", reason, event)
return result_deferred.errback(reason)
def socks_errback(reason, *args, **kwargs):
printError("SOCKS5", reason, event)
return result_deferred.errback(reason)
def tcp4_errback(reason, *args, **kwargs):
printError("TCP4", reason, event)
return result_deferred.errback(reason)
def result_errback(reason, *args, **kwargs):
"""To not report an error as unexpected in the log files"""
return True
def esmtp_sendError(self, exc):
if exc.code and exc.resp:
error_str = ""
error = re.match(r'^([0-9\.]+) ', exc.resp)
if error:
key = str(exc.code) + " " + error.group(1)
if key in smtp_errors:
error_str += " " + smtp_errors[key]
verb = '[unknown]'
if 'authentication' in exc.resp:
verb = 'autenticate'
if 'not support secure' in exc.resp:
verb = 'negotiate TLS'
log.err("Failed to %s to %s:%d (SMTP Code: %.3d) (%s)" %
(verb, smtp_host, smtp_port, exc.code, error_str))
SMTPClient.sendError(self, exc)
def esmtp_connectionLost(self, reason=protocol.connectionDone):
"""We are no longer connected"""
if isinstance(reason, Failure):
if not isinstance(reason.value, error.ConnectionDone):
verb = 'unknown_verb'
if 'OpenSSL' in str(reason.type):
verb = 'negotiate SSL'
log.err("Failed to %s to %s:%d (%s)"
% (verb, smtp_host, smtp_port, reason.type))
log.debug(reason)
self.setTimeout(None)
self.mailFile = None
# TODO: validation?
if from_address == '' or to_address == '':
log.err("Failed to init sendmail to %s:%s (Invalid from/to addresses)" %
(from_address, to_address))
return
if security != "SSL" and security != "disabled":
requireTransportSecurity = True
else:
requireTransportSecurity = False
try:
security = str(security)
result_deferred = Deferred()
result_deferred.addErrback(result_errback, event)
context_factory = ClientContextFactory()
# evilaliv3:
# this is the same solution I applied to tor2web:
# as discussed on https://trac.torproject.org/projects/tor/ticket/11598
# there is no way of enabling all TLS methods excluding SSL.
# the problem lies in the fact that SSL.TLSv1_METHOD | SSL.TLSv1_1_METHOD | SSL.TLSv1_2_METHOD
# is denied by OpenSSL.
#
# As spotted by nickm the only good solution right now is to enable SSL.SSLv23_METHOD then explicitly
# use options: SSL_OP_NO_SSLv2 and SSL_OP_NO_SSLv3
#
# This trick make openssl consider valid all TLS methods.
#
context_factory.method = SSL.SSLv23_METHOD
esmtp_deferred = Deferred()
esmtp_deferred.addErrback(esmtp_errback, event)
esmtp_deferred.addCallback(result_deferred.callback)
except Exception as excep:
log.err("Error in Twisted objects init - unexpected exception in sendmail: %s" % str(excep))
return fail()
try:
factory = ESMTPSenderFactory(
authentication_username,
authentication_password,
from_address,
to_address,
message_file,
esmtp_deferred,
contextFactory=context_factory,
requireAuthentication=(authentication_username and authentication_password),
requireTransportSecurity=requireTransportSecurity,
retries=notif_retries,
timeout=notif_timeout)
factory.protocol.sendError = esmtp_sendError
factory.protocol.connectionLost = esmtp_connectionLost
if security == "SSL":
factory = tls.TLSMemoryBIOFactory(context_factory, True, factory)
except Exception as excep:
log.err("Error in factory init - unexpected exception in sendmail: %s" % str(excep))
return fail()
try:
if not GLSettings.disable_mail_torification and GLSettings.memory_copy.notif_uses_tor:
socksProxy = TCP4ClientEndpoint(reactor, GLSettings.socks_host, GLSettings.socks_port, timeout=notif_timeout)
endpoint = SOCKS5ClientEndpoint(smtp_host.encode('utf-8'), smtp_port, socksProxy)
d = endpoint.connect(factory)
d.addErrback(socks_errback, event)
else:
endpoint = TCP4ClientEndpoint(reactor, smtp_host, smtp_port, timeout=notif_timeout)
d = endpoint.connect(factory)
d.addErrback(tcp4_errback, event)
except Exception as excep:
# we strongly need to avoid raising exception inside email logic to avoid chained errors
log.err("unexpected exception in sendmail: %s" % str(excep))
return fail()
return result_deferred
class ServiceProtocol(ProcessProtocol):
"""Start and stop a background service process.
This :class:`~twisted.internet.protocol.ProcessProtocol` manages the start
up and termination phases of a background service process. The process is
considered 'running' when it has stayed up for at least 0.1 seconds (or any
other non default value which `minUptime` is set too), and optionally when
it has emitted a certain string and/or it has started listening to a
certain port.
"""
#: The service process must stay up at least this amount of seconds, before
#: it's considered running. This allows to catch common issues like the
#: service process executable not being in PATH or not being executable.
minUptime = 0.2
def __init__(self, reactor, parser=None, timeout=TIMEOUT):
self.reactor = reactor
self.parser = parser or ServiceOutputParser()
#: Maximum amount of seconds to wait for the service to be ready. After
#: that, the 'ready' deferred will errback with a TimeoutError.
self.timeout = timeout
#: Optional text that we expect the process to emit in standard output
#: before we consider it ready.
self.expectedOutput = None
#: Optional port number that we expect the service process to listen,
#: before we consider it ready.
self.expectedPort = None
#: Deferred that will fire when the service is considered ready, i.e.
#: it has stayed up for at least minUptime seconds, has produced the
#: expected output (if any), and is listening to the expected port (if
#: any). Upon cancellation, any waiting activity will be stopped.
self.ready = Deferred(lambda _: self._stopWaitingForReady())
#: Deferred that will fire when the service has fully terminated, i.e.
#: it has exited and we parent process have read any outstanding data
#: in the pipes and have closed them.
self.terminated = Deferred()
# Delayed call that gets started right after the process has been
# spawned. Its purpose is to make the protocol "sleep" for a minUptime
# seconds (typically 0.1 seconds): if the process exits before this
# little time has elapsed, an error gets raised.
self._minUptimeCall = None
# Deferred that will be fired when the process emits the expected
# output (if any).
self._expectedOutputReady = Deferred()
# A LoopingCall instance that will periodically try to open the port
# that the process is supposed to start listening to.
self._probePortLoop = None
# A connector as returned by TCP4ClientEndpoint.connect() that can be
# used to abort an ongoing connection attempt as performed by the
# port probe loop.
self._probePortAttempt = None
def connectionMade(self):
# Called (indirectly) by `spawnProcess` after the `os.fork` call has
# succeeded.
logging.info("Service process spawned")
# XXX Replace with self.ready.addTimeout once support for Twisted<16.5
# gets dropped.
addTimeout(self.ready, self.timeout, self.reactor)
# The twisted.protocols.basic.LineOnlyReceiver class expects to know
# when the transport is disconnecting.
self.disconnecting = False
# Let's see if the process stays running for at least
self._minUptimeCall = self.reactor.callLater(self.minUptime,
self._minUptimeElapsed)
if self.expectedOutput:
# From this point on, be prepared to receive the expected output at
# any time.
self.parser.whenLineContains(self.expectedOutput,
self._expectedOutputReceived)
else:
# There's no output we expect, so we fire this Deferred right away.
# When _minUptimeElapsed will be called, the callback that gets
# attached to this Deferred will fire synchronously.
self._expectedOutputReady.callback(None)
self.parser.makeConnection(self)
def outReceived(self, data):
# Called when we receive data from the standard output of the service.
self.parser.dataReceived(data)
errReceived = outReceived
def processExited(self, reason):
# Called when the service process exited.
logging.info("Service process exited: %s", reason.getErrorMessage())
# If we did not reach the 'ready' state yet, the let's fire the 'ready'
# Deferred with an error.
if not self.ready.called:
self._stopWaitingForReady(reason)
def processEnded(self, reason):
# Called when the process has been reaped.
logging.info("Service process reaped")
self.terminated.callback(None)
def _minUptimeElapsed(self):
"""
Called if the process didn't exit in the first `minUptime` seconds
after having been spawned.
"""
logging.info("Service process alive for %.1f seconds", self.minUptime)
# Now wait for the expected output and then start polling the port
# we expect the service to listen to (if there's no expected output
# and/or no expected port, these deferreds will fire synchronously).
if self.expectedPort:
self._expectedOutputReady.addCallback(self._startProbePortLoop)
self._expectedOutputReady.addCallback(self._maybeFireReady)
def _expectedOutputReceived(self):
"""
Called after `_minUptimeElapsed` and the service process has emitted
the expected output string.
"""
# Let's fire the relevant deferred, so we can move forward to polling
# the expected port, or declaring the service as ready (if there's no
# expected port).
logging.info("Service process emitted '%s'", self.expectedOutput)
self._expectedOutputReady.callback(None)
@inlineCallbacks
def _startProbePortLoop(self, _):
"""
Called when the service process has stayed up for at least `minUptime`
seconds and it has emitted the expected output string (or there was no
expected output string at all).
"""
self._probePortLoop = LoopingCall(self._probePort)
self._probePortLoop.clock = self.reactor
# The LoopingCall.start() method returns a deferred that will fire
# when the loop stops, i.e. when we successfully probe the port.
yield self._probePortLoop.start(0.1)
@inlineCallbacks
def _probePort(self):
"""Perform a single attempt to connect to the expected port.
If the probe succeeds the probe loop will be stoped.
If the probe fails with a connection error, we'll just return
gracefully (we'll be invoked again at the next loop iteration).
"""
logging.info("Polling service port '%s'", self.expectedPort)
endpoint = TCP4ClientEndpoint(self.reactor, "localhost",
self.expectedPort)
try:
factory = Factory()
factory.protocol = Protocol
self._probePortAttempt = endpoint.connect(factory)
yield self._probePortAttempt
except ConnectionRefusedError as error:
logging.info("Service port probe failed: %s", error)
except ConnectingCancelledError as error:
# This happens if _stopWaitingForReady gets called while we are
# waiting for the enpoint connect() to succeed or fail.
logging.info("Service port probe cancelled: %s", error)
else:
if self._probePortLoop.running:
self._probePortLoop.stop()
logging.info("Service opened port %d", self.expectedPort)
finally:
self._probePortAttempt = None
def _maybeFireReady(self, result):
"""Fire the 'ready' deferred, unless we're aborting the startup.
If the startup sequence is aborting (either because the `ready`
deferred was cancelled by user code, or because the process died and
`processExited` was called), this will just be a no-op, as we rely
on the aborting code to errback the `ready` deferred.
"""
if not self.disconnecting:
logging.info("Service process ready")
self.ready.callback(result)
def _stopWaitingForReady(self, reason=None):
"""
Stop any delayed call or activity associated with the initial waiting
for the service to be ready.
If `reason` is passed, the `ready` deferred will errback with the given
failure.
"""
# This will prevent the ServiceOutputParser protocol from firing any
# further lineReceived event, so we don't fire _expectedOutputReady.
#
# It will also prevent _maybeFireReady from firing the 'ready'
# deferred, since we want to do it ourselves with the given reason (if
# any).
self.disconnecting = True
message = None
if self._minUptimeCall.active():
self._minUptimeCall.cancel()
message = "minimum uptime not yet elapsed"
elif self.expectedOutput and not self._expectedOutputReady.called:
message = "expected output not yet received"
elif self.expectedPort:
if self._probePortAttempt:
self._probePortAttempt.cancel()
self._probePortLoop.stop()
message = "expected port not yet open"
# We can safely assume that one of the conditions above is holding,
# because otherwise the 'ready' deferred would have already fired. In
# any case let's put an explicit assertion here for good measure.
assert message, "Unexpected protocol state while cancelling wait"
logging.info("Give up waiting for the service to be ready: %s",
message)
if reason:
self.ready.callback(reason)
def renderHTTP(self, ctx):
self.request = inevow.IRequest(ctx)
self.title_append_item = ""
self.search_segment = ""
# prep the glob with default info
glob = {
'include_comment_total': 1,
'order_by': 'date_uploaded',
'order_dir': 'desc',
'tag_list': True
}
self.search_items = []
self.user_info = {}
# ensure we have the correct feed types passed
if self.segments[0] == "rss" or self.segments[
0] == "atom" or self.segments[0] == "kml":
if len(self.segments) > 1 and self.segments[1]:
# parse out the search terms
self.search_items = self.segments[1].split(".")
if len(self.search_items) != 2:
# ummm, no - we have no idea what you are doing
return '<meta http-equiv="refresh" content="0;url=http://www.%s/not_found/">' % self.domain
else:
if self.search_items[0] == "SSQ":
# search query
glob['simple_search_query'] = self.search_items[1]
self.title_append_item = " - searching on '%s'" % self.search_items[
1]
self.search_segment = self.segments[1]
elif self.search_items[0] == "TUN":
# tag query - we'll need to update this when we get advanced search (intersections/unions on tags)
glob['tag_union'] = [self.search_items[1]]
self.title_append_item = " - tagged with '%s'" % self.search_items[
1]
self.search_segment = self.segments[1]
elif self.search_items[0] == "ALB":
# album query - we need to update this to support album names - but whatever - it's release time!!!
glob['album_id'] = int(self.search_items[1])
self.title_append_item = ""
self.search_segment = self.segments[1]
else:
# again, we have no idea what you are doing
return '<meta http-equiv="refresh" content="0;url=http://www.%s/not_found/">' % self.domain
else:
# unsupported feed type
return '<meta http-equiv="refresh" content="0;url=http://www.%s/not_found/">' % self.domain
def act(globber_result):
# grab some user info
d2 = self.app.api.users.get_user_id(self.username)
d2.addCallback(handle_user_id, globber_result)
return d2
def handle_user_id(result):
if result[0] != 0:
return "Bad username"
self.userid = result[1]
if self.userid:
d3 = self.app.api.users.get_info(self.userid, None)
else:
d3 = Deferred()
d3.callback((0, {}))
d3.addCallback(handle_user_info)
return d3
@stack
def handle_user_info(result):
if result[0] != 0:
return "NO USER INFO"
self.user_info = result[1]
limit = 20 # flickr does 20, so we do 20
offset = 0 # shouldn't ever be anything else
d4 = self.app.api.globber.get_images(self.userid, None, glob,
limit, offset)
d4.addCallback(handle_globber_result)
return d4
@stack
def handle_globber_result(result):
if result[0] != 0:
return "NO PHOTOS"
self.photo_list = result[1]
if self.userid:
if len(self.search_items) and self.search_items[0] == "ALB":
self.log.warning("we have an album search item: %s" %
self.search_items[1])
d5 = self.app.api.albums.get_info(
None, int(self.search_items[1]))
else:
self.log.warning("no album search item")
d5 = Deferred()
d5.callback((0, None))
else:
d5 = Deferred()
d5.callback((0, None))
d5.addCallback(handle_album_info)
return d5
@stack
def handle_album_info(result):
if result[0] != 0:
return "NO ALBUMS"
self.album_info = result[1]
if not self.userid:
self.user_info['last_login'] = datetime.datetime.now()
self.user_info['date_created'] = datetime.datetime.now()
globber_arg = (0, self.photo_list)
album_arg = (0, self.album_info)
if self.segments[0] == "rss":
return self._build_rss2_feed(self.user_info, globber_arg,
album_arg)
elif self.segments[0] == "atom":
return self._build_atom_feed(self.user_info, globber_arg,
album_arg)
elif self.segments[0] == "kml":
return self._build_kml_feed(self.user_info, globber_arg,
album_arg)
if self.username == "*ALL*":
d = Deferred()
d.callback((0, None))
else:
d = self.app.api.users.get_user_id(self.username)
d.addCallback(handle_user_id)
return d
class MultipageFetchSession(object):
"""
a page_callback(items,progress) function can be passed.
Two Issues are adressed:
* Ensure that a complete dataset is fetched --> Constantly query page 1 and check for changes, if change occures, reissue the last burst.
* End after no new items occur --> ProcessPage-callback returns true if no more data is needed
"""
_next_session = 0
def __init__(self,api,cmd,page_callback=lambda x,y:False,complete=False,**kwargs):
MultipageFetchSession._next_session += 1
self.sessionID = MultipageFetchSession._next_session
self.deferred = Deferred()
# self.deferred.addErrback(self.Errback,{"error":"global error","cmd":cmd,"sid":self.sessionID})
# Basic config
self.api = api
self.cmd = cmd
self.complete = complete # Ensure completeness of data that is fetched by constantly checking for new data
self.params = kwargs
# derrived config
self.bsize = int(max(3,(api.max_seen - 12)/3)) # Calculate number of sizes of the burst
print "Session %d"%self.sessionID,self.cmd,self.params,"burstsize:",self.bsize
# session-tracking
self.pages_fetched = {} # page -> number of times fetched
self.pages_pending = {} # page -> deferred dict
self.revert = False
# Data Tracking
self.items = {}
# Per Page callback
self.page_callback = page_callback # per Page-results callback, if one returns True in a burst, finish fetching!
self.burst_pages = {} # dict of page --> finished [true/false]
# Start First call
self.FetchPage(1,unique=True)
def AddCallback(self,callback):
self.deferred.addCallback(callback)
def FetchPage(self,page,unique = False):
"""Issue fetching a single page"""
deferred = self.api.APIRequest(self.cmd,page=page,unique=unique,**self.params).addErrback(self.DErrPage,page=page)
deferred.addCallback(self.DGetPage,page=page)
self.pages_pending[page] = deferred
def Errback(self,result):
print "Errback",result,page
return
def DErrPage(self,page):
self.revert = True
def DGetPage(self,result,page=0):
code = result.pop("code",None)
if code == None:
print "DGetPage has no code",result.keys()
phrase = result.pop("phrase","")
errors = result.pop("errors",None)
pages = result.pop("page",None)
# Handle page
try:
if pages == None:
self.pages_pending = {}
self.burst_pages = {}
self.revert = True
else:
self.pages_pending.pop(page,None)
fpage = self.pages_fetched.get(page,None)
if fpage == None:
self.pages_fetched[page] = 0
self.pages_fetched[page] += 1
print "\nSuccess",code,phrase,page,pages,result.get("call")
# print result.keys()
# print self.pages_fetched,self.pages_pending
# Process Results (Check if callback want's more data, register items, count unknowns)
p = {"fetched":self.pages_fetched,"pages":pages,"items":len(self.items)} # 'Progress'
pitems = self.ProcessPageResults(result) # Returns a dict of hash --> item for the current page
pfinished = self.page_callback(result,p) # Feed the dictified result to the callback, which decides if it want's more pages
self.burst_pages[page] = pfinished # Register finishing request for this burst
unknowns = self.RegisterPageResults(pitems) # Store results, get number of previously unknown items
if pfinished == False: # If the callback want's more data, check if page 1 indicates changed data
if page == 1:
if unknowns > 0:
self.revert = True
# Issue new Fetches
if len(self.pages_pending) == 0 :
fpages = []
# If revert = True, refetch last burst
if self.revert == True and self.complete == True:
fpages = self.burst_pages.keys() # Redo last fetch
if 1 not in fpages: # Ensure page 1 is present
fpages.append(1)
else:
maxfetched,lastpage = max(self.pages_fetched.keys()),pages["last"] # sets Range to be fetched
self.burst_pages.pop(1,None) # Don't care if page 1 returned "finished" from callback
finished = False
for pf in self.burst_pages.values():
if pf == True:
finished = True
if finished != True and maxfetched < lastpage:
fpages = range(maxfetched+1,min(maxfetched+1+self.bsize,lastpage+1))
if self.complete == True and 1 not in fpages:
fpages.append(1)
else:
self.deferred.callback(self.items)
print "Finished",self.pages_fetched
# Reset for new Burst
self.burst_pages = {} # Reset Burst
self.revert = False
# Issue new Burst
for p in fpages:
if p == 1:
self.FetchPage(p,unique=True)
else:
self.FetchPage(p)
except Exception as e:
import sys, os, traceback
print "DGetPage had an error"
exc_type, exc_obj, exc_tb = sys.exc_info()
# fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print exc_type, exc_tb.tb_lineno, exc_tb
traceback.print_tb(exc_tb)
self.deferred.errback(e)
def ProcessPageResults(self,result):
print "Implement ProcessPageResults"
def RegisterPageResults(self,items):
unknowns = 0
for h,item in items.items():
oitem = self.items.get(h,None)
if oitem == None:
unknowns += 1
self.items[h] = item
return unknowns
def callback_1(res):
print 'release', res
global deferred_1 # never do this in a real program
deferred_1 = Deferred()
return deferred_1
def callback_2(res):
print 'prepare release setup for', res
global deferred_2 # never do this in a real program
deferred_2 = Deferred()
return deferred_2
def callback_3(res):
print 'start release process for', res
return 3
d = Deferred()
d.addCallback(callback_1)
d.addCallback(callback_2)
d.addCallback(callback_3)
tag = 'ncs64-nma-cmn-CHN-r0'
d.callback(tag)
deferred_1.callback(tag)
deferred_2.callback(tag)
def render_PUT(self, request):
d = Deferred()
d.addCallback(self.handle_PUT)
d.callback(request)
return NOT_DONE_YET
def _callback(request, **kwargs):
d = Deferred()
d.addCallback(_throw)
self.reactor.callLater(1, d.callback, True)
return make_deferred_yieldable(d)
class Issue(Entity):
idCounter = 0
resolved = property(lambda self: self.deferred.called)
resolution = None
assignedTo = None
#serializable = True (not sure about this yet)
def __init__(self, description):
self.description = description
self.context = IssueContext(self)
self.deferred = Deferred()
# Get the next unique Issue ID
while Issue.byID(Issue.idCounter):
Issue.idCounter += 1
self.id = Issue.idCounter
Issue.idCounter += 1 #avoids duplicate id's caused by issues that go away quickly
def __getstate__(self):
return {'description': self.description, 'context': self.context.data}
@staticmethod
def construct(state):
return Issue(state['description']).withContext(state['context'])
@staticmethod
def byID(id):
for issue in Issue.objects:
if hasattr(issue, 'id'): # self won't have an id during __init__
if issue.id == id:
return issue
def withContext(self, context):
self.context.update(context)
return self
def whenResolved(self, callback):
self.deferred.addCallback(callback)
return self
def resolve(self, resolution=None):
if self.resolved:
return
else:
Issue.delete(self)
self.resolution = resolution
self.deferred.callback(self)
def assign(self, who):
if isinstance(who, str):
if Team.exists(who):
who = Team(who)
elif Conversation.byName(
who): #byName handles short usernames as well as full JIDs
who = SupportAgent(Conversation.byName(who).buddy)
else:
raise ValueError(
"I don't know any teams or people named \"%s\"" % who)
if not isinstance(who, (Team, SupportAgent)):
raise TypeError(
"You can only assign issues to Teams or SupportAgents")
self.assignedTo = who
if isinstance(who, SupportAgent) and who.available:
who.engage(self)
class CallFromThreadTests(unittest.TestCase):
"""
Task scheduling from threads tests.
"""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "Nothing to test without thread support"
def setUp(self):
self.counter = 0
self.deferred = Deferred()
def schedule(self, *args, **kwargs):
"""
Override in subclasses.
"""
reactor.callFromThread(*args, **kwargs)
def test_lotsOfThreadsAreScheduledCorrectly(self):
"""
L{IReactorThreads.callFromThread} can be used to schedule a large
number of calls in the reactor thread.
"""
def addAndMaybeFinish():
self.counter += 1
if self.counter == 100:
self.deferred.callback(True)
for i in range(100):
self.schedule(addAndMaybeFinish)
return self.deferred
def test_threadsAreRunInScheduledOrder(self):
"""
Callbacks should be invoked in the order they were scheduled.
"""
order = []
def check(_):
self.assertEqual(order, [1, 2, 3])
self.deferred.addCallback(check)
self.schedule(order.append, 1)
self.schedule(order.append, 2)
self.schedule(order.append, 3)
self.schedule(reactor.callFromThread, self.deferred.callback, None)
return self.deferred
def test_scheduledThreadsNotRunUntilReactorRuns(self):
"""
Scheduled tasks should not be run until the reactor starts running.
"""
def incAndFinish():
self.counter = 1
self.deferred.callback(True)
self.schedule(incAndFinish)
# Callback shouldn't have fired yet.
self.assertEqual(self.counter, 0)
return self.deferred
class CarbonClientFactory(ReconnectingClientFactory):
maxDelay = 5
def __init__(self, destination):
self.destination = destination
self.destinationName = ('%s:%d:%s' % destination).replace('.', '_')
self.host, self.port, self.carbon_instance = destination
self.addr = (self.host, self.port)
self.started = False
# This factory maintains protocol state across reconnects
self.queue = [] # including datapoints that still need to be sent
self.connectedProtocol = None
self.queueEmpty = Deferred()
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
self.connectFailed = Deferred()
self.connectionMade = Deferred()
self.connectionLost = Deferred()
# Define internal metric names
self.attemptedRelays = 'destinations.%s.attemptedRelays' % self.destinationName
self.fullQueueDrops = 'destinations.%s.fullQueueDrops' % self.destinationName
self.queuedUntilConnected = 'destinations.%s.queuedUntilConnected' % self.destinationName
def queueFullCallback(self, result):
state.events.cacheFull()
log.clients('%s send queue is full (%d datapoints)' % (self, result))
def queueSpaceCallback(self, result):
if self.queueFull.called:
log.clients('%s send queue has space available' %
self.connectedProtocol)
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
state.events.cacheSpaceAvailable()
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
def buildProtocol(self, addr):
self.connectedProtocol = CarbonClientProtocol()
self.connectedProtocol.factory = self
return self.connectedProtocol
def startConnecting(
self): # calling this startFactory yields recursion problems
self.started = True
self.connector = reactor.connectTCP(self.host, self.port, self)
def stopConnecting(self):
self.started = False
self.stopTrying()
if self.connectedProtocol and self.connectedProtocol.connected:
return self.connectedProtocol.disconnect()
@property
def queueSize(self):
return len(self.queue)
def hasQueuedDatapoints(self):
return bool(self.queue)
def takeSomeFromQueue(self):
datapoints = self.queue[:settings.MAX_DATAPOINTS_PER_MESSAGE]
self.queue = self.queue[settings.MAX_DATAPOINTS_PER_MESSAGE:]
return datapoints
def checkQueue(self):
if not self.queue:
self.queueEmpty.callback(0)
self.queueEmpty = Deferred()
def enqueue(self, metric, datapoint):
self.queue.append((metric, datapoint))
def sendDatapoint(self, metric, datapoint):
instrumentation.increment(self.attemptedRelays)
queueSize = self.queueSize
if queueSize >= settings.MAX_QUEUE_SIZE:
if not self.queueFull.called:
self.queueFull.callback(queueSize)
instrumentation.increment(self.fullQueueDrops)
elif self.connectedProtocol:
self.connectedProtocol.sendDatapoint(metric, datapoint)
else:
self.enqueue(metric, datapoint)
instrumentation.increment(self.queuedUntilConnected)
def startedConnecting(self, connector):
log.clients("%s::startedConnecting (%s:%d)" %
(self, connector.host, connector.port))
def clientConnectionLost(self, connector, reason):
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
log.clients(
"%s::clientConnectionLost (%s:%d) %s" %
(self, connector.host, connector.port, reason.getErrorMessage()))
self.connectedProtocol = None
self.connectionLost.callback(0)
self.connectionLost = Deferred()
def clientConnectionFailed(self, connector, reason):
ReconnectingClientFactory.clientConnectionFailed(
self, connector, reason)
log.clients(
"%s::clientConnectionFailed (%s:%d) %s" %
(self, connector.host, connector.port, reason.getErrorMessage()))
self.connectFailed.callback(dict(connector=connector, reason=reason))
self.connectFailed = Deferred()
def disconnect(self):
self.queueEmpty.addCallback(lambda result: self.stopConnecting())
readyToStop = DeferredList([self.connectionLost, self.connectFailed],
fireOnOneCallback=True,
fireOnOneErrback=True)
self.checkQueue()
# This can happen if the client is stopped before a connection is ever made
if (not readyToStop.called) and (not self.started):
readyToStop.callback(None)
return readyToStop
def __str__(self):
return 'CarbonClientFactory(%s:%d:%s)' % self.destination
__repr__ = __str__
# 3.
# d.addCallback(callback1)
# d.addCallback(callback2)
# d.addCallback(callback3)
# d.addErrback(errback3)
# d.callback("Test")
# 4.
# d.addErrback(errback1)
# d.errback(u"Test")
# 5.
# d.addErrback(errback1)
# d.addErrback(errback3)
# d.errback(u"Test")
# 6.
# d.addErrback(errback2)
# d.errback(u"Test")
# 7. vs 3. (none vs care exception)
# d.addCallback(callback1)
# d.addCallback(callback2)
# d.addCallbacks(callback3, errback3)
# d.callback(u"Test")
# 8.
d.addCallback(callback3)
d.addCallbacks(callback2, errback3)
d.addCallbacks(callback1, errback2)
d.callback(u"Test")
def _create_node(reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
name,
web_port,
storage=True,
magic_text=None,
needed=2,
happy=3,
total=4):
"""
Helper to create a single node, run it and return the instance
spawnProcess returned (ITransport)
"""
node_dir = join(temp_dir, name)
if web_port is None:
web_port = ''
if exists(node_dir):
created_d = succeed(None)
else:
print("creating", node_dir)
mkdir(node_dir)
done_proto = _ProcessExitedProtocol()
args = [
sys.executable,
'-m',
'allmydata.scripts.runner',
'create-node',
'--nickname',
name,
'--introducer',
introducer_furl,
'--hostname',
'localhost',
'--listen',
'tcp',
'--webport',
web_port,
'--shares-needed',
unicode(needed),
'--shares-happy',
unicode(happy),
'--shares-total',
unicode(total),
]
if not storage:
args.append('--no-storage')
args.append(node_dir)
reactor.spawnProcess(
done_proto,
sys.executable,
args,
)
created_d = done_proto.done
def created(_):
config_path = join(node_dir, 'tahoe.cfg')
config = get_config(config_path)
set_config(config, 'node', 'log_gatherer.furl', flog_gatherer)
write_config(config_path, config)
created_d.addCallback(created)
d = Deferred()
d.callback(None)
d.addCallback(lambda _: created_d)
d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text))
return d
class LocalService(automat.Automat):
"""
This class implements all the functionality of the ``local_service()``
state machine.
"""
service_name = ''
config_path = ''
def __init__(self):
if self.service_name == '':
raise RequireSubclass()
if self.service_name in services().keys():
raise ServiceAlreadyExist(self.service_name)
self.result_deferred = None
automat.Automat.__init__(
self,
name=self.service_name,
state='OFF',
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
)
#------------------------------------------------------------------------------
def dependent_on(self):
return []
def installed(self):
return True
def enabled(self):
from main import config
return config.conf().getBool(self.config_path)
def start(self):
raise RequireSubclass()
def stop(self):
raise RequireSubclass()
def request(self, newpacket, info):
raise RequireSubclass()
def cancel(self, newpacket, info):
raise RequireSubclass()
def add_callback(self, cb):
if not self.result_deferred:
self.result_deferred = Deferred()
if isinstance(cb, Deferred):
self.result_deferred.addCallback(cb.callback)
else:
self.result_deferred.addCallback(lambda *a, **kw: cb(*a, **kw))
return self.result_deferred
#------------------------------------------------------------------------------
def state_changed(self, oldstate, newstate, event, arg):
"""
Method to catch the moment when automat's state were changed.
"""
def state_not_changed(self, curstate, event, arg):
"""
This method intended to catch the moment when some event was fired but
automat's state was not changed.
"""
def A(self, event, arg):
#---ON---
if self.state == 'ON':
if event == 'shutdown':
self.state = 'CLOSED'
self.doStopService(arg)
self.doDestroyMe(arg)
elif event == 'stop':
self.state = 'INFLUENCE'
self.doSetCallback(arg)
self.doStopDependentServices(arg)
#---OFF---
elif self.state == 'OFF':
if event == 'stop':
self.doSetCallback(arg)
self.doNotifyStopped(arg)
elif event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(arg)
elif event == 'start':
self.state = 'STARTING'
self.NeedStart = False
self.NeedStop = False
self.doSetCallback(arg)
self.doStartService(arg)
#---NOT_INSTALLED---
elif self.state == 'NOT_INSTALLED':
if event == 'stop':
self.doSetCallback(arg)
self.doNotifyNotInstalled(arg)
elif event == 'start':
self.state = 'STARTING'
self.doSetCallback(arg)
self.doStartService(arg)
elif event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(arg)
#---INFLUENCE---
elif self.state == 'INFLUENCE':
if event == 'start':
self.doSetCallback(arg)
self.NeedStart = True
elif event == 'stop':
self.doSetCallback(arg)
elif event == 'shutdown':
self.state = 'CLOSED'
self.doStopService(arg)
self.doDestroyMe(arg)
elif event == 'depend-service-stopped' and self.isAllDependsStopped(
arg) and self.NeedStart:
self.state = 'STARTING'
self.NeedStart = False
self.doStopService(arg)
self.doNotifyStopped(arg)
self.doStartService(arg)
elif event == 'depend-service-stopped' and self.isAllDependsStopped(
arg) and not self.NeedStart:
self.state = 'STOPPING'
self.doStopService(arg)
#---STARTING---
elif self.state == 'STARTING':
if event == 'stop':
self.doSetCallback(arg)
self.NeedStop = True
elif event == 'start':
self.doSetCallback(arg)
elif event == 'service-started' and self.NeedStop:
self.state = 'INFLUENCE'
self.NeedStop = False
self.doStopDependentServices(arg)
elif event == 'service-failed':
self.state = 'OFF'
self.doNotifyFailed(arg)
elif event == 'shutdown':
self.state = 'CLOSED'
self.doStopService(arg)
self.doDestroyMe(arg)
elif event == 'service-not-installed':
self.state = 'NOT_INSTALLED'
self.doNotifyNotInstalled(arg)
elif event == 'service-depend-off':
self.state = 'DEPENDS_OFF'
self.doNotifyDependsOff(arg)
elif event == 'service-started' and not self.NeedStop:
self.state = 'ON'
self.doNotifyStarted(arg)
#---DEPENDS_OFF---
elif self.state == 'DEPENDS_OFF':
if event == 'stop':
self.doSetCallback(arg)
self.doNotifyDependsOff(arg)
elif event == 'start':
self.state = 'STARTING'
self.doSetCallback(arg)
self.doStartService(arg)
elif event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(arg)
#---STOPPING---
elif self.state == 'STOPPING':
if event == 'start':
self.doSetCallback(arg)
self.NeedStart = True
elif event == 'service-stopped':
self.state = 'OFF'
self.doNotifyStopped(arg)
elif event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(arg)
#---CLOSED---
elif self.state == 'CLOSED':
pass
return None
def isAllDependsStopped(self, arg):
"""
Condition method.
"""
for svc in services().values():
if self.service_name in svc.dependent_on():
if svc.state != 'OFF' and \
svc.state != 'DEPENDS_OFF' and \
svc.state != 'NOT_INSTALLED':
return False
return True
def doStartService(self, arg):
"""
Action method.
"""
if not self.installed():
self.automat('service-not-installed')
return
depends_results = []
for depend_name in self.dependent_on():
depend_service = services().get(depend_name, None)
if depend_service is None:
depends_results.append((depend_name, 'not found'))
continue
if depend_service.state != 'ON':
depends_results.append((depend_name, 'not started'))
continue
if len(depends_results) > 0:
self.automat('service-depend-off', depends_results)
return
lg.out(2, '[%s] STARTING' % self.service_name)
try:
result = self.start()
except:
lg.exc()
self.automat('service-failed', 'exception when starting')
return
if isinstance(result, Deferred):
result.addCallback(lambda x: self.automat('service-started'))
result.addErrback(lambda x: self.automat('service-failed', x))
return
if result:
self.automat('service-started')
else:
self.automat('service-failed', 'result is %r' % result)
def doStopService(self, arg):
"""
Action method.
"""
lg.out(2, '[%s] STOPPING' % self.service_name)
try:
result = self.stop()
except:
lg.exc()
self.automat('service-stopped',
'exception during stopping [%s]' % self.service_name)
return
if isinstance(result, Deferred):
result.addBoth(lambda x: self.automat('service-stopped', x))
else:
self.automat('service-stopped', result)
def doSetCallback(self, arg):
"""
Action method.
"""
if arg:
self.add_callback(arg)
def doStopDependentServices(self, arg):
"""
Action method.
"""
count = 0
for svc in services().values():
if self.service_name in svc.dependent_on():
lg.out(6, '%r sends "stop" to %r' % (self, svc))
svc.automat('stop')
count += 1
if count == 0:
self.automat('depend-service-stopped')
def doNotifyStarted(self, arg):
"""
Action method.
"""
if self.result_deferred:
self.result_deferred.callback('started')
self.result_deferred = None
on_service_callback('started', self.service_name)
def doNotifyStopped(self, arg):
"""
Action method.
"""
if self.result_deferred:
self.result_deferred.callback('stopped')
self.result_deferred = None
on_service_callback('stopped', self.service_name)
def doNotifyNotInstalled(self, arg):
"""
Action method.
"""
if self.result_deferred:
self.result_deferred.callback('not_installed')
self.result_deferred = None
on_service_callback('not_installed', self.service_name)
def doNotifyFailed(self, arg):
"""
Action method.
"""
if self.result_deferred:
self.result_deferred.callback('failed')
self.result_deferred = None
on_service_callback('failed', self.service_name)
def doNotifyDependsOff(self, arg):
"""
Action method.
"""
if self.result_deferred:
self.result_deferred.callback('depends_off')
self.result_deferred = None
on_service_callback('depends_off', self.service_name)
def doDestroyMe(self, arg):
"""
Action method.
"""
self.result_deferred = None
self.destroy()
class VNCDoToolClient(rfb.RFBClient):
encoding = rfb.RAW_ENCODING
x = 0
y = 0
buttons = 0
screen = None
image_mode = "RGBX"
deferred = None
cursor = None
cmask = None
SPECIAL_KEYS_US = "~!@#$%^&*()_+{}|:\"<>?"
def connectionMade(self):
rfb.RFBClient.connectionMade(self)
if self.transport.addressFamily == socket.AF_INET:
self.transport.setTcpNoDelay(True)
def _decodeKey(self, key):
if self.factory.force_caps:
if key.isupper() or key in self.SPECIAL_KEYS_US:
key = 'shift-%c' % key
if len(key) == 1:
keys = [key]
else:
keys = key.split('-')
keys = [KEYMAP.get(k) or ord(k) for k in keys]
return keys
def pause(self, duration):
d = Deferred()
reactor.callLater(duration, d.callback, self)
return d
def keyPress(self, key):
""" Send a key press to the server
key: string: either [a-z] or a from KEYMAP
"""
log.debug('keyPress %s', key)
self.keyDown(key)
self.keyUp(key)
return self
def keyDown(self, key):
log.debug('keyDown %s', key)
keys = self._decodeKey(key)
for k in keys:
self.keyEvent(k, down=1)
return self
def keyUp(self, key):
log.debug('keyUp %s', key)
keys = self._decodeKey(key)
for k in keys:
self.keyEvent(k, down=0)
return self
def mousePress(self, button):
""" Send a mouse click at the last set position
button: int: [1-n]
"""
log.debug('mousePress %s', button)
buttons = self.buttons | (1 << (button - 1))
self.mouseDown(button)
self.mouseUp(button)
return self
def mouseDown(self, button):
""" Send a mouse button down at the last set position
button: int: [1-n]
"""
log.debug('mouseDown %s', button)
self.buttons |= 1 << (button - 1)
self.pointerEvent(self.x, self.y, buttonmask=self.buttons)
return self
def mouseUp(self, button):
""" Send mouse button released at the last set position
button: int: [1-n]
"""
log.debug('mouseUp %s', button)
self.buttons &= ~(1 << (button - 1))
self.pointerEvent(self.x, self.y, buttonmask=self.buttons)
return self
def captureScreen(self, filename, incremental=0):
""" Save the current display to filename
"""
log.debug('captureScreen %s', filename)
return self._capture(filename, incremental)
def captureRegion(self, filename, x, y, w, h, incremental=0):
""" Save a region of the current display to filename
"""
log.debug('captureRegion %s', filename)
return self._capture(filename, incremental, x, y, x + w, y + h)
def refreshScreen(self, incremental=0):
d = self.deferred = Deferred()
self.framebufferUpdateRequest(incremental=incremental)
return d
def _capture(self, filename, incremental, *args):
d = self.refreshScreen(incremental)
d.addCallback(self._captureSave, filename, *args)
return d
def _captureSave(self, data, filename, *args):
log.debug('captureSave %s', filename)
if args:
capture = self.screen.crop(args)
else:
capture = self.screen
capture.save(filename)
return self
def expectScreen(self, filename, maxrms=0):
""" Wait until the display matches a target image
filename: an image file to read and compare against
maxrms: the maximum root mean square between histograms of the
screen and target image
"""
log.debug('expectScreen %s', filename)
return self._expectFramebuffer(filename, 0, 0, maxrms)
def expectRegion(self, filename, x, y, maxrms=0):
""" Wait until a portion of the screen matches the target image
The region compared is defined by the box
(x, y), (x + image.width, y + image.height)
"""
log.debug('expectRegion %s (%s, %s)', filename, x, y)
return self._expectFramebuffer(filename, x, y, maxrms)
def _expectFramebuffer(self, filename, x, y, maxrms):
self.framebufferUpdateRequest(incremental=1)
image = Image.open(filename)
w, h = image.size
self.expected = image.histogram()
self.deferred = Deferred()
self.deferred.addCallback(self._expectCompare, (x, y, x + w, y + h),
maxrms)
return self.deferred
def _expectCompare(self, data, box, maxrms):
image = self.screen.crop(box)
hist = image.histogram()
if len(hist) == len(self.expected):
sum_ = 0
for h, e in zip(hist, self.expected):
sum_ += (h - e)**2
rms = math.sqrt(sum_ / len(hist))
log.debug('rms:%s maxrms: %s', int(rms), int(maxrms))
if rms <= maxrms:
return self
self.deferred = Deferred()
self.deferred.addCallback(self._expectCompare, box, maxrms)
self.framebufferUpdateRequest(
incremental=1) # use box ~(x, y, w - x, h - y)?
return self.deferred
def mouseMove(self, x, y):
""" Move the mouse pointer to position (x, y)
"""
log.debug('mouseMove %d,%d', x, y)
self.x, self.y = x, y
self.pointerEvent(x, y, self.buttons)
return self
def mouseDrag(self, x, y, step=1):
""" Move the mouse point to position (x, y) in increments of step
"""
log.debug('mouseDrag %d,%d', x, y)
if x < self.x:
xsteps = [self.x - i for i in range(step, self.x - x + 1, step)]
else:
xsteps = range(self.x, x, step)
if y < self.y:
ysteps = [self.y - i for i in range(step, self.y - y + 1, step)]
else:
ysteps = range(self.y, y, step)
for ypos in ysteps:
time.sleep(.2)
self.mouseMove(self.x, ypos)
for xpos in xsteps:
time.sleep(.2)
self.mouseMove(xpos, self.y)
self.mouseMove(x, y)
return self
def setImageMode(self):
""" Extracts color ordering and 24 vs. 32 bpp info out of the pixel format information
"""
if self._version_server == 3.889:
self.setPixelFormat(bpp=16,
depth=16,
bigendian=0,
truecolor=1,
redmax=31,
greenmax=63,
bluemax=31,
redshift=11,
greenshift=5,
blueshift=0)
self.image_mode = "BGR;16"
elif (self.truecolor and (not self.bigendian) and self.depth == 24
and self.redmax == 255 and self.greenmax == 255
and self.bluemax == 255):
pixel = ["X"] * self.bypp
offsets = [
offset // 8
for offset in (self.redshift, self.greenshift, self.blueshift)
]
for offset, color in zip(offsets, "RGB"):
pixel[offset] = color
self.image_mode = "".join(pixel)
else:
self.setPixelFormat()
#
# base customizations
#
def vncRequestPassword(self):
if self.factory.password is None:
self.transport.loseConnection()
self.factory.clientConnectionFailed(
self,
AuthenticationError('password required, but none provided'))
return
self.sendPassword(self.factory.password)
def vncConnectionMade(self):
self.setImageMode()
encodings = [self.encoding]
if self.factory.pseudocursor or self.factory.nocursor:
encodings.append(rfb.PSEUDO_CURSOR_ENCODING)
if self.factory.pseudodesktop:
encodings.append(rfb.PSEUDO_DESKTOP_SIZE_ENCODING)
self.setEncodings(encodings)
self.factory.clientConnectionMade(self)
def bell(self):
print('ding')
def copy_text(self, text):
print('clipboard copy', repr(text))
def paste(self, message):
self.clientCutText(message)
return self
def updateRectangle(self, x, y, width, height, data):
# ignore empty updates
if not data:
return
size = (width, height)
update = Image.frombytes('RGB', size, data, 'raw', self.image_mode)
if not self.screen:
self.screen = update
# track upward screen resizes, often occurs during os boot of VMs
# When the screen is sent in chunks (as observed on VMWare ESXi), the canvas
# needs to be resized to fit all existing contents and the update.
elif self.screen.size[0] < (x + width) or self.screen.size[1] < (
y + height):
new_size = (max(x + width, self.screen.size[0]),
max(y + height, self.screen.size[1]))
new_screen = Image.new("RGB", new_size, "black")
new_screen.paste(self.screen, (0, 0))
new_screen.paste(update, (x, y))
self.screen = new_screen
else:
self.screen.paste(update, (x, y))
self.drawCursor()
def commitUpdate(self, rectangles):
if self.deferred:
d = self.deferred
self.deferred = None
d.callback(self)
def updateCursor(self, x, y, width, height, image, mask):
if self.factory.nocursor:
return
if not width or not height:
self.cursor = None
self.cursor = Image.frombytes('RGBX', (width, height), image)
self.cmask = Image.frombytes('1', (width, height), mask)
self.cfocus = x, y
self.drawCursor()
def drawCursor(self):
if not self.cursor:
return
if not self.screen:
return
x = self.x - self.cfocus[0]
y = self.y - self.cfocus[1]
self.screen.paste(self.cursor, (x, y), self.cmask)
def updateDesktopSize(self, width, height):
new_screen = Image.new("RGB", (width, height), "black")
if self.screen:
new_screen.paste(self.screen, (0, 0))
self.screen = new_screen
class CarbonClientFactory(
with_metaclass(PluginRegistrar, ReconnectingClientFactory, object)):
plugins = {}
maxDelay = 5
def __init__(self, destination, router):
self.destination = destination
self.router = router
self.destinationName = ('%s:%d:%s' % destination).replace('.', '_')
self.host, self.port, self.carbon_instance = destination
self.addr = (self.host, self.port)
self.started = False
# This factory maintains protocol state across reconnects
self.queue = deque(
) # Change to make this the sole source of metrics to be sent.
self.connectedProtocol = None
self.queueEmpty = Deferred()
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
# Args: {'connector': connector, 'reason': reason}
self.connectFailed = Deferred()
# Args: {'connector': connector, 'reason': reason}
self.connectionLost = Deferred()
# Args: protocol instance
self.connectionMade = Deferred()
self.connectionMade.addCallbacks(self.clientConnectionMade, log.err)
self.deferSendPending = None
# Define internal metric names
self.attemptedRelays = 'destinations.%s.attemptedRelays' % self.destinationName
self.fullQueueDrops = 'destinations.%s.fullQueueDrops' % self.destinationName
self.queuedUntilConnected = 'destinations.%s.queuedUntilConnected' % self.destinationName
self.relayMaxQueueLength = 'destinations.%s.relayMaxQueueLength' % self.destinationName
def clientProtocol(self):
raise NotImplementedError()
def scheduleSend(self):
if self.deferSendPending and self.deferSendPending.active():
return
self.deferSendPending = reactor.callLater(
settings.TIME_TO_DEFER_SENDING, self.sendQueued)
def sendQueued(self):
if self.connectedProtocol:
self.connectedProtocol.sendQueued()
def queueFullCallback(self, result):
state.events.cacheFull()
log.clients('%s send queue is full (%d datapoints)' % (self, result))
def queueSpaceCallback(self, result):
if self.queueFull.called:
log.clients('%s send queue has space available' %
self.connectedProtocol)
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
state.events.cacheSpaceAvailable()
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
def buildProtocol(self, addr):
self.connectedProtocol = self.clientProtocol()
self.connectedProtocol.factory = self
return self.connectedProtocol
def startConnecting(
self): # calling this startFactory yields recursion problems
self.started = True
self.connector = reactor.connectTCP(self.host, self.port, self)
def stopConnecting(self):
self.started = False
self.stopTrying()
if self.connectedProtocol and self.connectedProtocol.connected:
return self.connectedProtocol.disconnect()
@property
def queueSize(self):
return len(self.queue)
def hasQueuedDatapoints(self):
return bool(self.queue)
def takeSomeFromQueue(self):
"""Use self.queue, which is a collections.deque, to pop up to
settings.MAX_DATAPOINTS_PER_MESSAGE items from the left of the
queue.
"""
def yield_max_datapoints():
for _ in range(settings.MAX_DATAPOINTS_PER_MESSAGE):
try:
yield self.queue.popleft()
except IndexError:
return
return list(yield_max_datapoints())
def checkQueue(self):
"""Check if the queue is empty. If the queue isn't empty or
doesn't exist yet, then this will invoke the callback chain on the
self.queryEmpty Deferred chain with the argument 0, and will
re-set the queueEmpty callback chain with a new Deferred
object.
"""
if not self.queue:
self.queueEmpty.callback(0)
self.queueEmpty = Deferred()
def enqueue(self, metric, datapoint):
self.queue.append((metric, datapoint))
def enqueue_from_left(self, metric, datapoint):
self.queue.appendleft((metric, datapoint))
def sendDatapoint(self, metric, datapoint):
instrumentation.increment(self.attemptedRelays)
instrumentation.max(self.relayMaxQueueLength, self.queueSize)
if self.queueSize >= settings.MAX_QUEUE_SIZE:
if not self.queueFull.called:
self.queueFull.callback(self.queueSize)
instrumentation.increment(self.fullQueueDrops)
else:
self.enqueue(metric, datapoint)
if self.connectedProtocol:
self.scheduleSend()
else:
instrumentation.increment(self.queuedUntilConnected)
def sendHighPriorityDatapoint(self, metric, datapoint):
"""The high priority datapoint is one relating to the carbon
daemon itself. It puts the datapoint on the left of the deque,
ahead of other stats, so that when the carbon-relay, specifically,
is overwhelmed its stats are more likely to make it through and
expose the issue at hand.
In addition, these stats go on the deque even when the max stats
capacity has been reached. This relies on not creating the deque
with a fixed max size.
"""
instrumentation.increment(self.attemptedRelays)
self.enqueue_from_left(metric, datapoint)
if self.connectedProtocol:
self.scheduleSend()
else:
instrumentation.increment(self.queuedUntilConnected)
def startedConnecting(self, connector):
log.clients("%s::startedConnecting (%s:%d)" %
(self, connector.host, connector.port))
def clientConnectionMade(self, client):
log.clients("%s::connectionMade (%s)" % (self, client))
self.resetDelay()
self.destinationUp(client.factory.destination)
self.connectionMade.addCallbacks(self.clientConnectionMade, log.err)
return client
def clientConnectionLost(self, connector, reason):
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
log.clients(
"%s::clientConnectionLost (%s:%d) %s" %
(self, connector.host, connector.port, reason.getErrorMessage()))
self.connectedProtocol = None
self.destinationDown(connector.factory.destination)
args = dict(connector=connector, reason=reason)
d = self.connectionLost
self.connectionLost = Deferred()
d.callback(args)
def clientConnectionFailed(self, connector, reason):
ReconnectingClientFactory.clientConnectionFailed(
self, connector, reason)
log.clients(
"%s::clientConnectionFailed (%s:%d) %s" %
(self, connector.host, connector.port, reason.getErrorMessage()))
self.destinationDown(connector.factory.destination)
args = dict(connector=connector, reason=reason)
d = self.connectFailed
self.connectFailed = Deferred()
d.callback(args)
def destinationUp(self, destination):
log.clients("Destination is up: %s:%d:%s" % destination)
if not self.router.hasDestination(destination):
log.clients("Adding client %s:%d:%s to router" % destination)
self.router.addDestination(destination)
state.events.resumeReceivingMetrics()
def destinationDown(self, destination):
# Only blacklist the destination if we tried a lot.
log.clients("Destination is down: %s:%d:%s (%d/%d)" %
(destination[0], destination[1], destination[2],
self.retries, settings.DYNAMIC_ROUTER_MAX_RETRIES))
# Retries comes from the ReconnectingClientFactory.
if self.retries < settings.DYNAMIC_ROUTER_MAX_RETRIES:
return
if settings.DYNAMIC_ROUTER and self.router.hasDestination(destination):
log.clients("Removing client %s:%d:%s to router" % destination)
self.router.removeDestination(destination)
# Do not receive more metrics if we don't have any usable destinations.
if not self.router.countDestinations():
state.events.pauseReceivingMetrics()
# Re-inject queued metrics.
metrics = list(self.queue)
log.clients("Re-injecting %d metrics from %s" %
(len(metrics), self))
for metric, datapoint in metrics:
state.events.metricGenerated(metric, datapoint)
self.queue.clear()
def disconnect(self):
self.queueEmpty.addCallback(lambda result: self.stopConnecting())
readyToStop = DeferredList([self.connectionLost, self.connectFailed],
fireOnOneCallback=True,
fireOnOneErrback=True)
self.checkQueue()
# This can happen if the client is stopped before a connection is ever made
if (not readyToStop.called) and (not self.started):
readyToStop.callback(None)
return readyToStop
def __str__(self):
return 'CarbonClientFactory(%s:%d:%s)' % self.destination
__repr__ = __str__
class ChannelModifyTorrentEndpoint(BaseChannelsEndpoint):
"""
This class is responsible for methods that modify the list of torrents (adding/removing torrents).
"""
def __init__(self, session, cid, path):
BaseChannelsEndpoint.__init__(self, session)
self.cid = cid
self.path = path
self.deferred = Deferred()
def render_PUT(self, request):
"""
.. http:put:: /channels/discovered/(string: channelid)/torrents/http%3A%2F%2Ftest.com%2Ftest.torrent
Add a torrent by magnet or url to your channel. Returns error 500 if something is wrong with the torrent file
and DuplicateTorrentFileError if already added to your channel (except with magnet links).
**Example request**:
.. sourcecode:: none
curl -X PUT http://localhost:8085/channels/discovered/abcdefg/torrents/
http%3A%2F%2Ftest.com%2Ftest.torrent --data "description=nice video"
**Example response**:
.. sourcecode:: javascript
{
"added": "http://test.com/test.torrent"
}
:statuscode 404: if your channel does not exist.
:statuscode 500: if the specified torrent is already in your channel.
"""
channel = self.get_channel_from_db(self.cid)
if channel is None:
return BaseChannelsEndpoint.return_404(request)
parameters = http.parse_qs(request.content.read(), 1)
if 'description' not in parameters or len(
parameters['description']) == 0:
extra_info = {}
else:
extra_info = {'description': parameters['description'][0]}
def _on_url_fetched(data):
return TorrentDef.load_from_memory(data)
def _on_magnet_fetched(meta_info):
return TorrentDef.load_from_dict(meta_info)
@blocking_call_on_reactor_thread
def _on_torrent_def_loaded(torrent_def):
self.session.add_torrent_def_to_channel(channel[0],
torrent_def,
extra_info,
forward=True)
return self.path
def _on_added(added):
request.write(json.dumps({"added": added}))
request.finish()
def _on_add_failed(failure):
failure.trap(ValueError, DuplicateTorrentFileError,
SchemeNotSupported)
self._logger.exception(failure.value)
request.write(
BaseChannelsEndpoint.return_500(self, request, failure.value))
request.finish()
def _on_timeout(_):
request.write(
BaseChannelsEndpoint.return_500(
self, request, RuntimeError("Metainfo timeout")))
request.finish()
if self.path.startswith("http:") or self.path.startswith("https:"):
self.deferred = http_get(self.path)
self.deferred.addCallback(_on_url_fetched)
if self.path.startswith("magnet:"):
try:
self.session.lm.ltmgr.get_metainfo(
self.path,
callback=self.deferred.callback,
timeout=30,
timeout_callback=_on_timeout,
notify=True)
except Exception as ex:
self.deferred.errback(ex)
self.deferred.addCallback(_on_magnet_fetched)
self.deferred.addCallback(_on_torrent_def_loaded)
self.deferred.addCallback(_on_added)
self.deferred.addErrback(_on_add_failed)
return NOT_DONE_YET
def render_DELETE(self, request):
"""
.. http:delete:: /channels/discovered/(string: channelid)/torrents/(string: comma separated torrent infohashes)
Remove a single or multiple torrents with the given comma separated infohashes from a given channel.
**Example request**:
.. sourcecode:: none
curl -X DELETE http://localhost:8085/channels/discovered/abcdefg/torrents/
97d2d8f5d37e56cfaeaae151d55f05b077074779,971d55f05b077074779d2d8f5d37e56cfaeaae15
**Example response**:
.. sourcecode:: javascript
{
"removed": True
}
.. sourcecode:: javascript
{
"removed": False, "failed_torrents":["97d2d8f5d37e56cfaeaae151d55f05b077074779"]
}
:statuscode 404: if the channel is not found
"""
channel_info = self.get_channel_from_db(self.cid)
if channel_info is None:
return ChannelsTorrentsEndpoint.return_404(request)
channel_community = self.get_community_for_channel_id(channel_info[0])
if channel_community is None:
return BaseChannelsEndpoint.return_404(
request, message=UNKNOWN_COMMUNITY_MSG)
torrent_db_columns = [
'Torrent.torrent_id', 'infohash', 'Torrent.name', 'length',
'Torrent.category', 'num_seeders', 'num_leechers',
'last_tracker_check', 'ChannelTorrents.dispersy_id'
]
failed_torrents = []
for torrent_path in self.path.split(","):
torrent_info = self.channel_db_handler.getTorrentFromChannelId(
channel_info[0], torrent_path.decode('hex'),
torrent_db_columns)
if torrent_info is None:
failed_torrents.append(torrent_path)
else:
# the 8th index is the dispersy id of the channel torrent
channel_community.remove_torrents([torrent_info[8]])
if failed_torrents:
return json.dumps({
"removed": False,
"failed_torrents": failed_torrents
})
return json.dumps({"removed": True})
def render_GET(self, request):
"""
.. http:get:: /torrentinfo
A GET request to this endpoint will return information from a torrent found at a provided URI.
This URI can either represent a file location, a magnet link or a HTTP(S) url.
- torrent: the URI of the torrent file that should be downloaded. This parameter is required.
**Example request**:
.. sourcecode:: none
curl -X PUT http://localhost:8085/torrentinfo?torrent=file:/home/me/test.torrent
**Example response**:
.. sourcecode:: javascript
{"metainfo": <torrent metainfo dictionary>}
"""
metainfo_deferred = Deferred()
def on_got_metainfo(metainfo):
if not isinstance(metainfo, dict) or 'info' not in metainfo:
self._logger.warning("Received metainfo is not a valid dictionary")
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
request.write(json.dumps({"error": 'invalid response'}))
self.finish_request(request)
return
infohash = hashlib.sha1(bencode(metainfo['info'])).digest()
# Check if the torrent is already in the downloads
metainfo['download_exists'] = infohash in self.session.lm.downloads
# Save the torrent to our store
try:
self.session.save_collected_torrent(infohash, bencode(metainfo))
except TypeError:
# Note: in libtorrent 1.1.1, bencode throws a TypeError which is a known bug
pass
request.write(json.dumps({"metainfo": metainfo}, ensure_ascii=False))
self.finish_request(request)
def on_metainfo_timeout(_):
if not request.finished:
request.setResponseCode(http.REQUEST_TIMEOUT)
request.write(json.dumps({"error": "timeout"}))
# If the above request.write failed, the request will have already been finished
if not request.finished:
self.finish_request(request)
def on_lookup_error(failure):
failure.trap(ConnectError, DNSLookupError, HttpError, ConnectionLost)
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
request.write(json.dumps({"error": failure.getErrorMessage()}))
self.finish_request(request)
if 'uri' not in request.args or len(request.args['uri']) == 0:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "uri parameter missing"})
uri = unicode(request.args['uri'][0], 'utf-8')
if uri.startswith('file:'):
try:
filename = url2pathname(uri[5:].encode('utf-8') if isinstance(uri, unicode) else uri[5:])
metainfo_deferred.callback(bdecode(fix_torrent(filename)))
except TypeError:
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
return json.dumps({"error": "error while decoding torrent file"})
elif uri.startswith('http'):
def _on_loaded(metadata):
metainfo_deferred.callback(bdecode(metadata))
http_get(uri.encode('utf-8')).addCallback(_on_loaded).addErrback(on_lookup_error)
elif uri.startswith('magnet'):
infohash = parse_magnetlink(uri)[1]
if infohash is None:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "missing infohash"})
if self.session.has_collected_torrent(infohash):
try:
tdef = TorrentDef.load_from_memory(self.session.get_collected_torrent(infohash))
except ValueError as exc:
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
return json.dumps({"error": "invalid torrent file: %s" % str(exc)})
on_got_metainfo(tdef.get_metainfo())
return NOT_DONE_YET
self.session.lm.ltmgr.get_metainfo(uri, callback=metainfo_deferred.callback, timeout=20,
timeout_callback=on_metainfo_timeout, notify=True)
else:
request.setResponseCode(http.BAD_REQUEST)
return json.dumps({"error": "invalid uri"})
metainfo_deferred.addCallback(on_got_metainfo)
return NOT_DONE_YET
def test_disconnectWhileProducing(self):
"""
If C{loseConnection} is called while a producer is registered with the
transport, the connection is closed after the producer is unregistered.
"""
reactor = self.buildReactor()
# For some reason, pyobject/pygtk will not deliver the close
# notification that should happen after the unregisterProducer call in
# this test. The selectable is in the write notification set, but no
# notification ever arrives. Probably for the same reason #5233 led
# win32eventreactor to be broken.
skippedReactors = ["Glib2Reactor", "Gtk2Reactor"]
reactorClassName = reactor.__class__.__name__
if reactorClassName in skippedReactors and platform.isWindows():
raise SkipTest("A pygobject/pygtk bug disables this functionality "
"on Windows.")
class Producer:
def resumeProducing(self):
log.msg("Producer.resumeProducing")
self.listen(reactor, ServerFactory.forProtocol(Protocol))
finished = Deferred()
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
class ClientProtocol(Protocol):
"""
Protocol to connect, register a producer, try to lose the
connection, unregister the producer, and wait for the connection to
actually be lost.
"""
def connectionMade(self):
log.msg("ClientProtocol.connectionMade")
self.transport.registerProducer(Producer(), False)
self.transport.loseConnection()
# Let the reactor tick over, in case synchronously calling
# loseConnection and then unregisterProducer is the same as
# synchronously calling unregisterProducer and then
# loseConnection (as it is in several reactors).
reactor.callLater(0, reactor.callLater, 0, self.unregister)
def unregister(self):
log.msg("ClientProtocol unregister")
self.transport.unregisterProducer()
# This should all be pretty quick. Fail the test
# if we don't get a connectionLost event really
# soon.
reactor.callLater(
1.0, finished.errback,
Failure(Exception("Connection was not lost")))
def connectionLost(self, reason):
log.msg("ClientProtocol.connectionLost")
finished.callback(None)
clientFactory = ClientFactory()
clientFactory.protocol = ClientProtocol
self.connect(reactor, clientFactory)
self.runReactor(reactor)
class TestEventsEndpoint(AbstractApiTest):
@blocking_call_on_reactor_thread
@inlineCallbacks
def setUp(self, autoload_discovery=True):
yield super(TestEventsEndpoint,
self).setUp(autoload_discovery=autoload_discovery)
self.events_deferred = Deferred()
self.connection_pool = HTTPConnectionPool(reactor, False)
self.socket_open_deferred = self.tribler_started_deferred.addCallback(
self.open_events_socket)
self.messages_to_wait_for = 0
@blocking_call_on_reactor_thread
@inlineCallbacks
def tearDown(self, annotate=True):
yield self.close_connections()
# Wait to make sure the HTTPChannel is closed, see https://twistedmatrix.com/trac/ticket/2447
yield deferLater(reactor, 0.3, lambda: None)
yield super(TestEventsEndpoint, self).tearDown(annotate=annotate)
def on_event_socket_opened(self, response):
response.deliverBody(
EventDataProtocol(self.messages_to_wait_for, self.events_deferred,
response))
def open_events_socket(self, _):
agent = Agent(reactor, pool=self.connection_pool)
return agent.request('GET', 'http://localhost:%s/events' % self.session.get_http_api_port(),
Headers({'User-Agent': ['Tribler ' + version_id]}), None)\
.addCallback(self.on_event_socket_opened)
def close_connections(self):
return self.connection_pool.closeCachedConnections()
@deferred(timeout=20)
def test_search_results(self):
"""
Testing whether the event endpoint returns search results when we have search results available
"""
def verify_search_results(results):
self.assertEqual(len(results), 2)
self.messages_to_wait_for = 2
def send_notifications(_):
self.session.lm.api_manager.root_endpoint.events_endpoint.start_new_query(
)
results_dict = {
"keywords": ["test"],
"result_list": [('a', ) * 10]
}
self.session.notifier.notify(SIGNAL_CHANNEL,
SIGNAL_ON_SEARCH_RESULTS, None,
results_dict)
self.session.notifier.notify(SIGNAL_TORRENT,
SIGNAL_ON_SEARCH_RESULTS, None,
results_dict)
self.socket_open_deferred.addCallback(send_notifications)
return self.events_deferred.addCallback(verify_search_results)
@deferred(timeout=20)
def test_events(self):
"""
Testing whether various events are coming through the events endpoints
"""
self.messages_to_wait_for = 13
def send_notifications(_):
self.session.lm.api_manager.root_endpoint.events_endpoint.start_new_query(
)
results_dict = {
"keywords": ["test"],
"result_list": [('a', ) * 10]
}
self.session.notifier.notify(SIGNAL_TORRENT,
SIGNAL_ON_SEARCH_RESULTS, None,
results_dict)
self.session.notifier.notify(SIGNAL_CHANNEL,
SIGNAL_ON_SEARCH_RESULTS, None,
results_dict)
self.session.notifier.notify(NTFY_UPGRADER, NTFY_STARTED, None,
None)
self.session.notifier.notify(NTFY_UPGRADER_TICK, NTFY_STARTED,
None, None)
self.session.notifier.notify(NTFY_UPGRADER, NTFY_FINISHED, None,
None)
self.session.notifier.notify(NTFY_WATCH_FOLDER_CORRUPT_TORRENT,
NTFY_INSERT, None, None)
self.session.notifier.notify(NTFY_NEW_VERSION, NTFY_INSERT, None,
None)
self.session.notifier.notify(NTFY_CHANNEL, NTFY_DISCOVERED, None,
None)
self.session.notifier.notify(NTFY_TORRENT, NTFY_DISCOVERED, None,
{'a': 'Invalid character \xa1'})
self.session.notifier.notify(NTFY_TORRENT, NTFY_DELETE, None,
{'a': 'b'})
self.session.notifier.notify(NTFY_TORRENT, NTFY_FINISHED, 'a' * 10,
None)
self.session.notifier.notify(NTFY_TORRENT, NTFY_ERROR, 'a' * 10,
'This is an error message')
self.session.lm.api_manager.root_endpoint.events_endpoint.on_tribler_exception(
"hi")
self.socket_open_deferred.addCallback(send_notifications)
return self.events_deferred
@deferred(timeout=20)
def test_family_filter_search(self):
"""
Testing the family filter when searching for torrents and channels
"""
self.messages_to_wait_for = 2
def send_searches(_):
events_endpoint = self.session.lm.api_manager.root_endpoint.events_endpoint
channels = [[
'a',
] * 10, [
'a',
] * 10]
channels[0][2] = 'badterm'
events_endpoint.on_search_results_channels(None, None, None, {
"keywords": ["test"],
"result_list": channels
})
self.assertEqual(len(events_endpoint.channel_cids_sent), 1)
torrents = [[
'a',
] * 10, [
'a',
] * 10]
torrents[0][4] = 'xxx'
events_endpoint.on_search_results_torrents(None, None, None, {
"keywords": ["test"],
"result_list": torrents
})
self.assertEqual(len(events_endpoint.infohashes_sent), 1)
self.socket_open_deferred.addCallback(send_searches)
return self.events_deferred
@stack
def post_it(ids):
poster = beta_blogger_blog_poster(export_info, self.log, ids,
title, description, options)
d3 = poster.do_post()
d3.addCallback(handle_post_success)
d3.addErrback(handle_post_failure)
return d3
##
## Need to convert the image_ids to media_ids for posting
##
d = Deferred()
for id in image_ids:
d.addCallback(get_media_id, id)
d.addCallback(post_it)
d.callback(media_list)
return d
class beta_blogger_blog_list_getter(HTTPClientFactory):
def __init__(self, token, log):
self.re_ID = re.compile(
"^tag:blogger\.com\,[0-9]+:user-([0-9]+)\.blog-([0-9]+)$")
self.token = token
self.log = log
url = "http://www.blogger.com/feeds/default/blogs"
header_dict = {
'Accept': "*/*",
'Authorization': "AuthSub token=\"%s\"" % self.token,
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
d3 = Deferred()
def cb_will_fail(number):
print(1 / number)
d3.addCallback(cb_will_fail)
d3.addErrback(print)
d3.callback(0)
# Failure example
try:
1 / 0
except ZeroDivisionError:
f = Failure() # absorb an active exception and its traceback
print(f)
print(f.check(ValueError))
print(f.check(ValueError, ZeroDivisionError))
# Trapping the error
d4 = Deferred()
def eb_value_error(failure):
failure.trap(ValueError) # re-raise if doesn't match
print('failure was ValueError')
class H2Connection(Protocol, TimeoutMixin):
"""
A class representing a single HTTP/2 connection.
This implementation of L{IProtocol} works hand in hand with L{H2Stream}.
This is because we have the requirement to register multiple producers for
a single HTTP/2 connection, one for each stream. The standard Twisted
interfaces don't really allow for this, so instead there's a custom
interface between the two objects that allows them to work hand-in-hand here.
@ivar conn: The HTTP/2 connection state machine.
@type conn: L{h2.connection.H2Connection}
@ivar streams: A mapping of stream IDs to L{H2Stream} objects, used to call
specific methods on streams when events occur.
@type streams: L{dict}, mapping L{int} stream IDs to L{H2Stream} objects.
@ivar priority: A HTTP/2 priority tree used to ensure that responses are
prioritised appropriately.
@type priority: L{priority.PriorityTree}
@ivar _consumerBlocked: A flag tracking whether or not the L{IConsumer}
that is consuming this data has asked us to stop producing.
@type _consumerBlocked: L{bool}
@ivar _sendingDeferred: A L{Deferred} used to restart the data-sending loop
when more response data has been produced. Will not be present if there
is outstanding data still to send.
@type _consumerBlocked: A L{twisted.internet.defer.Deferred}, or L{None}
@ivar _outboundStreamQueues: A map of stream IDs to queues, used to store
data blocks that are yet to be sent on the connection. These are used
both to handle producers that do not respect L{IConsumer} but also to
allow priority to multiplex data appropriately.
@type _outboundStreamQueues: A L{dict} mapping L{int} stream IDs to
L{collections.deque} queues, which contain either L{bytes} objects or
C{_END_STREAM_SENTINEL}.
@ivar _sender: A handle to the data-sending loop, allowing it to be
terminated if needed.
@type _sender: L{twisted.internet.task.LoopingCall}
@ivar abortTimeout: The number of seconds to wait after we attempt to shut
the transport down cleanly to give up and forcibly terminate it. This
is only used when we time a connection out, to prevent errors causing
the FD to get leaked. If this is L{None}, we will wait forever.
@type abortTimeout: L{int}
@ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be
used to forcibly close the transport if it doesn't close cleanly.
@type _abortingCall: L{twisted.internet.base.DelayedCall
"""
factory = None
site = None
abortTimeout = 15
_log = Logger()
_abortingCall = None
def __init__(self, reactor=None):
config = h2.config.H2Configuration(client_side=False,
header_encoding=None)
self.conn = h2.connection.H2Connection(config=config)
self.streams = {}
self.priority = priority.PriorityTree()
self._consumerBlocked = None
self._sendingDeferred = None
self._outboundStreamQueues = {}
self._streamCleanupCallbacks = {}
self._stillProducing = True
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# Start the data sending function.
self._reactor.callLater(0, self._sendPrioritisedData)
# Implementation of IProtocol
def connectionMade(self):
"""
Called by the reactor when a connection is received. May also be called
by the L{twisted.web.http._GenericHTTPChannelProtocol} during upgrade
to HTTP/2.
"""
self.setTimeout(self.timeOut)
self.conn.initiate_connection()
self.transport.write(self.conn.data_to_send())
def dataReceived(self, data):
"""
Called whenever a chunk of data is received from the transport.
@param data: The data received from the transport.
@type data: L{bytes}
"""
self.resetTimeout()
try:
events = self.conn.receive_data(data)
except h2.exceptions.ProtocolError:
# A remote protocol error terminates the connection.
dataToSend = self.conn.data_to_send()
self.transport.write(dataToSend)
self.transport.loseConnection()
self.connectionLost("Protocol error from peer.")
return
for event in events:
if isinstance(event, h2.events.RequestReceived):
self._requestReceived(event)
elif isinstance(event, h2.events.DataReceived):
self._requestDataReceived(event)
elif isinstance(event, h2.events.StreamEnded):
self._requestEnded(event)
elif isinstance(event, h2.events.StreamReset):
self._requestAborted(event)
elif isinstance(event, h2.events.WindowUpdated):
self._handleWindowUpdate(event)
elif isinstance(event, h2.events.PriorityUpdated):
self._handlePriorityUpdate(event)
elif isinstance(event, h2.events.ConnectionTerminated):
self.transport.loseConnection()
self.connectionLost("Shutdown by remote peer")
dataToSend = self.conn.data_to_send()
if dataToSend:
self.transport.write(dataToSend)
def timeoutConnection(self):
"""
Called when the connection has been inactive for
L{self.timeOut<twisted.protocols.policies.TimeoutMixin.timeOut>}
seconds. Cleanly tears the connection down, attempting to notify the
peer if needed.
We override this method to add two extra bits of functionality:
- We want to log the timeout.
- We want to send a GOAWAY frame indicating that the connection is
being terminated, and whether it was clean or not. We have to do this
before the connection is torn down.
"""
self._log.info("Timing out client {client}",
client=self.transport.getPeer())
# Check whether there are open streams. If there are, we're going to
# want to use the error code PROTOCOL_ERROR. If there aren't, use
# NO_ERROR.
if (self.conn.open_outbound_streams > 0
or self.conn.open_inbound_streams > 0):
error_code = h2.errors.ErrorCodes.PROTOCOL_ERROR
else:
error_code = h2.errors.ErrorCodes.NO_ERROR
self.conn.close_connection(error_code=error_code)
self.transport.write(self.conn.data_to_send())
# Don't let the client hold this connection open too long.
if self.abortTimeout is not None:
# We use self.callLater because that's what TimeoutMixin does, even
# though we have a perfectly good reactor sitting around. See
# https://twistedmatrix.com/trac/ticket/8488.
self._abortingCall = self.callLater(self.abortTimeout,
self.forceAbortClient)
# We're done, throw the connection away.
self.transport.loseConnection()
def forceAbortClient(self):
"""
Called if C{abortTimeout} seconds have passed since the timeout fired,
and the connection still hasn't gone away. This can really only happen
on extremely bad connections or when clients are maliciously attempting
to keep connections open.
"""
self._log.info("Forcibly timing out client: {client}",
client=self.transport.getPeer())
self.transport.abortConnection()
def connectionLost(self, reason):
"""
Called when the transport connection is lost.
Informs all outstanding response handlers that the connection has been
lost, and cleans up all internal state.
"""
self._stillProducing = False
self.setTimeout(None)
for stream in self.streams.values():
stream.connectionLost(reason)
for streamID in list(self.streams.keys()):
self._requestDone(streamID)
# If we were going to force-close the transport, we don't have to now.
if self._abortingCall is not None:
self._abortingCall.cancel()
self._abortingCall = None
# Implementation of IPushProducer
#
# Here's how we handle IPushProducer. We have multiple outstanding
# H2Streams. Each of these exposes an IConsumer interface to the response
# handler that allows it to push data into the H2Stream. The H2Stream then
# writes the data into the H2Connection object.
#
# The H2Connection needs to manage these writes to account for:
#
# - flow control
# - priority
#
# We manage each of these in different ways.
#
# For flow control, we simply use the equivalent of the IPushProducer
# interface. We simply tell the H2Stream: "Hey, you can't send any data
# right now, sorry!". When that stream becomes unblocked, we free it up
# again. This allows the H2Stream to propagate this backpressure up the
# chain.
#
# For priority, we need to keep a backlog of data frames that we can send,
# and interleave them appropriately. This backlog is most sensibly kept in
# the H2Connection object itself. We keep one queue per stream, which is
# where the writes go, and then we have a loop that manages popping these
# streams off in priority order.
#
# Logically then, we go as follows:
#
# 1. Stream calls writeDataToStream(). This causes a DataFrame to be placed
# on the queue for that stream. It also informs the priority
# implementation that this stream is unblocked.
# 2. The _sendPrioritisedData() function spins in a tight loop. Each
# iteration it asks the priority implementation which stream should send
# next, and pops a data frame off that stream's queue. If, after sending
# that frame, there is no data left on that stream's queue, the function
# informs the priority implementation that the stream is blocked.
#
# If all streams are blocked, or if there are no outstanding streams, the
# _sendPrioritisedData function waits to be awoken when more data is ready
# to send.
#
# Note that all of this only applies to *data*. Headers and other control
# frames deliberately skip this processing as they are not subject to flow
# control or priority constraints.
def stopProducing(self):
"""
Stop producing data.
This tells the L{H2Connection} that its consumer has died, so it must
stop producing data for good.
"""
self.connectionLost("stopProducing")
def pauseProducing(self):
"""
Pause producing data.
Tells the L{H2Connection} that it has produced too much data to process
for the time being, and to stop until resumeProducing() is called.
"""
self._consumerBlocked = Deferred()
def resumeProducing(self):
"""
Resume producing data.
This tells the L{H2Connection} to re-add itself to the main loop and
produce more data for the consumer.
"""
if self._consumerBlocked is not None:
d = self._consumerBlocked
self._consumerBlocked = None
d.callback(None)
def _sendPrioritisedData(self, *args):
"""
The data sending loop. This function repeatedly calls itself, either
from L{Deferred}s or from
L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
This function sends data on streams according to the rules of HTTP/2
priority. It ensures that the data from each stream is interleved
according to the priority signalled by the client, making sure that the
connection is used with maximal efficiency.
This function will execute if data is available: if all data is
exhausted, the function will place a deferred onto the L{H2Connection}
object and wait until it is called to resume executing.
"""
# If producing has stopped, we're done. Don't reschedule ourselves
if not self._stillProducing:
return
stream = None
while stream is None:
try:
stream = next(self.priority)
except priority.DeadlockError:
# All streams are currently blocked or not progressing. Wait
# until a new one becomes available.
assert self._sendingDeferred is None
self._sendingDeferred = Deferred()
self._sendingDeferred.addCallback(self._sendPrioritisedData)
return
# Wait behind the transport.
if self._consumerBlocked is not None:
self._consumerBlocked.addCallback(self._sendPrioritisedData)
return
remainingWindow = self.conn.local_flow_control_window(stream)
frameData = self._outboundStreamQueues[stream].popleft()
maxFrameSize = min(self.conn.max_outbound_frame_size, remainingWindow)
if frameData is _END_STREAM_SENTINEL:
# There's no error handling here even though this can throw
# ProtocolError because we really shouldn't encounter this problem.
# If we do, that's a nasty bug.
self.conn.end_stream(stream)
self.transport.write(self.conn.data_to_send())
# Clean up the stream
self._requestDone(stream)
else:
# Respect the max frame size.
if len(frameData) > maxFrameSize:
excessData = frameData[maxFrameSize:]
frameData = frameData[:maxFrameSize]
self._outboundStreamQueues[stream].appendleft(excessData)
# There's deliberately no error handling here, because this just
# absolutely should not happen.
# If for whatever reason the max frame length is zero and so we
# have no frame data to send, don't send any.
if frameData:
self.conn.send_data(stream, frameData)
self.transport.write(self.conn.data_to_send())
# If there's no data left, this stream is now blocked.
if not self._outboundStreamQueues[stream]:
self.priority.block(stream)
# Also, if the stream's flow control window is exhausted, tell it
# to stop.
if self.remainingOutboundWindow(stream) <= 0:
self.streams[stream].flowControlBlocked()
self._reactor.callLater(0, self._sendPrioritisedData)
# Internal functions.
def _requestReceived(self, event):
"""
Internal handler for when a request has been received.
@param event: The Hyper-h2 event that encodes information about the
received request.
@type event: L{h2.events.RequestReceived}
"""
stream = H2Stream(event.stream_id, self, event.headers,
self.requestFactory, self.site, self.factory)
self.streams[event.stream_id] = stream
self._streamCleanupCallbacks[event.stream_id] = Deferred()
self._outboundStreamQueues[event.stream_id] = deque()
# Add the stream to the priority tree but immediately block it.
try:
self.priority.insert_stream(event.stream_id)
except priority.DuplicateStreamError:
# Stream already in the tree. This can happen if we received a
# PRIORITY frame before a HEADERS frame. Just move on: we set the
# stream up properly in _handlePriorityUpdate.
pass
else:
self.priority.block(event.stream_id)
def _requestDataReceived(self, event):
"""
Internal handler for when a chunk of data is received for a given
request.
@param event: The Hyper-h2 event that encodes information about the
received data.
@type event: L{h2.events.DataReceived}
"""
stream = self.streams[event.stream_id]
stream.receiveDataChunk(event.data, event.flow_controlled_length)
def _requestEnded(self, event):
"""
Internal handler for when a request is complete, and we expect no
further data for that request.
@param event: The Hyper-h2 event that encodes information about the
completed stream.
@type event: L{h2.events.StreamEnded}
"""
stream = self.streams[event.stream_id]
stream.requestComplete()
def _requestAborted(self, event):
"""
Internal handler for when a request is aborted by a remote peer.
@param event: The Hyper-h2 event that encodes information about the
reset stream.
@type event: L{h2.events.StreamReset}
"""
stream = self.streams[event.stream_id]
stream.connectionLost("Stream reset")
self._requestDone(event.stream_id)
def _handlePriorityUpdate(self, event):
"""
Internal handler for when a stream priority is updated.
@param event: The Hyper-h2 event that encodes information about the
stream reprioritization.
@type event: L{h2.events.PriorityUpdated}
"""
try:
self.priority.reprioritize(
stream_id=event.stream_id,
depends_on=event.depends_on or None,
weight=event.weight,
exclusive=event.exclusive,
)
except priority.MissingStreamError:
# A PRIORITY frame arrived before the HEADERS frame that would
# trigger us to insert the stream into the tree. That's fine: we
# can create the stream here and mark it as blocked.
self.priority.insert_stream(
stream_id=event.stream_id,
depends_on=event.depends_on or None,
weight=event.weight,
exclusive=event.exclusive,
)
self.priority.block(event.stream_id)
def writeHeaders(self, version, code, reason, headers, streamID):
"""
Called by L{twisted.web.http.Request} objects to write a complete set
of HTTP headers to a stream.
@param version: The HTTP version in use. Unused in HTTP/2.
@type version: L{bytes}
@param code: The HTTP status code to write.
@type code: L{bytes}
@param reason: The HTTP reason phrase to write. Unused in HTTP/2.
@type reason: L{bytes}
@param headers: The headers to write to the stream.
@type headers: L{twisted.web.http_headers.Headers}
@param streamID: The ID of the stream to write the headers to.
@type streamID: L{int}
"""
headers.insert(0, (b':status', code))
try:
self.conn.send_headers(streamID, headers)
except h2.exceptions.StreamClosedError:
# Stream was closed by the client at some point. We need to not
# explode here: just swallow the error. That's what write() does
# when a connection is lost, so that's what we do too.
return
else:
self.transport.write(self.conn.data_to_send())
def writeDataToStream(self, streamID, data):
"""
May be called by L{H2Stream} objects to write response data to a given
stream. Writes a single data frame.
@param streamID: The ID of the stream to write the data to.
@type streamID: L{int}
@param data: The data chunk to write to the stream.
@type data: L{bytes}
"""
self._outboundStreamQueues[streamID].append(data)
# There's obviously no point unblocking this stream and the sending
# loop if the data can't actually be sent, so confirm that there's
# some room to send data.
if self.conn.local_flow_control_window(streamID) > 0:
self.priority.unblock(streamID)
if self._sendingDeferred is not None:
d = self._sendingDeferred
self._sendingDeferred = None
d.callback(streamID)
if self.remainingOutboundWindow(streamID) <= 0:
self.streams[streamID].flowControlBlocked()
def endRequest(self, streamID):
"""
Called by L{H2Stream} objects to signal completion of a response.
@param streamID: The ID of the stream to write the data to.
@type streamID: L{int}
"""
self._outboundStreamQueues[streamID].append(_END_STREAM_SENTINEL)
self.priority.unblock(streamID)
if self._sendingDeferred is not None:
d = self._sendingDeferred
self._sendingDeferred = None
d.callback(streamID)
def abortRequest(self, streamID):
"""
Called by L{H2Stream} objects to request early termination of a stream.
This emits a RstStream frame and then removes all stream state.
@param streamID: The ID of the stream to write the data to.
@type streamID: L{int}
"""
self.conn.reset_stream(streamID)
self.transport.write(self.conn.data_to_send())
self._requestDone(streamID)
def _requestDone(self, streamID):
"""
Called internally by the data sending loop to clean up state that was
being used for the stream. Called when the stream is complete.
@param streamID: The ID of the stream to clean up state for.
@type streamID: L{int}
"""
del self._outboundStreamQueues[streamID]
self.priority.remove_stream(streamID)
del self.streams[streamID]
cleanupCallback = self._streamCleanupCallbacks.pop(streamID)
cleanupCallback.callback(streamID)
def remainingOutboundWindow(self, streamID):
"""
Called to determine how much room is left in the send window for a
given stream. Allows us to handle blocking and unblocking producers.
@param streamID: The ID of the stream whose flow control window we'll
check.
@type streamID: L{int}
@return: The amount of room remaining in the send window for the given
stream, including the data queued to be sent.
@rtype: L{int}
"""
# TODO: This involves a fair bit of looping and computation for
# something that is called a lot. Consider caching values somewhere.
windowSize = self.conn.local_flow_control_window(streamID)
sendQueue = self._outboundStreamQueues[streamID]
alreadyConsumed = sum(
len(chunk) for chunk in sendQueue
if chunk is not _END_STREAM_SENTINEL)
return windowSize - alreadyConsumed
def _handleWindowUpdate(self, event):
"""
Manage flow control windows.
Streams that are blocked on flow control will register themselves with
the connection. This will fire deferreds that wake those streams up and
allow them to continue processing.
@param event: The Hyper-h2 event that encodes information about the
flow control window change.
@type event: L{h2.events.WindowUpdated}
"""
streamID = event.stream_id
if streamID:
if not self._streamIsActive(streamID):
# We may have already cleaned up our stream state, making this
# a late WINDOW_UPDATE frame. That's fine: the update is
# unnecessary but benign. We'll ignore it.
return
# If we haven't got any data to send, don't unblock the stream. If
# we do, we'll eventually get an exception inside the
# _sendPrioritisedData loop some time later.
if self._outboundStreamQueues.get(streamID):
self.priority.unblock(streamID)
self.streams[streamID].windowUpdated()
else:
# Update strictly applies to all streams.
for stream in self.streams.values():
stream.windowUpdated()
# If we still have data to send for this stream, unblock it.
if self._outboundStreamQueues.get(stream.streamID):
self.priority.unblock(stream.streamID)
def getPeer(self):
"""
Get the remote address of this connection.
Treat this method with caution. It is the unfortunate result of the
CGI and Jabber standards, but should not be considered reliable for
the usual host of reasons; port forwarding, proxying, firewalls, IP
masquerading, etc.
@return: An L{IAddress} provider.
"""
return self.transport.getPeer()
def getHost(self):
"""
Similar to getPeer, but returns an address describing this side of the
connection.
@return: An L{IAddress} provider.
"""
return self.transport.getHost()
def openStreamWindow(self, streamID, increment):
"""
Open the stream window by a given increment.
@param streamID: The ID of the stream whose window needs to be opened.
@type streamID: L{int}
@param increment: The amount by which the stream window must be
incremented.
@type increment: L{int}
"""
self.conn.acknowledge_received_data(increment, streamID)
data = self.conn.data_to_send()
if data:
self.transport.write(data)
def _isSecure(self):
"""
Returns L{True} if this channel is using a secure transport.
@returns: L{True} if this channel is secure.
@rtype: L{bool}
"""
# A channel is secure if its transport is ISSLTransport.
return ISSLTransport(self.transport, None) is not None
def _send100Continue(self, streamID):
"""
Sends a 100 Continue response, used to signal to clients that further
processing will be performed.
@param streamID: The ID of the stream that needs the 100 Continue
response
@type streamID: L{int}
"""
headers = [(b':status', b'100')]
self.conn.send_headers(headers=headers, stream_id=streamID)
self.transport.write(self.conn.data_to_send())
def _respondToBadRequestAndDisconnect(self, streamID):
"""
This is a quick and dirty way of responding to bad requests.
As described by HTTP standard we should be patient and accept the
whole request from the client before sending a polite bad request
response, even in the case when clients send tons of data.
Unlike in the HTTP/1.1 case, this does not actually disconnect the
underlying transport: there's no need. This instead just sends a 400
response and terminates the stream.
@param streamID: The ID of the stream that needs the 100 Continue
response
@type streamID: L{int}
"""
headers = [(b':status', b'400')]
self.conn.send_headers(headers=headers,
stream_id=streamID,
end_stream=True)
self.transport.write(self.conn.data_to_send())
stream = self.streams[streamID]
stream.connectionLost("Stream reset")
self._requestDone(streamID)
def _streamIsActive(self, streamID):
"""
Checks whether Twisted has still got state for a given stream and so
can process events for that stream.
@param streamID: The ID of the stream that needs processing.
@type streamID: L{int}
@return: Whether the stream still has state allocated.
@rtype: L{bool}
"""
return streamID in self.streams
class CarbonClientFactory(ReconnectingClientFactory):
maxDelay = 5
def __init__(self, destination):
self.destination = destination
self.destinationName = ('%s:%d:%s' % destination).replace('.', '_')
self.host, self.port, self.carbon_instance = destination
self.addr = (self.host, self.port)
self.started = False
# This factory maintains protocol state across reconnects
self.queue = deque(
) # Change to make this the sole source of metrics to be sent.
self.connectedProtocol = None
self.queueEmpty = Deferred()
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
self.connectFailed = Deferred()
self.connectionMade = Deferred()
self.connectionLost = Deferred()
# Define internal metric names
self.attemptedRelays = 'destinations.%s.attemptedRelays' % self.destinationName
self.fullQueueDrops = 'destinations.%s.fullQueueDrops' % self.destinationName
self.queuedUntilConnected = 'destinations.%s.queuedUntilConnected' % self.destinationName
self.relayMaxQueueLength = 'destinations.%s.relayMaxQueueLength' % self.destinationName
def queueFullCallback(self, result):
state.events.cacheFull()
log.clients('%s send queue is full (%d datapoints)' % (self, result))
def queueSpaceCallback(self, result):
if self.queueFull.called:
log.clients('%s send queue has space available' %
self.connectedProtocol)
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
state.events.cacheSpaceAvailable()
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
def buildProtocol(self, addr):
self.connectedProtocol = CarbonClientProtocol()
self.connectedProtocol.factory = self
return self.connectedProtocol
def startConnecting(
self): # calling this startFactory yields recursion problems
self.started = True
self.connector = reactor.connectTCP(self.host, self.port, self)
def stopConnecting(self):
self.started = False
self.stopTrying()
if self.connectedProtocol and self.connectedProtocol.connected:
return self.connectedProtocol.disconnect()
@property
def queueSize(self):
return len(self.queue)
def hasQueuedDatapoints(self):
return bool(self.queue)
def takeSomeFromQueue(self):
"""Use self.queue, which is a collections.deque, to pop up to
settings.MAX_DATAPOINTS_PER_MESSAGE items from the left of the
queue.
"""
def yield_max_datapoints():
for count in range(settings.MAX_DATAPOINTS_PER_MESSAGE):
try:
yield self.queue.popleft()
except IndexError:
raise StopIteration
return list(yield_max_datapoints())
def checkQueue(self):
"""Check if the queue is empty. If the queue isn't empty or
doesn't exist yet, then this will invoke the callback chain on the
self.queryEmpty Deferred chain with the argument 0, and will
re-set the queueEmpty callback chain with a new Deferred
object.
"""
if not self.queue:
self.queueEmpty.callback(0)
self.queueEmpty = Deferred()
def enqueue(self, metric, datapoint):
self.queue.append((metric, datapoint))
def enqueue_from_left(self, metric, datapoint):
self.queue.appendleft((metric, datapoint))
def sendDatapoint(self, metric, datapoint):
instrumentation.increment(self.attemptedRelays)
instrumentation.max(self.relayMaxQueueLength, self.queueSize)
if self.queueSize >= settings.MAX_QUEUE_SIZE:
if not self.queueFull.called:
self.queueFull.callback(self.queueSize)
instrumentation.increment(self.fullQueueDrops)
else:
self.enqueue(metric, datapoint)
if self.connectedProtocol:
reactor.callLater(settings.TIME_TO_DEFER_SENDING,
self.connectedProtocol.sendQueued)
else:
instrumentation.increment(self.queuedUntilConnected)
def sendHighPriorityDatapoint(self, metric, datapoint):
"""The high priority datapoint is one relating to the carbon
daemon itself. It puts the datapoint on the left of the deque,
ahead of other stats, so that when the carbon-relay, specifically,
is overwhelmed its stats are more likely to make it through and
expose the issue at hand.
In addition, these stats go on the deque even when the max stats
capacity has been reached. This relies on not creating the deque
with a fixed max size.
"""
instrumentation.increment(self.attemptedRelays)
self.enqueue_from_left(metric, datapoint)
if self.connectedProtocol:
reactor.callLater(settings.TIME_TO_DEFER_SENDING,
self.connectedProtocol.sendQueued)
else:
instrumentation.increment(self.queuedUntilConnected)
def startedConnecting(self, connector):
log.clients("%s::startedConnecting (%s:%d)" %
(self, connector.host, connector.port))
def clientConnectionLost(self, connector, reason):
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
log.clients(
"%s::clientConnectionLost (%s:%d) %s" %
(self, connector.host, connector.port, reason.getErrorMessage()))
self.connectedProtocol = None
self.connectionLost.callback(0)
self.connectionLost = Deferred()
def clientConnectionFailed(self, connector, reason):
ReconnectingClientFactory.clientConnectionFailed(
self, connector, reason)
log.clients(
"%s::clientConnectionFailed (%s:%d) %s" %
(self, connector.host, connector.port, reason.getErrorMessage()))
self.connectFailed.callback(dict(connector=connector, reason=reason))
self.connectFailed = Deferred()
def disconnect(self):
self.queueEmpty.addCallback(lambda result: self.stopConnecting())
readyToStop = DeferredList([self.connectionLost, self.connectFailed],
fireOnOneCallback=True,
fireOnOneErrback=True)
self.checkQueue()
# This can happen if the client is stopped before a connection is ever made
if (not readyToStop.called) and (not self.started):
readyToStop.callback(None)
return readyToStop
def __str__(self):
return 'CarbonClientFactory(%s:%d:%s)' % self.destination
__repr__ = __str__
delayed_write('\nas does', 1)
delayed_write('\n the reactor', .6)
delayed_write('\nkeep that', 1)
delayed_write('\n factor', .6)
delayed_write('\nin', 1)
delayed_write(' mind', .4)
delayed_write('\n\n', 2)
def end_chain(_):
print "The end of the callback chain."
from twisted.internet import reactor
reactor.stop()
d = Deferred()
d.addCallback(start_chain)
d.addCallback(blocking_poem)
d.addCallback(end_chain)
def fire():
print 'Firing deferred.'
d.callback(True)
print 'Firing finished.'
from twisted.internet import reactor
reactor.callWhenRunning(fire)
print 'Starting reactor.'
reactor.run()
print 'Done.'
def run(self):
def onExperimentSucceeded(_):
msg("experiment suceeded")
reactor.stop()
def onExperimentFailed(failure):
err("Experiment execution failed, exiting with error.")
err(failure)
if reactor.running:
reactor.stop()
reactor.addSystemEventTrigger('after', 'shutdown', sys.exit, 1)
chdir(self._workspace_dir)
# Step 1:
# Inject all the config options as env variables to give sub-processes easy acces to them.
self.local_env = environ.copy()
self.local_env.update(configToEnv(self._cfg))
self.local_env['LOCAL_RUN'] = 'True'
# Step 2:
# Clear output dir before starting.
output_dir = path.join(self._workspace_dir, 'output')
if path.exists(output_dir):
rmtree(output_dir)
# Step 3:
# Sync the working dir with the head nodes
d = Deferred()
d.addCallback(lambda _: self.copyWorkspaceToHeadNodes())
# Step 4:
# Run the set up script, both locally and in the head nodes
d.addCallback(lambda _: self.runSetupScripts())
# Step 5:
# Start the tracker, either locally or on the first head node of the list.
d.addCallback(lambda _: self.startTracker())
# Step 6:
# Start the config server, always locally if running instances locally as the head nodes are firewalled and
# can only be reached from the outside trough SSH.
d.addCallback(lambda _: self.startExperimentServer())
# Step 7:
# Spawn both local and remote instance runner scripts, which will connect to the config server and wait for all
# of them to be ready before starting the experiment.
d.addCallback(lambda _: self.startInstances())
# Step 8:
# Collect all the data from the remote head nodes.
d.addCallback(lambda _: self.collectOutputFromHeadNodes())
# Step 9:
# Extract the data and graph stuff
d.addCallback(lambda _: self.runPostProcess())
# TODO: From here onwards
reactor.callLater(0, d.callback, None)
# reactor.callLater(60, reactor.stop)
return d.addCallbacks(onExperimentSucceeded, onExperimentFailed)
def handleGetKey_cb(response):
defer = Deferred()
defer.addCallback(method)
response.deliverBody(DataPrinter(defer, "getkey"))
return NOT_DONE_YET
def amqp_incoming_response_to_csr_request(self,
message=None,
properties=None,
correlation_info=None,
**kwargs):
"""
A response from a CSR request has been received. Lets process it.
:param properties: Properties of the AQMP message.
:param body: The message itself.
:param correlation: Any correlation data regarding the AQMP message. We can check for timing, etc.
:return:
"""
logger.info("Received a signed SSL/TLS certificate for: {sslname}",
sslname=self.sslname)
print("sslcert: processing 2")
logger.info("TLS cert body: {message}", message=message)
print("sslcert: processing 3")
if "csr_hash" not in message:
print("sslcert: processing 4")
logger.warn("'csr_hash' is missing from incoming amqp TLS key.")
print("sslcert: processing 5")
return
print("sslcert: processing 10")
csr_hash = message["csr_hash"]
print("sslcert: processing 10")
if csr_hash != self.next_csr_hash:
print("sslcert: processing 10")
logger.warn(
"Incoming TLS (SSL) key hash is mismatched. Discarding. "
"Have: {next_csr_hash}, received: {csr_hash}",
next_csr_hash=self.next_csr_hash,
csr_hash=csr_hash)
return
print("sslcert: processing 20")
self.next_status = message["status"]
self.next_status_msg = message["status_msg"]
if message["status"] == "signed":
self.next_chain_text = message["chain_text"]
self.next_cert_text = message["cert_text"]
self.next_signed_at = message["cert_signed_at"]
self.next_expires_at = message["cert_expires_at"]
self.next_is_valid = True
self.dirty = True
yield self.check_if_rotate_needed(
) # this will rotate next into current
print("sslcert: processing 30")
method = None
if self.current_is_valid is not True:
logger.warn(
"Received a new cert and rotated, but the new cert doesn't seem to be valid."
)
return
print("sslcert: processing 40")
if self.update_callback is not None and isinstance(
self.update_callback, collections.Callable):
method = self.update_callback
elif self.update_callback_type is not None and \
self.update_callback_component is not None and \
self.update_callback_function is not None:
try:
method = self._Parent._Loader.find_function(
self.update_callback_type,
self.update_callback_component,
self.update_callback_function,
)
except YomboWarning as e:
logger.warn(
"Invalid update_callback information provided: {e}", e=e)
print("sslcert: processing 50")
logger.info(
"Method to notify ssl requester that there's a new cert: {method}",
method=method)
if method is not None and isinstance(method, collections.Callable):
logger.info(
"About to tell the SSL/TLS cert requester know we have a new cert, from: {sslname}",
sslname=self.sslname)
print("sslcert: processing 60")
the_cert = self.get()
d = Deferred()
d.addCallback(lambda ignored: maybeDeferred(method, the_cert))
d.addErrback(self.tell_requester_failure)
d.callback(1)
yield d
def handleGetShareMData_cb(response):
defer = Deferred()
defer.addCallback(data[0])
response.deliverBody(DataPrinter(defer, "getmdata"))
return NOT_DONE_YET
def procResponse_cb(response):
defer = Deferred()
defer.addCallback(method)
response.deliverBody(DataPrinter(defer, "bool"))
return NOT_DONE_YET
