def test_unsetAfterCall(self):
"""
After a L{twisted.python.context.call} completes, keys specified in the
call are no longer associated with the values from that call.
"""
context.call({"x": "y"}, lambda: None)
self.assertIsNone(context.get("x"))
def test_unsetAfterCall(self):
"""
After a L{twisted.python.context.call} completes, keys specified in the
call are no longer associated with the values from that call.
"""
context.call({"x": "y"}, lambda: None)
self.assertIsNone(context.get("x"))
def _startSession(self, datagram, addr, mode):
# Set up a call context so that we can pass extra arbitrary
# information to interested backends without adding extra call
# arguments, or switching to using a request object, for example.
context = {}
if self.transport is not None:
# Add the local and remote addresses to the call context.
local = self.transport.getHost()
context["local"] = local.host, local.port
context["remote"] = addr
try:
if datagram.opcode == OP_WRQ:
fs_interface = yield call(context, self.backend.get_writer,
datagram.filename)
elif datagram.opcode == OP_RRQ:
fs_interface = yield call(context, self.backend.get_reader,
datagram.filename)
except Unsupported as e:
self.transport.write(
ERRORDatagram.from_code(
ERR_ILLEGAL_OP,
u"{}".format(e).encode("ascii", "replace")).to_wire(),
addr)
except AccessViolation:
self.transport.write(
ERRORDatagram.from_code(ERR_ACCESS_VIOLATION).to_wire(), addr)
except FileExists:
self.transport.write(
ERRORDatagram.from_code(ERR_FILE_EXISTS).to_wire(), addr)
except FileNotFound:
self.transport.write(
ERRORDatagram.from_code(ERR_FILE_NOT_FOUND).to_wire(), addr)
except BackendError as e:
self.transport.write(
ERRORDatagram.from_code(
ERR_NOT_DEFINED,
u"{}".format(e).encode("ascii", "replace")).to_wire(),
addr)
else:
if datagram.opcode == OP_WRQ:
if mode == b'netascii':
fs_interface = NetasciiReceiverProxy(fs_interface)
session = RemoteOriginWriteSession(addr,
fs_interface,
datagram.options,
_clock=self._clock)
reactor.listenUDP(0, session)
returnValue(session)
elif datagram.opcode == OP_RRQ:
if mode == b'netascii':
fs_interface = NetasciiSenderProxy(fs_interface)
session = RemoteOriginReadSession(addr,
fs_interface,
datagram.options,
_clock=self._clock)
reactor.listenUDP(0, session)
returnValue(session)
def testParentChild(self):
self.numExpectedUpdates = 1
sht = self.sheetHandles['wstest']
sht.attach(self)
cellH = sht().getCellHandle(Col('a'),3)
txnMgr = FormulaTxnManager(cellH)
ctx = {'cell':cellH(),'cache':True}
context.call({'ctx':ctx,'txnMgr':txnMgr},cellH()._ast.eval,1)
d = txnMgr.calcOnFinish()
d.addCallback(self.verifyUpdates,sht)
return d
def getChild(self, path, request):
# This is a bit of a hack, but it allows the `cssRewriter`
# processor to grab the request (which static.File.getChild sadly
# does not pass into it).
return context.call(
{'_BetterFile_last_request': request},
static.File.getChild, self, path, request)
def test_get_reader_config_file(self):
# For paths matching re_config_file, TFTPBackend.get_reader() returns
# a Deferred that will yield a BytesReader.
cluster_uuid = factory.getRandomUUID()
self.patch(tftp_module,
'get_cluster_uuid').return_value = (cluster_uuid)
mac = factory.getRandomMACAddress("-")
config_path = compose_config_path(mac)
backend = TFTPBackend(self.make_dir(), b"http://example.com/")
# python-tx-tftp sets up call context so that backends can discover
# more about the environment in which they're running.
call_context = {
"local": (factory.getRandomIPAddress(), factory.getRandomPort()),
"remote": (factory.getRandomIPAddress(), factory.getRandomPort()),
}
@partial(self.patch, backend, "get_config_reader")
def get_config_reader(params):
params_json = json.dumps(params)
params_json_reader = BytesReader(params_json)
return succeed(params_json_reader)
reader = yield context.call(call_context, backend.get_reader,
config_path)
output = reader.read(10000)
# The addresses provided by python-tx-tftp in the call context are
# passed over the wire as address:port strings.
expected_params = {
"mac": mac,
"local": call_context["local"][0], # address only.
"remote": call_context["remote"][0], # address only.
"cluster_uuid": cluster_uuid,
}
observed_params = json.loads(output)
self.assertEqual(expected_params, observed_params)
def _contextualize(contextFactory, contextReceiver):
"""
Invoke a callable with an argument derived from the current execution
context (L{twisted.python.context}), or automatically created if none is
yet present in the current context.
This function, with a better name and documentation, should probably be
somewhere in L{twisted.python.context}. Calling context.get() and
context.call() individually is perilous because you always have to handle
the case where the value you're looking for isn't present; this idiom
forces you to supply some behavior for that case.
@param contextFactory: An object which is both a 0-arg callable and
hashable; used to look up the value in the context, set the value in the
context, and create the value (by being called).
@param contextReceiver: A function that receives the value created or
identified by contextFactory. It is a 1-arg callable object, called with
the result of calling the contextFactory, or retrieving the contextFactory
from the context.
"""
value = context.get(contextFactory, _NOT_SPECIFIED)
if value is not _NOT_SPECIFIED:
return contextReceiver(value)
else:
return context.call({contextFactory: contextFactory()},
_contextualize, contextFactory, contextReceiver)
def prepareResponse(self, local, remote, file_name, over, *rest):
if over != b"$":
log.error(
"Message not properly terminated: local={local!r} "
"remote={remote!r} file_name={file_name!r} over={over!r} "
"rest={rest!r}",
local=local,
remote=remote,
file_name=file_name,
over=over,
rest=rest)
self.transport.loseConnection()
elif len(rest) != 0:
log.error(
"Message had trailing garbage: local={local!r} "
"remote={remote!r} file_name={file_name!r} over={over!r} "
"rest={rest!r}",
local=local,
remote=remote,
file_name=file_name,
over=over,
rest=rest)
self.transport.loseConnection()
else:
d = context.call(
{
"local": (local.decode(), 0),
"remote": (remote.decode(), 0)
}, self.backend.get_reader, file_name)
d.addCallbacks(self.prepareWriteResponse, self.writeError)
d.addBoth(call, self.transport.loseConnection)
d.addErrback(log.err, "Failure in TFTP back-end.")
def test_get_boot_method_render_substitutes_armhf_in_params(self):
# get_config_reader() should substitute "arm" for "armhf" in the
# arch field of the parameters (mapping from pxe to maas
# namespace).
config_path = b"pxelinux.cfg/default-arm"
backend = TFTPBackend(self.make_dir(), "http://example.com/")
# python-tx-tftp sets up call context so that backends can discover
# more about the environment in which they're running.
call_context = {
"local": (factory.make_ipv4_address(), factory.pick_port()),
"remote": (factory.make_ipv4_address(), factory.pick_port()),
}
@partial(self.patch, backend, "get_boot_method_reader")
def get_boot_method_reader(boot_method, params):
params_json = json.dumps(params).encode("ascii")
params_json_reader = BytesReader(params_json)
return succeed(params_json_reader)
reader = yield context.call(
call_context, backend.get_reader, config_path
)
output = reader.read(10000).decode("ascii")
observed_params = json.loads(output)
# XXX: GavinPanella 2015-11-25 bug=1519804: get_by_pxealias() on
# ArchitectureRegistry is not stable, so we permit either here.
self.assertIn(observed_params["arch"], ["armhf", "arm64"])
def do(self, work):
"""
Perform the given work with the context given to __init__.
@param work: the work to pass on to the real worker.
"""
super().do(lambda: call(self._context, work))
def wrapper(request, *args, **kwargs):
locale = select_locale_by_request(request)
translations = support.Translations.load('translations',
locales=locale,
domain='messages')
ctx = {'locale': locale, 'translations': translations}
return context.call(ctx, fn, request, *args, **kwargs)
def _contextualize(contextFactory, contextReceiver):
"""
Invoke a callable with an argument derived from the current execution
context (L{twisted.python.context}), or automatically created if none is
yet present in the current context.
This function, with a better name and documentation, should probably be
somewhere in L{twisted.python.context}. Calling context.get() and
context.call() individually is perilous because you always have to handle
the case where the value you're looking for isn't present; this idiom
forces you to supply some behavior for that case.
@param contextFactory: An object which is both a 0-arg callable and
hashable; used to look up the value in the context, set the value in the
context, and create the value (by being called).
@param contextReceiver: A function that receives the value created or
identified by contextFactory. It is a 1-arg callable object, called with
the result of calling the contextFactory, or retrieving the contextFactory
from the context.
"""
value = context.get(contextFactory, _NOT_SPECIFIED)
if value is not _NOT_SPECIFIED:
return contextReceiver(value)
else:
return context.call({contextFactory: contextFactory()}, _contextualize,
contextFactory, contextReceiver)
def testCustomRegistry(self):
from twisted.python import context
n = MetaNumber(0)
myReg = components.AdapterRegistry()
myReg.registerAdapter(BackwardsAdder, MetaNumber, IMeta)
self.assertEquals(context.call({components.AdapterRegistry: myReg},
IMeta, n).add(2), -2)
def _test_get_render_file(self, local, remote):
# For paths matching PXEBootMethod.match_path, TFTPBackend.get_reader()
# returns a Deferred that will yield a BytesReader.
mac = factory.make_mac_address("-")
config_path = compose_config_path(mac)
backend = TFTPBackend(self.make_dir(), Mock())
# python-tx-tftp sets up call context so that backends can discover
# more about the environment in which they're running.
call_context = {"local": local, "remote": remote}
@partial(self.patch, backend, "get_boot_method_reader")
def get_boot_method_reader(boot_method, params):
params_json = json.dumps(params).encode("ascii")
params_json_reader = BytesReader(params_json)
return succeed(params_json_reader)
reader = yield context.call(
call_context, backend.get_reader, config_path
)
output = reader.read(10000).decode("ascii")
# The addresses provided by python-tx-tftp in the call context are
# passed over the wire as address:port strings.
expected_params = {
"mac": mac,
"local_ip": call_context["local"][0], # address only.
"remote_ip": call_context["remote"][0], # address only.
"bios_boot_method": "pxe",
}
observed_params = json.loads(output)
self.assertEqual(expected_params, observed_params)
def do(self, work):
"""
Perform the given work with the context given to __init__.
@param work: the work to pass on to the real worker.
"""
super(ContextualWorker, self).do(lambda: call(self._context, work))
def callInThreadWithCallback(self, onResult, func, *args, **kw):
"""
Call a callable object in a separate thread and call C{onResult} with
the return value, or a L{twisted.python.failure.Failure} if the
callable raises an exception.
The callable is allowed to block, but the C{onResult} function must not
block and should perform as little work as possible.
A typical action for C{onResult} for a threadpool used with a Twisted
reactor would be to schedule a L{twisted.internet.defer.Deferred} to
fire in the main reactor thread using C{.callFromThread}. Note that
C{onResult} is called inside the separate thread, not inside the
reactor thread.
@param onResult: a callable with the signature C{(success, result)}.
If the callable returns normally, C{onResult} is called with
C{(True, result)} where C{result} is the return value of the
callable. If the callable throws an exception, C{onResult} is
called with C{(False, failure)}.
Optionally, C{onResult} may be L{None}, in which case it is not
called at all.
@param func: callable object to be called in separate thread
@param args: positional arguments to be passed to C{func}
@param kw: keyword arguments to be passed to C{func}
"""
if self.joined:
return
ctx = context.theContextTracker.currentContext().contexts[-1]
def inContext():
try:
result = inContext.theWork() # type: ignore[attr-defined]
ok = True
except:
result = Failure()
ok = False
inContext.theWork = None # type: ignore[attr-defined]
if inContext.onResult is not None: # type: ignore[attr-defined]
inContext.onResult(ok, result) # type: ignore[attr-defined]
inContext.onResult = None # type: ignore[attr-defined]
elif not ok:
log.err(result)
# Avoid closing over func, ctx, args, kw so that we can carefully
# manage their lifecycle. See
# test_threadCreationArgumentsCallInThreadWithCallback.
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
ctx, func, *args, **kw
)
inContext.onResult = onResult # type: ignore[attr-defined]
self._team.do(inContext)
def _worker(self):
ct = threading.currentThread()
self.threads.append(ct)
while 1:
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
if o == WorkerStop: break
self.working[ct] = ct
ctx, function, args, kwargs = o
try:
context.call(ctx, function, *args, **kwargs)
except:
context.call(ctx, log.deferr)
del self.working[ct]
self.threads.remove(ct)
self.workers = self.workers-1
def _worker(self, o):
ct = threading.currentThread()
while 1:
if o is WorkerStop:
break
elif o is not None:
self.working.append(ct)
ctx, function, args, kwargs = o
try:
context.call(ctx, function, *args, **kwargs)
except:
context.call(ctx, log.deferr)
self.working.remove(ct)
del o, ctx, function, args, kwargs
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def _startSession(self, datagram, addr, mode):
# Set up a call context so that we can pass extra arbitrary
# information to interested backends without adding extra call
# arguments, or switching to using a request object, for example.
context = {}
if self.transport is not None:
# Add the local and remote addresses to the call context.
local = self.transport.getHost()
context["local"] = local.host, local.port
context["remote"] = addr
try:
if datagram.opcode == OP_WRQ:
fs_interface = yield call(
context, self.backend.get_writer, datagram.filename)
elif datagram.opcode == OP_RRQ:
fs_interface = yield call(
context, self.backend.get_reader, datagram.filename)
except Unsupported as e:
self.transport.write(ERRORDatagram.from_code(ERR_ILLEGAL_OP,
u"{}".format(e).encode("ascii", "replace")).to_wire(), addr)
except AccessViolation:
self.transport.write(ERRORDatagram.from_code(ERR_ACCESS_VIOLATION).to_wire(), addr)
except FileExists:
self.transport.write(ERRORDatagram.from_code(ERR_FILE_EXISTS).to_wire(), addr)
except FileNotFound:
self.transport.write(ERRORDatagram.from_code(ERR_FILE_NOT_FOUND).to_wire(), addr)
except BackendError as e:
self.transport.write(ERRORDatagram.from_code(ERR_NOT_DEFINED,
u"{}".format(e).encode("ascii", "replace")).to_wire(), addr)
else:
if datagram.opcode == OP_WRQ:
if mode == b'netascii':
fs_interface = NetasciiReceiverProxy(fs_interface)
session = RemoteOriginWriteSession(addr, fs_interface,
datagram.options, _clock=self._clock)
reactor.listenUDP(0, session)
returnValue(session)
elif datagram.opcode == OP_RRQ:
if mode == b'netascii':
fs_interface = NetasciiSenderProxy(fs_interface)
session = RemoteOriginReadSession(addr, fs_interface,
datagram.options, _clock=self._clock)
reactor.listenUDP(0, session)
returnValue(session)
def callInThreadWithCallback(self, onResult, func, *args, **kw):
"""
Call a callable object in a separate thread and call C{onResult} with
the return value, or a L{twisted.python.failure.Failure} if the
callable raises an exception.
The callable is allowed to block, but the C{onResult} function must not
block and should perform as little work as possible.
A typical action for C{onResult} for a threadpool used with a Twisted
reactor would be to schedule a L{twisted.internet.defer.Deferred} to
fire in the main reactor thread using C{.callFromThread}. Note that
C{onResult} is called inside the separate thread, not inside the
reactor thread.
@param onResult: a callable with the signature C{(success, result)}.
If the callable returns normally, C{onResult} is called with
C{(True, result)} where C{result} is the return value of the
callable. If the callable throws an exception, C{onResult} is
called with C{(False, failure)}.
Optionally, C{onResult} may be C{None}, in which case it is not
called at all.
@param func: callable object to be called in separate thread
@param args: positional arguments to be passed to C{func}
@param kw: keyword arguments to be passed to C{func}
"""
if self.joined:
return
ctx = context.theContextTracker.currentContext().contexts[-1]
def inContext():
try:
result = inContext.theWork()
ok = True
except:
result = Failure()
ok = False
inContext.theWork = None
if inContext.onResult is not None:
inContext.onResult(ok, result)
inContext.onResult = None
elif not ok:
log.err(result)
# Avoid closing over func, ctx, args, kw so that we can carefully
# manage their lifecycle. See
# test_threadCreationArgumentsCallInThreadWithCallback.
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
inContext.onResult = onResult
self._team.do(inContext)
def _worker(self, o):
ct = threading.currentThread()
while 1:
if o is WorkerStop:
break
elif o is not None:
self.working.append(ct)
ctx, function, args, kwargs = o
try:
context.call(ctx, function, *args, **kwargs)
except:
context.call(ctx, log.deferr)
self.working.remove(ct)
del o, ctx, function, args, kwargs
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def test_get_remote_mac(self):
remote_host = factory.make_ipv4_address()
call_context = {
"local": (factory.make_ipv4_address(), factory.pick_port()),
"remote": (remote_host, factory.pick_port()),
}
mock_find = self.patch(boot, "find_mac_via_arp")
yield context.call(call_context, get_remote_mac)
self.assertThat(mock_find, MockCalledOnceWith(remote_host))
def twisted_threadpool_worker(self):
"""A reimplementation of the twisted threadpool worker
This logs the name of the function that is being executed into the thread
name to assist with debugging.
"""
ct = self.currentThread()
o = self.q.get()
while o is not threadpool.WorkerStop:
self.working.append(ct)
ctx, function, args, kwargs = o
# Calculate a function name
name = ct.getName()
try:
funcname = "<unknown>"
username = "******"
try:
adict = args[2][0]
if "name" in adict.keys():
funcname = adict["name"]
except: pass
try:
adict = args[2][1]
if "username" in adict.keys():
username = adict["username"]
except: pass
ct.setName("%s (%s:%s)" % (name, username, funcname))
ct.started_at = time.time()
except:
pass
try:
context.call(ctx, function, *args, **kwargs)
except:
context.call(ctx, twisted_log.deferr)
self.working.remove(ct)
ct.setName(name)
del o, ctx, function, args, kwargs
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def testImpliedCurrencyFormatting(self):
"""
verify that a web service that returns a currency value
is parsed as a dollar amount.
"""
self.numExpectedUpdates = 1
sht = self.sheetHandles['wstest']
sht.attach(self)
cellH = sht().getCellHandle(Col('a'),4)
txnMgr = FormulaTxnManager(cellH)
ctx = {'cell':cellH(),'cache':True}
context.call({'ctx':ctx,'txnMgr':txnMgr},cellH()._ast.eval,1)
d = txnMgr.calcOnFinish()
d.addCallback(self.verifyUpdates,sht)
def checkFormatting(arg,shtH):
self.failUnless(self.getFormat(shtH,'a4') == {'__sht':localedb.ParseCtx.currencyFormat})
d.addCallback(checkFormatting,sht)
return d
def render(self, request):
def finish(content):
content = content.encode('utf-8')
# FIXME: do we need to set the content-length header?
request.setHeader("Content-Type", "text/x-gwt-rpc; charset=utf-8")
request.setHeader("Content-length", len(content))
request.write(content)
request.finish()
procDeferred = context.call({iweb.IResource: request}, self.processRequest, request.content.read())
procDeferred.addBoth(finish).addErrback(log.err)
return server.NOT_DONE_YET
def _startSession(self, datagram, addr, mode):
# Set up a call context so that we can pass extra arbitrary
# information to interested backends without adding extra call
# arguments, or switching to using a request object, for example.
context = {}
if self.transport is not None:
# Add the local and remote addresses to the call context.
local = self.transport.getHost()
context["local"] = local.host, local.port
context["remote"] = addr
try:
if datagram.opcode == OP_WRQ:
fs_interface = yield call(
context, self.backend.get_writer, datagram.filename)
elif datagram.opcode == OP_RRQ:
fs_interface = yield call(
context, self.backend.get_reader, datagram.filename)
except Unsupported, e:
self.transport.write(ERRORDatagram.from_code(ERR_ILLEGAL_OP,
str(e)).to_wire(), addr)
def runHelper():
# Set up event context for the duration of the action
# run. Additionally, handle raised ActionFailures by
# adding their events to the revert event list and
# re-raising them so they will revert the transaction.
try:
return context.call(
{iimaginary.ITransactionalEventBroadcaster: broadcaster},
func, *args, **kwargs)
except eimaginary.ActionFailure, e:
broadcaster.addRevertEvent(e.event.reify())
raise
def _startSession(self, datagram, addr, mode):
# Set up a call context so that we can pass extra arbitrary
# information to interested backends without adding extra call
# arguments, or switching to using a request object, for example.
context = {}
if self.transport is not None:
# Add the local and remote addresses to the call context.
local = self.transport.getHost()
context["local"] = local.host, local.port
context["remote"] = addr
try:
if datagram.opcode == OP_WRQ:
fs_interface = yield call(context, self.backend.get_writer,
datagram.filename)
elif datagram.opcode == OP_RRQ:
fs_interface = yield call(context, self.backend.get_reader,
datagram.filename)
except Unsupported, e:
self.transport.write(
ERRORDatagram.from_code(ERR_ILLEGAL_OP, str(e)).to_wire(),
addr)
def runHelper():
# Set up event context for the duration of the action
# run. Additionally, handle raised ActionFailures by
# adding their events to the revert event list and
# re-raising them so they will revert the transaction.
try:
return context.call(
{iimaginary.ITransactionalEventBroadcaster: broadcaster}, func,
*args, **kwargs)
except eimaginary.ActionFailure, e:
broadcaster.addRevertEvent(e.event.reify())
raise
def _worker(self):
"""
Method used as target of the created threads: retrieve task to run
from the threadpool, run it, and proceed to the next task until
threadpool is stopped.
"""
ct = self.currentThread()
o = self.q.get()
while o is not WorkerStop:
self.working.append(ct)
ctx, function, args, kwargs = o
try:
context.call(ctx, function, *args, **kwargs)
except:
context.call(ctx, log.err)
self.working.remove(ct)
del o, ctx, function, args, kwargs
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def _worker(self):
"""
Method used as target of the created threads: retrieve task to run
from the threadpool, run it, and proceed to the next task until
threadpool is stopped.
"""
ct = self.currentThread()
o = self.q.get()
while o is not WorkerStop:
self.working.append(ct)
ctx, function, args, kwargs = o
try:
context.call(ctx, function, *args, **kwargs)
except:
context.call(ctx, log.err)
self.working.remove(ct)
del o, ctx, function, args, kwargs
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def _worker(self):
"""
Method used as target of the created threads: retrieve task to run
from the threadpool, run it, and proceed to the next task until
threadpool is stopped.
"""
ct = threading.current_thread()
ct.reactor = self._reactor
ct.stats = self._stats
ct.job_id = None
if self._stats:
self._stats.new_thread()
if callable(self._init_thread):
self._init_thread()
self._can_queue_get.wait()
o = self.q.get()
while o is not WorkerStop:
self.busy += 1
ctx, function, args, kwargs, callback, job_id = o
del o
ct.job_id = job_id
try:
ct.call_stats('started_item')
result = context.call(ctx, function, *args, **kwargs)
success = True
except:
success = False
result = failure.Failure()
del function, args, kwargs, job_id
ct.call_stats('finished_item')
self.busy -= 1
ct.job_id = None
callback(success, result)
del ctx, callback, result
self._can_queue_get.wait()
o = self.q.get()
if self._stats:
self._stats.exit_thread()
self._reactor.callFromThread(self._thread_finished, ct)
def _worker(self):
"""
Method used as target of the created threads: retrieve task to run
from the threadpool, run it, and proceed to the next task until
threadpool is stopped.
"""
ct = threading.current_thread()
ct.reactor = self._reactor
ct.stats = self._stats
ct.job_id = None
if self._stats:
self._stats.new_thread()
if callable(self._init_thread):
self._init_thread()
self._can_queue_get.wait()
o = self.q.get()
while o is not WorkerStop:
self.busy += 1
ctx, function, args, kwargs, callback, job_id = o
del o
ct.job_id = job_id
try:
ct.call_stats('started_item')
result = context.call(ctx, function, *args, **kwargs)
success = True
except:
success = False
result = failure.Failure()
del function, args, kwargs, job_id
ct.call_stats('finished_item')
self.busy -= 1
ct.job_id = None
callback(success, result)
del ctx, callback, result
self._can_queue_get.wait()
o = self.q.get()
if self._stats:
self._stats.exit_thread()
self._reactor.callFromThread(self._thread_finished, ct)
def _structured(self):
"""
Retrieve an opaque object which may be usable to construct the
client-side Widgets which correspond to this fragment and all of its
children.
"""
if self._structuredCache is not None:
return self._structuredCache
children = []
requiredModules = []
# Using the context here is terrible but basically necessary given the
# /current/ architecture of Athena and flattening. A better
# implementation which was not tied to the rendering system could avoid
# this.
markup = context.call(
{
'children': children,
'requiredModules': requiredModules
}, flat.flatten,
tags.div(
xmlns="http://www.w3.org/1999/xhtml")[self]).decode('utf-8')
del children[0]
self._structuredCache = {
u'requiredModules': [(name, flat.flatten(url).decode('utf-8'))
for (name, url) in requiredModules],
u'class':
self.jsClass,
u'id':
self._athenaID,
u'initArguments':
tuple(self.getInitialArguments()),
u'markup':
markup,
u'children':
children
}
return self._structuredCache
def callInThread(ctx, func, args, kwargs, d):
"""Call `func` in a newly created thread.
This function does not actually create the thread; this should be called
as the target of a newly created thread. Generally you won't call this
yourself, it will be called by `deferToNewThread`.
:param ctx: A context as a dict; see :module:`twisted.python.context`.
:param func: A callable, typically a function.
:param args: A tuple of positional arguments.
:param kwargs: A dict of keyword arguments.
:param d: A :class:`Deferred` that will be called back with the result of
the function call.
"""
try:
result = context.call(ctx, func, *args, **kwargs)
except:
# Failure() captures the exception information and trackback.
reactor.callFromThread(context.call, ctx, d.errback, Failure())
else:
reactor.callFromThread(context.call, ctx, d.callback, result)
def test_get_reader_config_file(self):
# For paths matching re_config_file, TFTPBackend.get_reader() returns
# a Deferred that will yield a BytesReader.
cluster_uuid = factory.getRandomUUID()
self.patch(tftp_module, 'get_cluster_uuid').return_value = (
cluster_uuid)
mac = factory.getRandomMACAddress(b"-")
config_path = compose_config_path(mac)
backend = TFTPBackend(self.make_dir(), b"http://example.com/")
# python-tx-tftp sets up call context so that backends can discover
# more about the environment in which they're running.
call_context = {
"local": (
factory.getRandomIPAddress(),
factory.getRandomPort()),
"remote": (
factory.getRandomIPAddress(),
factory.getRandomPort()),
}
@partial(self.patch, backend, "get_config_reader")
def get_config_reader(params):
params_json = json.dumps(params)
params_json_reader = BytesReader(params_json)
return succeed(params_json_reader)
reader = yield context.call(
call_context, backend.get_reader, config_path)
output = reader.read(10000)
# The addresses provided by python-tx-tftp in the call context are
# passed over the wire as address:port strings.
expected_params = {
"mac": mac,
"local": call_context["local"][0], # address only.
"remote": call_context["remote"][0], # address only.
"cluster_uuid": cluster_uuid,
}
observed_params = json.loads(output)
self.assertEqual(expected_params, observed_params)
def _worker(self):
"""
Method used as target of the created threads: retrieve a task to run
from the threadpool, run it, and proceed to the next task until
threadpool is stopped.
"""
ct = self.currentThread()
o = self.q.get()
while o is not WorkerStop:
self.working.append(ct)
ctx, function, args, kwargs, onResult = o
del o
try:
result = context.call(ctx, function, *args, **kwargs)
success = True
except:
success = False
if onResult is None:
context.call(ctx, log.err)
result = None
else:
result = failure.Failure()
del function, args, kwargs
self.working.remove(ct)
if onResult is not None:
try:
context.call(ctx, onResult, success, result)
except:
context.call(ctx, log.err)
del ctx, onResult, result
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def _worker(self):
"""
Method used as target of the created threads: retrieve task to run
from the threadpool, run it, and proceed to the next task until
threadpool is stopped.
"""
ct = self.currentThread()
o = self.q.get()
while o is not WorkerStop:
self.working.append(ct)
ctx, function, args, kwargs, onResult = o
del o
try:
result = context.call(ctx, function, *args, **kwargs)
success = True
except:
success = False
if onResult is None:
context.call(ctx, log.err)
result = None
else:
result = failure.Failure()
del function, args, kwargs
self.working.remove(ct)
if onResult is not None:
try:
context.call(ctx, onResult, success, result)
except:
context.call(ctx, log.err)
del ctx, onResult, result
self.waiters.append(ct)
o = self.q.get()
self.waiters.remove(ct)
self.threads.remove(ct)
def test_setByCall(self):
"""
Values may be associated with keys by passing them in a dictionary as
the first argument to L{twisted.python.context.call}.
"""
self.assertEqual(context.call({"x": "y"}, context.get, "x"), "y")
def wrapper(request, *args, **kwargs):
locale = select_locale_by_request(request)
translations = support.Translations.load(
'translations', locales=locale, domain='messages')
ctx = {'locale': locale, 'translations': translations}
return context.call(ctx, fn, request, *args, **kwargs)
def callWithContext(ctx, func, *args, **kw):
newCtx = context.get(ILogContext).copy()
newCtx.update(ctx)
return context.call({ILogContext: newCtx}, func, *args, **kw)
def render_GET(self, request):
# Be sure that the TFTP endpoint is running.
try:
tftp = services.getServiceNamed("tftp")
except KeyError:
# TFTP service is not installed cannot handle a boot request.
request.setResponseCode(503)
return b"HTTP boot service not ready."
# Extract the local servers IP/port of the request.
localHost = request.getHeader("X-Server-Addr")
try:
localPort = int(request.getHeader("X-Server-Port"))
except (TypeError, ValueError):
localPort = 0
# Extract the original clients IP/port of the request.
remoteHost = request.getHeader("X-Forwarded-For")
try:
remotePort = int(request.getHeader("X-Forwarded-Port"))
except (TypeError, ValueError):
remotePort = 0
# localHost and remoteHost are required headers.
if not localHost or not remoteHost:
request.setResponseCode(400)
return b"Missing X-Server-Addr and X-Forwarded-For HTTP headers."
def handleFailure(failure):
if failure.check(AccessViolation):
request.setResponseCode(403)
request.write(b"")
elif failure.check(FileNotFound):
request.setResponseCode(404)
request.write(b"")
else:
log.err(failure, "Failed to handle boot HTTP request.")
request.setResponseCode(500)
request.write(str(failure.value).encode("utf-8"))
request.finish()
def writeResponse(reader):
# Some readers from `tftp` do not provide a way to get the size
# of the generated content. Only set `Content-Length` when size
# can be determined for the response.
if hasattr(reader, "size"):
request.setHeader(b"Content-Length", reader.size)
# The readers from `tftp` use `finish` instead of `close`, but
# `NoRangeStaticProducer` expects `close` instead of `finish`. Map
# `finish` to `close` so the file handlers are cleaned up.
reader.close = reader.finish
# Produce the result without allowing range. This producer will
# call `close` on the reader and `finish` on the request when done.
producer = NoRangeStaticProducer(request, reader)
producer.start()
path = b"/".join(request.postpath)
d = context.call(
{
"local": (localHost, localPort),
"remote": (remoteHost, remotePort),
},
tftp.backend.get_reader,
path,
skip_logging=True,
)
d.addCallback(writeResponse)
d.addErrback(handleFailure)
d.addErrback(log.err, "Failed to handle boot HTTP request.")
# Log the HTTP request to rackd.log and push that event to the
# region controller.
log_path = path.decode("utf-8")
log.info(
"{path} requested by {remoteHost}",
path=log_path,
remoteHost=remoteHost,
)
d = deferLater(
reactor,
0,
send_node_event_ip_address,
event_type=EVENT_TYPES.NODE_HTTP_REQUEST,
ip_address=remoteHost,
description=log_path,
)
d.addErrback(log.err, "Logging HTTP request failed.")
# Response is handled in the defer.
return NOT_DONE_YET
def testBasicContext(self):
self.assertEquals(context.get("x"), None)
self.assertEquals(context.call({"x": "y"}, context.get, "x"), "y")
self.assertEquals(context.get("x"), None)
def test_setByCall(self):
"""
Values may be associated with keys by passing them in a dictionary as
the first argument to L{twisted.python.context.call}.
"""
self.assertEqual(context.call({"x": "y"}, context.get, "x"), "y")
def testBasicContext(self):
self.assertEquals(context.get("x"), None)
self.assertEquals(context.call({"x": "y"}, context.get, "x"), "y")
self.assertEquals(context.get("x"), None)
def getChild(self, path, request):
# This is a bit of a hack, but it allows the `cssRewriter`
# processor to grab the request (which static.File.getChild sadly
# does not pass into it).
return context.call({'_BetterFile_last_request': request},
static.File.getChild, self, path, request)
def test__returns_host_port_tuple_when_set(self):
host, port = factory.make_hostname(), factory.pick_port()
context = {self.context_key: (host, port)}
self.assertEqual((host, port), call(context, self.get_address))
def test_lwc(self):
## Basic implements testing
class IFoo(Interface):
pass
class Foo(object):
implements(IFoo)
assert IFoo in providedBy(Foo)
f = Foo()
assert IFoo(f) is f
## Adaption testing
class IBar(Interface):
pass
class Bar(object):
implements(IBar)
def __init__(self, original=None):
pass
registerAdapter(Bar, Foo, IBar)
b = Bar()
assert IBar in implementedBy(b)
fooBar = IBar(b)
self.assert_(type(fooBar) is Bar)
self.assert_(IBar in implementedBy(fooBar))
# "Bar" was a factory because it took a single argument and
# returned an implementor
# The adapter factory doesn't have to be a class
b2 = Bar()
def fooToBar(obj):
return b2
# Only for testing purposes do we overwrite an old registration
registerAdapter(fooToBar, Foo, IBar, overwrite=True)
self.assert_(IBar(f) is b2)
# "Faceted" lets us install components on an instance-by-instance
# basis
class Pluggable(Faceted):
pass
p = Pluggable()
p[IFoo] = f
p[IBar] = b
self.assert_(IFoo(p) is f)
self.assert_(IBar(p) is b)
# Faceted can have adapter factories registered against them,
# and "remembers" the output
class IBaz(Interface):
pass
def pluggableToBaz(obj):
return 1
registerAdapter(pluggableToBaz, Pluggable, IBaz)
self.assert_(IBaz not in p)
self.assert_(IBaz(p) == 1)
self.assert_(IBaz in p)
## Interface-to-interface adaption, a rarer use case
class IString(Interface):
pass
class IFile(Interface):
pass
class Stringlike(object):
implements(IString)
def stringToFile(s):
import StringIO
return StringIO.StringIO(s)
registerAdapter(stringToFile, IString, IFile)
self.assert_(hasattr(IFile(Stringlike()), 'read'))
registerAdapter(stringToFile, str, IFile)
self.assert_(hasattr(IFile("Hello"), 'read'))
## Declare that str implements IFoo
declareImplements(str, IFoo)
foo = "Foo"
self.assert_(IFoo in implementedBy(foo))
## Register an interface-to-interface adapter for IFoo to IBar
registerAdapter(fooToBar, IFoo, IBar)
## Now we can adapt strings to IBar
self.assert_(IBar("Foo") is b2)
if sys.version_info >= (2,4):
from shtoom.test.py24tests import decoratedLWCTest
takesABar = decoratedLWCTest(IBar, b2)
self.assert_(takesABar("This is not a bar, but can be adapted."))
## We have context-sensitive adapter registries
newRegistry = AdapterRegistry()
newRegistry.registerAdapter(lambda x: 1, str, IBar)
def tryToAdaptToIBar(something):
return IBar(something)
result = context.call({IAdapterRegistry: newRegistry},
tryToAdaptToIBar, "A string")
self.assert_(result == 1)
def test__returns_host_port_tuple_even_when_set_longer(self):
# Only the first two elements from the context's value are used.
host, port = factory.make_hostname(), factory.pick_port()
context = {self.context_key: (host, port, factory.make_name("thing"))}
self.assertEqual((host, port), call(context, self.get_address))
def callWithContext(ctx, func, *args, **kw):
newCtx = context.get(ILogContext).copy()
newCtx.update(ctx)
return context.call({ILogContext: newCtx}, func, *args, **kw)
def process_message(message_sender, client_application_message, server_application_message, async_client_factory,
encryption_context, server_response_id, system_auth_header, shard_id):
device_id = py23.b64encode_to_str(message_sender)
encryption_context = encryption_context
if client_application_message.HasField(CLIENT_SINGLE_REQUEST):
request_object = client_application_message.clientSingleRequest
response_object = server_application_message.serverSingleResponse
response_object.replyToMessageId = request_object.requestId
response_object.serverVersion = str(app_version())
processor = process_request
request_type = CLIENT_SINGLE_REQUEST
elif client_application_message.HasField(CLIENT_SUBSCRIPTION_MESSAGE):
request_object = client_application_message.clientSubscriptionMessage
# pings have a special case response, really they could be in their own modular input
if client_application_message.clientSubscriptionMessage.HasField(RequestType.CLIENT_SUBSCRIPTION_PING.value):
response_object = server_application_message.serverSubscriptionPing
else:
response_object = server_application_message.serverSubscriptionResponse
response_object.replyToMessageId = request_object.requestId
response_object.serverVersion = str(app_version())
processor = process_subscription
request_type = CLIENT_SUBSCRIPTION_MESSAGE
else:
LOGGER.warn("No suitable message type found client application request")
defer.returnValue("device_id={}".format(device_id))
response_object.requestId = server_response_id
if request_object.HasField("runAs"):
LOGGER.debug("run as credentials is present")
auth_header = yield parse_run_as_credentials(encryption_context, system_auth_header,
async_client_factory, request_object.runAs)
else:
auth_header = parse_session_token(encryption_context, request_object.sessionToken)
request_context = RequestContext(auth_header, device_id=device_id,
raw_device_id=message_sender, request_id=request_object.requestId,
current_user=auth_header.username, system_auth_header=system_auth_header,
client_version=request_object.clientVersion, user_agent=request_object.userAgent,
shard_id=shard_id)
try:
validate_client_version(request_object, response_object)
yield auth_header.validate(async_client_factory.splunk_client(), LOGGER, request_context)
should_send_response = yield context.call({'request_context': request_context}, processor, request_context,
encryption_context, request_object, response_object,
async_client_factory)
# If we aren't sending a response in this request clear the server_application_message
if not should_send_response:
server_application_message.Clear()
except OperationHaltedError:
server_application_message.ClearField('app_message')
except SpacebridgeError as e:
LOGGER.exception("SpacebridgeError during process_message")
e.set_proto(response_object)
except Exception as e:
LOGGER.exception("Unhandled exception during process_message")
response_object.error.code = common_pb2.Error.ERROR_UNKNOWN
response_object.error.message = str(e)
LOGGER.info('Finished processing message. {}'.format(request_context))
defer.returnValue(request_context)