Exemple #1
0
 def test_unsetAfterCall(self):
     """
     After a L{twisted.python.context.call} completes, keys specified in the
     call are no longer associated with the values from that call.
     """
     context.call({"x": "y"}, lambda: None)
     self.assertIsNone(context.get("x"))
 def test_unsetAfterCall(self):
     """
     After a L{twisted.python.context.call} completes, keys specified in the
     call are no longer associated with the values from that call.
     """
     context.call({"x": "y"}, lambda: None)
     self.assertIsNone(context.get("x"))
Exemple #3
0
 def _startSession(self, datagram, addr, mode):
     # Set up a call context so that we can pass extra arbitrary
     # information to interested backends without adding extra call
     # arguments, or switching to using a request object, for example.
     context = {}
     if self.transport is not None:
         # Add the local and remote addresses to the call context.
         local = self.transport.getHost()
         context["local"] = local.host, local.port
         context["remote"] = addr
     try:
         if datagram.opcode == OP_WRQ:
             fs_interface = yield call(context, self.backend.get_writer,
                                       datagram.filename)
         elif datagram.opcode == OP_RRQ:
             fs_interface = yield call(context, self.backend.get_reader,
                                       datagram.filename)
     except Unsupported as e:
         self.transport.write(
             ERRORDatagram.from_code(
                 ERR_ILLEGAL_OP,
                 u"{}".format(e).encode("ascii", "replace")).to_wire(),
             addr)
     except AccessViolation:
         self.transport.write(
             ERRORDatagram.from_code(ERR_ACCESS_VIOLATION).to_wire(), addr)
     except FileExists:
         self.transport.write(
             ERRORDatagram.from_code(ERR_FILE_EXISTS).to_wire(), addr)
     except FileNotFound:
         self.transport.write(
             ERRORDatagram.from_code(ERR_FILE_NOT_FOUND).to_wire(), addr)
     except BackendError as e:
         self.transport.write(
             ERRORDatagram.from_code(
                 ERR_NOT_DEFINED,
                 u"{}".format(e).encode("ascii", "replace")).to_wire(),
             addr)
     else:
         if datagram.opcode == OP_WRQ:
             if mode == b'netascii':
                 fs_interface = NetasciiReceiverProxy(fs_interface)
             session = RemoteOriginWriteSession(addr,
                                                fs_interface,
                                                datagram.options,
                                                _clock=self._clock)
             reactor.listenUDP(0, session)
             returnValue(session)
         elif datagram.opcode == OP_RRQ:
             if mode == b'netascii':
                 fs_interface = NetasciiSenderProxy(fs_interface)
             session = RemoteOriginReadSession(addr,
                                               fs_interface,
                                               datagram.options,
                                               _clock=self._clock)
             reactor.listenUDP(0, session)
             returnValue(session)
Exemple #4
0
 def testParentChild(self):
     self.numExpectedUpdates = 1
     
     sht = self.sheetHandles['wstest']
     sht.attach(self)
     cellH = sht().getCellHandle(Col('a'),3)
     txnMgr = FormulaTxnManager(cellH)
     ctx = {'cell':cellH(),'cache':True}
     context.call({'ctx':ctx,'txnMgr':txnMgr},cellH()._ast.eval,1)
     d = txnMgr.calcOnFinish()
     d.addCallback(self.verifyUpdates,sht)
     return d        
Exemple #5
0
	def getChild(self, path, request):
		# This is a bit of a hack, but it allows the `cssRewriter`
		# processor to grab the request (which static.File.getChild sadly
		# does not pass into it).
		return context.call(
			{'_BetterFile_last_request': request},
			static.File.getChild, self, path, request)
Exemple #6
0
    def test_get_reader_config_file(self):
        # For paths matching re_config_file, TFTPBackend.get_reader() returns
        # a Deferred that will yield a BytesReader.
        cluster_uuid = factory.getRandomUUID()
        self.patch(tftp_module,
                   'get_cluster_uuid').return_value = (cluster_uuid)
        mac = factory.getRandomMACAddress("-")
        config_path = compose_config_path(mac)
        backend = TFTPBackend(self.make_dir(), b"http://example.com/")
        # python-tx-tftp sets up call context so that backends can discover
        # more about the environment in which they're running.
        call_context = {
            "local": (factory.getRandomIPAddress(), factory.getRandomPort()),
            "remote": (factory.getRandomIPAddress(), factory.getRandomPort()),
        }

        @partial(self.patch, backend, "get_config_reader")
        def get_config_reader(params):
            params_json = json.dumps(params)
            params_json_reader = BytesReader(params_json)
            return succeed(params_json_reader)

        reader = yield context.call(call_context, backend.get_reader,
                                    config_path)
        output = reader.read(10000)
        # The addresses provided by python-tx-tftp in the call context are
        # passed over the wire as address:port strings.
        expected_params = {
            "mac": mac,
            "local": call_context["local"][0],  # address only.
            "remote": call_context["remote"][0],  # address only.
            "cluster_uuid": cluster_uuid,
        }
        observed_params = json.loads(output)
        self.assertEqual(expected_params, observed_params)
Exemple #7
0
def _contextualize(contextFactory, contextReceiver):
    """
    Invoke a callable with an argument derived from the current execution
    context (L{twisted.python.context}), or automatically created if none is
    yet present in the current context.

    This function, with a better name and documentation, should probably be
    somewhere in L{twisted.python.context}.  Calling context.get() and
    context.call() individually is perilous because you always have to handle
    the case where the value you're looking for isn't present; this idiom
    forces you to supply some behavior for that case.

    @param contextFactory: An object which is both a 0-arg callable and
    hashable; used to look up the value in the context, set the value in the
    context, and create the value (by being called).

    @param contextReceiver: A function that receives the value created or
    identified by contextFactory.  It is a 1-arg callable object, called with
    the result of calling the contextFactory, or retrieving the contextFactory
    from the context.
    """
    value = context.get(contextFactory, _NOT_SPECIFIED)
    if value is not _NOT_SPECIFIED:
        return contextReceiver(value)
    else:
        return context.call({contextFactory: contextFactory()},
                            _contextualize, contextFactory, contextReceiver)
Exemple #8
0
 def prepareResponse(self, local, remote, file_name, over, *rest):
     if over != b"$":
         log.error(
             "Message not properly terminated: local={local!r} "
             "remote={remote!r} file_name={file_name!r} over={over!r} "
             "rest={rest!r}",
             local=local,
             remote=remote,
             file_name=file_name,
             over=over,
             rest=rest)
         self.transport.loseConnection()
     elif len(rest) != 0:
         log.error(
             "Message had trailing garbage: local={local!r} "
             "remote={remote!r} file_name={file_name!r} over={over!r} "
             "rest={rest!r}",
             local=local,
             remote=remote,
             file_name=file_name,
             over=over,
             rest=rest)
         self.transport.loseConnection()
     else:
         d = context.call(
             {
                 "local": (local.decode(), 0),
                 "remote": (remote.decode(), 0)
             }, self.backend.get_reader, file_name)
         d.addCallbacks(self.prepareWriteResponse, self.writeError)
         d.addBoth(call, self.transport.loseConnection)
         d.addErrback(log.err, "Failure in TFTP back-end.")
Exemple #9
0
    def test_get_boot_method_render_substitutes_armhf_in_params(self):
        # get_config_reader() should substitute "arm" for "armhf" in the
        # arch field of the parameters (mapping from pxe to maas
        # namespace).
        config_path = b"pxelinux.cfg/default-arm"
        backend = TFTPBackend(self.make_dir(), "http://example.com/")
        # python-tx-tftp sets up call context so that backends can discover
        # more about the environment in which they're running.
        call_context = {
            "local": (factory.make_ipv4_address(), factory.pick_port()),
            "remote": (factory.make_ipv4_address(), factory.pick_port()),
        }

        @partial(self.patch, backend, "get_boot_method_reader")
        def get_boot_method_reader(boot_method, params):
            params_json = json.dumps(params).encode("ascii")
            params_json_reader = BytesReader(params_json)
            return succeed(params_json_reader)

        reader = yield context.call(
            call_context, backend.get_reader, config_path
        )
        output = reader.read(10000).decode("ascii")
        observed_params = json.loads(output)
        # XXX: GavinPanella 2015-11-25 bug=1519804: get_by_pxealias() on
        # ArchitectureRegistry is not stable, so we permit either here.
        self.assertIn(observed_params["arch"], ["armhf", "arm64"])
Exemple #10
0
    def do(self, work):
        """
        Perform the given work with the context given to __init__.

        @param work: the work to pass on to the real worker.
        """
        super().do(lambda: call(self._context, work))
Exemple #11
0
 def wrapper(request, *args, **kwargs):
     locale = select_locale_by_request(request)
     translations = support.Translations.load('translations',
                                              locales=locale,
                                              domain='messages')
     ctx = {'locale': locale, 'translations': translations}
     return context.call(ctx, fn, request, *args, **kwargs)
Exemple #12
0
def _contextualize(contextFactory, contextReceiver):
    """
    Invoke a callable with an argument derived from the current execution
    context (L{twisted.python.context}), or automatically created if none is
    yet present in the current context.

    This function, with a better name and documentation, should probably be
    somewhere in L{twisted.python.context}.  Calling context.get() and
    context.call() individually is perilous because you always have to handle
    the case where the value you're looking for isn't present; this idiom
    forces you to supply some behavior for that case.

    @param contextFactory: An object which is both a 0-arg callable and
    hashable; used to look up the value in the context, set the value in the
    context, and create the value (by being called).

    @param contextReceiver: A function that receives the value created or
    identified by contextFactory.  It is a 1-arg callable object, called with
    the result of calling the contextFactory, or retrieving the contextFactory
    from the context.
    """
    value = context.get(contextFactory, _NOT_SPECIFIED)
    if value is not _NOT_SPECIFIED:
        return contextReceiver(value)
    else:
        return context.call({contextFactory: contextFactory()}, _contextualize,
                            contextFactory, contextReceiver)
Exemple #13
0
 def testCustomRegistry(self):
     from twisted.python import context
     n = MetaNumber(0)
     myReg = components.AdapterRegistry()
     myReg.registerAdapter(BackwardsAdder, MetaNumber, IMeta)
     self.assertEquals(context.call({components.AdapterRegistry: myReg},
                                    IMeta, n).add(2), -2)
Exemple #14
0
    def _test_get_render_file(self, local, remote):
        # For paths matching PXEBootMethod.match_path, TFTPBackend.get_reader()
        # returns a Deferred that will yield a BytesReader.
        mac = factory.make_mac_address("-")
        config_path = compose_config_path(mac)
        backend = TFTPBackend(self.make_dir(), Mock())
        # python-tx-tftp sets up call context so that backends can discover
        # more about the environment in which they're running.
        call_context = {"local": local, "remote": remote}

        @partial(self.patch, backend, "get_boot_method_reader")
        def get_boot_method_reader(boot_method, params):
            params_json = json.dumps(params).encode("ascii")
            params_json_reader = BytesReader(params_json)
            return succeed(params_json_reader)

        reader = yield context.call(
            call_context, backend.get_reader, config_path
        )
        output = reader.read(10000).decode("ascii")
        # The addresses provided by python-tx-tftp in the call context are
        # passed over the wire as address:port strings.
        expected_params = {
            "mac": mac,
            "local_ip": call_context["local"][0],  # address only.
            "remote_ip": call_context["remote"][0],  # address only.
            "bios_boot_method": "pxe",
        }
        observed_params = json.loads(output)
        self.assertEqual(expected_params, observed_params)
Exemple #15
0
    def do(self, work):
        """
        Perform the given work with the context given to __init__.

        @param work: the work to pass on to the real worker.
        """
        super(ContextualWorker, self).do(lambda: call(self._context, work))
Exemple #16
0
    def callInThreadWithCallback(self, onResult, func, *args, **kw):
        """
        Call a callable object in a separate thread and call C{onResult} with
        the return value, or a L{twisted.python.failure.Failure} if the
        callable raises an exception.

        The callable is allowed to block, but the C{onResult} function must not
        block and should perform as little work as possible.

        A typical action for C{onResult} for a threadpool used with a Twisted
        reactor would be to schedule a L{twisted.internet.defer.Deferred} to
        fire in the main reactor thread using C{.callFromThread}.  Note that
        C{onResult} is called inside the separate thread, not inside the
        reactor thread.

        @param onResult: a callable with the signature C{(success, result)}.
            If the callable returns normally, C{onResult} is called with
            C{(True, result)} where C{result} is the return value of the
            callable.  If the callable throws an exception, C{onResult} is
            called with C{(False, failure)}.

            Optionally, C{onResult} may be L{None}, in which case it is not
            called at all.

        @param func: callable object to be called in separate thread

        @param args: positional arguments to be passed to C{func}

        @param kw: keyword arguments to be passed to C{func}
        """
        if self.joined:
            return
        ctx = context.theContextTracker.currentContext().contexts[-1]

        def inContext():
            try:
                result = inContext.theWork()  # type: ignore[attr-defined]
                ok = True
            except:
                result = Failure()
                ok = False

            inContext.theWork = None  # type: ignore[attr-defined]
            if inContext.onResult is not None:  # type: ignore[attr-defined]
                inContext.onResult(ok, result)  # type: ignore[attr-defined]
                inContext.onResult = None  # type: ignore[attr-defined]
            elif not ok:
                log.err(result)

        # Avoid closing over func, ctx, args, kw so that we can carefully
        # manage their lifecycle.  See
        # test_threadCreationArgumentsCallInThreadWithCallback.
        inContext.theWork = lambda: context.call(  # type: ignore[attr-defined]
            ctx, func, *args, **kw
        )
        inContext.onResult = onResult  # type: ignore[attr-defined]

        self._team.do(inContext)
Exemple #17
0
 def _worker(self):
     ct = threading.currentThread()
     self.threads.append(ct)
     
     while 1:
         self.waiters.append(ct)
         o = self.q.get()
         self.waiters.remove(ct)
         if o == WorkerStop: break
         self.working[ct] = ct
         ctx, function, args, kwargs = o
         try:
             context.call(ctx, function, *args, **kwargs)
         except:
             context.call(ctx, log.deferr)
         del self.working[ct]
     self.threads.remove(ct)
     self.workers = self.workers-1
Exemple #18
0
 def _worker(self, o):
     ct = threading.currentThread()
     while 1:
         if o is WorkerStop:
             break
         elif o is not None:
             self.working.append(ct)
             ctx, function, args, kwargs = o
             try:
                 context.call(ctx, function, *args, **kwargs)
             except:
                 context.call(ctx, log.deferr)
             self.working.remove(ct)
             del o, ctx, function, args, kwargs
         self.waiters.append(ct)
         o = self.q.get()
         self.waiters.remove(ct)
     self.threads.remove(ct)
Exemple #19
0
 def _startSession(self, datagram, addr, mode):
     # Set up a call context so that we can pass extra arbitrary
     # information to interested backends without adding extra call
     # arguments, or switching to using a request object, for example.
     context = {}
     if self.transport is not None:
         # Add the local and remote addresses to the call context.
         local = self.transport.getHost()
         context["local"] = local.host, local.port
         context["remote"] = addr
     try:
         if datagram.opcode == OP_WRQ:
             fs_interface = yield call(
                 context, self.backend.get_writer, datagram.filename)
         elif datagram.opcode == OP_RRQ:
             fs_interface = yield call(
                 context, self.backend.get_reader, datagram.filename)
     except Unsupported as e:
         self.transport.write(ERRORDatagram.from_code(ERR_ILLEGAL_OP,
             u"{}".format(e).encode("ascii", "replace")).to_wire(), addr)
     except AccessViolation:
         self.transport.write(ERRORDatagram.from_code(ERR_ACCESS_VIOLATION).to_wire(), addr)
     except FileExists:
         self.transport.write(ERRORDatagram.from_code(ERR_FILE_EXISTS).to_wire(), addr)
     except FileNotFound:
         self.transport.write(ERRORDatagram.from_code(ERR_FILE_NOT_FOUND).to_wire(), addr)
     except BackendError as e:
         self.transport.write(ERRORDatagram.from_code(ERR_NOT_DEFINED,
             u"{}".format(e).encode("ascii", "replace")).to_wire(), addr)
     else:
         if datagram.opcode == OP_WRQ:
             if mode == b'netascii':
                 fs_interface = NetasciiReceiverProxy(fs_interface)
             session = RemoteOriginWriteSession(addr, fs_interface,
                                                datagram.options, _clock=self._clock)
             reactor.listenUDP(0, session)
             returnValue(session)
         elif datagram.opcode == OP_RRQ:
             if mode == b'netascii':
                 fs_interface = NetasciiSenderProxy(fs_interface)
             session = RemoteOriginReadSession(addr, fs_interface,
                                               datagram.options, _clock=self._clock)
             reactor.listenUDP(0, session)
             returnValue(session)
Exemple #20
0
    def callInThreadWithCallback(self, onResult, func, *args, **kw):
        """
        Call a callable object in a separate thread and call C{onResult} with
        the return value, or a L{twisted.python.failure.Failure} if the
        callable raises an exception.

        The callable is allowed to block, but the C{onResult} function must not
        block and should perform as little work as possible.

        A typical action for C{onResult} for a threadpool used with a Twisted
        reactor would be to schedule a L{twisted.internet.defer.Deferred} to
        fire in the main reactor thread using C{.callFromThread}.  Note that
        C{onResult} is called inside the separate thread, not inside the
        reactor thread.

        @param onResult: a callable with the signature C{(success, result)}.
            If the callable returns normally, C{onResult} is called with
            C{(True, result)} where C{result} is the return value of the
            callable.  If the callable throws an exception, C{onResult} is
            called with C{(False, failure)}.

            Optionally, C{onResult} may be C{None}, in which case it is not
            called at all.

        @param func: callable object to be called in separate thread

        @param args: positional arguments to be passed to C{func}

        @param kw: keyword arguments to be passed to C{func}
        """
        if self.joined:
            return
        ctx = context.theContextTracker.currentContext().contexts[-1]

        def inContext():
            try:
                result = inContext.theWork()
                ok = True
            except:
                result = Failure()
                ok = False

            inContext.theWork = None
            if inContext.onResult is not None:
                inContext.onResult(ok, result)
                inContext.onResult = None
            elif not ok:
                log.err(result)

        # Avoid closing over func, ctx, args, kw so that we can carefully
        # manage their lifecycle.  See
        # test_threadCreationArgumentsCallInThreadWithCallback.
        inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
        inContext.onResult = onResult

        self._team.do(inContext)
Exemple #21
0
    def _worker(self, o):
        ct = threading.currentThread()
        while 1:
            if o is WorkerStop:
                break
            elif o is not None:
                self.working.append(ct)
                ctx, function, args, kwargs = o
                try:
                    context.call(ctx, function, *args, **kwargs)
                except:
                    context.call(ctx, log.deferr)
                self.working.remove(ct)
                del o, ctx, function, args, kwargs
            self.waiters.append(ct)
            o = self.q.get()
            self.waiters.remove(ct)

        self.threads.remove(ct)
Exemple #22
0
    def test_get_remote_mac(self):
        remote_host = factory.make_ipv4_address()
        call_context = {
            "local": (factory.make_ipv4_address(), factory.pick_port()),
            "remote": (remote_host, factory.pick_port()),
        }

        mock_find = self.patch(boot, "find_mac_via_arp")
        yield context.call(call_context, get_remote_mac)
        self.assertThat(mock_find, MockCalledOnceWith(remote_host))
Exemple #23
0
def twisted_threadpool_worker(self):
    """A reimplementation of the twisted threadpool worker

    This logs the name of the function that is being executed into the thread
    name to assist with debugging.
    """
    ct = self.currentThread()
    o = self.q.get()
    while o is not threadpool.WorkerStop:
        self.working.append(ct)
        ctx, function, args, kwargs = o
        # Calculate a function name
        name = ct.getName()
        try:
            funcname = "<unknown>"
            username = "******"
            try:
                adict = args[2][0]
                if "name" in adict.keys():
                    funcname = adict["name"]
            except: pass
            try:
                adict = args[2][1]
                if "username" in adict.keys():
                    username = adict["username"]
            except: pass
            ct.setName("%s (%s:%s)" % (name, username, funcname))
            ct.started_at = time.time()
        except:
            pass
        try:
            context.call(ctx, function, *args, **kwargs)
        except:
            context.call(ctx, twisted_log.deferr)
        self.working.remove(ct)
        ct.setName(name)
        del o, ctx, function, args, kwargs
        self.waiters.append(ct)
        o = self.q.get()
        self.waiters.remove(ct)
    
    self.threads.remove(ct)
Exemple #24
0
    def testImpliedCurrencyFormatting(self):
        """
        verify that a web service that returns a currency value
        is parsed as a dollar amount.
        """
        self.numExpectedUpdates = 1
        
        sht = self.sheetHandles['wstest']
        sht.attach(self)
        cellH = sht().getCellHandle(Col('a'),4)
        txnMgr = FormulaTxnManager(cellH)
        ctx = {'cell':cellH(),'cache':True}
        context.call({'ctx':ctx,'txnMgr':txnMgr},cellH()._ast.eval,1)
        d = txnMgr.calcOnFinish()
        d.addCallback(self.verifyUpdates,sht)

        def checkFormatting(arg,shtH):
            self.failUnless(self.getFormat(shtH,'a4') == {'__sht':localedb.ParseCtx.currencyFormat})
        d.addCallback(checkFormatting,sht)
        return d
Exemple #25
0
 def render(self, request):
     def finish(content):
         content = content.encode('utf-8')
         # FIXME: do we need to set the content-length header?
         request.setHeader("Content-Type", "text/x-gwt-rpc; charset=utf-8")
         request.setHeader("Content-length", len(content))
         request.write(content)
         request.finish()
     procDeferred = context.call({iweb.IResource: request}, self.processRequest, request.content.read())
     procDeferred.addBoth(finish).addErrback(log.err)
     return server.NOT_DONE_YET
Exemple #26
0
 def _startSession(self, datagram, addr, mode):
     # Set up a call context so that we can pass extra arbitrary
     # information to interested backends without adding extra call
     # arguments, or switching to using a request object, for example.
     context = {}
     if self.transport is not None:
         # Add the local and remote addresses to the call context.
         local = self.transport.getHost()
         context["local"] = local.host, local.port
         context["remote"] = addr
     try:
         if datagram.opcode == OP_WRQ:
             fs_interface = yield call(
                 context, self.backend.get_writer, datagram.filename)
         elif datagram.opcode == OP_RRQ:
             fs_interface = yield call(
                 context, self.backend.get_reader, datagram.filename)
     except Unsupported, e:
         self.transport.write(ERRORDatagram.from_code(ERR_ILLEGAL_OP,
                                 str(e)).to_wire(), addr)
Exemple #27
0
 def runHelper():
     # Set up event context for the duration of the action
     # run.  Additionally, handle raised ActionFailures by
     # adding their events to the revert event list and
     # re-raising them so they will revert the transaction.
     try:
         return context.call(
             {iimaginary.ITransactionalEventBroadcaster: broadcaster},
             func, *args, **kwargs)
     except eimaginary.ActionFailure, e:
         broadcaster.addRevertEvent(e.event.reify())
         raise
Exemple #28
0
 def _startSession(self, datagram, addr, mode):
     # Set up a call context so that we can pass extra arbitrary
     # information to interested backends without adding extra call
     # arguments, or switching to using a request object, for example.
     context = {}
     if self.transport is not None:
         # Add the local and remote addresses to the call context.
         local = self.transport.getHost()
         context["local"] = local.host, local.port
         context["remote"] = addr
     try:
         if datagram.opcode == OP_WRQ:
             fs_interface = yield call(context, self.backend.get_writer,
                                       datagram.filename)
         elif datagram.opcode == OP_RRQ:
             fs_interface = yield call(context, self.backend.get_reader,
                                       datagram.filename)
     except Unsupported, e:
         self.transport.write(
             ERRORDatagram.from_code(ERR_ILLEGAL_OP, str(e)).to_wire(),
             addr)
Exemple #29
0
 def runHelper():
     # Set up event context for the duration of the action
     # run.  Additionally, handle raised ActionFailures by
     # adding their events to the revert event list and
     # re-raising them so they will revert the transaction.
     try:
         return context.call(
             {iimaginary.ITransactionalEventBroadcaster: broadcaster}, func,
             *args, **kwargs)
     except eimaginary.ActionFailure, e:
         broadcaster.addRevertEvent(e.event.reify())
         raise
    def _worker(self):
        """
        Method used as target of the created threads: retrieve task to run
        from the threadpool, run it, and proceed to the next task until
        threadpool is stopped.
        """
        ct = self.currentThread()
        o = self.q.get()
        while o is not WorkerStop:
            self.working.append(ct)
            ctx, function, args, kwargs = o
            try:
                context.call(ctx, function, *args, **kwargs)
            except:
                context.call(ctx, log.err)
            self.working.remove(ct)
            del o, ctx, function, args, kwargs
            self.waiters.append(ct)
            o = self.q.get()
            self.waiters.remove(ct)

        self.threads.remove(ct)
Exemple #31
0
    def _worker(self):
        """
        Method used as target of the created threads: retrieve task to run
        from the threadpool, run it, and proceed to the next task until
        threadpool is stopped.
        """
        ct = self.currentThread()
        o = self.q.get()
        while o is not WorkerStop:
            self.working.append(ct)
            ctx, function, args, kwargs = o
            try:
                context.call(ctx, function, *args, **kwargs)
            except:
                context.call(ctx, log.err)
            self.working.remove(ct)
            del o, ctx, function, args, kwargs
            self.waiters.append(ct)
            o = self.q.get()
            self.waiters.remove(ct)

        self.threads.remove(ct)
Exemple #32
0
    def _worker(self):
        """
        Method used as target of the created threads: retrieve task to run
        from the threadpool, run it, and proceed to the next task until
        threadpool is stopped.
        """
        ct = threading.current_thread()
        ct.reactor = self._reactor
        ct.stats = self._stats
        ct.job_id = None
        if self._stats:
            self._stats.new_thread()

        if callable(self._init_thread):
            self._init_thread()

        self._can_queue_get.wait()
        o = self.q.get()
        while o is not WorkerStop:
            self.busy += 1

            ctx, function, args, kwargs, callback, job_id = o
            del o
            ct.job_id = job_id

            try:
                ct.call_stats('started_item')
                result = context.call(ctx, function, *args, **kwargs)
                success = True
            except:
                success = False
                result = failure.Failure()

            del function, args, kwargs, job_id

            ct.call_stats('finished_item')
            self.busy -= 1
            ct.job_id = None

            callback(success, result)

            del ctx, callback, result

            self._can_queue_get.wait()
            o = self.q.get()

        if self._stats:
            self._stats.exit_thread()
        self._reactor.callFromThread(self._thread_finished, ct)
Exemple #33
0
    def _worker(self):
        """
        Method used as target of the created threads: retrieve task to run
        from the threadpool, run it, and proceed to the next task until
        threadpool is stopped.
        """
        ct = threading.current_thread()
        ct.reactor = self._reactor
        ct.stats = self._stats
        ct.job_id = None
        if self._stats:
            self._stats.new_thread()

        if callable(self._init_thread):
            self._init_thread()

        self._can_queue_get.wait()
        o = self.q.get()
        while o is not WorkerStop:
            self.busy += 1

            ctx, function, args, kwargs, callback, job_id = o
            del o
            ct.job_id = job_id

            try:
                ct.call_stats('started_item')
                result = context.call(ctx, function, *args, **kwargs)
                success = True
            except:
                success = False
                result = failure.Failure()

            del function, args, kwargs, job_id

            ct.call_stats('finished_item')
            self.busy -= 1
            ct.job_id = None

            callback(success, result)

            del ctx, callback, result

            self._can_queue_get.wait()
            o = self.q.get()

        if self._stats:
            self._stats.exit_thread()
        self._reactor.callFromThread(self._thread_finished, ct)
Exemple #34
0
    def _structured(self):
        """
        Retrieve an opaque object which may be usable to construct the
        client-side Widgets which correspond to this fragment and all of its
        children.
        """
        if self._structuredCache is not None:
            return self._structuredCache

        children = []
        requiredModules = []

        # Using the context here is terrible but basically necessary given the
        # /current/ architecture of Athena and flattening.  A better
        # implementation which was not tied to the rendering system could avoid
        # this.
        markup = context.call(
            {
                'children': children,
                'requiredModules': requiredModules
            }, flat.flatten,
            tags.div(
                xmlns="http://www.w3.org/1999/xhtml")[self]).decode('utf-8')

        del children[0]

        self._structuredCache = {
            u'requiredModules': [(name, flat.flatten(url).decode('utf-8'))
                                 for (name, url) in requiredModules],
            u'class':
            self.jsClass,
            u'id':
            self._athenaID,
            u'initArguments':
            tuple(self.getInitialArguments()),
            u'markup':
            markup,
            u'children':
            children
        }
        return self._structuredCache
Exemple #35
0
def callInThread(ctx, func, args, kwargs, d):
    """Call `func` in a newly created thread.

    This function does not actually create the thread; this should be called
    as the target of a newly created thread. Generally you won't call this
    yourself, it will be called by `deferToNewThread`.

    :param ctx: A context as a dict; see :module:`twisted.python.context`.
    :param func: A callable, typically a function.
    :param args: A tuple of positional arguments.
    :param kwargs: A dict of keyword arguments.
    :param d: A :class:`Deferred` that will be called back with the result of
        the function call.

    """
    try:
        result = context.call(ctx, func, *args, **kwargs)
    except:
        # Failure() captures the exception information and trackback.
        reactor.callFromThread(context.call, ctx, d.errback, Failure())
    else:
        reactor.callFromThread(context.call, ctx, d.callback, result)
Exemple #36
0
    def test_get_reader_config_file(self):
        # For paths matching re_config_file, TFTPBackend.get_reader() returns
        # a Deferred that will yield a BytesReader.
        cluster_uuid = factory.getRandomUUID()
        self.patch(tftp_module, 'get_cluster_uuid').return_value = (
            cluster_uuid)
        mac = factory.getRandomMACAddress(b"-")
        config_path = compose_config_path(mac)
        backend = TFTPBackend(self.make_dir(), b"http://example.com/")
        # python-tx-tftp sets up call context so that backends can discover
        # more about the environment in which they're running.
        call_context = {
            "local": (
                factory.getRandomIPAddress(),
                factory.getRandomPort()),
            "remote": (
                factory.getRandomIPAddress(),
                factory.getRandomPort()),
            }

        @partial(self.patch, backend, "get_config_reader")
        def get_config_reader(params):
            params_json = json.dumps(params)
            params_json_reader = BytesReader(params_json)
            return succeed(params_json_reader)

        reader = yield context.call(
            call_context, backend.get_reader, config_path)
        output = reader.read(10000)
        # The addresses provided by python-tx-tftp in the call context are
        # passed over the wire as address:port strings.
        expected_params = {
            "mac": mac,
            "local": call_context["local"][0],  # address only.
            "remote": call_context["remote"][0],  # address only.
            "cluster_uuid": cluster_uuid,
            }
        observed_params = json.loads(output)
        self.assertEqual(expected_params, observed_params)
Exemple #37
0
    def _worker(self):
        """
        Method used as target of the created threads: retrieve a task to run
        from the threadpool, run it, and proceed to the next task until
        threadpool is stopped.
        """
        ct = self.currentThread()
        o = self.q.get()
        while o is not WorkerStop:
            self.working.append(ct)
            ctx, function, args, kwargs, onResult = o
            del o

            try:
                result = context.call(ctx, function, *args, **kwargs)
                success = True
            except:
                success = False
                if onResult is None:
                    context.call(ctx, log.err)
                    result = None
                else:
                    result = failure.Failure()

            del function, args, kwargs

            self.working.remove(ct)

            if onResult is not None:
                try:
                    context.call(ctx, onResult, success, result)
                except:
                    context.call(ctx, log.err)

            del ctx, onResult, result

            self.waiters.append(ct)
            o = self.q.get()
            self.waiters.remove(ct)

        self.threads.remove(ct)
Exemple #38
0
    def _worker(self):
        """
        Method used as target of the created threads: retrieve task to run
        from the threadpool, run it, and proceed to the next task until
        threadpool is stopped.
        """
        ct = self.currentThread()
        o = self.q.get()
        while o is not WorkerStop:
            self.working.append(ct)
            ctx, function, args, kwargs, onResult = o
            del o

            try:
                result = context.call(ctx, function, *args, **kwargs)
                success = True
            except:
                success = False
                if onResult is None:
                    context.call(ctx, log.err)
                    result = None
                else:
                    result = failure.Failure()

            del function, args, kwargs

            self.working.remove(ct)

            if onResult is not None:
                try:
                    context.call(ctx, onResult, success, result)
                except:
                    context.call(ctx, log.err)

            del ctx, onResult, result

            self.waiters.append(ct)
            o = self.q.get()
            self.waiters.remove(ct)

        self.threads.remove(ct)
Exemple #39
0
 def test_setByCall(self):
     """
     Values may be associated with keys by passing them in a dictionary as
     the first argument to L{twisted.python.context.call}.
     """
     self.assertEqual(context.call({"x": "y"}, context.get, "x"), "y")
Exemple #40
0
 def wrapper(request, *args, **kwargs):
     locale = select_locale_by_request(request)
     translations = support.Translations.load(
         'translations', locales=locale, domain='messages')
     ctx = {'locale': locale, 'translations': translations}
     return context.call(ctx, fn, request, *args, **kwargs)
Exemple #41
0
def callWithContext(ctx, func, *args, **kw):
    newCtx = context.get(ILogContext).copy()
    newCtx.update(ctx)
    return context.call({ILogContext: newCtx}, func, *args, **kw)
Exemple #42
0
    def render_GET(self, request):
        # Be sure that the TFTP endpoint is running.
        try:
            tftp = services.getServiceNamed("tftp")
        except KeyError:
            # TFTP service is not installed cannot handle a boot request.
            request.setResponseCode(503)
            return b"HTTP boot service not ready."

        # Extract the local servers IP/port of the request.
        localHost = request.getHeader("X-Server-Addr")
        try:
            localPort = int(request.getHeader("X-Server-Port"))
        except (TypeError, ValueError):
            localPort = 0

        # Extract the original clients IP/port of the request.
        remoteHost = request.getHeader("X-Forwarded-For")
        try:
            remotePort = int(request.getHeader("X-Forwarded-Port"))
        except (TypeError, ValueError):
            remotePort = 0

        # localHost and remoteHost are required headers.
        if not localHost or not remoteHost:
            request.setResponseCode(400)
            return b"Missing X-Server-Addr and X-Forwarded-For HTTP headers."

        def handleFailure(failure):
            if failure.check(AccessViolation):
                request.setResponseCode(403)
                request.write(b"")
            elif failure.check(FileNotFound):
                request.setResponseCode(404)
                request.write(b"")
            else:
                log.err(failure, "Failed to handle boot HTTP request.")
                request.setResponseCode(500)
                request.write(str(failure.value).encode("utf-8"))
            request.finish()

        def writeResponse(reader):
            # Some readers from `tftp` do not provide a way to get the size
            # of the generated content. Only set `Content-Length` when size
            # can be determined for the response.
            if hasattr(reader, "size"):
                request.setHeader(b"Content-Length", reader.size)

            # The readers from `tftp` use `finish` instead of `close`, but
            # `NoRangeStaticProducer` expects `close` instead of `finish`. Map
            # `finish` to `close` so the file handlers are cleaned up.
            reader.close = reader.finish

            # Produce the result without allowing range. This producer will
            # call `close` on the reader and `finish` on the request when done.
            producer = NoRangeStaticProducer(request, reader)
            producer.start()

        path = b"/".join(request.postpath)
        d = context.call(
            {
                "local": (localHost, localPort),
                "remote": (remoteHost, remotePort),
            },
            tftp.backend.get_reader,
            path,
            skip_logging=True,
        )
        d.addCallback(writeResponse)
        d.addErrback(handleFailure)
        d.addErrback(log.err, "Failed to handle boot HTTP request.")

        # Log the HTTP request to rackd.log and push that event to the
        # region controller.
        log_path = path.decode("utf-8")
        log.info(
            "{path} requested by {remoteHost}",
            path=log_path,
            remoteHost=remoteHost,
        )
        d = deferLater(
            reactor,
            0,
            send_node_event_ip_address,
            event_type=EVENT_TYPES.NODE_HTTP_REQUEST,
            ip_address=remoteHost,
            description=log_path,
        )
        d.addErrback(log.err, "Logging HTTP request failed.")

        # Response is handled in the defer.
        return NOT_DONE_YET
 def testBasicContext(self):
     self.assertEquals(context.get("x"), None)
     self.assertEquals(context.call({"x": "y"}, context.get, "x"), "y")
     self.assertEquals(context.get("x"), None)
 def test_setByCall(self):
     """
     Values may be associated with keys by passing them in a dictionary as
     the first argument to L{twisted.python.context.call}.
     """
     self.assertEqual(context.call({"x": "y"}, context.get, "x"), "y")
 def testBasicContext(self):
     self.assertEquals(context.get("x"), None)
     self.assertEquals(context.call({"x": "y"}, context.get, "x"), "y")
     self.assertEquals(context.get("x"), None)
Exemple #46
0
 def getChild(self, path, request):
     # This is a bit of a hack, but it allows the `cssRewriter`
     # processor to grab the request (which static.File.getChild sadly
     # does not pass into it).
     return context.call({'_BetterFile_last_request': request},
                         static.File.getChild, self, path, request)
Exemple #47
0
 def test__returns_host_port_tuple_when_set(self):
     host, port = factory.make_hostname(), factory.pick_port()
     context = {self.context_key: (host, port)}
     self.assertEqual((host, port), call(context, self.get_address))
Exemple #48
0
    def test_lwc(self):
        ## Basic implements testing
        class IFoo(Interface):
            pass

        class Foo(object):
            implements(IFoo)

        assert IFoo in providedBy(Foo)
        f = Foo()
        assert IFoo(f) is f

        ## Adaption testing
        class IBar(Interface):
            pass

        class Bar(object):
            implements(IBar)
            def __init__(self, original=None):
                pass

        registerAdapter(Bar, Foo, IBar)

        b = Bar()
        assert IBar in implementedBy(b)

        fooBar = IBar(b)
        self.assert_(type(fooBar) is Bar)
        self.assert_(IBar in implementedBy(fooBar))

        # "Bar" was a factory because it took a single argument and
        # returned an implementor
        # The adapter factory doesn't have to be a class
        b2 = Bar()
        def fooToBar(obj):
            return b2

        # Only for testing purposes do we overwrite an old registration
        registerAdapter(fooToBar, Foo, IBar, overwrite=True)
        self.assert_(IBar(f) is b2)

        # "Faceted" lets us install components on an instance-by-instance
        # basis
        class Pluggable(Faceted):
            pass

        p = Pluggable()
        p[IFoo] = f
        p[IBar] = b

        self.assert_(IFoo(p) is f)
        self.assert_(IBar(p) is b)

        # Faceted can have adapter factories registered against them,
        # and "remembers" the output
        class IBaz(Interface):
            pass

        def pluggableToBaz(obj):
            return 1

        registerAdapter(pluggableToBaz, Pluggable, IBaz)

        self.assert_(IBaz not in p)
        self.assert_(IBaz(p) == 1)
        self.assert_(IBaz in p)

        ## Interface-to-interface adaption, a rarer use case
        class IString(Interface):
            pass

        class IFile(Interface):
            pass

        class Stringlike(object):
            implements(IString)

        def stringToFile(s):
            import StringIO
            return StringIO.StringIO(s)

        registerAdapter(stringToFile, IString, IFile)

        self.assert_(hasattr(IFile(Stringlike()), 'read'))

        registerAdapter(stringToFile, str, IFile)

        self.assert_(hasattr(IFile("Hello"), 'read'))

        ## Declare that str implements IFoo
        declareImplements(str, IFoo)

        foo = "Foo"

        self.assert_(IFoo in implementedBy(foo))

        ## Register an interface-to-interface adapter for IFoo to IBar
        registerAdapter(fooToBar, IFoo, IBar)

        ## Now we can adapt strings to IBar
        self.assert_(IBar("Foo") is b2)

        if sys.version_info >= (2,4):
            from shtoom.test.py24tests import decoratedLWCTest
            takesABar = decoratedLWCTest(IBar, b2)
            self.assert_(takesABar("This is not a bar, but can be adapted."))

        ## We have context-sensitive adapter registries
        newRegistry = AdapterRegistry()
        newRegistry.registerAdapter(lambda x: 1, str, IBar)

        def tryToAdaptToIBar(something):
            return IBar(something)

        result = context.call({IAdapterRegistry: newRegistry},
                                        tryToAdaptToIBar, "A string")
        self.assert_(result == 1)
Exemple #49
0
 def test__returns_host_port_tuple_even_when_set_longer(self):
     # Only the first two elements from the context's value are used.
     host, port = factory.make_hostname(), factory.pick_port()
     context = {self.context_key: (host, port, factory.make_name("thing"))}
     self.assertEqual((host, port), call(context, self.get_address))
Exemple #50
0
def callWithContext(ctx, func, *args, **kw):
    newCtx = context.get(ILogContext).copy()
    newCtx.update(ctx)
    return context.call({ILogContext: newCtx}, func, *args, **kw)
def process_message(message_sender, client_application_message, server_application_message, async_client_factory,
                    encryption_context, server_response_id, system_auth_header, shard_id):
    device_id = py23.b64encode_to_str(message_sender)

    encryption_context = encryption_context

    if client_application_message.HasField(CLIENT_SINGLE_REQUEST):
        request_object = client_application_message.clientSingleRequest
        response_object = server_application_message.serverSingleResponse
        response_object.replyToMessageId = request_object.requestId
        response_object.serverVersion = str(app_version())

        processor = process_request
        request_type = CLIENT_SINGLE_REQUEST

    elif client_application_message.HasField(CLIENT_SUBSCRIPTION_MESSAGE):
        request_object = client_application_message.clientSubscriptionMessage

        # pings have a special case response, really they could be in their own modular input
        if client_application_message.clientSubscriptionMessage.HasField(RequestType.CLIENT_SUBSCRIPTION_PING.value):
            response_object = server_application_message.serverSubscriptionPing
        else:
            response_object = server_application_message.serverSubscriptionResponse
            response_object.replyToMessageId = request_object.requestId
            response_object.serverVersion = str(app_version())

        processor = process_subscription
        request_type = CLIENT_SUBSCRIPTION_MESSAGE

    else:
        LOGGER.warn("No suitable message type found client application request")
        defer.returnValue("device_id={}".format(device_id))

    response_object.requestId = server_response_id

    if request_object.HasField("runAs"):
        LOGGER.debug("run as credentials is present")
        auth_header = yield parse_run_as_credentials(encryption_context, system_auth_header,
                                                     async_client_factory, request_object.runAs)
    else:
        auth_header = parse_session_token(encryption_context, request_object.sessionToken)

    request_context = RequestContext(auth_header, device_id=device_id,
                                     raw_device_id=message_sender, request_id=request_object.requestId,
                                     current_user=auth_header.username, system_auth_header=system_auth_header,
                                     client_version=request_object.clientVersion, user_agent=request_object.userAgent,
                                     shard_id=shard_id)

    try:
        validate_client_version(request_object, response_object)
        yield auth_header.validate(async_client_factory.splunk_client(), LOGGER, request_context)
        should_send_response = yield context.call({'request_context': request_context}, processor, request_context,
                                                  encryption_context, request_object, response_object,
                                                  async_client_factory)

        # If we aren't sending a response in this request clear the server_application_message
        if not should_send_response:
            server_application_message.Clear()

    except OperationHaltedError:
        server_application_message.ClearField('app_message')
    except SpacebridgeError as e:
        LOGGER.exception("SpacebridgeError during process_message")
        e.set_proto(response_object)

    except Exception as e:
        LOGGER.exception("Unhandled exception during process_message")
        response_object.error.code = common_pb2.Error.ERROR_UNKNOWN
        response_object.error.message = str(e)

    LOGGER.info('Finished processing message. {}'.format(request_context))
    defer.returnValue(request_context)