Example #1
0
 def test_product(self):
     self.product = Product(store=self.siteStore)
     self.product.types = [
         n.decode('ascii') for n in [qual(Foo), qual(Baz)]]
     self.product.installProductOn(self.userStore)
     i = self.userStore.findUnique(Installation)
     self.assertEqual(i.types, self.product.types)
Example #2
0
 def startFactory(self):
     # disable coredumps
     if resource:
         resource.setrlimit(resource.RLIMIT_CORE, (0,0))
     else:
         log.msg('INSECURE: unable to disable core dumps.')
     if not hasattr(self,'publicKeys'):
         self.publicKeys = self.getPublicKeys()
     for keyType, value in self.publicKeys.items():
         if isinstance(value, str):
             warnings.warn("Returning a mapping from strings to "
                     "strings from getPublicKeys()/publicKeys (in %s) "
                     "is deprecated.  Return a mapping from "
                     "strings to Key objects instead." %
                     (qual(self.__class__)),
                     DeprecationWarning, stacklevel=1)
             self.publicKeys[keyType] = keys.Key.fromString(value)
     if not hasattr(self,'privateKeys'):
         self.privateKeys = self.getPrivateKeys()
     for keyType, value in self.privateKeys.items():
         if not isinstance(value, keys.Key):
             warnings.warn("Returning a mapping from strings to "
                     "PyCrypto key objects from "
                     "getPrivateKeys()/privateKeys (in %s) "
                     "is deprecated.  Return a mapping from "
                     "strings to Key objects instead." %
                     (qual(self.__class__),),
                     DeprecationWarning, stacklevel=1)
             self.privateKeys[keyType] = keys.Key(value)
     if not self.publicKeys or not self.privateKeys:
         raise error.ConchError('no host keys, failing')
     if not hasattr(self,'primes'):
         self.primes = self.getPrimes()
Example #3
0
 def startFactory(self):
     """
     Check for public and private keys.
     """
     if not hasattr(self, "publicKeys"):
         self.publicKeys = self.getPublicKeys()
     for keyType, value in self.publicKeys.items():
         if isinstance(value, str):
             warnings.warn(
                 "Returning a mapping from strings to "
                 "strings from getPublicKeys()/publicKeys (in %s) "
                 "is deprecated.  Return a mapping from "
                 "strings to Key objects instead." % (qual(self.__class__)),
                 DeprecationWarning,
                 stacklevel=1,
             )
             self.publicKeys[keyType] = keys.Key.fromString(value)
     if not hasattr(self, "privateKeys"):
         self.privateKeys = self.getPrivateKeys()
     for keyType, value in self.privateKeys.items():
         if not isinstance(value, keys.Key):
             warnings.warn(
                 "Returning a mapping from strings to "
                 "PyCrypto key objects from "
                 "getPrivateKeys()/privateKeys (in %s) "
                 "is deprecated.  Return a mapping from "
                 "strings to Key objects instead." % (qual(self.__class__),),
                 DeprecationWarning,
                 stacklevel=1,
             )
             self.privateKeys[keyType] = keys.Key(value)
     if not self.publicKeys or not self.privateKeys:
         raise error.ConchError("no host keys, failing")
     if not hasattr(self, "primes"):
         self.primes = self.getPrimes()
 def test_deprecatedPreservesName(self):
     """
     The decorated function has the same name as the original.
     """
     version = Version('Twisted', 8, 0, 0)
     dummy = deprecated(version)(dummyCallable)
     self.assertEqual(dummyCallable.__name__, dummy.__name__)
     self.assertEqual(qual(dummyCallable), qual(dummy))
Example #5
0
 def test_createProduct(self):
     """
     Verify that L{ProductConfiguration.createProduct} creates a
     correctly configured L{Product} and returns it.
     """
     conf = ProductConfiguration(store=self.userStore)
     product = conf.createProduct([Foo, Baz])
     self.assertEqual(product.types, [qual(Foo), qual(Baz)])
Example #6
0
    def reconfigServiceSlaves(self, new_config):

        timer = metrics.Timer("BotMaster.reconfigServiceSlaves")
        timer.start()

        # arrange slaves by name
        old_by_name = dict([ (s.slavename, s)
                            for s in list(self)
                            if interfaces.IBuildSlave.providedBy(s) ])
        old_set = set(old_by_name.iterkeys())
        new_by_name = dict([ (s.slavename, s)
                            for s in new_config.slaves ])
        new_set = set(new_by_name.iterkeys())

        # calculate new slaves, by name, and removed slaves
        removed_names, added_names = util.diffSets(old_set, new_set)

        # find any slaves for which the fully qualified class name has
        # changed, and treat those as an add and remove
        for n in old_set & new_set:
            old = old_by_name[n]
            new = new_by_name[n]
            # detect changed class name
            if reflect.qual(old.__class__) != reflect.qual(new.__class__):
                removed_names.add(n)
                added_names.add(n)

        if removed_names or added_names:
            log.msg("adding %d new slaves, removing %d" %
                    (len(added_names), len(removed_names)))

            for n in removed_names:
                slave = old_by_name[n]

                del self.slaves[n]
                slave.master = None
                slave.botmaster = None

                wfd = defer.waitForDeferred(
                    defer.maybeDeferred(lambda :
                        slave.disownServiceParent()))
                yield wfd
                wfd.getResult()

            for n in added_names:
                slave = new_by_name[n]
                slave.setServiceParent(self)

                slave.botmaster = self
                slave.master = self.master
                self.slaves[n] = slave

        metrics.MetricCountEvent.log("num_slaves",
                len(self.slaves), absolute=True)

        timer.stop()
Example #7
0
    def _checkRoundTrip(self, obj):
        """
        Make sure that an object will properly round-trip through 'qual' and
        'namedAny'.

        Raise a L{RuntimeError} if they aren't.
        """
        tripped = reflect.namedAny(reflect.qual(obj))
        if tripped is not obj:
            raise RuntimeError("importing %r is not the same as %r" %
                               (reflect.qual(obj), obj))
Example #8
0
File: web.py Project: fxia22/ASM_xf
 def makeConfigMenu(self, interface):
     """Make a menu for adding a new object to a collection."""
     l = []
     if 1:
         for realClass in coil.getImplementors(interface):
             cfgClass = coil.getConfiguratorClass(realClass)
             nm = getattr(cfgClass, 'configName', None) or reflect.qual(realClass)
             l.append(['new '+reflect.qual(realClass), 'new '+nm])
     for t in self.dispensers.getDispensers(interface):
         obj, methodName, desc = t
         l.append(['dis %d' % hash(t), desc])
     return l
Example #9
0
def spewer(frame, s, ignored):
    """A trace function for sys.settrace that prints every method call."""
    from twisted.python import reflect
    if frame.f_locals.has_key('self'):
        se = frame.f_locals['self']
        if hasattr(se, '__class__'):
            k = reflect.qual(se.__class__)
        else:
            k = reflect.qual(type(se))
        print 'method %s of %s at %s' % (
            frame.f_code.co_name, k, id(se)
        )
Example #10
0
    def versionUpgrade(self):
        """(internal) Do a version upgrade.
        """
        bases = _aybabtu(self.__class__)
        # put the bases in order so superclasses' persistenceVersion methods
        # will be called first.
        bases.reverse()
        bases.append(self.__class__)  # don't forget me!!
        # first let's look for old-skool versioned's
        if self.__dict__.has_key("persistenceVersion"):

            # Hacky heuristic: if more than one class subclasses Versioned,
            # we'll assume that the higher version number wins for the older
            # class, so we'll consider the attribute the version of the older
            # class.  There are obviously possibly times when this will
            # eventually be an incorrect assumption, but hopefully old-school
            # persistenceVersion stuff won't make it that far into multiple
            # classes inheriting from Versioned.

            pver = self.__dict__["persistenceVersion"]
            del self.__dict__["persistenceVersion"]
            highestVersion = 0
            highestBase = None
            for base in bases:
                if not base.__dict__.has_key("persistenceVersion"):
                    continue
                if base.persistenceVersion > highestVersion:
                    highestBase = base
                    highestVersion = base.persistenceVersion
            if highestBase:
                self.__dict__["%s.persistenceVersion" % reflect.qual(highestBase)] = pver
        for base in bases:
            # ugly hack, but it's what the user expects, really
            if Versioned not in base.__bases__ and not base.__dict__.has_key("persistenceVersion"):
                continue
            currentVers = base.persistenceVersion
            pverName = "%s.persistenceVersion" % reflect.qual(base)
            persistVers = self.__dict__.get(pverName) or 0
            if persistVers:
                del self.__dict__[pverName]
            assert persistVers <= currentVers, "Sorry, can't go backwards in time."
            while persistVers < currentVers:
                persistVers = persistVers + 1
                method = base.__dict__.get("upgradeToVersion%s" % persistVers, None)
                if method:
                    log.msg(
                        "Upgrading %s (of %s @ %s) to version %s"
                        % (reflect.qual(base), reflect.qual(self.__class__), id(self), persistVers)
                    )
                    method(self)
                else:
                    log.msg("Warning: cannot upgrade %s to version %s" % (base, persistVers))
Example #11
0
    def reconfigServiceWithBuildbotConfig(self, new_config):

        # arrange childs by name
        old_by_name = self.namedServices
        old_set = set(old_by_name.iterkeys())
        new_config_attr = getattr(new_config, self.config_attr)
        if isinstance(new_config_attr, list):
            new_by_name = dict([(s.name, s)
                                for s in new_config_attr])
        elif isinstance(new_config_attr, dict):
            new_by_name = new_config_attr
        else:
            raise TypeError("config.%s should be a list or dictionary" % (self.config_attr))
        new_set = set(new_by_name.iterkeys())

        # calculate new childs, by name, and removed childs
        removed_names, added_names = util.diffSets(old_set, new_set)

        # find any childs for which the fully qualified class name has
        # changed, and treat those as an add and remove
        for n in old_set & new_set:
            old = old_by_name[n]
            new = new_by_name[n]
            # detect changed class name
            if reflect.qual(old.__class__) != reflect.qual(new.__class__):
                removed_names.add(n)
                added_names.add(n)

        if removed_names or added_names:
            log.msg("adding %d new %s, removing %d" %
                    (len(added_names), self.config_attr, len(removed_names)))

            for n in removed_names:
                child = old_by_name[n]

                yield child.disownServiceParent()

            for n in added_names:
                child = new_by_name[n]
                yield child.setServiceParent(self)

        # get a list of child services to reconfigure
        reconfigurable_services = [svc for svc in self]

        # sort by priority
        reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)

        for svc in reconfigurable_services:
            if not svc.name:
                raise ValueError("%r: child %r should have a defined name attribute", self, svc)
            config_sibling = new_by_name.get(svc.name)
            yield svc.reconfigServiceWithSibling(config_sibling)
Example #12
0
 def jellyToAO(self, obj):
     """I turn an object into an AOT and return it."""
     objType = type(obj)
     self.stack.append(repr(obj))
     if objType in _SIMPLE_BUILTINS:
         retval = obj
     elif objType is types.MethodType:
         retval = InstanceMethod(obj.im_func.__name__, reflect.qual(obj.im_class),
                                 self.jellyToAO(obj.im_self))
     elif objType is types.ModuleType:
         retval = Module(obj.__name__)
     elif objType is types.ClassType:
         retval = Class(reflect.qual(obj))
     elif issubclass(objType, type):
         retval = Class(reflect.qual(obj))
     elif objType is types.FunctionType:
         retval = Function(reflect.fullFuncName(obj))
     else: #mutable! gotta watch for refs.
         if self.prepared.has_key(id(obj)):
             oldRef = self.prepared[id(obj)]
             if oldRef.refnum:
                 key = oldRef.refnum
             else:
                 self._ref_id = self._ref_id + 1
                 key = self._ref_id
                 oldRef.setRef(key)
             return Deref(key)
         retval = Ref()
         self.prepareForRef(retval, obj)
         if objType is types.ListType:
             retval.setObj(map(self.jellyToAO, obj)) #hah!
         elif objType is types.TupleType:
             retval.setObj(tuple(map(self.jellyToAO, obj)))
         elif objType is types.DictionaryType:
             d = {}
             for k,v in obj.items():
                 d[self.jellyToAO(k)] = self.jellyToAO(v)
             retval.setObj(d)
         elif objType is types.InstanceType:
             if hasattr(obj, "__getstate__"):
                 state = self.jellyToAO(obj.__getstate__())
             else:
                 state = self.jellyToAO(obj.__dict__)
             retval.setObj(Instance(reflect.qual(obj.__class__), state))
         elif copy_reg.dispatch_table.has_key(objType):
             unpickleFunc, state = copy_reg.dispatch_table[objType](obj)
             retval.setObj(Copyreg( reflect.fullFuncName(unpickleFunc),
                                    self.jellyToAO(state)))
         else:
             raise "Unsupported type: %s" % objType.__name__
     del self.stack[-1]
     return retval
Example #13
0
File: util.py Project: RichDijk/eXe
def spewer(frame, s, ignored):
    """A trace function for sys.settrace that prints every function or method call."""
    from twisted.python import reflect

    if frame.f_locals.has_key("self"):
        se = frame.f_locals["self"]
        if hasattr(se, "__class__"):
            k = reflect.qual(se.__class__)
        else:
            k = reflect.qual(type(se))
        print "method %s of %s at %s" % (frame.f_code.co_name, k, id(se))
    else:
        print "function %s in %s, line %s" % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)
Example #14
0
    def reconfigServiceSlaves(self, new_config):

        timer = metrics.Timer("BuildSlaveManager.reconfigServiceSlaves")
        timer.start()

        # first we deconfigure everything to let the slaves register again
        yield self.master.data.updates.deconfigureAllBuidslavesForMaster(self.master.masterid)

        # arrange slaves by name
        old_by_name = dict([(s.slavename, s)
                            for s in list(self)
                            if IBuildSlave.providedBy(s)])
        old_set = set(old_by_name.iterkeys())
        new_by_name = dict([(s.slavename, s)
                            for s in new_config.slaves])
        new_set = set(new_by_name.iterkeys())

        # calculate new slaves, by name, and removed slaves
        removed_names, added_names = util.diffSets(old_set, new_set)

        # find any slaves for which the fully qualified class name has
        # changed, and treat those as an add and remove
        for n in old_set & new_set:
            old = old_by_name[n]
            new = new_by_name[n]
            # detect changed class name
            if reflect.qual(old.__class__) != reflect.qual(new.__class__):
                removed_names.add(n)
                added_names.add(n)

        if removed_names or added_names:
            log.msg("adding %d new slaves, removing %d" %
                    (len(added_names), len(removed_names)))

            for n in removed_names:
                slave = old_by_name[n]

                del self.slaves[n]
                slave.master = None

                yield slave.disownServiceParent()

            for n in added_names:
                slave = new_by_name[n]
                yield slave.setServiceParent(self)
                self.slaves[n] = slave

        metrics.MetricCountEvent.log("num_slaves",
                                     len(self.slaves), absolute=True)

        timer.stop()
Example #15
0
    def display(self, request):
        """Implement me to represent your widget.

        I must return a list of strings and twisted.internet.defer.Deferred
        instances.
        """
        raise NotImplementedError("%s.display" % reflect.qual(self.__class__))
Example #16
0
 def __repr__(self):
     factoryName = reflect.qual(self.factory.__class__)
     if hasattr(self, 'socket'):
         return '<%s on %r>' % (
             factoryName, _coerceToFilesystemEncoding('', self.port))
     else:
         return '<%s (not listening)>' % (factoryName,)
Example #17
0
def proxyForInterface(iface, originalAttribute='original'):
    """
    Create a class which proxies all method calls which adhere to an interface
    to another provider of that interface.

    This function is intended for creating specialized proxies. The typical way
    to use it is by subclassing the result::

      class MySpecializedProxy(proxyForInterface(IFoo)):
          def someInterfaceMethod(self, arg):
              if arg == 3:
                  return 3
              return self.original.someInterfaceMethod(arg)

    @param iface: The Interface to which the resulting object will conform, and
        which the wrapped object must provide.

    @param originalAttribute: name of the attribute used to save the original
        object in the resulting class. Default to C{original}.
    @type originalAttribute: C{str}

    @return: A class whose constructor takes the original object as its only
        argument. Constructing the class creates the proxy.
    """
    def __init__(self, original):
        setattr(self, originalAttribute, original)
    contents = {"__init__": __init__}
    for name in iface:
        contents[name] = _ProxyDescriptor(name, originalAttribute)
    proxy = type("(Proxy for %s)"
                 % (reflect.qual(iface),), (object,), contents)
    directlyProvides(proxy, iface)
    return proxy
Example #18
0
    def _setUpStore(self):
        """
        Set up a store, install a L{HyperbolaPublicPresence} and its
        dependencies, and create a role
        """
        self.siteStore = Store(filesdir=self.mktemp())
        Mantissa().installSite(
            self.siteStore, u"localhost", u"", generateCert=False)

        # Make it standard so there's no port number in the generated URL.
        # This kind of sucks.  I don't want people assuming SSLPorts are
        # created by Mantissa().installSite().  Oh right, I should add a better
        # API for initializing a Mantissa server. -exarkun
        site = self.siteStore.findUnique(SiteConfiguration)
        ssls = list(site.store.query(SSLPort))
        ssls[0].portNumber = 443

        self.loginSystem = self.siteStore.findUnique(LoginSystem)

        product = Product(store=self.siteStore,
                          types=[qual(HyperbolaPublicPresence)])

        acct = self.loginSystem.addAccount(
            u'user', u'localhost', u'asdf', internal=True)
        self.userStore = acct.avatars.open()

        product.installProductOn(self.userStore)

        self.publicPresence = self.userStore.findUnique(
            HyperbolaPublicPresence)

        self.role = sharing.Role(
            store=self.userStore,
            externalID=u'foo@host', description=u'foo')
Example #19
0
    def _makeContext(self):
        ctx = SSL.Context(self.method)
        ctx.set_app_data(_SSLApplicationData())

        if self.certificate is not None and self.privateKey is not None:
            ctx.use_certificate(self.certificate)
            ctx.use_privatekey(self.privateKey)
            # Sanity check
            ctx.check_privatekey()

        verifyFlags = SSL.VERIFY_NONE
        if self.verify:
            verifyFlags = SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT
            if self.caCerts:
                store = ctx.get_cert_store()
                for cert in self.caCerts:
                    store.add_cert(cert)

        def _trackVerificationProblems(conn,cert,errno,depth,preverify_ok):
            return True
        
        ctx.set_verify(verifyFlags, _trackVerificationProblems)
        
        if self.enableSessions:
            sessionName = md5.md5("%s-%d" % (reflect.qual(self.__class__), _sessionCounter())).hexdigest()
            ctx.set_session_id(sessionName)
        
        return ctx
Example #20
0
    def spawn(self, hereProto, thereProto, childFDs=None):
        """
        Spawn a subprocess with a connected pair of protocol objects, one in
        the current process, one in the subprocess.

        @param hereProto: a L{Protocol} instance to listen in this process.

        @param thereProto: a top-level class or function that will be imported
            and called in the spawned subprocess.

        @param childFDs: File descriptors to share with the subprocess; same
            format as L{IReactorProcess.spawnProcess}.

        @return: a L{Deferred} that fires when C{hereProto} is ready.
        """
        if not self.running:
            self.pendingSpawns.append((hereProto, thereProto))
            return
        name = qual(thereProto)
        argv = [sys.executable, '-u', '-m', __name__, name]
        self.reactor.spawnProcess(
            BridgeProtocol(self, hereProto), sys.executable,
            argv, os.environ, childFDs=childFDs
        )
        return succeed(hereProto)
Example #21
0
 def rendered(req):
     self.assertIn(
         qual(ThemedSubclass),
         req.v)
     self.assertIn(
         'specified no <code>fragmentName</code> attribute.',
         req.v)
 def jellyFor(self, jellier):
     """
     @see: L{twisted.spread.interfaces.IJellyable.jellyFor}
     """
     sxp = jellier.prepare(self)
     sxp.extend([qual(self.__class__), jellier.jelly(self.getStateFor(jellier))])
     return jellier.preserve(self, sxp)
Example #23
0
 def fullyQualifiedName(self):
     # XXX: this is an example of silly redundancy, this really ought to be
     # refactored to work like any other attribute (including being
     # explicitly covered in the schema, which has other good qualities like
     # allowing tables to be VACUUM'd without destroying oid stability and
     # every storeID reference ever. --glyph
     return qual(self.type) + ".storeID"
Example #24
0
    def powerupsFor(self, interface):
        """
        Returns powerups installed using C{powerUp}, in order of descending
        priority.

        Powerups found to have been deleted, either during the course of this
        powerupsFor iteration, during an upgrader, or previously, will not be
        returned.
        """
        name = unicode(qual(interface), "ascii")
        for cable in self.store.query(
            _PowerupConnector,
            AND(_PowerupConnector.interface == name, _PowerupConnector.item == self),
            sort=_PowerupConnector.priority.descending,
        ):
            pup = cable.powerup
            if pup is None:
                # this powerup was probably deleted during an upgrader.
                cable.deleteFromStore()
            else:
                indirector = IPowerupIndirector(pup, None)
                if indirector is not None:
                    yield indirector.indirect(interface)
                else:
                    yield pup
Example #25
0
    def setUp(self):
        """
        Create an account and log in using it.
        """
        IntegrationTestsMixin.setUp(self)

        # Make an account to be already logged in.
        self.userAccount = self.login.addAccount(self.username, self.domain, u"password", internal=True)
        self.userStore = self.userAccount.avatars.open()

        # Make a product that includes PrivateApplication.  This is probably
        # the minimum requirement for web access.
        web = Product(store=self.store, types=[qual(PrivateApplication)])
        # Give it to Alice.
        web.installProductOn(self.userStore)

        # Log in to the web as Alice.
        login = getWithSession(
            self.factory,
            3,
            "/__login__?username=%s@%s&password=%s"
            % (self.username.encode("ascii"), self.domain.encode("ascii"), "password"),
            {"host": self.domain.encode("ascii")},
        )

        def loggedIn(request):
            self.cookies = request.cookies

        login.addCallback(loggedIn)
        return login
Example #26
0
    def doRead(self):
        """Called when data is avaliable for reading.

        Subclasses must override this method. The result will be interpreted
        in the same way as a result of doWrite().
        """
        raise NotImplementedError("%s does not implement doRead" % reflect.qual(self.__class__))
Example #27
0
    def operationError(self, error):
        """Example callback for database operation failure.

        Override this, and/or define your own callbacks.
        """
        log.msg("%s Operation Failed: %s" % (reflect.qual(self.__class__), error))
        log.err(error)
Example #28
0
    def format(self, form, write, request):
        """I display an HTML FORM according to the result of self.getFormFields.
        """
        write('<form ENCTYPE="multipart/form-data" METHOD="post" ACTION="%s">\n'
              '<table BORDER="0">\n' % (self.actionURI or request.uri))

        for field in form:
            if len(field) == 5:
                inputType, displayName, inputName, inputValue, description = field
            else:
                inputType, displayName, inputName, inputValue = field
                description = ""
            write('<tr>\n<td ALIGN="right" VALIGN="top"><B>%s</B></td>\n'
                  '<td VALIGN="%s">\n' %
                  (displayName, ((inputType == 'text') and 'top') or 'middle'))
            self.formGen[inputType](write, inputName, inputValue)
            write('\n<br />\n<font size="-1">%s</font></td>\n</tr>\n' % description)


        write('<tr><td></td><td ALIGN="left"><hr />\n')
        for submitName in self.submitNames:
            write('<INPUT TYPE="submit" NAME="submit" VALUE="%s" />\n' % submitName)
        write('</td></tr>\n</table>\n'
              '<INPUT TYPE="hidden" NAME="__formtype__" VALUE="%s" />\n'
              % (reflect.qual(self.__class__)))
        fid = self.getFormID()
        if fid:
            write('<INPUT TYPE="hidden" NAME="__formid__" VALUE="%s" />\n' % fid)
        write("</form>\n")
    def setUp(self):
        self.runners = []
        self.config = trial.Options()
        # whitebox hack a reporter in, because plugins are CACHED and will
        # only reload if the FILE gets changed.

        parts = reflect.qual(CapturingReporter).split('.')
        package = '.'.join(parts[:-1])
        klass = parts[-1]
        plugins = [twisted_trial._Reporter(
            "Test Helper Reporter",
            package,
            description="Utility for unit testing.",
            longOpt="capturing",
            shortOpt=None,
            klass=klass)]


        # XXX There should really be a general way to hook the plugin system
        # for tests.
        def getPlugins(iface, *a, **kw):
            self.assertEqual(iface, IReporter)
            return plugins + list(self.original(iface, *a, **kw))

        self.original = plugin.getPlugins
        plugin.getPlugins = getPlugins

        self.standardReport = ['startTest', 'addSuccess', 'stopTest',
                               'startTest', 'addSuccess', 'stopTest',
                               'startTest', 'addSuccess', 'stopTest',
                               'startTest', 'addSuccess', 'stopTest',
                               'startTest', 'addSuccess', 'stopTest',
                               'startTest', 'addSuccess', 'stopTest',
                               'startTest', 'addSuccess', 'stopTest']
Example #30
0
    def getComponent(self, interface, default=None):
        """Create or retrieve an adapter for the given interface.

        If such an adapter has already been created, retrieve it from the cache
        that this instance keeps of all its adapters.  Adapters created through
        this mechanism may safely store system-specific state.

        If you want to register an adapter that will be created through
        getComponent, but you don't require (or don't want) your adapter to be
        cached and kept alive for the lifetime of this Componentized object,
        set the attribute 'temporaryAdapter' to True on your adapter class.

        If you want to automatically register an adapter for all appropriate
        interfaces (with addComponent), set the attribute 'multiComponent' to
        True on your adapter class.
        """
        k = reflect.qual(interface)
        if self._adapterCache.has_key(k):
            return self._adapterCache[k]
        else:
            adapter = interface.__adapt__(self)
            if adapter is not None and not (
                hasattr(adapter, "temporaryAdapter") and
                adapter.temporaryAdapter):
                self._adapterCache[k] = adapter
                if (hasattr(adapter, "multiComponent") and
                    adapter.multiComponent):
                    self.addComponent(adapter)
            if adapter is None:
                return default
            return adapter
Example #31
0
def run_command_version(options, reactor=None, **kwargs):
    """
    Subcommand "crossbar version".
    """
    log = make_logger()

    # Python
    py_ver = '.'.join([str(x) for x in list(sys.version_info[:3])])
    py_ver_string = "[%s]" % sys.version.replace('\n', ' ')

    if 'pypy_version_info' in sys.__dict__:
        py_ver_detail = "{}-{}".format(
            platform.python_implementation(),
            '.'.join(str(x) for x in sys.pypy_version_info[:3]))
    else:
        py_ver_detail = platform.python_implementation()

    # Twisted / Reactor
    tx_ver = "%s-%s" % (pkg_resources.require("Twisted")[0].version,
                        reactor.__class__.__name__)
    tx_loc = "[%s]" % qual(reactor.__class__)

    # txaio
    txaio_ver = '%s' % pkg_resources.require("txaio")[0].version

    # Autobahn
    ab_ver = pkg_resources.require("autobahn")[0].version
    ab_loc = "[%s]" % qual(WebSocketProtocol)

    # UTF8 Validator
    s = qual(Utf8Validator)
    if 'wsaccel' in s:
        utf8_ver = 'wsaccel-%s' % pkg_resources.require('wsaccel')[0].version
    elif s.startswith('autobahn'):
        utf8_ver = 'autobahn'
    else:
        # could not detect UTF8 validator type/version
        utf8_ver = '?'
    utf8_loc = "[%s]" % qual(Utf8Validator)

    # XOR Masker
    s = qual(XorMaskerNull)
    if 'wsaccel' in s:
        xor_ver = 'wsaccel-%s' % pkg_resources.require('wsaccel')[0].version
    elif s.startswith('autobahn'):
        xor_ver = 'autobahn'
    else:
        # could not detect XOR masker type/version
        xor_ver = '?'
    xor_loc = "[%s]" % qual(XorMaskerNull)

    # JSON Serializer
    supported_serializers = ['JSON']
    from autobahn.wamp.serializer import JsonObjectSerializer
    json_ver = JsonObjectSerializer.JSON_MODULE.__name__

    # If it's just 'json' then it's the stdlib one...
    if json_ver == 'json':
        json_ver = 'stdlib'
    else:
        json_ver = (json_ver +
                    "-%s") % pkg_resources.require(json_ver)[0].version

    # MsgPack Serializer
    try:
        import umsgpack  # noqa
        msgpack_ver = 'u-msgpack-python-%s' % pkg_resources.require(
            'u-msgpack-python')[0].version
        supported_serializers.append('MessagePack')
    except ImportError:
        msgpack_ver = '-'

    # CBOR Serializer
    try:
        import cbor  # noqa
        cbor_ver = 'cbor-%s' % pkg_resources.require('cbor')[0].version
        supported_serializers.append('CBOR')
    except ImportError:
        cbor_ver = '-'

    # UBJSON Serializer
    try:
        import ubjson  # noqa
        ubjson_ver = 'ubjson-%s' % pkg_resources.require(
            'py-ubjson')[0].version
        supported_serializers.append('UBJSON')
    except ImportError:
        ubjson_ver = '-'

    # LMDB
    try:
        import lmdb  # noqa
        lmdb_lib_ver = '.'.join([str(x) for x in lmdb.version()])
        lmdb_ver = '{}/lmdb-{}'.format(
            pkg_resources.require('lmdb')[0].version, lmdb_lib_ver)
    except ImportError:
        lmdb_ver = '-'

    # Release Public Key
    release_pubkey = _read_release_pubkey()

    def decorate(text):
        return click.style(text, fg='yellow', bold=True)

    Node = node_classes[options.personality][u'class']

    for line in Node.BANNER.splitlines():
        log.info(decorate("{:>40}".format(line)))

    pad = " " * 22

    log.info(" Crossbar.io        : {ver} ({personality})",
             ver=decorate(crossbar.__version__),
             personality=Node.PERSONALITY)
    log.info("   Autobahn         : {ver} (with {serializers})",
             ver=decorate(ab_ver),
             serializers=', '.join(supported_serializers))
    log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(ab_loc))
    log.debug("     txaio             : {ver}", ver=decorate(txaio_ver))
    log.debug("     UTF8 Validator    : {ver}", ver=decorate(utf8_ver))
    log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(utf8_loc))
    log.debug("     XOR Masker        : {ver}", ver=decorate(xor_ver))
    log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(xor_loc))
    log.debug("     JSON Codec        : {ver}", ver=decorate(json_ver))
    log.debug("     MessagePack Codec : {ver}", ver=decorate(msgpack_ver))
    log.debug("     CBOR Codec        : {ver}", ver=decorate(cbor_ver))
    log.debug("     UBJSON Codec      : {ver}", ver=decorate(ubjson_ver))
    log.info("   Twisted          : {ver}", ver=decorate(tx_ver))
    log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(tx_loc))
    log.info("   LMDB             : {ver}", ver=decorate(lmdb_ver))
    log.info("   Python           : {ver}/{impl}",
             ver=decorate(py_ver),
             impl=decorate(py_ver_detail))
    log.trace("{pad}{debuginfo}", pad=pad, debuginfo=decorate(py_ver_string))
    log.info(" OS                 : {ver}", ver=decorate(platform.platform()))
    log.info(" Machine            : {ver}", ver=decorate(platform.machine()))
    log.info(" Release key        : {release_pubkey}",
             release_pubkey=decorate(release_pubkey[u'base64']))
    log.info("")
Example #32
0
 def __repr__(self):
     protocolName = reflect.qual(self.protocol.__class__, )
     if hasattr(self, 'socket'):
         return '<%s on %r>' % (protocolName, self.port)
     else:
         return '<%s (not listening)>' % (protocolName, )
Example #33
0
 def __repr__(self):
     factoryName = reflect.qual(self.factory.__class__)
     if hasattr(self, 'socket'):
         return '<%s on %r>' % (factoryName, self.port)
     else:
         return '<%s (not listening)>' % (factoryName, )
Example #34
0
 def jellyToDOM_1(self, jellier, element):
     from twisted.python.reflect import qual
     element.setAttribute("integer", str(self.integer))
     element.setAttribute("instance", qual(self.instance.__class__)) # not l33t enough
     element.setAttribute("name", str(self.name))
Example #35
0
 def addReader(self, reader):
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement addReader")
Example #36
0
 def jelly(self, obj):
     if isinstance(obj, Jellyable):
         preRef = self._checkMutable(obj)
         if preRef:
             return preRef
         return obj.jellyFor(self)
     objType = type(obj)
     if self.taster.isTypeAllowed(qual(objType).encode("utf-8")):
         # "Immutable" Types
         if objType in (bytes, int, float):
             return obj
         elif isinstance(obj, types.MethodType):
             aSelf = obj.__self__
             aFunc = obj.__func__
             aClass = aSelf.__class__
             return [
                 b"method",
                 aFunc.__name__,
                 self.jelly(aSelf),
                 self.jelly(aClass),
             ]
         elif objType is str:
             return [b"unicode", obj.encode("UTF-8")]
         elif isinstance(obj, type(None)):
             return [b"None"]
         elif isinstance(obj, types.FunctionType):
             return [b"function", obj.__module__ + "." + obj.__qualname__]
         elif isinstance(obj, types.ModuleType):
             return [b"module", obj.__name__]
         elif objType is bool:
             return [b"boolean", obj and b"true" or b"false"]
         elif objType is datetime.datetime:
             if obj.tzinfo:
                 raise NotImplementedError(
                     "Currently can't jelly datetime objects with tzinfo"
                 )
             return [
                 b"datetime",
                 " ".join(
                     [
                         str(x)
                         for x in (
                             obj.year,
                             obj.month,
                             obj.day,
                             obj.hour,
                             obj.minute,
                             obj.second,
                             obj.microsecond,
                         )
                     ]
                 ).encode("utf-8"),
             ]
         elif objType is datetime.time:
             if obj.tzinfo:
                 raise NotImplementedError(
                     "Currently can't jelly datetime objects with tzinfo"
                 )
             return [
                 b"time",
                 " ".join(
                     [
                         str(x)
                         for x in (obj.hour, obj.minute, obj.second, obj.microsecond)
                     ]
                 ).encode("utf-8"),
             ]
         elif objType is datetime.date:
             return [
                 b"date",
                 " ".join([str(x) for x in (obj.year, obj.month, obj.day)]).encode(
                     "utf-8"
                 ),
             ]
         elif objType is datetime.timedelta:
             return [
                 b"timedelta",
                 " ".join(
                     [str(x) for x in (obj.days, obj.seconds, obj.microseconds)]
                 ).encode("utf-8"),
             ]
         elif issubclass(objType, type):
             return [b"class", qual(obj).encode("utf-8")]
         elif objType is decimal.Decimal:
             return self.jelly_decimal(obj)
         else:
             preRef = self._checkMutable(obj)
             if preRef:
                 return preRef
             # "Mutable" Types
             sxp = self.prepare(obj)
             if objType is list:
                 sxp.extend(self._jellyIterable(list_atom, obj))
             elif objType is tuple:
                 sxp.extend(self._jellyIterable(tuple_atom, obj))
             elif objType in DictTypes:
                 sxp.append(dictionary_atom)
                 for key, val in obj.items():
                     sxp.append([self.jelly(key), self.jelly(val)])
             elif objType in _SetTypes:
                 sxp.extend(self._jellyIterable(set_atom, obj))
             elif objType in _ImmutableSetTypes:
                 sxp.extend(self._jellyIterable(frozenset_atom, obj))
             else:
                 className = qual(obj.__class__).encode("utf-8")
                 persistent = None
                 if self.persistentStore:
                     persistent = self.persistentStore(obj, self)
                 if persistent is not None:
                     sxp.append(persistent_atom)
                     sxp.append(persistent)
                 elif self.taster.isClassAllowed(obj.__class__):
                     sxp.append(className)
                     if hasattr(obj, "__getstate__"):
                         state = obj.__getstate__()
                     else:
                         state = obj.__dict__
                     sxp.append(self.jelly(state))
                 else:
                     self.unpersistable(
                         "instance of class %s deemed insecure"
                         % qual(obj.__class__),
                         sxp,
                     )
             return self.preserve(obj, sxp)
     else:
         raise InsecureJelly("Type not allowed for object: %s %s" % (objType, obj))
Example #37
0
 def doWrite(self):
     """Raises a RuntimeError"""
     raise RuntimeError("doWrite called on a %s" %
                        reflect.qual(self.__class__))
Example #38
0
 def installWaker(self):
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement installWaker")
Example #39
0
    def assertDetailedTraceback(self, captureVars=False, cleanFailure=False):
        """
        Assert that L{printDetailedTraceback} produces and prints a detailed
        traceback.

        The detailed traceback consists of a header::

          *--- Failure #20 ---

        The body contains the stacktrace::

          /twisted/trial/_synctest.py:1180: _run(...)
          /twisted/python/util.py:1076: runWithWarningsSuppressed(...)
          --- <exception caught here> ---
          /twisted/test/test_failure.py:39: getDivisionFailure(...)

        If C{captureVars} is enabled the body also includes a list of
        globals and locals::

           [ Locals ]
             exampleLocalVar : 'xyz'
             ...
           ( Globals )
             ...

        Or when C{captureVars} is disabled::

           [Capture of Locals and Globals disabled (use captureVars=True)]

        When C{cleanFailure} is enabled references to other objects are removed
        and replaced with strings.

        And finally the footer with the L{Failure}'s value::

          exceptions.ZeroDivisionError: float division
          *--- End of Failure #20 ---

        @param captureVars: Enables L{Failure.captureVars}.
        @type captureVars: C{bool}
        @param cleanFailure: Enables L{Failure.cleanFailure}.
        @type cleanFailure: C{bool}
        """
        if captureVars:
            exampleLocalVar = 'xyz'
            # Silence the linter as this variable is checked via
            # the traceback.
            exampleLocalVar

        f = getDivisionFailure(captureVars=captureVars)
        out = NativeStringIO()
        if cleanFailure:
            f.cleanFailure()
        f.printDetailedTraceback(out)

        tb = out.getvalue()
        start = "*--- Failure #%d%s---\n" % (f.count,
                                             (f.pickled and ' (pickled) ')
                                             or ' ')
        end = "%s: %s\n*--- End of Failure #%s ---\n" % (reflect.qual(
            f.type), reflect.safe_str(f.value), f.count)
        self.assertTracebackFormat(tb, start, end)

        # Variables are printed on lines with 2 leading spaces.
        linesWithVars = [
            line for line in tb.splitlines() if line.startswith('  ')
        ]

        if captureVars:
            self.assertNotEqual([], linesWithVars)
            if cleanFailure:
                line = '  exampleLocalVar : "\'xyz\'"'
            else:
                line = "  exampleLocalVar : 'xyz'"
            self.assertIn(line, linesWithVars)
        else:
            self.assertEqual([], linesWithVars)
            self.assertIn(
                ' [Capture of Locals and Globals disabled (use '
                'captureVars=True)]\n', tb)
Example #40
0
    def jellyToAO(self, obj):
        """I turn an object into an AOT and return it."""
        objType = type(obj)
        self.stack.append(repr(obj))

        # immutable: We don't care if these have multiple refs!
        if objType in _SIMPLE_BUILTINS:
            retval = obj

        elif issubclass(objType, types.MethodType):
            # TODO: make methods 'prefer' not to jelly the object internally,
            # so that the object will show up where it's referenced first NOT
            # by a method.
            retval = InstanceMethod(
                _funcOfMethod(obj).__name__,
                reflect.qual(_classOfMethod(obj)),
                self.jellyToAO(_selfOfMethod(obj)),
            )

        elif issubclass(objType, types.ModuleType):
            retval = Module(obj.__name__)

        elif issubclass(objType, type):
            retval = Class(reflect.qual(obj))

        elif objType is types.FunctionType:
            retval = Function(reflect.fullFuncName(obj))

        else:  # mutable! gotta watch for refs.

            # Marmalade had the nicety of being able to just stick a 'reference' attribute
            # on any Node object that was referenced, but in AOT, the referenced object
            # is *inside* of a Ref call (Ref(num, obj) instead of
            # <objtype ... reference="1">). The problem is, especially for built-in types,
            # I can't just assign some attribute to them to give them a refnum. So, I have
            # to "wrap" a Ref(..) around them later -- that's why I put *everything* that's
            # mutable inside one. The Ref() class will only print the "Ref(..)" around an
            # object if it has a Reference explicitly attached.

            if id(obj) in self.prepared:
                oldRef = self.prepared[id(obj)]
                if oldRef.refnum:
                    # it's been referenced already
                    key = oldRef.refnum
                else:
                    # it hasn't been referenced yet
                    self._ref_id = self._ref_id + 1
                    key = self._ref_id
                    oldRef.setRef(key)
                return Deref(key)

            retval = Ref()

            def _stateFrom(state):
                retval.setObj(
                    Instance(reflect.qual(obj.__class__),
                             self.jellyToAO(state)))

            self.prepareForRef(retval, obj)

            if objType is list:
                retval.setObj([self.jellyToAO(o) for o in obj])  # hah!

            elif objType is tuple:
                retval.setObj(tuple(map(self.jellyToAO, obj)))

            elif objType is dict:
                d = {}
                for k, v in obj.items():
                    d[self.jellyToAO(k)] = self.jellyToAO(v)
                retval.setObj(d)

            elif objType in copy_reg.dispatch_table:
                unpickleFunc, state = copy_reg.dispatch_table[objType](obj)

                retval.setObj(
                    Copyreg(reflect.fullFuncName(unpickleFunc),
                            self.jellyToAO(state)))

            elif hasattr(obj, "__getstate__"):
                _stateFrom(obj.__getstate__())
            elif hasattr(obj, "__dict__"):
                _stateFrom(obj.__dict__)
            else:
                raise TypeError("Unsupported type: %s" % objType.__name__)

        del self.stack[-1]
        return retval
Example #41
0
 def __init__(self, original):
     self.original = original
     key = reflect.qual(original.__class__)
     count = _counters.get(key, 0)
     _counters[key] = count + 1
     self._logFile = open(key + '-' + str(count), 'w')
Example #42
0
def run():
    """
    Entry point into (native) worker processes. This wires up stuff such that
    a worker instance is talking WAMP-over-stdio to the node controller.
    """
    import os
    import sys
    import platform
    import signal

    # make sure logging to something else than stdio is setup _first_
    #
    from crossbar._logging import make_JSON_observer, cb_logging_aware
    from txaio import make_logger, start_logging
    from twisted.logger import globalLogPublisher
    from twisted.python.reflect import qual

    log = make_logger()

    # Print a magic phrase that tells the capturing logger that it supports
    # Crossbar's rich logging
    print(cb_logging_aware, file=sys.__stderr__)
    sys.__stderr__.flush()

    flo = make_JSON_observer(sys.__stderr__)
    globalLogPublisher.addObserver(flo)

    # Ignore SIGINT so we get consistent behavior on control-C versus
    # sending SIGINT to the controller process. When the controller is
    # shutting down, it sends TERM to all its children but ctrl-C
    # handling will send a SIGINT to all the processes in the group
    # (so then the controller sends a TERM but the child already or
    # will very shortly get a SIGINT as well). Twisted installs signal
    # handlers, but not for SIGINT if there's already a custom one
    # present.

    def ignore(sig, frame):
        log.debug("Ignoring SIGINT in worker.")
    signal.signal(signal.SIGINT, ignore)

    # create the top-level parser
    #
    import argparse
    parser = argparse.ArgumentParser()

    parser.add_argument('--reactor',
                        default=None,
                        choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'],
                        help='Explicit Twisted reactor selection (optional).')

    parser.add_argument('--loglevel',
                        default="info",
                        choices=['none', 'error', 'warn', 'info', 'debug', 'trace'],
                        help='Initial log level.')

    parser.add_argument('-c',
                        '--cbdir',
                        type=six.text_type,
                        help="Crossbar.io node directory (required).")

    parser.add_argument('-r',
                        '--realm',
                        type=six.text_type,
                        help='Crossbar.io node (management) realm (required).')

    parser.add_argument('-k',
                        '--klass',
                        type=six.text_type,
                        help='Crossbar.io worker class (required).')

    parser.add_argument('-n',
                        '--node',
                        type=six.text_type,
                        help='Crossbar.io node ID (required).')

    parser.add_argument('-w',
                        '--worker',
                        type=six.text_type,
                        help='Crossbar.io worker ID (required).')

    parser.add_argument('--title',
                        type=six.text_type,
                        default=None,
                        help='Worker process title to set (optional).')

    parser.add_argument('--expose_controller',
                        type=bool,
                        default=False,
                        help='Expose node controller session to all components (this feature requires Crossbar.io Fabric extension).')

    parser.add_argument('--expose_shared',
                        type=bool,
                        default=False,
                        help='Expose a shared object to all components (this feature requires Crossbar.io Fabric extension).')

    parser.add_argument('--shutdown',
                        type=six.text_type,
                        default=None,
                        help='Shutdown method')

    options = parser.parse_args()

    # actually begin logging
    start_logging(None, options.loglevel)

    # we use an Autobahn utility to import the "best" available Twisted reactor
    #
    from autobahn.twisted.choosereactor import install_reactor
    reactor = install_reactor(options.reactor)

    # eg: crossbar.worker.container.ContainerWorkerSession
    l = options.klass.split('.')
    worker_module, worker_klass = '.'.join(l[:-1]), l[-1]

    # now load the worker module and class
    _mod = importlib.import_module(worker_module)
    klass = getattr(_mod, worker_klass)

    log.info(
        'Started {worker_title} worker "{worker_id}" on node "{node_id}" [{klass} / {python}-{reactor}]',
        worker_title=klass.WORKER_TITLE,
        klass=options.klass,
        node_id=options.node,
        worker_id=options.worker,
        pid=os.getpid(),
        python=platform.python_implementation(),
        reactor=qual(reactor.__class__).split('.')[-1],
    )

    # set process title if requested to
    #
    try:
        import setproctitle
    except ImportError:
        log.debug("Could not set worker process title (setproctitle not installed)")
    else:
        if options.title:
            setproctitle.setproctitle(options.title)
        else:
            setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass))

    # node directory
    #
    options.cbdir = os.path.abspath(options.cbdir)
    os.chdir(options.cbdir)
    # log.msg("Starting from node directory {}".format(options.cbdir))

    # set process title if requested to
    #
    try:
        import setproctitle
    except ImportError:
        log.debug("Could not set worker process title (setproctitle not installed)")
    else:
        if options.title:
            setproctitle.setproctitle(options.title)
        else:
            setproctitle.setproctitle(
                'crossbar-worker [{}]'.format(options.klass)
            )

    from twisted.internet.error import ConnectionDone
    from autobahn.twisted.websocket import WampWebSocketServerProtocol

    class WorkerServerProtocol(WampWebSocketServerProtocol):

        def connectionLost(self, reason):
            # the behavior here differs slightly whether we're shutting down orderly
            # or shutting down because of "issues"
            if isinstance(reason.value, ConnectionDone):
                was_clean = True
            else:
                was_clean = False

            try:
                # this log message is unlikely to reach the controller (unless
                # only stdin/stdout pipes were lost, but not stderr)
                if was_clean:
                    log.info("Connection to node controller closed cleanly")
                else:
                    log.warn("Connection to node controller lost: {reason}", reason=reason)

                # give the WAMP transport a change to do it's thing
                WampWebSocketServerProtocol.connectionLost(self, reason)
            except:
                # we're in the process of shutting down .. so ignore ..
                pass
            finally:
                # after the connection to the node controller is gone,
                # the worker is "orphane", and should exit

                # determine process exit code
                if was_clean:
                    exit_code = 0
                else:
                    exit_code = 1

                # exit the whole worker process when the reactor has stopped
                reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code)

                # stop the reactor
                try:
                    reactor.stop()
                except ReactorNotRunning:
                    pass

    try:
        # create a WAMP application session factory
        #
        from autobahn.twisted.wamp import ApplicationSessionFactory
        from autobahn.wamp.types import ComponentConfig

        session_config = ComponentConfig(realm=options.realm, extra=options)
        session_factory = ApplicationSessionFactory(session_config)
        session_factory.session = klass

        # create a WAMP-over-WebSocket transport server factory
        #
        from autobahn.twisted.websocket import WampWebSocketServerFactory
        transport_factory = WampWebSocketServerFactory(session_factory, u'ws://localhost')
        transport_factory.protocol = WorkerServerProtocol
        transport_factory.setProtocolOptions(failByDrop=False)

        # create a protocol instance and wire up to stdio
        #
        from twisted.python.runtime import platform as _platform
        from twisted.internet import stdio
        proto = transport_factory.buildProtocol(None)
        if _platform.isWindows():
            stdio.StandardIO(proto)
        else:
            stdio.StandardIO(proto, stdout=3)

        # now start reactor loop
        #
        if False:
            log.info("vmprof enabled.")

            import os
            import vmprof

            PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid())

            outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC)
            vmprof.enable(outfd, period=0.01)

            log.info("Entering event loop...")
            reactor.run()

            vmprof.disable()
        else:
            log.debug("Entering event loop...")
            reactor.run()

    except Exception as e:
        log.info("Unhandled exception: {e}", e=e)
        if reactor.running:
            reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1)
            reactor.stop()
        else:
            sys.exit(1)
Example #43
0
 def jellyFor(self, jellier):
     qual = reflect.qual(PBMind)
     if isinstance(qual, unicode):
         qual = qual.encode("utf-8")
     return qual, jellier.invoker.registerReference(self)
Example #44
0
def run_command_start(options, reactor=None):
    """
    Subcommand "crossbar start".
    """
    assert reactor

    # do not allow to run more than one Crossbar.io instance
    # from the same Crossbar.io node directory
    #
    pid_data = check_is_running(options.cbdir)
    if pid_data:
        print(
            "Crossbar.io is already running from node directory {} (PID {}).".
            format(options.cbdir, pid_data['pid']))
        sys.exit(1)
    else:
        fp = os.path.join(options.cbdir, _PID_FILENAME)
        with open(fp, 'wb') as fd:
            argv = options.argv
            options_dump = vars(options)
            pid_data = {
                'pid': os.getpid(),
                'argv': argv,
                'options': {
                    x: y
                    for x, y in options_dump.items()
                    if x not in ["func", "argv"]
                }
            }
            fd.write("{}\n".format(
                json.dumps(pid_data,
                           sort_keys=False,
                           indent=4,
                           separators=(', ', ': '),
                           ensure_ascii=False)).encode('utf8'))

    # remove node PID file when reactor exits
    #
    def remove_pid_file():
        fp = os.path.join(options.cbdir, _PID_FILENAME)
        if os.path.isfile(fp):
            os.remove(fp)

    reactor.addSystemEventTrigger('after', 'shutdown', remove_pid_file)

    log = make_logger()

    # represents the running Crossbar.io node
    #
    Node = node_classes[options.personality][u'class']
    node = Node(options.cbdir, reactor=reactor)

    # possibly generate new node key
    #
    pubkey = node.maybe_generate_key(options.cbdir)

    # Print the banner.
    #
    for line in Node.BANNER.splitlines():
        log.info(click.style(("{:>40}").format(line), fg='yellow', bold=True))

    bannerFormat = "{:<12} {:<24}"
    log.info(
        bannerFormat.format(
            "Version:",
            click.style('{} {}'.format(node.PERSONALITY, crossbar.__version__),
                        fg='yellow',
                        bold=True)))
    if pubkey:
        log.info(
            bannerFormat.format("Public Key:",
                                click.style(pubkey, fg='yellow', bold=True)))
    log.info()

    log.info(
        'Node created with personality "{node_personality}" [{node_class}]',
        node_personality=options.personality,
        node_class='{}.{}'.format(Node.__module__, Node.__name__))

    log.info('Running from node directory "{cbdir}"', cbdir=options.cbdir)

    # check and load the node configuration
    #
    try:
        node.load(options.config)
    except InvalidConfigException as e:
        log.error("Invalid node configuration")
        log.error("{e!s}", e=e)
        sys.exit(1)
    except:
        raise

    log.info("Controller process starting ({python}-{reactor}) ..",
             python=platform.python_implementation(),
             reactor=qual(reactor.__class__).split('.')[-1])

    # now actually start the node ..
    #
    def start_crossbar():
        d = node.start()

        def on_error(err):
            log.error("{e!s}", e=err.value)
            log.error("Could not start node: {err}", err=err)
            if reactor.running:
                reactor.stop()

        d.addErrback(on_error)

    reactor.callWhenRunning(start_crossbar)

    # enter event loop
    #
    try:
        reactor.run()
    except Exception:
        log.failure("Could not start reactor - {log_failure.value}")
Example #45
0
    def reconfigServiceWithBuildbotConfig(self, new_config):

        # arrange childs by name
        old_by_name = self.namedServices
        old_set = set(old_by_name)
        new_config_attr = getattr(new_config, self.config_attr)
        if isinstance(new_config_attr, list):
            new_by_name = {s.name: s for s in new_config_attr}
        elif isinstance(new_config_attr, dict):
            new_by_name = new_config_attr
        else:
            raise TypeError("config.{} should be a list or dictionary".format(
                self.config_attr))
        new_set = set(new_by_name)

        # calculate new childs, by name, and removed childs
        removed_names, added_names = util.diffSets(old_set, new_set)

        # find any childs for which the fully qualified class name has
        # changed, and treat those as an add and remove
        # While we're at it find any service that don't know how to reconfig,
        # and, if they have changed, add them to both removed and added, so that we
        # run the new version
        for n in old_set & new_set:
            old = old_by_name[n]
            new = new_by_name[n]
            # detect changed class name
            if reflect.qual(old.__class__) != reflect.qual(new.__class__):
                removed_names.add(n)
                added_names.add(n)
            # compare using ComparableMixin if they don't support reconfig
            elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
                if old != new:
                    removed_names.add(n)
                    added_names.add(n)

        if removed_names or added_names:
            log.msg("adding {} new {}, removing {}".format(
                len(added_names), self.config_attr, len(removed_names)))

            for n in removed_names:
                child = old_by_name[n]
                # disownServiceParent calls stopService after removing the relationship
                # as child might use self.master.data to stop itself, its better to stop it first
                # (this is related to the fact that self.master is found by recursively looking at
                # self.parent for a master)
                yield child.stopService()
                # it has already called, so do not call it again
                child.stopService = lambda: None
                yield child.disownServiceParent()

            for n in added_names:
                child = new_by_name[n]
                # setup service's objectid
                if hasattr(child, 'objectid'):
                    class_name = '{}.{}'.format(child.__class__.__module__,
                                                child.__class__.__name__)
                    objectid = yield self.master.db.state.getObjectId(
                        child.name, class_name)
                    child.objectid = objectid
                yield defer.maybeDeferred(child.setServiceParent, self)

        # As the services that were just added got
        # reconfigServiceWithSibling called by
        # setServiceParent->startService,
        # we avoid calling it again by selecting
        # in reconfigurable_services, services
        # that were not added just now
        reconfigurable_services = [
            svc for svc in self if svc.name not in added_names
        ]
        # sort by priority
        reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)

        for svc in reconfigurable_services:
            if not svc.name:
                raise ValueError(
                    "{}: child {} should have a defined name attribute".format(
                        self, svc))
            config_sibling = new_by_name.get(svc.name)
            try:
                yield svc.reconfigServiceWithSibling(config_sibling)
            except NotImplementedError:
                # legacy support. Its too painful to transition old code to new Service life cycle
                # so we implement switch of child when the service raises NotImplementedError
                # Note this means that self will stop, and sibling will take ownership
                # means that we have a small time where the service is unavailable.
                yield svc.disownServiceParent()
                config_sibling.objectid = svc.objectid
                yield config_sibling.setServiceParent(self)
Example #46
0
def rebuild(module, doLog=1):
    """
    Reload a module and do as much as possible to replace its references.
    """
    global lastRebuild
    lastRebuild = time.time()
    if hasattr(module, 'ALLOW_TWISTED_REBUILD'):
        # Is this module allowed to be rebuilt?
        if not module.ALLOW_TWISTED_REBUILD:
            raise RuntimeError("I am not allowed to be rebuilt.")
    if doLog:
        log.msg('Rebuilding {}...'.format(str(module.__name__)))

    # Safely handle adapter re-registration
    from twisted.python import components
    components.ALLOW_DUPLICATES = True

    d = module.__dict__
    _modDictIDMap[id(d)] = module
    newclasses = {}
    classes = {}
    functions = {}
    values = {}
    if doLog:
        log.msg('  (scanning {}): '.format(str(module.__name__)))
    for k, v in d.items():
        if _isClassType(type(v)):
            # ClassType exists on Python 2.x and earlier.
            # Failure condition -- instances of classes with buggy
            # __hash__/__cmp__ methods referenced at the module level...
            if v.__module__ == module.__name__:
                classes[v] = 1
                if doLog:
                    log.logfile.write("c")
                    log.logfile.flush()
        elif type(v) == types.FunctionType:
            if v.__globals__ is module.__dict__:
                functions[v] = 1
                if doLog:
                    log.logfile.write("f")
                    log.logfile.flush()
        elif isinstance(v, type):
            if v.__module__ == module.__name__:
                newclasses[v] = 1
                if doLog:
                    log.logfile.write("o")
                    log.logfile.flush()

    values.update(classes)
    values.update(functions)
    fromOldModule = values.__contains__
    newclasses = newclasses.keys()
    classes = classes.keys()
    functions = functions.keys()

    if doLog:
        log.msg('')
        log.msg('  (reload   {})'.format(str(module.__name__)))

    # Boom.
    reload(module)
    # Make sure that my traceback printing will at least be recent...
    linecache.clearcache()

    if doLog:
        log.msg('  (cleaning {}): '.format(str(module.__name__)))

    for clazz in classes:
        if getattr(module, clazz.__name__) is clazz:
            log.msg("WARNING: class {} not replaced by reload!".format(
                reflect.qual(clazz)))
        else:
            if doLog:
                log.logfile.write("x")
                log.logfile.flush()
            clazz.__bases__ = ()
            clazz.__dict__.clear()
            clazz.__getattr__ = __injectedgetattr__
            clazz.__module__ = module.__name__
    if newclasses:
        import gc
    for nclass in newclasses:
        ga = getattr(module, nclass.__name__)
        if ga is nclass:
            log.msg("WARNING: new-class {} not replaced by reload!".format(
                reflect.qual(nclass)))
        else:
            for r in gc.get_referrers(nclass):
                if getattr(r, '__class__', None) is nclass:
                    r.__class__ = ga
    if doLog:
        log.msg('')
        log.msg('  (fixing   {}): '.format(str(module.__name__)))
    modcount = 0
    for mk, mod in sys.modules.items():
        modcount = modcount + 1
        if mod == module or mod is None:
            continue

        if not hasattr(mod, '__file__'):
            # It's a builtin module; nothing to replace here.
            continue

        if hasattr(mod, '__bundle__'):
            # PyObjC has a few buggy objects which segfault if you hash() them.
            # It doesn't make sense to try rebuilding extension modules like
            # this anyway, so don't try.
            continue

        changed = 0

        for k, v in mod.__dict__.items():
            try:
                hash(v)
            except Exception:
                continue
            if fromOldModule(v):
                if _isClassType(type(v)):
                    if doLog:
                        log.logfile.write("c")
                        log.logfile.flush()
                    nv = latestClass(v)
                else:
                    if doLog:
                        log.logfile.write("f")
                        log.logfile.flush()
                    nv = latestFunction(v)
                changed = 1
                setattr(mod, k, nv)
            else:
                # Replace bases of non-module classes just to be sure.
                if _isClassType(type(v)):
                    for base in v.__bases__:
                        if fromOldModule(base):
                            latestClass(v)
        if doLog and not changed and ((modcount % 10) == 0):
            log.logfile.write(".")
            log.logfile.flush()

    components.ALLOW_DUPLICATES = False
    if doLog:
        log.msg('')
        log.msg('   Rebuilt {}.'.format(str(module.__name__)))
    return module
Example #47
0
 def removeAll(self):
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement removeAll")
Example #48
0
def install_optimal_reactor(verbose=False):
    """
    Try to install the optimal Twisted reactor for platform.

    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    import sys
    from twisted.python import reflect

    ## determine currently installed reactor, if any
    ##
    if 'twisted.internet.reactor' in sys.modules:
        current_reactor = reflect.qual(
            sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
    else:
        current_reactor = None

    ## depending on platform, install optimal reactor
    ##
    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):

        ## *BSD and MacOSX
        ##
        if current_reactor != 'KQueueReactor':
            try:
                v = sys.version_info
                if v[0] == 1 or (v[0] == 2
                                 and v[1] < 6) or (v[0] == 2 and v[1] == 6
                                                   and v[2] < 5):
                    raise Exception(
                        "Python version too old ({0}) to use kqueue reactor".
                        format(sys.version))
                from twisted.internet import kqreactor
                kqreactor.install()
            except Exception as e:
                print(
                    "WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0})."
                    .format(e))
            else:
                if verbose:
                    print(
                        "Running on *BSD or MacOSX and optimal reactor (kqueue) was installed."
                    )
        else:
            if verbose:
                print(
                    "Running on *BSD or MacOSX and optimal reactor (kqueue) already installed."
                )

    elif sys.platform in ['win32']:

        ## Windows
        ##
        if current_reactor != 'IOCPReactor':
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor
                iocpreactor.install()
            except Exception as e:
                print(
                    "WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0})."
                    .format(e))
            else:
                if verbose:
                    print(
                        "Running on Windows and optimal reactor (ICOP) was installed."
                    )
        else:
            if verbose:
                print(
                    "Running on Windows and optimal reactor (ICOP) already installed."
                )

    elif sys.platform.startswith('linux'):

        ## Linux
        ##
        if current_reactor != 'EPollReactor':
            try:
                from twisted.internet import epollreactor
                epollreactor.install()
            except Exception as e:
                print(
                    "WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0})."
                    .format(e))
            else:
                if verbose:
                    print(
                        "Running on Linux and optimal reactor (epoll) was installed."
                    )
        else:
            if verbose:
                print(
                    "Running on Linux and optimal reactor (epoll) already installed."
                )

    else:
        try:
            from twisted.internet import default as defaultreactor
            defaultreactor.install()
        except Exception as e:
            print(
                "WARNING: Could not install default Twisted reactor for this platform ({0})."
                .format(e))
Example #49
0
    def jelly(self, obj):
        if isinstance(obj, Jellyable):
            preRef = self._checkMutable(obj)
            if preRef:
                return preRef
            return obj.jellyFor(self)
        objType = type(obj)
        if self.taster.isTypeAllowed(qual(objType)):
            # "Immutable" Types
            if ((objType is StringType) or (objType is IntType)
                    or (objType is LongType) or (objType is FloatType)):
                return obj
            elif objType is MethodType:
                return [
                    "method", obj.im_func.__name__,
                    self.jelly(obj.im_self),
                    self.jelly(obj.im_class)
                ]

            elif UnicodeType and objType is UnicodeType:
                return ['unicode', obj.encode('UTF-8')]
            elif objType is NoneType:
                return ['None']
            elif objType is FunctionType:
                name = obj.__name__
                return [
                    'function',
                    str(pickle.whichmodule(obj, obj.__name__)) + '.' + name
                ]
            elif objType is ModuleType:
                return ['module', obj.__name__]
            elif objType is BooleanType:
                return ['boolean', obj and 'true' or 'false']
            elif objType is datetime.datetime:
                if obj.tzinfo:
                    raise NotImplementedError(
                        "Currently can't jelly datetime objects with tzinfo")
                return [
                    'datetime',
                    '%s %s %s %s %s %s %s' %
                    (obj.year, obj.month, obj.day, obj.hour, obj.minute,
                     obj.second, obj.microsecond)
                ]
            elif objType is datetime.time:
                if obj.tzinfo:
                    raise NotImplementedError(
                        "Currently can't jelly datetime objects with tzinfo")
                return [
                    'time',
                    '%s %s %s %s' %
                    (obj.hour, obj.minute, obj.second, obj.microsecond)
                ]
            elif objType is datetime.date:
                return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)]
            elif objType is datetime.timedelta:
                return [
                    'timedelta',
                    '%s %s %s' % (obj.days, obj.seconds, obj.microseconds)
                ]
            elif objType is ClassType or issubclass(objType, type):
                return ['class', qual(obj)]
            elif decimal is not None and objType is decimal.Decimal:
                return self.jelly_decimal(obj)
            else:
                preRef = self._checkMutable(obj)
                if preRef:
                    return preRef
                # "Mutable" Types
                sxp = self.prepare(obj)
                if objType is ListType:
                    sxp.extend(self._jellyIterable(list_atom, obj))
                elif objType is TupleType:
                    sxp.extend(self._jellyIterable(tuple_atom, obj))
                elif objType in DictTypes:
                    sxp.append(dictionary_atom)
                    for key, val in obj.items():
                        sxp.append([self.jelly(key), self.jelly(val)])
                elif (_set is not None and objType is set
                      or objType is _sets.Set):
                    sxp.extend(self._jellyIterable(set_atom, obj))
                elif (_set is not None and objType is frozenset
                      or objType is _sets.ImmutableSet):
                    sxp.extend(self._jellyIterable(frozenset_atom, obj))
                else:
                    className = qual(obj.__class__)
                    persistent = None
                    if self.persistentStore:
                        persistent = self.persistentStore(obj, self)
                    if persistent is not None:
                        sxp.append(persistent_atom)
                        sxp.append(persistent)
                    elif self.taster.isClassAllowed(obj.__class__):
                        sxp.append(className)
                        if hasattr(obj, "__getstate__"):
                            state = obj.__getstate__()
                        else:
                            state = obj.__dict__
                        sxp.append(self.jelly(state))
                    else:
                        self.unpersistable(
                            "instance of class %s deemed insecure" %
                            qual(obj.__class__), sxp)
                return self.preserve(obj, sxp)
        else:
            if objType is InstanceType:
                raise InsecureJelly("Class not allowed for instance: %s %s" %
                                    (obj.__class__, obj))
            raise InsecureJelly("Type not allowed for object: %s %s" %
                                (objType, obj))
Example #50
0
 def getWriters(self):
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement getWriters")
Example #51
0
 def __repr__(self):
     return "<%s instance at 0x%x %s %s>" % (reflect.qual(
         self.__class__), id(self), self.state, self.getDestination())
Example #52
0
 def removeWriter(self, writer):
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement removeWriter")
 def __repr__(self):
     return ('<%s run=%d errors=%d failures=%d todos=%d dones=%d skips=%d>'
             % (reflect.qual(self.__class__), self.testsRun,
                len(self.errors), len(self.failures),
                len(self.expectedFailures), len(self.skips),
                len(self.unexpectedSuccesses)))
Example #54
0
 def getDestination(self):
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement "
         "getDestination")
Example #55
0
 def logPrefix(self):
     """
     Returns the name of my class, to prefix log entries with.
     """
     return reflect.qual(self.factory.__class__)
Example #56
0
    def reconfigService(self, new_config):
        timer = metrics.Timer("SchedulerManager.reconfigService")
        timer.start()

        old_by_name = dict((sch.name, sch) for sch in self)
        old_set = set(old_by_name.iterkeys())
        new_by_name = new_config.schedulers
        new_set = set(new_by_name.iterkeys())

        removed_names, added_names = util.diffSets(old_set, new_set)

        # find any schedulers that don't know how to reconfig, and, if they
        # have changed, add them to both removed and added, so that we
        # run the new version.  While we're at it, find any schedulers whose
        # fully qualified class name has changed, and consider those a removal
        # and re-add as well.
        for n in old_set & new_set:
            old = old_by_name[n]
            new = new_by_name[n]
            # detect changed class name
            if reflect.qual(old.__class__) != reflect.qual(new.__class__):
                removed_names.add(n)
                added_names.add(n)

            # compare using ComparableMixin if they don't support reconfig
            elif not hasattr(old, 'reconfigService'):
                if old != new:
                    removed_names.add(n)
                    added_names.add(n)

        # removals first

        for sch_name in removed_names:
            log.msg("removing scheduler '%s'" % (sch_name,))
            sch = old_by_name[sch_name]
            yield defer.maybeDeferred(lambda :
                        sch.disownServiceParent())
            sch.master = None

        # .. then additions

        for sch_name in added_names:
            log.msg("adding scheduler '%s'" % (sch_name,))
            sch = new_by_name[sch_name]

            # get the scheduler's objectid
            class_name = '%s.%s' % (sch.__class__.__module__,
                                    sch.__class__.__name__)
            objectid = yield self.master.db.state.getObjectId(
                                    sch.name, class_name)

            # set up the scheduler
            sch.objectid = objectid
            sch.master = self.master

            # *then* attacah and start it
            sch.setServiceParent(self)

        metrics.MetricCountEvent.log("num_schedulers", len(list(self)),
                                    absolute=True)

        # reconfig any newly-added schedulers, as well as existing
        yield config.ReconfigurableServiceMixin.reconfigService(self,
                                                        new_config)

        timer.stop()
Example #57
0
    def _start_native_worker(self,
                             worker_type,
                             worker_id,
                             worker_options=None,
                             details=None):

        # prohibit starting a worker twice
        #
        if worker_id in self._workers:
            emsg = "Could not start worker: a worker with ID '{}' is already running (or starting)".format(
                worker_id)
            self.log.error(emsg)
            raise ApplicationError('crossbar.error.worker_already_running',
                                   emsg)

        # check worker options
        #
        options = worker_options or {}
        if 'extra' in options:
            worker_options_extra = binascii.b2a_hex(
                cbor2.dumps(dict(options['extra'])))
        else:
            worker_options_extra = None

        try:
            if worker_type in self._node._native_workers:
                if self._node._native_workers[worker_type][
                        'checkconfig_options']:
                    self._node._native_workers[worker_type][
                        'checkconfig_options'](self.personality, options)
                else:
                    raise Exception(
                        'No checkconfig_options for worker type "{worker_type}" implemented!'
                        .format(worker_type=worker_type))
            else:
                raise Exception('invalid worker type "{}"'.format(worker_type))
        except Exception as e:
            emsg = "Could not start native worker: invalid configuration ({})".format(
                e)
            self.log.error(emsg)
            raise ApplicationError('crossbar.error.invalid_configuration',
                                   emsg)

        # the fully qualified worker class as a string
        worker_class = qual(
            self._node._native_workers[worker_type]['worker_class'])

        # allow override Python executable from options
        #
        if 'python' in options:
            exe = options['python']

            # the executable must be an absolute path, e.g. /home/oberstet/pypy-2.2.1-linux64/bin/pypy
            #
            if not os.path.isabs(exe):
                emsg = "Invalid worker configuration: python executable '{}' must be an absolute path".format(
                    exe)
                self.log.error(emsg)
                raise ApplicationError('crossbar.error.invalid_configuration',
                                       emsg)

            # of course the path must exist and actually be executable
            #
            if not (os.path.isfile(exe) and os.access(exe, os.X_OK)):
                emsg = "Invalid worker configuration: python executable '{}' does not exist or isn't an executable".format(
                    exe)
                self.log.error(emsg)
                raise ApplicationError('crossbar.error.invalid_configuration',
                                       emsg)
        else:
            exe = sys.executable

        # allow override default Python module search paths from options
        #
        if 'pythonpath' in options:
            pythonpaths_to_add = [
                os.path.abspath(os.path.join(self._node._cbdir, p))
                for p in options.get('pythonpath', [])
            ]
        else:
            pythonpaths_to_add = []

        # assemble command line for forking the worker
        #
        # all native workers (routers and containers for now) start
        # from the same script in crossbar/worker/process.py or
        # from the command "crossbar _exec_worker" when crossbar is
        # running from a frozen executable (single-file, pyinstaller, etc)
        #
        if getattr(sys, 'frozen', False):
            # if we are inside a frozen crossbar executable, we need to invoke
            # the crossbar executable with a command ("_exec_worker")
            args = [exe, self._node.personality.NAME, "_exec_worker"]
        else:
            # we are invoking via "-m" so that .pyc files, __pycache__
            # etc work properly. this works everywhere, but frozen executables
            args = [exe, "-u", "-m", "crossbar.worker.main"]
        args.extend(["--cbdir", self._node._cbdir])
        args.extend(["--node", str(self._node._node_id)])
        args.extend(["--worker", str(worker_id)])
        args.extend(["--realm", self._realm])
        args.extend(["--personality", class_name(self._node.personality)])
        args.extend(["--klass", worker_class])
        args.extend(["--loglevel", get_global_log_level()])
        if self._node.options.debug_lifecycle:
            args.append("--debug-lifecycle")
        if self._node.options.debug_programflow:
            args.append("--debug-programflow")
        if self._node.options.enable_vmprof:
            args.append("--vmprof")
        if "shutdown" in options:
            args.extend(["--shutdown", options["shutdown"]])
        if "restart" in options:
            args.extend(["--restart", options["restart"]])
        if worker_options_extra:
            args.extend(["--extra", worker_options_extra])

        # Node-level callback to inject worker arguments
        #
        self._node._extend_worker_args(args, options)

        # allow override worker process title from options
        #
        if options.get('title', None):
            args.extend(['--title', options['title']])

        # forward explicit reactor selection
        #
        if 'reactor' in options and sys.platform in options['reactor']:
            args.extend(['--reactor', options['reactor'][sys.platform]])
        # FIXME
        # elif self._node.options.reactor:
        #    args.extend(['--reactor', self._node.options.reactor])

        # create worker process environment
        #
        worker_env = create_process_env(options)

        # We need to use the same PYTHONPATH we were started with, so we can
        # find the Crossbar we're working with -- it may not be the same as the
        # one on the default path
        worker_env["PYTHONPATH"] = os.pathsep.join(pythonpaths_to_add +
                                                   sys.path)

        # log name of worker
        #
        worker_logname = self._node._native_workers[worker_type]['logname']

        # each worker is run under its own dedicated WAMP auth role
        #
        worker_auth_role = 'crossbar.worker.{}'.format(worker_id)

        # topic URIs used (later)
        #
        starting_topic = self._node._native_workers[worker_type]['topics'][
            'starting']
        started_topic = self._node._native_workers[worker_type]['topics'][
            'started']

        # add worker tracking instance to the worker map ..
        #
        WORKER = self._node._native_workers[worker_type]['class']
        worker = WORKER(self,
                        worker_id,
                        details.caller,
                        keeplog=options.get('traceback', None))
        self._workers[worker_id] = worker

        # create a (custom) process endpoint.
        #
        if platform.isWindows():
            childFDs = None  # Use the default Twisted ones
        else:
            # The communication between controller and container workers is
            # using WAMP running over 2 pipes.
            # For controller->native-worker traffic this runs over FD 0 (`stdin`)
            # and for the native-worker->controller traffic, this runs over FD 3.
            #
            # Note: We use FD 3, not FD 1 (`stdout`) or FD 2 (`stderr`) for
            # container->controller traffic, so that components running in the
            # container which happen to write to `stdout` or `stderr` do not
            # interfere with the container-controller communication.
            childFDs = {0: "w", 1: "r", 2: "r", 3: "r"}

        ep = WorkerProcessEndpoint(self._node._reactor,
                                   exe,
                                   args,
                                   env=worker_env,
                                   worker=worker,
                                   childFDs=childFDs)

        # ready handling
        #
        def on_ready_success(worker_id):
            self.log.debug(
                '{worker_type} worker "{worker_id}" process {pid} started',
                worker_type=worker_logname,
                worker_id=worker.id,
                pid=worker.pid)

            self._node._reactor.addSystemEventTrigger(
                'before',
                'shutdown',
                self._cleanup_worker,
                self._node._reactor,
                worker,
            )

            worker.on_worker_started()

            started_info = {
                'id':
                worker.id,
                'status':
                worker.status,
                'started':
                utcstr(worker.started),
                'who':
                worker.who,
                'pid':
                worker.pid,
                'startup_time':
                (worker.started -
                 worker.created).total_seconds() if worker.started else None
            }

            # FIXME: make start of stats printer dependent on log level ..
            if False:
                worker.log_stats(5.)

            self.publish(started_topic,
                         started_info,
                         options=PublishOptions(exclude=details.caller))

            return started_info

        def on_ready_error(err):
            del self._workers[worker.id]
            emsg = 'Failed to start native worker: {}'.format(err.value)
            self.log.error(emsg)
            raise ApplicationError("crossbar.error.cannot_start", emsg,
                                   worker.getlog())

        worker.ready.addCallbacks(on_ready_success, on_ready_error)

        def on_exit_success(_):
            self.log.info("Node worker {worker.id} ended successfully",
                          worker=worker)

            # clear worker log
            worker.log_stats(0)

            # remove the dedicated node router authrole we dynamically
            # added for the worker
            self._node._drop_worker_role(worker_auth_role)

            # remove our metadata tracking for the worker
            del self._workers[worker.id]

            # indicate that the worker excited successfully
            return True

        def on_exit_error(err):
            self.log.info("Node worker {worker.id} ended with error ({err})",
                          worker=worker,
                          err=err)

            # clear worker log
            worker.log_stats(0)

            # remove the dedicated node router authrole we dynamically
            # added for the worker
            self._node._drop_worker_role(worker_auth_role)

            # remove our metadata tracking for the worker
            del self._workers[worker.id]

            # indicate that the worker excited with error
            return False

        def check_for_shutdown(was_successful):
            self.log.info(
                'Checking for node shutdown: worker_exit_success={worker_exit_success}, shutdown_requested={shutdown_requested}, node_shutdown_triggers={node_shutdown_triggers}',
                worker_exit_success=was_successful,
                shutdown_requested=self._shutdown_requested,
                node_shutdown_triggers=self._node._node_shutdown_triggers)

            shutdown = self._shutdown_requested

            # automatically shutdown node whenever a worker ended (successfully, or with error)
            #
            if NODE_SHUTDOWN_ON_WORKER_EXIT in self._node._node_shutdown_triggers:
                self.log.info(
                    "Node worker ended, and trigger '{trigger}' is active: will shutdown node ..",
                    trigger=NODE_SHUTDOWN_ON_WORKER_EXIT)
                term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT')
                shutdown = True

            # automatically shutdown node when worker ended with error
            #
            elif not was_successful and NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR in self._node._node_shutdown_triggers:
                self.log.info(
                    "Node worker ended with error, and trigger '{trigger}' is active: will shutdown node ..",
                    trigger=NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR)
                term_print('CROSSBAR:NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR')
                shutdown = True

            # automatically shutdown node when no more workers are left
            #
            elif len(
                    self._workers
            ) == 0 and NODE_SHUTDOWN_ON_LAST_WORKER_EXIT in self._node._node_shutdown_triggers:
                self.log.info(
                    "No more node workers running, and trigger '{trigger}' is active: will shutdown node ..",
                    trigger=NODE_SHUTDOWN_ON_LAST_WORKER_EXIT)
                term_print('CROSSBAR:NODE_SHUTDOWN_ON_LAST_WORKER_EXIT')
                shutdown = True

            # initiate shutdown (but only if we are not already shutting down)
            #
            if shutdown:
                self.shutdown()
            else:
                self.log.info('Node will continue to run!')

        d_on_exit = worker.exit.addCallbacks(on_exit_success, on_exit_error)
        d_on_exit.addBoth(check_for_shutdown)

        # create a transport factory for talking WAMP to the native worker
        #
        transport_factory = create_native_worker_client_factory(
            self._node._router_session_factory, worker_auth_role, worker.ready,
            worker.exit)
        transport_factory.noisy = False
        self._workers[worker_id].factory = transport_factory

        # now (immediately before actually forking) signal the starting of the worker
        #
        starting_info = {
            'id': worker_id,
            'status': worker.status,
            'created': utcstr(worker.created),
            'who': worker.who,
        }

        # the caller gets a progressive result ..
        if details.progress:
            details.progress(starting_info)

        # .. while all others get an event
        self.publish(starting_topic,
                     starting_info,
                     options=PublishOptions(exclude=details.caller))

        # only the following line will actually exec a new worker process - everything before is just setup
        # for this moment:
        self.log.debug(
            'Starting new managed worker process for {worker_logname} worker "{worker_id}" using {exe} with args {args}',
            worker_id=worker_id,
            worker_logname=worker_logname,
            exe=exe,
            args=args)
        d = ep.connect(transport_factory)

        def on_connect_success(proto):

            # this seems to be called immediately when the child process
            # has been forked. even if it then immediately fails because
            # e.g. the executable doesn't even exist. in other words,
            # I'm not sure under what conditions the deferred will errback ..

            self.log.debug('Native worker "{worker_id}" connected',
                           worker_id=worker_id)

            worker.on_worker_connected(proto)

            # dynamically add a dedicated authrole to the router
            # for the worker we've just started
            self._node._add_worker_role(worker_auth_role, options)

        def on_connect_error(err):

            # not sure when this errback is triggered at all ..
            self.log.error(
                "Internal error: connection to forked native worker failed ({err})",
                err=err)

            # in any case, forward the error ..
            worker.ready.errback(err)

        d.addCallbacks(on_connect_success, on_connect_error)

        return worker.ready
Example #58
0
 def setLogStr(self):
     self.logstr = reflect.qual(self.protocol.__class__) + " (TUNTAP)"
Example #59
0
 def _stateFrom(state):
     retval.setObj(
         Instance(reflect.qual(obj.__class__),
                  self.jellyToAO(state)))
Example #60
0
 def doIteration(self, delay):
     """
     Do one iteration over the readers and writers which have been added.
     """
     raise NotImplementedError(
         reflect.qual(self.__class__) + " did not implement doIteration")