Beispiel #1
0
    def prevalidate(self, contents):
        for name, ents in contents:
            if name == 'Server':
                for k, v, _ in ents:
                    if k == 'Descriptor-Version' and v.strip() != '0.2':
                        raise ConfigError(
                            "Unrecognized descriptor version: %s" % v.strip())

        # Remove any sections with unrecognized versions.
        revisedContents = []
        for name, ents in contents:
            v = self.expected_versions.get(name)
            if not v:
                revisedContents.append((name, ents))
                continue
            versionkey, versionval = v
            for k, v, _ in ents:
                if k == versionkey and v.strip() != versionval:
                    LOG.warn(
                        "Skipping %s section with unrecognized version %s",
                        name, v.strip())
                    break
            else:
                revisedContents.append((name, ents))

        return revisedContents
Beispiel #2
0
    def processMessage(self, packet):
        assert packet.getExitType() == 0xFFFE
        exitInfo = packet.getAddress()

        if exitInfo == 'fail':
            return DELIVER_FAIL_RETRY
        elif exitInfo == 'FAIL!':
            return DELIVER_FAIL_NORETRY

        LOG.debug("Delivering test message")

        m = _escapeMessageForEmail(packet)
        if m is None:
            # Ordinarily, we'd drop corrupt messages, but this module is
            # meant for debugging.
            m = """\
==========CORRUPT OR UNDECODABLE MESSAGE
Decoding handle: %s%s==========MESSAGE ENDS""" % (
                      base64.encodestring(packet.getTag()),
                      base64.encodestring(packet.getContents()))

        f = open(os.path.join(self.loc, str(self.next)), 'w')
        self.next += 1
        f.write(m)
        f.close()
        return DELIVER_OK
Beispiel #3
0
 def regenerateDescriptors(self):
     """Regenerate all server descriptors for all keysets in this
        keyring, but keep all old keys intact."""
     LOG.info("Regenerating server descriptors; keeping old keys.")
     identityKey = self.getIdentityKey()
     for _,_,ks in self.keySets:
         ks.regenerateServerDescriptor(self.config, identityKey)
Beispiel #4
0
    def checkDescriptorConsistency(self, regen=1):
        """Check whether the server descriptors in this keyring are
           consistent with the server's configuration.  If 'regen' is
           true, inconsistent descriptors are regenerated."""
        identity = None
        state = []
        for _,_,ks in self.keySets:
            ok = ks.checkConsistency(self.config, 0)
            if ok == 'good':
                continue
            state.append((ok, ks))

        if not state:
            return

        LOG.warn("Some generated keysets do not match "
                  "current configuration...")

        for ok, ks in state:
            va,vu = ks.getLiveness()
            LOG.warn("Keyset %s (%s--%s):",ks.keyname,formatTime(va,1),
                     formatTime(vu,1))
            ks.checkConsistency(self.config, 1)
            if regen and ok == 'bad':
                if not identity: identity = self.getIdentityKey()
                ks.regenerateServerDescriptor(self.config, identity)
Beispiel #5
0
    def getSigners(self):
        #DOCDOC -- returns members of self.dirInfo.voters with valid signatures.
        if self.signers is not None:
            return self.signers

        sigs = {}
        self.signers = []
        for s in self.signatures:
            sigs[s.getKeyFingerprint()] = s
        for digest, url in self.dirInfo.voters:
            try:
                s = sigs[digest]
            except KeyError:
                #XXXX008 log something.
                continue
            if s.checkSignature():
                LOG.trace("Found valid signature from %s at %s",
                          digest, url)
                self.signers.append((digest, url))
            else:
                LOG.trace("Signature claiming to be from %s was not valid",
                          digest)
                continue

        return self.signers
Beispiel #6
0
    def processMessage(self, packet):
        assert packet.getExitType() == 0xFFFE
        exitInfo = packet.getAddress()

        if exitInfo == 'fail':
            return DELIVER_FAIL_RETRY
        elif exitInfo == 'FAIL!':
            return DELIVER_FAIL_NORETRY

        LOG.debug("Delivering test message")

        m = _escapeMessageForEmail(packet)
        if m is None:
            # Ordinarily, we'd drop corrupt messages, but this module is
            # meant for debugging.
            m = """\
==========CORRUPT OR UNDECODABLE MESSAGE
Decoding handle: %s%s==========MESSAGE ENDS""" % (base64.encodestring(
                packet.getTag()), base64.encodestring(packet.getContents()))

        f = open(os.path.join(self.loc, str(self.next)), 'w')
        self.next += 1
        f.write(m)
        f.close()
        return DELIVER_OK
Beispiel #7
0
    def lookup(self,name,cb):
        """Look up the name 'name', and pass the result to the callback
           function 'cb' when we're done.  The result will be of the
           same form as the return value of NetUtils.getIP: either
           (Family, Address, Time) or ('NOENT', Reason, Time).

           Note: The callback may be invoked from a different thread.  Either
           this thread or a DNS thread will block until the callback finishes,
           so it shouldn't be especially time-consuming.
        """
        # Check for a static IP first; no need to resolve that.
        v = mixminion.NetUtils.nameIsStaticIP(name)
        if v is not None:
            cb(name,v)
            return

        try:
            self.lock.acquire()
            v = self.cache.get(name)
            # If we don't have a cached answer, add cb to self.callbacks
            if v is None or v is PENDING:
                self.callbacks.setdefault(name, []).append(cb)
            # If we aren't looking up the answer, start looking it up.
            if v is None:
                LOG.trace("DNS cache starting lookup of %r", name)
                self._beginLookup(name)
        finally:
            self.lock.release()
        # If we _did_ have an answer, invoke the callback now.
        if v is not None and v is not PENDING:
            LOG.trace("DNS cache returning cached value %s for %r",
                      v,name)
            cb(name,v)
Beispiel #8
0
    def addChunk(self, h, fm):
        """Register a chunk with handle h and FragmentMetadata fm.  If the
           chunk is inconsistent with other fragments of this message,
           raise MismatchedFragment."""
        assert fm.isChunk
        assert fm.messageid == self.messageid
        if fm.size != self.params.length:
            raise MismatchedFragment("Mismatched message length")
        if fm.overhead != self.overhead:
            raise MismatchedFragment("Mismatched packet overhead")
        if self.chunks.has_key(fm.chunkNum):
            raise MismatchedFragment("Duplicate chunks")
        if fm.nym != self.nym:
            raise MismatchedFragment("Fragments received for differing identities")

        if self.inserted > fm.insertedDate:
            self.inserted = fm.insertedDate
        self.chunks[fm.chunkNum] = (h,fm)

        if self.fragmentsByChunk[fm.chunkNum]:
            LOG.warn("Found a chunk with unneeded fragments for message %r",
                     self.messageid)

        if self.readyChunks.get(fm.chunkNum):
            del self.readyChunks[fm.chunkNum]
Beispiel #9
0
    def addChunk(self, h, fm):
        """Register a chunk with handle h and FragmentMetadata fm.  If the
           chunk is inconsistent with other fragments of this message,
           raise MismatchedFragment."""
        assert fm.isChunk
        assert fm.messageid == self.messageid
        if fm.size != self.params.length:
            raise MismatchedFragment("Mismatched message length")
        if fm.overhead != self.overhead:
            raise MismatchedFragment("Mismatched packet overhead")
        if self.chunks.has_key(fm.chunkNum):
            raise MismatchedFragment("Duplicate chunks")
        if fm.nym != self.nym:
            raise MismatchedFragment(
                "Fragments received for differing identities")

        if self.inserted > fm.insertedDate:
            self.inserted = fm.insertedDate
        self.chunks[fm.chunkNum] = (h, fm)

        if self.fragmentsByChunk[fm.chunkNum]:
            LOG.warn("Found a chunk with unneeded fragments for message %r",
                     self.messageid)

        if self.readyChunks.get(fm.chunkNum):
            del self.readyChunks[fm.chunkNum]
Beispiel #10
0
    def getInbufLine(self,
                     maxBytes=None,
                     terminator="\r\n",
                     clear=0,
                     allowExtra=0):
        """Return the first prefix of the current inbuf that ends with the
           'terminator' string.

           Returns the string on success, None if no such string is
           found, and -1 on error.  Errors occur when: there are 'maxBytes'
           bytes available but the terminator is not found; or when
           'allowExtra' is false and there is data on the input buffer
           following the terminator."""
        s = self.getInbuf(maxBytes)
        idx = s.find(terminator)
        if idx < 0:
            if len(s) == maxBytes:
                LOG.warn("Too much data without EOL from %s", self.address)
                return -1
            else:
                return None
        if not allowExtra and idx + len(terminator) < self.inbuflen:
            LOG.warn("Trailing data after EOL from %s", self.address)
            return -1

        return self.getInbuf(idx + len(terminator), clear=clear)
Beispiel #11
0
def sendPackets(routing, packetList, timeout=300, callback=None):
    """Sends a list of packets to a server.  Raise MixProtocolError on
       failure.

       routing -- an instance of mixminion.Packet.IPV4Info or
                  mixminion.Packet.MMTPHostInfo.
                  If routing.keyinfo == '\000'*20, we ignore the server's
                  keyid.
       packetList -- a list of 32KB packets and control strings.  Control
           strings must be one of "JUNK" to send a 32KB padding chunk,
           or "RENEGOTIATE" to renegotiate the connection key.
       connectTimeout -- None, or a number of seconds to wait for data
           on the connection before raising TimeoutError.
       callback -- None, or a function to call with a index into packetList
           after each successful packet delivery.
    """
    # Find out where we're connecting to.
    serverName = mixminion.ServerInfo.displayServerByRouting(routing)
    if isinstance(routing, IPV4Info):
        family, addr = socket.AF_INET, routing.ip
    else:
        assert isinstance(routing, MMTPHostInfo)
        LOG.trace("Looking up %s...",routing.hostname)
        family, addr, _ = mixminion.NetUtils.getIP(routing.hostname)
        if family == "NOENT":
            raise MixProtocolError("Couldn't resolve hostname %s: %s" % (
                                   routing.hostname, addr))

    # Create an MMTPClientConnection
    try:
        con = MMTPClientConnection(
            family, addr, routing.port, routing.keyinfo, serverName=serverName)
    except socket.error, e:
        raise MixProtocolError(str(e))
Beispiel #12
0
    def checkDescriptorConsistency(self, regen=1):
        """Check whether the server descriptors in this keyring are
           consistent with the server's configuration.  If 'regen' is
           true, inconsistent descriptors are regenerated."""
        identity = None
        state = []
        for _, _, ks in self.keySets:
            ok = ks.checkConsistency(self.config, 0)
            if ok == 'good':
                continue
            state.append((ok, ks))

        if not state:
            return

        LOG.warn("Some generated keysets do not match "
                 "current configuration...")

        for ok, ks in state:
            va, vu = ks.getLiveness()
            LOG.warn("Keyset %s (%s--%s):", ks.keyname, formatTime(va, 1),
                     formatTime(vu, 1))
            ks.checkConsistency(self.config, 1)
            if regen and ok == 'bad':
                if not identity: identity = self.getIdentityKey()
                ks.regenerateServerDescriptor(self.config, identity)
Beispiel #13
0
 def regenerateDescriptors(self):
     """Regenerate all server descriptors for all keysets in this
        keyring, but keep all old keys intact."""
     LOG.info("Regenerating server descriptors; keeping old keys.")
     identityKey = self.getIdentityKey()
     for _, _, ks in self.keySets:
         ks.regenerateServerDescriptor(self.config, identityKey)
Beispiel #14
0
def _validateZlib():
    """Internal function:  Make sure that zlib is a recognized version, and
       that it compresses things as expected.  (This check is important,
       because using a zlib version that compressed differently from zlib1.1.4
       would make senders partitionable by payload compression.)
    """
    global _ZLIB_LIBRARY_OK
    ver = getattr(zlib, "ZLIB_VERSION", None)
    if ver and ver < "1.1.2":
        raise MixFatalError("Zlib version %s is not supported"%ver)

    _ZLIB_LIBRARY_OK = 0.5
    if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2",
               "1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7",
               "1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2",
               "1.2.3", "1.2.7", "1.2.8"):
        _ZLIB_LIBRARY_OK = 1
        return

    LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver)
    # This test is inadequate, but it _might_ catch future incompatible
    # changes.
    _ZLIB_LIBRARY_OK = 0.5
    good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
           '`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
           '\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
           'q\x8d\n\x93'
    if compressData("aZbAAcdefg"*1000) == good:
        _ZLIB_LIBRARY_OK = 1
    else:
        _ZLIB_LIBRARY_OK = 0
        raise MixFatalError("Zlib output not as exected.")
Beispiel #15
0
class ServerInbox:
    """A ServerInbox holds server descriptors received from the outside
       world that are not yet ready to be included in the directory.
       """
    ## Fields:
    # newQueue: IncomingQueue object to hold descriptors for previously
    #      unknown servers.
    # updateQueue:  IncomingQueue object to hold descriptors for currently
    #      known servers.
    def __init__(self, base, idCache):
        """Initialize a ServerInbox to store its files in 'base', and
           check server descriptors against the IDCache 'idCache'."""
        self.newQueue = IncomingQueue(os.path.join(base, "new"),
                                      os.path.join(base, "reject"))
        self.updateQueue = IncomingQueue(os.path.join(base, "updates"),
                                         os.path.join(base, "reject"))
        self.idCache = idCache

    def receiveServer(self, text, source):
        """Process a new server descriptor and store it for later action.
           (To be run by the CGI user.)

           If the server will be automatically inserted, return true.
           If the server will be inserted (given administrator intervention),
              raise ServerQueuedException.
           If there is a problem, log it, and raise UIError.

           text -- a string containing a new server descriptor.
           source -- a (human readable) string describing the source
               of the descriptor, used in error messages.

           """
        try:
            server = ServerInfo(string=text,assumeValid=0)
        except MixError, e:
            LOG.warn("Rejected invalid server from %s: %s", source,e)
            raise UIError("Server descriptor was not valid: %s"%e)

        nickname = server.getNickname()

        try:
            known = self.idCache.containsServer(server)
        except MismatchedID:
            LOG.warn("Rejected server with mismatched identity from %s",
                     source)
            self.updateQueue.queueRejectedServer(text,server)
            raise UIError(("I already know a server named "
                           "%s with a different key.")%nickname)

        if not known:
            LOG.info("Received previously unknown server %s from %s",
                     nickname, source)
            self.newQueue.queueIncomingServer(text,server)
            raise ServerQueuedException(
                "Server queued pending manual checking")
        else:
            LOG.info("Received update for server %s from %s",
                     nickname, source)
            self.updateQueue.queueIncomingServer(text,server)
            return 1
Beispiel #16
0
    def readProtocol(self):
        s = self.getInbufLine(4096,clear=1)
        if s is None:
            return
        elif s == -1:
            self.startShutdown()
            #failed
            return

        self.stopReading()

        m = PROTOCOL_RE.match(s)
        if not m:
            LOG.warn("Bad MMTP protocol string format from %s", self.address)
            #failed
            self.startShutdown()
            return

        protocols = m.group(1).split(",")
        for p in self.PROTOCOL_VERSIONS:
            if p in protocols:
                self.protocol = p
                self.onWrite = self.protocolWritten
                self.beginWriting("MMTP %s\r\n"%p)
                return
        LOG.warn("No common protocols with %s", self.address)
        #failed
        self.startShutdown()
Beispiel #17
0
    def prevalidate(self, contents):
        for name, ents in contents:
            if name == 'Server':
                for k,v,_ in ents:
                    if k == 'Descriptor-Version' and v.strip() != '0.2':
                        raise ConfigError("Unrecognized descriptor version: %s"
                                          % v.strip())


        # Remove any sections with unrecognized versions.
        revisedContents = []
        for name, ents in contents:
            v = self.expected_versions.get(name)
            if not v:
                revisedContents.append((name, ents))
                continue
            versionkey, versionval = v
            for k,v,_ in ents:
                if k == versionkey and v.strip() != versionval:
                    LOG.warn("Skipping %s section with unrecognized version %s"
                             , name, v.strip())
                    break
            else:
                revisedContents.append((name, ents))

        return revisedContents
Beispiel #18
0
    def readProtocol(self):
        s = self.getInbufLine(4096, clear=1)
        if s is None:
            return
        elif s == -1:
            self.startShutdown()
            #failed
            return

        self.stopReading()

        m = PROTOCOL_RE.match(s)
        if not m:
            LOG.warn("Bad MMTP protocol string format from %s", self.address)
            #failed
            self.startShutdown()
            return

        protocols = m.group(1).split(",")
        for p in self.PROTOCOL_VERSIONS:
            if p in protocols:
                self.protocol = p
                self.onWrite = self.protocolWritten
                self.beginWriting("MMTP %s\r\n" % p)
                return
        LOG.warn("No common protocols with %s", self.address)
        #failed
        self.startShutdown()
Beispiel #19
0
def _validateZlib():
    """Internal function:  Make sure that zlib is a recognized version, and
       that it compresses things as expected.  (This check is important,
       because using a zlib version that compressed differently from zlib1.1.4
       would make senders partitionable by payload compression.)
    """
    global _ZLIB_LIBRARY_OK
    ver = getattr(zlib, "ZLIB_VERSION", None)
    if ver and ver < "1.1.2":
        raise MixFatalError("Zlib version %s is not supported" % ver)

    _ZLIB_LIBRARY_OK = 0.5
    if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2",
               "1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7",
               "1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2",
               "1.2.3"):
        _ZLIB_LIBRARY_OK = 1
        return

    LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver)
    # This test is inadequate, but it _might_ catch future incompatible
    # changes.
    _ZLIB_LIBRARY_OK = 0.5
    good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
           '`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
           '\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
           'q\x8d\n\x93'
    if compressData("aZbAAcdefg" * 1000) == good:
        _ZLIB_LIBRARY_OK = 1
    else:
        _ZLIB_LIBRARY_OK = 0
        raise MixFatalError("Zlib output not as exected.")
Beispiel #20
0
 def __readTooMuch(self):
     """Helper function -- called if we read too much data while we're
        shutting down."""
     LOG.error("Read over 128 bytes of unexpected data from closing "
               "connection to %s", self.address)
     self.onTLSError()
     raise _Closing()
Beispiel #21
0
 def run(self):
     """Thread body: pull questions from the DNS thread queue and
        answer them."""
     queue = self.dnscache.queue
     _lookupDone = self.dnscache._lookupDone
     _adjBusyThreads = self.dnscache._adjBusyThreads
     _adjLiveThreads = self.dnscache._adjLiveThreads
     try:
         _adjLiveThreads(1)
         try:
             while 1:
                 # Get a question from the queue, but don't wait more than
                 # MAX_THREAD_IDLE seconds
                 hostname = queue.get(timeout=MAX_THREAD_IDLE)
                 # If the question is None, shutdown.
                 if hostname is None:
                     return
                 # Else, resolve the IP and send the answer to the dnscache
                 _adjBusyThreads(1)
                 result = mixminion.NetUtils.getIP(hostname)
                 _lookupDone(hostname, result)
                 _adjBusyThreads(-1)
         except QueueEmpty:
             LOG.debug("DNS thread shutting down: idle for %s seconds.",
                      MAX_THREAD_IDLE)
         except:
             LOG.error_exc(sys.exc_info(),
                           "Exception in DNS thread; shutting down.")
     finally:
         _adjLiveThreads(-1)
Beispiel #22
0
    def getHandlesByDestAndAge(self,
                               destList,
                               directory,
                               notAfter=None,
                               warnUnused=1):
        """Return a list of handles for all messages queued for servers in a
           given list before a given date.

              destList -- A list of hostnames, ips, keyids, or nicknames
                for servers whose messages should be included in the result.
              directory -- An instance of ClientDirectory used to resolve
                nicknames.  This may be None if no nicknames are included.
              notAfter -- If provided, a time such that no messages queued
                later should be included
              warnUnused -- If true, we log a message for every element in
                destList that has no matching messages in the queue.
        """
        destSet = {}
        reverse = {}
        for d in destList:
            if directory:
                keyid = directory.getKeyIDByNickname(d)
                if keyid:
                    destSet[keyid] = 1
                    reverse[keyid] = d
                    continue
            destSet[d] = 1

        self.loadMetadata()
        result = []
        foundAny = {}
        foundMatch = {}
        for h in self.store.getAllMessages():
            _, r, when = self.store.getMetadata(h)
            if (destSet.has_key(r.keyinfo)
                    or (hasattr(r, 'hostname') and destSet.has_key(r.hostname))
                    or (hasattr(r, 'ip') and destSet.has_key(r.ip))):

                keys = [
                    getattr(r, 'hostname', None),
                    getattr(r, 'ip', None),
                    reverse.get(r.keyinfo, None), r.keyinfo
                ]
                for k in keys:
                    foundAny[k] = 1
                if notAfter and when > notAfter:
                    continue
                for k in keys:
                    foundMatch[k] = 1
                result.append(h)
        if warnUnused:
            for d in destList:
                if foundMatch.get(d):
                    continue
                elif foundAny.get(d):
                    LOG.warn("No expired packets found for %r", d)
                else:
                    LOG.warn("No pending packets found for %r", d)
        return result
Beispiel #23
0
 def removeDeadKeys(self, now=None):
     """Remove all keys that have expired."""
     self.checkKeys()
     keys = self.getDeadKeys(now)
     for message, keyset in keys:
         LOG.info(message)
         keyset.delete()
     self.checkKeys()
Beispiel #24
0
 def process(self, r, w, x, cap):
     #XXXX007 do something with x
     try:
         con, addr = self.sock.accept()
         LOG.debug("Accepted connection from %s", addr)
         self.connectionFactory(con)
     except socket.error, e:
         LOG.warn("Socket error while accepting connection: %s", e)
Beispiel #25
0
 def process(self, r, w, x, cap):
     #XXXX007 do something with x
     try:
         con, addr = self.sock.accept()
         LOG.debug("Accepted connection from %s", addr)
         self.connectionFactory(con)
     except socket.error, e:
         LOG.warn("Socket error while accepting connection: %s", e)
Beispiel #26
0
 def __call__(self, *args):
     self.called = 1
     self.errors += 1
     if not self.published:
         args = list(args)
         args[0] = args[0].replace("published", "in unpublished descriptor")
     if not self.silence:
         LOG.warn(*args)
Beispiel #27
0
 def __call__(self, *args):
     self.called = 1
     self.errors += 1
     if not self.published:
         args = list(args)
         args[0] = args[0].replace("published", "in unpublished descriptor")
     if not self.silence:
         LOG.warn(*args)
Beispiel #28
0
 def getHeaders(self):
     """Return a dict containing the headers for this message."""
     if self.type is None:
         self.decode()
     if self.headers is None:
         LOG.warn("getHeaders found no decoded headers")
         return {}
     return self.headers
Beispiel #29
0
 def removeDeadKeys(self, now=None):
     """Remove all keys that have expired."""
     self.checkKeys()
     keys = self.getDeadKeys(now)
     for message, keyset in keys:
         LOG.info(message)
         keyset.delete()
     self.checkKeys()
Beispiel #30
0
 def __clientFinished(self, addr):
     """Called when a client connection runs out of packets to send,
        or halts."""
     try:
         del self.clientConByAddr[addr]
     except KeyError:
         LOG.warn("Didn't find client connection to %s in address map",
                  addr)
Beispiel #31
0
 def getHeaders(self):
     """Return a dict containing the headers for this message."""
     if self.type is None:
         self.decode()
     if self.headers is None:
         LOG.warn("getHeaders found no decoded headers")
         return {}
     return self.headers
Beispiel #32
0
 def __readTooMuch(self):
     """Helper function -- called if we read too much data while we're
        shutting down."""
     LOG.error(
         "Read over 128 bytes of unexpected data from closing "
         "connection to %s", self.address)
     self.onTLSError()
     raise _Closing()
Beispiel #33
0
 def __clientFinished(self, addr):
     """Called when a client connection runs out of packets to send,
        or halts."""
     try:
         del self.clientConByAddr[addr]
     except KeyError:
         LOG.warn("Didn't find client connection to %s in address map",
                  addr)
Beispiel #34
0
    def deliveryFailed(self, handle, retriable=0, now=None):
        """Removes a message from the outgoing queue, or requeues it
           for delivery at a later time.  This method should be
           invoked after the corresponding message has been
           unsuccessfully delivered."""
        assert self.retrySchedule is not None
        LOG.trace("DeliveryQueue failed to deliver %s from %s",
                  handle, self.qname)
        try:
            self._lock.acquire()
            try:
                ds = self.store.getMetadata(handle)
            except KeyError:
                ds = None
            except CorruptedFile:
                return

            if ds is None:
                # This should never happen
                LOG.error_exc(sys.exc_info(),
                              "Handle %s had no state", handle)
                ds = _DeliveryState(now)
                ds.setNextAttempt(self.retrySchedule, now)
                self.store.setMetadata(handle, ds)
                return

            if not ds.isPending():
                LOG.error("Handle %s was not pending", handle)
                return

            last = ds.pending
            ds.setNonPending()

            if retriable:
                # If we can retry the message, update the deliveryState
                # with the most recent attempt, and see if there's another
                # attempt in the future.
                ds.setLastAttempt(last)
                ds.setNextAttempt(self.retrySchedule, now)
                if ds.nextAttempt is not None:
                    # There is another scheduled delivery attempt.  Remember
                    # it, mark the message sendable again, and save our state.
                    LOG.trace("     (We'll try %s again at %s)", handle,
                              formatTime(ds.nextAttempt, 1))

                    self.store.setMetadata(handle, ds)
                    return
                else:
                    assert ds.isRemovable()
                # Otherwise, fallthrough.

            # If we reach this point, the message is undeliverable, either
            # because 'retriable' is false, or because we've run out of
            # retries.
            LOG.trace("     (Giving up on %s)", handle)
            self.removeMessage(handle)
        finally:
            self._lock.release()
Beispiel #35
0
def configure_trng(config):
    """Initialize the true entropy source from a given Config object.  If
       none is provided, tries some sane defaults."""
    global _TRNG_FILENAME
    global _theTrueRNG

    if sys.platform == 'win32':
        # We have two entropy sources on windows: openssl's built-in
        # entropy generator that takes data from the screen, and
        # Windows's CryptGenRandom function.  Because the former is
        # insecure, and the latter is closed-source, we xor them.
        _ml.win32_openssl_seed()
        _ml.openssl_seed(_ml.win32_get_random_bytes(32))
        _theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG())
        return

    if config is not None:
        requestedFile = config['Host'].get('EntropySource')
    else:
        requestedFile = None

    # Build a list of candidates
    defaults =  PLATFORM_TRNG_DEFAULTS.get(sys.platform,
                           PLATFORM_TRNG_DEFAULTS['***'])
    files = [ requestedFile ] + defaults

    # Now find the first of our candidates that exists and is a character
    # device.
    randFile = None
    for filename in files:
        if filename is None:
            continue

        verbose = (filename == requestedFile)
        if not os.path.exists(filename):
            if verbose:
                LOG.warn("No such file as %s", filename)
        else:
            st = os.stat(filename)
            if not (st[stat.ST_MODE] & stat.S_IFCHR):
                if verbose:
                    LOG.error("Entropy source %s isn't a character device",
                                   filename)
            else:
                randFile = filename
                break

    if randFile is None and _TRNG_FILENAME is None:
        LOG.fatal("No entropy source available: Tried all of %s",
                  files)
        raise MixFatalError("No entropy source available")
    elif randFile is None:
        LOG.warn("Falling back to previous entropy source %s",
                 _TRNG_FILENAME)
    else:
        LOG.info("Setting entropy source to %r", randFile)
        _TRNG_FILENAME = randFile
        _theTrueRNG = _TrueRNG(1024)
Beispiel #36
0
 def getBaseDir(self):
     """Return the base directory for this configuration."""
     v = self["Server"]["BaseDir"]
     if v is None:
         v = self["Server"]["Homedir"]
     if v is None:
         LOG.warn("Defaulting base directory to /var/spool/minion; this will change.")
         v = "/var/spool/minion"
     return v
Beispiel #37
0
    def deliverySucceeded(self, handle, now=None):
        """Removes a message from the outgoing queue.  This method
           should be invoked after the corresponding message has been
           successfully delivered.
        """
        assert self.retrySchedule is not None

        LOG.trace("DeliveryQueue got successful delivery for %s from %s", handle, self.qname)
        self.removeMessage(handle)
Beispiel #38
0
    def deliveryFailed(self, handle, retriable=0, now=None):
        """Removes a message from the outgoing queue, or requeues it
           for delivery at a later time.  This method should be
           invoked after the corresponding message has been
           unsuccessfully delivered."""
        assert self.retrySchedule is not None
        LOG.trace("DeliveryQueue failed to deliver %s from %s", handle,
                  self.qname)
        try:
            self._lock.acquire()
            try:
                ds = self.store.getMetadata(handle)
            except KeyError:
                ds = None
            except CorruptedFile:
                return

            if ds is None:
                # This should never happen
                LOG.error_exc(sys.exc_info(), "Handle %s had no state", handle)
                ds = _DeliveryState(now)
                ds.setNextAttempt(self.retrySchedule, now)
                self.store.setMetadata(handle, ds)
                return

            if not ds.isPending():
                LOG.error("Handle %s was not pending", handle)
                return

            last = ds.pending
            ds.setNonPending()

            if retriable:
                # If we can retry the message, update the deliveryState
                # with the most recent attempt, and see if there's another
                # attempt in the future.
                ds.setLastAttempt(last)
                ds.setNextAttempt(self.retrySchedule, now)
                if ds.nextAttempt is not None:
                    # There is another scheduled delivery attempt.  Remember
                    # it, mark the message sendable again, and save our state.
                    LOG.trace("     (We'll try %s again at %s)", handle,
                              formatTime(ds.nextAttempt, 1))

                    self.store.setMetadata(handle, ds)
                    return
                else:
                    assert ds.isRemovable()
                # Otherwise, fallthrough.

            # If we reach this point, the message is undeliverable, either
            # because 'retriable' is false, or because we've run out of
            # retries.
            LOG.trace("     (Giving up on %s)", handle)
            self.removeMessage(handle)
        finally:
            self._lock.release()
Beispiel #39
0
def encodeMessage(message, overhead, uncompressedFragmentPrefix="",
                  paddingPRNG=None):
    """Given a message, compress it, fragment it into individual payloads,
       and add extra fields (size, hash, etc) as appropriate.  Return a list
       of strings, each of which is a message payload suitable for use in
       build*Message.

              message: the initial message
              overhead: number of bytes to omit from each payload,
                        given the type ofthe message encoding.
                        (0 or ENC_FWD_OVERHEAD)
              uncompressedFragmentPrefix: If we fragment the message,
                we add this string to the message after compression but
                before whitening and fragmentation.
              paddingPRNG: generator for padding.

       Note: If multiple strings are returned, be sure to shuffle them
       before transmitting them to the network.
    """
    assert overhead in (0, ENC_FWD_OVERHEAD)
    if paddingPRNG is None:
        paddingPRNG = Crypto.getCommonPRNG()
    origLength = len(message)
    payload = compressData(message)
    length = len(payload)

    if length > 1024 and length*20 <= origLength:
        LOG.warn("Message is very compressible and will look like a zlib bomb")

    paddingLen = PAYLOAD_LEN - SINGLETON_PAYLOAD_OVERHEAD - overhead - length

    # If the compressed payload fits in 28K, we're set.
    if paddingLen >= 0:
        # We pad the payload, and construct a new SingletonPayload,
        # including this payload's size and checksum.
        payload += paddingPRNG.getBytes(paddingLen)
        p = SingletonPayload(length, None, payload)
        p.computeHash()
        return [ p.pack() ]

    # Okay, we need to fragment the message.  First, add the prefix if needed.
    if uncompressedFragmentPrefix:
        payload = uncompressedFragmentPrefix+payload
    # Now generate a message ID
    messageid = Crypto.getCommonPRNG().getBytes(FRAGMENT_MESSAGEID_LEN)
    # Figure out how many chunks to divide it into...
    p = mixminion.Fragments.FragmentationParams(len(payload), overhead)
    # ... fragment the payload into chunks...
    rawFragments = p.getFragments(payload)
    fragments = []
    # ... and annotate each chunk with appropriate payload header info.
    for i in xrange(len(rawFragments)):
        pyld = FragmentPayload(i, None, messageid, p.length, rawFragments[i])
        pyld.computeHash()
        fragments.append(pyld.pack())
        rawFragments[i] = None
    return fragments
Beispiel #40
0
    def onProtocolWritten(self,n):
        if self.outbuf:
            # Not done writing outgoing data.
            return

        LOG.debug("Sent MMTP protocol string to %s", self.address)
        self.stopWriting()
        self.beginReading()
        self.onRead = self.onProtocolRead
Beispiel #41
0
def encodeMessage(message, overhead, uncompressedFragmentPrefix="",
                  paddingPRNG=None):
    """Given a message, compress it, fragment it into individual payloads,
       and add extra fields (size, hash, etc) as appropriate.  Return a list
       of strings, each of which is a message payload suitable for use in
       build*Message.

              message: the initial message
              overhead: number of bytes to omit from each payload,
                        given the type ofthe message encoding.
                        (0 or ENC_FWD_OVERHEAD)
              uncompressedFragmentPrefix: If we fragment the message,
                we add this string to the message after compression but
                before whitening and fragmentation.
              paddingPRNG: generator for padding.

       Note: If multiple strings are returned, be sure to shuffle them
       before transmitting them to the network.
    """
    assert overhead in (0, ENC_FWD_OVERHEAD)
    if paddingPRNG is None:
        paddingPRNG = Crypto.getCommonPRNG()
    origLength = len(message)
    payload = compressData(message)
    length = len(payload)

    if length > 1024 and length*20 <= origLength:
        LOG.warn("Message is very compressible and will look like a zlib bomb")

    paddingLen = PAYLOAD_LEN - SINGLETON_PAYLOAD_OVERHEAD - overhead - length

    # If the compressed payload fits in 28K, we're set.
    if paddingLen >= 0:
        # We pad the payload, and construct a new SingletonPayload,
        # including this payload's size and checksum.
        payload += paddingPRNG.getBytes(paddingLen)
        p = SingletonPayload(length, None, payload)
        p.computeHash()
        return [ p.pack() ]

    # Okay, we need to fragment the message.  First, add the prefix if needed.
    if uncompressedFragmentPrefix:
        payload = uncompressedFragmentPrefix+payload
    # Now generate a message ID
    messageid = Crypto.getCommonPRNG().getBytes(FRAGMENT_MESSAGEID_LEN)
    # Figure out how many chunks to divide it into...
    p = mixminion.Fragments.FragmentationParams(len(payload), overhead)
    # ... fragment the payload into chunks...
    rawFragments = p.getFragments(payload)
    fragments = []
    # ... and annotate each chunk with appropriate payload header info.
    for i in xrange(len(rawFragments)):
        pyld = FragmentPayload(i, None, messageid, p.length, rawFragments[i])
        pyld.computeHash()
        fragments.append(pyld.pack())
        rawFragments[i] = None
    return fragments
Beispiel #42
0
    def onProtocolWritten(self, n):
        if self.outbuf:
            # Not done writing outgoing data.
            return

        LOG.debug("Sent MMTP protocol string to %s", self.address)
        self.stopWriting()
        self.beginReading()
        self.onRead = self.onProtocolRead
Beispiel #43
0
def configure_trng(config):
    """Initialize the true entropy source from a given Config object.  If
       none is provided, tries some sane defaults."""
    global _TRNG_FILENAME
    global _theTrueRNG

    if sys.platform == 'win32':
        # We have two entropy sources on windows: openssl's built-in
        # entropy generator that takes data from the screen, and
        # Windows's CryptGenRandom function.  Because the former is
        # insecure, and the latter is closed-source, we xor them.
        _ml.win32_openssl_seed()
        _ml.openssl_seed(_ml.win32_get_random_bytes(32))
        _theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG())
        return

    if config is not None:
        requestedFile = config['Host'].get('EntropySource')
    else:
        requestedFile = None

    # Build a list of candidates
    defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform,
                                          PLATFORM_TRNG_DEFAULTS['***'])
    files = [requestedFile] + defaults

    # Now find the first of our candidates that exists and is a character
    # device.
    randFile = None
    for filename in files:
        if filename is None:
            continue

        verbose = (filename == requestedFile)
        if not os.path.exists(filename):
            if verbose:
                LOG.warn("No such file as %s", filename)
        else:
            st = os.stat(filename)
            if not (st[stat.ST_MODE] & stat.S_IFCHR):
                if verbose:
                    LOG.error("Entropy source %s isn't a character device",
                              filename)
            else:
                randFile = filename
                break

    if randFile is None and _TRNG_FILENAME is None:
        LOG.fatal("No entropy source available: Tried all of %s", files)
        raise MixFatalError("No entropy source available")
    elif randFile is None:
        LOG.warn("Falling back to previous entropy source %s", _TRNG_FILENAME)
    else:
        LOG.info("Setting entropy source to %r", randFile)
        _TRNG_FILENAME = randFile
        _theTrueRNG = _TrueRNG(1024)
Beispiel #44
0
 def onConnected(self):
     LOG.debug("Completed MMTP client connection to %s",self.address)
     # Is the certificate correct?
     try:
         self.certCache.check(self.tls, self.targetKeyID, self.address)
     except MixProtocolBadAuth, e:
         LOG.warn("Certificate error: %s. Shutting down connection.", e)
         self._failPendingPackets()
         self.startShutdown()
         return
Beispiel #45
0
    def __loadModules(self, section, sectionEntries):
        """Callback from the [Server] section of a config file.  Parses
           the module options, and adds new sections to the syntax
           accordingly."""
        self.moduleManager.setPath(section.get('ModulePath'))
        for mod in section.get('Module', []):
            LOG.info("Loading module %s", mod)
            self.moduleManager.loadExtModule(mod)

        self._syntax.update(self.moduleManager.getConfigSyntax())
Beispiel #46
0
    def deliverySucceeded(self, handle, now=None):
        """Removes a message from the outgoing queue.  This method
           should be invoked after the corresponding message has been
           successfully delivered.
        """
        assert self.retrySchedule is not None

        LOG.trace("DeliveryQueue got successful delivery for %s from %s",
                  handle, self.qname)
        self.removeMessage(handle)
Beispiel #47
0
 def onConnected(self):
     LOG.debug("Completed MMTP client connection to %s", self.address)
     # Is the certificate correct?
     try:
         self.certCache.check(self.tls, self.targetKeyID, self.address)
     except MixProtocolBadAuth, e:
         LOG.warn("Certificate error: %s. Shutting down connection.", e)
         self._failPendingPackets()
         self.startShutdown()
         return
Beispiel #48
0
 def rebuildIDCache(self):
     for fn in os.listdir(self.serverIDDir):
         fname = os.path.join(self.serverIDDir, fn)
         tp, val = readPickled(fname)
         if tp != "V0":
             LOG.warn("Weird file version %s on %s", tp, fname)
             continue
         nickname, ident = val
         ID = mixminion.Crypto.sha1(ident)
         self.idCache.insertID(nickname, ID)
Beispiel #49
0
 def rebuildIDCache(self):
     for fn in os.listdir(self.serverIDDir):
         fname = os.path.join(self.serverIDDir, fn)
         tp,val = readPickled(fname)
         if tp != "V0":
             LOG.warn("Weird file version %s on %s",tp,fname)
             continue
         nickname, ident = val
         ID = mixminion.Crypto.sha1(ident)
         self.idCache.insertID(nickname, ID)
Beispiel #50
0
    def __loadModules(self, section, sectionEntries):
        """Callback from the [Server] section of a config file.  Parses
           the module options, and adds new sections to the syntax
           accordingly."""
        self.moduleManager.setPath(section.get('ModulePath'))
        for mod in section.get('Module', []):
            LOG.info("Loading module %s", mod)
            self.moduleManager.loadExtModule(mod)

        self._syntax.update(self.moduleManager.getConfigSyntax())
Beispiel #51
0
 def tryTimeout(self, cutoff):
     """Close self.sock if the last activity on this connection was
        before 'cutoff'.  Returns true iff the connection is timed out.
     """
     if self.lastActivity <= cutoff:
         LOG.warn("Connection to %s timed out: %.2f seconds without activity",
                  self.address, time.time()-self.lastActivity)
         self.onTimeout()
         self.__close()
         return 1
     return 0
Beispiel #52
0
 def getBaseDir(self):
     """Return the base directory for this configuration."""
     v = self["Server"]["BaseDir"]
     if v is None:
         v = self["Server"]["Homedir"]
     if v is None:
         LOG.warn(
             "Defaulting base directory to /var/spool/minion; this will change."
         )
         v = "/var/spool/minion"
     return v
Beispiel #53
0
def main(cmd, args):
    """[Entry point] Multiplex among subcommands."""
    if len(args)<1 or ('-h', '--help') in args:
        usageAndExit()
    command = args[0]
    args = args[1:]
    if not SUBCOMMANDS.has_key(command):
        print "Unknown command", command
        usageAndExit()
    init_crypto()
    LOG.setMinSeverity("INFO")
    SUBCOMMANDS[command](args)
Beispiel #54
0
def main(cmd, args):
    """[Entry point] Multiplex among subcommands."""
    if len(args) < 1 or ('-h', '--help') in args:
        usageAndExit()
    command = args[0]
    args = args[1:]
    if not SUBCOMMANDS.has_key(command):
        print "Unknown command", command
        usageAndExit()
    init_crypto()
    LOG.setMinSeverity("INFO")
    SUBCOMMANDS[command](args)
Beispiel #55
0
    def getHandlesByDestAndAge(self, destList, directory, notAfter=None,
                               warnUnused=1):
        """Return a list of handles for all messages queued for servers in a
           given list before a given date.

              destList -- A list of hostnames, ips, keyids, or nicknames
                for servers whose messages should be included in the result.
              directory -- An instance of ClientDirectory used to resolve
                nicknames.  This may be None if no nicknames are included.
              notAfter -- If provided, a time such that no messages queued
                later should be included
              warnUnused -- If true, we log a message for every element in
                destList that has no matching messages in the queue.
        """
        destSet = {}
        reverse = {}
        for d in destList:
            if directory:
                keyid = directory.getKeyIDByNickname(d)
                if keyid:
                    destSet[keyid] = 1
                    reverse[keyid] = d
                    continue
            destSet[d] = 1

        self.loadMetadata()
        result = []
        foundAny = {}
        foundMatch = {}
        for h in self.store.getAllMessages():
            _, r, when = self.store.getMetadata(h)
            if (destSet.has_key(r.keyinfo) or
                (hasattr(r, 'hostname') and destSet.has_key(r.hostname)) or
                (hasattr(r, 'ip') and destSet.has_key(r.ip))):

                keys = [ getattr(r, 'hostname', None),
                         getattr(r, 'ip', None),
                         reverse.get(r.keyinfo, None),
                         r.keyinfo ]
                for k in keys: foundAny[k]=1
                if notAfter and when > notAfter:
                    continue
                for k in keys: foundMatch[k]=1
                result.append(h)
        if warnUnused:
            for d in destList:
                if foundMatch.get(d):
                    continue
                elif foundAny.get(d):
                    LOG.warn("No expired packets found for %r", d)
                else:
                    LOG.warn("No pending packets found for %r", d)
        return result
Beispiel #56
0
 def tryTimeout(self, cutoff):
     """Close self.sock if the last activity on this connection was
        before 'cutoff'.  Returns true iff the connection is timed out.
     """
     if self.lastActivity <= cutoff:
         LOG.warn(
             "Connection to %s timed out: %.2f seconds without activity",
             self.address,
             time.time() - self.lastActivity)
         self.onTimeout()
         self.__close()
         return 1
     return 0
Beispiel #57
0
 def run(self):
     """Internal: main body of processing thread."""
     try:
         while 1:
             job = self.mqueue.get()
             job()
     except ProcessingThread._Shutdown:
         LOG.info("Shutting down %s", self.threadName)
         return
     except:
         LOG.error_exc(sys.exc_info(),
                       "Exception in %s; shutting down thread.",
                       self.threadName)
Beispiel #58
0
 def run(self):
     """Internal: main body of processing thread."""
     try:
         while 1:
             job = self.mqueue.get()
             job()
     except ProcessingThread._Shutdown:
         LOG.info("Shutting down %s",self.threadName)
         return
     except:
         LOG.error_exc(sys.exc_info(),
                       "Exception in %s; shutting down thread.",
                       self.threadName)
Beispiel #59
0
 def __close(self, gotClose=0):
     """helper: close the underlying socket without cleaning up the TLS
        connection."""
     if gotClose:
         if self.__stateFn == self.__connectFn:
             LOG.warn("Couldn't connect to %s", self.address)
         else:
             LOG.warn("Unexpectedly closed connection to %s", self.address)
         self.onTLSError()
     self.sock.close()
     self.sock = None
     self.tls = None
     self.__stateFn = self.__closedFn
     self.onClosed()
Beispiel #60
0
 def getPacket(self, handle):
     """Given a handle, return a 3-tuple of the corresponding
        32K packet, {IPV4/Host}Info, and time of first queueing.  (The time
        is rounded down to the closest midnight GMT.)  May raise
        CorruptedFile."""
     obj = self.store.getObject(handle)
     try:
         magic, packet, routing, when = obj
     except (ValueError, TypeError):
         magic = None
     if magic != "PACKET-0":
         LOG.error("Unrecognized packet format for %s", handle)
         return None
     return packet, routing, when