def run(self): """Thread body: pull questions from the DNS thread queue and answer them.""" queue = self.dnscache.queue _lookupDone = self.dnscache._lookupDone _adjBusyThreads = self.dnscache._adjBusyThreads _adjLiveThreads = self.dnscache._adjLiveThreads try: _adjLiveThreads(1) try: while 1: # Get a question from the queue, but don't wait more than # MAX_THREAD_IDLE seconds hostname = queue.get(timeout=MAX_THREAD_IDLE) # If the question is None, shutdown. if hostname is None: return # Else, resolve the IP and send the answer to the dnscache _adjBusyThreads(1) result = mixminion.NetUtils.getIP(hostname) _lookupDone(hostname, result) _adjBusyThreads(-1) except QueueEmpty: LOG.debug("DNS thread shutting down: idle for %s seconds.", MAX_THREAD_IDLE) except: LOG.error_exc(sys.exc_info(), "Exception in DNS thread; shutting down.") finally: _adjLiveThreads(-1)
def processMessage(self, packet): assert packet.getExitType() == 0xFFFE exitInfo = packet.getAddress() if exitInfo == 'fail': return DELIVER_FAIL_RETRY elif exitInfo == 'FAIL!': return DELIVER_FAIL_NORETRY LOG.debug("Delivering test message") m = _escapeMessageForEmail(packet) if m is None: # Ordinarily, we'd drop corrupt messages, but this module is # meant for debugging. m = """\ ==========CORRUPT OR UNDECODABLE MESSAGE Decoding handle: %s%s==========MESSAGE ENDS""" % ( base64.encodestring(packet.getTag()), base64.encodestring(packet.getContents())) f = open(os.path.join(self.loc, str(self.next)), 'w') self.next += 1 f.write(m) f.close() return DELIVER_OK
def processMessage(self, packet): assert packet.getExitType() == 0xFFFE exitInfo = packet.getAddress() if exitInfo == 'fail': return DELIVER_FAIL_RETRY elif exitInfo == 'FAIL!': return DELIVER_FAIL_NORETRY LOG.debug("Delivering test message") m = _escapeMessageForEmail(packet) if m is None: # Ordinarily, we'd drop corrupt messages, but this module is # meant for debugging. m = """\ ==========CORRUPT OR UNDECODABLE MESSAGE Decoding handle: %s%s==========MESSAGE ENDS""" % (base64.encodestring( packet.getTag()), base64.encodestring(packet.getContents())) f = open(os.path.join(self.loc, str(self.next)), 'w') self.next += 1 f.write(m) f.close() return DELIVER_OK
def process(self, r, w, x, cap): #XXXX007 do something with x try: con, addr = self.sock.accept() LOG.debug("Accepted connection from %s", addr) self.connectionFactory(con) except socket.error, e: LOG.warn("Socket error while accepting connection: %s", e)
def onProtocolWritten(self,n): if self.outbuf: # Not done writing outgoing data. return LOG.debug("Sent MMTP protocol string to %s", self.address) self.stopWriting() self.beginReading() self.onRead = self.onProtocolRead
def onProtocolWritten(self, n): if self.outbuf: # Not done writing outgoing data. return LOG.debug("Sent MMTP protocol string to %s", self.address) self.stopWriting() self.beginReading() self.onRead = self.onProtocolRead
def _sendPackets(self, family, ip, port, keyID, deliverable, serverName): """Begin sending a set of packets to a given server. 'deliverable' is a list of objects obeying the DeliverableMessage interface. """ try: # Is there an existing connection open to the right server? con = self.clientConByAddr[(ip, port, keyID)] except KeyError: pass else: # No exception: There is an existing connection. But is that # connection currently sending packets? if con.isActive(): LOG.debug("Queueing %s packets on open connection to %s", len(deliverable), con.address) for d in deliverable: con.addPacket(d) return if len(self.clientConByAddr) >= self.maxClientConnections: LOG.debug( "We already have %s open client connections; delaying %s packets for %s", len(self.clientConByAddr), len(deliverable), serverName) self.pendingPackets.append( (family, ip, port, keyID, deliverable, serverName)) return try: # There isn't any connection to the right server. Open one... addr = (ip, port, keyID) finished = lambda addr=addr, self=self: self.__clientFinished(addr) con = _ClientCon(family, ip, port, keyID, serverName=serverName, context=self.clientContext, certCache=self.certificateCache) nickname = mixminion.ServerInfo.getNicknameByKeyID(keyID) if nickname is not None: # If we recognize this server, then we'll want to tell # the ping log what happens to our connection attempt. con.configurePingLog(self.pingLog, keyID) #con.allPacketsSent = finished #XXXX007 wrong! con.onClosed = finished except (socket.error, MixProtocolError), e: LOG.error("Unexpected socket error connecting to %s: %s", serverName, e) EventStats.log.failedConnect() #FFFF addr for m in deliverable: try: m.failed(1) except AttributeError: pass
def onConnected(self): LOG.debug("Completed MMTP client connection to %s", self.address) # Is the certificate correct? try: self.certCache.check(self.tls, self.targetKeyID, self.address) except MixProtocolBadAuth, e: LOG.warn("Certificate error: %s. Shutting down connection.", e) self._failPendingPackets() self.startShutdown() return
def onConnected(self): LOG.debug("Completed MMTP client connection to %s",self.address) # Is the certificate correct? try: self.certCache.check(self.tls, self.targetKeyID, self.address) except MixProtocolBadAuth, e: LOG.warn("Certificate error: %s. Shutting down connection.", e) self._failPendingPackets() self.startShutdown() return
def checkKeys(self): """Internal method: read information about all this server's currently-prepared keys from disk. May raise ConfigError if any of the server descriptors on disk are invalid. """ self.keySets = [] badKeySets = [] firstKey = sys.maxint lastKey = 0 LOG.debug("Scanning server keystore at %s", self.keyDir) if not os.path.exists(self.keyDir): LOG.info("Creating server keystore at %s", self.keyDir) createPrivateDir(self.keyDir) # Iterate over the entires in HOME/keys for dirname in os.listdir(self.keyDir): # Skip any that aren't directories named "key_INT" if not os.path.isdir(os.path.join(self.keyDir, dirname)): continue if not dirname.startswith('key_'): LOG.warn("Unexpected directory %s under %s", dirname, self.keyDir) continue keysetname = dirname[4:] try: setNum = int(keysetname) # keep trace of the first and last used key number if setNum < firstKey: firstKey = setNum if setNum > lastKey: lastKey = setNum except ValueError: LOG.warn("Unexpected directory %s under %s", dirname, self.keyDir) continue # Find the server descriptor... keyset = ServerKeyset(self.keyDir, keysetname, self.hashDir) ok = 1 try: keyset.checkKeys() except MixError, e: LOG.warn("Error checking private keys in keyset %s: %s", keysetname, str(e)) ok = 0 try: if ok: keyset.getServerDescriptor() except (ConfigError, IOError), e: LOG.warn("Key set %s has invalid/missing descriptor: %s", keysetname, str(e)) ok = 0
def checkKeys(self): """Internal method: read information about all this server's currently-prepared keys from disk. May raise ConfigError if any of the server descriptors on disk are invalid. """ self.keySets = [] badKeySets = [] firstKey = sys.maxint lastKey = 0 LOG.debug("Scanning server keystore at %s", self.keyDir) if not os.path.exists(self.keyDir): LOG.info("Creating server keystore at %s", self.keyDir) createPrivateDir(self.keyDir) # Iterate over the entires in HOME/keys for dirname in os.listdir(self.keyDir): # Skip any that aren't directories named "key_INT" if not os.path.isdir(os.path.join(self.keyDir,dirname)): continue if not dirname.startswith('key_'): LOG.warn("Unexpected directory %s under %s", dirname, self.keyDir) continue keysetname = dirname[4:] try: setNum = int(keysetname) # keep trace of the first and last used key number if setNum < firstKey: firstKey = setNum if setNum > lastKey: lastKey = setNum except ValueError: LOG.warn("Unexpected directory %s under %s", dirname, self.keyDir) continue # Find the server descriptor... keyset = ServerKeyset(self.keyDir, keysetname, self.hashDir) ok = 1 try: keyset.checkKeys() except MixError, e: LOG.warn("Error checking private keys in keyset %s: %s", keysetname, str(e)) ok = 0 try: if ok: keyset.getServerDescriptor() except (ConfigError, IOError), e: LOG.warn("Key set %s has invalid/missing descriptor: %s", keysetname, str(e)) ok = 0
def rescan(self): """Check all fragment metadata objects on disk, and reconstruct our internal view of message states. """ # Delete all internal state; reload FragmentMetadatas from disk. self.store.loadAllMetadata(lambda: None) meta = self.store._metadata_cache self.states = {} badMessageIDs = {} # map from bad messageID to 1 unneededHandles = [] # list of handles that aren't needed. for h, fm in meta.items(): if not fm: LOG.debug("Removing fragment %s with missing metadata", h) self.store.removeMessage(h) continue try: mid = fm.messageid if badMessageIDs.has_key(mid): # We've already decided to reject fragments with this ID. pass else: # All is well; try to register the fragment/chunk. If it's # redundant or inconsistent, raise an exception. state = self._getState(fm) if fm.isChunk: state.addChunk(h, fm) else: state.addFragment(h, fm) except MismatchedFragment: # Mark the message ID for this fragment as inconsistent. badMessageIDs[mid] = 1 except UnneededFragment: LOG.warn("Found redundant fragment %s in pool", h) # Remember that this message is unneeded. unneededHandles.append(h) # Check for fragments superseded by chunks -- those are unneeded too. for s in self.states.values(): unneededHandles.extend(s.getUnneededFragmentHandles()) # Delete unneeded fragments. for h in unneededHandles: try: fm = meta[h] except KeyError: continue LOG.debug("Removing unneeded fragment %s from message ID %r", fm.idx, fm.messageid) self.store.removeMessage(h) # Now nuke inconsistent messages. self._deleteMessageIDs(badMessageIDs, "REJECTED")
def onDataRead(self): while self.inbuflen >= self.MESSAGE_LEN: data = self.getInbuf(self.MESSAGE_LEN, clear=1) control = data[:SEND_CONTROL_LEN] pkt = data[SEND_CONTROL_LEN:-DIGEST_LEN] digest = data[-DIGEST_LEN:] if control == JUNK_CONTROL: expectedDigest = sha1(pkt + "JUNK") replyDigest = sha1(pkt + "RECEIVED JUNK") replyControl = RECEIVED_CONTROL isJunk = 1 elif control == SEND_CONTROL: expectedDigest = sha1(pkt + "SEND") if self.rejectPackets: replyDigest = sha1(pkt + "REJECTED") replyControl = REJECTED_CONTROL else: replyDigest = sha1(pkt + "RECEIVED") replyControl = RECEIVED_CONTROL isJunk = 0 else: LOG.warn( "Unrecognized command (%r) from %s. Closing connection.", control, self.address) #failed self.startShutdown() return if expectedDigest != digest: LOG.warn("Invalid checksum from %s. Closing connection.", self.address) #failed self.startShutdown() return else: if isJunk: LOG.debug("Link padding received from %s; Checksum valid.", self.address) else: LOG.debug("Packet received from %s; Checksum valid.", self.address) # Make sure we process the packet before we queue the ack. if isJunk: self.junkCallback() elif self.rejectPackets: self.rejectCallback() else: self.packetConsumer(pkt) # Queue the ack. self.beginWriting(replyControl + replyDigest)
def _save(self, now=None): """Implements 'save' method. For internal use. Must hold self._lock to invoke.""" LOG.debug("Syncing statistics to disk") if not now: now = time() tmpfile = self.filename + "_tmp" tryUnlink(tmpfile) self.accumulatedTime += int(now-self.lastSave) self.lastSave = now writePickled(self.filename, { 'count' : self.count, 'lastRotation' : self.lastRotation, 'accumulatedTime' : self.accumulatedTime, })
def onDataRead(self): while self.inbuflen >= self.MESSAGE_LEN: data = self.getInbuf(self.MESSAGE_LEN, clear=1) control = data[:SEND_CONTROL_LEN] pkt = data[SEND_CONTROL_LEN:-DIGEST_LEN] digest = data[-DIGEST_LEN:] if control == JUNK_CONTROL: expectedDigest = sha1(pkt+"JUNK") replyDigest = sha1(pkt+"RECEIVED JUNK") replyControl = RECEIVED_CONTROL isJunk = 1 elif control == SEND_CONTROL: expectedDigest = sha1(pkt+"SEND") if self.rejectPackets: replyDigest = sha1(pkt+"REJECTED") replyControl = REJECTED_CONTROL else: replyDigest = sha1(pkt+"RECEIVED") replyControl = RECEIVED_CONTROL isJunk = 0 else: LOG.warn("Unrecognized command (%r) from %s. Closing connection.", control, self.address) #failed self.startShutdown() return if expectedDigest != digest: LOG.warn("Invalid checksum from %s. Closing connection.", self.address) #failed self.startShutdown() return else: if isJunk: LOG.debug("Link padding received from %s; Checksum valid.", self.address) else: LOG.debug("Packet received from %s; Checksum valid.", self.address) # Make sure we process the packet before we queue the ack. if isJunk: self.junkCallback() elif self.rejectPackets: self.rejectCallback() else: self.packetConsumer(pkt) # Queue the ack. self.beginWriting(replyControl+replyDigest)
def _buildReplyBlockImpl(path, exitType, exitInfo, expiryTime=0, secretPRNG=None, tag=None): """Helper function: makes a reply block, given a tag and a PRNG to generate secrets. Returns a 3-tuple containing (1) a newly-constructed reply block, (2) a list of secrets used to make it, (3) a tag. path: A list of ServerInfo exitType: Routing type to use for the final node exitInfo: Routing info for the final node, not including tag. expiryTime: The time at which this block should expire. secretPRNG: A PRNG to use for generating secrets. If not provided, uses an AES counter-mode stream seeded from our entropy source. Note: the secrets are generated so that they will be used to encrypt the message in reverse order. tag: If provided, a 159-bit tag. If not provided, a new one is generated. """ if secretPRNG is None: secretPRNG = Crypto.getCommonPRNG() if expiryTime is None: # XXXX This is dangerous, and should go away; the user should # XXXX *always* specify an expiry time. LOG.warn("Inferring expiry time for reply block") expiryTime = min([s.getValidUntil() for s in path]) checkPathLength(None, path, exitType, exitInfo, explicitSwap=0) LOG.debug("Building reply block for path %s", [s.getNickname() for s in path]) LOG.debug(" Delivering to %04x:%r", exitType, exitInfo) # The message is encrypted first by the end-to-end key, then by # each of the path keys in order. We need to reverse these steps, so we # generate the path keys back-to-front, followed by the end-to-end key. secrets = [ secretPRNG.getBytes(SECRET_LEN) for _ in range(len(path)+1) ] headerSecrets = secrets[:-1] headerSecrets.reverse() sharedKey = secrets[-1] # (This will go away when we deprecate 'stateful' reply blocks if tag is None: tag = _getRandomTag(secretPRNG) header = _buildHeader(path, headerSecrets, exitType, tag+exitInfo, paddingPRNG=Crypto.getCommonPRNG()) return ReplyBlock(header, expiryTime, SWAP_FWD_HOST_TYPE, path[0].getMMTPHostInfo().pack(), sharedKey), secrets, tag
def _sendPackets(self, family, ip, port, keyID, deliverable, serverName): """Begin sending a set of packets to a given server. 'deliverable' is a list of objects obeying the DeliverableMessage interface. """ try: # Is there an existing connection open to the right server? con = self.clientConByAddr[(ip,port,keyID)] except KeyError: pass else: # No exception: There is an existing connection. But is that # connection currently sending packets? if con.isActive(): LOG.debug("Queueing %s packets on open connection to %s", len(deliverable), con.address) for d in deliverable: con.addPacket(d) return if len(self.clientConByAddr) >= self.maxClientConnections: LOG.debug("We already have %s open client connections; delaying %s packets for %s", len(self.clientConByAddr), len(deliverable), serverName) self.pendingPackets.append((family,ip,port,keyID,deliverable,serverName)) return try: # There isn't any connection to the right server. Open one... addr = (ip, port, keyID) finished = lambda addr=addr, self=self: self.__clientFinished(addr) con = _ClientCon( family, ip, port, keyID, serverName=serverName, context=self.clientContext, certCache=self.certificateCache) nickname = mixminion.ServerInfo.getNicknameByKeyID(keyID) if nickname is not None: # If we recognize this server, then we'll want to tell # the ping log what happens to our connection attempt. con.configurePingLog(self.pingLog, keyID) #con.allPacketsSent = finished #XXXX007 wrong! con.onClosed = finished except (socket.error, MixProtocolError), e: LOG.error("Unexpected socket error connecting to %s: %s", serverName, e) EventStats.log.failedConnect() #FFFF addr for m in deliverable: try: m.failed(1) except AttributeError: pass
def _save(self, now=None): """Implements 'save' method. For internal use. Must hold self._lock to invoke.""" LOG.debug("Syncing statistics to disk") if not now: now = time() tmpfile = self.filename + "_tmp" tryUnlink(tmpfile) self.accumulatedTime += int(now - self.lastSave) self.lastSave = now writePickled( self.filename, { 'count': self.count, 'lastRotation': self.lastRotation, 'accumulatedTime': self.accumulatedTime, })
def _getDHFile(self): """Return the filename for the diffie-helman parameters for the server. Creates the file if it doesn't yet exist.""" dhdir = os.path.split(self.dhFile)[0] createPrivateDir(dhdir) if not os.path.exists(self.dhFile): # ???? This is only using 512-bit Diffie-Hellman! That isn't # ???? remotely enough. LOG.info("Generating Diffie-Helman parameters for TLS...") mixminion._minionlib.generate_dh_parameters(self.dhFile, verbose=0) LOG.info("...done") else: LOG.debug("Using existing Diffie-Helman parameter from %s", self.dhFile) return self.dhFile
def _sendQueuedPackets(self): """Helper function: Find all DNS lookup results and packets in self.msgQueue, and begin sending packets to the resulting servers. This function should only be called from the main thread. """ while len(self.clientConByAddr) < self.maxClientConnections and self.pendingPackets: args = self.pendingPackets.pop(0) LOG.debug("Sending %s delayed packets...",len(args[5])) self._sendPackets(*args) while 1: try: family,addr,port,keyID,deliverable,serverName = \ self.msgQueue.get(block=0) except QueueEmpty: return self._sendPackets(family,addr,port,keyID,deliverable,serverName)
def _updateRWState(self): """Helper: if we have any queued packets that haven't been sent yet, and we aren't waiting for WRITEAHEAD acks, and we're connected, start sending the pending packets. """ if not self._isConnected: return while self.nPacketsSent < self.nPacketsAcked + self.WRITEAHEAD: if not self.packets: break LOG.trace("Queueing new packet for %s", self.address) self._startSendingNextPacket() if self.nPacketsAcked == self.nPacketsSent: LOG.debug("Successfully relayed all packets to %s", self.address) self.allPacketsSent() self._isConnected = 0 self._isAlive = 0 self.startShutdown()
def _updateRWState(self): """Helper: if we have any queued packets that haven't been sent yet, and we aren't waiting for WRITEAHEAD acks, and we're connected, start sending the pending packets. """ if not self._isConnected: return while self.nPacketsSent < self.nPacketsAcked + self.WRITEAHEAD: if not self.packets: break LOG.trace("Queueing new packet for %s",self.address) self._startSendingNextPacket() if self.nPacketsAcked == self.nPacketsSent: LOG.debug("Successfully relayed all packets to %s",self.address) self.allPacketsSent() self._isConnected = 0 self._isAlive = 0 self.startShutdown()
def _sendQueuedPackets(self): """Helper function: Find all DNS lookup results and packets in self.msgQueue, and begin sending packets to the resulting servers. This function should only be called from the main thread. """ while len(self.clientConByAddr ) < self.maxClientConnections and self.pendingPackets: args = self.pendingPackets.pop(0) LOG.debug("Sending %s delayed packets...", len(args[5])) self._sendPackets(*args) while 1: try: family,addr,port,keyID,deliverable,serverName = \ self.msgQueue.get(block=0) except QueueEmpty: return self._sendPackets(family, addr, port, keyID, deliverable, serverName)
def buildReplyPacket(payload, path1, replyBlock, paddingPRNG=None): """Build a message using a reply block. 'path1' is a sequence of ServerInfo for the nodes on the first leg of the path. 'payload' must be exactly 28K long. """ if paddingPRNG is None: paddingPRNG = Crypto.getCommonPRNG() LOG.debug("Encoding reply message for %s-byte payload", len(payload)) LOG.debug(" Using path %s/??", [s.getNickname() for s in path1]) assert len(payload) == PAYLOAD_LEN # Encrypt the payload so that it won't appear as plaintext to the # crossover note. (We use 'decrypt' so that the message recipient can # simply use 'encrypt' to reverse _all_ the steps of the reply path.) k = Crypto.Keyset(replyBlock.encryptionKey).getLionessKeys(Crypto.PAYLOAD_ENCRYPT_MODE) payload = Crypto.lioness_decrypt(payload, k) return _buildPacket(payload, None, None, path1=path1, path2=replyBlock)
def __shutdownFn(self, r, w, cap): """state function: TLS shutdonw""" while 1: if self.__awaitingShutdown: # We've already sent a 'shutdown' once. Read until we # get another shutdown, or until we get enough data to # give up. s = "x" while s != 0: #XXXX007 respect cap. s = self.tls.read(_READLEN) # might raise TLSWant* if s == 0: LOG.debug("Read returned 0; shutdown to %s done", self.address) else: self.__bytesReadOnShutdown += len(s) if self.__bytesReadOnShutdown > 128: self.__readTooMuch() return 0 done = self.tls.shutdown() if not done and self.__awaitingShutdown: # This should neer actually happen, but let's cover the # possibility. LOG.error("Shutdown returned zero twice from %s--bailing", self.address) done = 1 if done: LOG.debug("Got a completed shutdown from %s", self.address) self.shutdownFinished() raise _Closing() else: LOG.trace("Shutdown returned zero -- entering read mode.") self.__awaitingShutdown = 1 self.__bytesReadOnShutdown = 0 self.wantRead = 1 return 1 raise AssertionError() # unreached; appease pychecker
def buildEncryptedForwardPacket(payload, exitType, exitInfo, path1, path2, key, paddingPRNG=None, secretRNG=None): """Construct a forward message encrypted with the public key of a given user. payload: The payload to deliver. Must be 28K-42b long. exitType: The routing type for the final node. (2 bytes, >=0x100) exitInfo: The routing info for the final node, not including tag. path1: Sequence of ServerInfo objects for the first leg of the path path2: Sequence of ServerInfo objects for the 2nd leg of the path key: Public key of this message's recipient. paddingPRNG: random number generator used to generate padding. If None, a new PRNG is initialized. """ if paddingPRNG is None: paddingPRNG = Crypto.getCommonPRNG() if secretRNG is None: secretRNG = paddingPRNG LOG.debug("Encoding encrypted forward message for %s-byte payload", len(payload)) LOG.debug(" Using path %s/%s", [s.getNickname() for s in path1], [s.getNickname() for s in path2]) LOG.debug(" Delivering to %04x:%r", exitType, exitInfo) # (For encrypted-forward messages, we have overhead for OAEP padding # and the session key, but we save 20 bytes by spilling into the tag.) assert len(payload) == PAYLOAD_LEN - ENC_FWD_OVERHEAD # Generate the session key, and prepend it to the payload. sessionKey = secretRNG.getBytes(SECRET_LEN) payload = sessionKey+payload # We'll encrypt the first part of the new payload with RSA, and the # second half with Lioness, based on the session key. rsaDataLen = key.get_modulus_bytes()-OAEP_OVERHEAD rsaPart = payload[:rsaDataLen] lionessPart = payload[rsaDataLen:] # RSA encryption: To avoid leaking information about our RSA modulus, # we keep trying to encrypt until the MSBit of our encrypted value is # zero. while 1: encrypted = Crypto.pk_encrypt(rsaPart, key) if not (ord(encrypted[0]) & 0x80): break # Lioness encryption. k= Crypto.Keyset(sessionKey).getLionessKeys(Crypto.END_TO_END_ENCRYPT_MODE) lionessPart = Crypto.lioness_encrypt(lionessPart, k) # Now we re-divide the payload into the part that goes into the tag, and # the 28K of the payload proper... payload = encrypted + lionessPart tag = payload[:TAG_LEN] payload = payload[TAG_LEN:] exitInfo = tag + exitInfo assert len(payload) == 28*1024 # And now, we can finally build the message. return _buildPacket(payload, exitType, exitInfo, path1, path2,paddingPRNG)
def _rotate(self, now=None): """Flush all events since the last rotation to the history file, and clears the current event log.""" # Must hold lock LOG.debug("Flushing statistics log") if now is None: now = time() starting = not os.path.exists(self.historyFilename) f = open(self.historyFilename, 'a') if starting: f.write(BOILERPLATE) self.dump(f, now) f.close() self.count = {} for e in _EVENTS: self.count[e] = {} self.lastRotation = now self._save(now) self.accumulatedTime = 0 self._setNextRotation(now)
def onDataRead(self): # We got some data from the server: it'll be 0 or more acks. if self.inbuflen < self.ACK_LEN: # If we have no acks at all, do nothing. return while self.inbuflen >= self.ACK_LEN: if not self.expectedAcks: LOG.warn( "Received acknowledgment from %s with no corresponding message", self.address) self._failPendingPackets() self.startShutdown() return ack = self.getInbuf(self.ACK_LEN, clear=1) good, bad = self.expectedAcks.pop(0) if ack == good: LOG.debug("Packet delivered to %s", self.address) self.nPacketsAcked += 1 if not self.pendingPackets[0].isJunk(): EventStats.log.successfulRelay() self.pendingPackets[0].succeeded() del self.pendingPackets[0] elif ack == bad: LOG.warn("Packet rejected by %s", self.address) self.nPacketsAcked += 1 if not self.pendingPackets[0].isJunk(): EventStats.log.failedRelay() self.pendingPackets[0].failed(1) del self.pendingPackets[0] else: # The control string and digest are wrong for an accepted # or rejected packet! LOG.warn("Bad acknowledgement received from %s", self.address) self._failPendingPackets() self.startShutdown() return # Start sending more packets, if we were waiting for an ACK to do so. self._updateRWState()
def generateConsensusDirectory(identity, voters, validAfter, directories, validatedDigests=None): # directories is (source, stringable) list # First -- whom shall we vote with? goodDirectories = {} # {fingerprint: (src,SignedDirectory)} serverMap = {} # digest->server info serversByDir = {} # keyid->list of digest for src, val in directories: LOG.debug("Checking vote directory from %s", src) val = str(val) try: directory = mixminion.ServerInfo.SignedDirectory( string=val, validatedDigests=validatedDigests, _keepServerContents=1 ) except ConfigError, e: LOG.warn("Rejecting malformed vote directory from %s: %s", src, e) continue try: checkVoteDirectory(voters, validAfter, directory) except BadVote, e: LOG.warn("Rejecting vote directory from %s: %s", src, e) continue
def onProtocolRead(self): # Pull the contents of the buffer up to the first CRLF s = self.getInbufLine(4096, clear=1) if s is None: # We have <4096 bytes, and no CRLF yet return elif s == -1: # We got 4096 bytes with no CRLF, or a CRLF with more data # after it. self._failPendingPackets() self.startShutdown() return # Find which protocol the server chose. self.protocol = None for p in self.PROTOCOL_VERSIONS: if s == "MMTP %s\r\n" % p: self.protocol = p break if not self.protocol: LOG.warn("Protocol negotiation failed with %s", self.address) self._failPendingPackets() self.startShutdown() return LOG.debug("MMTP protocol negotiated with %s: version %s", self.address, self.protocol) # Now that we're connected, optimize for throughput. mixminion.NetUtils.optimizeThroughput(self.sock) self.onRead = self.onDataRead self.onWrite = self.onDataWritten self.beginReading() self._isConnected = 1 # Now that we're connected, start sending packets. self._updateRWState()
def onProtocolRead(self): # Pull the contents of the buffer up to the first CRLF s = self.getInbufLine(4096,clear=1) if s is None: # We have <4096 bytes, and no CRLF yet return elif s == -1: # We got 4096 bytes with no CRLF, or a CRLF with more data # after it. self._failPendingPackets() self.startShutdown() return # Find which protocol the server chose. self.protocol = None for p in self.PROTOCOL_VERSIONS: if s == "MMTP %s\r\n"%p: self.protocol = p break if not self.protocol: LOG.warn("Protocol negotiation failed with %s", self.address) self._failPendingPackets() self.startShutdown() return LOG.debug("MMTP protocol negotiated with %s: version %s", self.address, self.protocol) # Now that we're connected, optimize for throughput. mixminion.NetUtils.optimizeThroughput(self.sock) self.onRead = self.onDataRead self.onWrite = self.onDataWritten self.beginReading() self._isConnected = 1 # Now that we're connected, start sending packets. self._updateRWState()
def buildReplyPacket(payload, path1, replyBlock, paddingPRNG=None): """Build a message using a reply block. 'path1' is a sequence of ServerInfo for the nodes on the first leg of the path. 'payload' must be exactly 28K long. """ if paddingPRNG is None: paddingPRNG = Crypto.getCommonPRNG() LOG.debug("Encoding reply message for %s-byte payload", len(payload)) LOG.debug(" Using path %s/??",[s.getNickname() for s in path1]) assert len(payload) == PAYLOAD_LEN # Encrypt the payload so that it won't appear as plaintext to the # crossover note. (We use 'decrypt' so that the message recipient can # simply use 'encrypt' to reverse _all_ the steps of the reply path.) k = Crypto.Keyset(replyBlock.encryptionKey).getLionessKeys( Crypto.PAYLOAD_ENCRYPT_MODE) payload = Crypto.lioness_decrypt(payload, k) return _buildPacket(payload, None, None, path1=path1, path2=replyBlock)
def generateConsensusDirectory(identity, voters, validAfter, directories, validatedDigests=None): # directories is (source, stringable) list # First -- whom shall we vote with? goodDirectories = {} # {fingerprint: (src,SignedDirectory)} serverMap = {} # digest->server info serversByDir = {} # keyid->list of digest for src, val in directories: LOG.debug("Checking vote directory from %s",src) val = str(val) try: directory = mixminion.ServerInfo.SignedDirectory(string=val, validatedDigests=validatedDigests, _keepServerContents=1) except ConfigError,e: LOG.warn("Rejecting malformed vote directory from %s: %s",src,e) continue try: checkVoteDirectory(voters, validAfter, directory) except BadVote, e: LOG.warn("Rejecting vote directory from %s: %s", src, e) continue
def onDataRead(self): # We got some data from the server: it'll be 0 or more acks. if self.inbuflen < self.ACK_LEN: # If we have no acks at all, do nothing. return while self.inbuflen >= self.ACK_LEN: if not self.expectedAcks: LOG.warn("Received acknowledgment from %s with no corresponding message", self.address) self._failPendingPackets() self.startShutdown() return ack = self.getInbuf(self.ACK_LEN, clear=1) good, bad = self.expectedAcks.pop(0) if ack == good: LOG.debug("Packet delivered to %s",self.address) self.nPacketsAcked += 1 if not self.pendingPackets[0].isJunk(): EventStats.log.successfulRelay() self.pendingPackets[0].succeeded() del self.pendingPackets[0] elif ack == bad: LOG.warn("Packet rejected by %s", self.address) self.nPacketsAcked += 1 if not self.pendingPackets[0].isJunk(): EventStats.log.failedRelay() self.pendingPackets[0].failed(1) del self.pendingPackets[0] else: # The control string and digest are wrong for an accepted # or rejected packet! LOG.warn("Bad acknowledgement received from %s",self.address) self._failPendingPackets() self.startShutdown() return # Start sending more packets, if we were waiting for an ACK to do so. self._updateRWState()
def _deleteMessageIDs(self, messageIDSet, why, today=None): """Helper function. Remove all the fragments and chunks associated with a given message, and mark the message as delivered or undeliverable. messageIDSet -- a map from 20-byte messageID to 1. why -- 'REJECTED' or 'COMPLETED' or '?' """ assert why in ("REJECTED", "COMPLETED", "?") if not messageIDSet: return if today is None: today = time.time() today = previousMidnight(today) if why == 'REJECTED': LOG.debug("Removing bogus messages by IDs: %s", messageIDSet.keys()) elif why == "COMPLETED": LOG.debug("Removing completed messages by IDs: %s", messageIDSet.keys()) else: LOG.debug("Removing messages by IDs: %s", messageIDSet.keys()) for mid in messageIDSet.keys(): if why == "?": state = self.states[mid] if state.isDone: whythis = "COMPLETED" else: whythis = "REJECTED" else: whythis = why self.db.markStatus(mid, whythis, today) try: del self.states[mid] except KeyError: pass for h, fm in self.store._metadata_cache.items(): if messageIDSet.has_key(fm.messageid): self.store.removeMessage(h)
def shutdown(self): LOG.debug("Closing listener connection (fd %s)", self.sock.fileno()) self.isOpen = 0 self.sock.close() LOG.info("Server connection closed")
createPrivateDir(parent) # If the file exists, but can't be read, bail. try: st = os.stat(filename) except OSError, e: if e.errno != errno.ENOENT: raise st = None # If the file is empty, delete it and start over. if st and st[stat.ST_SIZE] == 0: LOG.warn("Half-created database %s found; cleaning up.", filename) tryUnlink(filename) dbtype = whichdb.whichdb(filename) LOG.debug("Opening %s database at %s", purpose, filename) try: if dbtype != 'dbhash': db = _openDBHash(filename, 'c', 0600) else: db = anydbm.open(filename, 'c', 0600) except anydbm.error, e: raise MixFatalError("Can't open %s database: %s"%(purpose,e)) except ImportError: raise MixFatalError("Unsupported type for %s database: %s" %(purpose, dbtype)) if hasattr(db, 'sync'): syncLog = db.sync elif hasattr(db, '_commit'): # Workaround for dumbdbm to allow syncing. (Standard in
class MMTPClientConnection(mixminion.TLSConnection.TLSConnection): """A nonblocking MMTP connection sending packets and padding to a single server.""" # Which MMTP versions do we understand? PROTOCOL_VERSIONS = ['0.3'] # If we've written WRITEAHEAD packets without receiving any acks, we wait # for an ack before sending any more. WRITEAHEAD = 6 # Length of a single transmission unit (control string, packet, checksum) MESSAGE_LEN = 6 + (1 << 15) + 20 # Length of a single acknowledgment (control string, digest) ACK_LEN = 10 + 20 ## Fields: # targetAddr, targetPort, targetKeyID: the address and keyid of the # server we're trying to connect to. # certCache: an instance of PeerCertificateCache to use to check the # peer server's certificate # packets: a list of DeliverableMessage objects that have not yet been # sent to the TLS connection, in the order they should be sent. # pendingPackets: a list of DeliverableMessage objects that have been # sent to the TLS connection, but which have not yet been acknowledged. # nPacketsTotal: total number of packets we've ever been asked to send. # nPacketsSent: total number of packets sent across the TLS connection # nPacketsAcked: total number of acks received from the TLS connection # expectedAcks: list of acceptAck,rejectAck tuples for the packets # that we've sent but haven't gotten acks for. # _isConnected: flag: true if the TLS connection been completed, # and no errors have been encountered. # _isFailed: flag: has this connection encountered any errors? # _isAlive: flag: if we put another packet on this connection, will the # packet maybe get delivered? #### # External interface #### def __init__(self, targetFamily, targetAddr, targetPort, targetKeyID, serverName=None, context=None, certCache=None): """Initialize a new MMTPClientConnection.""" assert targetFamily in (mixminion.NetUtils.AF_INET, mixminion.NetUtils.AF_INET6) if context is None: context = _ml.TLSContext_new() if serverName is None: serverName = mixminion.ServerInfo.displayServerByRouting( IPV4Info(targetAddr, targetPort, targetKeyID)) if certCache is None: certCache = PeerCertificateCache() self.targetAddr = targetAddr self.targetPort = targetPort sock = socket.socket(targetFamily, socket.SOCK_STREAM) serverName += " (fd %s)" % sock.fileno() sock.setblocking(0) try: sock.connect((targetAddr, targetPort)) except socket.error, e: # This will always raise an error, since we're nonblocking. That's # okay... but it had better be EINPROGRESS or the local equivalent. if e[0] not in mixminion.NetUtils.IN_PROGRESS_ERRNOS: raise e tls = context.sock(sock) mixminion.TLSConnection.TLSConnection.__init__(self, tls, sock, serverName) if targetKeyID != '\x00' * 20: self.targetKeyID = targetKeyID else: self.targetKeyID = None self.certCache = certCache self.packets = [] self.pendingPackets = [] self.expectedAcks = [] self.nPacketsSent = self.nPacketsAcked = self.nPacketsTotal = 0 self._isConnected = 0 self._isFailed = 0 self._isAlive = 1 EventStats.log.attemptedConnect() LOG.debug("Opening client connection to %s", self.address) self.beginConnecting()
keyset.getServerDescriptor() except (ConfigError, IOError), e: LOG.warn("Key set %s has invalid/missing descriptor: %s", keysetname, str(e)) ok = 0 if ok: t1, t2 = keyset.getLiveness() self.keySets.append((t1, t2, keyset)) LOG.trace("Found key %s (valid from %s to %s)", dirname, formatDate(t1), formatDate(t2)) else: badKeySets.append(keyset) LOG.debug("Found %s keysets: %s were incomplete or invalid.", len(self.keySets), len(badKeySets)) if badKeySets: LOG.warn("Removing %s invalid keysets", len(badKeySets)) for b in badKeySets: b.delete() # Now, sort the key intervals by starting time. self.keySets.sort() self.keyRange = (firstKey, lastKey) # Now we try to see whether we have more or less than 1 key in effect # for a given time. for idx in xrange(len(self.keySets) - 1): end = self.keySets[idx][1] start = self.keySets[idx + 1][0]
#### # Implementation: hooks #### def onConnected(self): LOG.debug("Completed MMTP client connection to %s",self.address) # Is the certificate correct? try: self.certCache.check(self.tls, self.targetKeyID, self.address) except MixProtocolBadAuth, e: LOG.warn("Certificate error: %s. Shutting down connection.", e) self._failPendingPackets() self.startShutdown() return else: LOG.debug("KeyID is valid from %s", self.address) EventStats.log.successfulConnect() # The certificate is fine; start protocol negotiation. self.beginWriting("MMTP %s\r\n" % ",".join(self.PROTOCOL_VERSIONS)) self.onWrite = self.onProtocolWritten def onProtocolWritten(self,n): if self.outbuf: # Not done writing outgoing data. return LOG.debug("Sent MMTP protocol string to %s", self.address) self.stopWriting() self.beginReading()
keyset.getServerDescriptor() except (ConfigError, IOError), e: LOG.warn("Key set %s has invalid/missing descriptor: %s", keysetname, str(e)) ok = 0 if ok: t1, t2 = keyset.getLiveness() self.keySets.append( (t1, t2, keyset) ) LOG.trace("Found key %s (valid from %s to %s)", dirname, formatDate(t1), formatDate(t2)) else: badKeySets.append(keyset) LOG.debug("Found %s keysets: %s were incomplete or invalid.", len(self.keySets), len(badKeySets)) if badKeySets: LOG.warn("Removing %s invalid keysets", len(badKeySets)) for b in badKeySets: b.delete() # Now, sort the key intervals by starting time. self.keySets.sort() self.keyRange = (firstKey, lastKey) # Now we try to see whether we have more or less than 1 key in effect # for a given time. for idx in xrange(len(self.keySets)-1): end = self.keySets[idx][1] start = self.keySets[idx+1][0]
createPrivateDir(parent) # If the file exists, but can't be read, bail. try: st = os.stat(filename) except OSError, e: if e.errno != errno.ENOENT: raise st = None # If the file is empty, delete it and start over. if st and st[stat.ST_SIZE] == 0: LOG.warn("Half-created database %s found; cleaning up.", filename) tryUnlink(filename) dbtype = whichdb.whichdb(filename) LOG.debug("Opening %s database at %s", purpose, filename) try: if dbtype != 'dbhash': db = _openDBHash(filename, 'c', 0600) else: db = anydbm.open(filename, 'c', 0600) except anydbm.error, e: raise MixFatalError("Can't open %s database: %s" % (purpose, e)) except ImportError: raise MixFatalError("Unsupported type for %s database: %s" % (purpose, dbtype)) if hasattr(db, 'sync'): syncLog = db.sync elif hasattr(db, '_commit'): # Workaround for dumbdbm to allow syncing. (Standard in
#### # Implementation: hooks #### def onConnected(self): LOG.debug("Completed MMTP client connection to %s", self.address) # Is the certificate correct? try: self.certCache.check(self.tls, self.targetKeyID, self.address) except MixProtocolBadAuth, e: LOG.warn("Certificate error: %s. Shutting down connection.", e) self._failPendingPackets() self.startShutdown() return else: LOG.debug("KeyID is valid from %s", self.address) EventStats.log.successfulConnect() # The certificate is fine; start protocol negotiation. self.beginWriting("MMTP %s\r\n" % ",".join(self.PROTOCOL_VERSIONS)) self.onWrite = self.onProtocolWritten def onProtocolWritten(self, n): if self.outbuf: # Not done writing outgoing data. return LOG.debug("Sent MMTP protocol string to %s", self.address) self.stopWriting() self.beginReading()