def prevalidate(self, contents): for name, ents in contents: if name == 'Server': for k, v, _ in ents: if k == 'Descriptor-Version' and v.strip() != '0.2': raise ConfigError( "Unrecognized descriptor version: %s" % v.strip()) # Remove any sections with unrecognized versions. revisedContents = [] for name, ents in contents: v = self.expected_versions.get(name) if not v: revisedContents.append((name, ents)) continue versionkey, versionval = v for k, v, _ in ents: if k == versionkey and v.strip() != versionval: LOG.warn( "Skipping %s section with unrecognized version %s", name, v.strip()) break else: revisedContents.append((name, ents)) return revisedContents
def processMessage(self, packet): assert packet.getExitType() == 0xFFFE exitInfo = packet.getAddress() if exitInfo == 'fail': return DELIVER_FAIL_RETRY elif exitInfo == 'FAIL!': return DELIVER_FAIL_NORETRY LOG.debug("Delivering test message") m = _escapeMessageForEmail(packet) if m is None: # Ordinarily, we'd drop corrupt messages, but this module is # meant for debugging. m = """\ ==========CORRUPT OR UNDECODABLE MESSAGE Decoding handle: %s%s==========MESSAGE ENDS""" % ( base64.encodestring(packet.getTag()), base64.encodestring(packet.getContents())) f = open(os.path.join(self.loc, str(self.next)), 'w') self.next += 1 f.write(m) f.close() return DELIVER_OK
def regenerateDescriptors(self): """Regenerate all server descriptors for all keysets in this keyring, but keep all old keys intact.""" LOG.info("Regenerating server descriptors; keeping old keys.") identityKey = self.getIdentityKey() for _,_,ks in self.keySets: ks.regenerateServerDescriptor(self.config, identityKey)
def checkDescriptorConsistency(self, regen=1): """Check whether the server descriptors in this keyring are consistent with the server's configuration. If 'regen' is true, inconsistent descriptors are regenerated.""" identity = None state = [] for _,_,ks in self.keySets: ok = ks.checkConsistency(self.config, 0) if ok == 'good': continue state.append((ok, ks)) if not state: return LOG.warn("Some generated keysets do not match " "current configuration...") for ok, ks in state: va,vu = ks.getLiveness() LOG.warn("Keyset %s (%s--%s):",ks.keyname,formatTime(va,1), formatTime(vu,1)) ks.checkConsistency(self.config, 1) if regen and ok == 'bad': if not identity: identity = self.getIdentityKey() ks.regenerateServerDescriptor(self.config, identity)
def getSigners(self): #DOCDOC -- returns members of self.dirInfo.voters with valid signatures. if self.signers is not None: return self.signers sigs = {} self.signers = [] for s in self.signatures: sigs[s.getKeyFingerprint()] = s for digest, url in self.dirInfo.voters: try: s = sigs[digest] except KeyError: #XXXX008 log something. continue if s.checkSignature(): LOG.trace("Found valid signature from %s at %s", digest, url) self.signers.append((digest, url)) else: LOG.trace("Signature claiming to be from %s was not valid", digest) continue return self.signers
def processMessage(self, packet): assert packet.getExitType() == 0xFFFE exitInfo = packet.getAddress() if exitInfo == 'fail': return DELIVER_FAIL_RETRY elif exitInfo == 'FAIL!': return DELIVER_FAIL_NORETRY LOG.debug("Delivering test message") m = _escapeMessageForEmail(packet) if m is None: # Ordinarily, we'd drop corrupt messages, but this module is # meant for debugging. m = """\ ==========CORRUPT OR UNDECODABLE MESSAGE Decoding handle: %s%s==========MESSAGE ENDS""" % (base64.encodestring( packet.getTag()), base64.encodestring(packet.getContents())) f = open(os.path.join(self.loc, str(self.next)), 'w') self.next += 1 f.write(m) f.close() return DELIVER_OK
def lookup(self,name,cb): """Look up the name 'name', and pass the result to the callback function 'cb' when we're done. The result will be of the same form as the return value of NetUtils.getIP: either (Family, Address, Time) or ('NOENT', Reason, Time). Note: The callback may be invoked from a different thread. Either this thread or a DNS thread will block until the callback finishes, so it shouldn't be especially time-consuming. """ # Check for a static IP first; no need to resolve that. v = mixminion.NetUtils.nameIsStaticIP(name) if v is not None: cb(name,v) return try: self.lock.acquire() v = self.cache.get(name) # If we don't have a cached answer, add cb to self.callbacks if v is None or v is PENDING: self.callbacks.setdefault(name, []).append(cb) # If we aren't looking up the answer, start looking it up. if v is None: LOG.trace("DNS cache starting lookup of %r", name) self._beginLookup(name) finally: self.lock.release() # If we _did_ have an answer, invoke the callback now. if v is not None and v is not PENDING: LOG.trace("DNS cache returning cached value %s for %r", v,name) cb(name,v)
def addChunk(self, h, fm): """Register a chunk with handle h and FragmentMetadata fm. If the chunk is inconsistent with other fragments of this message, raise MismatchedFragment.""" assert fm.isChunk assert fm.messageid == self.messageid if fm.size != self.params.length: raise MismatchedFragment("Mismatched message length") if fm.overhead != self.overhead: raise MismatchedFragment("Mismatched packet overhead") if self.chunks.has_key(fm.chunkNum): raise MismatchedFragment("Duplicate chunks") if fm.nym != self.nym: raise MismatchedFragment("Fragments received for differing identities") if self.inserted > fm.insertedDate: self.inserted = fm.insertedDate self.chunks[fm.chunkNum] = (h,fm) if self.fragmentsByChunk[fm.chunkNum]: LOG.warn("Found a chunk with unneeded fragments for message %r", self.messageid) if self.readyChunks.get(fm.chunkNum): del self.readyChunks[fm.chunkNum]
def addChunk(self, h, fm): """Register a chunk with handle h and FragmentMetadata fm. If the chunk is inconsistent with other fragments of this message, raise MismatchedFragment.""" assert fm.isChunk assert fm.messageid == self.messageid if fm.size != self.params.length: raise MismatchedFragment("Mismatched message length") if fm.overhead != self.overhead: raise MismatchedFragment("Mismatched packet overhead") if self.chunks.has_key(fm.chunkNum): raise MismatchedFragment("Duplicate chunks") if fm.nym != self.nym: raise MismatchedFragment( "Fragments received for differing identities") if self.inserted > fm.insertedDate: self.inserted = fm.insertedDate self.chunks[fm.chunkNum] = (h, fm) if self.fragmentsByChunk[fm.chunkNum]: LOG.warn("Found a chunk with unneeded fragments for message %r", self.messageid) if self.readyChunks.get(fm.chunkNum): del self.readyChunks[fm.chunkNum]
def getInbufLine(self, maxBytes=None, terminator="\r\n", clear=0, allowExtra=0): """Return the first prefix of the current inbuf that ends with the 'terminator' string. Returns the string on success, None if no such string is found, and -1 on error. Errors occur when: there are 'maxBytes' bytes available but the terminator is not found; or when 'allowExtra' is false and there is data on the input buffer following the terminator.""" s = self.getInbuf(maxBytes) idx = s.find(terminator) if idx < 0: if len(s) == maxBytes: LOG.warn("Too much data without EOL from %s", self.address) return -1 else: return None if not allowExtra and idx + len(terminator) < self.inbuflen: LOG.warn("Trailing data after EOL from %s", self.address) return -1 return self.getInbuf(idx + len(terminator), clear=clear)
def sendPackets(routing, packetList, timeout=300, callback=None): """Sends a list of packets to a server. Raise MixProtocolError on failure. routing -- an instance of mixminion.Packet.IPV4Info or mixminion.Packet.MMTPHostInfo. If routing.keyinfo == '\000'*20, we ignore the server's keyid. packetList -- a list of 32KB packets and control strings. Control strings must be one of "JUNK" to send a 32KB padding chunk, or "RENEGOTIATE" to renegotiate the connection key. connectTimeout -- None, or a number of seconds to wait for data on the connection before raising TimeoutError. callback -- None, or a function to call with a index into packetList after each successful packet delivery. """ # Find out where we're connecting to. serverName = mixminion.ServerInfo.displayServerByRouting(routing) if isinstance(routing, IPV4Info): family, addr = socket.AF_INET, routing.ip else: assert isinstance(routing, MMTPHostInfo) LOG.trace("Looking up %s...",routing.hostname) family, addr, _ = mixminion.NetUtils.getIP(routing.hostname) if family == "NOENT": raise MixProtocolError("Couldn't resolve hostname %s: %s" % ( routing.hostname, addr)) # Create an MMTPClientConnection try: con = MMTPClientConnection( family, addr, routing.port, routing.keyinfo, serverName=serverName) except socket.error, e: raise MixProtocolError(str(e))
def checkDescriptorConsistency(self, regen=1): """Check whether the server descriptors in this keyring are consistent with the server's configuration. If 'regen' is true, inconsistent descriptors are regenerated.""" identity = None state = [] for _, _, ks in self.keySets: ok = ks.checkConsistency(self.config, 0) if ok == 'good': continue state.append((ok, ks)) if not state: return LOG.warn("Some generated keysets do not match " "current configuration...") for ok, ks in state: va, vu = ks.getLiveness() LOG.warn("Keyset %s (%s--%s):", ks.keyname, formatTime(va, 1), formatTime(vu, 1)) ks.checkConsistency(self.config, 1) if regen and ok == 'bad': if not identity: identity = self.getIdentityKey() ks.regenerateServerDescriptor(self.config, identity)
def regenerateDescriptors(self): """Regenerate all server descriptors for all keysets in this keyring, but keep all old keys intact.""" LOG.info("Regenerating server descriptors; keeping old keys.") identityKey = self.getIdentityKey() for _, _, ks in self.keySets: ks.regenerateServerDescriptor(self.config, identityKey)
def _validateZlib(): """Internal function: Make sure that zlib is a recognized version, and that it compresses things as expected. (This check is important, because using a zlib version that compressed differently from zlib1.1.4 would make senders partitionable by payload compression.) """ global _ZLIB_LIBRARY_OK ver = getattr(zlib, "ZLIB_VERSION", None) if ver and ver < "1.1.2": raise MixFatalError("Zlib version %s is not supported"%ver) _ZLIB_LIBRARY_OK = 0.5 if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2", "1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7", "1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2", "1.2.3", "1.2.7", "1.2.8"): _ZLIB_LIBRARY_OK = 1 return LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver) # This test is inadequate, but it _might_ catch future incompatible # changes. _ZLIB_LIBRARY_OK = 0.5 good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\ '`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\ '\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\ 'q\x8d\n\x93' if compressData("aZbAAcdefg"*1000) == good: _ZLIB_LIBRARY_OK = 1 else: _ZLIB_LIBRARY_OK = 0 raise MixFatalError("Zlib output not as exected.")
class ServerInbox: """A ServerInbox holds server descriptors received from the outside world that are not yet ready to be included in the directory. """ ## Fields: # newQueue: IncomingQueue object to hold descriptors for previously # unknown servers. # updateQueue: IncomingQueue object to hold descriptors for currently # known servers. def __init__(self, base, idCache): """Initialize a ServerInbox to store its files in 'base', and check server descriptors against the IDCache 'idCache'.""" self.newQueue = IncomingQueue(os.path.join(base, "new"), os.path.join(base, "reject")) self.updateQueue = IncomingQueue(os.path.join(base, "updates"), os.path.join(base, "reject")) self.idCache = idCache def receiveServer(self, text, source): """Process a new server descriptor and store it for later action. (To be run by the CGI user.) If the server will be automatically inserted, return true. If the server will be inserted (given administrator intervention), raise ServerQueuedException. If there is a problem, log it, and raise UIError. text -- a string containing a new server descriptor. source -- a (human readable) string describing the source of the descriptor, used in error messages. """ try: server = ServerInfo(string=text,assumeValid=0) except MixError, e: LOG.warn("Rejected invalid server from %s: %s", source,e) raise UIError("Server descriptor was not valid: %s"%e) nickname = server.getNickname() try: known = self.idCache.containsServer(server) except MismatchedID: LOG.warn("Rejected server with mismatched identity from %s", source) self.updateQueue.queueRejectedServer(text,server) raise UIError(("I already know a server named " "%s with a different key.")%nickname) if not known: LOG.info("Received previously unknown server %s from %s", nickname, source) self.newQueue.queueIncomingServer(text,server) raise ServerQueuedException( "Server queued pending manual checking") else: LOG.info("Received update for server %s from %s", nickname, source) self.updateQueue.queueIncomingServer(text,server) return 1
def readProtocol(self): s = self.getInbufLine(4096,clear=1) if s is None: return elif s == -1: self.startShutdown() #failed return self.stopReading() m = PROTOCOL_RE.match(s) if not m: LOG.warn("Bad MMTP protocol string format from %s", self.address) #failed self.startShutdown() return protocols = m.group(1).split(",") for p in self.PROTOCOL_VERSIONS: if p in protocols: self.protocol = p self.onWrite = self.protocolWritten self.beginWriting("MMTP %s\r\n"%p) return LOG.warn("No common protocols with %s", self.address) #failed self.startShutdown()
def prevalidate(self, contents): for name, ents in contents: if name == 'Server': for k,v,_ in ents: if k == 'Descriptor-Version' and v.strip() != '0.2': raise ConfigError("Unrecognized descriptor version: %s" % v.strip()) # Remove any sections with unrecognized versions. revisedContents = [] for name, ents in contents: v = self.expected_versions.get(name) if not v: revisedContents.append((name, ents)) continue versionkey, versionval = v for k,v,_ in ents: if k == versionkey and v.strip() != versionval: LOG.warn("Skipping %s section with unrecognized version %s" , name, v.strip()) break else: revisedContents.append((name, ents)) return revisedContents
def readProtocol(self): s = self.getInbufLine(4096, clear=1) if s is None: return elif s == -1: self.startShutdown() #failed return self.stopReading() m = PROTOCOL_RE.match(s) if not m: LOG.warn("Bad MMTP protocol string format from %s", self.address) #failed self.startShutdown() return protocols = m.group(1).split(",") for p in self.PROTOCOL_VERSIONS: if p in protocols: self.protocol = p self.onWrite = self.protocolWritten self.beginWriting("MMTP %s\r\n" % p) return LOG.warn("No common protocols with %s", self.address) #failed self.startShutdown()
def _validateZlib(): """Internal function: Make sure that zlib is a recognized version, and that it compresses things as expected. (This check is important, because using a zlib version that compressed differently from zlib1.1.4 would make senders partitionable by payload compression.) """ global _ZLIB_LIBRARY_OK ver = getattr(zlib, "ZLIB_VERSION", None) if ver and ver < "1.1.2": raise MixFatalError("Zlib version %s is not supported" % ver) _ZLIB_LIBRARY_OK = 0.5 if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2", "1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7", "1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2", "1.2.3"): _ZLIB_LIBRARY_OK = 1 return LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver) # This test is inadequate, but it _might_ catch future incompatible # changes. _ZLIB_LIBRARY_OK = 0.5 good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\ '`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\ '\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\ 'q\x8d\n\x93' if compressData("aZbAAcdefg" * 1000) == good: _ZLIB_LIBRARY_OK = 1 else: _ZLIB_LIBRARY_OK = 0 raise MixFatalError("Zlib output not as exected.")
def __readTooMuch(self): """Helper function -- called if we read too much data while we're shutting down.""" LOG.error("Read over 128 bytes of unexpected data from closing " "connection to %s", self.address) self.onTLSError() raise _Closing()
def run(self): """Thread body: pull questions from the DNS thread queue and answer them.""" queue = self.dnscache.queue _lookupDone = self.dnscache._lookupDone _adjBusyThreads = self.dnscache._adjBusyThreads _adjLiveThreads = self.dnscache._adjLiveThreads try: _adjLiveThreads(1) try: while 1: # Get a question from the queue, but don't wait more than # MAX_THREAD_IDLE seconds hostname = queue.get(timeout=MAX_THREAD_IDLE) # If the question is None, shutdown. if hostname is None: return # Else, resolve the IP and send the answer to the dnscache _adjBusyThreads(1) result = mixminion.NetUtils.getIP(hostname) _lookupDone(hostname, result) _adjBusyThreads(-1) except QueueEmpty: LOG.debug("DNS thread shutting down: idle for %s seconds.", MAX_THREAD_IDLE) except: LOG.error_exc(sys.exc_info(), "Exception in DNS thread; shutting down.") finally: _adjLiveThreads(-1)
def getHandlesByDestAndAge(self, destList, directory, notAfter=None, warnUnused=1): """Return a list of handles for all messages queued for servers in a given list before a given date. destList -- A list of hostnames, ips, keyids, or nicknames for servers whose messages should be included in the result. directory -- An instance of ClientDirectory used to resolve nicknames. This may be None if no nicknames are included. notAfter -- If provided, a time such that no messages queued later should be included warnUnused -- If true, we log a message for every element in destList that has no matching messages in the queue. """ destSet = {} reverse = {} for d in destList: if directory: keyid = directory.getKeyIDByNickname(d) if keyid: destSet[keyid] = 1 reverse[keyid] = d continue destSet[d] = 1 self.loadMetadata() result = [] foundAny = {} foundMatch = {} for h in self.store.getAllMessages(): _, r, when = self.store.getMetadata(h) if (destSet.has_key(r.keyinfo) or (hasattr(r, 'hostname') and destSet.has_key(r.hostname)) or (hasattr(r, 'ip') and destSet.has_key(r.ip))): keys = [ getattr(r, 'hostname', None), getattr(r, 'ip', None), reverse.get(r.keyinfo, None), r.keyinfo ] for k in keys: foundAny[k] = 1 if notAfter and when > notAfter: continue for k in keys: foundMatch[k] = 1 result.append(h) if warnUnused: for d in destList: if foundMatch.get(d): continue elif foundAny.get(d): LOG.warn("No expired packets found for %r", d) else: LOG.warn("No pending packets found for %r", d) return result
def removeDeadKeys(self, now=None): """Remove all keys that have expired.""" self.checkKeys() keys = self.getDeadKeys(now) for message, keyset in keys: LOG.info(message) keyset.delete() self.checkKeys()
def process(self, r, w, x, cap): #XXXX007 do something with x try: con, addr = self.sock.accept() LOG.debug("Accepted connection from %s", addr) self.connectionFactory(con) except socket.error, e: LOG.warn("Socket error while accepting connection: %s", e)
def __call__(self, *args): self.called = 1 self.errors += 1 if not self.published: args = list(args) args[0] = args[0].replace("published", "in unpublished descriptor") if not self.silence: LOG.warn(*args)
def getHeaders(self): """Return a dict containing the headers for this message.""" if self.type is None: self.decode() if self.headers is None: LOG.warn("getHeaders found no decoded headers") return {} return self.headers
def __clientFinished(self, addr): """Called when a client connection runs out of packets to send, or halts.""" try: del self.clientConByAddr[addr] except KeyError: LOG.warn("Didn't find client connection to %s in address map", addr)
def __readTooMuch(self): """Helper function -- called if we read too much data while we're shutting down.""" LOG.error( "Read over 128 bytes of unexpected data from closing " "connection to %s", self.address) self.onTLSError() raise _Closing()
def deliveryFailed(self, handle, retriable=0, now=None): """Removes a message from the outgoing queue, or requeues it for delivery at a later time. This method should be invoked after the corresponding message has been unsuccessfully delivered.""" assert self.retrySchedule is not None LOG.trace("DeliveryQueue failed to deliver %s from %s", handle, self.qname) try: self._lock.acquire() try: ds = self.store.getMetadata(handle) except KeyError: ds = None except CorruptedFile: return if ds is None: # This should never happen LOG.error_exc(sys.exc_info(), "Handle %s had no state", handle) ds = _DeliveryState(now) ds.setNextAttempt(self.retrySchedule, now) self.store.setMetadata(handle, ds) return if not ds.isPending(): LOG.error("Handle %s was not pending", handle) return last = ds.pending ds.setNonPending() if retriable: # If we can retry the message, update the deliveryState # with the most recent attempt, and see if there's another # attempt in the future. ds.setLastAttempt(last) ds.setNextAttempt(self.retrySchedule, now) if ds.nextAttempt is not None: # There is another scheduled delivery attempt. Remember # it, mark the message sendable again, and save our state. LOG.trace(" (We'll try %s again at %s)", handle, formatTime(ds.nextAttempt, 1)) self.store.setMetadata(handle, ds) return else: assert ds.isRemovable() # Otherwise, fallthrough. # If we reach this point, the message is undeliverable, either # because 'retriable' is false, or because we've run out of # retries. LOG.trace(" (Giving up on %s)", handle) self.removeMessage(handle) finally: self._lock.release()
def configure_trng(config): """Initialize the true entropy source from a given Config object. If none is provided, tries some sane defaults.""" global _TRNG_FILENAME global _theTrueRNG if sys.platform == 'win32': # We have two entropy sources on windows: openssl's built-in # entropy generator that takes data from the screen, and # Windows's CryptGenRandom function. Because the former is # insecure, and the latter is closed-source, we xor them. _ml.win32_openssl_seed() _ml.openssl_seed(_ml.win32_get_random_bytes(32)) _theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG()) return if config is not None: requestedFile = config['Host'].get('EntropySource') else: requestedFile = None # Build a list of candidates defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform, PLATFORM_TRNG_DEFAULTS['***']) files = [ requestedFile ] + defaults # Now find the first of our candidates that exists and is a character # device. randFile = None for filename in files: if filename is None: continue verbose = (filename == requestedFile) if not os.path.exists(filename): if verbose: LOG.warn("No such file as %s", filename) else: st = os.stat(filename) if not (st[stat.ST_MODE] & stat.S_IFCHR): if verbose: LOG.error("Entropy source %s isn't a character device", filename) else: randFile = filename break if randFile is None and _TRNG_FILENAME is None: LOG.fatal("No entropy source available: Tried all of %s", files) raise MixFatalError("No entropy source available") elif randFile is None: LOG.warn("Falling back to previous entropy source %s", _TRNG_FILENAME) else: LOG.info("Setting entropy source to %r", randFile) _TRNG_FILENAME = randFile _theTrueRNG = _TrueRNG(1024)
def getBaseDir(self): """Return the base directory for this configuration.""" v = self["Server"]["BaseDir"] if v is None: v = self["Server"]["Homedir"] if v is None: LOG.warn("Defaulting base directory to /var/spool/minion; this will change.") v = "/var/spool/minion" return v
def deliverySucceeded(self, handle, now=None): """Removes a message from the outgoing queue. This method should be invoked after the corresponding message has been successfully delivered. """ assert self.retrySchedule is not None LOG.trace("DeliveryQueue got successful delivery for %s from %s", handle, self.qname) self.removeMessage(handle)
def encodeMessage(message, overhead, uncompressedFragmentPrefix="", paddingPRNG=None): """Given a message, compress it, fragment it into individual payloads, and add extra fields (size, hash, etc) as appropriate. Return a list of strings, each of which is a message payload suitable for use in build*Message. message: the initial message overhead: number of bytes to omit from each payload, given the type ofthe message encoding. (0 or ENC_FWD_OVERHEAD) uncompressedFragmentPrefix: If we fragment the message, we add this string to the message after compression but before whitening and fragmentation. paddingPRNG: generator for padding. Note: If multiple strings are returned, be sure to shuffle them before transmitting them to the network. """ assert overhead in (0, ENC_FWD_OVERHEAD) if paddingPRNG is None: paddingPRNG = Crypto.getCommonPRNG() origLength = len(message) payload = compressData(message) length = len(payload) if length > 1024 and length*20 <= origLength: LOG.warn("Message is very compressible and will look like a zlib bomb") paddingLen = PAYLOAD_LEN - SINGLETON_PAYLOAD_OVERHEAD - overhead - length # If the compressed payload fits in 28K, we're set. if paddingLen >= 0: # We pad the payload, and construct a new SingletonPayload, # including this payload's size and checksum. payload += paddingPRNG.getBytes(paddingLen) p = SingletonPayload(length, None, payload) p.computeHash() return [ p.pack() ] # Okay, we need to fragment the message. First, add the prefix if needed. if uncompressedFragmentPrefix: payload = uncompressedFragmentPrefix+payload # Now generate a message ID messageid = Crypto.getCommonPRNG().getBytes(FRAGMENT_MESSAGEID_LEN) # Figure out how many chunks to divide it into... p = mixminion.Fragments.FragmentationParams(len(payload), overhead) # ... fragment the payload into chunks... rawFragments = p.getFragments(payload) fragments = [] # ... and annotate each chunk with appropriate payload header info. for i in xrange(len(rawFragments)): pyld = FragmentPayload(i, None, messageid, p.length, rawFragments[i]) pyld.computeHash() fragments.append(pyld.pack()) rawFragments[i] = None return fragments
def onProtocolWritten(self,n): if self.outbuf: # Not done writing outgoing data. return LOG.debug("Sent MMTP protocol string to %s", self.address) self.stopWriting() self.beginReading() self.onRead = self.onProtocolRead
def onProtocolWritten(self, n): if self.outbuf: # Not done writing outgoing data. return LOG.debug("Sent MMTP protocol string to %s", self.address) self.stopWriting() self.beginReading() self.onRead = self.onProtocolRead
def configure_trng(config): """Initialize the true entropy source from a given Config object. If none is provided, tries some sane defaults.""" global _TRNG_FILENAME global _theTrueRNG if sys.platform == 'win32': # We have two entropy sources on windows: openssl's built-in # entropy generator that takes data from the screen, and # Windows's CryptGenRandom function. Because the former is # insecure, and the latter is closed-source, we xor them. _ml.win32_openssl_seed() _ml.openssl_seed(_ml.win32_get_random_bytes(32)) _theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG()) return if config is not None: requestedFile = config['Host'].get('EntropySource') else: requestedFile = None # Build a list of candidates defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform, PLATFORM_TRNG_DEFAULTS['***']) files = [requestedFile] + defaults # Now find the first of our candidates that exists and is a character # device. randFile = None for filename in files: if filename is None: continue verbose = (filename == requestedFile) if not os.path.exists(filename): if verbose: LOG.warn("No such file as %s", filename) else: st = os.stat(filename) if not (st[stat.ST_MODE] & stat.S_IFCHR): if verbose: LOG.error("Entropy source %s isn't a character device", filename) else: randFile = filename break if randFile is None and _TRNG_FILENAME is None: LOG.fatal("No entropy source available: Tried all of %s", files) raise MixFatalError("No entropy source available") elif randFile is None: LOG.warn("Falling back to previous entropy source %s", _TRNG_FILENAME) else: LOG.info("Setting entropy source to %r", randFile) _TRNG_FILENAME = randFile _theTrueRNG = _TrueRNG(1024)
def onConnected(self): LOG.debug("Completed MMTP client connection to %s",self.address) # Is the certificate correct? try: self.certCache.check(self.tls, self.targetKeyID, self.address) except MixProtocolBadAuth, e: LOG.warn("Certificate error: %s. Shutting down connection.", e) self._failPendingPackets() self.startShutdown() return
def __loadModules(self, section, sectionEntries): """Callback from the [Server] section of a config file. Parses the module options, and adds new sections to the syntax accordingly.""" self.moduleManager.setPath(section.get('ModulePath')) for mod in section.get('Module', []): LOG.info("Loading module %s", mod) self.moduleManager.loadExtModule(mod) self._syntax.update(self.moduleManager.getConfigSyntax())
def onConnected(self): LOG.debug("Completed MMTP client connection to %s", self.address) # Is the certificate correct? try: self.certCache.check(self.tls, self.targetKeyID, self.address) except MixProtocolBadAuth, e: LOG.warn("Certificate error: %s. Shutting down connection.", e) self._failPendingPackets() self.startShutdown() return
def rebuildIDCache(self): for fn in os.listdir(self.serverIDDir): fname = os.path.join(self.serverIDDir, fn) tp, val = readPickled(fname) if tp != "V0": LOG.warn("Weird file version %s on %s", tp, fname) continue nickname, ident = val ID = mixminion.Crypto.sha1(ident) self.idCache.insertID(nickname, ID)
def rebuildIDCache(self): for fn in os.listdir(self.serverIDDir): fname = os.path.join(self.serverIDDir, fn) tp,val = readPickled(fname) if tp != "V0": LOG.warn("Weird file version %s on %s",tp,fname) continue nickname, ident = val ID = mixminion.Crypto.sha1(ident) self.idCache.insertID(nickname, ID)
def tryTimeout(self, cutoff): """Close self.sock if the last activity on this connection was before 'cutoff'. Returns true iff the connection is timed out. """ if self.lastActivity <= cutoff: LOG.warn("Connection to %s timed out: %.2f seconds without activity", self.address, time.time()-self.lastActivity) self.onTimeout() self.__close() return 1 return 0
def getBaseDir(self): """Return the base directory for this configuration.""" v = self["Server"]["BaseDir"] if v is None: v = self["Server"]["Homedir"] if v is None: LOG.warn( "Defaulting base directory to /var/spool/minion; this will change." ) v = "/var/spool/minion" return v
def main(cmd, args): """[Entry point] Multiplex among subcommands.""" if len(args)<1 or ('-h', '--help') in args: usageAndExit() command = args[0] args = args[1:] if not SUBCOMMANDS.has_key(command): print "Unknown command", command usageAndExit() init_crypto() LOG.setMinSeverity("INFO") SUBCOMMANDS[command](args)
def main(cmd, args): """[Entry point] Multiplex among subcommands.""" if len(args) < 1 or ('-h', '--help') in args: usageAndExit() command = args[0] args = args[1:] if not SUBCOMMANDS.has_key(command): print "Unknown command", command usageAndExit() init_crypto() LOG.setMinSeverity("INFO") SUBCOMMANDS[command](args)
def getHandlesByDestAndAge(self, destList, directory, notAfter=None, warnUnused=1): """Return a list of handles for all messages queued for servers in a given list before a given date. destList -- A list of hostnames, ips, keyids, or nicknames for servers whose messages should be included in the result. directory -- An instance of ClientDirectory used to resolve nicknames. This may be None if no nicknames are included. notAfter -- If provided, a time such that no messages queued later should be included warnUnused -- If true, we log a message for every element in destList that has no matching messages in the queue. """ destSet = {} reverse = {} for d in destList: if directory: keyid = directory.getKeyIDByNickname(d) if keyid: destSet[keyid] = 1 reverse[keyid] = d continue destSet[d] = 1 self.loadMetadata() result = [] foundAny = {} foundMatch = {} for h in self.store.getAllMessages(): _, r, when = self.store.getMetadata(h) if (destSet.has_key(r.keyinfo) or (hasattr(r, 'hostname') and destSet.has_key(r.hostname)) or (hasattr(r, 'ip') and destSet.has_key(r.ip))): keys = [ getattr(r, 'hostname', None), getattr(r, 'ip', None), reverse.get(r.keyinfo, None), r.keyinfo ] for k in keys: foundAny[k]=1 if notAfter and when > notAfter: continue for k in keys: foundMatch[k]=1 result.append(h) if warnUnused: for d in destList: if foundMatch.get(d): continue elif foundAny.get(d): LOG.warn("No expired packets found for %r", d) else: LOG.warn("No pending packets found for %r", d) return result
def tryTimeout(self, cutoff): """Close self.sock if the last activity on this connection was before 'cutoff'. Returns true iff the connection is timed out. """ if self.lastActivity <= cutoff: LOG.warn( "Connection to %s timed out: %.2f seconds without activity", self.address, time.time() - self.lastActivity) self.onTimeout() self.__close() return 1 return 0
def run(self): """Internal: main body of processing thread.""" try: while 1: job = self.mqueue.get() job() except ProcessingThread._Shutdown: LOG.info("Shutting down %s", self.threadName) return except: LOG.error_exc(sys.exc_info(), "Exception in %s; shutting down thread.", self.threadName)
def run(self): """Internal: main body of processing thread.""" try: while 1: job = self.mqueue.get() job() except ProcessingThread._Shutdown: LOG.info("Shutting down %s",self.threadName) return except: LOG.error_exc(sys.exc_info(), "Exception in %s; shutting down thread.", self.threadName)
def __close(self, gotClose=0): """helper: close the underlying socket without cleaning up the TLS connection.""" if gotClose: if self.__stateFn == self.__connectFn: LOG.warn("Couldn't connect to %s", self.address) else: LOG.warn("Unexpectedly closed connection to %s", self.address) self.onTLSError() self.sock.close() self.sock = None self.tls = None self.__stateFn = self.__closedFn self.onClosed()
def getPacket(self, handle): """Given a handle, return a 3-tuple of the corresponding 32K packet, {IPV4/Host}Info, and time of first queueing. (The time is rounded down to the closest midnight GMT.) May raise CorruptedFile.""" obj = self.store.getObject(handle) try: magic, packet, routing, when = obj except (ValueError, TypeError): magic = None if magic != "PACKET-0": LOG.error("Unrecognized packet format for %s", handle) return None return packet, routing, when