def prevalidate(self, contents):
for name, ents in contents:
if name == 'Server':
for k, v, _ in ents:
if k == 'Descriptor-Version' and v.strip() != '0.2':
raise ConfigError(
"Unrecognized descriptor version: %s" % v.strip())
# Remove any sections with unrecognized versions.
revisedContents = []
for name, ents in contents:
v = self.expected_versions.get(name)
if not v:
revisedContents.append((name, ents))
continue
versionkey, versionval = v
for k, v, _ in ents:
if k == versionkey and v.strip() != versionval:
LOG.warn(
"Skipping %s section with unrecognized version %s",
name, v.strip())
break
else:
revisedContents.append((name, ents))
return revisedContents
def processMessage(self, packet):
assert packet.getExitType() == 0xFFFE
exitInfo = packet.getAddress()
if exitInfo == 'fail':
return DELIVER_FAIL_RETRY
elif exitInfo == 'FAIL!':
return DELIVER_FAIL_NORETRY
LOG.debug("Delivering test message")
m = _escapeMessageForEmail(packet)
if m is None:
# Ordinarily, we'd drop corrupt messages, but this module is
# meant for debugging.
m = """\
==========CORRUPT OR UNDECODABLE MESSAGE
Decoding handle: %s%s==========MESSAGE ENDS""" % (
base64.encodestring(packet.getTag()),
base64.encodestring(packet.getContents()))
f = open(os.path.join(self.loc, str(self.next)), 'w')
self.next += 1
f.write(m)
f.close()
return DELIVER_OK
def regenerateDescriptors(self):
"""Regenerate all server descriptors for all keysets in this
keyring, but keep all old keys intact."""
LOG.info("Regenerating server descriptors; keeping old keys.")
identityKey = self.getIdentityKey()
for _,_,ks in self.keySets:
ks.regenerateServerDescriptor(self.config, identityKey)
def checkDescriptorConsistency(self, regen=1):
"""Check whether the server descriptors in this keyring are
consistent with the server's configuration. If 'regen' is
true, inconsistent descriptors are regenerated."""
identity = None
state = []
for _,_,ks in self.keySets:
ok = ks.checkConsistency(self.config, 0)
if ok == 'good':
continue
state.append((ok, ks))
if not state:
return
LOG.warn("Some generated keysets do not match "
"current configuration...")
for ok, ks in state:
va,vu = ks.getLiveness()
LOG.warn("Keyset %s (%s--%s):",ks.keyname,formatTime(va,1),
formatTime(vu,1))
ks.checkConsistency(self.config, 1)
if regen and ok == 'bad':
if not identity: identity = self.getIdentityKey()
ks.regenerateServerDescriptor(self.config, identity)
def getSigners(self):
#DOCDOC -- returns members of self.dirInfo.voters with valid signatures.
if self.signers is not None:
return self.signers
sigs = {}
self.signers = []
for s in self.signatures:
sigs[s.getKeyFingerprint()] = s
for digest, url in self.dirInfo.voters:
try:
s = sigs[digest]
except KeyError:
#XXXX008 log something.
continue
if s.checkSignature():
LOG.trace("Found valid signature from %s at %s",
digest, url)
self.signers.append((digest, url))
else:
LOG.trace("Signature claiming to be from %s was not valid",
digest)
continue
return self.signers
def processMessage(self, packet):
assert packet.getExitType() == 0xFFFE
exitInfo = packet.getAddress()
if exitInfo == 'fail':
return DELIVER_FAIL_RETRY
elif exitInfo == 'FAIL!':
return DELIVER_FAIL_NORETRY
LOG.debug("Delivering test message")
m = _escapeMessageForEmail(packet)
if m is None:
# Ordinarily, we'd drop corrupt messages, but this module is
# meant for debugging.
m = """\
==========CORRUPT OR UNDECODABLE MESSAGE
Decoding handle: %s%s==========MESSAGE ENDS""" % (base64.encodestring(
packet.getTag()), base64.encodestring(packet.getContents()))
f = open(os.path.join(self.loc, str(self.next)), 'w')
self.next += 1
f.write(m)
f.close()
return DELIVER_OK
def lookup(self,name,cb):
"""Look up the name 'name', and pass the result to the callback
function 'cb' when we're done. The result will be of the
same form as the return value of NetUtils.getIP: either
(Family, Address, Time) or ('NOENT', Reason, Time).
Note: The callback may be invoked from a different thread. Either
this thread or a DNS thread will block until the callback finishes,
so it shouldn't be especially time-consuming.
"""
# Check for a static IP first; no need to resolve that.
v = mixminion.NetUtils.nameIsStaticIP(name)
if v is not None:
cb(name,v)
return
try:
self.lock.acquire()
v = self.cache.get(name)
# If we don't have a cached answer, add cb to self.callbacks
if v is None or v is PENDING:
self.callbacks.setdefault(name, []).append(cb)
# If we aren't looking up the answer, start looking it up.
if v is None:
LOG.trace("DNS cache starting lookup of %r", name)
self._beginLookup(name)
finally:
self.lock.release()
# If we _did_ have an answer, invoke the callback now.
if v is not None and v is not PENDING:
LOG.trace("DNS cache returning cached value %s for %r",
v,name)
cb(name,v)
def addChunk(self, h, fm):
"""Register a chunk with handle h and FragmentMetadata fm. If the
chunk is inconsistent with other fragments of this message,
raise MismatchedFragment."""
assert fm.isChunk
assert fm.messageid == self.messageid
if fm.size != self.params.length:
raise MismatchedFragment("Mismatched message length")
if fm.overhead != self.overhead:
raise MismatchedFragment("Mismatched packet overhead")
if self.chunks.has_key(fm.chunkNum):
raise MismatchedFragment("Duplicate chunks")
if fm.nym != self.nym:
raise MismatchedFragment("Fragments received for differing identities")
if self.inserted > fm.insertedDate:
self.inserted = fm.insertedDate
self.chunks[fm.chunkNum] = (h,fm)
if self.fragmentsByChunk[fm.chunkNum]:
LOG.warn("Found a chunk with unneeded fragments for message %r",
self.messageid)
if self.readyChunks.get(fm.chunkNum):
del self.readyChunks[fm.chunkNum]
def addChunk(self, h, fm):
"""Register a chunk with handle h and FragmentMetadata fm. If the
chunk is inconsistent with other fragments of this message,
raise MismatchedFragment."""
assert fm.isChunk
assert fm.messageid == self.messageid
if fm.size != self.params.length:
raise MismatchedFragment("Mismatched message length")
if fm.overhead != self.overhead:
raise MismatchedFragment("Mismatched packet overhead")
if self.chunks.has_key(fm.chunkNum):
raise MismatchedFragment("Duplicate chunks")
if fm.nym != self.nym:
raise MismatchedFragment(
"Fragments received for differing identities")
if self.inserted > fm.insertedDate:
self.inserted = fm.insertedDate
self.chunks[fm.chunkNum] = (h, fm)
if self.fragmentsByChunk[fm.chunkNum]:
LOG.warn("Found a chunk with unneeded fragments for message %r",
self.messageid)
if self.readyChunks.get(fm.chunkNum):
del self.readyChunks[fm.chunkNum]
def getInbufLine(self,
maxBytes=None,
terminator="\r\n",
clear=0,
allowExtra=0):
"""Return the first prefix of the current inbuf that ends with the
'terminator' string.
Returns the string on success, None if no such string is
found, and -1 on error. Errors occur when: there are 'maxBytes'
bytes available but the terminator is not found; or when
'allowExtra' is false and there is data on the input buffer
following the terminator."""
s = self.getInbuf(maxBytes)
idx = s.find(terminator)
if idx < 0:
if len(s) == maxBytes:
LOG.warn("Too much data without EOL from %s", self.address)
return -1
else:
return None
if not allowExtra and idx + len(terminator) < self.inbuflen:
LOG.warn("Trailing data after EOL from %s", self.address)
return -1
return self.getInbuf(idx + len(terminator), clear=clear)
def sendPackets(routing, packetList, timeout=300, callback=None):
"""Sends a list of packets to a server. Raise MixProtocolError on
failure.
routing -- an instance of mixminion.Packet.IPV4Info or
mixminion.Packet.MMTPHostInfo.
If routing.keyinfo == '\000'*20, we ignore the server's
keyid.
packetList -- a list of 32KB packets and control strings. Control
strings must be one of "JUNK" to send a 32KB padding chunk,
or "RENEGOTIATE" to renegotiate the connection key.
connectTimeout -- None, or a number of seconds to wait for data
on the connection before raising TimeoutError.
callback -- None, or a function to call with a index into packetList
after each successful packet delivery.
"""
# Find out where we're connecting to.
serverName = mixminion.ServerInfo.displayServerByRouting(routing)
if isinstance(routing, IPV4Info):
family, addr = socket.AF_INET, routing.ip
else:
assert isinstance(routing, MMTPHostInfo)
LOG.trace("Looking up %s...",routing.hostname)
family, addr, _ = mixminion.NetUtils.getIP(routing.hostname)
if family == "NOENT":
raise MixProtocolError("Couldn't resolve hostname %s: %s" % (
routing.hostname, addr))
# Create an MMTPClientConnection
try:
con = MMTPClientConnection(
family, addr, routing.port, routing.keyinfo, serverName=serverName)
except socket.error, e:
raise MixProtocolError(str(e))
def checkDescriptorConsistency(self, regen=1):
"""Check whether the server descriptors in this keyring are
consistent with the server's configuration. If 'regen' is
true, inconsistent descriptors are regenerated."""
identity = None
state = []
for _, _, ks in self.keySets:
ok = ks.checkConsistency(self.config, 0)
if ok == 'good':
continue
state.append((ok, ks))
if not state:
return
LOG.warn("Some generated keysets do not match "
"current configuration...")
for ok, ks in state:
va, vu = ks.getLiveness()
LOG.warn("Keyset %s (%s--%s):", ks.keyname, formatTime(va, 1),
formatTime(vu, 1))
ks.checkConsistency(self.config, 1)
if regen and ok == 'bad':
if not identity: identity = self.getIdentityKey()
ks.regenerateServerDescriptor(self.config, identity)
def regenerateDescriptors(self):
"""Regenerate all server descriptors for all keysets in this
keyring, but keep all old keys intact."""
LOG.info("Regenerating server descriptors; keeping old keys.")
identityKey = self.getIdentityKey()
for _, _, ks in self.keySets:
ks.regenerateServerDescriptor(self.config, identityKey)
def _validateZlib():
"""Internal function: Make sure that zlib is a recognized version, and
that it compresses things as expected. (This check is important,
because using a zlib version that compressed differently from zlib1.1.4
would make senders partitionable by payload compression.)
"""
global _ZLIB_LIBRARY_OK
ver = getattr(zlib, "ZLIB_VERSION", None)
if ver and ver < "1.1.2":
raise MixFatalError("Zlib version %s is not supported"%ver)
_ZLIB_LIBRARY_OK = 0.5
if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2",
"1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7",
"1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2",
"1.2.3", "1.2.7", "1.2.8"):
_ZLIB_LIBRARY_OK = 1
return
LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver)
# This test is inadequate, but it _might_ catch future incompatible
# changes.
_ZLIB_LIBRARY_OK = 0.5
good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
'`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
'\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
'q\x8d\n\x93'
if compressData("aZbAAcdefg"*1000) == good:
_ZLIB_LIBRARY_OK = 1
else:
_ZLIB_LIBRARY_OK = 0
raise MixFatalError("Zlib output not as exected.")
class ServerInbox:
"""A ServerInbox holds server descriptors received from the outside
world that are not yet ready to be included in the directory.
"""
## Fields:
# newQueue: IncomingQueue object to hold descriptors for previously
# unknown servers.
# updateQueue: IncomingQueue object to hold descriptors for currently
# known servers.
def __init__(self, base, idCache):
"""Initialize a ServerInbox to store its files in 'base', and
check server descriptors against the IDCache 'idCache'."""
self.newQueue = IncomingQueue(os.path.join(base, "new"),
os.path.join(base, "reject"))
self.updateQueue = IncomingQueue(os.path.join(base, "updates"),
os.path.join(base, "reject"))
self.idCache = idCache
def receiveServer(self, text, source):
"""Process a new server descriptor and store it for later action.
(To be run by the CGI user.)
If the server will be automatically inserted, return true.
If the server will be inserted (given administrator intervention),
raise ServerQueuedException.
If there is a problem, log it, and raise UIError.
text -- a string containing a new server descriptor.
source -- a (human readable) string describing the source
of the descriptor, used in error messages.
"""
try:
server = ServerInfo(string=text,assumeValid=0)
except MixError, e:
LOG.warn("Rejected invalid server from %s: %s", source,e)
raise UIError("Server descriptor was not valid: %s"%e)
nickname = server.getNickname()
try:
known = self.idCache.containsServer(server)
except MismatchedID:
LOG.warn("Rejected server with mismatched identity from %s",
source)
self.updateQueue.queueRejectedServer(text,server)
raise UIError(("I already know a server named "
"%s with a different key.")%nickname)
if not known:
LOG.info("Received previously unknown server %s from %s",
nickname, source)
self.newQueue.queueIncomingServer(text,server)
raise ServerQueuedException(
"Server queued pending manual checking")
else:
LOG.info("Received update for server %s from %s",
nickname, source)
self.updateQueue.queueIncomingServer(text,server)
return 1
def readProtocol(self):
s = self.getInbufLine(4096,clear=1)
if s is None:
return
elif s == -1:
self.startShutdown()
#failed
return
self.stopReading()
m = PROTOCOL_RE.match(s)
if not m:
LOG.warn("Bad MMTP protocol string format from %s", self.address)
#failed
self.startShutdown()
return
protocols = m.group(1).split(",")
for p in self.PROTOCOL_VERSIONS:
if p in protocols:
self.protocol = p
self.onWrite = self.protocolWritten
self.beginWriting("MMTP %s\r\n"%p)
return
LOG.warn("No common protocols with %s", self.address)
#failed
self.startShutdown()
def prevalidate(self, contents):
for name, ents in contents:
if name == 'Server':
for k,v,_ in ents:
if k == 'Descriptor-Version' and v.strip() != '0.2':
raise ConfigError("Unrecognized descriptor version: %s"
% v.strip())
# Remove any sections with unrecognized versions.
revisedContents = []
for name, ents in contents:
v = self.expected_versions.get(name)
if not v:
revisedContents.append((name, ents))
continue
versionkey, versionval = v
for k,v,_ in ents:
if k == versionkey and v.strip() != versionval:
LOG.warn("Skipping %s section with unrecognized version %s"
, name, v.strip())
break
else:
revisedContents.append((name, ents))
return revisedContents
def readProtocol(self):
s = self.getInbufLine(4096, clear=1)
if s is None:
return
elif s == -1:
self.startShutdown()
#failed
return
self.stopReading()
m = PROTOCOL_RE.match(s)
if not m:
LOG.warn("Bad MMTP protocol string format from %s", self.address)
#failed
self.startShutdown()
return
protocols = m.group(1).split(",")
for p in self.PROTOCOL_VERSIONS:
if p in protocols:
self.protocol = p
self.onWrite = self.protocolWritten
self.beginWriting("MMTP %s\r\n" % p)
return
LOG.warn("No common protocols with %s", self.address)
#failed
self.startShutdown()
def _validateZlib():
"""Internal function: Make sure that zlib is a recognized version, and
that it compresses things as expected. (This check is important,
because using a zlib version that compressed differently from zlib1.1.4
would make senders partitionable by payload compression.)
"""
global _ZLIB_LIBRARY_OK
ver = getattr(zlib, "ZLIB_VERSION", None)
if ver and ver < "1.1.2":
raise MixFatalError("Zlib version %s is not supported" % ver)
_ZLIB_LIBRARY_OK = 0.5
if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2",
"1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7",
"1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2",
"1.2.3"):
_ZLIB_LIBRARY_OK = 1
return
LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver)
# This test is inadequate, but it _might_ catch future incompatible
# changes.
_ZLIB_LIBRARY_OK = 0.5
good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
'`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
'\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
'q\x8d\n\x93'
if compressData("aZbAAcdefg" * 1000) == good:
_ZLIB_LIBRARY_OK = 1
else:
_ZLIB_LIBRARY_OK = 0
raise MixFatalError("Zlib output not as exected.")
def __readTooMuch(self):
"""Helper function -- called if we read too much data while we're
shutting down."""
LOG.error("Read over 128 bytes of unexpected data from closing "
"connection to %s", self.address)
self.onTLSError()
raise _Closing()
def run(self):
"""Thread body: pull questions from the DNS thread queue and
answer them."""
queue = self.dnscache.queue
_lookupDone = self.dnscache._lookupDone
_adjBusyThreads = self.dnscache._adjBusyThreads
_adjLiveThreads = self.dnscache._adjLiveThreads
try:
_adjLiveThreads(1)
try:
while 1:
# Get a question from the queue, but don't wait more than
# MAX_THREAD_IDLE seconds
hostname = queue.get(timeout=MAX_THREAD_IDLE)
# If the question is None, shutdown.
if hostname is None:
return
# Else, resolve the IP and send the answer to the dnscache
_adjBusyThreads(1)
result = mixminion.NetUtils.getIP(hostname)
_lookupDone(hostname, result)
_adjBusyThreads(-1)
except QueueEmpty:
LOG.debug("DNS thread shutting down: idle for %s seconds.",
MAX_THREAD_IDLE)
except:
LOG.error_exc(sys.exc_info(),
"Exception in DNS thread; shutting down.")
finally:
_adjLiveThreads(-1)
def getHandlesByDestAndAge(self,
destList,
directory,
notAfter=None,
warnUnused=1):
"""Return a list of handles for all messages queued for servers in a
given list before a given date.
destList -- A list of hostnames, ips, keyids, or nicknames
for servers whose messages should be included in the result.
directory -- An instance of ClientDirectory used to resolve
nicknames. This may be None if no nicknames are included.
notAfter -- If provided, a time such that no messages queued
later should be included
warnUnused -- If true, we log a message for every element in
destList that has no matching messages in the queue.
"""
destSet = {}
reverse = {}
for d in destList:
if directory:
keyid = directory.getKeyIDByNickname(d)
if keyid:
destSet[keyid] = 1
reverse[keyid] = d
continue
destSet[d] = 1
self.loadMetadata()
result = []
foundAny = {}
foundMatch = {}
for h in self.store.getAllMessages():
_, r, when = self.store.getMetadata(h)
if (destSet.has_key(r.keyinfo)
or (hasattr(r, 'hostname') and destSet.has_key(r.hostname))
or (hasattr(r, 'ip') and destSet.has_key(r.ip))):
keys = [
getattr(r, 'hostname', None),
getattr(r, 'ip', None),
reverse.get(r.keyinfo, None), r.keyinfo
]
for k in keys:
foundAny[k] = 1
if notAfter and when > notAfter:
continue
for k in keys:
foundMatch[k] = 1
result.append(h)
if warnUnused:
for d in destList:
if foundMatch.get(d):
continue
elif foundAny.get(d):
LOG.warn("No expired packets found for %r", d)
else:
LOG.warn("No pending packets found for %r", d)
return result
def removeDeadKeys(self, now=None):
"""Remove all keys that have expired."""
self.checkKeys()
keys = self.getDeadKeys(now)
for message, keyset in keys:
LOG.info(message)
keyset.delete()
self.checkKeys()
def process(self, r, w, x, cap):
#XXXX007 do something with x
try:
con, addr = self.sock.accept()
LOG.debug("Accepted connection from %s", addr)
self.connectionFactory(con)
except socket.error, e:
LOG.warn("Socket error while accepting connection: %s", e)
def process(self, r, w, x, cap):
#XXXX007 do something with x
try:
con, addr = self.sock.accept()
LOG.debug("Accepted connection from %s", addr)
self.connectionFactory(con)
except socket.error, e:
LOG.warn("Socket error while accepting connection: %s", e)
def __call__(self, *args):
self.called = 1
self.errors += 1
if not self.published:
args = list(args)
args[0] = args[0].replace("published", "in unpublished descriptor")
if not self.silence:
LOG.warn(*args)
def __call__(self, *args):
self.called = 1
self.errors += 1
if not self.published:
args = list(args)
args[0] = args[0].replace("published", "in unpublished descriptor")
if not self.silence:
LOG.warn(*args)
def getHeaders(self):
"""Return a dict containing the headers for this message."""
if self.type is None:
self.decode()
if self.headers is None:
LOG.warn("getHeaders found no decoded headers")
return {}
return self.headers
def removeDeadKeys(self, now=None):
"""Remove all keys that have expired."""
self.checkKeys()
keys = self.getDeadKeys(now)
for message, keyset in keys:
LOG.info(message)
keyset.delete()
self.checkKeys()
def __clientFinished(self, addr):
"""Called when a client connection runs out of packets to send,
or halts."""
try:
del self.clientConByAddr[addr]
except KeyError:
LOG.warn("Didn't find client connection to %s in address map",
addr)
def getHeaders(self):
"""Return a dict containing the headers for this message."""
if self.type is None:
self.decode()
if self.headers is None:
LOG.warn("getHeaders found no decoded headers")
return {}
return self.headers
def __readTooMuch(self):
"""Helper function -- called if we read too much data while we're
shutting down."""
LOG.error(
"Read over 128 bytes of unexpected data from closing "
"connection to %s", self.address)
self.onTLSError()
raise _Closing()
def __clientFinished(self, addr):
"""Called when a client connection runs out of packets to send,
or halts."""
try:
del self.clientConByAddr[addr]
except KeyError:
LOG.warn("Didn't find client connection to %s in address map",
addr)
def deliveryFailed(self, handle, retriable=0, now=None):
"""Removes a message from the outgoing queue, or requeues it
for delivery at a later time. This method should be
invoked after the corresponding message has been
unsuccessfully delivered."""
assert self.retrySchedule is not None
LOG.trace("DeliveryQueue failed to deliver %s from %s",
handle, self.qname)
try:
self._lock.acquire()
try:
ds = self.store.getMetadata(handle)
except KeyError:
ds = None
except CorruptedFile:
return
if ds is None:
# This should never happen
LOG.error_exc(sys.exc_info(),
"Handle %s had no state", handle)
ds = _DeliveryState(now)
ds.setNextAttempt(self.retrySchedule, now)
self.store.setMetadata(handle, ds)
return
if not ds.isPending():
LOG.error("Handle %s was not pending", handle)
return
last = ds.pending
ds.setNonPending()
if retriable:
# If we can retry the message, update the deliveryState
# with the most recent attempt, and see if there's another
# attempt in the future.
ds.setLastAttempt(last)
ds.setNextAttempt(self.retrySchedule, now)
if ds.nextAttempt is not None:
# There is another scheduled delivery attempt. Remember
# it, mark the message sendable again, and save our state.
LOG.trace(" (We'll try %s again at %s)", handle,
formatTime(ds.nextAttempt, 1))
self.store.setMetadata(handle, ds)
return
else:
assert ds.isRemovable()
# Otherwise, fallthrough.
# If we reach this point, the message is undeliverable, either
# because 'retriable' is false, or because we've run out of
# retries.
LOG.trace(" (Giving up on %s)", handle)
self.removeMessage(handle)
finally:
self._lock.release()
def configure_trng(config):
"""Initialize the true entropy source from a given Config object. If
none is provided, tries some sane defaults."""
global _TRNG_FILENAME
global _theTrueRNG
if sys.platform == 'win32':
# We have two entropy sources on windows: openssl's built-in
# entropy generator that takes data from the screen, and
# Windows's CryptGenRandom function. Because the former is
# insecure, and the latter is closed-source, we xor them.
_ml.win32_openssl_seed()
_ml.openssl_seed(_ml.win32_get_random_bytes(32))
_theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG())
return
if config is not None:
requestedFile = config['Host'].get('EntropySource')
else:
requestedFile = None
# Build a list of candidates
defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform,
PLATFORM_TRNG_DEFAULTS['***'])
files = [ requestedFile ] + defaults
# Now find the first of our candidates that exists and is a character
# device.
randFile = None
for filename in files:
if filename is None:
continue
verbose = (filename == requestedFile)
if not os.path.exists(filename):
if verbose:
LOG.warn("No such file as %s", filename)
else:
st = os.stat(filename)
if not (st[stat.ST_MODE] & stat.S_IFCHR):
if verbose:
LOG.error("Entropy source %s isn't a character device",
filename)
else:
randFile = filename
break
if randFile is None and _TRNG_FILENAME is None:
LOG.fatal("No entropy source available: Tried all of %s",
files)
raise MixFatalError("No entropy source available")
elif randFile is None:
LOG.warn("Falling back to previous entropy source %s",
_TRNG_FILENAME)
else:
LOG.info("Setting entropy source to %r", randFile)
_TRNG_FILENAME = randFile
_theTrueRNG = _TrueRNG(1024)
def getBaseDir(self):
"""Return the base directory for this configuration."""
v = self["Server"]["BaseDir"]
if v is None:
v = self["Server"]["Homedir"]
if v is None:
LOG.warn("Defaulting base directory to /var/spool/minion; this will change.")
v = "/var/spool/minion"
return v
def deliverySucceeded(self, handle, now=None):
"""Removes a message from the outgoing queue. This method
should be invoked after the corresponding message has been
successfully delivered.
"""
assert self.retrySchedule is not None
LOG.trace("DeliveryQueue got successful delivery for %s from %s", handle, self.qname)
self.removeMessage(handle)
def deliveryFailed(self, handle, retriable=0, now=None):
"""Removes a message from the outgoing queue, or requeues it
for delivery at a later time. This method should be
invoked after the corresponding message has been
unsuccessfully delivered."""
assert self.retrySchedule is not None
LOG.trace("DeliveryQueue failed to deliver %s from %s", handle,
self.qname)
try:
self._lock.acquire()
try:
ds = self.store.getMetadata(handle)
except KeyError:
ds = None
except CorruptedFile:
return
if ds is None:
# This should never happen
LOG.error_exc(sys.exc_info(), "Handle %s had no state", handle)
ds = _DeliveryState(now)
ds.setNextAttempt(self.retrySchedule, now)
self.store.setMetadata(handle, ds)
return
if not ds.isPending():
LOG.error("Handle %s was not pending", handle)
return
last = ds.pending
ds.setNonPending()
if retriable:
# If we can retry the message, update the deliveryState
# with the most recent attempt, and see if there's another
# attempt in the future.
ds.setLastAttempt(last)
ds.setNextAttempt(self.retrySchedule, now)
if ds.nextAttempt is not None:
# There is another scheduled delivery attempt. Remember
# it, mark the message sendable again, and save our state.
LOG.trace(" (We'll try %s again at %s)", handle,
formatTime(ds.nextAttempt, 1))
self.store.setMetadata(handle, ds)
return
else:
assert ds.isRemovable()
# Otherwise, fallthrough.
# If we reach this point, the message is undeliverable, either
# because 'retriable' is false, or because we've run out of
# retries.
LOG.trace(" (Giving up on %s)", handle)
self.removeMessage(handle)
finally:
self._lock.release()
def encodeMessage(message, overhead, uncompressedFragmentPrefix="",
paddingPRNG=None):
"""Given a message, compress it, fragment it into individual payloads,
and add extra fields (size, hash, etc) as appropriate. Return a list
of strings, each of which is a message payload suitable for use in
build*Message.
message: the initial message
overhead: number of bytes to omit from each payload,
given the type ofthe message encoding.
(0 or ENC_FWD_OVERHEAD)
uncompressedFragmentPrefix: If we fragment the message,
we add this string to the message after compression but
before whitening and fragmentation.
paddingPRNG: generator for padding.
Note: If multiple strings are returned, be sure to shuffle them
before transmitting them to the network.
"""
assert overhead in (0, ENC_FWD_OVERHEAD)
if paddingPRNG is None:
paddingPRNG = Crypto.getCommonPRNG()
origLength = len(message)
payload = compressData(message)
length = len(payload)
if length > 1024 and length*20 <= origLength:
LOG.warn("Message is very compressible and will look like a zlib bomb")
paddingLen = PAYLOAD_LEN - SINGLETON_PAYLOAD_OVERHEAD - overhead - length
# If the compressed payload fits in 28K, we're set.
if paddingLen >= 0:
# We pad the payload, and construct a new SingletonPayload,
# including this payload's size and checksum.
payload += paddingPRNG.getBytes(paddingLen)
p = SingletonPayload(length, None, payload)
p.computeHash()
return [ p.pack() ]
# Okay, we need to fragment the message. First, add the prefix if needed.
if uncompressedFragmentPrefix:
payload = uncompressedFragmentPrefix+payload
# Now generate a message ID
messageid = Crypto.getCommonPRNG().getBytes(FRAGMENT_MESSAGEID_LEN)
# Figure out how many chunks to divide it into...
p = mixminion.Fragments.FragmentationParams(len(payload), overhead)
# ... fragment the payload into chunks...
rawFragments = p.getFragments(payload)
fragments = []
# ... and annotate each chunk with appropriate payload header info.
for i in xrange(len(rawFragments)):
pyld = FragmentPayload(i, None, messageid, p.length, rawFragments[i])
pyld.computeHash()
fragments.append(pyld.pack())
rawFragments[i] = None
return fragments
def onProtocolWritten(self,n):
if self.outbuf:
# Not done writing outgoing data.
return
LOG.debug("Sent MMTP protocol string to %s", self.address)
self.stopWriting()
self.beginReading()
self.onRead = self.onProtocolRead
def encodeMessage(message, overhead, uncompressedFragmentPrefix="",
paddingPRNG=None):
"""Given a message, compress it, fragment it into individual payloads,
and add extra fields (size, hash, etc) as appropriate. Return a list
of strings, each of which is a message payload suitable for use in
build*Message.
message: the initial message
overhead: number of bytes to omit from each payload,
given the type ofthe message encoding.
(0 or ENC_FWD_OVERHEAD)
uncompressedFragmentPrefix: If we fragment the message,
we add this string to the message after compression but
before whitening and fragmentation.
paddingPRNG: generator for padding.
Note: If multiple strings are returned, be sure to shuffle them
before transmitting them to the network.
"""
assert overhead in (0, ENC_FWD_OVERHEAD)
if paddingPRNG is None:
paddingPRNG = Crypto.getCommonPRNG()
origLength = len(message)
payload = compressData(message)
length = len(payload)
if length > 1024 and length*20 <= origLength:
LOG.warn("Message is very compressible and will look like a zlib bomb")
paddingLen = PAYLOAD_LEN - SINGLETON_PAYLOAD_OVERHEAD - overhead - length
# If the compressed payload fits in 28K, we're set.
if paddingLen >= 0:
# We pad the payload, and construct a new SingletonPayload,
# including this payload's size and checksum.
payload += paddingPRNG.getBytes(paddingLen)
p = SingletonPayload(length, None, payload)
p.computeHash()
return [ p.pack() ]
# Okay, we need to fragment the message. First, add the prefix if needed.
if uncompressedFragmentPrefix:
payload = uncompressedFragmentPrefix+payload
# Now generate a message ID
messageid = Crypto.getCommonPRNG().getBytes(FRAGMENT_MESSAGEID_LEN)
# Figure out how many chunks to divide it into...
p = mixminion.Fragments.FragmentationParams(len(payload), overhead)
# ... fragment the payload into chunks...
rawFragments = p.getFragments(payload)
fragments = []
# ... and annotate each chunk with appropriate payload header info.
for i in xrange(len(rawFragments)):
pyld = FragmentPayload(i, None, messageid, p.length, rawFragments[i])
pyld.computeHash()
fragments.append(pyld.pack())
rawFragments[i] = None
return fragments
def onProtocolWritten(self, n):
if self.outbuf:
# Not done writing outgoing data.
return
LOG.debug("Sent MMTP protocol string to %s", self.address)
self.stopWriting()
self.beginReading()
self.onRead = self.onProtocolRead
def configure_trng(config):
"""Initialize the true entropy source from a given Config object. If
none is provided, tries some sane defaults."""
global _TRNG_FILENAME
global _theTrueRNG
if sys.platform == 'win32':
# We have two entropy sources on windows: openssl's built-in
# entropy generator that takes data from the screen, and
# Windows's CryptGenRandom function. Because the former is
# insecure, and the latter is closed-source, we xor them.
_ml.win32_openssl_seed()
_ml.openssl_seed(_ml.win32_get_random_bytes(32))
_theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG())
return
if config is not None:
requestedFile = config['Host'].get('EntropySource')
else:
requestedFile = None
# Build a list of candidates
defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform,
PLATFORM_TRNG_DEFAULTS['***'])
files = [requestedFile] + defaults
# Now find the first of our candidates that exists and is a character
# device.
randFile = None
for filename in files:
if filename is None:
continue
verbose = (filename == requestedFile)
if not os.path.exists(filename):
if verbose:
LOG.warn("No such file as %s", filename)
else:
st = os.stat(filename)
if not (st[stat.ST_MODE] & stat.S_IFCHR):
if verbose:
LOG.error("Entropy source %s isn't a character device",
filename)
else:
randFile = filename
break
if randFile is None and _TRNG_FILENAME is None:
LOG.fatal("No entropy source available: Tried all of %s", files)
raise MixFatalError("No entropy source available")
elif randFile is None:
LOG.warn("Falling back to previous entropy source %s", _TRNG_FILENAME)
else:
LOG.info("Setting entropy source to %r", randFile)
_TRNG_FILENAME = randFile
_theTrueRNG = _TrueRNG(1024)
def onConnected(self):
LOG.debug("Completed MMTP client connection to %s",self.address)
# Is the certificate correct?
try:
self.certCache.check(self.tls, self.targetKeyID, self.address)
except MixProtocolBadAuth, e:
LOG.warn("Certificate error: %s. Shutting down connection.", e)
self._failPendingPackets()
self.startShutdown()
return
def __loadModules(self, section, sectionEntries):
"""Callback from the [Server] section of a config file. Parses
the module options, and adds new sections to the syntax
accordingly."""
self.moduleManager.setPath(section.get('ModulePath'))
for mod in section.get('Module', []):
LOG.info("Loading module %s", mod)
self.moduleManager.loadExtModule(mod)
self._syntax.update(self.moduleManager.getConfigSyntax())
def deliverySucceeded(self, handle, now=None):
"""Removes a message from the outgoing queue. This method
should be invoked after the corresponding message has been
successfully delivered.
"""
assert self.retrySchedule is not None
LOG.trace("DeliveryQueue got successful delivery for %s from %s",
handle, self.qname)
self.removeMessage(handle)
def onConnected(self):
LOG.debug("Completed MMTP client connection to %s", self.address)
# Is the certificate correct?
try:
self.certCache.check(self.tls, self.targetKeyID, self.address)
except MixProtocolBadAuth, e:
LOG.warn("Certificate error: %s. Shutting down connection.", e)
self._failPendingPackets()
self.startShutdown()
return
def rebuildIDCache(self):
for fn in os.listdir(self.serverIDDir):
fname = os.path.join(self.serverIDDir, fn)
tp, val = readPickled(fname)
if tp != "V0":
LOG.warn("Weird file version %s on %s", tp, fname)
continue
nickname, ident = val
ID = mixminion.Crypto.sha1(ident)
self.idCache.insertID(nickname, ID)
def rebuildIDCache(self):
for fn in os.listdir(self.serverIDDir):
fname = os.path.join(self.serverIDDir, fn)
tp,val = readPickled(fname)
if tp != "V0":
LOG.warn("Weird file version %s on %s",tp,fname)
continue
nickname, ident = val
ID = mixminion.Crypto.sha1(ident)
self.idCache.insertID(nickname, ID)
def __loadModules(self, section, sectionEntries):
"""Callback from the [Server] section of a config file. Parses
the module options, and adds new sections to the syntax
accordingly."""
self.moduleManager.setPath(section.get('ModulePath'))
for mod in section.get('Module', []):
LOG.info("Loading module %s", mod)
self.moduleManager.loadExtModule(mod)
self._syntax.update(self.moduleManager.getConfigSyntax())
def tryTimeout(self, cutoff):
"""Close self.sock if the last activity on this connection was
before 'cutoff'. Returns true iff the connection is timed out.
"""
if self.lastActivity <= cutoff:
LOG.warn("Connection to %s timed out: %.2f seconds without activity",
self.address, time.time()-self.lastActivity)
self.onTimeout()
self.__close()
return 1
return 0
def getBaseDir(self):
"""Return the base directory for this configuration."""
v = self["Server"]["BaseDir"]
if v is None:
v = self["Server"]["Homedir"]
if v is None:
LOG.warn(
"Defaulting base directory to /var/spool/minion; this will change."
)
v = "/var/spool/minion"
return v
def main(cmd, args):
"""[Entry point] Multiplex among subcommands."""
if len(args)<1 or ('-h', '--help') in args:
usageAndExit()
command = args[0]
args = args[1:]
if not SUBCOMMANDS.has_key(command):
print "Unknown command", command
usageAndExit()
init_crypto()
LOG.setMinSeverity("INFO")
SUBCOMMANDS[command](args)
def main(cmd, args):
"""[Entry point] Multiplex among subcommands."""
if len(args) < 1 or ('-h', '--help') in args:
usageAndExit()
command = args[0]
args = args[1:]
if not SUBCOMMANDS.has_key(command):
print "Unknown command", command
usageAndExit()
init_crypto()
LOG.setMinSeverity("INFO")
SUBCOMMANDS[command](args)
def getHandlesByDestAndAge(self, destList, directory, notAfter=None,
warnUnused=1):
"""Return a list of handles for all messages queued for servers in a
given list before a given date.
destList -- A list of hostnames, ips, keyids, or nicknames
for servers whose messages should be included in the result.
directory -- An instance of ClientDirectory used to resolve
nicknames. This may be None if no nicknames are included.
notAfter -- If provided, a time such that no messages queued
later should be included
warnUnused -- If true, we log a message for every element in
destList that has no matching messages in the queue.
"""
destSet = {}
reverse = {}
for d in destList:
if directory:
keyid = directory.getKeyIDByNickname(d)
if keyid:
destSet[keyid] = 1
reverse[keyid] = d
continue
destSet[d] = 1
self.loadMetadata()
result = []
foundAny = {}
foundMatch = {}
for h in self.store.getAllMessages():
_, r, when = self.store.getMetadata(h)
if (destSet.has_key(r.keyinfo) or
(hasattr(r, 'hostname') and destSet.has_key(r.hostname)) or
(hasattr(r, 'ip') and destSet.has_key(r.ip))):
keys = [ getattr(r, 'hostname', None),
getattr(r, 'ip', None),
reverse.get(r.keyinfo, None),
r.keyinfo ]
for k in keys: foundAny[k]=1
if notAfter and when > notAfter:
continue
for k in keys: foundMatch[k]=1
result.append(h)
if warnUnused:
for d in destList:
if foundMatch.get(d):
continue
elif foundAny.get(d):
LOG.warn("No expired packets found for %r", d)
else:
LOG.warn("No pending packets found for %r", d)
return result
def tryTimeout(self, cutoff):
"""Close self.sock if the last activity on this connection was
before 'cutoff'. Returns true iff the connection is timed out.
"""
if self.lastActivity <= cutoff:
LOG.warn(
"Connection to %s timed out: %.2f seconds without activity",
self.address,
time.time() - self.lastActivity)
self.onTimeout()
self.__close()
return 1
return 0
def run(self):
"""Internal: main body of processing thread."""
try:
while 1:
job = self.mqueue.get()
job()
except ProcessingThread._Shutdown:
LOG.info("Shutting down %s", self.threadName)
return
except:
LOG.error_exc(sys.exc_info(),
"Exception in %s; shutting down thread.",
self.threadName)
def run(self):
"""Internal: main body of processing thread."""
try:
while 1:
job = self.mqueue.get()
job()
except ProcessingThread._Shutdown:
LOG.info("Shutting down %s",self.threadName)
return
except:
LOG.error_exc(sys.exc_info(),
"Exception in %s; shutting down thread.",
self.threadName)
def __close(self, gotClose=0):
"""helper: close the underlying socket without cleaning up the TLS
connection."""
if gotClose:
if self.__stateFn == self.__connectFn:
LOG.warn("Couldn't connect to %s", self.address)
else:
LOG.warn("Unexpectedly closed connection to %s", self.address)
self.onTLSError()
self.sock.close()
self.sock = None
self.tls = None
self.__stateFn = self.__closedFn
self.onClosed()
def getPacket(self, handle):
"""Given a handle, return a 3-tuple of the corresponding
32K packet, {IPV4/Host}Info, and time of first queueing. (The time
is rounded down to the closest midnight GMT.) May raise
CorruptedFile."""
obj = self.store.getObject(handle)
try:
magic, packet, routing, when = obj
except (ValueError, TypeError):
magic = None
if magic != "PACKET-0":
LOG.error("Unrecognized packet format for %s", handle)
return None
return packet, routing, when
