def _validateZlib():
"""Internal function: Make sure that zlib is a recognized version, and
that it compresses things as expected. (This check is important,
because using a zlib version that compressed differently from zlib1.1.4
would make senders partitionable by payload compression.)
"""
global _ZLIB_LIBRARY_OK
ver = getattr(zlib, "ZLIB_VERSION", None)
if ver and ver < "1.1.2":
raise MixFatalError("Zlib version %s is not supported" % ver)
_ZLIB_LIBRARY_OK = 0.5
if ver in ("1.1.2", "1.1.3", "1.1.4", "1.2.0", "1.2.0.1", "1.2.0.2",
"1.2.0.3", "1.2.0.4", "1.2.0.5", "1.2.0.6", "1.2.0.7",
"1.2.0.8", "1.2.1", "1.2.1.1", "1.2.1.2", "1.2.2", "1.2.2.2",
"1.2.3"):
_ZLIB_LIBRARY_OK = 1
return
LOG.info("Unrecognized zlib version: %r. Spot-checking output", ver)
# This test is inadequate, but it _might_ catch future incompatible
# changes.
_ZLIB_LIBRARY_OK = 0.5
good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
'`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
'\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
'q\x8d\n\x93'
if compressData("aZbAAcdefg" * 1000) == good:
_ZLIB_LIBRARY_OK = 1
else:
_ZLIB_LIBRARY_OK = 0
raise MixFatalError("Zlib output not as exected.")
def __init__(self, incomingDir, rejectDir):
"""Create an IncomingQueue to hold incoming servers in incomingDir
and rejected servers in rejectDir."""
self.incomingDir = incomingDir
self.rejectDir = rejectDir
if not os.path.exists(incomingDir):
raise MixFatalError("Incoming directory doesn't exist"+incomingDir)
if not os.path.exists(rejectDir):
raise MixFatalError("Reject directory doesn't exist"+rejectDir)
def _prng(self, n):
"""Implementation: uses the AES counter stream to generate entropy."""
c = self.counter
self.counter += n
# On python2.0, we overflow and wrap around.
if (self.counter < c) or (self.counter >> 32):
raise MixFatalError("Exhausted period of PRNG.")
return prng(self.key, n, c)
def __setstate__(self, state):
if state[0] == 'ADDR-V1':
_, self.address, self.lastSuccess, self.lastFailure, \
self.firstFailure = state
else:
#XXXX008 This is way too extreme.
raise MixFatalError("Unrecognized delivery state")
self.nextAttempt = None
def configure_trng(config):
"""Initialize the true entropy source from a given Config object. If
none is provided, tries some sane defaults."""
global _TRNG_FILENAME
global _theTrueRNG
if sys.platform == 'win32':
# We have two entropy sources on windows: openssl's built-in
# entropy generator that takes data from the screen, and
# Windows's CryptGenRandom function. Because the former is
# insecure, and the latter is closed-source, we xor them.
_ml.win32_openssl_seed()
_ml.openssl_seed(_ml.win32_get_random_bytes(32))
_theTrueRNG = _XorRNG(_OpensslRNG(), _WinTrueRNG())
return
if config is not None:
requestedFile = config['Host'].get('EntropySource')
else:
requestedFile = None
# Build a list of candidates
defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform,
PLATFORM_TRNG_DEFAULTS['***'])
files = [requestedFile] + defaults
# Now find the first of our candidates that exists and is a character
# device.
randFile = None
for filename in files:
if filename is None:
continue
verbose = (filename == requestedFile)
if not os.path.exists(filename):
if verbose:
LOG.warn("No such file as %s", filename)
else:
st = os.stat(filename)
if not (st[stat.ST_MODE] & stat.S_IFCHR):
if verbose:
LOG.error("Entropy source %s isn't a character device",
filename)
else:
randFile = filename
break
if randFile is None and _TRNG_FILENAME is None:
LOG.fatal("No entropy source available: Tried all of %s", files)
raise MixFatalError("No entropy source available")
elif randFile is None:
LOG.warn("Falling back to previous entropy source %s", _TRNG_FILENAME)
else:
LOG.info("Setting entropy source to %r", randFile)
_TRNG_FILENAME = randFile
_theTrueRNG = _TrueRNG(1024)
def __init__(self, filename, keyid):
mixminion.Filestore.BooleanJournaledDBBase.__init__(
self, filename, "digest hash", 20)
self.keyid = keyid
try:
if self.log["KEYID"] != keyid:
raise MixFatalError("Log KEYID does not match current KEYID")
except KeyError:
self.log["KEYID"] = keyid
self._syncLog()
def __setstate__(self, state):
if state[0] == 'V0':
(_, self.messageid, self.idx, self.size, self.isChunk,
self.chunkNum, self.overhead, self.insertedDate, self.nym) = state
self.digest = None
elif state[0] == 'V1':
(_, self.messageid, self.idx, self.size, self.isChunk,
self.chunkNum, self.overhead, self.insertedDate, self.nym,
self.digest) = state
else:
raise MixFatalError("Unrecognized fragment state")
def init_crypto(config=None):
"""Initialize the crypto subsystem."""
configure_trng(config)
try:
# Try to read /dev/urandom
trng(1)
except MixFatalError:
raise
except:
info = sys.exc_info()
raise MixFatalError("Error initializing entropy source: %s" % info[1])
openssl_seed(40)
def __setstate__(self, state):
# For pickling.
if state[0] == "V1":
self.queuedTime = state[1]
self.lastAttempt = state[2]
self.address = state[3]
else:
#XXXX008 This is way too extreme.
raise MixFatalError("Unrecognized delivery state")
self.pending = None
self.nextAttempt = None
self.remove = 0
def rescan(self):
"""Reconstruct this ServerList object's internal state."""
try:
self._lock()
# First, build self.servers
self.servers = {}
for filename in os.listdir(self.serverDir):
path = os.path.join(self.serverDir, filename)
try:
self.servers[filename] = ServerInfo(fname=path)
except ConfigError, e:
LOG.warn("Somehow, a bad server named %s got in our store",
filename)
LOG.warn(" (Error was: %s)", str(e))
_moveServer(self.serverDir, self.rejectDir, filename)
# Next, rebuild self.serverIDs:
self.serverIDs = {}
for filename in os.listdir(self.serverIDDir):
path = os.path.join(self.serverIDDir, filename)
t = readPickled(path)
if t[0] != 'V0':
LOG.warn("Skipping confusing stored key in file %s",
filename)
continue
nickname, key = t[1]
key = pk_decode_public_key(key)
if self.serverIDs.has_key(nickname.lower()):
LOG.warn("Eeek! Multiple entries for %s", nickname)
if not pk_same_public_key(self.serverIDs[nickname.lower()],
key):
raise MixFatalError(
"Multiple conflicting entries for %s" % nickname)
self.serverIDs[nickname.lower()] = key
# (check for consistency)
for s in self.servers.values():
lcn = s.getNickname().lower()
try:
ident = self.serverIDs[lcn]
except KeyError:
raise UIError("No stored key for server %s" %
s.getNickname())
if not pk_same_public_key(ident, s.getIdentity()):
raise UIError("Inconsistent stored key for server %s" %
s.getNickname())
# Then, rebuild self.serversByNickname
self.__buildNicknameMap()
def parseRelayInfoByType(routingType, routingInfo):
"""Parse the routingInfo contained in the string 'routinginfo',
according to the type in 'routingType'. Only relay types are
supported."""
if routingType in (FWD_IPV4_TYPE, SWAP_FWD_IPV4_TYPE):
parseFn = parseIPV4Info
parsedType = IPV4Info
elif routingType in (FWD_HOST_TYPE, SWAP_FWD_HOST_TYPE):
parseFn = parseMMTPHostInfo
parsedType = MMTPHostInfo
else:
raise MixFatalError("Unrecognized relay type 0x%04X" % routingType)
if type(routingInfo) == types.StringType:
routingInfo = parseFn(routingInfo)
assert isinstance(routingInfo, parsedType)
return routingInfo
def getHashLog(filename, keyid):
"""Given a filename and keyid, return a HashLog object with that fname
and ID, opening a new one if necessary. This function is needed to
implement key rotation: we want to assemble a list of current
hashlogs, but we can't open the same HashLog database twice at once."""
try:
_HASHLOG_DICT_LOCK.acquire()
try:
keyid_orig, hl = _OPEN_HASHLOGS[filename]
if keyid != keyid_orig:
raise MixFatalError("KeyID changed for hashlog %s" % filename)
LOG.trace("getHashLog() returning open hashlog at %s", filename)
except KeyError:
LOG.trace("getHashLog() opening hashlog at %s", filename)
hl = HashLog(filename, keyid)
_OPEN_HASHLOGS[filename] = (keyid, hl)
return hl
finally:
_HASHLOG_DICT_LOCK.release()
def setKeys(self, keys, hashlogs):
"""Change the keys and hashlogs used by this PacketHandler.
Arguments are as to PacketHandler.__init__
"""
self.lock.acquire()
newKeys = {}
try:
# Build a set of asn.1-encoded public keys in *new* set.
for k in keys:
newKeys[k.encode_key(1)] = 1
if k.get_modulus_bytes() != PACKET_KEY_BYTES:
raise MixFatalError("Incorrect packet key length")
# For all old public keys, if they aren't in the new set, close
# their hashlogs.
for k, h in self.privatekeys:
if not newKeys.get(k.encode_key(1)):
h.close()
# Now, set the keys.
self.privatekeys = zip(keys, hashlogs)
finally:
self.lock.release()
def learnServerID(self, server):
"""Mark the ID for a server descriptor as the canonical
identity key associated with that server's nickname."""
try:
self._lock()
ident = server.getIdentity()
nickname = server.getNickname()
try:
if self.idCache.containsServer(server):
LOG.warn("Server %s already known", nickname)
except mixminion.directory.MismatchedID:
raise MixFatalError("Mismatched ID for server %s" % nickname)
LOG.info("Learning identity for new server %s", nickname)
self.idCache.insertServer(server)
writePickled(
os.path.join(self.serverIDDir,
nickname + "-" + formatFnameTime()),
("V0", (nickname, pk_encode_public_key(ident))))
self.idCache.save()
finally:
self._unlock()
def __init__(self, location, create=0, scrub=0):
"""Creates a file store object for a given directory, 'location'. If
'create' is true, creates the directory if necessary. If 'scrub'
is true, removes any incomplete or invalidated messages from the
store."""
secureDelete([]) # Make sure secureDelete is configured. HACK!
self._lock = threading.RLock()
self.dir = location
if not os.path.isabs(location):
LOG.warn("Directory path %s isn't absolute.", location)
if os.path.exists(location) and not os.path.isdir(location):
raise MixFatalError("%s is not a directory" % location)
createPrivateDir(location, nocreate=(not create))
if scrub:
self.cleanQueue()
# Count messages on first time through.
self.n_entries = -1
class RNG:
'''Base implementation class for random number generators. Works
by requesting a bunch of bytes via self._prng, and doling them
out piecemeal via self.getBytes.'''
def __init__(self, chunksize):
"""Initializes a RNG. Bytes will be fetched from _prng by 'chunkSize'
bytes at a time."""
self.bytes = ""
self.chunksize = chunksize
def getBytes(self, n):
"""Returns a string of 'n' random bytes."""
assert n >= 0
if n > len(self.bytes):
# If we don't have enough bytes, fetch enough so that we'll have
# a full chunk left over.
nMore = n + self.chunksize - len(self.bytes)
morebytes = self._prng(nMore)
res = self.bytes + morebytes[:n - len(self.bytes)]
self.bytes = morebytes[n - len(self.bytes):]
return res
else:
res = self.bytes[:n]
self.bytes = self.bytes[n:]
return res
def pick(self, lst):
"""Return a member of 'lst', chosen randomly according to a uniform
distribution. Raises IndexError if lst is empty."""
if not lst:
raise IndexError("rng.pick([])")
return lst[self.getInt(len(lst))]
def shuffle(self, lst, n=None):
"""Rearranges the elements of lst so that the first n elements
are randomly chosen from lst. Returns the first n elements.
(Other elements are still in lst, but may be in a nonrandom
order.) If n is None, shuffles and returns the entire list"""
size = len(lst)
if n is None:
n = size
else:
n = min(n, size)
if n == size:
series = xrange(n - 1)
else:
series = xrange(n)
# This permutation algorithm yields all permutation with equal
# probability (assuming a good rng); others do not.
getInt = self.getInt
for i in series:
swap = i + getInt(size - i)
lst[swap], lst[i] = lst[i], lst[swap]
return lst[:n]
def getInt(self, max):
"""Returns a random integer i s.t. 0 <= i < max.
The value of max must be less than 2**30."""
# FFFF This implementation is about 2-4x as good as the last one, but
# FFFF still could be better. It's faster than getFloat()*max.
# FFFF (This code assumes that integers are at least 32 bits. Maybe
# FFFF we could do better.)
assert 0 < max < 0x3fffffff
_ord = ord
cutoff = 0x7fffffff - (0x7fffffff % max)
while 1:
# Get a random positive int between 0 and 0x7fffffff.
b = self.getBytes(4)
o = (((((((_ord(b[0]) & 0x7f) << 8) + _ord(b[1])) << 8) +
_ord(b[2])) << 8) + _ord(b[3]))
# Retry if we got a value that would fall in an incomplete
# run of 'max' elements.
if o < cutoff:
return o % max
raise AssertionError # unreached; appease pychecker
def getNormal(self, m, s):
"""Return a random value with mean m and standard deviation s.
"""
# Lifted from random.py in standard python dist.
while 1:
u1 = self.getFloat()
u2 = 1.0 - self.getFloat()
z = NV_MAGICCONST * (u1 - 0.5) / u2
zz = z * z / 4.0
if zz <= -math.log(u2):
break
return m + z * s
def getFloat(self):
"""Return a floating-point number between 0 and 1."""
b = self.getBytes(4)
_ord = ord
o = ((((((
(_ord(b[0]) & 0x7f) << 8) + _ord(b[1])) << 8) + _ord(b[2])) << 8) +
_ord(b[3]))
# return o / float(0x7fffffff)
return o / 2147483647.0
def openNewFile(self, dir, prefix="", binary=1, conflictPrefix=None):
"""Generate a new random filename within a directory with a given
prefix within a directory, and open a new file within the directory
with that filename. Return 2-tuple of a file object and the
random portion of the filename.
Random portions are generated by choosing 8 random characters
from the set 'A-Za-z0-9+-'.
If 'conflictPrefix' is set, do not return any file named
prefix+H if a file named conflictPrefix+H already exists.
"""
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
mode = "w"
if binary:
flags |= getattr(os, 'O_BINARY', 0)
mode = "wb"
while 1:
b = self.getBytes(6)
base = binascii.b2a_base64(b).strip().replace("/", "-")
if FS_IS_CASEI:
base = base.lower()
fname = os.path.join(dir, "%s%s" % (prefix, base))
if conflictPrefix and os.path.exists(
os.path.join(dir, conflictPrefix + base)):
continue
try:
fd = os.open(fname, flags, 0600)
return os.fdopen(fd, mode), base
except OSError, e:
if e.errno != errno.EEXIST:
raise e
# If the file exists (a rare event!) we pass through, and
# try again. This paranoia is brought to you by user
# request. :)
raise MixFatalError("Unreachable") # appease pychecker.
if msg is not None:
try:
if force:
maxSize = None
else:
maxSize = len(msg) * 20
return mixminion.Packet.uncompressData(msg, maxSize)
except mixminion.Packet.ParseError, e:
raise UIError("Invalid message %s: %s" % (msgid, e))
if state is None:
raise UIError("No such message as '%s'" % msgid)
elif not state.isDone():
raise UIError("Message '%s' is still missing fragments." % msgid)
else:
raise MixFatalError("Can't decode message %s; I don't know why!" %
msgid)
def removeMessages(self, msgids):
"""Remove all the messages whose IDs are in the list 'msgIDs'. If the
messages were reassembled, mark them as 'COMPLETED'; else mark them
as 'REJECTED'."""
pool = self.__getPool()
idSet = {}
for i in msgids:
state = pool.getStateByMsgID(i)
if state is None:
raise UIError("No such message as %s")
idSet[state.messageid] = 1
pool._deleteMessageIDs(idSet, "?")
pool.cleanQueue()
raise
st = None
# If the file is empty, delete it and start over.
if st and st[stat.ST_SIZE] == 0:
LOG.warn("Half-created database %s found; cleaning up.", filename)
tryUnlink(filename)
dbtype = whichdb.whichdb(filename)
LOG.debug("Opening %s database at %s", purpose, filename)
try:
if dbtype != 'dbhash':
db = _openDBHash(filename, 'c', 0600)
else:
db = anydbm.open(filename, 'c', 0600)
except anydbm.error, e:
raise MixFatalError("Can't open %s database: %s" % (purpose, e))
except ImportError:
raise MixFatalError("Unsupported type for %s database: %s" %
(purpose, dbtype))
if hasattr(db, 'sync'):
syncLog = db.sync
elif hasattr(db, '_commit'):
# Workaround for dumbdbm to allow syncing. (Standard in
# Python 2.3.)
syncLog = db._commit
else:
# Otherwise, force a no-op sync method.
syncLog = lambda: None
if isinstance(db, dumbdbm._Database):