def fillPool(self): """Starts the random pool. Dont do in constructor since this is computationally intensive.""" # A cryptographically safe source of random data. self.pool = RandomPool(384)
def fillPool(self):
"""Starts the random pool. Dont do in constructor since this
is computationally intensive."""
# A cryptographically safe source of random data.
self.pool = RandomPool(384)
class Service411:
"""Can retrieve a 411 file from a set of master servers. Has the
ability to encrypt, and decrypt files using a hybrid RSA-Symmetric
cryptographic technique. Only as safe as your sysadmin."""
def __init__(self):
self.priv_filename = '/etc/411-security/master.key'
self.pub_filename = '/etc/411-security/master.pub'
self.shared_filename = '/etc/411-security/shared.key'
self.masters = []
# Our current favorite
self.master = None
self.disable = 0
self.verbose = 0
# The directory from which we place our 411 files. Used
# when translating 411 file paths.
self.rootdir="/"
self.sym = None
self.pool = None
# Master keys
self.priv = None
self.pub = None
# Shared key, 256bit + 64bit IV
self.shared = None
self.conn = None
# Default URL base path to find files.
self.urldir = '411.d'
# Groups we are interested in
self.groups = ['']
# Store attributes which we can use to filter on.
self.attrs = {}
self.config = Conf(self)
self.config.parse()
self.plugin = None
# A regex for our header search.
pattern = "\n*(?P<comment>.*?)\$411id\$"
self.header_pattern = re.compile(pattern)
pattern = "<a href=.+>(?P<filename>.+)</a> +(?P<date>\d+.*) +(?P<size>\d+.*)"
# Make the pattern matching engine case-insensitive.
self.dir_pattern = re.compile(pattern, re.I)
# Use Blowfish with fast Cipher Block Chaining.
self.sym = POW.Symmetric(POW.BF_CBC)
def fillPool(self):
"""Starts the random pool. Dont do in constructor since this
is computationally intensive."""
# A cryptographically safe source of random data.
self.pool = RandomPool(384)
def setConf(self, conf):
self.config = conf(self)
def setConfHandler(self, handler):
self.config.setHandler(handler)
self.config.parse()
def four11Path(self, filename411):
"""Translates 411 file names into UNIX absolute filenames."""
# Warning this is UNIX dependant
n = string.replace(filename411, ".", "/")
n = string.replace(n, "//", ".")
return os.path.normpath(os.path.join(self.rootdir, n))
def path411(self, filename):
"""Turns an absolute UNIX path into a 411 filename. Every
period ('.') is a filesystem delimeter (like /), and all
filenames are assumed to be absolute, to start with a /. A
literal period is coded as a double period ('..'). """
# Warning, this is UNIX dependant.
n = string.replace(filename[1:], ".", "..")
n = string.replace(n, "/", ".")
return n
def connect(self, master=None):
"""Opens a HTTP 1.1 connection to the first live master server.
Will use the master argument if specified, otherwise consults
internal master server list. This connection can service
multiple requests."""
if self.conn:
self.disconnect()
if master:
masters = [master]
else:
masters = self.getMasters()
if not masters:
raise ValueError, \
"We have no master servers to connect to."
conn = None
for master in masters:
# Split address into host & port
m = master.getAddress().split(':')
if len(m) == 2:
conn = HTTPConnection(m[0], int(m[1]))
else:
conn = HTTPConnection(m[0])
# Test the connection.
try:
# Set variable so that connection originates
# from privileged port only. This way, only
# root can initialize this connection
conn.privileged_port = True
conn.connect()
except:
# If we cannot connect, devalue this server.
master.decScore()
conn = None
continue
else:
# We pick the first master that accepts a
# connection.
self.master = master
master.incScore()
break
if conn:
self.conn = conn
else:
raise Error411, \
"Could not reach a master server. Masters: %s" \
% masters
def disconnect(self):
"""Closes the master HTTP connection. """
if self.conn:
self.conn.close()
self.conn = None
def get(self, file):
""" Retrives a 411 file. If arg is empty, a directory listing
dictionary is returned. If file is valid, returns the decrypted
contents and its associated meta data. """
if not file:
raise Error411, "I need a file to get"
# Allow you to get a full URL.
if file.count("http://"):
m = Master(file)
self.connect(m)
file = os.path.basename(file)
else:
# We use our internal configuration.
self.connect()
path = self.master.getDir() + file
self.conn.request('GET', urllib.quote(path))
headers = self.conn.getresponse()
status = headers['status']
reason = headers['reason']
if status != 200:
raise Error411, "Could not get file '%s%s': %s %s" % \
(self.master.getUrl(), file, status, reason)
contents = self.conn.read()
self.disconnect()
if file[-1] == '/': # A directory
return contents
else:
return self.decrypt(contents)
def find(self, path="/"):
"""Finds all relevant files to retrieve on a master server.
Returns dict indexed by file path containing mtime and size."""
self.files = {}
self.findHelper(path)
return self.files
def findHelper(self, path, depth=0):
listing = self.get(path)
lines = listing.split("\n")
for line in lines:
m = self.dir_pattern.search(line)
if not m:
continue
#print line
filename = m.group('filename')
if filename == "Parent Directory":
continue
if filename[-1] == '/':
if self.verbose:
print "Found directory %s (%s)" \
% (path+filename, depth)
if self.isInteresting(path+filename, depth):
self.findHelper(path+filename, depth+1)
continue
date = m.group('date').strip()
size = m.group('size').strip()
self.files[path + filename] = {"Name": filename,
"Modified": date, "Size" : size}
def isInteresting(self, path, level):
"""Matches an offered group our registered groups. Slow,
search time is squared with the number of registered groups.
"""
offered = path[1:-1].split('/')
elements = len(offered)
for r in self.groups:
if not r:
continue
if self.verbose:
print "Matching %s to %s level %s" \
% (offered, r[:len(offered)], level)
try:
# Shorten the registered group to match
# the offerred one, then compare. Allows
# group inheritance.
if offered == r[:len(offered)]:
return 1
except:
continue
return 0
def readKeys(self):
"""Loads the 411 shared and master RSA keys"""
pub_file = open(self.pub_filename, 'r')
self.pub = POW.pemRead(POW.RSA_PUBLIC_KEY, pub_file.read())
pub_file.close()
shared_file = open(self.shared_filename, 'r')
self.shared = self.readSharedKey(shared_file.read())
shared_file.close()
if os.path.exists(self.priv_filename):
priv_file = open(self.priv_filename, 'r')
self.priv = POW.pemRead(POW.RSA_PRIVATE_KEY, priv_file.read())
priv_file.close()
def makeSharedKey(self):
"""Uses our cryptographically safe random number generator to
give us a 256bit session key and 64bit Init Vector, in 411 format."""
if not self.pool:
self.fillPool()
while 1:
self.pool.stir()
randomkey = self.pool.get_bytes(40)
# First 256 bits are for the session key
sessionkey = randomkey[:32]
# Last 64 bits are for the CBC initial value.
initialvalue = randomkey[-8:]
try:
self.sym.encryptInit(sessionkey, initialvalue)
except TypeError:
# Need a new session key (null chars in our key)
continue
else:
break
key = "-----BEGIN 411 SHARED KEY-----\n"
key += base64.encodestring(randomkey)
key += "-----END 411 SHARED KEY-----\n"
return key
def readSharedKey(self, key64):
"""Read a 411 shared key in base64 encoding and return the
binary bits"""
header = "-----BEGIN 411 SHARED KEY-----\n"
footer = "-----END 411 SHARED KEY-----\n"
try:
a = key64.index(header) + len(header)
b = key64.index(footer)
key = base64.decodestring(key64[a:b])
except:
raise Error411, \
"This does not appear to be a 411 shared key."
return key
def encrypt(self, plaintext,
header = "-----BEGIN 411 MESSAGE-----\n",
footer = "-----END 411 MESSAGE-----\n",
sign = 1):
"""Encrypts the plain text message using a hybrid cryptography
technique: a 256-bit random session key is encrypted with the
cluster shared key. The session key is used to quickly encrypt
the message with the Blowfish symmetrical algorithm."""
if not self.shared:
self.readKeys()
# First 256 bits are for the session key,
# Last 64 bits are for the CBC initial value.
try:
self.sym.encryptInit(self.shared[:32], self.shared[-8:])
except TypeError:
raise Error411, "Invalid Shared Key"
# Sign the text with the master private key
if sign:
if not self.priv:
raise Error411, "I need the master private key to sign messages"
sig = self.sign(plaintext)
else:
sig = "Not Signed"
# Encrypt the text with Blowfish for speed.
ciphertext = self.sym.update(plaintext) + self.sym.final()
ciphertext_base64 = base64.encodestring(ciphertext)
# Message format (v2.0):
# digital signature
# <blank line>
# symmetrically-encrypted message
msg = header
msg += sig + "\n"
msg += ciphertext_base64
msg += footer
return msg
def decrypt(self, contents,
header = "-----BEGIN 411 MESSAGE-----\n",
footer = "-----END 411 MESSAGE-----\n",
type411 = 1):
"""Uses the shared key to read 411 messages. For 411
type messages, returns the tuple (contents, meta) where meta is
a dictionary containing the 411 headers. If not type 411,
returns a (plaintext, sig_base64). No verification of signature
is performed."""
if not self.sym:
self.fillPool()
ciphersig_base64 = ''
try:
a = string.index(contents, header) + len(header)
b = string.index(contents, footer)
msg = contents[a:b]
ciphersig_base64, ciphertext_base64 = msg.split('\n\n')
ciphertext = base64.decodestring(ciphertext_base64)
except:
raise Error411, \
"This file does not appear to be in 411 format."
if not self.shared:
self.readKeys()
sessionkey = self.shared[:32]
initialvalue = self.shared[-8:]
self.sym.decryptInit(sessionkey, initialvalue)
try:
text = self.sym.update(ciphertext) + self.sym.final()
except POW.SSLError:
raise Error411, "Could not decrypt file, wrong key?"
if type411:
if not self.verify(text, ciphersig_base64):
raise Error411, "Signature does not verify."
return self.decode(text)
else:
return (text, ciphersig_base64)
def decode(self, plaintext):
meta = {}
p = Parser(plaintext, self.attrs)
meta = p.get_filtered_content()
self.plugin = p.get_plugin()
return meta['content'], meta
def verify(self, msg, sig_base64):
"""Verifies that the plaintext message was signed with the
(base64 encoded) signature, and has not been altered since
signing. Returns true if message verifies."""
if not self.pub:
try:
self.readKeys()
except IOError, e:
syslog.syslog(syslog.LOG_ERR, '411-error: ' \
+ str(e))
return 0
digest = POW.Digest(POW.MD5_DIGEST)
digest.update(msg)
try:
sig = base64.decodestring(sig_base64)
except:
return 0
return self.pub.verify(sig, digest.digest(), POW.MD5_DIGEST)
class Service411:
"""Can retrieve a 411 file from a set of master servers. Has the
ability to encrypt, and decrypt files using a hybrid RSA-Symmetric
cryptographic technique. Only as safe as your sysadmin."""
def __init__(self):
self.priv_filename = '/etc/411-security/master.key'
self.pub_filename = '/etc/411-security/master.pub'
self.shared_filename = '/etc/411-security/shared.key'
self.masters = []
# Our current favorite
self.master = None
self.disable = 0
self.verbose = 0
# The directory from which we place our 411 files. Used
# when translating 411 file paths.
self.rootdir = "/"
self.sym = None
self.pool = None
# Master keys
self.priv = None
self.pub = None
# Shared key, 256bit + 64bit IV
self.shared = None
self.conn = None
# Default URL base path to find files.
self.urldir = '411.d'
# Groups we are interested in
self.groups = ['']
# Store attributes which we can use to filter on.
self.attrs = {}
self.config = Conf(self)
self.config.parse()
self.plugin = None
# A regex for our header search.
pattern = "\n*(?P<comment>.*?)\$411id\$"
self.header_pattern = re.compile(pattern)
pattern = "<a href=.+>(?P<filename>.+)</a> +(?P<date>\d+.*) +(?P<size>\d+.*)"
# Make the pattern matching engine case-insensitive.
self.dir_pattern = re.compile(pattern, re.I)
# Use Blowfish with fast Cipher Block Chaining.
self.sym = POW.Symmetric(POW.BF_CBC)
def fillPool(self):
"""Starts the random pool. Dont do in constructor since this
is computationally intensive."""
# A cryptographically safe source of random data.
self.pool = RandomPool(384)
def setConf(self, conf):
self.config = conf(self)
def setConfHandler(self, handler):
self.config.setHandler(handler)
self.config.parse()
def four11Path(self, filename411):
"""Translates 411 file names into UNIX absolute filenames."""
# Warning this is UNIX dependant
n = string.replace(filename411, ".", "/")
n = string.replace(n, "//", ".")
return os.path.normpath(os.path.join(self.rootdir, n))
def path411(self, filename):
"""Turns an absolute UNIX path into a 411 filename. Every
period ('.') is a filesystem delimeter (like /), and all
filenames are assumed to be absolute, to start with a /. A
literal period is coded as a double period ('..'). """
# Warning, this is UNIX dependant.
n = string.replace(filename[1:], ".", "..")
n = string.replace(n, "/", ".")
return n
def connect(self, master=None):
"""Opens a HTTP 1.1 connection to the first live master server.
Will use the master argument if specified, otherwise consults
internal master server list. This connection can service
multiple requests."""
if self.conn:
self.disconnect()
if master:
masters = [master]
else:
masters = self.getMasters()
if not masters:
raise ValueError, \
"We have no master servers to connect to."
conn = None
for master in masters:
# Split address into host & port
m = master.getAddress().split(':')
if len(m) == 2:
conn = HTTPConnection(m[0], int(m[1]))
else:
conn = HTTPConnection(m[0])
# Test the connection.
try:
# Set variable so that connection originates
# from privileged port only. This way, only
# root can initialize this connection
conn.privileged_port = True
conn.connect()
except:
# If we cannot connect, devalue this server.
master.decScore()
conn = None
continue
else:
# We pick the first master that accepts a
# connection.
self.master = master
master.incScore()
break
if conn:
self.conn = conn
else:
raise Error411, \
"Could not reach a master server. Masters: %s" \
% masters
def disconnect(self):
"""Closes the master HTTP connection. """
if self.conn:
self.conn.close()
self.conn = None
def get(self, file):
""" Retrives a 411 file. If arg is empty, a directory listing
dictionary is returned. If file is valid, returns the decrypted
contents and its associated meta data. """
if not file:
raise Error411, "I need a file to get"
# Allow you to get a full URL.
if file.count("http://"):
m = Master(file)
self.connect(m)
file = os.path.basename(file)
else:
# We use our internal configuration.
self.connect()
path = self.master.getDir() + file
self.conn.request('GET', urllib.quote(path))
headers = self.conn.getresponse()
status = headers['status']
reason = headers['reason']
if status != 200:
raise Error411, "Could not get file '%s%s': %s %s" % \
(self.master.getUrl(), file, status, reason)
contents = self.conn.read()
self.disconnect()
if file[-1] == '/': # A directory
return contents
else:
return self.decrypt(contents)
def find(self, path="/"):
"""Finds all relevant files to retrieve on a master server.
Returns dict indexed by file path containing mtime and size."""
self.files = {}
self.findHelper(path)
return self.files
def findHelper(self, path, depth=0):
listing = self.get(path)
lines = listing.split("\n")
for line in lines:
m = self.dir_pattern.search(line)
if not m:
continue
#print line
filename = m.group('filename')
if filename == "Parent Directory":
continue
if filename[-1] == '/':
if self.verbose:
print "Found directory %s (%s)" \
% (path+filename, depth)
if self.isInteresting(path + filename, depth):
self.findHelper(path + filename, depth + 1)
continue
date = m.group('date').strip()
size = m.group('size').strip()
self.files[path + filename] = {
"Name": filename,
"Modified": date,
"Size": size
}
def isInteresting(self, path, level):
"""Matches an offered group our registered groups. Slow,
search time is squared with the number of registered groups.
"""
offered = path[1:-1].split('/')
elements = len(offered)
for r in self.groups:
if not r:
continue
if self.verbose:
print "Matching %s to %s level %s" \
% (offered, r[:len(offered)], level)
try:
# Shorten the registered group to match
# the offerred one, then compare. Allows
# group inheritance.
if offered == r[:len(offered)]:
return 1
except:
continue
return 0
def readKeys(self):
"""Loads the 411 shared and master RSA keys"""
pub_file = open(self.pub_filename, 'r')
self.pub = POW.pemRead(POW.RSA_PUBLIC_KEY, pub_file.read())
pub_file.close()
shared_file = open(self.shared_filename, 'r')
self.shared = self.readSharedKey(shared_file.read())
shared_file.close()
if os.path.exists(self.priv_filename):
priv_file = open(self.priv_filename, 'r')
self.priv = POW.pemRead(POW.RSA_PRIVATE_KEY, priv_file.read())
priv_file.close()
def makeSharedKey(self):
"""Uses our cryptographically safe random number generator to
give us a 256bit session key and 64bit Init Vector, in 411 format."""
if not self.pool:
self.fillPool()
while 1:
self.pool.stir()
randomkey = self.pool.get_bytes(40)
# First 256 bits are for the session key
sessionkey = randomkey[:32]
# Last 64 bits are for the CBC initial value.
initialvalue = randomkey[-8:]
try:
self.sym.encryptInit(sessionkey, initialvalue)
except TypeError:
# Need a new session key (null chars in our key)
continue
else:
break
key = "-----BEGIN 411 SHARED KEY-----\n"
key += base64.encodestring(randomkey)
key += "-----END 411 SHARED KEY-----\n"
return key
def readSharedKey(self, key64):
"""Read a 411 shared key in base64 encoding and return the
binary bits"""
header = "-----BEGIN 411 SHARED KEY-----\n"
footer = "-----END 411 SHARED KEY-----\n"
try:
a = key64.index(header) + len(header)
b = key64.index(footer)
key = base64.decodestring(key64[a:b])
except:
raise Error411, \
"This does not appear to be a 411 shared key."
return key
def encrypt(self,
plaintext,
header="-----BEGIN 411 MESSAGE-----\n",
footer="-----END 411 MESSAGE-----\n",
sign=1):
"""Encrypts the plain text message using a hybrid cryptography
technique: a 256-bit random session key is encrypted with the
cluster shared key. The session key is used to quickly encrypt
the message with the Blowfish symmetrical algorithm."""
if not self.shared:
self.readKeys()
# First 256 bits are for the session key,
# Last 64 bits are for the CBC initial value.
try:
self.sym.encryptInit(self.shared[:32], self.shared[-8:])
except TypeError:
raise Error411, "Invalid Shared Key"
# Sign the text with the master private key
if sign:
if not self.priv:
raise Error411, "I need the master private key to sign messages"
sig = self.sign(plaintext)
else:
sig = "Not Signed"
# Encrypt the text with Blowfish for speed.
ciphertext = self.sym.update(plaintext) + self.sym.final()
ciphertext_base64 = base64.encodestring(ciphertext)
# Message format (v2.0):
# digital signature
# <blank line>
# symmetrically-encrypted message
msg = header
msg += sig + "\n"
msg += ciphertext_base64
msg += footer
return msg
def decrypt(self,
contents,
header="-----BEGIN 411 MESSAGE-----\n",
footer="-----END 411 MESSAGE-----\n",
type411=1):
"""Uses the shared key to read 411 messages. For 411
type messages, returns the tuple (contents, meta) where meta is
a dictionary containing the 411 headers. If not type 411,
returns a (plaintext, sig_base64). No verification of signature
is performed."""
if not self.sym:
self.fillPool()
ciphersig_base64 = ''
try:
a = string.index(contents, header) + len(header)
b = string.index(contents, footer)
msg = contents[a:b]
ciphersig_base64, ciphertext_base64 = msg.split('\n\n')
ciphertext = base64.decodestring(ciphertext_base64)
except:
raise Error411, \
"This file does not appear to be in 411 format."
if not self.shared:
self.readKeys()
sessionkey = self.shared[:32]
initialvalue = self.shared[-8:]
self.sym.decryptInit(sessionkey, initialvalue)
try:
text = self.sym.update(ciphertext) + self.sym.final()
except POW.SSLError:
raise Error411, "Could not decrypt file, wrong key?"
if type411:
if not self.verify(text, ciphersig_base64):
raise Error411, "Signature does not verify."
return self.decode(text)
else:
return (text, ciphersig_base64)
def decode(self, plaintext):
meta = {}
p = Parser(plaintext, self.attrs)
meta = p.get_filtered_content()
self.plugin = p.get_plugin()
return meta['content'], meta
def verify(self, msg, sig_base64):
"""Verifies that the plaintext message was signed with the
(base64 encoded) signature, and has not been altered since
signing. Returns true if message verifies."""
if not self.pub:
try:
self.readKeys()
except IOError, e:
syslog.syslog(syslog.LOG_ERR, '411-error: ' \
+ str(e))
return 0
digest = POW.Digest(POW.MD5_DIGEST)
digest.update(msg)
try:
sig = base64.decodestring(sig_base64)
except:
return 0
return self.pub.verify(sig, digest.digest(), POW.MD5_DIGEST)