class DecryptingStreamer(InputCoprocess):
"""
This class creates a coprocess for decrypting data.
"""
BEGIN_PGP = "-----BEGIN PGP MESSAGE-----"
END_PGP = "-----END PGP MESSAGE-----"
BEGIN_MED = "-----BEGIN MAILPILE ENCRYPTED DATA-----"
BEGIN_MED2 = "X-Mailpile-Encrypted-Data: "
END_MED = "-----END MAILPILE ENCRYPTED DATA-----"
PREFERRED_CIPHER = None
STATE_BEGIN = 0
STATE_HEADER = 1
STATE_DATA = 2
STATE_ONLY_DATA = 3
STATE_END = 4
STATE_RAW_DATA = 5
STATE_PGP_DATA = 6
STATE_ERROR = -1
@classmethod
def StartEncrypted(cls, line):
return (line.startswith(cls.BEGIN_MED) or
line.startswith(cls.BEGIN_MED2) or
line.startswith(cls.BEGIN_PGP))
@classmethod
def EndEncrypted(cls, line):
return (line.startswith(cls.END_MED) or
line.startswith(cls.END_PGP))
def __init__(self, fd,
mep_key=None, gpg_pass=None, sha256=None, cipher=None,
name=None, long_running=False, gpgi=None, md_alg=None):
self.expected_outer_sha256 = sha256
self.expected_inner_sha256 = None
self.expected_inner_md5sum = None
self.name = name
self.outer_sha = hashlib.sha256()
self.inner_sha = hashlib.sha256()
self.inner_md5 = hashlib.md5()
self.cipher = self.PREFERRED_CIPHER or PREFERRED_CIPHER
self.md_alg = md_alg or OPENSSL_MD_ALG
self.state = self.STATE_BEGIN
self.buffered = ''
self.mep_version = None
self.mep_mutated = None
self.mep_key = mep_key
self.gpg_pass = gpg_pass
self.gpgi = gpgi
self.decryptor = None
self.decoder = None
self.decoder_data_bytes = 0 # Not counting white-space
# Start reading our data...
self.startup_lock = CryptoLock()
self.startup_lock.acquire()
self.data_filter = self._mk_data_filter(fd, self._read_data,
self.startup_lock.release)
self.read_fd = self.data_filter.reader()
try:
# Once the header has been processed (_read_data() will release
# the lock), fork out our coprocess.
self.startup_lock.acquire()
InputCoprocess.__init__(self, self._mk_command(), self.read_fd,
name=name, long_running=long_running)
self.startup_lock = None
except:
try:
self.data_filter.join(aborting=True)
self.data_filter.close()
except (IOError, OSError):
pass
raise
def _read_filter(self, data):
if data:
if self.expected_inner_sha256:
self.inner_sha.update(data)
if self.expected_inner_md5sum:
self.inner_md5.update(data)
return data
def close(self):
self.data_filter.join()
self.read_fd.close()
return InputCoprocess.close(self)
def verify(self, testing=False, _raise=None):
if self.close() != 0:
if testing:
print 'Close returned nonzero'
if _raise:
raise _raise('Non-zero exit code from coprocess')
return False
if self.expected_inner_sha256:
mac = mac_sha256(self.mep_mutated, self.inner_sha.digest())
if self.expected_inner_sha256 != mac:
if testing:
print 'Inner %s != %s' % (self.expected_inner_sha256, mac)
if _raise:
raise _raise('Invalid inner SHA256')
return False
elif self.expected_inner_md5sum:
if self.expected_inner_md5sum != self.inner_md5.hexdigest():
if testing:
print 'Inner %s != %s' % (self.expected_inner_md5sum,
self.inner_md5.hexdigest())
if _raise:
raise _raise('Invalid inner MD5 sum')
return False
elif testing and not self.expected_inner_md5sum:
print 'No inner MD5 sum or SHA256 expected'
if self.expected_outer_sha256:
mac = mac_sha256(self.mep_mutated, self.outer_sha.digest())
if self.expected_outer_sha256 != mac:
if testing:
print 'Outer %s != %s' % (self.expected_outer_sha256, mac)
if _raise:
raise _raise('Invalid outer SHA256')
return False
elif testing and not self.expected_outer_sha256:
print 'No outer SHA256 expected'
return True
def _mk_data_filter(self, fd, cb, ecb):
return IOFilter(fd, cb, error_callback=ecb,
name='%s/filter' % (self.name or 'ds'))
def _read_data(self, data):
def process(data):
if self.decryptor is not None:
eof = not data
if self.decoder_data_bytes and data:
self.buffered += ''.join([c for c in data if c not in
(' ', '\t', '\r', '\n')])
else:
self.buffered += (data or '')
data = ''
for i in (256, 64, 8, 1):
batch = max(1, i * self.decoder_data_bytes)
while eof or (len(self.buffered) >= batch):
if self.decoder_data_bytes:
d = self.buffered[:batch]
b = self.buffered[batch:]
self.buffered = b
else:
d, self.buffered = self.buffered, ''
try:
data += self.decryptor(self.decoder(d))
eof = False
except TypeError:
raise IOError('%s: Bad data, failed to decode'
% self.name)
return (data or '')
if data is None:
# EOF!
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
data, self.buffered = self.buffered, ''
return process(data) + process(None)
return process(None)
if self.expected_outer_sha256:
# The outer MD5 sum is calculated over all data, but with any
# CRLF sequences normalized to only LF and the sha256 header
# itself replaced with a placeholder.
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
sum_data = re.sub(MD5_SUM_RE, MD5_SUM_PLACEHOLDER,
re.sub(SHA_256_RE, SHA_256_PLACEHOLDER, data))
else:
sum_data = data
sum_data = sum_data.replace('\r', '').replace('\n', '\r\n')
self.outer_sha.update(sum_data)
if self.state in (
self.STATE_RAW_DATA, self.STATE_PGP_DATA, self.STATE_ONLY_DATA):
return process(data)
if self.state == self.STATE_BEGIN:
self.buffered += data
if (len(self.buffered) >= len(self.BEGIN_PGP)
and self.buffered.startswith(self.BEGIN_PGP)):
self.state = self.STATE_PGP_DATA
if self.gpg_pass:
self.gpg_pass.seek(0, 0)
passphrase, c = [], self.gpg_pass.read(1)
while c != '':
passphrase.append(c)
c = self.gpg_pass.read(1)
self.startup_lock.release()
return ''.join(passphrase + ['\n', self.buffered])
else:
self.startup_lock.release()
return self.buffered
# Note: The max() check is OK, because both formats add more
# data which covers the difference.
if len(self.buffered) >= max(len(self.BEGIN_MED),
len(self.BEGIN_MED2)):
if not (self.buffered.startswith(self.BEGIN_MED) or
self.buffered.startswith(self.BEGIN_MED2)):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
return self.buffered
if '\r\n\r\n' in self.buffered:
header, data = self.buffered.split('\r\n\r\n', 1)
headlines = header.strip().split('\r\n')
self.state = self.STATE_HEADER
elif '\n\n' in self.buffered:
header, data = self.buffered.split('\n\n', 1)
headlines = header.strip().split('\n')
self.state = self.STATE_HEADER
else:
return ''
else:
return ''
if self.state == self.STATE_HEADER:
# State: header and data have been set, header is complete.
self.buffered = ''
headers = dict([l.split(': ', 1) for l in headlines if ': ' in l])
nonce = headers.get('nonce', '')
self.mep_mutated = self._mutate_key(self.mep_key, nonce)
self.mep_version = headers.get('X-Mailpile-Encrypted-Data', 'v1')
self.cipher = headers.get('cipher', self.cipher)
data_fmt = '%s:%s' % (self.mep_version, self.cipher)
if data_fmt != PREFERRED_FORMAT:
DETECTED_OBSOLETE_FORMATS.add(data_fmt)
eim = self.expected_inner_md5sum = headers.get('md5sum')
if eim and eim == ('0' * LEN_MD5_SUM):
self.expected_inner_md5sum = None
if self.expected_inner_md5sum:
self.inner_md5.update(self.mep_mutated)
self.inner_md5.update(nonce)
eis = self.expected_inner_sha256 = headers.get('sha256')
if eis and eis == ('0' * LEN_SHA_256):
self.expected_inner_sha256 = None
if self.expected_inner_sha256:
self.inner_sha.update(self.mep_mutated)
self.inner_sha.update(nonce)
if self.buffered.startswith(self.BEGIN_MED2):
self.state = self.STATE_ONLY_DATA
else:
self.state = self.STATE_DATA
if self.cipher == 'aes-128-ctr':
self.decryptor = aes_ctr_decryptor(self.mep_mutated, nonce)
self.decoder = base64.b64decode
# Decode data in chunks this big (multiple of 4 and 16);
# guarantees workable chunks for both AES-CTR and base64.
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'none':
self.decryptor = lambda d: d
self.decoder = base64.b64decode
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'broken':
self.decryptor = lambda d: d
self.decoder = lambda d: d
self.expected_inner_md5sum = None
self.expected_inner_sha256 = None
else:
self.decryptor = None
data = '\n'.join((self.mep_mutated, data))
self.startup_lock.release()
if self.state == self.STATE_ONLY_DATA:
return process(data)
if self.state == self.STATE_DATA:
for delim in (self.END_MED, self.END_PGP):
if delim in data:
for pf in ('\r\n', '\n', ''):
if pf + delim in data:
data = data.split(pf + delim, 1)[0]
self.state = self.STATE_END
return process(data)
return process(data)
# Error, end and unknown states...
return ''
def _mutate_key(self, key, nonce):
return genkey(key or '', nonce)[:32].strip()
def _mk_command(self):
if self.state == self.STATE_RAW_DATA:
return None
elif self.decryptor is not None:
return None
elif self.state == self.STATE_PGP_DATA:
safe_assert(self.gpgi is not None)
if self.gpg_pass:
return self.gpgi.common_args(will_send_passphrase=True)
else:
return self.gpgi.common_args()
return [OPENSSL_COMMAND(), "enc", "-d", "-a", "-%s" % self.cipher,
"-pass", "stdin", "-md", self.md_alg]
class DecryptingStreamer(InputCoprocess):
"""
This class creates a coprocess for decrypting data.
"""
BEGIN_PGP = "-----BEGIN PGP MESSAGE-----"
END_PGP = "-----END PGP MESSAGE-----"
BEGIN_MED = "-----BEGIN MAILPILE ENCRYPTED DATA-----\n"
BEGIN_MED2 = "X-Mailpile-Encrypted-Data: "
END_MED = "-----END MAILPILE ENCRYPTED DATA-----\n"
DEFAULT_CIPHER = "aes-256-cbc"
STATE_BEGIN = 0
STATE_HEADER = 1
STATE_DATA = 2
STATE_ONLY_DATA = 3
STATE_END = 4
STATE_RAW_DATA = 5
STATE_PGP_DATA = 6
STATE_ERROR = -1
@classmethod
def StartEncrypted(cls, line):
return (line.startswith(cls.BEGIN_MED[:-1]) or
line.startswith(cls.BEGIN_MED2) or
line.startswith(cls.BEGIN_PGP[:-1]))
@classmethod
def EndEncrypted(cls, line):
return (line.startswith(cls.END_MED[:-1]) or
line.startswith(cls.END_PGP[:-1]))
def __init__(self, fd,
mep_key=None, gpg_pass=None, md5sum=None, cipher=None,
name=None, long_running=False):
self.expected_outer_md5sum = md5sum
self.expected_inner_md5sum = None
self.name = name
self.outer_md5 = hashlib.md5()
self.inner_md5 = hashlib.md5()
self.cipher = self.DEFAULT_CIPHER
self.state = self.STATE_BEGIN
self.buffered = ''
self.mep_key = mep_key
self.gpg_pass = gpg_pass
# Start reading our data...
self.startup_lock = CryptoLock()
self.startup_lock.acquire()
self.data_filter = self._mk_data_filter(fd, self._read_data,
self.startup_lock.release)
self.read_fd = self.data_filter.reader()
try:
# Once the header has been processed (_read_data() will release
# the lock), fork out our coprocess.
self.startup_lock.acquire()
InputCoprocess.__init__(self, self._mk_command(), self.read_fd,
name=name, long_running=long_running)
self.startup_lock = None
except:
try:
self.data_filter.join(aborting=True)
self.data_filter.close()
except (IOError, OSError):
pass
raise
def _read_filter(self, data):
if data:
self.inner_md5.update(data)
return data
def close(self):
self.data_filter.join()
self.read_fd.close()
return InputCoprocess.close(self)
def verify(self, testing=False, _raise=None):
if self.close() != 0:
if testing:
print 'Close returned nonzero'
if _raise:
raise _raise('Non-zero exit code from coprocess')
return False
if (self.expected_inner_md5sum and
self.expected_inner_md5sum != self.inner_md5.hexdigest()):
if testing:
print 'Inner %s != %s' % (self.expected_inner_md5sum,
self.inner_md5.hexdigest())
if _raise:
raise _raise('Invalid inner MD5 sum')
return False
elif testing and not self.expected_inner_md5sum:
print 'No inner MD5 sum expected'
if (self.expected_outer_md5sum and
self.expected_outer_md5sum != self.outer_md5.hexdigest()):
if testing:
print 'Outer %s != %s' % (self.expected_outer_md5sum,
self.outer_md5.hexdigest())
if _raise:
raise _raise('Invalid outer MD5 sum')
return False
elif testing and not self.expected_outer_md5sum:
print 'No outer MD5 sum expected'
return True
def _mk_data_filter(self, fd, cb, ecb):
return IOFilter(fd, cb, error_callback=ecb,
name='%s/filter' % (self.name or 'ds'))
def _read_data(self, data):
if data is None:
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
data, self.buffered = self.buffered, ''
return data
return ''
if self.expected_outer_md5sum:
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
sum_data = re.sub(MD5_SUM_RE, MD5_SUM_PLACEHOLDER, data)
else:
sum_data = data
sum_data = sum_data.replace('\r', '').replace('\n', '\r\n')
self.outer_md5.update(sum_data)
if self.state in (self.STATE_RAW_DATA, self.STATE_PGP_DATA):
return data
if self.state == self.STATE_BEGIN:
self.buffered += data
if (len(self.buffered) >= len(self.BEGIN_PGP)
and self.buffered.startswith(self.BEGIN_PGP)):
self.state = self.STATE_PGP_DATA
if self.gpg_pass:
passphrase, c = [], self.gpg_pass.read(1)
while c != '':
passphrase.append(c)
c = self.gpg_pass.read(1)
self.startup_lock.release()
return ''.join(passphrase + ['\n', self.buffered])
else:
self.startup_lock.release()
return self.buffered
# Note: The max() check is OK, because both formats add more
# data which covers the difference.
if len(self.buffered) >= max(len(self.BEGIN_MED),
len(self.BEGIN_MED2)):
if not (self.buffered.startswith(self.BEGIN_MED) or
self.buffered.startswith(self.BEGIN_MED2)):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
return self.buffered
if '\r\n\r\n' in self.buffered:
header, data = self.buffered.split('\r\n\r\n', 1)
headlines = header.strip().split('\r\n')
self.state = self.STATE_HEADER
elif '\n\n' in self.buffered:
header, data = self.buffered.split('\n\n', 1)
headlines = header.strip().split('\n')
self.state = self.STATE_HEADER
else:
return ''
else:
return ''
if self.state == self.STATE_HEADER:
headers = dict([l.split(': ', 1) for l in headlines[1:]])
nonce = headers.get('nonce', '')
mutated = self._mutate_key(self.mep_key, nonce)
self.cipher = headers.get('cipher', self.cipher)
eim = self.expected_inner_md5sum = headers.get('md5sum')
if eim == '00000000000000000000000000000000':
self.expected_inner_md5sum = None
self.inner_md5.update(mutated)
self.inner_md5.update(nonce)
data = '\n'.join((mutated, data))
if self.buffered.startswith(self.BEGIN_MED2):
self.state = self.STATE_ONLY_DATA
else:
self.state = self.STATE_DATA
self.startup_lock.release()
if self.state == self.STATE_ONLY_DATA:
return data
if self.state == self.STATE_DATA:
if '\n\n-' in data:
data = data.split('\n\n-', 1)[0]
self.state = self.STATE_END
elif '\r\n\r\n-' in data:
data = data.split('\r\n\r\n-', 1)[0]
self.state = self.STATE_END
return data
# Error, end and unknown states...
return ''
def _mutate_key(self, key, nonce):
return genkey(key or '', nonce)[:32].strip()
def _mk_command(self):
if self.state == self.STATE_RAW_DATA:
return None
elif self.state == self.STATE_PGP_DATA:
gpg = [GPG_BINARY, "--batch"]
if self.gpg_pass:
gpg.extend(["--no-use-agent", "--passphrase-fd=0"])
return gpg
return [OPENSSL_COMMAND, "enc", "-d", "-a", "-%s" % self.cipher,
"-pass", "stdin"]
class DecryptingStreamer(InputCoprocess):
"""
This class creates a coprocess for decrypting data.
"""
BEGIN_PGP = "-----BEGIN PGP MESSAGE-----"
END_PGP = "-----END PGP MESSAGE-----"
BEGIN_MED = "-----BEGIN MAILPILE ENCRYPTED DATA-----"
BEGIN_MED2 = "X-Mailpile-Encrypted-Data: "
END_MED = "-----END MAILPILE ENCRYPTED DATA-----"
PREFERRED_CIPHER = None
STATE_BEGIN = 0
STATE_HEADER = 1
STATE_DATA = 2
STATE_ONLY_DATA = 3
STATE_END = 4
STATE_RAW_DATA = 5
STATE_PGP_DATA = 6
STATE_ERROR = -1
@classmethod
def StartEncrypted(cls, line):
return (line.startswith(cls.BEGIN_MED)
or line.startswith(cls.BEGIN_MED2)
or line.startswith(cls.BEGIN_PGP))
@classmethod
def EndEncrypted(cls, line):
return (line.startswith(cls.END_MED) or line.startswith(cls.END_PGP))
def __init__(self,
fd,
mep_key=None,
gpg_pass=None,
sha256=None,
cipher=None,
name=None,
long_running=False,
gpgi=None,
md_alg=None):
self.expected_outer_sha256 = sha256
self.expected_inner_sha256 = None
self.expected_inner_md5sum = None
self.name = name
self.outer_sha = hashlib.sha256()
self.inner_sha = hashlib.sha256()
self.inner_md5 = hashlib.md5()
self.cipher = self.PREFERRED_CIPHER or PREFERRED_CIPHER
self.md_alg = md_alg or OPENSSL_MD_ALG
self.state = self.STATE_BEGIN
self.buffered = ''
self.mep_version = None
self.mep_mutated = None
self.mep_key = mep_key
self.gpg_pass = gpg_pass
self.gpgi = gpgi
self.decryptor = None
self.decoder = None
self.decoder_data_bytes = 0 # Not counting white-space
# Start reading our data...
self.startup_lock = CryptoLock()
self.startup_lock.acquire()
self.data_filter = self._mk_data_filter(fd, self._read_data,
self.startup_lock.release)
self.read_fd = self.data_filter.reader()
try:
# Once the header has been processed (_read_data() will release
# the lock), fork out our coprocess.
self.startup_lock.acquire()
InputCoprocess.__init__(self,
self._mk_command(),
self.read_fd,
name=name,
long_running=long_running)
self.startup_lock = None
except:
try:
self.data_filter.join(aborting=True)
self.data_filter.close()
except (IOError, OSError):
pass
raise
def _read_filter(self, data):
if data:
if self.expected_inner_sha256:
self.inner_sha.update(data)
if self.expected_inner_md5sum:
self.inner_md5.update(data)
return data
def close(self):
self.data_filter.join()
self.read_fd.close()
return InputCoprocess.close(self)
def verify(self, testing=False, _raise=None):
if self.close() != 0:
if testing:
print 'Close returned nonzero'
if _raise:
raise _raise('Non-zero exit code from coprocess')
return False
if self.expected_inner_sha256:
mac = mac_sha256(self.mep_mutated, self.inner_sha.digest())
if self.expected_inner_sha256 != mac:
if testing:
print 'Inner %s != %s' % (self.expected_inner_sha256, mac)
if _raise:
raise _raise('Invalid inner SHA256')
return False
elif self.expected_inner_md5sum:
if self.expected_inner_md5sum != self.inner_md5.hexdigest():
if testing:
print 'Inner %s != %s' % (self.expected_inner_md5sum,
self.inner_md5.hexdigest())
if _raise:
raise _raise('Invalid inner MD5 sum')
return False
elif testing and not self.expected_inner_md5sum:
print 'No inner MD5 sum or SHA256 expected'
if self.expected_outer_sha256:
mac = mac_sha256(self.mep_mutated, self.outer_sha.digest())
if self.expected_outer_sha256 != mac:
if testing:
print 'Outer %s != %s' % (self.expected_outer_sha256, mac)
if _raise:
raise _raise('Invalid outer SHA256')
return False
elif testing and not self.expected_outer_sha256:
print 'No outer SHA256 expected'
return True
def _mk_data_filter(self, fd, cb, ecb):
return IOFilter(fd,
cb,
error_callback=ecb,
name='%s/filter' % (self.name or 'ds'))
def _read_data(self, data):
def process(data):
if self.decryptor is not None:
eof = not data
if self.decoder_data_bytes and data:
self.buffered += ''.join(
[c for c in data if c not in (' ', '\t', '\r', '\n')])
else:
self.buffered += (data or '')
data = ''
for i in (256, 64, 8, 1):
batch = max(1, i * self.decoder_data_bytes)
while eof or (len(self.buffered) >= batch):
if self.decoder_data_bytes:
d = self.buffered[:batch]
b = self.buffered[batch:]
self.buffered = b
else:
d, self.buffered = self.buffered, ''
try:
data += self.decryptor(self.decoder(d))
eof = False
except TypeError:
raise IOError('%s: Bad data, failed to decode' %
self.name)
return (data or '')
if data is None:
# EOF!
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
data, self.buffered = self.buffered, ''
return process(data) + process(None)
return process(None)
if self.expected_outer_sha256:
# The outer MD5 sum is calculated over all data, but with any
# CRLF sequences normalized to only LF and the sha256 header
# itself replaced with a placeholder.
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
sum_data = re.sub(
MD5_SUM_RE, MD5_SUM_PLACEHOLDER,
re.sub(SHA_256_RE, SHA_256_PLACEHOLDER, data))
else:
sum_data = data
sum_data = sum_data.replace('\r', '').replace('\n', '\r\n')
self.outer_sha.update(sum_data)
if self.state in (self.STATE_RAW_DATA, self.STATE_PGP_DATA,
self.STATE_ONLY_DATA):
return process(data)
if self.state == self.STATE_BEGIN:
self.buffered += data
if (len(self.buffered) >= len(self.BEGIN_PGP)
and self.buffered.startswith(self.BEGIN_PGP)):
self.state = self.STATE_PGP_DATA
if self.gpg_pass:
self.gpg_pass.seek(0, 0)
passphrase, c = [], self.gpg_pass.read(1)
while c != '':
passphrase.append(c)
c = self.gpg_pass.read(1)
self.startup_lock.release()
return ''.join(passphrase + ['\n', self.buffered])
else:
self.startup_lock.release()
return self.buffered
# Note: The max() check is OK, because both formats add more
# data which covers the difference.
if len(self.buffered) >= max(len(self.BEGIN_MED),
len(self.BEGIN_MED2)):
if not (self.buffered.startswith(self.BEGIN_MED)
or self.buffered.startswith(self.BEGIN_MED2)):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
return self.buffered
if '\r\n\r\n' in self.buffered:
header, data = self.buffered.split('\r\n\r\n', 1)
headlines = header.strip().split('\r\n')
self.state = self.STATE_HEADER
elif '\n\n' in self.buffered:
header, data = self.buffered.split('\n\n', 1)
headlines = header.strip().split('\n')
self.state = self.STATE_HEADER
else:
return ''
else:
return ''
if self.state == self.STATE_HEADER:
# State: header and data have been set, header is complete.
self.buffered = ''
headers = dict([l.split(': ', 1) for l in headlines if ': ' in l])
nonce = headers.get('nonce', '')
self.mep_mutated = self._mutate_key(self.mep_key, nonce)
self.mep_version = headers.get('X-Mailpile-Encrypted-Data', 'v1')
self.cipher = headers.get('cipher', self.cipher)
data_fmt = '%s:%s' % (self.mep_version, self.cipher)
if data_fmt != PREFERRED_FORMAT:
DETECTED_OBSOLETE_FORMATS.add(data_fmt)
eim = self.expected_inner_md5sum = headers.get('md5sum')
if eim and eim == ('0' * LEN_MD5_SUM):
self.expected_inner_md5sum = None
if self.expected_inner_md5sum:
self.inner_md5.update(self.mep_mutated)
self.inner_md5.update(nonce)
eis = self.expected_inner_sha256 = headers.get('sha256')
if eis and eis == ('0' * LEN_SHA_256):
self.expected_inner_sha256 = None
if self.expected_inner_sha256:
self.inner_sha.update(self.mep_mutated)
self.inner_sha.update(nonce)
if self.buffered.startswith(self.BEGIN_MED2):
self.state = self.STATE_ONLY_DATA
else:
self.state = self.STATE_DATA
if self.cipher == 'aes-128-ctr':
self.decryptor = aes_ctr_decryptor(self.mep_mutated, nonce)
self.decoder = base64.b64decode
# Decode data in chunks this big (multiple of 4 and 16);
# guarantees workable chunks for both AES-CTR and base64.
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'none':
self.decryptor = lambda d: d
self.decoder = base64.b64decode
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'broken':
self.decryptor = lambda d: d
self.decoder = lambda d: d
self.expected_inner_md5sum = None
self.expected_inner_sha256 = None
else:
self.decryptor = None
data = '\n'.join((self.mep_mutated, data))
self.startup_lock.release()
if self.state == self.STATE_ONLY_DATA:
return process(data)
if self.state == self.STATE_DATA:
for delim in (self.END_MED, self.END_PGP):
if delim in data:
for pf in ('\r\n', '\n', ''):
if pf + delim in data:
data = data.split(pf + delim, 1)[0]
self.state = self.STATE_END
return process(data)
return process(data)
# Error, end and unknown states...
return ''
def _mutate_key(self, key, nonce):
return genkey(key or '', nonce)[:32].strip()
def _mk_command(self):
if self.state == self.STATE_RAW_DATA:
return None
elif self.decryptor is not None:
return None
elif self.state == self.STATE_PGP_DATA:
safe_assert(self.gpgi is not None)
if self.gpg_pass:
return self.gpgi.common_args(will_send_passphrase=True)
else:
return self.gpgi.common_args()
return [
OPENSSL_COMMAND(), "enc", "-d", "-a",
"-%s" % self.cipher, "-pass", "stdin", "-md", self.md_alg
]
class DecryptingStreamer(InputCoprocess):
"""
This class creates a coprocess for decrypting data.
"""
BEGIN_PGP = "-----BEGIN PGP MESSAGE-----"
END_PGP = "-----END PGP MESSAGE-----"
BEGIN_MED = "-----BEGIN MAILPILE ENCRYPTED DATA-----\n"
BEGIN_MED2 = "X-Mailpile-Encrypted-Data: "
END_MED = "-----END MAILPILE ENCRYPTED DATA-----\n"
DEFAULT_CIPHER = "aes-256-cbc"
STATE_BEGIN = 0
STATE_HEADER = 1
STATE_DATA = 2
STATE_ONLY_DATA = 3
STATE_END = 4
STATE_RAW_DATA = 5
STATE_PGP_DATA = 6
STATE_ERROR = -1
@classmethod
def StartEncrypted(cls, line):
return (line.startswith(cls.BEGIN_MED[:-1]) or
line.startswith(cls.BEGIN_MED2) or
line.startswith(cls.BEGIN_PGP[:-1]))
@classmethod
def EndEncrypted(cls, line):
return (line.startswith(cls.END_MED[:-1]) or
line.startswith(cls.END_PGP[:-1]))
def __init__(self, fd,
mep_key=None, gpg_pass=None, md5sum=None, cipher=None,
name=None, long_running=False):
self.expected_outer_md5sum = md5sum
self.expected_inner_md5sum = None
self.name = name
self.outer_md5 = hashlib.md5()
self.inner_md5 = hashlib.md5()
self.cipher = self.DEFAULT_CIPHER
self.state = self.STATE_BEGIN
self.buffered = ''
self.mep_key = mep_key
self.gpg_pass = gpg_pass
# Start reading our data...
self.startup_lock = CryptoLock()
self.startup_lock.acquire()
self.data_filter = self._mk_data_filter(fd, self._read_data,
self.startup_lock.release)
self.read_fd = self.data_filter.reader()
try:
# Once the header has been processed (_read_data() will release
# the lock), fork out our coprocess.
self.startup_lock.acquire()
InputCoprocess.__init__(self, self._mk_command(), self.read_fd,
name=name, long_running=long_running)
self.startup_lock = None
except:
try:
self.data_filter.join(aborting=True)
self.data_filter.close()
except (IOError, OSError):
pass
raise
def _read_filter(self, data):
if data:
self.inner_md5.update(data)
return data
def close(self):
self.data_filter.join()
self.read_fd.close()
return InputCoprocess.close(self)
def verify(self, testing=False, _raise=None):
if self.close() != 0:
if testing:
print 'Close returned nonzero'
if _raise:
raise _raise('Non-zero exit code from coprocess')
return False
if (self.expected_inner_md5sum and
self.expected_inner_md5sum != self.inner_md5.hexdigest()):
if testing:
print 'Inner %s != %s' % (self.expected_inner_md5sum,
self.inner_md5.hexdigest())
if _raise:
raise _raise('Invalid inner MD5 sum')
return False
elif testing and not self.expected_inner_md5sum:
print 'No inner MD5 sum expected'
if (self.expected_outer_md5sum and
self.expected_outer_md5sum != self.outer_md5.hexdigest()):
if testing:
print 'Outer %s != %s' % (self.expected_outer_md5sum,
self.outer_md5.hexdigest())
if _raise:
raise _raise('Invalid outer MD5 sum')
return False
elif testing and not self.expected_outer_md5sum:
print 'No outer MD5 sum expected'
return True
def _mk_data_filter(self, fd, cb, ecb):
return IOFilter(fd, cb, error_callback=ecb,
name='%s/filter' % (self.name or 'ds'))
def _read_data(self, data):
if data is None:
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
data, self.buffered = self.buffered, ''
return data
return ''
if self.expected_outer_md5sum:
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
sum_data = re.sub(MD5_SUM_RE, MD5_SUM_PLACEHOLDER, data)
else:
sum_data = data
sum_data = sum_data.replace('\r', '').replace('\n', '\r\n')
self.outer_md5.update(sum_data)
if self.state in (self.STATE_RAW_DATA, self.STATE_PGP_DATA):
return data
if self.state == self.STATE_BEGIN:
self.buffered += data
if (len(self.buffered) >= len(self.BEGIN_PGP)
and self.buffered.startswith(self.BEGIN_PGP)):
self.state = self.STATE_PGP_DATA
if self.gpg_pass:
passphrase, c = [], self.gpg_pass.read(1)
while c != '':
passphrase.append(c)
c = self.gpg_pass.read(1)
self.startup_lock.release()
return ''.join(passphrase + ['\n', self.buffered])
else:
self.startup_lock.release()
return self.buffered
# Note: The max() check is OK, because both formats add more
# data which covers the difference.
if len(self.buffered) >= max(len(self.BEGIN_MED),
len(self.BEGIN_MED2)):
if not (self.buffered.startswith(self.BEGIN_MED) or
self.buffered.startswith(self.BEGIN_MED2)):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
return self.buffered
if '\r\n\r\n' in self.buffered:
header, data = self.buffered.split('\r\n\r\n', 1)
headlines = header.strip().split('\r\n')
self.state = self.STATE_HEADER
elif '\n\n' in self.buffered:
header, data = self.buffered.split('\n\n', 1)
headlines = header.strip().split('\n')
self.state = self.STATE_HEADER
else:
return ''
else:
return ''
if self.state == self.STATE_HEADER:
headers = dict([l.split(': ', 1) for l in headlines[1:]])
nonce = headers.get('nonce', '')
mutated = self._mutate_key(self.mep_key, nonce)
self.cipher = headers.get('cipher', self.cipher)
eim = self.expected_inner_md5sum = headers.get('md5sum')
if eim == '00000000000000000000000000000000':
self.expected_inner_md5sum = None
self.inner_md5.update(mutated)
self.inner_md5.update(nonce)
data = '\n'.join((mutated, data))
if self.buffered.startswith(self.BEGIN_MED2):
self.state = self.STATE_ONLY_DATA
else:
self.state = self.STATE_DATA
self.startup_lock.release()
if self.state == self.STATE_ONLY_DATA:
return data
if self.state == self.STATE_DATA:
if '\n\n-' in data:
data = data.split('\n\n-', 1)[0]
self.state = self.STATE_END
elif '\r\n\r\n-' in data:
data = data.split('\r\n\r\n-', 1)[0]
self.state = self.STATE_END
return data
# Error, end and unknown states...
return ''
def _mutate_key(self, key, nonce):
return genkey(key or '', nonce)[:32].strip()
def _mk_command(self):
if self.state == self.STATE_RAW_DATA:
return None
elif self.state == self.STATE_PGP_DATA:
gpg = [GPG_BINARY, "--batch"]
if self.gpg_pass:
gpg.extend(["--no-use-agent", "--passphrase-fd=0"])
return gpg
return [OPENSSL_COMMAND, "enc", "-d", "-a", "-%s" % self.cipher,
"-pass", "stdin"]