def _read_data(self, data):
def process(data):
if self.decryptor is not None:
eof = not data
if self.decoder_data_bytes and data:
self.buffered += ''.join([c for c in data if c not in
(' ', '\t', '\r', '\n')])
else:
self.buffered += (data or '')
data = ''
for i in (256, 64, 8, 1):
batch = max(1, i * self.decoder_data_bytes)
while eof or (len(self.buffered) >= batch):
if self.decoder_data_bytes:
d = self.buffered[:batch]
b = self.buffered[batch:]
self.buffered = b
else:
d, self.buffered = self.buffered, ''
try:
data += self.decryptor(self.decoder(d))
eof = False
except TypeError:
raise IOError('%s: Bad data, failed to decode'
% self.name)
return (data or '')
if data is None:
# EOF!
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
data, self.buffered = self.buffered, ''
return process(data) + process(None)
return process(None)
if self.expected_outer_sha256:
# The outer MD5 sum is calculated over all data, but with any
# CRLF sequences normalized to only LF and the sha256 header
# itself replaced with a placeholder.
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
sum_data = re.sub(MD5_SUM_RE, MD5_SUM_PLACEHOLDER,
re.sub(SHA_256_RE, SHA_256_PLACEHOLDER, data))
else:
sum_data = data
sum_data = sum_data.replace('\r', '').replace('\n', '\r\n')
self.outer_sha.update(sum_data)
if self.state in (
self.STATE_RAW_DATA, self.STATE_PGP_DATA, self.STATE_ONLY_DATA):
return process(data)
if self.state == self.STATE_BEGIN:
self.buffered += data
if (len(self.buffered) >= len(self.BEGIN_PGP)
and self.buffered.startswith(self.BEGIN_PGP)):
self.state = self.STATE_PGP_DATA
if self.gpg_pass:
self.gpg_pass.seek(0, 0)
passphrase, c = [], self.gpg_pass.read(1)
while c != '':
passphrase.append(c)
c = self.gpg_pass.read(1)
self.startup_lock.release()
return ''.join(passphrase + ['\n', self.buffered])
else:
self.startup_lock.release()
return self.buffered
# Note: The max() check is OK, because both formats add more
# data which covers the difference.
if len(self.buffered) >= max(len(self.BEGIN_MED),
len(self.BEGIN_MED2)):
if not (self.buffered.startswith(self.BEGIN_MED) or
self.buffered.startswith(self.BEGIN_MED2)):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
return self.buffered
if '\r\n\r\n' in self.buffered:
header, data = self.buffered.split('\r\n\r\n', 1)
headlines = header.strip().split('\r\n')
self.state = self.STATE_HEADER
elif '\n\n' in self.buffered:
header, data = self.buffered.split('\n\n', 1)
headlines = header.strip().split('\n')
self.state = self.STATE_HEADER
else:
return ''
else:
return ''
if self.state == self.STATE_HEADER:
# State: header and data have been set, header is complete.
self.buffered = ''
headers = dict([l.split(': ', 1) for l in headlines if ': ' in l])
nonce = headers.get('nonce', '')
self.mep_mutated = self._mutate_key(self.mep_key, nonce)
self.mep_version = headers.get('X-Mailpile-Encrypted-Data', 'v1')
self.cipher = headers.get('cipher', self.cipher)
data_fmt = '%s:%s' % (self.mep_version, self.cipher)
if data_fmt != PREFERRED_FORMAT:
DETECTED_OBSOLETE_FORMATS.add(data_fmt)
eim = self.expected_inner_md5sum = headers.get('md5sum')
if eim and eim == ('0' * LEN_MD5_SUM):
self.expected_inner_md5sum = None
if self.expected_inner_md5sum:
self.inner_md5.update(self.mep_mutated)
self.inner_md5.update(nonce)
eis = self.expected_inner_sha256 = headers.get('sha256')
if eis and eis == ('0' * LEN_SHA_256):
self.expected_inner_sha256 = None
if self.expected_inner_sha256:
self.inner_sha.update(self.mep_mutated)
self.inner_sha.update(nonce)
if self.buffered.startswith(self.BEGIN_MED2):
self.state = self.STATE_ONLY_DATA
else:
self.state = self.STATE_DATA
if self.cipher == 'aes-128-ctr':
self.decryptor = aes_ctr_decryptor(self.mep_mutated, nonce)
self.decoder = base64.b64decode
# Decode data in chunks this big (multiple of 4 and 16);
# guarantees workable chunks for both AES-CTR and base64.
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'none':
self.decryptor = lambda d: d
self.decoder = base64.b64decode
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'broken':
self.decryptor = lambda d: d
self.decoder = lambda d: d
self.expected_inner_md5sum = None
self.expected_inner_sha256 = None
else:
self.decryptor = None
data = '\n'.join((self.mep_mutated, data))
self.startup_lock.release()
if self.state == self.STATE_ONLY_DATA:
return process(data)
if self.state == self.STATE_DATA:
for delim in (self.END_MED, self.END_PGP):
if delim in data:
for pf in ('\r\n', '\n', ''):
if pf + delim in data:
data = data.split(pf + delim, 1)[0]
self.state = self.STATE_END
return process(data)
return process(data)
# Error, end and unknown states...
return ''
def _read_data(self, data):
def process(data):
if self.decryptor is not None:
eof = not data
if self.decoder_data_bytes and data:
self.buffered += ''.join(
[c for c in data if c not in (' ', '\t', '\r', '\n')])
else:
self.buffered += (data or '')
data = ''
for i in (256, 64, 8, 1):
batch = max(1, i * self.decoder_data_bytes)
while eof or (len(self.buffered) >= batch):
if self.decoder_data_bytes:
d = self.buffered[:batch]
b = self.buffered[batch:]
self.buffered = b
else:
d, self.buffered = self.buffered, ''
try:
data += self.decryptor(self.decoder(d))
eof = False
except TypeError:
raise IOError('%s: Bad data, failed to decode' %
self.name)
return (data or '')
if data is None:
# EOF!
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
data, self.buffered = self.buffered, ''
return process(data) + process(None)
return process(None)
if self.expected_outer_sha256:
# The outer MD5 sum is calculated over all data, but with any
# CRLF sequences normalized to only LF and the sha256 header
# itself replaced with a placeholder.
if self.state in (self.STATE_BEGIN, self.STATE_HEADER):
sum_data = re.sub(
MD5_SUM_RE, MD5_SUM_PLACEHOLDER,
re.sub(SHA_256_RE, SHA_256_PLACEHOLDER, data))
else:
sum_data = data
sum_data = sum_data.replace('\r', '').replace('\n', '\r\n')
self.outer_sha.update(sum_data)
if self.state in (self.STATE_RAW_DATA, self.STATE_PGP_DATA,
self.STATE_ONLY_DATA):
return process(data)
if self.state == self.STATE_BEGIN:
self.buffered += data
if (len(self.buffered) >= len(self.BEGIN_PGP)
and self.buffered.startswith(self.BEGIN_PGP)):
self.state = self.STATE_PGP_DATA
if self.gpg_pass:
self.gpg_pass.seek(0, 0)
passphrase, c = [], self.gpg_pass.read(1)
while c != '':
passphrase.append(c)
c = self.gpg_pass.read(1)
self.startup_lock.release()
return ''.join(passphrase + ['\n', self.buffered])
else:
self.startup_lock.release()
return self.buffered
# Note: The max() check is OK, because both formats add more
# data which covers the difference.
if len(self.buffered) >= max(len(self.BEGIN_MED),
len(self.BEGIN_MED2)):
if not (self.buffered.startswith(self.BEGIN_MED)
or self.buffered.startswith(self.BEGIN_MED2)):
self.state = self.STATE_RAW_DATA
self.startup_lock.release()
return self.buffered
if '\r\n\r\n' in self.buffered:
header, data = self.buffered.split('\r\n\r\n', 1)
headlines = header.strip().split('\r\n')
self.state = self.STATE_HEADER
elif '\n\n' in self.buffered:
header, data = self.buffered.split('\n\n', 1)
headlines = header.strip().split('\n')
self.state = self.STATE_HEADER
else:
return ''
else:
return ''
if self.state == self.STATE_HEADER:
# State: header and data have been set, header is complete.
self.buffered = ''
headers = dict([l.split(': ', 1) for l in headlines if ': ' in l])
nonce = headers.get('nonce', '')
self.mep_mutated = self._mutate_key(self.mep_key, nonce)
self.mep_version = headers.get('X-Mailpile-Encrypted-Data', 'v1')
self.cipher = headers.get('cipher', self.cipher)
data_fmt = '%s:%s' % (self.mep_version, self.cipher)
if data_fmt != PREFERRED_FORMAT:
DETECTED_OBSOLETE_FORMATS.add(data_fmt)
eim = self.expected_inner_md5sum = headers.get('md5sum')
if eim and eim == ('0' * LEN_MD5_SUM):
self.expected_inner_md5sum = None
if self.expected_inner_md5sum:
self.inner_md5.update(self.mep_mutated)
self.inner_md5.update(nonce)
eis = self.expected_inner_sha256 = headers.get('sha256')
if eis and eis == ('0' * LEN_SHA_256):
self.expected_inner_sha256 = None
if self.expected_inner_sha256:
self.inner_sha.update(self.mep_mutated)
self.inner_sha.update(nonce)
if self.buffered.startswith(self.BEGIN_MED2):
self.state = self.STATE_ONLY_DATA
else:
self.state = self.STATE_DATA
if self.cipher == 'aes-128-ctr':
self.decryptor = aes_ctr_decryptor(self.mep_mutated, nonce)
self.decoder = base64.b64decode
# Decode data in chunks this big (multiple of 4 and 16);
# guarantees workable chunks for both AES-CTR and base64.
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'none':
self.decryptor = lambda d: d
self.decoder = base64.b64decode
self.decoder_data_bytes = 32 * 1024
elif self.cipher == 'broken':
self.decryptor = lambda d: d
self.decoder = lambda d: d
self.expected_inner_md5sum = None
self.expected_inner_sha256 = None
else:
self.decryptor = None
data = '\n'.join((self.mep_mutated, data))
self.startup_lock.release()
if self.state == self.STATE_ONLY_DATA:
return process(data)
if self.state == self.STATE_DATA:
for delim in (self.END_MED, self.END_PGP):
if delim in data:
for pf in ('\r\n', '\n', ''):
if pf + delim in data:
data = data.split(pf + delim, 1)[0]
self.state = self.STATE_END
return process(data)
return process(data)
# Error, end and unknown states...
return ''