def log_message(message): ''' Log a message to the local log. ''' global log if log is None: log = LogFile() log.write_and_flush(message)
def log_message(message): ''' Log a message. >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.utils.log')) True ''' global _log if _log is None: _log = LogFile() _log.write_and_flush(message)
def log_message(message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.sync_db_with_keyring.log')) True ''' global _log if _log is None: _log = LogFile() _log.write_and_flush(message)
def log_message(message): ''' Log a message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.user_keys.log')) True ''' global log if log is None: log = LogFile() log.write_and_flush(message)
def log_message(message): ''' Log a message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.options.log')) True ''' global log if log is None: log = LogFile() log.write_and_flush(message)
class EmailMessage(object): ''' Email Message. Messages should be converted to EmailMessage as soon as possible, to check whether the message is parsable as part of validating input. If a MIME message is not parsable, a new Message will be created that does conform and contains the original unparsable message in the body. ''' DEBUGGING = False def __init__(self, message_or_file=None): ''' Creates an EmailMessage from a Message or a file. Non-mime messages are converted to MIME "text/plain". >>> email_message = EmailMessage() >>> type(email_message) <class 'goodcrypto.mail.message.email_message.EmailMessage'> ''' self.bad_header_lines = [] self.parser = Parser() self._last_charset = constants.DEFAULT_CHAR_SET self._log = self._message = None if message_or_file is None: self._message = Message() elif isinstance(message_or_file, Message): self._message = message_or_file elif isinstance(message_or_file, EmailMessage): self._message = message_or_file.get_message() else: try: if isinstance(message_or_file, IOBase) or isinstance(message_or_file, StringIO): self.log_message('about to parse a message from a file') try: self._message = self.parser.parse(message_or_file) self.log_message('parsed message from file') except TypeError: message_or_file.seek(0, os.SEEK_SET) self.parser = BytesParser() self._message = self.parser.parse(message_or_file) self.log_message('parsed message from file as bytes') else: try: self.log_message('about to parse a message from a string') self._message = self.parser.parsestr(message_or_file) self.log_message('parsed message from string') except TypeError: self.parser = BytesParser() self._message = self.parser.parsebytes(message_or_file) self.log_message('parsed message from bytes') if not self.validate_message(): self._create_good_message_from_bad(message_or_file) except Exception: try: self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() self._create_good_message_from_bad(message_or_file) # if we still don't have a good message, then blow up if not self.validate_message(): self.log_message('unable to create a valid message') raise MessageException() except Exception: record_exception() if self.DEBUGGING: try: self.log_message(self.to_string()) except: pass def get_header(self, key): ''' Get a header from an existing message. >>> from goodcrypto_tests.mail.message_utils import get_encrypted_message_name >>> with open(get_encrypted_message_name('basic.txt')) as input_file: ... email_message = EmailMessage(input_file) ... crypto_software = email_message.get_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) >>> crypto_software == 'GPG' True ''' try: value = self.get_message().__getitem__(key) except Exception: value = None return value def add_header(self, key, value): ''' Add a header to an existing message. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('basic.txt')) as input_file: ... email_message = EmailMessage(input_file) ... email_message.add_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER, 'GPG') ... crypto_software = email_message.get_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) >>> crypto_software == 'GPG' True ''' self._message.__setitem__(key, value) def change_header(self, key, value): ''' Change a header to an existing message. >>> from goodcrypto_tests.mail.message_utils import get_encrypted_message_name >>> with open(get_encrypted_message_name('bouncy-castle.txt')) as input_file: ... email_message = EmailMessage(input_file) ... email_message.change_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER, 'TestGPG') ... crypto_software = email_message.get_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) >>> crypto_software == 'TestGPG' True ''' if key in self._message: self._message.replace_header(key, value) else: self.add_header(key, value) def delete_header(self, key): ''' Delete a header to an existing message. >>> from goodcrypto_tests.mail.message_utils import get_encrypted_message_name >>> with open(get_encrypted_message_name('bouncy-castle.txt')) as input_file: ... email_message = EmailMessage(input_file) ... email_message.delete_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) ... email_message.get_header(constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) is None True ''' self._message.__delitem__(key) def get_message(self): ''' Get the message. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> from goodcrypto.oce.test_constants import EDWARD_LOCAL_USER >>> email_message = get_basic_email_message() >>> email_message.get_message() is not None True >>> email_message.get_message().get(mime_constants.FROM_KEYWORD) == EDWARD_LOCAL_USER True ''' return self._message def set_message(self, new_message): ''' Set the new message. # Get a basic message first so we can avoid recursion >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> from goodcrypto.oce.test_constants import EDWARD_LOCAL_USER >>> basic_email_message = get_basic_email_message().get_message() >>> email_message = EmailMessage() >>> email_message.get_message().get(mime_constants.FROM_KEYWORD) is None True >>> email_message.set_message(basic_email_message) >>> email_message.get_message().get(mime_constants.FROM_KEYWORD) == EDWARD_LOCAL_USER True ''' old_message = self._message if is_string(new_message): try: if isinstance(self.parser, Parser): self._message = self.parser.parsestr(new_message) else: self._message = self.parser.parsebytes(new_message.encode()) except: self._message = old_message record_exception() else: self._message = new_message # restore the old message if the new one isn't valid. if not self.validate_message(): self._message = old_message self.log_message('restored previous message') def validate_message(self): ''' Validate a message. Python's parser frequently accepts a message that has garbage in the header by simply adding all header items after the bad header line(s) to the body text; this can leave a pretty unmanageable message so we apply our own validation. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> from goodcrypto.oce.test_constants import EDWARD_LOCAL_USER >>> email_message = get_basic_email_message() >>> email_message.validate_message() True ''' try: validator = Validator(self) if validator.is_message_valid(): valid = True self.log_message('message is valid') else: valid = False self.log_message('message is invalid') self.log_message(validator.get_why()) except Exception as AttributeError: valid = False record_exception() return valid def get_text(self): ''' Gets text from the current Message. This method works with both plain and MIME messages, except open pgp mime. If the message is MIMEMultipart, the text is from the first text/plain part. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> email_message = get_basic_email_message() >>> text = email_message.get_text() >>> text == 'Test message text' True ''' text = None message = self.get_message() if is_open_pgp_mime(message): self.log_message("unable to get text from openpgp mime message") else: if message.is_multipart(): self.log_message("message is a MIMEMultipart") # get the first text/plain part result_ok = False part_index = 0 parts = message.get_payload() while part_index < len(parts) and not result_ok: part = message.get_payload(part_index) content_type = part.get_content_type() if content_type == mime_constants.TEXT_PLAIN_TYPE: text = self._get_decoded_payload(part) result_ok = True else: self.log_message("body part type is " + content_type) part_index += 1 else: text = self._get_decoded_payload(message) self.log_message("payload is a: {}".format(type(text))) return text def set_text(self, text, charset=None): ''' Sets text in the current Message. This method works with both plain and MIME messages, except open pgp mime. If the message is MIMEMultipart, the text is set in the first text/plain part. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> email_message = get_basic_email_message() >>> email_message.set_text('New test message text') True >>> text = email_message.get_text() >>> text == 'New test message text' True ''' if self.DEBUGGING: self.log_message("setting text:\n{}".format(text)) text_set = False message = self.get_message() if message.is_multipart(): # set the first text/plain part text_set = False part_index = 0 parts = message.get_payload() while part_index < len(parts) and not text_set: part = message.get_payload(part_index) content_type = part.get_content_type() if content_type == mime_constants.TEXT_PLAIN_TYPE: part.set_payload(text) text_set = True self.log_message('the first text/plain part found') else: self.log_message('body part type is {}'.format(content_type)) part_index += 1 if not text_set: charset, __ = get_charset(self._message, self._last_charset) self.log_message('no text_set char set: {}'.format(charset)) new_part = MIMEText(text, mime_constants.PLAIN_SUB_TYPE, charset) message.attach(new_part) text_set = True self.log_message('added a new text/plain part with text') elif is_open_pgp_mime(message): self.log_message("unable to set text from openpgp mime message") else: self.set_content(text, mime_constants.TEXT_PLAIN_TYPE, charset=charset) text_set = True if self.DEBUGGING: self.log_message("message after setting text:\n" + self.to_string()) self.log_message("set text:\n{}".format(text_set)) return text_set def get_content(self): ''' Get the message's content, decoding if bas64 or print-quoted encoded. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> email_message = get_basic_email_message() >>> text = email_message.get_content() >>> text == 'Test message text' True ''' decode = False msg = self.get_message() encoding = self.get_header(mime_constants.CONTENT_XFER_ENCODING_KEYWORD) if encoding is not None: encoding = encoding.lower() self.log_message('payloaded encoded with {}'.format(encoding)) # only use the encoding if it's not a multipart message if (encoding == mime_constants.QUOTED_PRINTABLE_ENCODING or encoding == mime_constants.BASE64_ENCODING): current_content_type = self.get_message().get_content_type() if (current_content_type is not None and current_content_type.lower().find(mime_constants.MULTIPART_PRIMARY_TYPE) < 0): decode = True self.log_message('decoding payload with {}'.format(encoding)) try: payload = self._get_decoded_payload(self.get_message(), decode=decode) if self.DEBUGGING: self.log_message('decoded payloaded:\n{}'.format(payload)) self.log_message('type of payload: {}'.format(type(payload))) except: record_exception() payload = message.get_payload() return payload def set_content(self, payload, content_type, charset=None): ''' Set the content of the message. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> email_message = get_basic_email_message() >>> email_message.set_content('New test message text', mime_constants.TEXT_PLAIN_TYPE) >>> text = email_message.get_content() >>> text == 'New test message text' True ''' # create a new message if one doesn't exist if self._message is None: self._message = Message() current_content_type = self.get_message().get_content_type() if current_content_type is None: current_content_type = content_type self.log_message('current content type: {}'.format(current_content_type)) self.log_message('setting content type: {}'.format(content_type)) if self.DEBUGGING: self.log_message('content:\n{}'.format(payload)) current_encoding = self.get_header(mime_constants.CONTENT_XFER_ENCODING_KEYWORD) if current_encoding is None: self._message.__setitem__(mime_constants.CONTENT_XFER_ENCODING_KEYWORD, mime_constants.BITS_8) self.log_message('setting content encoding: {}'.format(mime_constants.BITS_8)) # if this is a simple text or html message, then just update the payload if (content_type == current_content_type and (content_type == mime_constants.TEXT_PLAIN_TYPE or content_type == mime_constants.TEXT_HTML_TYPE)): if charset is None: charset, self._last_charset = get_charset(payload, self._last_charset) self.log_message('getting charset from payload: {}'.format(charset)) elif self._last_charset is None: self._last_charset = constants.DEFAULT_CHAR_SET self.log_message('setting last charset to default: {}'.format()) else: self.log_message('using preset charset: {}'.format(charset)) try: self.get_message().set_payload( self.encode_payload(payload, current_encoding), charset=charset) self.log_message('set payload with {} charset'.format(charset)) if self.DEBUGGING: self.log_message('payload set:\n{}'.format(payload)) except UnicodeEncodeError as error: self.log_message(error.reason) self.log_message('start: {} end: {}'.format(error.start, error.end)) self.log_message('object: {}'.format(error.object)) self.get_message().set_payload(self.encode_payload(payload, current_encoding)) self.log_message('setting payload without charset') self.get_message().set_type(content_type) else: from goodcrypto.mail.message.inspect_utils import is_content_type_mime self.log_message('attaching payload for {}'.format(content_type)) if content_type == mime_constants.OCTET_STREAM_TYPE: part = MIMEBase(mime_constants.APPLICATION_TYPE, mime_constants.OCTET_STREAM_SUB_TYPE) part.set_payload(open(payload,"rb").read()) encode_base64(part) part.add_header('Content-Disposition', 'attachment; filename="%s"' % os.path.basename(payload)) self.get_message().attach(part) elif is_content_type_mime(self.get_message()): if not self.get_message().is_multipart(): if charset is None: charset, self._last_charset = get_charset(payload, self._last_charset) self.log_message('setting content with char set: {}'.format(charset)) else: if self._last_charset is None: self._last_charset = constants.DEFAULT_CHAR_SET self.get_message().set_payload(self.encode_payload(payload, current_encoding), charset) self.log_message('set payload with {} charset'.format(charset)) self.get_message().set_type(content_type) elif content_type == mime_constants.TEXT_PLAIN_TYPE: if self.DEBUGGING: self.log_message('mime text payload:\n{}'.format(payload)) part = MIMEText(payload) if self.DEBUGGING: self.log_message('mime text part:\n{}'.format(part)) part.set_payload(self.encode_payload(payload, current_encoding)) if self.DEBUGGING: self.log_message('mime text part with payload:\n{}'.format(part)) self.get_message().attach(part) else: primary, __, secondary = content_type.partition(mime_constants.PRIMARY_TYPE_DELIMITER) part = MIMEBase(primary, secondary) part.set_payload(self.encode_payload(payload, current_encoding)) self.get_message().attach(part) def encode_payload(self, payload, current_encoding): ''' Encode the payload. Test extreme case. >>> email_message = EmailMessage() >>> email_message.encode_payload(None, None) ''' new_payload = payload if payload is not None and current_encoding is not None: """ """ if current_encoding == mime_constants.BASE64_ENCODING: if isinstance(payload, str): payload = payload.encode() new_payload = b64encode(payload) self.log_message('encoding payload with {}'.format(current_encoding)) elif current_encoding == mime_constants.QUOTED_PRINTABLE_ENCODING: if isinstance(payload, str): payload = payload.encode() new_payload = encodestring(payload) self.log_message('encoding payload with {}'.format(current_encoding)) return new_payload def is_probably_pgp(self): ''' Returns true if this is probably an OpenPGP message. >>> from goodcrypto_tests.mail.message_utils import get_encrypted_message_name >>> with open(get_encrypted_message_name('open-pgp-mime.txt')) as input_file: ... mime_message = EmailMessage(input_file) ... mime_message.is_probably_pgp() True ''' is_pgp = is_open_pgp_mime(self.get_message()) if not is_pgp: content = self.get_content() if is_string(content): is_pgp = self.contains_pgp_message_delimters(content) self.log_message('message uses in line pgp: {}'.format(is_pgp)) elif isinstance(content, list): for part in content: if isinstance(part, Message): part_content = part.get_payload() else: part_content = part if is_string(part_content): is_pgp = self.contains_pgp_message_delimters(part_content) if is_pgp: self.log_message('part of message uses in line pgp: {}'.format(is_pgp)) break else: self.log_message('part of content type is: {}'.format(repr(part_content))) else: self.log_message('content type is: {}'.format(type(content))) return is_pgp def contains_pgp_message_delimters(self, text): ''' Returns true if text contains PGP message delimiters. >>> from goodcrypto_tests.mail.message_utils import get_encrypted_message_name >>> with open(get_encrypted_message_name('open-pgp-mime.txt')) as input_file: ... text = input_file.read() ... email_message = EmailMessage() ... email_message.contains_pgp_message_delimters(text) True ''' return (isinstance(text, str) and text.find(oce_constants.BEGIN_PGP_MESSAGE) >= 0 and text.find(oce_constants.END_PGP_MESSAGE) >= 0) def contains_pgp_signature_delimeters(self, text): ''' Returns true if text contains PGP signature delimiters. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('pgp-signature.txt')) as input_file: ... text = input_file.read() ... email_message = EmailMessage() ... email_message.contains_pgp_signature_delimeters(text) True ''' return (isinstance(text, str) and text.find(oce_constants.BEGIN_PGP_SIGNATURE) >= 0 and text.find(oce_constants.END_PGP_SIGNATURE) >= 0) def get_pgp_signature_blocks(self): ''' Returns the PGP signature blocks with text, if there are any. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('pgp-signature.txt')) as input_file: ... mime_message = EmailMessage(input_file) ... signature_blocks = mime_message.get_pgp_signature_blocks() ... len(signature_blocks) > 0 True ''' def get_signed_data(content): ''' Get the signed data. ''' signature_block = None start_index = content.find(oce_constants.BEGIN_PGP_SIGNED_MESSAGE) if start_index < 0: start_index = content.find(oce_constants.BEGIN_PGP_SIGNATURE) end_index = content.find(oce_constants.END_PGP_SIGNATURE) if start_index >= 0 and end_index > start_index: signature_block = content[start_index:end_index + len(oce_constants.END_PGP_SIGNATURE)] return signature_block signature_blocks = [] if self.get_message().is_multipart(): self.log_message('check each of {} parts of message for a signature'.format( len(self.get_message().get_payload()))) part_index = 0 parts = self.get_message().get_payload() for part in parts: part_index += 1 if isinstance(part, str): content = part else: content = part.get_payload() if self.contains_pgp_signature_delimeters(content): is_signed = True signature_block = get_signed_data(content) if signature_block is not None: signature_blocks.append(signature_block) self.log_message('found signature block in part {}'.format(part_index)) part_index += 1 else: content = self._get_decoded_payload(self.get_message()) if isinstance(content, str) and self.contains_pgp_signature_delimeters(content): is_signed = True signature_block = get_signed_data(content) if signature_block is not None: signature_blocks.append(signature_block) self.log_message('found signature block in content') self.log_message('total signature blocks: {}'.format(len(signature_blocks))) return signature_blocks def remove_pgp_signature_blocks(self): ''' Remove the PGP signature blocks, if there are any. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('pgp-signature.txt')) as input_file: ... mime_message = EmailMessage(input_file) ... mime_message.remove_pgp_signature_blocks() ... signature_blocks = mime_message.get_pgp_signature_blocks() ... len(signature_blocks) == 0 True ''' def remove_signature(content): ''' Remove the signature from the content. ''' # remove the beginning signature lines if content.startswith(oce_constants.BEGIN_PGP_SIGNED_MESSAGE): begin_sig_lines = '' for line in content.split('\n'): if len(line.strip()) <= 0: break else: begin_sig_lines += '{}\n'.format(line) content = content[len(begin_sig_lines):] # remove the signature itself start_index = content.find(oce_constants.BEGIN_PGP_SIGNATURE) end_index = content.find(oce_constants.END_PGP_SIGNATURE) content = content[0:start_index] + content[end_index + len(oce_constants.END_PGP_SIGNATURE):] # remove the extra characters added around the message itself content = content.replace('- {}'.format(oce_constants.BEGIN_PGP_MESSAGE), oce_constants.BEGIN_PGP_MESSAGE) content = content.replace('- {}'.format(oce_constants.END_PGP_MESSAGE), oce_constants.END_PGP_MESSAGE) return content try: if self.get_message().is_multipart(): self.log_message('check each of {} parts of message for a signature'.format( len(self.get_message().get_payload()))) part_index = 0 parts = self.get_message().get_payload() for part in parts: part_index += 1 if isinstance(part, str): content = part else: content = self._get_decoded_payload(part) if self.contains_pgp_signature_delimeters(content): charset, __ = get_charset(part) self.log_message('set payload after removing sig with char set: {}'.format(charset)) part.set_payload(remove_signature(content), charset=charset) self.log_message('extracted signature block from part {}'.format(part_index)) else: content = self._get_decoded_payload(self.get_message()) if isinstance(content, str) and self.contains_pgp_signature_delimeters(content): charset, __ = get_charset(part) self.get_message().set_payload(remove_signature(content), charset=charset) self.log_message('extracted signature block from content with char set: {}'.format(charset)) except: self.log_message('EXCEPTION see syr.exception.log') record_exception() def write_to(self, output_file): ''' Write message to the specified file. >>> from goodcrypto.mail.utils.dirs import get_test_directory >>> from goodcrypto_tests.mail.message_utils import get_encrypted_message_name >>> filename = get_encrypted_message_name('iso-8859-1-binary.txt') >>> with open(filename) as input_file: ... output_dir = get_test_directory() ... output_filename = os.path.join(output_dir, 'test-message.txt') ... mime_message = EmailMessage(input_file) ... with open(output_filename, 'w') as out: ... mime_message.write_to(out) ... os.path.exists(output_filename) ... mime_message.write_to(out) ... os.path.exists(output_filename) ... os.remove(output_filename) True True True True if os.path.exists(output_filename): os.remove(output_filename) ''' result_ok = False try: if isinstance(output_file, IOBase): if output_file.closed: with open(output_file.name, 'w') as out: out.write(self.to_string()) out.flush() else: output_file.write(self.to_string()) output_file.flush() elif isinstance(output_file, StringIO): output_file.write(self.to_string()) else: with open(output_file, 'w') as out: out.write(self.to_string()) out.flush() result_ok = True except Exception: record_exception() raise Exception return result_ok def to_string(self, charset=None, mangle_from=False): ''' Convert message to a string. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> filename = get_plain_message_name('basic.txt') >>> with open(filename) as input_file: ... file_content = input_file.read().replace('\\r\\n', '\\n') ... position = input_file.seek(os.SEEK_SET) ... email_message = EmailMessage(input_file) ... file_content.strip() == email_message.to_string().strip() True ''' string = None try: msg = self._message if charset is None: charset, __ = get_charset(msg, self._last_charset) self.log_message('char set in to_string(): {}'.format(charset)) # convert the message try: file_pointer = StringIO() message_generator = Generator(file_pointer, mangle_from_=mangle_from, maxheaderlen=78) message_generator.flatten(msg) string = file_pointer.getvalue() except Exception as AttributeError: try: self.log_message('unable to flatten message') record_exception(AttributeError) msg = self._message string = msg.as_string() except Exception as AttributeError: # we explicitly want to catch everything here, even NPE self.log_message('unable to convert message as_string') string = '{}\n\n{}'.format( '\n'.join(self.get_header_lines()), '\n'.join(self.get_content_lines())) if self.DEBUGGING: self.log_message("message string:\n{}".format(string)) except IOError as io_error: self.last_error = io_error self.log_message(io_error) except MessageException as msg_exception: self.last_error = msg_exception self.log_message(msg_exception) return string def get_header_lines(self): ''' Get message headers as a list of lines. The lines follow RFC 2822, with a maximum of 998 characters per line. Longer headers are folded using a leading tab. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> filename = get_plain_message_name('basic.txt') >>> with open(filename) as input_file: ... email_message = EmailMessage(input_file) ... len(email_message.get_header_lines()) > 0 True ''' max_line_length = 998 lines = [] keys = self._message.keys() for key in keys: value = self.get_header(key) if value is None: value = '' raw_line = '{}: {}'.format(key, value) if len(raw_line) > max_line_length: # add first line from this header part_line = raw_line[0:max_line_length] lines.append(part_line) raw_line = raw_line[:max_line_length] # add continuation lines while len(raw_line) > max_line_length: # make space for leading tab part_line = raw_line[0:max_line_length - 1] lines.append("\t" + part_line) raw_line = raw_line[max_line_length - 1:] if len(raw_line) > 0: lines.append(raw_line) return lines def get_content_lines(self): ''' Gets the message content as a list of lines. This is the part of the message after the header and the separating blank line, with no decoding. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> filename = get_plain_message_name('basic.txt') >>> with open(filename) as input_file: ... email_message = EmailMessage(input_file) ... len(email_message.get_content_lines()) > 0 True ''' lines = [] payloads = self._message.get_payload() if payloads is None: self.log_message('No content') else: if isinstance(payloads, str): lines = payloads.split('\n') else: for payload in payloads: if isinstance(payload, Message): lines += payload.as_string() else: lines += payload.split('\n') return lines def _parse_header_line(self, line, last_name): ''' Parse a header line (internal user only). >>> email_message = EmailMessage() >>> name, value, last_name = email_message._parse_header_line( ... 'Mime-Version: 1.0', 'Subject') >>> name == 'Mime-Version' True >>> value == '1.0' True ''' if line is None: name = value = last_name = None else: name, __, value = line.partition(':') if name is not None: name = name.strip() if name is None or len(name) <= 0: self.log_message("no header name in line: " + line) if last_name is not None: old_value = self.get_header(last_name) self.add_header(name, '{} {}\n'.format(old_value.strip('\n'), value.strip())) else: last_name = name if value is None: value = '' else: value = value.strip() try: # try adding the header line and see if python can parse it test_message = Message() test_message.__setitem__(name, value) if isinstance(self.parser, Parser): temp_header = self.parser.parsestr(test_message.as_string(unixfrom=False)) else: temp_header = self.parser.parsebytes(test_message.as_string(unixfrom=False).encode()) if temp_header.__len__() == 0: self.log_message('bad header: {}'.format(line)) self.bad_header_lines.append(line) else: # if the parser accept this header line, then keep it self.add_header(name, value) except Exception: record_exception() self.bad_header_lines.append(line) return name, value, last_name def _set_content_encoding(self, name, value): ''' Set encoding in content (internal use only). >>> email_message = EmailMessage() >>> email_message._set_content_encoding( ... mime_constants.CONTENT_TYPE_KEYWORD, 'charset=utf-8') ''' if name is None or value is None: self.log_message('no name or value defined while trying to set content encoding') elif name == mime_constants.CONTENT_TYPE_KEYWORD: try: # try to set the charset index = value.find('charset=') if index >= 0: charset = value[index + len('charset='):] if charset.startswith('"') and charset.endswith('"'): charset = charset[1:len(charset)-1] self._message.set_charset(charset) except Exception: record_exception() self._message.set_charset(constants.DEFAULT_CHAR_SET) elif name == mime_constants.CONTENT_XFER_ENCODING_KEYWORD: encoding_value = self._message.get( mime_constants.CONTENT_XFER_ENCODING_KEYWORD) self.log_message('message encoding: {}'.format(encoding_value)) if encoding_value is None or encoding_value.lower() != value.lower(): self._message.__delitem__(name) self._message.__setitem__(name, value) self.log_message('set message encoding: {}'.format(value)) def _get_decoded_payload(self, msg, decode=True): ''' Get the payload and decode it if necessary. >>> email_message = EmailMessage() >>> email_message._get_decoded_payload(None) ''' if msg is None: payload = None else: payload = msg.get_payload(decode=decode) if isinstance(payload, bytearray) or isinstance(payload, bytes): charset, __ = get_charset(msg, self._last_charset) self.log_message('decoding payload with char set: {}'.format(charset)) try: payload = payload.decode(encoding=charset) except: payload = payload.decode(encoding=charset, errors='replace') return payload def _create_new_header(self, message_string): ''' Create a new header from a corrupted message (internal use only). >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('basic.txt')) as input_file: ... message_string = ''.join(input_file.readlines()) ... email_message = EmailMessage() ... body_text_lines = email_message._create_new_header(message_string) ... len(body_text_lines) > 0 True ''' last_name = None body_text_lines = None if message_string is None: self.log_message('no message string defined to create new header') else: self.log_message('starting to parse headers') lines = message_string.split('\n') header_count = 0 for line in lines: if line is None or len(line.strip()) <= 0: self.log_message('finished parsing headers') if header_count + 1 <= len(lines): body_text_lines = lines[header_count + 1:] else: body_text_lines = [] break else: header_count += 1 name, value, last_name = self._parse_header_line(line, last_name) if (name is not None and (name == mime_constants.CONTENT_TYPE_KEYWORD or name == mime_constants.CONTENT_XFER_ENCODING_KEYWORD) ): self._set_content_encoding(name, value) return body_text_lines def _create_new_body_text(self, body): ''' Create the body text from a corrupted message. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('basic.txt')) as input_file: ... email_message = EmailMessage(input_file.readlines()) ... email_message._create_new_body_text('Test new body text') ''' charset, __ = get_charset(self._message, self._last_charset) self.log_message('creating new body text with char set: {}'.format(charset)) try: body_text = '' for line in body: body_text += line.encode(charset) except Exception as body_exception: self.log_message(body_exception) record_exception() body_text = ''.join(body) if len(self.bad_header_lines) > 0: body_text += '\n\n{}\n'.format(i18n('Removed bad header lines')) for bad_header_line in self.bad_header_lines: body_text += ' {}\n'.format(bad_header_line) self._message.set_payload(body_text, charset=charset) def _create_good_message_from_bad(self, source): ''' Create a good message from a source that contains a corrupted message. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> with open(get_plain_message_name('bad-basic.txt')) as input_file: ... email_message = EmailMessage() ... email_message._create_good_message_from_bad(input_file) ''' try: # start with a fresh message self._message = Message() if isinstance(source, IOBase): source.seek(os.SEEK_SET) message_string = source.read() else: message_string = source body_text = self._create_new_header(message_string) if body_text: self._create_new_body_text(body_text) except Exception as message_exception: self.log_message(message_exception) record_exception() raise MessageException(message_exception) def init_new_message(self, from_addr, to_addr, subject, text=None): ''' Initialize a basic new message. Used primarily for testing. >>> # In honor of Kirk Wiebe, a whistleblower about Trailblazer, an NSA mass surveillance project. >>> from_user = '******' >>> to_user = '******' >>> email_message = EmailMessage() >>> email_message.init_new_message(from_user, to_user, "Test message", 'Test body text') ''' self.add_header(mime_constants.FROM_KEYWORD, from_addr) self.add_header(mime_constants.TO_KEYWORD, to_addr) self.add_header(mime_constants.SUBJECT_KEYWORD, subject) if text: self.set_text(text) def log_message_exception(self, exception_error, message, log_msg): ''' Log an exception. >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> email_message = EmailMessage() >>> email_message.log_message_exception(Exception, 'message', 'log message') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.email_message.log')) True >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'syr.exception.log')) True ''' self.log_exception(log_msg, message_exception=exception_error) if message != None: try: self.log_message("message:\n" + message.to_string()) except Exception as exception_error2: self.log_message("unable to log message: {}".format(exception_error2)) def log_exception(self, log_msg, message_exception=None): ''' Log an exception. >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> email_message = EmailMessage() >>> email_message.log_exception('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.email_message.log')) True >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'syr.exception.log')) True >>> email_message.log_exception('test', message_exception='message exception') ''' record_exception() self.log_message(log_msg) record_exception(message=log_msg) if message_exception is not None: if type(message_exception) == Exception: self.log_message(message_exception.value) record_exception(message=message_exception.value) elif type(message_exception) == str: self.log_message(message_exception) record_exception(message=message_exception) def log_message(self, message): ''' Log a message. >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> email_message = EmailMessage() >>> email_message.log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.email_message.log')) True ''' if self._log is None: self._log = LogFile() self._log.write_and_flush(message)
class HeaderKeys(object): ''' Manage keys from a message's header. It's important that we only import a key if we don't have a key for this user. If we already do, then we must be very careful deciding whether to use the key or not. The following table shows the conditions that we must handle if there's a key in the header. If we decide not to use a key, then a message is sent to the recipient explaining our reason for not processing the message. The original offending message is included as an attachment. Received key in header so check db and crypto package. | saved in db | crypto package | action | unit test | |--------------|----------------|------------|---------------------------------------| | match | match | use | test_all_matching_keys | | match | no match | do not use | test_db_fingerprint_bad_crypto_key | | match | missing | do not use | test_db_fingerprint_no_crypto_key | | no match | match | do not use | test_bad_fingerprint_matching_crypto | | no match | no match | do not use | test_bad_fingerprint_bad_crypto | | no match | missing | do not use | test_bad_fingerprint_missing_crypto | | missing | match | add to db | test_crypto_key_no_db_fingerprint | | missing | no match | do not use | test_bad_crypto_key_no_db_fingerprint | | missing | missing | import | test_no_existing_keys | ''' DEBUGGING = False UNEXPECTED_ERROR = i18n('An unexpected error ocurred while processing this message') def __init__(self): ''' >>> header_keys = HeaderKeys() >>> header_keys != None True ''' self.log = LogFile() self.recipient_to_notify = None self.new_key_imported = False def manage_keys_in_header(self, crypto_message): ''' Manage all the public keys in the message's header. ''' header_contains_key_info = False try: from_user = crypto_message.smtp_sender() self.recipient_to_notify = crypto_message.smtp_recipient() # all notices about a metadata address goes to the admin if is_metadata_address(self.recipient_to_notify): self.recipient_to_notify = get_admin_email() name, address = parse_address(from_user) if address is None or crypto_message is None or crypto_message.get_email_message() is None: self.log_message('missing data so cannot import key') self.log_message(' from user: {}'.format(from_user)) self.log_message(' address: {}'.format(address)) self.log_message(' crypto message: {}'.format(crypto_message)) if crypto_message is not None: self.log_message(' email message: {}'.format(crypto_message.get_email_message())) else: accepted_crypto_packages = crypto_message.get_accepted_crypto_software() if accepted_crypto_packages is None or len(accepted_crypto_packages) <= 0: self.log_message("checking for default key for {} <{}>".format(name, address)) tag = self._manage_key_header(address, crypto_message, KeyFactory.get_default_encryption_name(), PUBLIC_KEY_HEADER) else: self.log_message("checking for {} keys".format(accepted_crypto_packages)) for encryption_name in accepted_crypto_packages: # see if there's a the key block for this encryption program header_name = get_public_key_header_name(encryption_name) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), header_name) # see if there's a plain key block if ((key_block is None or len(key_block) <= 0) and len(accepted_crypto_packages) == 1): self.log_message("no {} public key in header so trying generic header".format(encryption_name)) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), PUBLIC_KEY_HEADER) tag = self._manage_key_header( address, crypto_message, encryption_name, key_block) header_contains_key_info = True self.update_accepted_crypto(from_user, accepted_crypto_packages) except MessageException as message_exception: self.log_message(message_exception.value) raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if crypto_message is not None: crypto_message.add_error_tag_once(self.UNEXPECTED_ERROR) self.log_message('header_contains_key_info: {}'.format(header_contains_key_info)) return header_contains_key_info def keys_in_header(self, crypto_message): ''' Return true if there are public keys in the message's header. ''' header_contains_key_info = False try: good_key = True from_user = crypto_message.smtp_sender() accepted_crypto_packages = crypto_message.get_accepted_crypto_software() if accepted_crypto_packages is not None and len(accepted_crypto_packages) > 0: self.log_message("checking for {} keys".format(accepted_crypto_packages)) for encryption_name in accepted_crypto_packages: # see if there's a the key block for this encryption program header_name = get_public_key_header_name(encryption_name) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), header_name) # see if there's a plain key block if ((key_block is None or len(key_block) <= 0) and len(accepted_crypto_packages) == 1): self.log_message("no {} public key in header so trying generic header".format(encryption_name)) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), PUBLIC_KEY_HEADER) if key_block is not None and len(key_block) > 0: header_contains_key_info = True user_ids = [] key_crypto = KeyFactory.get_crypto( encryption_name, crypto_software.get_key_classname(encryption_name)) if key_crypto is None: id_fingerprint_pairs = None self.log_message('no key crypto for {}'.format(encryption_name)) else: id_fingerprint_pairs = key_crypto.get_id_fingerprint_pairs(key_block) for (user_id, __) in id_fingerprint_pairs: user_ids.append(user_id) self.log_message("key block includes key for {}".format(user_ids)) # don't consider a key valid if it's not from the sender if from_user not in user_ids: tag = notices.report_bad_header_key( self.recipient_to_notify, from_user, user_ids, encryption_name, crypto_message) self.log_message('tag: {}'.format(tag)) if crypto_message.get_email_message().is_probably_pgp(): crypto_message.drop(dropped=True) self.log_message('serious error in header so original message sent as attchment') raise MessageException(value=i18n(tag)) self.update_accepted_crypto(from_user, accepted_crypto_packages) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if crypto_message is not None: crypto_message.add_error_tag_once(self.UNEXPECTED_ERROR) return header_contains_key_info def new_key_imported_from_header(self): ''' Return true if a new key was imported from the header. ''' return self.new_key_imported def _manage_key_header(self, from_user, crypto_message, encryption_name, key_block): ''' Manage a key in the header for the encryption software (internal use only). ''' tag = None try: if key_block == None or len(key_block.strip()) <= 0: self.log_message("no {} public key in header".format(encryption_name)) else: if self.DEBUGGING: self.log_message("{} key from message:\n{}".format(encryption_name, key_block)) key_crypto = KeyFactory.get_crypto( encryption_name, crypto_software.get_key_classname(encryption_name)) if key_crypto is None: id_fingerprint_pairs = None self.log_message('no key crypto for {}'.format(encryption_name)) else: id_fingerprint_pairs = key_crypto.get_id_fingerprint_pairs(key_block) if id_fingerprint_pairs is None or len(id_fingerprint_pairs) <= 0: tag = None self.log_message('no user keys in key block') else: tag = self._manage_public_key( from_user, crypto_message, key_crypto, key_block, id_fingerprint_pairs) if tag is not None and len(tag.strip()) > 0: crypto_message.add_tag_once(tag) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return tag def _manage_public_key(self, from_user, crypto_message, key_crypto, key_block, id_fingerprint_pairs): ''' Manage a public key for the encryption software (internal use only). ''' tag = None drop = False encryption_name = key_crypto.get_name() user_ids= [] for (user_id, __) in id_fingerprint_pairs: user_ids.append(user_id) self.log_message("key block includes key for {}".format(user_ids)) if from_user in user_ids: # see if we already have a key for this ID or if it has expired saved_fingerprint, verified, __ = contacts.get_fingerprint(from_user, encryption_name) crypto_fingerprint, expiration = key_crypto.get_fingerprint(from_user) self.log_message("{} {} saved fingerprint {}".format(from_user, encryption_name, saved_fingerprint)) self.log_message("{} {} crypto fingerprint {} expires {}".format( from_user, encryption_name, crypto_fingerprint, expiration)) self.log_message("{} {} id fingerprint pairs {}".format(from_user, encryption_name, id_fingerprint_pairs)) if crypto_fingerprint is None: if saved_fingerprint is None: self.log_message("importing new key") tag = self._import_new_key( from_user, encryption_name, key_block, id_fingerprint_pairs) else: self.log_message("checking if key matches") drop = crypto_message.get_email_message().is_probably_pgp() key_matches, key_error = self._key_matches( encryption_name, saved_fingerprint, id_fingerprint_pairs) self.log_message("key matches: {} / key error: {}".format(key_matches, key_error)) if key_error: tag = notices.report_error_verifying_key( self.recipient_to_notify, from_user, encryption_name, crypto_message) else: tag = notices.report_missing_key(self.recipient_to_notify, from_user, key_matches, id_fingerprint_pairs, crypto_message) else: if saved_fingerprint is None: # remember the fingerprint for the future saved_fingerprint = crypto_fingerprint if (strip_fingerprint(crypto_fingerprint).lower() == strip_fingerprint(saved_fingerprint).lower()): key_matches, key_error = self._key_matches( encryption_name, crypto_fingerprint, id_fingerprint_pairs) if key_error: tag = notices.report_error_verifying_key( self.recipient_to_notify, from_user, encryption_name, crypto_message) elif key_matches: if key_crypto.fingerprint_expired(expiration): drop = crypto_message.get_email_message().is_probably_pgp() tag = notices.report_expired_key(self.recipient_to_notify, from_user, encryption_name, expiration, crypto_message) else: self.log_message(' same fingerprint: {}'.format(saved_fingerprint)) # warn user if unable to save the fingerprint, but proceed if not self.update_fingerprint(from_user, encryption_name, crypto_fingerprint): tag = notices.report_db_error( self.recipient_to_notify, from_user, encryption_name, crypto_message) else: if self.DEBUGGING: self.log_message('{} key block\n{}'.format( from_user, key_crypto.export_public(from_user))) drop = crypto_message.get_email_message().is_probably_pgp() tag = notices.report_replacement_key(self.recipient_to_notify, from_user, encryption_name, id_fingerprint_pairs, crypto_message) else: drop = crypto_message.get_email_message().is_probably_pgp() tag = notices.report_mismatched_keys( self.recipient_to_notify, from_user, encryption_name, crypto_message) else: if len(user_ids) > 0: drop = crypto_message.get_email_message().is_probably_pgp() tag = notices.report_bad_header_key( self.recipient_to_notify, from_user, user_ids, encryption_name, crypto_message) else: self.log_message('no importable keys found\n{}'.format(key_block)) self.log_message('tag: {}'.format(tag)) if drop: crypto_message.drop(dropped=True) self.log_message('serious error in header so original message sent as attchment') raise MessageException(value=i18n(tag)) return tag def _import_new_key(self, from_user, encryption_name, key_block, id_fingerprint_pairs): ''' Import a new key (internal use only). ''' tag = None result_ok = False if encryption_name is None: encryption_name = '' self.new_key_imported = False try: self.log_message("starting to import new {} key for {}".format(encryption_name, from_user)) if from_user is None or len(encryption_name) == 0 or id_fingerprint_pairs is None: self.log_message('missing key data so unable to import new key') else: key_crypto = KeyFactory.get_crypto( encryption_name, crypto_software.get_key_classname(encryption_name)) # make sure that we don't have a key for any of the user ids included with this key result_ok = True if id_fingerprint_pairs is None or len(id_fingerprint_pairs) <= 0: result_ok = False elif len(id_fingerprint_pairs) > 1: for (user_id, __) in id_fingerprint_pairs: crypto_fingerprint, expiration = key_crypto.get_fingerprint(user_id) if crypto_fingerprint is not None: result_ok = False self.log_message('key exists for {} so unable to import key for {}'.format(user_id, from_user)) break if result_ok: result_ok = key_crypto.import_public(key_block, id_fingerprint_pairs=id_fingerprint_pairs) self.log_message('imported key: {}'.format(result_ok)) if result_ok: self.new_key_imported = True result_ok = self._add_contacts_and_notify(encryption_name, id_fingerprint_pairs) self.log_message('added contacts: {}'.format(result_ok)) if not result_ok: self.log_message('Unable to import new public key for {}; probably taking longer than expected; check Contacts later'.format(from_user)) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_message('import new key ok: {}'.format(result_ok)) return tag def _add_contacts_and_notify(self, encryption_name, id_fingerprint_pairs): result_ok = True # use the first email address from the imported key email, __ = id_fingerprint_pairs[0] for (user_id, fingerprint) in id_fingerprint_pairs: self.log_message( 'adding contact for {} with {} fingerprint'.format(user_id, fingerprint)) contact = contacts.add(user_id, encryption_name, fingerprint=fingerprint, source=MESSAGE_HEADER) if contact is None: result_ok = False self.log_message('unable to add contact while trying to import key for {}'.format(email)) else: self.log_message('successfully added contact after importing key for {}'.format(email)) if result_ok: notices.notify_new_key_arrived(self.recipient_to_notify, id_fingerprint_pairs) return result_ok def _key_matches(self, encryption_name, old_fingerprint, id_fingerprint_pairs): ''' Does the new key's fingerprint match the old fingerprint? (internal use only) ''' matches = error = False if encryption_name is None: error = True self.log_message('unable to compare fingerprints because basic data is missing') else: try: if id_fingerprint_pairs is None or old_fingerprint is None: error = True self.log_message('missing fingerprint for comparison') self.log_message('old fingerprint: {}'.format(old_fingerprint)) self.log_message('id fingerprint pairs: {}'.format(id_fingerprint_pairs)) else: fingerprints = [] for (__, fingerprint) in id_fingerprint_pairs: fingerprints.append(fingerprint.replace(' ', '')) matches = old_fingerprint.replace(' ', '') in fingerprints except: error = True matches = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return matches, error def _import_accepted_crypto_software(self, from_user, crypto_message): ''' Import the encryption software the contact can use (internal use only). ''' accepted_crypto_packages = crypto_message.get_accepted_crypto_software() self.update_accepted_crypto(from_user, accepted_crypto_packages) return accepted_crypto_packages def update_accepted_crypto(self, email, encryption_software_list): ''' Update the list of encryption software accepted by user. ''' if email is None: self.log_message("email not defined so no need to update accepted crypto") elif encryption_software_list is None or len(encryption_software_list) <= 0: self.log_message('no encryption programs defined for {}'.format(email)) else: contact = contacts.get(email) if contact is None: # if the contact doesn't exist, then add them with the first encryption program encryption_program = encryption_software_list[0] contact = contacts.add(email, encryption_program, source=MESSAGE_HEADER) self.log_message("added {} to contacts".format(email)) # associate each encryption program in the list with this contact for encryption_program in encryption_software_list: try: contacts_crypto = contacts.get_contacts_crypto(email, encryption_program) if contacts_crypto is None: encryption_software = crypto_software.get(encryption_program) if encryption_software is None: self.log_message('{} encryption software unknown'.format(encryption_program)) self.log_message( 'unable to add contacts crypt for {} using {} encryption software unknown'.format(email, encryption_program)) else: contacts_crypto = contacts.add_contacts_crypto( contact=contact, encryption_software=encryption_software, source=MESSAGE_HEADER) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def update_fingerprint(self, email, encryption_name, new_fingerprint, verified=False): ''' Set the fingerprint for the encryption software for this email. If the fingerprint doesn't match the crypto's fingerprint, it won't be saved in the database. ''' if email is None or encryption_name is None: result_ok = False self.log_message("missing data to save {} fingerprint for {}".format(encryption_name, email)) else: self.log_message('updating {} fingerprint for {}'.format(encryption_name, email)) contacts_crypto = contacts.get_contacts_crypto(email, encryption_name=encryption_name) if contacts_crypto is None: contact = contacts.add(email, encryption_name, source=MESSAGE_HEADER) contacts_crypto = contacts.get_contacts_crypto(email, encryption_name=encryption_name) if contacts_crypto is None: result_ok = False self.log_message("unable to save contact's {} fingerprint".format(encryption_name)) else: try: need_update = False if new_fingerprint != contacts_crypto.fingerprint: contacts_crypto.fingerprint = new_fingerprint need_update = True self.log_message("contacts_crypto fingerprint: {}".format(contacts_crypto.fingerprint)) if contacts_crypto.verified != verified: contacts_crypto.verified = verified need_update = True self.log_message('Updated verification status: {}'.format(verified)) if need_update: contacts_crypto.save() self.log_message('saved changes') result_ok = True except: self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() result_ok = False return result_ok def log_message(self, message): ''' Log the message to the local log. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class Validator(object): ''' Validates an EmailMessage. ''' DEBUGGING = False def __init__(self, email_message): ''' Unparsable messages are wrapped in a valid message. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator != None True ''' self.log = LogFile() self.email_message = email_message self.why = None def is_message_valid(self): ''' Returns true if the message is parsable, else false. If a MIME message is not parsable, you should still be able to process it. As we find different errors in messages, we should make sure this method catches them. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator.is_message_valid() True ''' is_valid = False self.why = None if self.email_message is None: raise MessageException('Message is None') else: try: self.email_message.write_to(StringIO()) if Validator.DEBUGGING: self.log_message("message after check:\n{}".format(self.email_message.to_string())) self._check_content(self.email_message) is_valid = True except Exception: is_valid = False # we explicitly want to catch everything here, even NPE record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_message('message is valid: {}'.format(is_valid)) return is_valid def _check_content(self, part): ''' Make sure we can read the content. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator._check_content(validator.email_message) ''' content_type = part.get_header('Content-Type') self.log_message("MIME part content type: {}".format(content_type)) content = part.get_content() if isinstance(content, MIMEMultipart): count = 0 parts = content for sub_part in parts: self._check_content(sub_part) count += 1 self.log_message('parts in message: {}'.format(count)) if count != parts.getCount(): self.why = "Unable to read all content. Reported: '{}', read: {}".format( parts.getCount(), count) raise MessageException(self.why) def get_why(self): ''' Gets why a message is invalid. Returns null if the message is valid. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator.get_why() is None True ''' return self.why def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator.log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.validator.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class CryptoMessage(object): ''' Crypto email_message. This class does not extend EmailMessage because we want a copy of the original EmailMessage so we can change it without impacting the original. See unittests for most of the functions. ''' DEBUGGING = False SEPARATOR = ': ' def __init__(self, email_message=None, sender=None, recipient=None): ''' >>> crypto_message = CryptoMessage() >>> crypto_message != None True >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message != None True ''' super(CryptoMessage, self).__init__() self.log = LogFile() if email_message is None: self.email_message = EmailMessage() self.log_message('starting crypto message with a blank email message') else: self.email_message = email_message self.log_message('starting crypto message with an existing email message') # initialize a few key elements self.set_smtp_sender(sender) self.set_smtp_recipient(recipient) self.set_filtered(False) self.set_crypted(False) self.set_crypted_with([]) self.set_metadata_crypted(False) self.set_metadata_crypted_with([]) self.drop(False) self.set_processed(False) self.set_tag('') self.set_error_tag('') self.set_private_signed(False) self.set_private_signers([]) self.set_clear_signed(False) self.set_clear_signers([]) self.set_dkim_signed(False) self.set_dkim_sig_verified(False) def get_email_message(self): ''' Returns the email message. >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message.get_email_message() is not None True ''' return self.email_message def set_email_message(self, email_message): ''' Sets the email_message. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> crypto_message = CryptoMessage() >>> crypto_message.set_email_message(get_basic_email_message()) >>> crypto_message.get_email_message().get_message() is not None True ''' self.email_message = email_message def smtp_sender(self): ''' Returns the SMTP sender. >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message.smtp_sender() is None True ''' return self.sender def set_smtp_sender(self, email_address): ''' Sets the SMTP sender email address. If a message had its metadata protected, then we'll set the "smtp sender" as the inner, protected messages are set. This address is never derived from the "header" section of a message. # In honor of Sister Megan Rice, an anti-nuclear activist who was # initially sentenced for breaking into a US nuclear facility as a protest. # Fortunately, she was finally released when federal appeals court acknowledged a # little old lady had embarrassed the gov't, not threatened them. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> crypto_message = CryptoMessage() >>> crypto_message.set_smtp_sender('*****@*****.**') >>> sender = crypto_message.smtp_sender() >>> sender == '*****@*****.**' True ''' self.sender = get_email(email_address) if self.DEBUGGING: self.log_message('set sender: {}'.format(self.sender)) def smtp_recipient(self): ''' Returns the SMTP recipient. >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message.smtp_recipient() is None True ''' return self.recipient def set_smtp_recipient(self, email_address): ''' Sets the SMTP recipient email address. If a message had its metadata protected, then we'll set the "smtp recipient" as the inner, protected messages are set. This address is never derived from the "header" section of a message. >>> # In honor of the Navy nurse who refused to torture prisoners >>> # in Guantanamo by force feeding them. >>> crypto_message = CryptoMessage() >>> crypto_message.set_smtp_recipient('*****@*****.**') >>> recipient = crypto_message.smtp_recipient() >>> recipient == '*****@*****.**' True ''' self.recipient = get_email(email_address) if self.DEBUGGING: self.log_message('set recipient: {}'.format(self.recipient)) def get_public_key_header(self, from_user): ''' Get the public key header lines. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> from goodcrypto.oce.test_constants import EDWARD_LOCAL_USER >>> auto_exchange = options.auto_exchange_keys() >>> options.set_auto_exchange_keys(True) >>> filename = get_plain_message_name('basic.txt') >>> with open(filename) as input_file: ... crypto_message = CryptoMessage(email_message=EmailMessage(input_file)) ... key_block = crypto_message.get_public_key_header(EDWARD_LOCAL_USER) ... key_block is not None ... len(key_block) > 0 True True >>> options.set_auto_exchange_keys(auto_exchange) ''' header_lines = [] if options.auto_exchange_keys(): encryption_software_list = contacts.get_encryption_names(from_user) # if no crypto and we're creating keys, then do so now if (len(encryption_software_list) <= 0 and email_in_domain(from_user) and options.create_private_keys()): add_private_key(from_user) self.log_message("started to create a new key for {}".format(from_user)) encryption_software_list = contacts.get_encryption_names(from_user) if len(encryption_software_list) > 0: self.log_message("getting header with public keys for {}: {}".format( from_user, encryption_software_list)) for encryption_software in encryption_software_list: key_block = self.create_public_key_block(encryption_software, from_user) if len(key_block) > 0: header_lines += key_block else: self.log_message("Warning: auto-exchange of keys is not active") return header_lines def create_public_key_block(self, encryption_software, from_user): ''' Create a public key block for the user if the header doesn't already have one. ''' key_block = [] try: if from_user is None or encryption_software is None or self.has_public_key_header(encryption_software): self.log_message('public {} key block already exists'.format(encryption_software)) else: key_block = utils.make_public_key_block(from_user, encryption_software=encryption_software) except MessageException: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_exception("Unable to get {} public key header for {}".format(encryption_software, from_user)) return key_block def extract_public_key_block(self, encryption_software): ''' Extract a public key block from the header, if there is one. ''' key_block = None try: if self.has_public_key_header(encryption_software): header_name = utils.get_public_key_header_name(encryption_software) self.log_message("getting {} public key header block using header {}".format( encryption_software, header_name)) key_block = inspect_utils.get_multientry_header( self.get_email_message().get_message(), header_name) if key_block: self.log_message("len key_block: {}".format(len(key_block))) else: self.log_exception("No valid key {} block in header".format(encryption_software)) except MessageException: record_exception() self.log_exception("Unable to get {} public key block".format(encryption_software)) self.log_message('EXCEPTION - see syr.exception.log for details') return key_block def add_public_key_to_header(self, from_user): ''' Add public key and accepted crypto to header if automatically exchanging keys. ''' if options.auto_exchange_keys(): header_lines = self.get_public_key_header(from_user) if header_lines and len(header_lines) > 0: for line in header_lines: # we can't just use split() because some lines have no value index = line.find(CryptoMessage.SEPARATOR) if index > 0: header_name = line[0:index] value_index = index + len(CryptoMessage.SEPARATOR) if len(line) > value_index: value = line[value_index:] else: value = '' else: header_name = line value = '' self.email_message.add_header(header_name, value) self.add_accepted_crypto_software(from_user) self.add_fingerprint(from_user) self.log_message("added key for {} to header".format(from_user)) else: encryption_name = CryptoFactory.DEFAULT_ENCRYPTION_NAME if options.create_private_keys(): add_private_key(from_user, encryption_software=encryption_name) self.log_message("creating a new {} key for {}".format(encryption_name, from_user)) else: self.log_message("not creating a new {} key for {} because auto-create disabled".format( encryption_name, from_user_id)) else: self.log_message("not adding key for {} to header because auto-exchange disabled".format(from_user)) def add_accepted_crypto_software(self, from_user): ''' Add accepted encryption software to email message header. ''' # check whether we've already added them existing_crypto_software = self.get_accepted_crypto_software() if len(existing_crypto_software) > 0: self.log_message("attempted to add accepted encryption software to email_message that already has them") else: encryption_software_list = contacts.get_encryption_names(from_user) if len(encryption_software_list) <= 0: self.log_message("No encryption software for {}".format(from_user)) else: self.email_message.add_header( constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER, ','.join(encryption_software_list)) def get_accepted_crypto_software(self): ''' Gets list of accepted encryption software from email message header. Crypto services are comma delimited. ''' encryption_software_list = [] try: # !!!! the accepted services list is unsigned! fix this! encryption_software_header = inspect_utils.get_first_header( self.email_message.get_message(), constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) if encryption_software_header != None and len(encryption_software_header) > 0: self.log_message("accepted encryption software from email_message: {}".format(encryption_software_header)) encryption_software_list = encryption_software_header.split(',') except Exception as exception: self.log_message(exception) return encryption_software_list def add_fingerprint(self, from_user): ''' Add the fingerprint for each type of crypto used to the email message header. ''' try: encryption_software_list = contacts.get_encryption_names(from_user) if len(encryption_software_list) <= 0: self.log_message("Not adding fingerprint for {} because no crypto software".format(from_user)) else: for encryption_name in encryption_software_list: fingerprint, __, active = contacts.get_fingerprint(from_user, encryption_name) if active and fingerprint is not None and len(fingerprint.strip()) > 0: self.email_message.add_header(constants.PUBLIC_FINGERPRINT_HEADER.format( encryption_name.upper()), format_fingerprint(fingerprint)) self.log_message('added {} fingerprint'.format(encryption_name)) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def get_default_key_from_header(self): ''' Gets the default public key from the email_message header. ''' return self.get_public_key_from_header(constants.PUBLIC_KEY_HEADER) def get_public_key_from_header(self, header_name): ''' Gets the public key from the email_message header. ''' key = None try: key = inspect_utils.get_multientry_header(self.email_message.get_message(), header_name) if key is not None and len(key.strip()) <= 0: key = None except Exception: self.log_message("No public key found in email message") return key def has_public_key_header(self, encryption_name): ''' Return true if a public key header exists for the encryption software. ''' has_key = False try: header_name = utils.get_public_key_header_name(encryption_name) email_message_key = inspect_utils.get_multientry_header(self.email_message.get_message(), header_name) has_key = email_message_key != None and len(email_message_key) > 0 except Exception as exception: # whatever the error, the point is we didn't get a public key header self.log_message(exception) if has_key: self.log_message("email_message already has public key header for encryption program {}".format(encryption_name)) return has_key def set_filtered(self, filtered): ''' Sets whether this email_message has been changed by a filter. >>> crypto_message = CryptoMessage() >>> crypto_message.set_filtered(True) >>> crypto_message.is_filtered() True ''' if self.DEBUGGING: self.log_message("set filtered: {}".format(filtered)) self.filtered = filtered def is_filtered(self): ''' Gets whether this email_message has been changed by a filter. >>> crypto_message = CryptoMessage() >>> crypto_message.is_filtered() False ''' return self.filtered def set_crypted(self, crypted): ''' Sets whether this email_message has been encrypted or decrypted, even partially. You can check whether an inner email_message is still encrypted with email_email_message.is_probably_pgp(). >>> crypto_message = CryptoMessage() >>> crypto_message.set_crypted(True) >>> crypto_message.is_crypted() True ''' if self.DEBUGGING: self.log_message("set crypted: {}".format(crypted)) self.crypted = crypted def is_crypted(self): ''' Returns whether this email_message has been encrypted or decrypted, even partially. You can check whether an inner email_message is still encrypted with email_email_message.is_probably_pgp(). >>> crypto_message = CryptoMessage() >>> crypto_message.is_crypted() False ''' return self.crypted def set_metadata_crypted(self, crypted): ''' Sets whether this email_message has its metadata encrypted or decrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.set_metadata_crypted(True) >>> crypto_message.is_metadata_crypted() True ''' if self.DEBUGGING: self.log_message("set metadata crypted: {}".format(crypted)) self.metadata_crypted = crypted def is_metadata_crypted(self): ''' Returns whether this email_message has been encrypted or decrypted, even partially. You can check whether an inner email_message is still encrypted with email_email_message.is_probably_pgp(). >>> crypto_message = CryptoMessage() >>> crypto_message.is_metadata_crypted() False ''' return self.metadata_crypted def is_signed(self): ''' Returns whether this email_message has any type of signature. >>> crypto_message = CryptoMessage() >>> crypto_message.is_signed() False ''' return self.is_private_signed() or self.is_clear_signed() or self.is_dkim_signed() def set_private_signed(self, signed): ''' Sets whether this email_message has been signed when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.set_private_signed(True) >>> crypto_message.is_private_signed() True ''' if self.DEBUGGING: self.log_message("set private signed: {}".format(signed)) self.private_signed = signed def is_private_signed(self): ''' Returns whether this email_message has been signed when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.is_private_signed() False ''' return self.private_signed def is_private_sig_verified(self): ''' Returns whether this email_message's signature has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.is_private_sig_verified() False ''' return is_sig_verified(self.private_signers_list()) def set_private_signers(self, signers): ''' Set who signed this email_message when encrypted. >>> private_signers = [{'signer': '*****@*****.**', 'verified': True}] >>> crypto_message = CryptoMessage() >>> crypto_message.set_private_signers( ... [{u'signer': u'*****@*****.**', u'verified': True}]) >>> signers = crypto_message.private_signers_list() >>> signers == private_signers True ''' self.private_signers = signers def add_private_signer(self, signer): ''' Add who signed this email_message when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.add_private_signer( ... {constants.SIGNER: '*****@*****.**', constants.SIGNER_VERIFIED: True}) >>> signers = crypto_message.private_signers_list() >>> signers == [{'signer': '*****@*****.**', 'verified': True}] True ''' self.add_signer(signer, self.private_signers_list()) def private_signers_list(self): ''' Returns a list of signers when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.private_signers_list() [] ''' if self.private_signers is None: self.set_private_signers([]) return self.private_signers def set_clear_signed(self, signed): ''' Sets whether this email_message has been clear signed. >>> crypto_message = CryptoMessage() >>> crypto_message.set_clear_signed(True) >>> crypto_message.is_clear_signed() True ''' if self.DEBUGGING: self.log_message("set clear signed: {}".format(signed)) self.clear_signed = signed def is_clear_signed(self): ''' Returns whether this email_message has been clear signed. >>> crypto_message = CryptoMessage() >>> crypto_message.is_clear_signed() False ''' return self.clear_signed def is_clear_sig_verified(self): ''' Returns whether this email_message's clear signature has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.is_clear_sig_verified() False ''' return is_sig_verified(self.clear_signers_list()) def set_clear_signers(self, signers): ''' Set who clear signed this email_message. >>> crypto_message = CryptoMessage() >>> crypto_message.set_clear_signers( ... [{'signer': '*****@*****.**', 'verified': True}]) >>> signers = crypto_message.clear_signers_list() >>> signers == [{'signer': '*****@*****.**', 'verified': True}] True ''' self.clear_signers = signers def add_clear_signer(self, signer_dict): ''' Add who clear signed this email_message. >>> crypto_message = CryptoMessage() >>> crypto_message.add_clear_signer({'signer': '*****@*****.**', 'verified': True}) >>> signers = crypto_message.clear_signers_list() >>> signers == [{'signer': '*****@*****.**', 'verified': True}] True ''' self.add_signer(signer_dict, self.clear_signers_list()) def clear_signers_list(self): ''' Returns a list of clear signers. >>> crypto_message = CryptoMessage() >>> crypto_message.clear_signers_list() [] ''' if self.clear_signers is None: self.set_clear_signers([]) return self.clear_signers def set_dkim_signed(self, signed): ''' Sets whether this email_message has been signed using DKIM. >>> crypto_message = CryptoMessage() >>> crypto_message.set_dkim_signed(True) >>> crypto_message.is_dkim_signed() True ''' if self.DEBUGGING: self.log_message("set dkim signed: {}".format(signed)) self.dkim_signed = signed def is_dkim_signed(self): ''' Returns whether this email_message has been signed using DKIM. >>> crypto_message = CryptoMessage() >>> crypto_message.is_dkim_signed() False ''' return self.dkim_signed def set_dkim_sig_verified(self, verified): ''' Sets whether this email_message's DKIM sig has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.set_dkim_sig_verified(True) >>> crypto_message.is_dkim_sig_verified() True ''' if self.DEBUGGING: self.log_message("set dkim sig verified: {}".format(verified)) self.dkim_verified = verified def is_dkim_sig_verified(self): ''' Returns whether this email_message's DKIM sig has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.is_dkim_sig_verified() False ''' return self.dkim_verified def add_signer(self, signer_dict, signer_list): ''' Add who signed this email_message. >>> crypto_message = CryptoMessage() >>> clear_signers = crypto_message.clear_signers_list() >>> crypto_message.add_signer({'signer': '*****@*****.**', 'verified': True}, clear_signers) ''' if signer_dict is not None: signer = signer_dict[constants.SIGNER] if signer is not None: signer = get_email(signer) # now make the signer readable if unknown if signer == None: signer = 'unknown user' signer_dict[constants.SIGNER] = signer if signer_dict not in signer_list: if self.DEBUGGING: self.log_message("add signer: {}".format(signer_dict)) signer_list.append(signer_dict) def drop(self, dropped=True): ''' Sets whether this email_message has been dropped by a filter. If the message is dropped, then it's never returned to postfix. >>> crypto_message = CryptoMessage() >>> crypto_message.drop() >>> crypto_message.is_dropped() True ''' if self.DEBUGGING: self.log_message("set dropped: {}".format(dropped)) self.dropped = dropped def is_dropped(self): ''' Gets whether this email_message has been dropped by a filter. If the message is dropped, then it's never returned to postfix. >>> crypto_message = CryptoMessage() >>> crypto_message.is_dropped() False ''' return self.dropped def set_processed(self, processed): ''' Sets whether this email message has been processed by a filter. A processed message does not need any further processing by the caller. >>> crypto_message = CryptoMessage() >>> crypto_message.set_processed(True) >>> crypto_message.is_processed() True ''' if self.DEBUGGING: self.log_message("set processed: {}".format(processed)) self.processed = processed def is_processed(self): ''' Gets whether this email_message has been processed by a filter. >>> crypto_message = CryptoMessage() >>> crypto_message.is_processed() False ''' return self.processed def set_crypted_with(self, crypted_with): ''' Sets the encryption programs message was crypted.. >>> crypto_message = CryptoMessage() >>> crypto_message.set_crypted_with(['GPG']) >>> crypted_with = crypto_message.get_crypted_with() >>> crypted_with == ['GPG'] True ''' self.crypted_with = crypted_with if self.DEBUGGING: self.log_message("set crypted_with: {}".format(self.get_crypted_with())) def get_crypted_with(self): ''' Returns the encryption programs message was crypted. >>> crypto_message = CryptoMessage() >>> crypto_message.get_crypted_with() [] ''' return self.crypted_with def set_metadata_crypted_with(self, crypted_with): ''' Sets whether this email_message has its metadata encrypted or decrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.set_metadata_crypted_with(['GPG']) >>> crypted_with = crypto_message.get_metadata_crypted_with() >>> crypted_with == ['GPG'] True ''' self.metadata_crypted_with = crypted_with if self.DEBUGGING: self.log_message("set metadata crypted_with: {}".format(self.get_metadata_crypted_with())) def get_metadata_crypted_with(self): ''' Returns the encryption programs the metadata was encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.get_metadata_crypted_with() [] ''' return self.metadata_crypted_with def is_create_private_keys_active(self): ''' Gets whether creating private keys on the fly is active. >>> crypto_message = CryptoMessage() >>> current_setting = options.create_private_keys() >>> options.set_create_private_keys(True) >>> crypto_message.is_create_private_keys_active() True >>> options.set_create_private_keys(False) >>> crypto_message.is_create_private_keys_active() False >>> options.set_create_private_keys(current_setting) ''' active = options.create_private_keys() if self.DEBUGGING: self.log_message("Create private keys: {}".format(active)) return active def add_tags_to_message(self, tags): ''' Add tag to a message. >>> crypto_message = CryptoMessage() >>> crypto_message.add_tags_to_message('') False ''' def add_tags_to_text(content, tags): ''' Add the tag to the text content. ''' text_content = content text_content = '{}\n\n\n{}\n'.format(text_content, str(tags)) self.log_message('added tags to text content') return text_content def add_tags_to_html(content, tags): ''' Add the tag to the html content. ''' for tag in tags: tag = tag.replace('\n', '<br/>') tag = tag.replace(' ', ' ') html_content = content index = html_content.lower().find('</body>') if index < 0: index = html_content.lower().find('</html>') if index < 0: html_content = '{}<div><hr>\n{}<br/></div>'.format(html_content, str(tags)) else: html_content = '{}<div><hr>\n{}<br/></div>\n{}'.format(html_content[0:index], str(tags), html_content[:index]) self.log_message('added tags to html content') return html_content tags_added = False if tags is None or len(tags.strip()) <= 0: self.log_message('no tags need to be added to message') elif self.get_email_message() is None or self.get_email_message().get_message() is None: self.log_message('email message not formed correctly') else: msg_charset, self._last_charset = inspect_utils.get_charset(self.get_email_message()) content_type = self.get_email_message().get_message().get_content_type() self.log_message("content type: {}".format(content_type)) if content_type is None: pass elif (content_type == mime_constants.TEXT_PLAIN_TYPE or content_type == mime_constants.TEXT_HTML_TYPE): content = self.get_email_message().get_content() if content is None: self.get_email_message().set_content(tags, content_type, charset=msg_charset) tags_added = True else: if content.lower().find('<html>') > 0: content = add_tags_to_html(content, tags) else: content = add_tags_to_text(self.get_email_message().get_content(), tags) self.get_email_message().set_content(content, content_type, charset=msg_charset) tags_added = True elif content_type.startswith(mime_constants.MULTIPART_PRIMARY_TYPE): added_tags_to_text = False added_tags_to_html = False message = self.get_email_message().get_message() for part in message.get_payload(): part_content_type = part.get_content_type().lower() self.log_message('part_content_type: {}'.format(part_content_type)) #DEBUG if part_content_type == mime_constants.TEXT_PLAIN_TYPE and not added_tags_to_text: content = add_tags_to_text(part.get_payload(), tags) charset, __ = inspect_utils.get_charset(content) if charset.lower() == msg_charset.lower(): part.set_payload(content) else: part.set_payload(content, charset=charset) added_tags_to_text = True elif part_content_type == mime_constants.TEXT_HTML_TYPE and not added_tags_to_html: content = add_tags_to_html(part.get_payload(), tags) charset, __ = inspect_utils.get_charset(content) part.set_payload(content, charset=charset) added_tags_to_html = True # no need to keep getting payloads if we've added the tags if added_tags_to_text and added_tags_to_html: break tags_added = added_tags_to_text or added_tags_to_html if not tags_added: msg = MIMENonMultipart(mime_constants.TEXT_PRIMARY_TYPE, mime_constants.PLAIN_SUB_TYPE) msg.set_payload(tags) self.get_email_message().get_message().attach(msg) self.log_message('attached new payload\n{}'.format(msg)) tags_added = True self.log_message('added tags to multipart message: {}'.format(tags_added)) else: self.log_message('unable to add tags to message with {} content type'.format(content_type)) return tags_added def get_tags(self): ''' Returns the list of tags to be added to the email_message text. >>> regular_tags = ['test tag'] >>> crypto_message = CryptoMessage() >>> crypto_message.set_tag('test tag') >>> tags = crypto_message.get_tags() >>> crypto_message.log_message('tags: {}'.format(tags)) >>> tags == regular_tags True ''' if self.DEBUGGING: self.log_message("tags:\n{}".format(self.tags)) return self.tags def get_tag(self): ''' Returns the tags as a string to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_tag('test tag') >>> tags = crypto_message.get_tag() >>> tags == 'test tag' True ''' if self.tags is None: tag = '' else: tag = '\n'.join(self.tags) if self.DEBUGGING: self.log_message("tag:\n{}".format(tag)) return tag def set_tag(self, new_tag): ''' Sets the tag to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_tag(None) >>> tag = crypto_message.get_tag() >>> tag == '' True ''' if new_tag is None: if self.DEBUGGING: self.log_message("tried to set blank tag") elif new_tag == '': self.tags = [] if self.DEBUGGING: self.log_message("reset tags") else: if is_string(new_tag): new_tag = new_tag.strip('\n') self.tags = [new_tag] else: self.tags = new_tag if self.DEBUGGING: self.log_message("new tag:\n{}".format(new_tag)) def add_tag(self, new_tag): ''' Add new tag to the existing tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_tag(None) >>> tag = crypto_message.get_tag() >>> tag == '' True ''' if new_tag is None or len(new_tag) <= 0: if self.DEBUGGING: self.log_message("tried to add empty tag") else: new_tag = new_tag.strip('\n') if self.tags == None or len(self.tags) <= 0: if self.DEBUGGING: self.log_message("adding to an empty tag:\n{}".format(new_tag)) self.tags = [new_tag] else: if self.DEBUGGING: self.log_message("adding to tag:\n{}".format(new_tag)) if new_tag.startswith('.'): self.tags[len(self.tags) - 1] += new_tag else: self.tags.append(new_tag) def add_tag_once(self, new_tag): ''' Add new tag only if it isn't already in the tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_tag_once(None) >>> tag = crypto_message.get_tag().strip() >>> tag == '' True ''' if new_tag is None: pass elif self.tags is None or new_tag not in self.tags: self.add_tag(new_tag) def add_prefix_to_tag_once(self, new_tag): ''' Add new tag prefix only if it isn't already in the tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_prefix_to_tag_once(None) >>> tag = crypto_message.get_tag() >>> tag == '' True ''' if new_tag is None: pass elif self.tags is None or new_tag not in self.tags: new_tag = new_tag.strip('\n') if self.DEBUGGING: self.log_message("adding prefix to tag:\n{}".format(new_tag)) if self.tags == None or len(self.tags) <= 0: self.tags = [new_tag] else: old_tags = self.tags self.tags = [new_tag] for tag in old_tags: self.tags.append(tag) def get_error_tags(self): ''' Returns the list of error tags to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_error_tag('test error tag') >>> tags = crypto_message.get_error_tags() >>> tags == ['test error tag'] True ''' if self.DEBUGGING: self.log_message("error tags:\n{}".format(self.error_tags)) return self.error_tags def get_error_tag(self): ''' Returns the error tags as a string to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_error_tag('test error tag') >>> tag = crypto_message.get_error_tag() >>> tag == 'test error tag' True ''' if self.error_tags is None: error_tag = '' else: error_tag = '\n'.join(self.error_tags) if self.DEBUGGING: self.log_message("error tag:\n{}".format(error_tag)) return error_tag def set_error_tag(self, new_tag): ''' Sets the error tag to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_error_tag(None) >>> tag = crypto_message.get_error_tag() >>> tag == '' True ''' if new_tag is None: if self.DEBUGGING: self.log_message("tried to set blank error tag") elif new_tag == '': self.error_tags = [] if self.DEBUGGING: self.log_message("reset error tags") else: if is_string(new_tag): new_tag = new_tag.strip('\n') self.error_tags = [new_tag] else: self.error_tags = new_tag if self.DEBUGGING: self.log_message("new tag:\n{}".format(self.error_tags)) def add_error_tag_once(self, new_tag): ''' Add new error tag only if it isn't already in the error tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_error_tag_once(None) >>> tag = crypto_message.get_error_tag().strip() >>> tag == '' True ''' if new_tag is None: pass elif self.error_tags is None or new_tag not in self.error_tags: new_tag = new_tag.strip('\n') if len(self.error_tags) <= 0: if self.DEBUGGING: self.log_message("adding to an empty error tag:\n{}".format(new_tag)) self.error_tags = [new_tag] else: self.log_message("adding to error tag:\n{}".format(new_tag)) if new_tag.startswith('.'): self.error_tags[len(self.tags) - 1] += new_tag else: self.error_tags.append(new_tag) def log_exception(self, exception): ''' Log the message to the local and Exception logs. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> CryptoMessage().log_exception('test message') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.crypto_message.log')) True ''' self.log_message(exception) record_exception(message=exception) def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> CryptoMessage().log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.crypto_message.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class MailAPI(object): '''Handle the API for GoodCrypto Mail.''' def __init__(self): self.log = None def interface(self, request): '''Interface with the server through the API. All requests must be via a POST. ''' # final results and error_messages of the actions result = None ok = False response = None try: self.action = self.domain = self.mail_server_address = None self.public_key = self.encryption_name = self.email = self.fingerprint = None self.user_name = self.admin = self.password = None self.ip = get_remote_ip(request) self.log_message('attempting mail api call from {}'.format(self.ip)) if request.method == 'POST': try: form = APIForm(request.POST) if form.is_valid(): cleaned_data = form.cleaned_data self.action = cleaned_data.get(api_constants.ACTION_KEY) self.log_message('action: {}'.format(self.action)) if self.action == api_constants.CREATE_SUPERUSER: self.admin = strip_input(cleaned_data.get(api_constants.SYSADMIN_KEY)) self.log_message('admin: {}'.format(self.admin)) elif self.action == api_constants.CONFIGURE: self.domain = strip_input(cleaned_data.get(api_constants.DOMAIN_KEY)) self.log_message('domain: {}'.format(self.domain)) self.mail_server_address = strip_input(cleaned_data.get(api_constants.MTA_ADDRESS_KEY)) self.log_message('mail_server_address: {}'.format(self.mail_server_address)) elif self.action == api_constants.GET_FINGERPRINT: self.encryption_name = strip_input(cleaned_data.get(api_constants.ENCRYPTION_NAME_KEY)) self.log_message('encryption_name: {}'.format(self.encryption_name)) self.email = strip_input(cleaned_data.get(api_constants.EMAIL_KEY)) self.log_message('email: {}'.format(self.email)) self.password = strip_input(cleaned_data.get(api_constants.PASSWORD_KEY)) elif self.action == api_constants.IMPORT_KEY: self.public_key = strip_input(cleaned_data.get(api_constants.PUBLIC_KEY)) self.encryption_name = strip_input(cleaned_data.get(api_constants.ENCRYPTION_NAME_KEY)) self.log_message('encryption_name: {}'.format(self.encryption_name)) self.fingerprint = strip_input(cleaned_data.get(api_constants.FINGERPRINT_KEY)) self.log_message('fingerprint: {}'.format(self.fingerprint)) self.user_name = strip_input(cleaned_data.get(api_constants.USER_NAME_KEY)) self.log_message('user_name: {}'.format(self.user_name)) self.admin = strip_input(cleaned_data.get(api_constants.SYSADMIN_KEY)) self.log_message('admin: {}'.format(self.admin)) self.password = strip_input(cleaned_data.get(api_constants.PASSWORD_KEY)) elif self.action == api_constants.GET_CONTACT_LIST: self.encryption_name = strip_input(cleaned_data.get(api_constants.ENCRYPTION_NAME_KEY)) self.log_message('encryption_name: {}'.format(self.encryption_name)) self.admin = strip_input(cleaned_data.get(api_constants.SYSADMIN_KEY)) self.log_message('admin: {}'.format(self.admin)) self.password = strip_input(cleaned_data.get(api_constants.PASSWORD_KEY)) result = self.take_api_action() else: result = self.format_bad_result('Invalid form') self.log_attempted_access(result) self.log_message('api form is not valid') self.log_bad_form(request, form) except: result = self.format_bad_result('Unknown error') self.log_attempted_access(result) record_exception() self.log_message('unexpected error while parsing input') self.log_message('EXCEPTION - see syr.exception.log for details') else: self.log_attempted_access('Attempted GET connection') self.log_message('redirecting api GET request to website') response = HttpResponsePermanentRedirect('/') if response is None: response = self.get_api_response(request, result) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') response = HttpResponsePermanentRedirect('/') return response def take_api_action(self): result = None ok, error_message = self.is_data_ok() if ok: if self.action == api_constants.CONFIGURE: mail_options = options.get_options() mail_options.domain = self.domain mail_options.mail_server_address = self.mail_server_address options.save_options(mail_options) result = self.format_result(api_constants.CONFIGURE, ok) self.log_message('configure result: {}'.format(result)) elif self.action == api_constants.CREATE_SUPERUSER: user, password, error_message = create_superuser(self.admin) if error_message is None: if password is None: result = self.format_bad_result(error_message) else: result = self.format_message_result(api_constants.CREATE_SUPERUSER, ok, password) else: result = self.format_bad_result(error_message) self.log_message('create user result: {}'.format(result)) elif self.action == api_constants.STATUS: result = self.format_result(api_constants.STATUS, get_mail_status()) self.log_message('status result: {}'.format(result)) elif self.action == api_constants.IMPORT_KEY: from goodcrypto.mail.import_key import import_key_now result_ok, status, fingerprint_ok = import_key_now( self.encryption_name, self.public_key, self.user_name, self.fingerprint, self.password) if result_ok: __, email = status.split(':') result = self.format_message_result(api_constants.IMPORT_KEY, True, email) else: result = self.format_bad_result(status) self.log_message('import key result: {}'.format(result)) elif self.action == api_constants.GET_CONTACT_LIST: email_addresses = contacts.get_contact_list(self.encryption_name) addresses = '\n'.join(email_addresses) result = self.format_message_result(api_constants.GET_CONTACT_LIST, True, addresses) self.log_message('{} {} contacts found'.format(len(email_addresses), self.encryption_name)) elif self.action == api_constants.GET_FINGERPRINT: fingerprint, verified, __ = contacts.get_fingerprint(self.email, self.encryption_name) if fingerprint is None: ok = False error_message = 'No {} fingerprint for {}'.format(self.encryption_name, self.email) result = self.format_bad_result(error_message) self.log_message('bad result: {}'.format(result)) else: message = 'Fingerprint: {} verified: {}'.format(format_fingerprint(fingerprint), verified) result = self.format_message_result(api_constants.GET_FINGERPRINT, True, message) self.log_message(message) else: ok = False error_message = 'Bad action: {}'.format(self.action) result = self.format_bad_result(error_message) self.log_message('bad action result: {}'.format(result)) else: result = self.format_bad_result(error_message) self.log_message('data is bad') return result def is_data_ok(self): '''Check if all the required data is present.''' error_message = '' ok = False if self.has_content(self.action): if self.action == api_constants.CONFIGURE: if self.has_content(self.domain) and self.has_content(self.mail_server_address): ok = True self.log_message('minimum configure data found') elif self.action == api_constants.CREATE_SUPERUSER: if self.has_content(self.admin): ok = True self.log_message('minimum create user data found: {}'.format(self.admin)) elif self.action == api_constants.STATUS: ok = True self.log_message('status request found') elif self.action == api_constants.GET_FINGERPRINT: if (self.has_content(self.encryption_name) and self.has_content(self.email)): ok = True self.log_message('minimum get fingerprint data found') elif self.action == api_constants.IMPORT_KEY: if (self.has_content(self.public_key) and self.has_content(self.encryption_name) and self.has_content(self.admin) and self.has_content(self.password)): ok = True self.log_message('minimum import key data found') elif self.action == api_constants.GET_CONTACT_LIST: if (self.has_content(self.encryption_name) and self.has_content(self.admin) and self.has_content(self.password)): ok = True self.log_message('minimum get contact list data found') if not ok: error_message = 'Missing required data' self.log_message('missing required data') else: ok = False error_message = 'Missing required action' self.log_message('missing required action') return ok, error_message def has_content(self, value): '''Check that the value has content.''' try: str_value = str(value) if str_value is None or len(str_value.strip()) <= 0: ok = False else: ok = True except: ok = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return ok def format_result(self, action, ok, error_message=None): '''Format the action's result.''' if error_message is None: result = {api_constants.ACTION_KEY: action, api_constants.OK_KEY: ok} else: result = { api_constants.ACTION_KEY: action, api_constants.OK_KEY: ok, api_constants.ERROR_KEY: error_message } return result def format_message_result(self, action, ok, message): '''Format the action's result.''' result = { api_constants.ACTION_KEY: action, api_constants.OK_KEY: ok, api_constants.MESSAGE_KEY: message } return result def format_bad_result(self, error_message): '''Format the bad result for the action.''' result = None if self.action and len(self.action) > 0: result = self.format_result(self.action, False, error_message=error_message) else: result = self.format_result('Unknown', False, error_message=error_message) self.log_message('action result: {}'.format(error_message)) return result def get_api_response(self, request, result): ''' Get API reponse as JSON. ''' json_result = json.dumps(result) self.log_message('json results: {}'.format(''.join(json_result))) response = render_to_response('mail/api_response.html', {'result': ''.join(json_result),}, context_instance=RequestContext(request)) return response def log_attempted_access(self, results): '''Log an attempted access to the api.''' self.log_message('attempted access from {} for {}'.format(self.ip, results)) def log_bad_form(self, request, form): ''' Log the bad fields entered.''' # see django.contrib.formtools.utils.security_hash() # for example of form traversal for field in form: if (hasattr(form, 'cleaned_data') and field.name in form.cleaned_data): name = field.name else: # mark invalid data name = '__invalid__' + field.name self.log_message('name: {}; data: {}'.format(name, field.data)) try: if form.name.errors: self.log_message(' ' + form.name.errors) if form.email.errors: self.log_message(' ' + form.email.errors) except: pass self.log_message('logged bad api form') def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> MailAPI().log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.api.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class HeaderKeys(object): ''' Manage keys from a message's header. It's important that we only import a key if we don't have a key for this user. If we already do, then we must be very careful deciding whether to use the key or not. The following table shows the conditions that we must handle if there's a key in the header. If we decide not to use a key, then a message is sent to the recipient explaining our reason for not processing the message. The original offending message is included as an attachment. Received key in header so check db and crypto package. | saved in db | crypto package | action | unit test | |--------------|----------------|------------|---------------------------------------| | match | match | use | test_all_matching_keys | | match | no match | do not use | test_db_fingerprint_bad_crypto_key | | match | missing | do not use | test_db_fingerprint_no_crypto_key | | no match | match | do not use | test_bad_fingerprint_matching_crypto | | no match | no match | do not use | test_bad_fingerprint_bad_crypto | | no match | missing | do not use | test_bad_fingerprint_missing_crypto | | missing | match | add to db | test_crypto_key_no_db_fingerprint | | missing | no match | do not use | test_bad_crypto_key_no_db_fingerprint | | missing | missing | import | test_no_existing_keys | ''' DEBUGGING = False UNEXPECTED_ERROR = i18n( 'An unexpected error ocurred while processing this message') def __init__(self): ''' >>> header_keys = HeaderKeys() >>> header_keys != None True ''' self.log = LogFile() self.recipient_to_notify = None self.new_key_imported = False def manage_keys_in_header(self, crypto_message): ''' Manage all the public keys in the message's header. ''' header_contains_key_info = False try: from_user = crypto_message.smtp_sender() self.recipient_to_notify = crypto_message.smtp_recipient() # all notices about a metadata address goes to the admin if is_metadata_address(self.recipient_to_notify): self.recipient_to_notify = get_admin_email() name, address = parse_address(from_user) if address is None or crypto_message is None or crypto_message.get_email_message( ) is None: self.log_message('missing data so cannot import key') self.log_message(' from user: {}'.format(from_user)) self.log_message(' address: {}'.format(address)) self.log_message( ' crypto message: {}'.format(crypto_message)) if crypto_message is not None: self.log_message(' email message: {}'.format( crypto_message.get_email_message())) else: accepted_crypto_packages = crypto_message.get_accepted_crypto_software( ) if accepted_crypto_packages is None or len( accepted_crypto_packages) <= 0: self.log_message( "checking for default key for {} <{}>".format( name, address)) tag = self._manage_key_header( address, crypto_message, KeyFactory.get_default_encryption_name(), PUBLIC_KEY_HEADER) else: self.log_message("checking for {} keys".format( accepted_crypto_packages)) for encryption_name in accepted_crypto_packages: # see if there's a the key block for this encryption program header_name = get_public_key_header_name( encryption_name) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), header_name) # see if there's a plain key block if ((key_block is None or len(key_block) <= 0) and len(accepted_crypto_packages) == 1): self.log_message( "no {} public key in header so trying generic header" .format(encryption_name)) key_block = get_multientry_header( crypto_message.get_email_message().get_message( ), PUBLIC_KEY_HEADER) tag = self._manage_key_header(address, crypto_message, encryption_name, key_block) header_contains_key_info = True self.update_accepted_crypto(from_user, accepted_crypto_packages) except MessageException as message_exception: self.log_message(message_exception.value) raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if crypto_message is not None: crypto_message.add_error_tag_once(self.UNEXPECTED_ERROR) self.log_message( 'header_contains_key_info: {}'.format(header_contains_key_info)) return header_contains_key_info def keys_in_header(self, crypto_message): ''' Return true if there are public keys in the message's header. ''' header_contains_key_info = False try: good_key = True from_user = crypto_message.smtp_sender() accepted_crypto_packages = crypto_message.get_accepted_crypto_software( ) if accepted_crypto_packages is not None and len( accepted_crypto_packages) > 0: self.log_message( "checking for {} keys".format(accepted_crypto_packages)) for encryption_name in accepted_crypto_packages: # see if there's a the key block for this encryption program header_name = get_public_key_header_name(encryption_name) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), header_name) # see if there's a plain key block if ((key_block is None or len(key_block) <= 0) and len(accepted_crypto_packages) == 1): self.log_message( "no {} public key in header so trying generic header" .format(encryption_name)) key_block = get_multientry_header( crypto_message.get_email_message().get_message(), PUBLIC_KEY_HEADER) if key_block is not None and len(key_block) > 0: header_contains_key_info = True user_ids = [] key_crypto = KeyFactory.get_crypto( encryption_name, crypto_software.get_key_classname(encryption_name)) if key_crypto is None: id_fingerprint_pairs = None self.log_message( 'no key crypto for {}'.format(encryption_name)) else: id_fingerprint_pairs = key_crypto.get_id_fingerprint_pairs( key_block) for (user_id, __) in id_fingerprint_pairs: user_ids.append(user_id) self.log_message( "key block includes key for {}".format( user_ids)) # don't consider a key valid if it's not from the sender if from_user not in user_ids: tag = notices.report_bad_header_key( self.recipient_to_notify, from_user, user_ids, encryption_name, crypto_message) self.log_message('tag: {}'.format(tag)) if crypto_message.get_email_message( ).is_probably_pgp(): crypto_message.drop(dropped=True) self.log_message( 'serious error in header so original message sent as attchment' ) raise MessageException(value=i18n(tag)) self.update_accepted_crypto(from_user, accepted_crypto_packages) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if crypto_message is not None: crypto_message.add_error_tag_once(self.UNEXPECTED_ERROR) return header_contains_key_info def new_key_imported_from_header(self): ''' Return true if a new key was imported from the header. ''' return self.new_key_imported def _manage_key_header(self, from_user, crypto_message, encryption_name, key_block): ''' Manage a key in the header for the encryption software (internal use only). ''' tag = None try: if key_block == None or len(key_block.strip()) <= 0: self.log_message( "no {} public key in header".format(encryption_name)) else: if self.DEBUGGING: self.log_message("{} key from message:\n{}".format( encryption_name, key_block)) key_crypto = KeyFactory.get_crypto( encryption_name, crypto_software.get_key_classname(encryption_name)) if key_crypto is None: id_fingerprint_pairs = None self.log_message( 'no key crypto for {}'.format(encryption_name)) else: id_fingerprint_pairs = key_crypto.get_id_fingerprint_pairs( key_block) if id_fingerprint_pairs is None or len( id_fingerprint_pairs) <= 0: tag = None self.log_message('no user keys in key block') else: tag = self._manage_public_key(from_user, crypto_message, key_crypto, key_block, id_fingerprint_pairs) if tag is not None and len(tag.strip()) > 0: crypto_message.add_tag_once(tag) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return tag def _manage_public_key(self, from_user, crypto_message, key_crypto, key_block, id_fingerprint_pairs): ''' Manage a public key for the encryption software (internal use only). ''' tag = None drop = False encryption_name = key_crypto.get_name() user_ids = [] for (user_id, __) in id_fingerprint_pairs: user_ids.append(user_id) self.log_message("key block includes key for {}".format(user_ids)) if from_user in user_ids: # see if we already have a key for this ID or if it has expired saved_fingerprint, verified, __ = contacts.get_fingerprint( from_user, encryption_name) crypto_fingerprint, expiration = key_crypto.get_fingerprint( from_user) self.log_message("{} {} saved fingerprint {}".format( from_user, encryption_name, saved_fingerprint)) self.log_message("{} {} crypto fingerprint {} expires {}".format( from_user, encryption_name, crypto_fingerprint, expiration)) self.log_message("{} {} id fingerprint pairs {}".format( from_user, encryption_name, id_fingerprint_pairs)) if crypto_fingerprint is None: if saved_fingerprint is None: self.log_message("importing new key") tag = self._import_new_key(from_user, encryption_name, key_block, id_fingerprint_pairs) else: self.log_message("checking if key matches") drop = crypto_message.get_email_message().is_probably_pgp() key_matches, key_error = self._key_matches( encryption_name, saved_fingerprint, id_fingerprint_pairs) self.log_message("key matches: {} / key error: {}".format( key_matches, key_error)) if key_error: tag = notices.report_error_verifying_key( self.recipient_to_notify, from_user, encryption_name, crypto_message) else: tag = notices.report_missing_key( self.recipient_to_notify, from_user, key_matches, id_fingerprint_pairs, crypto_message) else: if saved_fingerprint is None: # remember the fingerprint for the future saved_fingerprint = crypto_fingerprint if (strip_fingerprint(crypto_fingerprint).lower() == strip_fingerprint(saved_fingerprint).lower()): key_matches, key_error = self._key_matches( encryption_name, crypto_fingerprint, id_fingerprint_pairs) if key_error: tag = notices.report_error_verifying_key( self.recipient_to_notify, from_user, encryption_name, crypto_message) elif key_matches: if key_crypto.fingerprint_expired(expiration): drop = crypto_message.get_email_message( ).is_probably_pgp() tag = notices.report_expired_key( self.recipient_to_notify, from_user, encryption_name, expiration, crypto_message) else: self.log_message(' same fingerprint: {}'.format( saved_fingerprint)) # warn user if unable to save the fingerprint, but proceed if not self.update_fingerprint( from_user, encryption_name, crypto_fingerprint): tag = notices.report_db_error( self.recipient_to_notify, from_user, encryption_name, crypto_message) else: if self.DEBUGGING: self.log_message('{} key block\n{}'.format( from_user, key_crypto.export_public(from_user))) drop = crypto_message.get_email_message( ).is_probably_pgp() tag = notices.report_replacement_key( self.recipient_to_notify, from_user, encryption_name, id_fingerprint_pairs, crypto_message) else: drop = crypto_message.get_email_message().is_probably_pgp() tag = notices.report_mismatched_keys( self.recipient_to_notify, from_user, encryption_name, crypto_message) else: if len(user_ids) > 0: drop = crypto_message.get_email_message().is_probably_pgp() tag = notices.report_bad_header_key(self.recipient_to_notify, from_user, user_ids, encryption_name, crypto_message) else: self.log_message( 'no importable keys found\n{}'.format(key_block)) self.log_message('tag: {}'.format(tag)) if drop: crypto_message.drop(dropped=True) self.log_message( 'serious error in header so original message sent as attchment' ) raise MessageException(value=i18n(tag)) return tag def _import_new_key(self, from_user, encryption_name, key_block, id_fingerprint_pairs): ''' Import a new key (internal use only). ''' tag = None result_ok = False if encryption_name is None: encryption_name = '' self.new_key_imported = False try: self.log_message("starting to import new {} key for {}".format( encryption_name, from_user)) if from_user is None or len( encryption_name) == 0 or id_fingerprint_pairs is None: self.log_message( 'missing key data so unable to import new key') else: key_crypto = KeyFactory.get_crypto( encryption_name, crypto_software.get_key_classname(encryption_name)) # make sure that we don't have a key for any of the user ids included with this key result_ok = True if id_fingerprint_pairs is None or len( id_fingerprint_pairs) <= 0: result_ok = False elif len(id_fingerprint_pairs) > 1: for (user_id, __) in id_fingerprint_pairs: crypto_fingerprint, expiration = key_crypto.get_fingerprint( user_id) if crypto_fingerprint is not None: result_ok = False self.log_message( 'key exists for {} so unable to import key for {}' .format(user_id, from_user)) break if result_ok: result_ok = key_crypto.import_public( key_block, id_fingerprint_pairs=id_fingerprint_pairs) self.log_message('imported key: {}'.format(result_ok)) if result_ok: self.new_key_imported = True result_ok = self._add_contacts_and_notify( encryption_name, id_fingerprint_pairs) self.log_message('added contacts: {}'.format(result_ok)) if not result_ok: self.log_message( 'Unable to import new public key for {}; probably taking longer than expected; check Contacts later' .format(from_user)) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_message('import new key ok: {}'.format(result_ok)) return tag def _add_contacts_and_notify(self, encryption_name, id_fingerprint_pairs): result_ok = True # use the first email address from the imported key email, __ = id_fingerprint_pairs[0] for (user_id, fingerprint) in id_fingerprint_pairs: self.log_message( 'adding contact for {} with {} fingerprint'.format( user_id, fingerprint)) contact = contacts.add(user_id, encryption_name, fingerprint=fingerprint, source=MESSAGE_HEADER) if contact is None: result_ok = False self.log_message( 'unable to add contact while trying to import key for {}'. format(email)) else: self.log_message( 'successfully added contact after importing key for {}'. format(email)) if result_ok: notices.notify_new_key_arrived(self.recipient_to_notify, id_fingerprint_pairs) return result_ok def _key_matches(self, encryption_name, old_fingerprint, id_fingerprint_pairs): ''' Does the new key's fingerprint match the old fingerprint? (internal use only) ''' matches = error = False if encryption_name is None: error = True self.log_message( 'unable to compare fingerprints because basic data is missing') else: try: if id_fingerprint_pairs is None or old_fingerprint is None: error = True self.log_message('missing fingerprint for comparison') self.log_message( 'old fingerprint: {}'.format(old_fingerprint)) self.log_message('id fingerprint pairs: {}'.format( id_fingerprint_pairs)) else: fingerprints = [] for (__, fingerprint) in id_fingerprint_pairs: fingerprints.append(fingerprint.replace(' ', '')) matches = old_fingerprint.replace(' ', '') in fingerprints except: error = True matches = False record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') return matches, error def _import_accepted_crypto_software(self, from_user, crypto_message): ''' Import the encryption software the contact can use (internal use only). ''' accepted_crypto_packages = crypto_message.get_accepted_crypto_software( ) self.update_accepted_crypto(from_user, accepted_crypto_packages) return accepted_crypto_packages def update_accepted_crypto(self, email, encryption_software_list): ''' Update the list of encryption software accepted by user. ''' if email is None: self.log_message( "email not defined so no need to update accepted crypto") elif encryption_software_list is None or len( encryption_software_list) <= 0: self.log_message( 'no encryption programs defined for {}'.format(email)) else: contact = contacts.get(email) if contact is None: # if the contact doesn't exist, then add them with the first encryption program encryption_program = encryption_software_list[0] contact = contacts.add(email, encryption_program, source=MESSAGE_HEADER) self.log_message("added {} to contacts".format(email)) # associate each encryption program in the list with this contact for encryption_program in encryption_software_list: try: contacts_crypto = contacts.get_contacts_crypto( email, encryption_program) if contacts_crypto is None: encryption_software = crypto_software.get( encryption_program) if encryption_software is None: self.log_message( '{} encryption software unknown'.format( encryption_program)) self.log_message( 'unable to add contacts crypt for {} using {} encryption software unknown' .format(email, encryption_program)) else: contacts_crypto = contacts.add_contacts_crypto( contact=contact, encryption_software=encryption_software, source=MESSAGE_HEADER) except Exception: record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') def update_fingerprint(self, email, encryption_name, new_fingerprint, verified=False): ''' Set the fingerprint for the encryption software for this email. If the fingerprint doesn't match the crypto's fingerprint, it won't be saved in the database. ''' if email is None or encryption_name is None: result_ok = False self.log_message( "missing data to save {} fingerprint for {}".format( encryption_name, email)) else: self.log_message('updating {} fingerprint for {}'.format( encryption_name, email)) contacts_crypto = contacts.get_contacts_crypto( email, encryption_name=encryption_name) if contacts_crypto is None: contact = contacts.add(email, encryption_name, source=MESSAGE_HEADER) contacts_crypto = contacts.get_contacts_crypto( email, encryption_name=encryption_name) if contacts_crypto is None: result_ok = False self.log_message( "unable to save contact's {} fingerprint".format( encryption_name)) else: try: need_update = False if new_fingerprint != contacts_crypto.fingerprint: contacts_crypto.fingerprint = new_fingerprint need_update = True self.log_message( "contacts_crypto fingerprint: {}".format( contacts_crypto.fingerprint)) if contacts_crypto.verified != verified: contacts_crypto.verified = verified need_update = True self.log_message( 'Updated verification status: {}'.format(verified)) if need_update: contacts_crypto.save() self.log_message('saved changes') result_ok = True except: self.log_message( 'EXCEPTION - see syr.exception.log for details') record_exception() result_ok = False return result_ok def log_message(self, message): ''' Log the message to the local log. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
def execute_gpg_command(home_dir, initial_args, passphrase=None, data=None): ''' Issue a GPG command in its own worker so there are no concurrency challenges. ''' log = LogFile(filename='goodcrypto.oce.gpg_exec_queue.log') gpg_exec = None try: if initial_args is not None: new_args = [] for arg in initial_args: new_args.append(arg) initial_args = new_args log.write_and_flush('gpg exec: {}'.format(initial_args)) auto_check_trustdb = gpg_constants.CHECK_TRUSTDB in initial_args # different higher levels may try to generate the same key # so only allow one key to be generated if gpg_constants.GEN_KEY in initial_args: command_ready = need_private_key(home_dir, data) if not command_ready: result_code = gpg_constants.GOOD_RESULT gpg_output = gpg_constants.KEY_EXISTS gpg_error = None log.write_and_flush('{}'.format(gpg_output)) # if deleting a key, get the fingerprint because gpg # only allows deletion in batch mode with the fingerprint elif gpg_constants.DELETE_KEYS in initial_args: fingerprint = prep_to_delete_key(home_dir, initial_args) if fingerprint is not None: initial_args = [gpg_constants.DELETE_KEYS, fingerprint] log.write_and_flush('ready to delete key: {}'.format(fingerprint)) command_ready = True else: command_ready = True if command_ready: gpg_exec = GPGExec(home_dir, auto_check_trustdb=auto_check_trustdb) result_code, gpg_output, gpg_error = gpg_exec.execute( initial_args, passphrase, data) log.write_and_flush('result code: {}'.format(result_code)) except JobTimeoutException as job_exception: log.write_and_flush('gpg exec {}'.format(str(job_exception))) result_code = gpg_constants.TIMED_OUT_RESULT gpg_error = str(job_exception) gpg_output = None log.write_and_flush('EXCEPTION - see syr.exception.log for details') record_exception() except Exception as exception: result_code = gpg_constants.ERROR_RESULT gpg_output = None gpg_error = str(exception) log.write_and_flush('EXCEPTION - see syr.exception.log for details') record_exception() if gpg_exec is not None and gpg_exec.gpg_home is not None: gpg_exec.clear_gpg_lock_files() gpg_exec.clear_gpg_tmp_files() log.flush() return result_code, gpg_output, gpg_error
class Encrypt(object): """ Encrypt message filter. This class encodes MIME messages using RFC 3156 "MIME Security with OpenPGP", specifically section 6.2 "Combined method". We send PGP public keys in the header instead of in an application/pgp-keys MIME body part so the keys are transparent to users who do not yet encrypt, and so we can specify when a key is for use with a particular encryption software. Section 6.1 of RFC 3156 gives a way to specify what was encrypted. We include the encrypted content type in a header instead, apparently because section 6.1 looked like it was just for message signatures. But that's not how it is used by OpenPGP MIME packages such as EnigMail. We need to apply section 6.1 the same way. """ DEBUGGING = False MUST_ENCRYPT_ALL_MAIL = i18n("Message not sent because all messages must be encrypted. Contact your mail administrator if you'd like to be able to send plain text messages to {to_email}.") MUST_ENCRYPT_MAIL_TO_USER = i18n("Message not sent because all messages to {to_email} must be encrypted. Contact your mail administrator if you'd like to be able to send plain text messages to this contact.") UNABLE_TO_ENCRYPT = i18n("Error while trying to encrypt message from {from_email} to {to_email} using {encryption}") MESSAGE_TOO_LARGE = i18n('Message too large to send. The maximum size, including attachments, is {kb_size} KB.') POSSIBLE_ENCRYPT_SOLUTION = i18n("Report this error to your mail administrator.") def __init__(self, crypto_message): ''' >>> encrypt = Encrypt(None) >>> encrypt != None True ''' self._log = LogFile() self.crypto_message = crypto_message self.verification_code = None self.ready_to_protect_metadata = False def process_message(self): ''' Process a message and encrypt if possible, bounce if unable to encrypt and encryption required, or add a warning about the danger of sending unencrypted messages. Also, add clear sig and DKIM sig if user opted for either or both. ''' filtered = encrypted = False inner_encrypted_with = [] # a place for the original message in case metadata is protected original_crypto_message = None try: if self.crypto_message is None: self._log_message('no crypto message defined') self.crypto_message = CryptoMessage() from_user = to_user = None else: from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() self._log_message("trying to encrypt message from {} to {}".format(from_user, to_user)) self.verification_code = gen_verification_code() self.ready_to_protect_metadata = is_ready_to_protect_metadata(from_user, to_user) contact = contacts.get(to_user) if contact is None: never_encrypt = True encryption_names = [] if options.use_keyservers(): # if there's no contact, then start another job # that can determine whether we can find a key self._start_check_for_encryption(from_user, to_user) else: never_encrypt = contact.outbound_encrypt_policy == NEVER_ENCRYPT_OUTBOUND encryption_names = contacts.get_encryption_names(to_user) if len(encryption_names) <= 0 or never_encrypt: self._process_plain_message(from_user, to_user, never_encrypt, encryption_names) else: inner_encrypted_with = self._process_encrypt_message(encryption_names) if self.ready_to_protect_metadata: original_crypto_message = copy.copy(self.crypto_message) self._protect_metadata(from_user, to_user, inner_encrypted_with) else: self._process_no_metadata_message(to_user) if self.crypto_message.is_processed(): self._log_message('message processed and awaiting bundling') else: self._finish_processing_message(inner_encrypted_with, original_crypto_message) except MessageException as message_exception: raise MessageException(value=message_exception.value) except Exception as IOError: record_exception() self._log_message('EXCEPTION - see syr.exception.log for details') if self.crypto_message is not None: self._log_message(' final status: filtered: {} encrypted: {}'.format( self.crypto_message.is_filtered(), self.crypto_message.is_crypted())) return self.crypto_message def _process_encrypt_message(self, encryption_names): ''' Handle the initial processing of a message that needs encryption. ''' inner_encrypted_with = [] try: if self.DEBUGGING: self._log_message('message before encryption:\n{}'.format( self.crypto_message.get_email_message().to_string())) self._log_message('trying to encrypt using {}'.format(encryption_names)) inner_encrypted_with = self._encrypt_message_with_all(encryption_names) if self.DEBUGGING: self._log_message('message after encryption:\n{}'.format( self.crypto_message.get_email_message().to_string())) except MessageException as message_exception: raise MessageException(value=message_exception.value) except Exception as exception: self._log_error(str(exception)) except IOError as io_error: self._log_error(io_error.value) return inner_encrypted_with def _process_plain_message(self, from_user, to_user, never_encrypt, encryption_names): ''' Handle the initial processing of a message that does not need encryption. ''' self._log_message('{} uses {} known encryption'.format(to_user, len(encryption_names))) if not self.ready_to_protect_metadata: if never_encrypt: self._log_message('{} set not to use any encryption'.format(to_user)) # fail if encryption is required globally or for this individual elif options.require_outbound_encryption(): self._log_message('message not sent because global encryption required') raise MessageException(value=self.MUST_ENCRYPT_ALL_MAIL.format(to_email=to_user)) elif contacts.always_encrypt_outbound(to_user): self._log_message('message not sent because encryption required for {}'.format(to_user)) raise MessageException(value=self.MUST_ENCRYPT_MAIL_TO_USER.format(to_email=to_user)) # see if message must be clear signed if options.clear_sign_email(): self._clear_sign_crypto_message(from_user) # add the public key so the receiver can use crypto with us in the future if options.auto_exchange_keys(): self.crypto_message.add_public_key_to_header(from_user) self._log_message('added {} public key to header'.format(from_user)) # if we're not exchanging keys, but we are creating them, # then start the process in background elif options.create_private_keys(): add_private_key(from_user) self._log_message('created private key for {} if needed'.format(from_user)) self.crypto_message.set_filtered(True) def _process_no_metadata_message(self, to_user): ''' Handle a message without metadata protection. ''' if self.crypto_message.is_crypted(): if self.DEBUGGING: self._log_message('full encrypted message:\n{}'.format( self.crypto_message.get_email_message().to_string())) elif contacts.never_encrypt_outbound(to_user): self._log_message('{} not using encryption'.format(to_user)) self.crypto_message.add_error_tag_once(USE_ENCRYPTION_WARNING) elif options.require_outbound_encryption(): self._log_message('message not sent because global encryption required') raise MessageException(value=self.MUST_ENCRYPT_ALL_MAIL.format(to_email=to_user)) elif contacts.always_encrypt_outbound(to_user): self._log_message('message not sent because encryption required for {}'.format(to_user)) raise MessageException(value=self.MUST_ENCRYPT_MAIL_TO_USER.format(to_email=to_user)) else: self.crypto_message.add_error_tag_once(USE_ENCRYPTION_WARNING) def _finish_processing_message(self, inner_encrypted_with, original_crypto_message): ''' Finish processing message that aren't bundled for later delivery. ''' tags_added = add_encrypted_tags_to_message(self.crypto_message) self._log_message('tags added to encrypted message: {}'.format(tags_added)) # add the DKIM signature if user opted for it self.crypto_message = encrypt_utils.add_dkim_sig_optionally(self.crypto_message) self.crypto_message.set_filtered(True) # finally save a record so the user can verify what security measures were added if (self.crypto_message.is_crypted() or self.crypto_message.is_private_signed() or self.crypto_message.is_clear_signed()): self._add_outbound_record(original_crypto_message, inner_encrypted_with) self._log_message('added outbound history record') return tags_added def _protect_metadata(self, from_user, to_user, inner_encrypted_with): ''' Protect the message's metadata and resist traffic analysis. ''' if options.bundle_and_pad(): tags_added = add_encrypted_tags_to_message(self.crypto_message) self._log_message('tags added to encrypted message before bundling: {}'.format(tags_added)) # add the DKIM signature if user opted for it self.crypto_message = encrypt_utils.add_dkim_sig_optionally(self.crypto_message) message_name = packetize( self.crypto_message, inner_encrypted_with, self.verification_code) if os.stat(message_name).st_size > options.bundled_message_max_size(): self._log_message('Message too large to bundle so throwing MessageException') if os.path.exists(message_name): os.remove(message_name) error_message = self.MESSAGE_TOO_LARGE.format(kb_size=options.bundle_message_kb()) self._log_message(error_message) raise MessageException(value=error_message) else: self._log_message('message waiting for bundling: {}'.format( self.crypto_message.is_processed())) else: self._log_message('protecting metadata') # add the original sender and recipient in the header self.crypto_message.get_email_message().add_header(constants.ORIGINAL_FROM, from_user) self.crypto_message.get_email_message().add_header(constants.ORIGINAL_TO, to_user) # add the DKIM signature to the inner message if user opted for it self.crypto_message = encrypt_utils.add_dkim_sig_optionally(self.crypto_message) self._log_message('DEBUG: logged headers before encrypting with metadata key in goodcrypto.message.utils.log') log_message_headers(self.crypto_message, tag='headers before encrypting with metadata key') # even if the inner message isn't encrypted, encrypt # the entire message to protect the metadata message_id = get_message_id(self.crypto_message.get_email_message()) self.crypto_message = encrypt_utils.create_protected_message( self.crypto_message.smtp_sender(), self.crypto_message.smtp_recipient(), self.crypto_message.get_email_message().to_string(), message_id) self._log_message('added protective layer for metadata') def _encrypt_message_with_all(self, encryption_names): ''' Encrypt the message with each encryption program. ''' encrypted = fatal_error = False error_message = '' encrypted_with = [] encrypted_classnames = [] to_user = self.crypto_message.smtp_recipient() original_payload = self.crypto_message.get_email_message().get_message().get_payload() self._log_message("encrypting using {} with {}'s key".format(encryption_names, to_user)) for encryption_name in encryption_names: encryption_classname = get_key_classname(encryption_name) if encryption_classname not in encrypted_classnames: try: if options.require_key_verified(): __, key_ok, __ = contacts.get_fingerprint(to_user, encryption_name) self._log_message("{} {} key verified: {}".format(to_user, encryption_name, key_ok)) else: key_ok = True if key_ok: if self._encrypt_message(encryption_name): encrypted_classnames.append(encryption_classname) encrypted_with.append(encryption_name) else: error_message += i18n('You need to verify the {encryption} key for {email} before you can use it.'.format( encryption=encryption_name, email=to_user)) self._log_message(error_message) except MessageException as message_exception: fatal_error = True error_message += message_exception.value self._log_exception(error_message) break # if the user has encryption software defined, then the message # must be encrypted or bounced to the sender if len(encrypted_classnames) > 0: encrypted = True else: MSG_NOT_SET = i18n('Message not sent to {email} because there was a problem encrypting.'.format( email=to_user)) fatal_error = True if error_message is None or len(error_message) <= 0: error_message = '{} {}\n{}'.format(MSG_NOT_SET, i18n("It's possible the recipient's key is missing."), self.POSSIBLE_ENCRYPT_SOLUTION) else: error_message = '{} {}'.format(MSG_NOT_SET, error_message) # restore the payload self.crypto_message.get_email_message().get_message().set_payload(original_payload) if fatal_error: self._log_message('raising message exception in _encrypt_message_with_all') self._log_message(error_message) raise MessageException(value=error_message) return encrypted_with def _encrypt_message(self, encryption_name): ''' Encrypt the message if both To and From have keys. Otherwise, if we just have a key for the From, then add it to the header. ''' result_ok = False try: self._log_message("encrypting message with {}".format(encryption_name)) crypto = CryptoFactory.get_crypto(encryption_name, get_classname(encryption_name)) if crypto is None: self._log_message("{} is not ready to use".format(encryption_name)) elif (self.crypto_message is None or self.crypto_message.smtp_sender() is None or self.crypto_message.smtp_recipient() is None): self._log_message("missing key data to encrypt message") else: # get all the user ids that have encryption keys user_ids = crypto.get_user_ids() if self.DEBUGGING: self._log_message('user ids: {}'.format(user_ids)) private_user_ids = crypto.get_private_user_ids() if self.DEBUGGING: self._log_message('private user ids: {}'.format(private_user_ids)) self._prep_from_user(private_user_ids, encryption_name) users_dict, result_ok = self._get_crypto_details(encryption_name, user_ids) if result_ok or options.create_private_keys(): # regardless if we can encrypt this message, # add From's public key if we have it and we're exchanging keys if options.auto_exchange_keys() and users_dict[encrypt_utils.FROM_KEYWORD] is not None: self.crypto_message.add_public_key_to_header(users_dict[encrypt_utils.FROM_KEYWORD]) else: self._log_message("user_ids is not None and len(user_ids) > 0: {}".format( user_ids is not None and len(user_ids) > 0)) if result_ok: self._log_message("from user ID: {}".format(users_dict[encrypt_utils.FROM_KEYWORD])) self._log_message("to user ID: {}".format(users_dict[encrypt_utils.TO_KEYWORD])) self._log_message("subject: {}".format( self.crypto_message.get_email_message().get_header(mime_constants.SUBJECT_KEYWORD))) result_ok = self._encrypt_message_with_keys(crypto, users_dict) self._log_message('encrypted message: {}'.format(result_ok)) if result_ok: self._log_message('encrypted with: {}'.format(encryption_name)) else: error_message = self._get_encrypt_error_message(users_dict, encryption_name) raise MessageException(value=error_message) else: # the message hasn't been encrypted, but we have # successfully processed it self.crypto_message.set_filtered(True) except MessageException as message_exception: raise MessageException(value=message_exception.value) except Exception: result_ok = False record_exception() self._log_message('EXCEPTION - see syr.exception.log for details') return result_ok def _encrypt_message_with_keys(self, crypto, users_dict): ''' Encrypt a message with the To and From keys. ''' if is_content_type_text(self.crypto_message.get_email_message().get_message()): encrypt_utils.encrypt_text_message(self.crypto_message, crypto, users_dict) else: encrypt_utils.encrypt_mime_message(self.crypto_message, crypto, users_dict) return self.crypto_message.is_crypted() def _get_crypto_details(self, encryption_name, user_ids): ''' Get the details needed to encrypt a message. ''' from_user_id = to_user_id = passcode = None ready_to_encrypt = user_ids is not None and len(user_ids) > 0 if ready_to_encrypt: to_user_id, ready_to_encrypt = self._get_to_crypto_details(encryption_name, user_ids) if ready_to_encrypt or options.auto_exchange_keys(): from_user_id, passcode, error_message = self._get_from_crypto_details(encryption_name, user_ids) if ready_to_encrypt and passcode is None and options.clear_sign_email(): if error_message is not None: error_message += i18n(' and clear signing is required.') if options.create_private_keys(): error_message += '\n\n{}'.format(i18n( "You should wait 5-10 minutes and try again. If that doesn't help, then contact your mail administrator.")) else: error_message += '\n\n{}'.format(i18n( 'Ask your mail administrator to create a private key for you.')) self._log_exception(error_message) raise MessageException(error_message) if self.crypto_message is None or self.crypto_message.get_email_message() is None: charset = 'UTF-8' else: charset, __ = get_charset(self.crypto_message.get_email_message().get_message()) self._log_message('char set in crypto message: {}'.format(charset)) users_dict = {encrypt_utils.TO_KEYWORD: to_user_id, encrypt_utils.FROM_KEYWORD: from_user_id, encrypt_utils.PASSCODE_KEYWORD: passcode, encrypt_utils.CHARSET_KEYWORD: charset} self._log_message('got crypto details and ready to encrypt: {}'.format(ready_to_encrypt)) return users_dict, ready_to_encrypt def _get_from_crypto_details(self, encryption_name, user_ids): ''' Get the from user's details needed to encrypt a message. ''' error_message = passcode = None from_user = self.crypto_message.smtp_sender() self._log_message('user ids: {}'.format(user_ids)) from_user_id = utils.get_user_id_matching_email(from_user, user_ids) self._log_message('from_user_id: {}'.format(from_user_id)) if from_user_id is None: passcode = None error_message = i18n("There isn't a {encryption_name} key for {email}").format( encryption_name=encryption_name, email=from_user) self._log_message(error_message) else: passcode = user_keys.get_passcode(from_user, encryption_name) if passcode is None: error_message = i18n("There isn't a private {encryption_name} key for {email}").format( encryption_name=encryption_name, email=from_user) self._log_message(error_message) return from_user_id, passcode, error_message def _get_to_crypto_details(self, encryption_name, user_ids): ''' Get the recipient details needed to encrypt a message. ''' from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() to_user_id = utils.get_user_id_matching_email(to_user, user_ids) if to_user_id is None: ready_to_encrypt = False self._log_message("No {} key for {}".format(encryption_name, to_user)) else: try: contacts.is_key_ok(to_user_id, encryption_name) ready_to_encrypt = True except CryptoException as exception: to_user_id = None ready_to_encrypt = False self._log_exception(exception.value) self._log_message('raising message exception in _get_to_crypto_details') raise MessageException(value=exception.value) return to_user_id, ready_to_encrypt def _prep_from_user(self, private_user_ids, encryption_name): result_ok = private_user_ids is not None and len(private_user_ids) > 0 if result_ok or options.create_private_keys(): if result_ok: from_user_id = utils.get_user_id_matching_email( self.crypto_message.smtp_sender(), private_user_ids) else: from_user_id = None if from_user_id is None: # add a key if there isn't one for the sender and we're creating keys if options.create_private_keys(): add_private_key( self.crypto_message.smtp_sender(), encryption_software=encryption_name) else: self._log_message("not creating a new {} key for {} because auto-create disabled".format( encryption_name, from_user_id)) else: self._log_message('found matching user id: {}'.format(from_user_id)) self._log_message('_prep_from_user: {}'.format(result_ok)) return result_ok def _get_encrypt_error_message(self, users_dict, encryption_name): to_user = users_dict[encrypt_utils.TO_KEYWORD] from_user = users_dict[encrypt_utils.FROM_KEYWORD] passcode = users_dict[encrypt_utils.PASSCODE_KEYWORD] if options.clear_sign_email(): if from_user is None or passcode is None: error_message = '{} '.format( i18n("Message not sent to {email} because currently there isn't a private {encryption} key for you and your mail administrator requires all encrypted messages also be clear signed.".format( email=to_user, encryption=encryption_name))) if options.create_private_keys(): error_message += i18n("GoodCrypto is creating a private key now. You will receive email when your keys are ready so you can resend your message.") else: error_message += '\n\n{}'.format(i18n( 'Ask your mail administrator to create a private key for you and then try resending the message.')) else: error_message = '{}\n{}'.format( i18n(self.UNABLE_TO_ENCRYPT.format( from_email=from_user, to_email=to_user, encryption=encryption_name)), self.POSSIBLE_ENCRYPT_SOLUTION) else: error_message = '{}\n{}'.format( i18n(self.UNABLE_TO_ENCRYPT.format( from_email=from_user, to_email=to_user, encryption=encryption_name)), self.POSSIBLE_ENCRYPT_SOLUTION) return error_message def _clear_sign_crypto_message(self, from_user): ''' Clear sign the message. ''' """ !!!!!! clear_sign_policy = options.clear_sign_policy() if clear_sign_policy == constants.CLEAR_SIGN_WITH_DOMAIN_KEY: encryption_names = contacts.get_encryption_names(utils.get_domain_email()) elif clear_sign_policy == constants.CLEAR_SIGN_WITH_SENDER_KEY: encryption_names = contacts.get_encryption_names(from_user) else: encryption_names = contacts.get_encryption_names(from_user) if encryption_names is None or len(encryption_names) <= 0: encryption_names = contacts.get_encryption_names(from_user) """ encryption_names = contacts.get_encryption_names(from_user) if len(encryption_names) <= 0: signed = False self._log_message('unable to clear sign message because no encryption software defined for: {}'.format(from_user)) else: signed = self._clear_sign_message_with_all(encryption_names) # the message isn't really encrypted, just signed if signed: self.crypto_message.set_crypted(False) self.crypto_message.set_clear_signed(True) self.crypto_message.add_clear_signer( {constants.SIGNER: from_user, constants.SIGNER_VERIFIED: True}) self._log_message('message clear signed, but not encrypted') if self.DEBUGGING: self._log_message('message after clear signature:\n{}'.format( self.crypto_message.get_email_message().to_string())) return signed def _clear_sign_message_with_all(self, encryption_names): ''' Clear sign the message with each encryption program. ''' signed = fatal_error = False error_message = '' signed_with = [] encrypted_classnames = [] self._log_message("signing using {} encryption software".format(encryption_names)) for encryption_name in encryption_names: encryption_classname = get_key_classname(encryption_name) if encryption_classname not in encrypted_classnames: try: if self._clear_sign_message(encryption_name): signed_with.append(encryption_name) encrypted_classnames.append(encryption_classname) except MessageException as message_exception: error_message += message_exception.value self._log_exception(error_message) break return signed_with def _clear_sign_message(self, encryption_name): ''' Clear sign the message if From has a key. ''' result_ok = False try: self._log_message("signing message with {}".format(encryption_name)) crypto = CryptoFactory.get_crypto(encryption_name, get_classname(encryption_name)) if crypto is None: self._log_message("{} is not ready to use".format(encryption_name)) elif (self.crypto_message is None or self.crypto_message.smtp_sender() is None): self._log_message("missing key data to sign message") else: # get all the private user ids that have encryption keys private_user_ids = crypto.get_private_user_ids() if self.DEBUGGING: self._log_message('private user ids: {}'.format(private_user_ids)) result_ok = self._prep_from_user(private_user_ids, encryption_name) if result_ok: from_user_id, passcode, error_message = self._get_from_crypto_details( encryption_name, private_user_ids) result_ok = from_user_id is not None and passcode is not None else: self._log_message("private_user_ids is not None and len(private_user_ids) > 0: {}".format( private_user_ids is not None and len(private_user_ids) > 0)) if result_ok: self._log_message("from user ID: {}".format(from_user_id)) self._log_message("subject: {}".format( self.crypto_message.get_email_message().get_header(mime_constants.SUBJECT_KEYWORD))) result_ok = self._clear_sign_message_with_keys(crypto, from_user_id, passcode) self._log_message('signed message: {}'.format(result_ok)) else: # the message hasn't been signed, but we have # successfully processed it self.crypto_message.set_filtered(True) except Exception: result_ok = False record_exception() self._log_message('EXCEPTION - see syr.exception.log for details') return result_ok def _clear_sign_message_with_keys(self, crypto, from_user_id, passcode): ''' Sign a message with the From key. ''' if is_content_type_text(self.crypto_message.get_email_message().get_message()): encrypt_utils.sign_text_message(self.crypto_message, crypto, from_user_id, passcode) else: encrypt_utils.sign_mime_message(self.crypto_message, crypto, from_user_id, passcode) return self.crypto_message.is_crypted() def _add_outbound_record(self, original_crypto_message, inner_encrypted_with): ''' Add an outbound record that a message was sent privately.''' self.crypto_message.set_crypted_with(inner_encrypted_with) # use the original message, not the protected one if original_crypto_message is None: original_crypto_message = copy.copy(self.crypto_message) self._log_message('copying crypto message before saving history record') else: original_crypto_message.set_crypted(True) original_crypto_message.set_crypted_with(inner_encrypted_with) original_crypto_message.set_metadata_crypted(True) original_crypto_message.set_metadata_crypted_with(self.crypto_message.get_metadata_crypted_with()) if self.crypto_message.is_private_signed(): original_crypto_message.set_private_signed(True) if self.crypto_message.is_private_sig_verified(): original_crypto_message.set_private_signers(self.crypto_message.private_signers_list()) if self.crypto_message.is_clear_signed(): original_crypto_message.set_clear_signed(True) if self.crypto_message.is_clear_sig_verified(): original_crypto_message.set_clear_signers(self.crypto_message.clear_signers_list()) if self.crypto_message.is_dkim_signed(): original_crypto_message.set_dkim_signed(True) if self.crypto_message.is_dkim_sig_verified(): original_crypto_message.set_dkim_sig_verified(True) history.add_outbound_record(original_crypto_message, self.verification_code) def _start_check_for_encryption(self, from_user, to_user): ''' Start checking if the user uses encryption. ''' # don't look for keys if the to_user's domain is also using goodcrypto if contacts.get(get_metadata_address(email=to_user)) is None: # start by adding a record so we don't search for a key again contact = contacts.add(to_user, None) # search for the keys; if one's found, send email to the from # user so they can verify the fingerprint self._log_message('queuing search for a key for {}'.format(to_user)) queue_keyserver_search(to_user, from_user) else: self._log_message('{} is also using goodcrypto so not searching for a key'.format(to_user)) def _log_exception(self, msg): ''' Log the message to the local and Exception logs. ''' self._log_message(msg) record_exception(message=msg) def _log_error(error_message): ''' Log an error. ''' record_exception() self._log_message('EXCEPTION - see syr.exception.log for details') try: self.crypto_message.add_error_tag_once('{} {}'.format( SERIOUS_ERROR_PREFIX, error_message)) except Exception: record_exception() self._log_message('EXCEPTION - see syr.exception.log for details') def _log_message(self, message): ''' Log the message to the local log. ''' if self._log is None: self._log = LogFile() self._log.write_and_flush(message)
class Validator(object): ''' Validates an EmailMessage. ''' DEBUGGING = False def __init__(self, email_message): ''' Unparsable messages are wrapped in a valid message. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator != None True ''' self.log = LogFile() self.email_message = email_message self.why = None def is_message_valid(self): ''' Returns true if the message is parsable, else false. If a MIME message is not parsable, you should still be able to process it. As we find different errors in messages, we should make sure this method catches them. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator.is_message_valid() True ''' is_valid = False self.why = None if self.email_message is None: raise MessageException('Message is None') else: try: self.email_message.write_to(StringIO()) if Validator.DEBUGGING: self.log_message("message after check:\n{}".format( self.email_message.to_string())) self._check_content(self.email_message) is_valid = True except Exception: is_valid = False # we explicitly want to catch everything here, even NPE record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') self.log_message('message is valid: {}'.format(is_valid)) return is_valid def _check_content(self, part): ''' Make sure we can read the content. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator._check_content(validator.email_message) ''' content_type = part.get_header('Content-Type') self.log_message("MIME part content type: {}".format(content_type)) content = part.get_content() if isinstance(content, MIMEMultipart): count = 0 parts = content for sub_part in parts: self._check_content(sub_part) count += 1 self.log_message('parts in message: {}'.format(count)) if count != parts.getCount(): self.why = "Unable to read all content. Reported: '{}', read: {}".format( parts.getCount(), count) raise MessageException(self.why) def get_why(self): ''' Gets why a message is invalid. Returns null if the message is valid. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator.get_why() is None True ''' return self.why def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> good_message = get_basic_email_message() >>> validator = Validator(good_message) >>> validator.log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.validator.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class Filter(object): ''' Filters a message through the encrypt or decrypt filter as needed. The overall design of this module is trying to emulate a milter so if we ever decide to convert from a content filter it will be easier. ''' DEBUGGING = False def __init__(self, from_user, to_user, message): ''' >>> # In honor of John Kiriakou, who went to prison for exposing CIA torture. >>> # In honor of asn, developer of Obfsproxy. >>> filter = Filter('*****@*****.**', '*****@*****.**', 'message') >>> filter is not None True ''' self.log = self.out_message = None self.sender = from_user self.recipient = to_user self.in_message = message def process(self): ''' Encrypt/decrypt the message, or decide not to. ''' self.log_message('=== starting to filter mail from {} to {} ==='.format(self.sender, self.recipient)) self.out_message = self.in_message crypto_message = CryptoMessage( email_message=EmailMessage(self.in_message), sender=self.sender, recipient=self.recipient) if self.possibly_needs_encryption(): self.process_outbound_message(crypto_message) else: self.process_inbound_message(crypto_message) result_code = self.reinject_message() self.log_message('mail filtered ok: {}'.format(result_code)) self.log_message('=== finished filtering mail from {} to {} ==='.format(self.sender, self.recipient)) return result_code def process_outbound_message(self, crypto_message): ''' Process an outbound message, encrypting if approriate. ''' try: # something is wrong if an outbound message is from the metadata address # any messages from a metadata address don't pass through the filter if is_metadata_address(self.sender): self.sender = get_admin_email() self.bounce_outbound_message(i18n('Message originating from your metadata address')) self.out_message = None else: encrypt = Encrypt(crypto_message) encrypted_message = encrypt.process_message() filtered = encrypted_message.is_filtered() crypted = encrypted_message.is_crypted() processed = encrypted_message.is_processed() if encrypted_message.is_dropped(): self.out_message = None self.log_message('outbound message dropped') elif processed: # nothing to re-inject at this time self.out_message = None self.log_message('outbound message processed') else: self.out_message = encrypted_message.get_email_message().to_string() self.sender = encrypted_message.smtp_sender() self.recipient = encrypted_message.smtp_recipient() self.log_message( 'outbound sender: {} recipient: {}'.format(self.sender, self.recipient)) self.log_message('outbound final status: filtered: {} crypted: {} queued: {}'.format( filtered, crypted, processed)) except MessageException as message_exception: self.bounce_outbound_message(message_exception.value) self.out_message = None except Exception as exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') try: self.bounce_outbound_message(exception.value) except: self.bounce_outbound_message(exception) self.out_message = None except IOError as ioerror: try: self.bounce_outbound_message(ioerror.value) except: self.bounce_outbound_message(ioerror) self.out_message = None def process_inbound_message(self, crypto_message): ''' Process an inbound message, decrypting if appropriate. ''' try: debundled_message = None filtered = crypted = processed = False if self.DEBUGGING: self.log_message(crypto_message.get_email_message().to_string()) debundle = Debundle(crypto_message) debundled_message = debundle.process_message() filtered = debundled_message.is_filtered() crypted = debundled_message.is_crypted() processed = debundled_message.is_processed() if debundled_message.is_dropped(): self.out_message = None self.log_message('inbound message dropped') elif processed: # nothing to re-inject at this time self.out_message = None self.log_message('inbound message processed') else: self.out_message = debundled_message.get_email_message().to_string() self.sender = debundled_message.smtp_sender() self.recipient = debundled_message.smtp_recipient() self.log_message( 'inbound sender: {} recipient: {}'.format(self.sender, self.recipient)) self.log_message('inbound final status: filtered: {} crypted: {} queued: {}'.format( filtered, crypted, processed)) except DKIMException as dkim_exception: self.drop_message(dkim_exception) self.out_message = None except CryptoException as crypto_exception: if debundled_message is not None and not debundled_message.is_dropped(): self.wrap_inbound_message(crypto_exception, debundled_message) self.out_message = None except MessageException as message_exception: if debundled_message is not None and not debundled_message.is_dropped(): self.wrap_inbound_message(message_exception, debundled_message) self.out_message = None except Exception as exception: self.wrap_inbound_message(exception, debundled_message) self.out_message = None except IOError as ioerror: self.wrap_inbound_message(ioerror, debundled_message) self.out_message = None def possibly_needs_encryption(self): ''' Determine if the message might need to be encrypted or not. It could need encrytion if the sender has the same domain as the domain defined in goodcrypto mail server's options. If we're not using an SMTP proxy, then it never needs encryption if the message is going to and from a user with the domain defined in goodcrypto mail server's options. ''' maybe_needs_encryption = email_in_domain(self.sender) self.log_message('possibly needs encryption: {}'.format(maybe_needs_encryption)) return maybe_needs_encryption def get_processed_message(self): ''' Get the message after it has been processed. >>> filter = Filter('root', 'root', 'bad message') >>> filter.get_processed_message() ''' return self.out_message def reinject_message(self, message=None): ''' Re-inject message back into queue. ''' if message is None: message = self.out_message try: if message is None: # set to result_ok to True because we've already bounced the message or # there isn't anything to bounce because it failed validation result_ok = True self.log_message('nothing to reinject') else: self.log_message('starting to re-inject message into postfix queue') result_ok = send_message(self.sender, self.recipient, message) self.log_message('re-injected message: {}'.format(result_ok)) if self.DEBUGGING: self.log_message('\n==================\n') self.log_message(message) self.log_message('\n==================\n') except Exception as exception: result_ok = False self.log_message('error while re-injecting message into postfix queue') self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() try: error_message = exception.value to_address = self.reject_message(error_message) self.log_message('sent notice to {} about {}'.format(to_address, error_message)) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return result_ok def reject_message(self, error_message, message=None): ''' Reject a message that had an unexpected error and return the to address. >>> # This message will fail if testing on dev system >>> to_address = get_admin_email() >>> filter = Filter('root', 'root', 'bad message') >>> filter.reject_message('Unknown message') == to_address True ''' try: if message is None: message = self.out_message if email_in_domain(self.sender): to_address = self.sender subject = i18n('Undelivered Mail: Unable to send message') elif email_in_domain(self.recipient): to_address = self.recipient subject = i18n('Error: Unable to receive message') else: to_address = get_admin_email() subject = i18n('Message rejected.') notice = '{}'.format(error_message) if message is not None: notice += '\n\n===================\n{}'.format(message) notify_user(to_address, subject, notice) except: raise return to_address def bounce_outbound_message(self, error_message): ''' Bounce a message that a local user originated. ''' self.log_message(error_message) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() subject = i18n('{} - Undelivered Mail: Unable to protect message'.format(TAG_ERROR)) utils.bounce_message(self.in_message, self.sender, subject, error_message) def wrap_inbound_message(self, error_message, debundled_message): ''' Wrap an inbound message that had a serious error. ''' self.log_message(error_message) body = error_message try: processed_message = debundled_message.crypto_message error_tags = processed_message.get_error_tags() if error_tags and len(error_tags) > 0: for tag in error_tags: body += '\n{}'.format(tag) if options.add_long_tags(): long_tags = processed_message.get_tags() if long_tags and len(long_tags) > 0: for tag in long_tags: body += '\n{}'.format(tag) except: pass subject = i18n('{} - Unable to decrypt message'.format(TAG_ERROR)) utils.bounce_message(self.in_message, self.recipient, subject, body) def drop_message(self, error_message): ''' Drop a message that we shouldn't process from a remote user. ''' self.log_message(error_message) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() subject = i18n('{} - Unable to decrypt message'.format(TAG_ERROR)) utils.drop_message(self.in_message, self.recipient, subject, error_message) def log_message(self, message): ''' Record debugging messages. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> filter = Filter('*****@*****.**', ['*****@*****.**'], 'message') >>> filter.log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.filter.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class SearchKeyserver(object): ''' Search for a key from keyserver. ''' def __init__(self, email, encryption_name, keyserver, user_initiated_search): ''' >>> # In honor of Werner Koch, developer of gpg. >>> email = '*****@*****.**' >>> crypto_name = 'GPG' >>> srk_class = SearchKeyserver(email, crypto_name, 'pgp.mit.edu', '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(None, crypto_name, 'pgp.mit.edu', '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(email, None, 'pgp.mit.edu', '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(email, crypto_name, None, '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(None, None, None, None) >>> srk_class != None True ''' self.log = LogFile() self.email = email self.encryption_name = encryption_name self.keyserver = keyserver self.user_initiated_search = user_initiated_search self.key_plugin = None def start_search(self): ''' Queue searching the keyserver. When the job finishes, the key will be retrieved from another queued job which is dependent on the search's job. Test extreme case. >>> srk_class = SearchKeyserver(None, None, None, None) >>> srk_class.start_search() False ''' try: if self._is_ready_for_search(): result_ok = True # start the search, but don't wait for the results self.key_plugin.search_for_key(self.email, self.keyserver) search_job = self.key_plugin.get_job() queue = self.key_plugin.get_queue() # if the search job or queue are done, then retrieve the key if queue is None or search_job is None: get_key(self.email, self.encryption_name, self.keyserver, self.user_initiated_search, search_job, queue) else: from goodcrypto.mail.keyserver_utils import get_key ONE_MINUTE = 60 # one minute, in seconds DEFAULT_TIMEOUT = 10 * ONE_MINUTE # otherwise, set up another job in the queue to retrieve the # key as soon as the search for the key id is finished result_ok = get_key(self.email, self.encryption_name, self.keyserver, self.user_initiated_search, search_job.get_id(), queue.key) self.log_message('retrieving {} key for {}: {}'.format( self.encryption_name, self.email, result_ok)) else: result_ok = False except Exception as exception: result_ok = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_message('finished starting search on {} for {} ok: {}'.format( self.keyserver, self.email, result_ok)) return result_ok def _is_ready_for_search(self): ''' Verify that we're ready to search for this key. Test extreme case. >>> srk_class = SearchKeyserver(None, None, None, None) >>> srk_class._is_ready_for_search() False ''' ready = False try: ready = (self.email is not None and self.encryption_name is not None and self.keyserver is not None and self.user_initiated_search is not None and not email_in_domain(self.email)) if ready: self.key_plugin = KeyFactory.get_crypto( self.encryption_name, crypto_software.get_key_classname(self.encryption_name)) ready = self.key_plugin is not None if ready: # make sure we don't already have crypto defined for this user contacts_crypto = contacts.get_contacts_crypto(self.email, self.encryption_name) if contacts_crypto is None or contacts_crypto.fingerprint is None: fingerprint, expiration = self.key_plugin.get_fingerprint(self.email) if fingerprint is not None: ready = False self.log_message('{} public key exists for {}: {}'.format( self.encryption_name, self.email, fingerprint)) else: ready = False self.log_message('crypto for {} already defined'.format(self.email)) except Exception as exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') ready = False return ready def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> SearchKeyserver(None, None, None, None).log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.search_keyserver.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class CryptoMessage(object): ''' Crypto email_message. This class does not extend EmailMessage because we want a copy of the original EmailMessage so we can change it without impacting the original. See unittests for most of the functions. ''' DEBUGGING = False SEPARATOR = ': ' def __init__(self, email_message=None, sender=None, recipient=None): ''' >>> crypto_message = CryptoMessage() >>> crypto_message != None True >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message != None True ''' super(CryptoMessage, self).__init__() self.log = LogFile() if email_message is None: self.email_message = EmailMessage() self.log_message( 'starting crypto message with a blank email message') else: self.email_message = email_message self.log_message( 'starting crypto message with an existing email message') # initialize a few key elements self.set_smtp_sender(sender) self.set_smtp_recipient(recipient) self.set_filtered(False) self.set_crypted(False) self.set_crypted_with([]) self.set_metadata_crypted(False) self.set_metadata_crypted_with([]) self.drop(False) self.set_processed(False) self.set_tag('') self.set_error_tag('') self.set_private_signed(False) self.set_private_signers([]) self.set_clear_signed(False) self.set_clear_signers([]) self.set_dkim_signed(False) self.set_dkim_sig_verified(False) def get_email_message(self): ''' Returns the email message. >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message.get_email_message() is not None True ''' return self.email_message def set_email_message(self, email_message): ''' Sets the email_message. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> crypto_message = CryptoMessage() >>> crypto_message.set_email_message(get_basic_email_message()) >>> crypto_message.get_email_message().get_message() is not None True ''' self.email_message = email_message def smtp_sender(self): ''' Returns the SMTP sender. >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message.smtp_sender() is None True ''' return self.sender def set_smtp_sender(self, email_address): ''' Sets the SMTP sender email address. If a message had its metadata protected, then we'll set the "smtp sender" as the inner, protected messages are set. This address is never derived from the "header" section of a message. # In honor of Sister Megan Rice, an anti-nuclear activist who was # initially sentenced for breaking into a US nuclear facility as a protest. # Fortunately, she was finally released when federal appeals court acknowledged a # little old lady had embarrassed the gov't, not threatened them. >>> from goodcrypto_tests.mail.message_utils import get_basic_email_message >>> crypto_message = CryptoMessage() >>> crypto_message.set_smtp_sender('*****@*****.**') >>> sender = crypto_message.smtp_sender() >>> sender == '*****@*****.**' True ''' self.sender = get_email(email_address) if self.DEBUGGING: self.log_message('set sender: {}'.format(self.sender)) def smtp_recipient(self): ''' Returns the SMTP recipient. >>> crypto_message = CryptoMessage(email_message=EmailMessage()) >>> crypto_message.smtp_recipient() is None True ''' return self.recipient def set_smtp_recipient(self, email_address): ''' Sets the SMTP recipient email address. If a message had its metadata protected, then we'll set the "smtp recipient" as the inner, protected messages are set. This address is never derived from the "header" section of a message. >>> # In honor of the Navy nurse who refused to torture prisoners >>> # in Guantanamo by force feeding them. >>> crypto_message = CryptoMessage() >>> crypto_message.set_smtp_recipient('*****@*****.**') >>> recipient = crypto_message.smtp_recipient() >>> recipient == '*****@*****.**' True ''' self.recipient = get_email(email_address) if self.DEBUGGING: self.log_message('set recipient: {}'.format(self.recipient)) def get_public_key_header(self, from_user): ''' Get the public key header lines. >>> from goodcrypto_tests.mail.message_utils import get_plain_message_name >>> from goodcrypto.oce.test_constants import EDWARD_LOCAL_USER >>> auto_exchange = options.auto_exchange_keys() >>> options.set_auto_exchange_keys(True) >>> filename = get_plain_message_name('basic.txt') >>> with open(filename) as input_file: ... crypto_message = CryptoMessage(email_message=EmailMessage(input_file)) ... key_block = crypto_message.get_public_key_header(EDWARD_LOCAL_USER) ... key_block is not None ... len(key_block) > 0 True True >>> options.set_auto_exchange_keys(auto_exchange) ''' header_lines = [] if options.auto_exchange_keys(): encryption_software_list = contacts.get_encryption_names(from_user) # if no crypto and we're creating keys, then do so now if (len(encryption_software_list) <= 0 and email_in_domain(from_user) and options.create_private_keys()): add_private_key(from_user) self.log_message( "started to create a new key for {}".format(from_user)) encryption_software_list = contacts.get_encryption_names( from_user) if len(encryption_software_list) > 0: self.log_message( "getting header with public keys for {}: {}".format( from_user, encryption_software_list)) for encryption_software in encryption_software_list: key_block = self.create_public_key_block( encryption_software, from_user) if len(key_block) > 0: header_lines += key_block else: self.log_message("Warning: auto-exchange of keys is not active") return header_lines def create_public_key_block(self, encryption_software, from_user): ''' Create a public key block for the user if the header doesn't already have one. ''' key_block = [] try: if from_user is None or encryption_software is None or self.has_public_key_header( encryption_software): self.log_message('public {} key block already exists'.format( encryption_software)) else: key_block = utils.make_public_key_block( from_user, encryption_software=encryption_software) except MessageException: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_exception( "Unable to get {} public key header for {}".format( encryption_software, from_user)) return key_block def extract_public_key_block(self, encryption_software): ''' Extract a public key block from the header, if there is one. ''' key_block = None try: if self.has_public_key_header(encryption_software): header_name = utils.get_public_key_header_name( encryption_software) self.log_message( "getting {} public key header block using header {}". format(encryption_software, header_name)) key_block = inspect_utils.get_multientry_header( self.get_email_message().get_message(), header_name) if key_block: self.log_message("len key_block: {}".format( len(key_block))) else: self.log_exception( "No valid key {} block in header".format( encryption_software)) except MessageException: record_exception() self.log_exception("Unable to get {} public key block".format( encryption_software)) self.log_message('EXCEPTION - see syr.exception.log for details') return key_block def add_public_key_to_header(self, from_user): ''' Add public key and accepted crypto to header if automatically exchanging keys. ''' if options.auto_exchange_keys(): header_lines = self.get_public_key_header(from_user) if header_lines and len(header_lines) > 0: for line in header_lines: # we can't just use split() because some lines have no value index = line.find(CryptoMessage.SEPARATOR) if index > 0: header_name = line[0:index] value_index = index + len(CryptoMessage.SEPARATOR) if len(line) > value_index: value = line[value_index:] else: value = '' else: header_name = line value = '' self.email_message.add_header(header_name, value) self.add_accepted_crypto_software(from_user) self.add_fingerprint(from_user) self.log_message( "added key for {} to header".format(from_user)) else: encryption_name = CryptoFactory.DEFAULT_ENCRYPTION_NAME if options.create_private_keys(): add_private_key(from_user, encryption_software=encryption_name) self.log_message("creating a new {} key for {}".format( encryption_name, from_user)) else: self.log_message( "not creating a new {} key for {} because auto-create disabled" .format(encryption_name, from_user_id)) else: self.log_message( "not adding key for {} to header because auto-exchange disabled" .format(from_user)) def add_accepted_crypto_software(self, from_user): ''' Add accepted encryption software to email message header. ''' # check whether we've already added them existing_crypto_software = self.get_accepted_crypto_software() if len(existing_crypto_software) > 0: self.log_message( "attempted to add accepted encryption software to email_message that already has them" ) else: encryption_software_list = contacts.get_encryption_names(from_user) if len(encryption_software_list) <= 0: self.log_message( "No encryption software for {}".format(from_user)) else: self.email_message.add_header( constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER, ','.join(encryption_software_list)) def get_accepted_crypto_software(self): ''' Gets list of accepted encryption software from email message header. Crypto services are comma delimited. ''' encryption_software_list = [] try: # !!!! the accepted services list is unsigned! fix this! encryption_software_header = inspect_utils.get_first_header( self.email_message.get_message(), constants.ACCEPTED_CRYPTO_SOFTWARE_HEADER) if encryption_software_header != None and len( encryption_software_header) > 0: self.log_message( "accepted encryption software from email_message: {}". format(encryption_software_header)) encryption_software_list = encryption_software_header.split( ',') except Exception as exception: self.log_message(exception) return encryption_software_list def add_fingerprint(self, from_user): ''' Add the fingerprint for each type of crypto used to the email message header. ''' try: encryption_software_list = contacts.get_encryption_names(from_user) if len(encryption_software_list) <= 0: self.log_message( "Not adding fingerprint for {} because no crypto software". format(from_user)) else: for encryption_name in encryption_software_list: fingerprint, __, active = contacts.get_fingerprint( from_user, encryption_name) if active and fingerprint is not None and len( fingerprint.strip()) > 0: self.email_message.add_header( constants.PUBLIC_FINGERPRINT_HEADER.format( encryption_name.upper()), format_fingerprint(fingerprint)) self.log_message( 'added {} fingerprint'.format(encryption_name)) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def get_default_key_from_header(self): ''' Gets the default public key from the email_message header. ''' return self.get_public_key_from_header(constants.PUBLIC_KEY_HEADER) def get_public_key_from_header(self, header_name): ''' Gets the public key from the email_message header. ''' key = None try: key = inspect_utils.get_multientry_header( self.email_message.get_message(), header_name) if key is not None and len(key.strip()) <= 0: key = None except Exception: self.log_message("No public key found in email message") return key def has_public_key_header(self, encryption_name): ''' Return true if a public key header exists for the encryption software. ''' has_key = False try: header_name = utils.get_public_key_header_name(encryption_name) email_message_key = inspect_utils.get_multientry_header( self.email_message.get_message(), header_name) has_key = email_message_key != None and len(email_message_key) > 0 except Exception as exception: # whatever the error, the point is we didn't get a public key header self.log_message(exception) if has_key: self.log_message( "email_message already has public key header for encryption program {}" .format(encryption_name)) return has_key def set_filtered(self, filtered): ''' Sets whether this email_message has been changed by a filter. >>> crypto_message = CryptoMessage() >>> crypto_message.set_filtered(True) >>> crypto_message.is_filtered() True ''' if self.DEBUGGING: self.log_message("set filtered: {}".format(filtered)) self.filtered = filtered def is_filtered(self): ''' Gets whether this email_message has been changed by a filter. >>> crypto_message = CryptoMessage() >>> crypto_message.is_filtered() False ''' return self.filtered def set_crypted(self, crypted): ''' Sets whether this email_message has been encrypted or decrypted, even partially. You can check whether an inner email_message is still encrypted with email_email_message.is_probably_pgp(). >>> crypto_message = CryptoMessage() >>> crypto_message.set_crypted(True) >>> crypto_message.is_crypted() True ''' if self.DEBUGGING: self.log_message("set crypted: {}".format(crypted)) self.crypted = crypted def is_crypted(self): ''' Returns whether this email_message has been encrypted or decrypted, even partially. You can check whether an inner email_message is still encrypted with email_email_message.is_probably_pgp(). >>> crypto_message = CryptoMessage() >>> crypto_message.is_crypted() False ''' return self.crypted def set_metadata_crypted(self, crypted): ''' Sets whether this email_message has its metadata encrypted or decrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.set_metadata_crypted(True) >>> crypto_message.is_metadata_crypted() True ''' if self.DEBUGGING: self.log_message("set metadata crypted: {}".format(crypted)) self.metadata_crypted = crypted def is_metadata_crypted(self): ''' Returns whether this email_message has been encrypted or decrypted, even partially. You can check whether an inner email_message is still encrypted with email_email_message.is_probably_pgp(). >>> crypto_message = CryptoMessage() >>> crypto_message.is_metadata_crypted() False ''' return self.metadata_crypted def is_signed(self): ''' Returns whether this email_message has any type of signature. >>> crypto_message = CryptoMessage() >>> crypto_message.is_signed() False ''' return self.is_private_signed() or self.is_clear_signed( ) or self.is_dkim_signed() def set_private_signed(self, signed): ''' Sets whether this email_message has been signed when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.set_private_signed(True) >>> crypto_message.is_private_signed() True ''' if self.DEBUGGING: self.log_message("set private signed: {}".format(signed)) self.private_signed = signed def is_private_signed(self): ''' Returns whether this email_message has been signed when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.is_private_signed() False ''' return self.private_signed def is_private_sig_verified(self): ''' Returns whether this email_message's signature has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.is_private_sig_verified() False ''' return is_sig_verified(self.private_signers_list()) def set_private_signers(self, signers): ''' Set who signed this email_message when encrypted. >>> private_signers = [{'signer': '*****@*****.**', 'verified': True}] >>> crypto_message = CryptoMessage() >>> crypto_message.set_private_signers( ... [{u'signer': u'*****@*****.**', u'verified': True}]) >>> signers = crypto_message.private_signers_list() >>> signers == private_signers True ''' self.private_signers = signers def add_private_signer(self, signer): ''' Add who signed this email_message when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.add_private_signer( ... {constants.SIGNER: '*****@*****.**', constants.SIGNER_VERIFIED: True}) >>> signers = crypto_message.private_signers_list() >>> signers == [{'signer': '*****@*****.**', 'verified': True}] True ''' self.add_signer(signer, self.private_signers_list()) def private_signers_list(self): ''' Returns a list of signers when encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.private_signers_list() [] ''' if self.private_signers is None: self.set_private_signers([]) return self.private_signers def set_clear_signed(self, signed): ''' Sets whether this email_message has been clear signed. >>> crypto_message = CryptoMessage() >>> crypto_message.set_clear_signed(True) >>> crypto_message.is_clear_signed() True ''' if self.DEBUGGING: self.log_message("set clear signed: {}".format(signed)) self.clear_signed = signed def is_clear_signed(self): ''' Returns whether this email_message has been clear signed. >>> crypto_message = CryptoMessage() >>> crypto_message.is_clear_signed() False ''' return self.clear_signed def is_clear_sig_verified(self): ''' Returns whether this email_message's clear signature has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.is_clear_sig_verified() False ''' return is_sig_verified(self.clear_signers_list()) def set_clear_signers(self, signers): ''' Set who clear signed this email_message. >>> crypto_message = CryptoMessage() >>> crypto_message.set_clear_signers( ... [{'signer': '*****@*****.**', 'verified': True}]) >>> signers = crypto_message.clear_signers_list() >>> signers == [{'signer': '*****@*****.**', 'verified': True}] True ''' self.clear_signers = signers def add_clear_signer(self, signer_dict): ''' Add who clear signed this email_message. >>> crypto_message = CryptoMessage() >>> crypto_message.add_clear_signer({'signer': '*****@*****.**', 'verified': True}) >>> signers = crypto_message.clear_signers_list() >>> signers == [{'signer': '*****@*****.**', 'verified': True}] True ''' self.add_signer(signer_dict, self.clear_signers_list()) def clear_signers_list(self): ''' Returns a list of clear signers. >>> crypto_message = CryptoMessage() >>> crypto_message.clear_signers_list() [] ''' if self.clear_signers is None: self.set_clear_signers([]) return self.clear_signers def set_dkim_signed(self, signed): ''' Sets whether this email_message has been signed using DKIM. >>> crypto_message = CryptoMessage() >>> crypto_message.set_dkim_signed(True) >>> crypto_message.is_dkim_signed() True ''' if self.DEBUGGING: self.log_message("set dkim signed: {}".format(signed)) self.dkim_signed = signed def is_dkim_signed(self): ''' Returns whether this email_message has been signed using DKIM. >>> crypto_message = CryptoMessage() >>> crypto_message.is_dkim_signed() False ''' return self.dkim_signed def set_dkim_sig_verified(self, verified): ''' Sets whether this email_message's DKIM sig has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.set_dkim_sig_verified(True) >>> crypto_message.is_dkim_sig_verified() True ''' if self.DEBUGGING: self.log_message("set dkim sig verified: {}".format(verified)) self.dkim_verified = verified def is_dkim_sig_verified(self): ''' Returns whether this email_message's DKIM sig has been verified. >>> crypto_message = CryptoMessage() >>> crypto_message.is_dkim_sig_verified() False ''' return self.dkim_verified def add_signer(self, signer_dict, signer_list): ''' Add who signed this email_message. >>> crypto_message = CryptoMessage() >>> clear_signers = crypto_message.clear_signers_list() >>> crypto_message.add_signer({'signer': '*****@*****.**', 'verified': True}, clear_signers) ''' if signer_dict is not None: signer = signer_dict[constants.SIGNER] if signer is not None: signer = get_email(signer) # now make the signer readable if unknown if signer == None: signer = 'unknown user' signer_dict[constants.SIGNER] = signer if signer_dict not in signer_list: if self.DEBUGGING: self.log_message("add signer: {}".format(signer_dict)) signer_list.append(signer_dict) def drop(self, dropped=True): ''' Sets whether this email_message has been dropped by a filter. If the message is dropped, then it's never returned to postfix. >>> crypto_message = CryptoMessage() >>> crypto_message.drop() >>> crypto_message.is_dropped() True ''' if self.DEBUGGING: self.log_message("set dropped: {}".format(dropped)) self.dropped = dropped def is_dropped(self): ''' Gets whether this email_message has been dropped by a filter. If the message is dropped, then it's never returned to postfix. >>> crypto_message = CryptoMessage() >>> crypto_message.is_dropped() False ''' return self.dropped def set_processed(self, processed): ''' Sets whether this email message has been processed by a filter. A processed message does not need any further processing by the caller. >>> crypto_message = CryptoMessage() >>> crypto_message.set_processed(True) >>> crypto_message.is_processed() True ''' if self.DEBUGGING: self.log_message("set processed: {}".format(processed)) self.processed = processed def is_processed(self): ''' Gets whether this email_message has been processed by a filter. >>> crypto_message = CryptoMessage() >>> crypto_message.is_processed() False ''' return self.processed def set_crypted_with(self, crypted_with): ''' Sets the encryption programs message was crypted.. >>> crypto_message = CryptoMessage() >>> crypto_message.set_crypted_with(['GPG']) >>> crypted_with = crypto_message.get_crypted_with() >>> crypted_with == ['GPG'] True ''' self.crypted_with = crypted_with if self.DEBUGGING: self.log_message("set crypted_with: {}".format( self.get_crypted_with())) def get_crypted_with(self): ''' Returns the encryption programs message was crypted. >>> crypto_message = CryptoMessage() >>> crypto_message.get_crypted_with() [] ''' return self.crypted_with def set_metadata_crypted_with(self, crypted_with): ''' Sets whether this email_message has its metadata encrypted or decrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.set_metadata_crypted_with(['GPG']) >>> crypted_with = crypto_message.get_metadata_crypted_with() >>> crypted_with == ['GPG'] True ''' self.metadata_crypted_with = crypted_with if self.DEBUGGING: self.log_message("set metadata crypted_with: {}".format( self.get_metadata_crypted_with())) def get_metadata_crypted_with(self): ''' Returns the encryption programs the metadata was encrypted. >>> crypto_message = CryptoMessage() >>> crypto_message.get_metadata_crypted_with() [] ''' return self.metadata_crypted_with def is_create_private_keys_active(self): ''' Gets whether creating private keys on the fly is active. >>> crypto_message = CryptoMessage() >>> current_setting = options.create_private_keys() >>> options.set_create_private_keys(True) >>> crypto_message.is_create_private_keys_active() True >>> options.set_create_private_keys(False) >>> crypto_message.is_create_private_keys_active() False >>> options.set_create_private_keys(current_setting) ''' active = options.create_private_keys() if self.DEBUGGING: self.log_message("Create private keys: {}".format(active)) return active def add_tags_to_message(self, tags): ''' Add tag to a message. >>> crypto_message = CryptoMessage() >>> crypto_message.add_tags_to_message('') False ''' def add_tags_to_text(content, tags): ''' Add the tag to the text content. ''' text_content = content text_content = '{}\n\n\n{}\n'.format(text_content, str(tags)) self.log_message('added tags to text content') return text_content def add_tags_to_html(content, tags): ''' Add the tag to the html content. ''' for tag in tags: tag = tag.replace('\n', '<br/>') tag = tag.replace(' ', ' ') html_content = content index = html_content.lower().find('</body>') if index < 0: index = html_content.lower().find('</html>') if index < 0: html_content = '{}<div><hr>\n{}<br/></div>'.format( html_content, str(tags)) else: html_content = '{}<div><hr>\n{}<br/></div>\n{}'.format( html_content[0:index], str(tags), html_content[:index]) self.log_message('added tags to html content') return html_content tags_added = False if tags is None or len(tags.strip()) <= 0: self.log_message('no tags need to be added to message') elif self.get_email_message() is None or self.get_email_message( ).get_message() is None: self.log_message('email message not formed correctly') else: msg_charset, self._last_charset = inspect_utils.get_charset( self.get_email_message()) content_type = self.get_email_message().get_message( ).get_content_type() self.log_message("content type: {}".format(content_type)) if content_type is None: pass elif (content_type == mime_constants.TEXT_PLAIN_TYPE or content_type == mime_constants.TEXT_HTML_TYPE): content = self.get_email_message().get_content() if content is None: self.get_email_message().set_content(tags, content_type, charset=msg_charset) tags_added = True else: if content.lower().find('<html>') > 0: content = add_tags_to_html(content, tags) else: content = add_tags_to_text( self.get_email_message().get_content(), tags) self.get_email_message().set_content(content, content_type, charset=msg_charset) tags_added = True elif content_type.startswith( mime_constants.MULTIPART_PRIMARY_TYPE): added_tags_to_text = False added_tags_to_html = False message = self.get_email_message().get_message() for part in message.get_payload(): part_content_type = part.get_content_type().lower() self.log_message('part_content_type: {}'.format( part_content_type)) #DEBUG if part_content_type == mime_constants.TEXT_PLAIN_TYPE and not added_tags_to_text: content = add_tags_to_text(part.get_payload(), tags) charset, __ = inspect_utils.get_charset(content) if charset.lower() == msg_charset.lower(): part.set_payload(content) else: part.set_payload(content, charset=charset) added_tags_to_text = True elif part_content_type == mime_constants.TEXT_HTML_TYPE and not added_tags_to_html: content = add_tags_to_html(part.get_payload(), tags) charset, __ = inspect_utils.get_charset(content) part.set_payload(content, charset=charset) added_tags_to_html = True # no need to keep getting payloads if we've added the tags if added_tags_to_text and added_tags_to_html: break tags_added = added_tags_to_text or added_tags_to_html if not tags_added: msg = MIMENonMultipart(mime_constants.TEXT_PRIMARY_TYPE, mime_constants.PLAIN_SUB_TYPE) msg.set_payload(tags) self.get_email_message().get_message().attach(msg) self.log_message('attached new payload\n{}'.format(msg)) tags_added = True self.log_message( 'added tags to multipart message: {}'.format(tags_added)) else: self.log_message( 'unable to add tags to message with {} content type'. format(content_type)) return tags_added def get_tags(self): ''' Returns the list of tags to be added to the email_message text. >>> regular_tags = ['test tag'] >>> crypto_message = CryptoMessage() >>> crypto_message.set_tag('test tag') >>> tags = crypto_message.get_tags() >>> crypto_message.log_message('tags: {}'.format(tags)) >>> tags == regular_tags True ''' if self.DEBUGGING: self.log_message("tags:\n{}".format(self.tags)) return self.tags def get_tag(self): ''' Returns the tags as a string to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_tag('test tag') >>> tags = crypto_message.get_tag() >>> tags == 'test tag' True ''' if self.tags is None: tag = '' else: tag = '\n'.join(self.tags) if self.DEBUGGING: self.log_message("tag:\n{}".format(tag)) return tag def set_tag(self, new_tag): ''' Sets the tag to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_tag(None) >>> tag = crypto_message.get_tag() >>> tag == '' True ''' if new_tag is None: if self.DEBUGGING: self.log_message("tried to set blank tag") elif new_tag == '': self.tags = [] if self.DEBUGGING: self.log_message("reset tags") else: if is_string(new_tag): new_tag = new_tag.strip('\n') self.tags = [new_tag] else: self.tags = new_tag if self.DEBUGGING: self.log_message("new tag:\n{}".format(new_tag)) def add_tag(self, new_tag): ''' Add new tag to the existing tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_tag(None) >>> tag = crypto_message.get_tag() >>> tag == '' True ''' if new_tag is None or len(new_tag) <= 0: if self.DEBUGGING: self.log_message("tried to add empty tag") else: new_tag = new_tag.strip('\n') if self.tags == None or len(self.tags) <= 0: if self.DEBUGGING: self.log_message( "adding to an empty tag:\n{}".format(new_tag)) self.tags = [new_tag] else: if self.DEBUGGING: self.log_message("adding to tag:\n{}".format(new_tag)) if new_tag.startswith('.'): self.tags[len(self.tags) - 1] += new_tag else: self.tags.append(new_tag) def add_tag_once(self, new_tag): ''' Add new tag only if it isn't already in the tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_tag_once(None) >>> tag = crypto_message.get_tag().strip() >>> tag == '' True ''' if new_tag is None: pass elif self.tags is None or new_tag not in self.tags: self.add_tag(new_tag) def add_prefix_to_tag_once(self, new_tag): ''' Add new tag prefix only if it isn't already in the tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_prefix_to_tag_once(None) >>> tag = crypto_message.get_tag() >>> tag == '' True ''' if new_tag is None: pass elif self.tags is None or new_tag not in self.tags: new_tag = new_tag.strip('\n') if self.DEBUGGING: self.log_message("adding prefix to tag:\n{}".format(new_tag)) if self.tags == None or len(self.tags) <= 0: self.tags = [new_tag] else: old_tags = self.tags self.tags = [new_tag] for tag in old_tags: self.tags.append(tag) def get_error_tags(self): ''' Returns the list of error tags to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_error_tag('test error tag') >>> tags = crypto_message.get_error_tags() >>> tags == ['test error tag'] True ''' if self.DEBUGGING: self.log_message("error tags:\n{}".format(self.error_tags)) return self.error_tags def get_error_tag(self): ''' Returns the error tags as a string to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_error_tag('test error tag') >>> tag = crypto_message.get_error_tag() >>> tag == 'test error tag' True ''' if self.error_tags is None: error_tag = '' else: error_tag = '\n'.join(self.error_tags) if self.DEBUGGING: self.log_message("error tag:\n{}".format(error_tag)) return error_tag def set_error_tag(self, new_tag): ''' Sets the error tag to be added to the email_message text. >>> crypto_message = CryptoMessage() >>> crypto_message.set_error_tag(None) >>> tag = crypto_message.get_error_tag() >>> tag == '' True ''' if new_tag is None: if self.DEBUGGING: self.log_message("tried to set blank error tag") elif new_tag == '': self.error_tags = [] if self.DEBUGGING: self.log_message("reset error tags") else: if is_string(new_tag): new_tag = new_tag.strip('\n') self.error_tags = [new_tag] else: self.error_tags = new_tag if self.DEBUGGING: self.log_message("new tag:\n{}".format(self.error_tags)) def add_error_tag_once(self, new_tag): ''' Add new error tag only if it isn't already in the error tag. >>> crypto_message = CryptoMessage() >>> crypto_message.add_error_tag_once(None) >>> tag = crypto_message.get_error_tag().strip() >>> tag == '' True ''' if new_tag is None: pass elif self.error_tags is None or new_tag not in self.error_tags: new_tag = new_tag.strip('\n') if len(self.error_tags) <= 0: if self.DEBUGGING: self.log_message( "adding to an empty error tag:\n{}".format(new_tag)) self.error_tags = [new_tag] else: self.log_message("adding to error tag:\n{}".format(new_tag)) if new_tag.startswith('.'): self.error_tags[len(self.tags) - 1] += new_tag else: self.error_tags.append(new_tag) def log_exception(self, exception): ''' Log the message to the local and Exception logs. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> CryptoMessage().log_exception('test message') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.crypto_message.log')) True ''' self.log_message(exception) record_exception(message=exception) def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> CryptoMessage().log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.crypto_message.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class MailAPI(object): '''Handle the API for GoodCrypto Mail.''' def __init__(self): self.log = None def interface(self, request): '''Interface with the server through the API. All requests must be via a POST. ''' # final results and error_messages of the actions result = None ok = False response = None try: self.action = self.domain = self.mail_server_address = None self.public_key = self.encryption_name = self.email = self.fingerprint = None self.user_name = self.admin = self.password = None self.ip = get_remote_ip(request) self.log_message('attempting mail api call from {}'.format( self.ip)) if request.method == 'POST': try: form = APIForm(request.POST) if form.is_valid(): cleaned_data = form.cleaned_data self.action = cleaned_data.get( api_constants.ACTION_KEY) self.log_message('action: {}'.format(self.action)) if self.action == api_constants.CREATE_SUPERUSER: self.admin = strip_input( cleaned_data.get(api_constants.SYSADMIN_KEY)) self.log_message('admin: {}'.format(self.admin)) elif self.action == api_constants.CONFIGURE: self.domain = strip_input( cleaned_data.get(api_constants.DOMAIN_KEY)) self.log_message('domain: {}'.format(self.domain)) self.mail_server_address = strip_input( cleaned_data.get( api_constants.MTA_ADDRESS_KEY)) self.log_message('mail_server_address: {}'.format( self.mail_server_address)) elif self.action == api_constants.GET_FINGERPRINT: self.encryption_name = strip_input( cleaned_data.get( api_constants.ENCRYPTION_NAME_KEY)) self.log_message('encryption_name: {}'.format( self.encryption_name)) self.email = strip_input( cleaned_data.get(api_constants.EMAIL_KEY)) self.log_message('email: {}'.format(self.email)) self.password = strip_input( cleaned_data.get(api_constants.PASSWORD_KEY)) elif self.action == api_constants.IMPORT_KEY: self.public_key = strip_input( cleaned_data.get(api_constants.PUBLIC_KEY)) self.encryption_name = strip_input( cleaned_data.get( api_constants.ENCRYPTION_NAME_KEY)) self.log_message('encryption_name: {}'.format( self.encryption_name)) self.fingerprint = strip_input( cleaned_data.get( api_constants.FINGERPRINT_KEY)) self.log_message('fingerprint: {}'.format( self.fingerprint)) self.user_name = strip_input( cleaned_data.get(api_constants.USER_NAME_KEY)) self.log_message('user_name: {}'.format( self.user_name)) self.admin = strip_input( cleaned_data.get(api_constants.SYSADMIN_KEY)) self.log_message('admin: {}'.format(self.admin)) self.password = strip_input( cleaned_data.get(api_constants.PASSWORD_KEY)) elif self.action == api_constants.GET_CONTACT_LIST: self.encryption_name = strip_input( cleaned_data.get( api_constants.ENCRYPTION_NAME_KEY)) self.log_message('encryption_name: {}'.format( self.encryption_name)) self.admin = strip_input( cleaned_data.get(api_constants.SYSADMIN_KEY)) self.log_message('admin: {}'.format(self.admin)) self.password = strip_input( cleaned_data.get(api_constants.PASSWORD_KEY)) result = self.take_api_action() else: result = self.format_bad_result('Invalid form') self.log_attempted_access(result) self.log_message('api form is not valid') self.log_bad_form(request, form) except: result = self.format_bad_result('Unknown error') self.log_attempted_access(result) record_exception() self.log_message('unexpected error while parsing input') self.log_message( 'EXCEPTION - see syr.exception.log for details') else: self.log_attempted_access('Attempted GET connection') self.log_message('redirecting api GET request to website') response = HttpResponsePermanentRedirect('/') if response is None: response = self.get_api_response(request, result) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') response = HttpResponsePermanentRedirect('/') return response def take_api_action(self): result = None ok, error_message = self.is_data_ok() if ok: if self.action == api_constants.CONFIGURE: mail_options = options.get_options() mail_options.domain = self.domain mail_options.mail_server_address = self.mail_server_address options.save_options(mail_options) result = self.format_result(api_constants.CONFIGURE, ok) self.log_message('configure result: {}'.format(result)) elif self.action == api_constants.CREATE_SUPERUSER: user, password, error_message = create_superuser(self.admin) if error_message is None: if password is None: result = self.format_bad_result(error_message) else: result = self.format_message_result( api_constants.CREATE_SUPERUSER, ok, password) else: result = self.format_bad_result(error_message) self.log_message('create user result: {}'.format(result)) elif self.action == api_constants.STATUS: result = self.format_result(api_constants.STATUS, get_mail_status()) self.log_message('status result: {}'.format(result)) elif self.action == api_constants.IMPORT_KEY: from goodcrypto.mail.import_key import import_key_now result_ok, status, fingerprint_ok = import_key_now( self.encryption_name, self.public_key, self.user_name, self.fingerprint, self.password) if result_ok: __, email = status.split(':') result = self.format_message_result( api_constants.IMPORT_KEY, True, email) else: result = self.format_bad_result(status) self.log_message('import key result: {}'.format(result)) elif self.action == api_constants.GET_CONTACT_LIST: email_addresses = contacts.get_contact_list( self.encryption_name) addresses = '\n'.join(email_addresses) result = self.format_message_result( api_constants.GET_CONTACT_LIST, True, addresses) self.log_message('{} {} contacts found'.format( len(email_addresses), self.encryption_name)) elif self.action == api_constants.GET_FINGERPRINT: fingerprint, verified, __ = contacts.get_fingerprint( self.email, self.encryption_name) if fingerprint is None: ok = False error_message = 'No {} fingerprint for {}'.format( self.encryption_name, self.email) result = self.format_bad_result(error_message) self.log_message('bad result: {}'.format(result)) else: message = 'Fingerprint: {} verified: {}'.format( format_fingerprint(fingerprint), verified) result = self.format_message_result( api_constants.GET_FINGERPRINT, True, message) self.log_message(message) else: ok = False error_message = 'Bad action: {}'.format(self.action) result = self.format_bad_result(error_message) self.log_message('bad action result: {}'.format(result)) else: result = self.format_bad_result(error_message) self.log_message('data is bad') return result def is_data_ok(self): '''Check if all the required data is present.''' error_message = '' ok = False if self.has_content(self.action): if self.action == api_constants.CONFIGURE: if self.has_content(self.domain) and self.has_content( self.mail_server_address): ok = True self.log_message('minimum configure data found') elif self.action == api_constants.CREATE_SUPERUSER: if self.has_content(self.admin): ok = True self.log_message( 'minimum create user data found: {}'.format( self.admin)) elif self.action == api_constants.STATUS: ok = True self.log_message('status request found') elif self.action == api_constants.GET_FINGERPRINT: if (self.has_content(self.encryption_name) and self.has_content(self.email)): ok = True self.log_message('minimum get fingerprint data found') elif self.action == api_constants.IMPORT_KEY: if (self.has_content(self.public_key) and self.has_content(self.encryption_name) and self.has_content(self.admin) and self.has_content(self.password)): ok = True self.log_message('minimum import key data found') elif self.action == api_constants.GET_CONTACT_LIST: if (self.has_content(self.encryption_name) and self.has_content(self.admin) and self.has_content(self.password)): ok = True self.log_message('minimum get contact list data found') if not ok: error_message = 'Missing required data' self.log_message('missing required data') else: ok = False error_message = 'Missing required action' self.log_message('missing required action') return ok, error_message def has_content(self, value): '''Check that the value has content.''' try: str_value = str(value) if str_value is None or len(str_value.strip()) <= 0: ok = False else: ok = True except: ok = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return ok def format_result(self, action, ok, error_message=None): '''Format the action's result.''' if error_message is None: result = { api_constants.ACTION_KEY: action, api_constants.OK_KEY: ok } else: result = { api_constants.ACTION_KEY: action, api_constants.OK_KEY: ok, api_constants.ERROR_KEY: error_message } return result def format_message_result(self, action, ok, message): '''Format the action's result.''' result = { api_constants.ACTION_KEY: action, api_constants.OK_KEY: ok, api_constants.MESSAGE_KEY: message } return result def format_bad_result(self, error_message): '''Format the bad result for the action.''' result = None if self.action and len(self.action) > 0: result = self.format_result(self.action, False, error_message=error_message) else: result = self.format_result('Unknown', False, error_message=error_message) self.log_message('action result: {}'.format(error_message)) return result def get_api_response(self, request, result): ''' Get API reponse as JSON. ''' json_result = json.dumps(result) self.log_message('json results: {}'.format(''.join(json_result))) response = render_to_response('mail/api_response.html', { 'result': ''.join(json_result), }, context_instance=RequestContext(request)) return response def log_attempted_access(self, results): '''Log an attempted access to the api.''' self.log_message('attempted access from {} for {}'.format( self.ip, results)) def log_bad_form(self, request, form): ''' Log the bad fields entered.''' # see django.contrib.formtools.utils.security_hash() # for example of form traversal for field in form: if (hasattr(form, 'cleaned_data') and field.name in form.cleaned_data): name = field.name else: # mark invalid data name = '__invalid__' + field.name self.log_message('name: {}; data: {}'.format(name, field.data)) try: if form.name.errors: self.log_message(' ' + form.name.errors) if form.email.errors: self.log_message(' ' + form.email.errors) except: pass self.log_message('logged bad api form') def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> MailAPI().log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.api.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class Filter(object): ''' Filters a message through the encrypt or decrypt filter as needed. The overall design of this module is trying to emulate a milter so if we ever decide to convert from a content filter it will be easier. ''' DEBUGGING = False def __init__(self, from_user, to_user, message): ''' >>> # In honor of John Kiriakou, who went to prison for exposing CIA torture. >>> # In honor of asn, developer of Obfsproxy. >>> filter = Filter('*****@*****.**', '*****@*****.**', 'message') >>> filter is not None True ''' self.log = self.out_message = None self.sender = from_user self.recipient = to_user self.in_message = message def process(self): ''' Encrypt/decrypt the message, or decide not to. ''' self.log_message( '=== starting to filter mail from {} to {} ==='.format( self.sender, self.recipient)) self.out_message = self.in_message crypto_message = CryptoMessage(email_message=EmailMessage( self.in_message), sender=self.sender, recipient=self.recipient) if self.possibly_needs_encryption(): self.process_outbound_message(crypto_message) else: self.process_inbound_message(crypto_message) result_code = self.reinject_message() self.log_message('mail filtered ok: {}'.format(result_code)) self.log_message( '=== finished filtering mail from {} to {} ==='.format( self.sender, self.recipient)) return result_code def process_outbound_message(self, crypto_message): ''' Process an outbound message, encrypting if approriate. ''' try: # something is wrong if an outbound message is from the metadata address # any messages from a metadata address don't pass through the filter if is_metadata_address(self.sender): self.sender = get_admin_email() self.bounce_outbound_message( i18n('Message originating from your metadata address')) self.out_message = None else: encrypt = Encrypt(crypto_message) encrypted_message = encrypt.process_message() filtered = encrypted_message.is_filtered() crypted = encrypted_message.is_crypted() processed = encrypted_message.is_processed() if encrypted_message.is_dropped(): self.out_message = None self.log_message('outbound message dropped') elif processed: # nothing to re-inject at this time self.out_message = None self.log_message('outbound message processed') else: self.out_message = encrypted_message.get_email_message( ).to_string() self.sender = encrypted_message.smtp_sender() self.recipient = encrypted_message.smtp_recipient() self.log_message( 'outbound sender: {} recipient: {}'.format( self.sender, self.recipient)) self.log_message( 'outbound final status: filtered: {} crypted: {} queued: {}' .format(filtered, crypted, processed)) except MessageException as message_exception: self.bounce_outbound_message(message_exception.value) self.out_message = None except Exception as exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') try: self.bounce_outbound_message(exception.value) except: self.bounce_outbound_message(exception) self.out_message = None except IOError as ioerror: try: self.bounce_outbound_message(ioerror.value) except: self.bounce_outbound_message(ioerror) self.out_message = None def process_inbound_message(self, crypto_message): ''' Process an inbound message, decrypting if appropriate. ''' try: debundled_message = None filtered = crypted = processed = False if self.DEBUGGING: self.log_message( crypto_message.get_email_message().to_string()) debundle = Debundle(crypto_message) debundled_message = debundle.process_message() filtered = debundled_message.is_filtered() crypted = debundled_message.is_crypted() processed = debundled_message.is_processed() if debundled_message.is_dropped(): self.out_message = None self.log_message('inbound message dropped') elif processed: # nothing to re-inject at this time self.out_message = None self.log_message('inbound message processed') else: self.out_message = debundled_message.get_email_message( ).to_string() self.sender = debundled_message.smtp_sender() self.recipient = debundled_message.smtp_recipient() self.log_message('inbound sender: {} recipient: {}'.format( self.sender, self.recipient)) self.log_message( 'inbound final status: filtered: {} crypted: {} queued: {}'. format(filtered, crypted, processed)) except DKIMException as dkim_exception: self.drop_message(dkim_exception) self.out_message = None except CryptoException as crypto_exception: if debundled_message is not None and not debundled_message.is_dropped( ): self.wrap_inbound_message(crypto_exception, debundled_message) self.out_message = None except MessageException as message_exception: if debundled_message is not None and not debundled_message.is_dropped( ): self.wrap_inbound_message(message_exception, debundled_message) self.out_message = None except Exception as exception: self.wrap_inbound_message(exception, debundled_message) self.out_message = None except IOError as ioerror: self.wrap_inbound_message(ioerror, debundled_message) self.out_message = None def possibly_needs_encryption(self): ''' Determine if the message might need to be encrypted or not. It could need encrytion if the sender has the same domain as the domain defined in goodcrypto mail server's options. If we're not using an SMTP proxy, then it never needs encryption if the message is going to and from a user with the domain defined in goodcrypto mail server's options. ''' maybe_needs_encryption = email_in_domain(self.sender) self.log_message( 'possibly needs encryption: {}'.format(maybe_needs_encryption)) return maybe_needs_encryption def get_processed_message(self): ''' Get the message after it has been processed. >>> filter = Filter('root', 'root', 'bad message') >>> filter.get_processed_message() ''' return self.out_message def reinject_message(self, message=None): ''' Re-inject message back into queue. ''' if message is None: message = self.out_message try: if message is None: # set to result_ok to True because we've already bounced the message or # there isn't anything to bounce because it failed validation result_ok = True self.log_message('nothing to reinject') else: self.log_message( 'starting to re-inject message into postfix queue') result_ok = send_message(self.sender, self.recipient, message) self.log_message('re-injected message: {}'.format(result_ok)) if self.DEBUGGING: self.log_message('\n==================\n') self.log_message(message) self.log_message('\n==================\n') except Exception as exception: result_ok = False self.log_message( 'error while re-injecting message into postfix queue') self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() try: error_message = exception.value to_address = self.reject_message(error_message) self.log_message('sent notice to {} about {}'.format( to_address, error_message)) except: record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') return result_ok def reject_message(self, error_message, message=None): ''' Reject a message that had an unexpected error and return the to address. >>> # This message will fail if testing on dev system >>> to_address = get_admin_email() >>> filter = Filter('root', 'root', 'bad message') >>> filter.reject_message('Unknown message') == to_address True ''' try: if message is None: message = self.out_message if email_in_domain(self.sender): to_address = self.sender subject = i18n('Undelivered Mail: Unable to send message') elif email_in_domain(self.recipient): to_address = self.recipient subject = i18n('Error: Unable to receive message') else: to_address = get_admin_email() subject = i18n('Message rejected.') notice = '{}'.format(error_message) if message is not None: notice += '\n\n===================\n{}'.format(message) notify_user(to_address, subject, notice) except: raise return to_address def bounce_outbound_message(self, error_message): ''' Bounce a message that a local user originated. ''' self.log_message(error_message) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() subject = i18n( '{} - Undelivered Mail: Unable to protect message'.format( TAG_ERROR)) utils.bounce_message(self.in_message, self.sender, subject, error_message) def wrap_inbound_message(self, error_message, debundled_message): ''' Wrap an inbound message that had a serious error. ''' self.log_message(error_message) body = error_message try: processed_message = debundled_message.crypto_message error_tags = processed_message.get_error_tags() if error_tags and len(error_tags) > 0: for tag in error_tags: body += '\n{}'.format(tag) if options.add_long_tags(): long_tags = processed_message.get_tags() if long_tags and len(long_tags) > 0: for tag in long_tags: body += '\n{}'.format(tag) except: pass subject = i18n('{} - Unable to decrypt message'.format(TAG_ERROR)) utils.bounce_message(self.in_message, self.recipient, subject, body) def drop_message(self, error_message): ''' Drop a message that we shouldn't process from a remote user. ''' self.log_message(error_message) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() subject = i18n('{} - Unable to decrypt message'.format(TAG_ERROR)) utils.drop_message(self.in_message, self.recipient, subject, error_message) def log_message(self, message): ''' Record debugging messages. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> filter = Filter('*****@*****.**', ['*****@*****.**'], 'message') >>> filter.log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.message.filter.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class RetrieveKey(object): ''' Retrieve a key from a keyserver. ''' def __init__(self, email, encryption_name, keyserver, key_id, user_initiated_search): ''' >>> # In honor of Werner Koch, developer of gpg. >>> email = '*****@*****.**' >>> crypto_name = 'GPG' >>> srk_class = RetrieveKey(email, crypto_name, 'pgp.mit.edu', 'F2AD85AC1E42B367', '*****@*****.**') >>> srk_class = RetrieveKey(None, crypto_name, 'pgp.mit.edu', 'F2AD85AC1E42B367', '*****@*****.**') >>> srk_class = RetrieveKey(email, None, 'pgp.mit.edu', 'F2AD85AC1E42B367', '*****@*****.**') >>> srk_class = RetrieveKey(email, crypto_name, None, 'F2AD85AC1E42B367', '*****@*****.**') >>> srk_class = RetrieveKey(None, None, None, None, None) ''' self.log = LogFile() self.email = email self.encryption_name = encryption_name self.keyserver = keyserver self.key_id = key_id self.user_initiated_search = user_initiated_search self.key_plugin = None def start_retrieval(self): ''' Queue retrieving key from the keyserver. When the job finishes, associated database entries will be made from another queued job which is dependent on the key retrieval's job. Test extreme case. >>> rk = RetrieveKey(None, None, None, None, None) >>> rk.start_retrieval() False ''' if self.email == UNKNOWN_EMAIL: email_or_fingerprint = self.key_id else: email_or_fingerprint = self.email try: result_ok = (self.email is not None and self.encryption_name is not None and self.keyserver is not None and self.key_id is not None and self.user_initiated_search is not None) if result_ok: self.key_plugin = KeyFactory.get_crypto( self.encryption_name, crypto_software.get_key_classname(self.encryption_name)) result_ok = self.key_plugin is not None if result_ok: from goodcrypto.mail.keyserver_utils import add_contact_records self.key_plugin.retrieve_key(self.key_id, self.keyserver) retrieve_job = self.key_plugin.get_job() queue = self.key_plugin.get_queue() if queue is None or retrieve_job is None: self.log_message('unable to queue job to add contact recs for {}'.format( self.key_id)) result_ok = False else: self.log_message('starting to add contact records for {} (after job: {})'.format( self.key_id, retrieve_job.get_id())) add_contact_records(email_or_fingerprint, self.encryption_name, self.user_initiated_search, retrieve_job.get_id(), queue.key) result_ok = True else: result_ok = False self.log_message('unable to queue retrieving {} key for {}'.format( self.encryption_name, self.key_id)) except Exception as exception: result_ok = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_message('finished queueing retreival for {} ok: {}'.format(email_or_fingerprint, result_ok)) return result_ok def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> RetrieveKey(None, None, None, None, None).log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.retrieve_key.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class GPGExec(object): ''' Execute a gpg command. gpg expects single tasks so we use redis to queue tasks. -fd: 0 = stdin 1 = stdout 2 = stderr ''' DEBUGGING = False def __init__(self, home_dir, auto_check_trustdb=False): ''' Create a new GPGExec object. >>> gpg_exec = GPGExec('/var/local/projects/goodcrypto/server/data/oce/.gnupg') >>> gpg_exec != None True ''' self.log = LogFile() self.gpg_home = home_dir self.result_code = gpg_constants.ERROR_RESULT self.gpg_output = None self.gpg_error = None self.set_up_conf() # --no-tty: Do not write anything to TTY # --homedir: home directory for gpg's keyring files # --verbose: give details if error # --ignore-time-conflict: Since different machines have different ideas of what time it is, we want to ignore time conflicts. # --ignore-valid-from: "valid-from" is just a different kind of time conflict. # --batch: We're always in batch mode. # --lock-once: Lock the databases the first time a lock is requested and do not release the lock until the process terminates. # --no-auto-key-locate: Don't look for keys outside our system # --no-auto-check-trustdb: Do not always update the trust db because it goes online # --always-trust: We don't have any trust infrastructure yet. # --utf8-strings: Assume all arguments are in UTF-8 format. # redirect stdout and stderr so we can exam the results as needed kwargs = dict(no_tty=True, verbose=True, homedir=self.gpg_home, ignore_time_conflict=True, ignore_valid_from=True, batch=True, no_auto_key_locate=True, lock_once=True, utf8_strings=True, _env=minimal_env()) # gpg tries to go online when it updates the trustdb # so we don't want to check on every command if auto_check_trustdb: kwargs['auto_check_trustdb'] = True self.log_message('auto_check_trustdb') else: kwargs['no_auto_check_trustdb'] = True self.log_message('no_auto_check_trustdb') if (self.gpg_home is not None and os.path.exists(os.path.join(self.gpg_home, gpg_constants.TRUST_DB_FILENAME))): kwargs['always_trust'] = True self.gpg = sh.gpg.bake(**kwargs) # make sure no old job has left locked files self.clear_gpg_lock_files() self.clear_gpg_tmp_files() def execute(self, initial_args, passphrase=None, data=None): ''' Prepare and then run a gpg command. ''' result_ok = False timeout = None try: stdin_file = StringIO() args = initial_args if GPGExec.DEBUGGING: self.log_message("executing: {}".format(args)) if passphrase and len(passphrase) > 0: self.log_message('passphrase supplied') # passphrase will be passed on stdin, file descriptor 0 is stdin passphraseOptions = ['--passphrase-fd', '0'] args.append(passphraseOptions) stdin_file.write(passphrase) stdin_file.write(gpg_constants.EOL) if data: if isinstance(data, bytearray) or isinstance(data, bytes): data = data.decode() stdin_file.write(data) data_length = len(data) self.log_message('data length: {}'.format(data_length)) if data_length > gpg_constants.LARGE_DATA_CHUNK: timeout = int( (data_length / gpg_constants.LARGE_DATA_CHUNK) * gpg_constants.TIMEOUT_PER_CHUNK) * 1000 # in ms self.log_message('timeout in ms: {}'.format(data_length)) if GPGExec.DEBUGGING: self.log_message("data: {}".format(data)) stdin = stdin_file.getvalue() stdin_file.close() if GPGExec.DEBUGGING: self.log_message("gpg args:") for arg in args: self.log_message(' {}'.format(arg)) result_ok = self.run_gpg(args, stdin, timeout=timeout) self.log_message("gpg command result_ok: {}".format(result_ok)) except Exception as exception: result_ok = False self.result_code = gpg_constants.ERROR_RESULT self.gpg_error = str(exception) self.log_message('result code: {}'.format(self.result_code)) if self.gpg_error and len(self.gpg_error.strip()) > 0: self.log_message("gpg error: {}".format(self.gpg_error)) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() self.log.flush() return self.result_code, self.gpg_output, self.gpg_error def run_gpg(self, args, stdin, timeout=None): ''' Run the GPG command. ''' try: self.gpg_output = self.gpg_error = None if gpg_constants.ENCRYPT_DATA in args: command = gpg_constants.ENCRYPT_DATA elif gpg_constants.DECRYPT_DATA in args: command = gpg_constants.DECRYPT_DATA elif gpg_constants.SEARCH_KEYSERVER in args: command = gpg_constants.SEARCH_KEYSERVER elif gpg_constants.RETRIEVE_KEYS in args: command = gpg_constants.RETRIEVE_KEYS else: command = args[0] self.log_message('--- started executing: {} ---'.format(command)) with elapsed_time() as gpg_time: if timeout is None: if stdin and len(stdin) > 0: gpg_results = self.gpg(*args, _in=stdin, _ok_code=[0,2]) else: gpg_results = self.gpg(*args, _ok_code=[0,2]) else: if stdin and len(stdin) > 0: gpg_results = self.gpg(*args, _in=stdin, _ok_code=[0,2], _timeout=timeout) else: gpg_results = self.gpg(*args, _ok_code=[0,2], _timeout=timeout) self.log_message('{} command elapsed time: {}'.format(command, gpg_time)) self.log_message('{} exit code: {}'.format(command, gpg_results.exit_code)) self.log_message('--- finished executing: {} ---'.format(command)) self.result_code = gpg_results.exit_code self.gpg_output = self.get_decoded_results(gpg_results.stdout) self.gpg_error = self.get_decoded_results(gpg_results.stderr) if GPGExec.DEBUGGING: if self.gpg_output: self.log_message('stdout: {}'.format(self.gpg_output)) if self.gpg_error: self.log_message('stderr:{}'.format(self.gpg_error)) except sh.ErrorReturnCode as exception: self.result_code = exception.exit_code if self.gpg_error is None: self.gpg_error = exception.stderr # get the essence of the error self.gpg_error = exception.stderr if self.gpg_error and self.gpg_error.find(':'): self.gpg_error = self.gpg_error[self.gpg_error.find(':') + 1:] if self.gpg_error and self.gpg_error.find(':'): self.gpg_error = self.gpg_error[self.gpg_error.find(':') + 1:] self.log_message('exception result code: {}'.format(self.result_code)) if exception: self.log_message("exception:\n==============\n{}\n============".format(exception)) except JobTimeoutException as job_exception: self.log_message('run_gpg exception: {}'.format(str(job_exception))) self.result_code = gpg_constants.TIMED_OUT_RESULT self.gpg_error = str(job_exception) self.gpg_output = None self.log_message('--- timedout executing {} ---'.format(command)) return (self.result_code == gpg_constants.GOOD_RESULT or self.result_code == gpg_constants.CONDITIONAL_RESULT) def get_decoded_results(self, results): ''' Get the decoded results. >>> exec = GPGExec('/var/local/projects/goodcrypto/server/data/oce/.gnupg') >>> exec.get_decoded_results(b'test') 'test' ''' def decode_with(results, charset): try: decoded_results = results.decode(encoding=charset) except ValueError as ve: decoded_results = None return decoded_results decoded_results = decode_with(results, 'utf-8') if decoded_results is None: decoded_results = decode_with(results, 'iso-8859-1') if decoded_results is None: decoded_results = decode_with(results, 'iso-8859-2') if decoded_results is None: decoded_results = decode_with(results, 'iso-8859-15') if decoded_results is None: decoded_results = decode_with(results, 'koi8-r') if decoded_results is None: decoded_results = results self.log_message('unable to decode results "{}" with standard gpg charsets'.format(results)) return decoded_results def set_up_conf(self): ''' Set up the GPG conf file, if it doesn't exist. ''' try: if self.gpg_home is None: self.log_message('gpg home not defined yet') else: gpg_conf = os.path.join(self.gpg_home, gpg_constants.CONF_FILENAME) if not os.path.exists(gpg_conf): lines = [] lines.append('#\n') lines.append('# This is an adpation of the Riseup OpenPGP Best Practices\n') lines.append('# https://help.riseup.net/en/security/message-security/openpgp/best-practices\n') lines.append('#\n') lines.append('#-----------------------------\n') lines.append('# behavior\n') lines.append('#-----------------------------\n') lines.append('# Disable inclusion of the version string in ASCII armored output\n') lines.append('no-emit-version\n') lines.append('# Disable comment string in clear text signatures and ASCII armored messages\n') lines.append('no-comments\n') lines.append('# Display long key IDs\n') lines.append('keyid-format 0xlong\n') lines.append('# List all keys (or the specified ones) along with their fingerprints\n') lines.append('with-fingerprint\n') lines.append('# Display the calculated validity of user IDs during key listings\n') lines.append('list-options show-uid-validity\n') lines.append('verify-options show-uid-validity\n') lines.append('# Try to use the GnuPG-Agent. With this option, GnuPG first tries to connect to\n') lines.append('# the agent before it asks for a passphrase.\n') lines.append('use-agent\n') lines.append('#-----------------------------\n') lines.append('# keyserver -- goodcrypto relies on per-to-per key exchange, not key servers\n') lines.append('#-----------------------------\n') lines.append('# This is the server that --recv-keys, --send-keys, and --search-keys will\n') lines.append('# communicate with to receive keys from, send keys to, and search for keys on\n') lines.append('# keyserver hkps://hkps.pool.sks-keyservers.net\n') lines.append('# Provide a certificate store to override the system default\n') lines.append('# Get this from https://sks-keyservers.net/sks-keyservers.netCA.pem\n') lines.append('# keyserver-options ca-cert-file=/usr/local/etc/ssl/certs/hkps.pool.sks-keyservers.net.pem\n') lines.append('# Set the proxy to use for HTTP and HKP keyservers - default to the standard\n') lines.append('# local Tor socks proxy\n') lines.append('# It is encouraged to use Tor for improved anonymity. Preferrably use either a\n') lines.append('# dedicated SOCKSPort for GnuPG and/or enable IsolateDestPort and\n') lines.append('# IsolateDestAddr\n') lines.append('#keyserver-options http-proxy=socks5-hostname://127.0.0.1:9050\n') lines.append("# Don't leak DNS, see https://trac.torproject.org/projects/tor/ticket/2846\n") lines.append('keyserver-options no-try-dns-srv\n') lines.append('# When using --refresh-keys, if the key in question has a preferred keyserver\n') lines.append('# URL, then disable use of that preferred keyserver to refresh the key from\n') lines.append('keyserver-options no-honor-keyserver-url\n') lines.append('# When searching for a key with --search-keys, include keys that are marked on\n') lines.append('# the keyserver as revoked\n') lines.append('keyserver-options include-revoked\n') lines.append('#-----------------------------\n') lines.append('# algorithm and ciphers\n') lines.append('#-----------------------------\n') lines.append('# list of personal digest preferences. When multiple digests are supported by\n') lines.append('# all recipients, choose the strongest one\n') lines.append('personal-cipher-preferences AES256 AES192 AES CAST5\n') lines.append('# list of personal digest preferences. When multiple ciphers are supported by\n') lines.append('# all recipients, choose the strongest one\n') lines.append('personal-digest-preferences SHA512 SHA384 SHA256 SHA224\n') lines.append('# message digest algorithm used when signing a key\n') lines.append('cert-digest-algo SHA512\n') lines.append('# This preference list is used for new keys and becomes the default for\n') lines.append('# "setpref" in the edit menu\n') lines.append('default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed\n') ''' lines.append('# when outputting certificates, view user IDs distinctly from keys:\n') lines.append('fixed-list-mode\n') lines.append("# long keyids are more collision-resistant than short keyids (it's trivial to make a key with any desired short keyid)") lines.append('keyid-format 0xlong\n') lines.append('# when multiple digests are supported by all recipients, choose the strongest one:\n') lines.append('personal-digest-preferences SHA512 SHA384 SHA256 SHA224\n') lines.append('# preferences chosen for new keys should prioritize stronger algorithms: \n') lines.append('default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 BZIP2 ZLIB ZIP Uncompressed\n') lines.append("# If you use a graphical environment (and even if you don't) you should be using an agent:") lines.append('# (similar arguments as https://www.debian-administration.org/users/dkg/weblog/64)\n') lines.append('use-agent\n') lines.append('# You should always know at a glance which User IDs gpg thinks are legitimately bound to the keys in your keyring:\n') lines.append('verify-options show-uid-validity\n') lines.append('list-options show-uid-validity\n') lines.append('# include an unambiguous indicator of which key made a signature:\n') lines.append('# (see http://thread.gmane.org/gmane.mail.notmuch.general/3721/focus=7234)\n') lines.append('sig-notation [email protected]=%g\n') lines.append('# when making an OpenPGP certification, use a stronger digest than the default SHA1:\n') lines.append('cert-digest-algo SHA256\n') ''' self.log_message('creating {}'.format(gpg_conf)) with open(gpg_conf, 'wt') as f: for line in lines: f.write(line) sh.chmod('0400', gpg_conf) self.log_message('created {}'.format(gpg_conf)) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def clear_gpg_lock_files(self): ''' Delete gpg lock files. Warning: Calling this method when a valid lock file exists can have very serious consequences. Lock files are in gpg home directory and are in the form ".*.lock", ".?*", or possibly "trustdb.gpg.lock". ''' try: if self.gpg_home is None: self.log_message("unable to clear gpg's lock files because home dir unknown") else: filenames = os.listdir(self.gpg_home) if filenames and len(filenames) > 0: for name in filenames: if name.endswith(gpg_constants.LOCK_FILE_SUFFIX): os.remove(os.path.join(self.gpg_home, name)) self.log_message("deleted lock file " + name) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def clear_gpg_tmp_files(self): ''' Delete gpg tmp files. ''' TmpPREFIX = 'tmp' TmpSUFFIX = '~' try: if self.gpg_home is None: self.log_message("unable to clear gpg's tmp files because home dir unknown") else: filenames = os.listdir(self.gpg_home) if filenames and len(filenames) > 0: for name in filenames: if name.startswith(TmpPREFIX) and name.endswith(TmpSUFFIX): os.remove(os.path.join(self.gpg_home, name)) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def log_message(self, message): ''' Log the message. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class Decrypt(object): ''' Decrypt message filter. This filter tries all known encryption software for the recipient. Because encryption may be nested, this class keeps trying until the message is decrypted, or no valid encryption program can decrypt it further. !!!! If part of a message is plaintext and part encrypted, the decrypted text replaces the entire text, and the plaintext part is lost. !!!! A multiply-encrypted message may be tagged decrypted if any layer is successfully decrypted, even if an inner layer is still encrypted. See the unit tests to see how to use the Decrypt class. ''' DEBUGGING = False # the encrypted content is the second part; indexing starts at 0 ENCRYPTED_BODY_PART_INDEX = 1 def __init__(self, crypto_message): ''' >>> decrypt = Decrypt(None) >>> decrypt != None True ''' self.log = LogFile() self.crypto_message = crypto_message self.need_to_send_metadata_key = False def process_message(self): ''' If the message is encrypted, try to decrypt it. If it's not encrypted, add a warning about the dangers of unencrypted messages. See unittests for usage as the test set up is too complex for a doctest. ''' try: filtered = decrypted = False from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() if options.verify_dkim_sig(): self.crypto_message, dkim_sig_verified = decrypt_utils.verify_dkim_sig( self.crypto_message) if options.dkim_delivery_policy() == DKIM_DROP_POLICY: self.log_message('verified dkim signature ok: {}'.format( dkim_sig_verified)) elif dkim_sig_verified: self.log_message('verified dkim signature') else: self.log_message( 'unable to verify dkim signature, but dkim policy is to just warn' ) self.log_message( "checking if message from {} to {} needs decryption".format( from_user, to_user)) if Decrypt.DEBUGGING: self.log_message( 'logged original message headers in goodcrypto.message.utils.log' ) utils.log_message_headers(self.crypto_message, tag='original message headers') header_keys = HeaderKeys() if options.auto_exchange_keys(): header_contains_key_info = header_keys.manage_keys_in_header( self.crypto_message) else: header_contains_key_info = header_keys.keys_in_header( self.crypto_message) if self.crypto_message.is_dropped(): decrypted = False self.log_message( "message dropped because of bad key in header") self.log_message( 'logged dropped message headers in goodcrypto.message.utils.log' ) utils.log_message_headers(self.crypto_message, tag='dropped message headers') else: message_char_set, __ = get_charset( self.crypto_message.get_email_message()) self.log_message( 'message char set: {}'.format(message_char_set)) if self.crypto_message.get_email_message().is_probably_pgp(): decrypted, decrypted_with = self.decrypt_message() if not decrypted and self.crypto_message.is_metadata_crypted( ): tags.add_metadata_tag(self.crypto_message) self.log_message( "message only encrypted with metadata key") else: decrypt_utils.verify_clear_signed(from_user, self.crypto_message) if self.crypto_message.is_metadata_crypted(): tags.add_metadata_tag(self.crypto_message) self.log_message( "message only encrypted with metadata key") else: tags.add_unencrypted_warning(self.crypto_message) filtered = True self.log_message( "message doesn't appear to be encrypted at all") # create a private key for the recipient if there isn't one already add_private_key(to_user) self.need_to_send_metadata_key = ( # if the metadata wasn't encrypted not self.crypto_message.is_metadata_crypted() and # but the sender's key was in the header so we know the sender uses GoodCrypto private server header_contains_key_info and # and we don't have the sender's metadata key len( contacts.get_encryption_names( get_metadata_address(email=from_user))) < 1) self.log_message('need to send metadata key: {}'.format( self.need_to_send_metadata_key)) self.log_message('message content decrypted: {}'.format(decrypted)) # finally save a record so the user can verify the message was received securely if (decrypted or self.crypto_message.is_metadata_crypted() or self.crypto_message.is_private_signed() or self.crypto_message.is_clear_signed()): decrypt_utils.add_history_and_verification(self.crypto_message) if decrypted or self.crypto_message.is_metadata_crypted(): self.crypto_message.set_crypted(True) if not decrypted: self.log_message('metadata tag: {}'.format( self.crypto_message.get_tag())) analyzer = OpenPGPAnalyzer() if analyzer.is_encrypted( self.crypto_message.get_email_message().get_content()): tags.add_extra_layer_warning(self.crypto_message) else: self.crypto_message.set_crypted(False) decrypted_tags_filtered = tags.add_decrypted_tags_to_message( self.crypto_message) if decrypted_tags_filtered: filtered = True if filtered and not self.crypto_message.is_filtered(): self.crypto_message.set_filtered(True) self.log_message( "finished adding tags to decrypted message; filtered: {}". format(self.crypto_message.is_filtered())) self.log_message("message originally encrypted with: {}".format( self.crypto_message.get_crypted_with())) self.log_message("metadata originally encrypted with: {}".format( self.crypto_message.get_metadata_crypted_with())) decrypt_utils.re_mime_encode(self.crypto_message) if self.DEBUGGING: self.log_message( 'logged final decrypted headers in goodcrypto.message.utils.log' ) utils.log_message_headers(self.crypto_message, tag='final decrypted headers') self.log_message( ' final status: filtered: {} decrypted: {}'.format( self.crypto_message.is_filtered(), self.crypto_message.is_crypted())) except MessageException as message_exception: self.log_message(message_exception) raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return self.crypto_message def get_crypto_message(self): ''' Get the crypto message. ''' return self.crypto_message def decrypt_message(self, filter_msg=True): ''' Decrypt a message and add a tag if unsuccessful (internal use only). ''' encryption_names = self._get_recipient_encryption_software() if encryption_names is None or len( encryption_names) < 1 or self.crypto_message is None: decrypted = False decrypted_with = [] self.log_message('unable to decrypt message when missing data') else: try: self.log_message( 'trying to decrypt with: {}'.format(encryption_names)) decrypted, decrypted_with = self._decrypt_with_all_encryption( encryption_names) if decrypted: self.crypto_message.set_crypted_with(decrypted_with) # if the message is still encrypted, log it and tell the user elif self.crypto_message.get_email_message().is_probably_pgp(): if len(encryption_names) > 1: software = encryption_names.__str__() else: software = str(encryption_names[0]) log_msg = "Failed to decrypt with {}".format(software) self.log_message(log_msg) self.log_message( 'logged failed message headers in goodcrypto.message.utils.log' ) utils.log_message_headers(self.crypto_message, 'failed message headers') record_exception(message=log_msg) tag = i18n( 'Unable to decrypt message with {encryption}'.format( encryption=software)) self.crypto_message.add_error_tag_once(tag) # don't filter bundled messages; each message will be filtered separately if filter_msg and options.filter_html(): self._filter_html() else: self.log_message("html filter disabled") except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: decrypted = False record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') return decrypted, decrypted_with def needs_metadata_key(self): ''' Gets whether the metadata key should be sent. ''' return self.need_to_send_metadata_key def _get_recipient_encryption_software(self): ''' Get the software to decrypt a message for the recipient (internal use only). If the user doesn't have a key, then configure one and return None. ''' encryption_software = None if self.crypto_message is None: self.log_message("missing crypto_message".format( self.crypto_message)) else: try: from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() encryption_software = contacts.get_encryption_names(to_user) if len(encryption_software) > 0: self.log_message( "encryption software: {}".format(encryption_software)) elif email_in_domain( to_user) and options.create_private_keys(): add_private_key(to_user, encryption_software=encryption_software) self.log_message( "started to create a new {} key for {}".format( encryption_software, to_user)) else: self.log_message( "no encryption software for {}".format(to_user)) self.crypto_message.add_error_tag_once( i18n( 'Message appears encrypted, but {email} does not use any known encryption' .format(email=to_user))) report_unable_to_decrypt( to_user, self.crypto_message.get_email_message().to_string()) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception as IOError: utils.log_crypto_exception(MessageException(format_exc())) record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') try: self.crypto_message.add_error_tag_once( SERIOUS_ERROR_PREFIX) except Exception: record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') return encryption_software def _decrypt_with_all_encryption(self, encryption_names): ''' Decrypt a message using all known encryption (internal use only). ''' decrypted = False decrypted_with = [] try: # the sender probably used the order of services in the # AcceptedEncryptionSoftware header we sent out, so we want to # use them in reverse order # move to the end of the list, and back up i = len(encryption_names) self.log_message("encrypted {} time(s)".format(i)) while i > 0: i -= 1 encryption_name = encryption_names[i] if self.crypto_message.get_email_message().is_probably_pgp(): try: if self._decrypt_message_with_crypto(encryption_name): # if any encryption decrypts, the message was decrypted decrypted = True decrypted_with.append(encryption_name) self.crypto_message.set_crypted(decrypted) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: msg = 'could not decrypt with {}.'.format( encryption_name) self.log_message(msg) self.log_message( 'EXCEPTION - see syr.exception.log for details') record_exception() else: self.log_message( "message already decrypted, so did not try {}".format( encryption_name)) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return decrypted, decrypted_with def _decrypt_message_with_crypto(self, encryption_name): ''' Decrypt a message using the encryption software (internal use only). ''' decrypted = False self.log_message("encryption program: {}".format(encryption_name)) from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() passcode = user_keys.get_passcode(to_user, encryption_name) if passcode == None or len(passcode) <= 0: tag = '{email} does not have a private key configured.'.format( email=to_user) self.log_message(tag) self.crypto_message.add_error_tag_once(tag) else: # make sure that the key for the recipient is ok; if it's not, a CryptoException is thrown __, verified, __ = contacts.is_key_ok(to_user, encryption_name) self.log_message('{} {} key pinned'.format(to_user, encryption_name)) self.log_message('{} {} key verified: {}'.format( to_user, encryption_name, verified)) crypto = CryptoFactory.get_crypto( encryption_name, crypto_software.get_classname(encryption_name)) # try to verify signature in case it was clear signed after it was encrypted decrypt_utils.verify_clear_signed( from_user, self.crypto_message, encryption_name=crypto.get_name(), crypto=crypto) self.log_message( 'trying to decrypt using {} private {} key.'.format( to_user, encryption_name)) if is_open_pgp_mime( self.crypto_message.get_email_message().get_message()): decrypted = self._decrypt_open_pgp_mime( from_user, crypto, passcode) else: decrypted = self._decrypt_inline_pgp(from_user, crypto, passcode) self.log_message('decrypted using {} private {} key: {}'.format( to_user, encryption_name, decrypted)) # try to verify signature in case it was clear signed before it was encrypted if decrypted: decrypt_utils.verify_clear_signed( from_user, self.crypto_message, encryption_name=crypto.get_name(), crypto=crypto) if self.DEBUGGING: self.log_message('decrypted message:\n{}'.format( self.crypto_message.get_email_message().get_message())) self.log_message('decrypted message char set: {}'.format( get_charset(self.crypto_message.get_email_message()))) return decrypted def _decrypt_open_pgp_mime(self, from_user, crypto, passcode): ''' Decrypt an open PGP MIME message (internal use only). ''' decrypted = False plaintext = None encrypted_part = None try: self.log_message("message is in OpenPGP MIME format") if self.DEBUGGING: self.log_message( 'logged OpenPGP mime headers in goodcrypto.message.utils.log' ) utils.log_message_headers(self.crypto_message, tag='OpenPGP mime headers') # remove any clear signed section before decrypting message self.crypto_message.get_email_message( ).remove_pgp_signature_blocks() payloads = self.crypto_message.get_email_message().get_message( ).get_payload() self.log_message("{} parts in message".format(len(payloads))) encrypted_part = payloads[self.ENCRYPTED_BODY_PART_INDEX] if isinstance(encrypted_part, Message): encrypted_part = encrypted_part.get_payload() if Decrypt.DEBUGGING: self.log_message("encrypted_part\n{}".format(encrypted_part)) charset, __ = get_charset(encrypted_part) self.log_message('encrypted part char set: {}'.format(charset)) plaintext = self._decrypt(from_user, encrypted_part, charset, crypto, passcode) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if plaintext == None or encrypted_part is None or plaintext == encrypted_part: decrypted = False self.log_message("unable to decrypt message") else: filtered = self._extract_embedded_message(plaintext) self.crypto_message.set_filtered(filtered) decrypted = self.crypto_message.is_crypted() return decrypted def _decrypt_inline_pgp(self, from_user, crypto, passcode): ''' Decrypt an inline PGP message (internal use only). ''' def adjust_attachment_name(part): '''Adjust the filename for the attachment.''' try: filename = part.get_filename() if filename and filename.endswith('.pgp'): self.log_message( 'original attachment filename: {}'.format(filename)) end = len(filename) - len('.pgp') part.replace_header( mime_constants.CONTENT_DISPOSITION_KEYWORD, 'attachment; filename="{}"'.format(filename[:end])) filename = part.__getitem__( mime_constants.CONTENT_DISPOSITION_KEYWORD) self.log_message( 'new attachment filename: {}'.format(filename)) else: self.log_message( 'attachment filename: {}'.format(filename)) except: record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') decrypted = False self.log_message("message is inline PGP format") message = self.crypto_message.get_email_message().get_message() self.log_message("message content type is {}".format( message.get_content_type())) # remove any clear signed section before decrypting message self.crypto_message.get_email_message().remove_pgp_signature_blocks() if message.is_multipart(): for part in message.get_payload(): content_type = part.get_content_type() ciphertext = part.get_payload(decode=True) charset, __ = get_charset(part) self.log_message( 'multipart message char set: {}'.format(charset)) plaintext = self._decrypt(from_user, ciphertext, charset, crypto, passcode) if plaintext is not None and plaintext != ciphertext: decrypted = True charset, __ = part.get_charset() self.log_message( "message part charset is {}".format(charset)) part.set_payload(plaintext, charset=charset) try: content_keyword_in_part = part.__getitem__( mime_constants.CONTENT_DISPOSITION_KEYWORD) except Exception: content_keyword_in_part = None if content_keyword_in_part is not None: adjust_attachment_name(part) try: encoding = part.__getitem__( mime_constants.CONTENT_XFER_ENCODING_KEYWORD) except Exception: encoding = None if encoding == mime_constants.QUOTED_PRINTABLE_ENCODING: encode_quopri(part) self.log_message( "{} encoded message part".format(encoding)) elif encoding == mime_constants.BASE64_ENCODING: encode_base64(part) self.log_message( "{} encoded message part".format(encoding)) else: charset, __ = get_charset(self.crypto_message.get_email_message()) self.log_message('inline pgp char set: {}'.format(charset)) ciphertext = self.crypto_message.get_email_message().get_content() plaintext = self._decrypt(from_user, ciphertext, charset, crypto, passcode) if plaintext is None or ciphertext is None or plaintext == ciphertext: decrypted = False self.log_message("unable to decrypt {} message".format( message.get_content_type())) else: # do not specify the codec self.crypto_message.get_email_message().get_message( ).set_payload(plaintext) try: encoding = self.crypto_message.get_email_message( ).get_message().__getitem__( mime_constants.CONTENT_XFER_ENCODING_KEYWORD) except Exception: encoding = None if encoding == mime_constants.QUOTED_PRINTABLE_ENCODING: encode_quopri( self.crypto_message.get_email_message().get_message()) elif encoding == mime_constants.BASE64_ENCODING: encode_base64( self.crypto_message.get_email_message().get_message()) decrypted = True return decrypted def _decrypt(self, from_user, data, charset, crypto, passcode): ''' Decrypt the data from a message (internal use only). ''' decrypted_data = signed_by = None if crypto is None or data is None: decrypted_data = None self.log_message("no crypto defined") else: if is_string(data): encrypted_data = data else: encrypted_data = data.encode(errors='replace') # ASCII armored plaintext looks just like armored ciphertext, # so check that we actually have encrypted data if (OpenPGPAnalyzer().is_encrypted(encrypted_data, passphrase=passcode, crypto=crypto)): if self.DEBUGGING: self.log_message( 'encrypted data before decryption:\n{}'.format( encrypted_data)) decrypted_data, signed_by, result_code = crypto.decrypt( encrypted_data, passcode) if (decrypted_data == None or (is_string(decrypted_data) and len(decrypted_data) <= 0)): if self.DEBUGGING: self.log_message( 'decrypted data:\n{}'.format(decrypted_data)) decrypted_data = None self.log_message("unable to decrypt data") if self.DEBUGGING: self.log_message( 'data bytearray after decryption:\n{}'.format( decrypted_data)) else: if result_code == 0: tag = tags.get_decrypt_signature_tag( self.crypto_message, from_user, signed_by, crypto.get_name()) if tag is not None: self.crypto_message.add_prefix_to_tag_once(tag) self.log_message('decrypt sig tag: {}'.format(tag)) elif result_code == 2: self.crypto_message.add_error_tag_once( i18n( "Can't verify signature because the message was encrypted using an unknown key. Ask the sender to send their key if they aren't using GoodCrypto." )) if is_string(decrypted_data): self.log_message('plaintext length: {}'.format( len(decrypted_data))) if self.DEBUGGING: self.log_message( 'plaintext:\n{}'.format(decrypted_data)) else: decrypted_data = None self.log_message("data appeared encrypted, but wasn't") if self.DEBUGGING: self.log_message('data:\n{}'.format(data)) return decrypted_data def _extract_embedded_message(self, plaintext): ''' Extract an embedded message. If the message includes an Open PGP header, then save the plaintext in the email message. Otherwise, create a new email message from the embedded message. ''' extracted_embedded_message = False try: if self.DEBUGGING: self.log_message('embbedded message:\n{}'.format(plaintext)) encrypted_type = get_first_header( self.crypto_message.get_email_message().get_message(), PGP_ENCRYPTED_CONTENT_TYPE) if encrypted_type is None: old_message = self.crypto_message.get_email_message( ).get_message() new_message = plaintext_to_message(old_message, plaintext) self.crypto_message.get_email_message().set_message( new_message) self.crypto_message.set_crypted(True) self.log_message("created a new message from the plaintext") if self.DEBUGGING: self.log_message( 'logged final embedded message headers in goodcrypto.message.utils.log' ) utils.log_message_headers( self.crypto_message, tag='final embedded message headers') else: # this assumes an embedded mime message self.log_message( "openpgp mime type: {}".format(encrypted_type)) embedded_message = EmailMessage(plaintext) self.crypto_message.set_email_message(embedded_message) self.crypto_message.set_crypted(True) extracted_embedded_message = True self.log_message("embedded message type is {}".format( embedded_message.get_message().get_content_type())) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return extracted_embedded_message def _filter_html(self): ''' Filter HTML to remove malious code (internal use only). ''' try: message = self.crypto_message.get_email_message().get_message() for part in message.walk(): part_content_type = part.get_content_type() # filter html and plain text if (part_content_type == mime_constants.TEXT_HTML_TYPE or part_content_type == mime_constants.TEXT_PLAIN_TYPE): original_payload = part.get_payload() safe_payload = firewall_html(original_payload) if original_payload != safe_payload: try: # strip extraneous </html> HTML_CLOSE = '</html>' if (part_content_type == mime_constants.TEXT_PLAIN_TYPE and safe_payload.lower().find('<html>') < 0): index = safe_payload.find(HTML_CLOSE) if index >= 0: safe_payload = '{} {}'.format( safe_payload[0:index], safe_payload[index + len(HTML_CLOSE):]) except: record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details' ) pass charset, __ = get_charset(part) self.log_message('html char set: {}'.format(charset)) part.set_payload(safe_payload, charset=charset) self.log_message( "html filtered {} part".format(part_content_type)) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def log_message(self, message): ''' Log the message to the local log. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class Debundle(object): ''' Filter to debundle one or more messages, including messages that have been bundled together to stop traffic analysis. See the unit tests to see how to use the Debundle class. ''' DEBUGGING = False def __init__(self, crypto_message): ''' >>> decrypt = Debundle(None) >>> decrypt != None True ''' self.log = LogFile() self.crypto_message = crypto_message self.messages_sent = 0 def process_message(self): ''' If the message is an encrypted message to a single individual, then decrypt the message and send it to the recipient. If the message is an encrypted message that protects metadata and contains one or more messages inside, then decrypt the metadata, decrypt the inner message(s) and deliver each message to the intended recipient. If the message is not encrypted, then just add a warning about receiving unencrypted messages. See unittests for usage as the test set up is too complex for a doctest. ''' try: filtered = decrypted = dkim_sig_verified = False from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() # see if there's a metadata wrapper if is_metadata_address(from_user) and is_metadata_address(to_user): dkim_sig_verified = self.decrypt_metadata() # if only one of to/from is a metadata address, the message is bad elif is_metadata_address(from_user) or is_metadata_address(to_user): self.log_message("bad envelope: from user: {} to_user: {}".format(from_user, to_user)) self.crypto_message.drop() if not self.crypto_message.is_dropped(): # now see if there's a bundle of padded messages if (is_metadata_address(self.crypto_message.smtp_sender()) and is_metadata_address(self.crypto_message.smtp_recipient())): self.unbundle_wrapped_messages() else: if not dkim_sig_verified and options.verify_dkim_sig(): self.crypto_message, dkim_sig_verified = decrypt_utils.verify_dkim_sig( self.crypto_message) self.log_message('dkim signature must be verified: {}'.format(dkim_sig_verified)) self.decrypt_message() except CryptoException as crypto_exception: raise MessageException(value=crypto_exception.value) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return self.crypto_message def decrypt_metadata(self): ''' Decrypt the wrapper message that protects metadata. ''' dkim_sig_verified = False from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() self.log_message("decrypting metadata protected message from {} to {}".format(from_user, to_user)) if self.DEBUGGING: self.log_message('DEBUG: logged original metadata headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, tag='original metadata headers') if options.verify_dkim_sig(): # verify dkim sig before any changes to message happen self.crypto_message, dkim_sig_verified = decrypt_utils.verify_dkim_sig(self.crypto_message) if options.dkim_delivery_policy() == DKIM_DROP_POLICY: self.log_message('verified metadata dkim signature ok: {}'.format(dkim_sig_verified)) elif dkim_sig_verified: self.log_message('verified metadata dkim signature') else: self.log_message('unable to verify metadata dkim signature, but dkim policy is to just warn') # the metadata wrapper always includes its own pub key in the header # and it must be imported if we don't already have it auto_exchange_keys = options.auto_exchange_keys() options.set_auto_exchange_keys(True) header_keys = HeaderKeys() # import the key if it's new, and # verify the key matches the sender's email address and our database header_keys.manage_keys_in_header(self.crypto_message) new_metadata_key = header_keys.new_key_imported_from_header() options.set_auto_exchange_keys(auto_exchange_keys) # before we change anything, send our metadata key to the recipient # this could result in a sending the key twice, but until we keep a # long term record of who we've sent the key, this is better than not sending it if new_metadata_key: metadata_key_sent = send_metadata_key(from_user, to_user) self.log_message('sent metadata key: {}'.format(metadata_key_sent)) decrypt = Decrypt(self.crypto_message) decrypted, decrypted_with = decrypt.decrypt_message(filter_msg=False) inner_crypto_message = decrypt.get_crypto_message() if inner_crypto_message.is_dropped(): self.log_message('public metadata wrapper dropped') wrapped_crypto_message = inner_crypto_message else: if decrypted: wrapped_crypto_message = self.get_bundled_message(inner_crypto_message) wrapped_crypto_message.set_metadata_crypted(True) wrapped_crypto_message.set_metadata_crypted_with(decrypted_with) self.log_message('created decrypted wrapped message') if self.DEBUGGING: self.log_message('DEBUG: logged decrypted wrapped headers in goodcrypto.message.utils.log') utils.log_message_headers(wrapped_crypto_message, tag='decrypted wrapped headers') else: # if it's not encypted, then drop the message inner_crypto_message.drop(True) wrapped_crypto_message = inner_crypto_message self.log_message('public metadata wrapper message not encrypted so dropping message') # use the new inner crypto message self.crypto_message = wrapped_crypto_message self.log_message("metadata decrypted with: {}".format( self.crypto_message.get_metadata_crypted_with())) return dkim_sig_verified def unbundle_wrapped_messages(self): ''' Unbundle messages and send them to their intended recipients. ''' result_ok = False self.log_message('unbundling wrapped messages') if self.crypto_message is None: self.log_message('no crypto message to unbundle') else: message = self.crypto_message.get_email_message().get_message() if self.DEBUGGING: self.log_message('DEBUG: logged wrapped headers in goodcrypto.message.utils.log') utils.log_message_headers(message, tag='wrapped headers') if message.get_content_type() == mime_constants.MULTIPART_MIXED_TYPE: result_ok = self.split_and_send_messages(message) # no need to re-inject this message as we've already sent the inner messages to users self.crypto_message.set_processed(True) result_ok = True else: result_ok = False self.crypto_message.drop() self.log_message('dropping message because there are no valid bundled messages; content type: {}'.format( message.get_content_type())) return result_ok def decrypt_message(self): ''' Decrypt a message for the original recipient. ''' try: self.log_message('decrypting message') if self.DEBUGGING and self.crypto_message is not None: self.log_message( 'original message:\n{}'.format(self.crypto_message.get_email_message().to_string())) decrypt = Decrypt(self.crypto_message) self.crypto_message = decrypt.process_message() self.log_message('decrypted message: {}'.format(self.crypto_message.is_crypted())) # send our metadata key to the recipient if needed if decrypt.needs_metadata_key(): self.log_message('need to send metadata key') metadata_key_sent = send_metadata_key( self.crypto_message.smtp_sender(), self.crypto_message.smtp_recipient()) self.log_message('sent metadata key: {}'.format(metadata_key_sent)) except CryptoException as crypto_exception: raise MessageException(value=crypto_exception.value) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') try: self.crypto_message.add_error_tag_once(SERIOUS_ERROR_PREFIX) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') raise MessageException() def get_bundled_message(self, crypto_message): ''' Get the message which contains one or more bundled messages. ''' try: self.log_message('getting message which contains one or more messages') if self.DEBUGGING: self.log_message('DEBUG: logged bundled crypto headers in goodcrypto.message.utils.log') utils.log_message_headers(crypto_message, 'bundled crypto headers') if self.DEBUGGING: self.log_message('crypto message before getting inner message\n{}'.format(crypto_message.get_email_message().to_string())) inner_message = crypto_message.get_email_message().get_content() if self.DEBUGGING: self.log_message('raw inner message\n{}'.format(inner_message)) inner_crypto_message = CryptoMessage(email_message=EmailMessage(inner_message)) if options.verify_dkim_sig(): # verify dkim sig before any changes to message happen inner_crypto_message, dkim_sig_verified = decrypt_utils.verify_dkim_sig(inner_crypto_message) if options.dkim_delivery_policy() == DKIM_DROP_POLICY: self.log_message('verified bundled dkim signature ok: {}'.format(dkim_sig_verified)) elif dkim_sig_verified: self.log_message('verified bundled dkim signature') else: self.log_message('unable to verify bundled dkim signature, but dkim policy is to just warn') if self.DEBUGGING: self.log_message('DEBUG: logged bundled inner headers in goodcrypto.message.utils.log') utils.log_message_headers(crypto_message, 'bundled inner headers') original_sender = inner_crypto_message.get_email_message().get_header(ORIGINAL_FROM) original_recipient = inner_crypto_message.get_email_message().get_header(ORIGINAL_TO) original_subject = inner_crypto_message.get_email_message().get_header(mime_constants.SUBJECT_KEYWORD) # if this message is an internal message with a subject, then send it to the admin if (original_sender == inner_crypto_message.smtp_sender() and original_recipient == inner_crypto_message.smtp_recipient() and original_subject is not None): admin = get_admin_email() inner_crypto_message.set_smtp_recipient(admin) else: inner_crypto_message.set_smtp_sender(original_sender) inner_crypto_message.set_smtp_recipient(original_recipient) # remove the original keywords from the message inner_crypto_message.get_email_message().delete_header(ORIGINAL_FROM) inner_crypto_message.get_email_message().delete_header(ORIGINAL_TO) if self.DEBUGGING: self.log_message('DEBUG: logged inner crypto headers in goodcrypto.message.utils.log') utils.log_message_headers(inner_crypto_message, 'inner crypto headers') except Exception: record_exception() inner_crypto_message = None self.log_message('EXCEPTION - see syr.exception.log for details') return inner_crypto_message def split_and_send_messages(self, message): ''' Split a message apart and send each to the intended recipient. ''' result_ok = False self.messages_sent = 0 try: for part in message.walk(): try: if part.get_content_type() == mime_constants.APPLICATION_ALT_TYPE: content = b64decode(part.get_payload(decode=True)) inner_crypto_message = self.create_inner_message(content) if inner_crypto_message is not None: self.log_message('logged inner message headers in goodcrypto.message.utils.log') utils.log_message_headers(inner_crypto_message, tag='inner message headers') ok, __ = self.decrypt_and_send_message(inner_crypto_message) if ok: self.messages_sent += 1 except CryptoException as crypto_exception: raise MessageException(value=crypto_exception.value) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: self.log_message('bad part of message discarded') self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() result_ok = True self.log_message('good bundled message contains {} inner message(s)'.format(self.messages_sent)) except CryptoException as crypto_exception: raise MessageException(value=crypto_exception.value) except MessageException as message_exception: raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') result_ok = False return result_ok def create_inner_message(self, content): ''' Create the inner crypto message. ''' inner_crypto_message = None try: original_message, addendum = parse_bundled_message(content) if original_message is None or len(original_message.strip()) <= 0 or addendum is None: self.log_message('discarded padding') else: sender = addendum[mime_constants.FROM_KEYWORD] recipient = addendum[mime_constants.TO_KEYWORD] if sender is None or recipient is None: self.log_message('discarded badly formatted message') else: if self.DEBUGGING: self.log_message('DEBUG: content of part: {}'.format(content)) metadata_crypted_with = self.crypto_message.get_metadata_crypted_with() inner_crypto_message = CryptoMessage(EmailMessage(original_message)) inner_crypto_message.set_smtp_sender(sender) inner_crypto_message.set_smtp_recipient(recipient) inner_crypto_message.set_metadata_crypted(True) inner_crypto_message.set_metadata_crypted_with(metadata_crypted_with) self.log_message('created message from {}'.format(sender)) self.log_message('created message to {}'.format(recipient)) self.log_message("metadata decrypted with: {}".format( inner_crypto_message.get_metadata_crypted_with())) if self.DEBUGGING: self.log_message('original message: {}'.format(original_message)) except: inner_crypto_message = None record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return inner_crypto_message def decrypt_and_send_message(self, inner_crypto_message): ''' Decrypt and send a message. ''' result_ok = False sender = recipient = message = decrypted_crypto_message = None try: decrypt = Decrypt(inner_crypto_message) decrypted_crypto_message = decrypt.process_message() sender = decrypted_crypto_message.smtp_sender() recipient = decrypted_crypto_message.smtp_recipient() message = decrypted_crypto_message.get_email_message().get_message().as_string() self.log_message('message to {} decrypted: {}'.format( recipient, decrypted_crypto_message.is_crypted())) if send_message(sender, recipient, message): result_ok = True self.log_message('sent message to {}'.format(recipient)) if self.DEBUGGING: self.log_message('DEBUG: message:\n{}'.format(message)) else: result_ok = False except AttributeError as attribute_error: result_ok = False self.log_message(attribute_error) except: result_ok = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if not result_ok: report_message_undeliverable(message, sender) self.log_message('reported message undeliverable') # we're returning the decrypted message to allow tests to verify everything went smoothly return result_ok, decrypted_crypto_message def log_message(self, message): ''' Log the message to the local log. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class Bundle(object): ''' Bundle and pad messages to each domain we have their metadata address. Unlike much of GoodCrypto, the results of many functions in this class are python style email.Messages, not EmailMessages or CryptoMessages. Comments try to use camel back to make the disticion clear (i.e., Message referrs to an email.Message(). ''' def __init__(self): ''' >>> bundle = Bundle() >>> bundle is not None True ''' self.DEBUGGING = False self.log = None self.bundled_messages = [] self.crypted_with = [] def ready_to_run(self): ''' Check to see if it's time to run the bundle and again. ''' if options.encrypt_metadata() and options.bundle_and_pad(): if options.bundle_frequency() == options.bundle_hourly(): if WARNING_WARNING_WARNING_TESTING_ONLY_DO_NOT_SHIP: frequency = timedelta(minutes=10) else: frequency = timedelta(hours=1) elif options.bundle_frequency() == options.bundle_daily(): frequency = timedelta(days=1) elif options.bundle_frequency() == options.bundle_weekly(): frequency = timedelta(weeks=1) else: frequency = timedelta(hour=1) next_run = get_date_queue_last_active() + frequency ready = next_run <= datetime.utcnow() if not ready: self.log_message( 'bundle and pad messages next: {}'.format(next_run)) else: self.log_message('not padding and packetizing messages') ready = False return ready def bundle_and_pad_messages(self): ''' Bundle and pad messages. ''' try: self.log_message('starting to bundle and pad messages') self.bundle_and_pad() set_date_queue_last_active(datetime.utcnow()) ok = True self.log_message('finished bundling and padding messages') except Exception as exception: ok = False report_unable_to_send_bundled_messages(exception) self.log_message('EXCEPTION - see syr.exception.log for details') return ok def bundle_and_pad(self): ''' Bundle and pad messages to reduce tracking. ''' packet_dir = get_packet_directory() dirnames = os.listdir(packet_dir) if dirnames is None or len(dirnames) <= 0: self.log_message('no pending packets') else: self.log_message('starting to bundle and pad packets') for dirname in dirnames: # reset variables used per domain self.bundled_messages = [] self.crypted_with = [] path = os.path.join(packet_dir, dirname) if os.path.isdir(path) and dirname.startswith('.'): to_domain = dirname[1:] message = self.create_message(path, to_domain) if message is None: self.log_message( 'no message to send to {}'.format(to_domain)) elif self.send_bundled_message(message, to_domain): self.add_history_and_remove(to_domain) self.log_message('finished bundling and paddding packets') def send_bundled_message(self, message, to_domain): ''' Send a Message to the domain. ''' try: if message is None: result_ok = False self.log_message('nothing to send to {}'.format(to_domain)) else: sender = get_email(get_metadata_address(domain=get_domain())) recipient = get_email(get_metadata_address(domain=to_domain)) self.log_message( 'starting to send message from {} to {}'.format( sender, recipient)) result_ok = send_message(sender, recipient, message.as_string()) self.log_message('finished sending message') except Exception as exception: result_ok = False self.log_message('error while sending message') self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() return result_ok def create_message(self, dirname, to_domain): ''' Create a Message to send that contains any other messages that are ready. ''' inner_message = self.create_inner_message(dirname, to_domain) if inner_message is None: encrypted_message = None self.log_message('no inner message for {}'.format(to_domain)) else: encrypted_message = self.create_encrypted_message( inner_message, to_domain) return encrypted_message def create_encrypted_message(self, inner_message, to_domain): ''' Create an encrypted Message. ''' message = None if to_domain is None: self.log_message('domain is not defined') elif inner_message is None: self.log_message('no inner message defined') else: from_user = get_email(get_metadata_address(domain=get_domain())) to_user = get_email(get_metadata_address(domain=to_domain)) crypto_message = create_protected_message( from_user, to_user, inner_message.as_string(), utils.get_message_id()) if crypto_message.is_crypted(): # add the DKIM signature to the inner message if user opted for it crypto_message = add_dkim_sig_optionally(crypto_message) message = crypto_message.get_email_message().get_message() self.crypted_with = crypto_message.get_metadata_crypted_with() self.log_message('crypted with: {}'.format(self.crypted_with)) for part in message.walk(): self.log_message('Content type of part: {}'.format( part.get_content_type())) if self.DEBUGGING: self.log_message(part.get_payload()) else: report_bad_bundled_encrypted_message(to_domain, self.bundled_messages) return message def create_inner_message(self, dirname, to_domain): ''' Create a Message that contains other messages plus padding. ''' message = None if dirname is None: self.log_message('no dir name defined in create_inner_message') elif to_domain is None: self.log_message('domain is not defined in create_inner_message') elif not os.path.exists(dirname): self.log_message('{} does not exist'.format(dirname)) else: parts = [] filenames = os.listdir(dirname) if filenames is None or len(filenames) <= 0: self.log_message( 'no pending messages for {}'.format(to_domain)) else: estimated_size = 0 # it's imporant to process in sorted order so larger messages don't get stuck # because smaller ones filled up the queue before the larger ones could be processed for filename in sorted(filenames): part, filesize = self.get_mime_part( dirname, filename, estimated_size) if part is not None: parts.append(part) estimated_size += filesize self.bundled_messages.append( os.path.join(dirname, filename)) self.log_message('bundling {} messages for {}'.format( len(parts), to_domain)) parts = self.pad_message(parts, to_domain) if len(parts) > 0: boundary = 'Part{}{}--'.format(random(), random()) params = { mime_constants.PROTOCOL_KEYWORD: mime_constants.MULTIPART_MIXED_TYPE, mime_constants.CHARSET_KEYWORD: constants.DEFAULT_CHAR_SET, } message = self.prep_message_header( MIMEMultipart(mime_constants.MIXED_SUB_TYPE, boundary, parts, **params), to_domain) if self.DEBUGGING: self.log_message('message: {}'.format(str(message))) for part in message.walk(): self.log_message('Content type: {}'.format( part.get_content_type())) if self.DEBUGGING: self.log_message(part.get_payload()) else: self.log_message( 'Unable to get any parts so no inner message created') return message def pad_message(self, parts, to_domain): ''' Pad the Message with random characters. ''' # determine how large the bundled messages are original_size = 0 for part in parts: original_size += len(part.as_string()) # then pad the message with random characters current_size = original_size target_size = options.bundled_message_max_size() while current_size < target_size: # using urandom because it's less likely to lock up # a messaging program can't afford the potential locks up of /dev/random with open('/dev/urandom', 'rb') as rnd: rnd_bytes = rnd.read(target_size - current_size) if len(rnd_bytes) > target_size - current_size: rnd_bytes = rnd_bytes[:target_size - current_size] part = MIMEApplication(b64encode(rnd_bytes), mime_constants.ALTERNATIVE_SUB_TYPE, encode_base64) current_size += len(part.as_string()) parts.append(part) padded_bytes = target_size - original_size if padded_bytes < 0: self.log_message('original size: {}'.format(original_size)) self.log_message('target size: {}'.format(target_size)) self.log_message('padded with {} random bytes'.format(padded_bytes)) return parts def get_mime_part(self, dirname, filename, estimated_size): ''' Get a MIME part with the Message. If any errors occur, then bounce the message to the original sender. ''' part = None fullname = os.path.join(dirname, filename) filesize = os.stat(fullname).st_size max_size = options.bundled_message_max_size() # only look at the message files that won't make the overall size too large if (filename.startswith(constants.MESSAGE_PREFIX) and filename.endswith(constants.MESSAGE_SUFFIX)): # if the message is too large, then bounce it back to the user if filesize > max_size: self.bounce_message( fullname, i18n( 'Message too large to send. It must be {size} KB or smaller' .format(size=options.bundle_message_kb()))) self.log_message( '{} message too large; max size: {} KB'.format( filesize, options.bundle_message_kb())) elif (filesize + estimated_size) < max_size: try: with open(fullname, 'rb') as input_file: content = input_file.read() if content.endswith(constants.END_ADDENDUM.encode()): part = MIMEApplication( b64encode(content), mime_constants.ALTERNATIVE_SUB_TYPE, encode_base64) if self.DEBUGGING: self.log_message( 'message part:\n{}'.format(part)) else: self.log_message( '{} is not ready to send'.format(filename)) except: record_exception() self.log_message( 'EXCEPTION - see syr.exception.log for details') else: self.log_message( '{} is too large for this batch of messages (estimated size with this message: {})' .format(filename, filesize + estimated_size)) return part, filesize def prep_message_header(self, message, to_domain): ''' Prepare the header of a Message. ''' if message is None: self.log_message('no message defined in prep_message_header') elif to_domain is None: self.log_message('domain is not defined in prep_message_header') else: message_date = datetime.utcnow().replace(tzinfo=utc) from_user = get_metadata_address(domain=get_domain()) to_user = get_metadata_address(domain=to_domain) message.__setitem__(mime_constants.FROM_KEYWORD, from_user) message.__setitem__(mime_constants.TO_KEYWORD, to_user) message.__setitem__(constants.ORIGINAL_FROM, from_user) message.__setitem__(constants.ORIGINAL_TO, to_user) message.__setitem__(mime_constants.DATE_KEYWORD, message_date.__str__()) message.__setitem__(mime_constants.MESSAGE_ID_KEYWORD, utils.get_message_id()) self.log_message("message's content type: {}".format( message.get_content_type())) self.log_message("message's boundary: {}".format( message.get_boundary())) if self.DEBUGGING: self.log_message("message's key/value pair") for key in message.keys(): self.log_message('{}: {}'.format(key, message.get(key))) return message def add_history_and_remove(self, to_domain): ''' Add history records for the messages sent and then remove the associated file. ''' def get_addendum_value(addendum, keyword): value = addendum[keyword] if is_string(value): value = value.strip() if value == 'True': value = True elif value == 'False': value = False return value if len(self.bundled_messages) > 0: for bundled_message in self.bundled_messages: self.log_message( 'message included in bundled: {}'.format(bundled_message)) with open(bundled_message) as f: original_message, addendum = parse_bundled_message( f.read()) self.log_message('addendum: {}'.format(addendum)) encrypted = get_addendum_value(addendum, constants.CRYPTED_KEYWORD) private_signed = get_addendum_value( addendum, constants.PRIVATE_SIGNED_KEYWORD) clear_signed = get_addendum_value( addendum, constants.CLEAR_SIGNED_KEYWORD) dkim_signed = get_addendum_value( addendum, constants.DKIM_SIGNED_KEYWORD) if encrypted or private_signed or clear_signed or dkim_signed: sender = get_addendum_value( addendum, mime_constants.FROM_KEYWORD) recipient = get_addendum_value( addendum, mime_constants.TO_KEYWORD) verification_code = get_addendum_value( addendum, constants.VERIFICATION_KEYWORD) crypto_message = CryptoMessage( email_message=EmailMessage(original_message)) crypto_message.set_smtp_sender(sender) crypto_message.set_smtp_recipient(recipient) crypto_message.set_crypted(encrypted) crypto_message.set_crypted_with( addendum[constants.CRYPTED_WITH_KEYWORD]) crypto_message.set_metadata_crypted(True) crypto_message.set_metadata_crypted_with( self.crypted_with) crypto_message.set_private_signed(private_signed) if private_signed: if encrypted: crypto_message.add_private_signer({ constants.SIGNER: sender, constants.SIGNER_VERIFIED: True }) crypto_message.add_private_signer({ constants.SIGNER: get_metadata_address(sender), constants.SIGNER_VERIFIED: True }) crypto_message.set_clear_signed(clear_signed) if clear_signed: if encrypted: crypto_message.add_clear_signer({ constants.SIGNER: sender, constants.SIGNER_VERIFIED: True }) else: crypto_message.add_clear_signer({ constants.SIGNER: get_metadata_address(sender), constants.SIGNER_VERIFIED: True }) crypto_message.set_dkim_signed(dkim_signed) if dkim_signed: crypto_message.set_dkim_sig_verified(True) history.add_outbound_record(crypto_message, verification_code) self.log_message( 'added outbound history record from {}'.format( sender)) if self.DEBUGGING: self.log_message( 'logged headers in goodcrypto.message.utils.log' ) utils.log_message_headers(crypto_message, tag='bundled headers') if os.path.exists(bundled_message): os.remove(bundled_message) else: self.log_message( 'tried to delete message after bundling it, but message no longer exists on disk' ) else: self.log_message('no bundled messages') def bounce_message(self, fullname, error_message): ''' Bounce a Message to the original user. ''' notified_user = False if fullname is not None and os.path.exists(fullname): with open(fullname) as f: content = f.read() original_message, addendum = parse_bundled_message(content) subject = i18n('{} - Unable to send message to {email}'.format( TAG_ERROR, email=addendum[mime_constants.TO_KEYWORD])) notified_user = utils.bounce_message( original_message, addendum[mime_constants.FROM_KEYWORD], subject, error_message) self.log_message(subject) return notified_user def log_message(self, message): ''' Log the message to the local log. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class SearchKeyserver(object): ''' Search for a key from keyserver. ''' def __init__(self, email, encryption_name, keyserver, user_initiated_search): ''' >>> # In honor of Werner Koch, developer of gpg. >>> email = '*****@*****.**' >>> crypto_name = 'GPG' >>> srk_class = SearchKeyserver(email, crypto_name, 'pgp.mit.edu', '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(None, crypto_name, 'pgp.mit.edu', '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(email, None, 'pgp.mit.edu', '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(email, crypto_name, None, '*****@*****.**') >>> srk_class != None True >>> srk_class = SearchKeyserver(None, None, None, None) >>> srk_class != None True ''' self.log = LogFile() self.email = email self.encryption_name = encryption_name self.keyserver = keyserver self.user_initiated_search = user_initiated_search self.key_plugin = None def start_search(self): ''' Queue searching the keyserver. When the job finishes, the key will be retrieved from another queued job which is dependent on the search's job. Test extreme case. >>> srk_class = SearchKeyserver(None, None, None, None) >>> srk_class.start_search() False ''' try: if self._is_ready_for_search(): result_ok = True # start the search, but don't wait for the results self.key_plugin.search_for_key(self.email, self.keyserver) search_job = self.key_plugin.get_job() queue = self.key_plugin.get_queue() # if the search job or queue are done, then retrieve the key if queue is None or search_job is None: get_key(self.email, self.encryption_name, self.keyserver, self.user_initiated_search, search_job, queue) else: from goodcrypto.mail.keyserver_utils import get_key ONE_MINUTE = 60 # one minute, in seconds DEFAULT_TIMEOUT = 10 * ONE_MINUTE # otherwise, set up another job in the queue to retrieve the # key as soon as the search for the key id is finished result_ok = get_key(self.email, self.encryption_name, self.keyserver, self.user_initiated_search, search_job.get_id(), queue.key) self.log_message('retrieving {} key for {}: {}'.format( self.encryption_name, self.email, result_ok)) else: result_ok = False except Exception as exception: result_ok = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') self.log_message('finished starting search on {} for {} ok: {}'.format( self.keyserver, self.email, result_ok)) return result_ok def _is_ready_for_search(self): ''' Verify that we're ready to search for this key. Test extreme case. >>> srk_class = SearchKeyserver(None, None, None, None) >>> srk_class._is_ready_for_search() False ''' ready = False try: ready = (self.email is not None and self.encryption_name is not None and self.keyserver is not None and self.user_initiated_search is not None and not email_in_domain(self.email)) if ready: self.key_plugin = KeyFactory.get_crypto( self.encryption_name, crypto_software.get_key_classname(self.encryption_name)) ready = self.key_plugin is not None if ready: # make sure we don't already have crypto defined for this user contacts_crypto = contacts.get_contacts_crypto( self.email, self.encryption_name) if contacts_crypto is None or contacts_crypto.fingerprint is None: fingerprint, expiration = self.key_plugin.get_fingerprint( self.email) if fingerprint is not None: ready = False self.log_message( '{} public key exists for {}: {}'.format( self.encryption_name, self.email, fingerprint)) else: ready = False self.log_message('crypto for {} already defined'.format( self.email)) except Exception as exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') ready = False return ready def log_message(self, message): ''' Log the message to the local log. >>> import os.path >>> from syr.log import BASE_LOG_DIR >>> from syr.user import whoami >>> SearchKeyserver(None, None, None, None).log_message('test') >>> os.path.exists(os.path.join(BASE_LOG_DIR, whoami(), 'goodcrypto.mail.search_keyserver.log')) True ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)
class GPGPlugin(AbstractPlugin): ''' Gnu Privacy Guard crypto plugin. Be careful with how you specify a user ID to GPG. Case insensitive substring matching is the default. For example if you specify the email address "*****@*****.**" as a user ID, you will match a user ID such as "*****@*****.**". You can specify an exact match on the entire user ID by prefixing your user ID spec with "=", e.g. "=John Heinrich <*****@*****.**>". Another option is to tell GPG you want an exact match on an email address using "<" and ">", e.g. "<*****@*****.**>". Debug note: If there seems to be an extra blank line at the top of decrypted text, check whether we should be using "passphrase + \r" instead of "passphrase + EOL". ''' DEBUGGING = False # use a queue to insure that GPG is only run one instance at a time USE_RQ = True # Match email addresses of user IDs. This is the default. # If a user ID does not include "@", acts like CASE_INSENSITVE_MATCH. # EMAIL_MATCH = 1 # Match user IDs exactly. EXACT_MATCH = 2 # # Match case insensitive substrings of user IDs. # This is GPG's default, but not the default for this class. # CASE_INSENSITVE_MATCH = 3 GPG_COMMAND_NAME = "gpg" GOOD_SIGNATURE_PREFIX = "gpg: Good signature from " BAD_SIGNATURE_PREFIX = "gpg: BAD signature from " ONE_MINUTE = 60 # one minute, in seconds DEFAULT_TIMEOUT = 10 * ONE_MINUTE GPG_HOME_DIR = os.path.join(OCE_DATA_DIR, '.gnupg') def __init__(self): ''' Creates a new GPGPlugin object. ''' super(GPGPlugin, self).__init__() self.log = LogFile() self.name = gpg_constants.ENCRYPTION_NAME self._executable_pathname = self.GPG_COMMAND_NAME self._user_id_match_method = self.EMAIL_MATCH # keep the info so other classes can add dependencies self.queue = None self.queue_connection = None self.job = None self.gpg_home = self.GPG_HOME_DIR def get_job_count(self): ''' Get the jobs in the queue. ''' return manage_queues.get_job_count(self.get_queue_name(), self.get_queue_port()) def get_job(self): ''' Get the job from the queue if this plugin uses one. ''' return self.job def get_queue_connection(self): ''' Get the connection to the queue if this plugin uses one. ''' if self.queue_connection is None: self.queue_connection = Redis(REDIS_HOST, self.get_queue_port()) return self.queue_connection def get_queue(self): ''' Get the queue if this plugin uses one. ''' return self.queue def get_queue_name(self): ''' Get the queue's name if this plugin uses one. >>> plugin = GPGPlugin() >>> plugin.get_queue_name() 'gpg' ''' return GPG_RQ def get_queue_port(self): ''' Get the queue's port if this plugin uses one. >>> plugin = GPGPlugin() >>> plugin.get_queue_port() 6381 ''' return GPG_REDIS_PORT def wait_until_queue_empty(self): ''' Wait until the queue is empty. ''' return manage_queues.wait_until_queue_empty(self.get_queue_name(), self.get_queue_port()) def clear_failed_queue(self): ''' Clear all the jobs in the failed queue. ''' return manage_queues.clear_failed_queue(self.get_queue_name(), self.get_queue_port()) def set_user_id_match_method(self, method): ''' Set user ID match method. ''' if method == self.EMAIL_MATCH or method == self.EXACT_MATCH: self._user_id_match_method = method def get_user_id_match_method(self): ''' Get user ID match method. The default is to match email addresses of user IDs. ''' return self._user_id_match_method def set_executable(self, pathname): ''' Set executable pathname. >>> plugin = GPGPlugin() >>> plugin.set_executable('/usr/bin/gpg') >>> plugin.get_executable() == '/usr/bin/gpg' True >>> plugin.set_executable(plugin.GPG_COMMAND_NAME) ''' self._executable_pathname = pathname def get_executable(self): ''' Get executable pathname. >>> plugin = GPGPlugin() >>> plugin.get_executable() != None True ''' if self._executable_pathname is None: self._executable_pathname = self.GPG_COMMAND_NAME return self._executable_pathname def get_default_executable(self): ''' Get default executable pathname. >>> plugin = GPGPlugin() >>> executable = plugin.get_executable() >>> executable == 'gpg' True ''' return self.GPG_COMMAND_NAME def get_crypto_version(self): ''' Get the version of the underlying crypto. >>> from shutil import rmtree >>> plugin = GPGPlugin() >>> original_home_dir = plugin.get_home_dir() >>> plugin.set_home_dir('/var/local/projects/goodcrypto/server/data/test_oce/.gnupg') True >>> plugin.get_crypto_version() is not None True >>> if os.path.exists('/var/local/projects/goodcrypto/server/data/test_oce/.gnupg'): ... rmtree('/var/local/projects/goodcrypto/server/data/test_oce/.gnupg') >>> if os.path.exists('/var/local/projects/goodcrypto/server/data/test_oce'): ... rmtree('/var/local/projects/goodcrypto/server/data/test_oce') >>> plugin.set_home_dir(original_home_dir) True ''' version_number = None try: args = [gpg_constants.GET_VERSION] result_code, gpg_output, gpg_error= self.gpg_command(args) if result_code == gpg_constants.GOOD_RESULT: version_number = self._parse_version(gpg_output) self.log_message("version number is {}".format(version_number)) except Exception as exception: self.log_message(exception) return version_number def get_name(self): ''' Get the crypto's short name. >>> plugin = GPGPlugin() >>> name = plugin.get_name() >>> name == gpg_constants.ENCRYPTION_NAME True ''' return self.name def get_plugin_name(self): ''' Get the plugin's name. >>> plugin = GPGPlugin() >>> plugin.get_plugin_name().startswith('goodcrypto.oce') True ''' return gpg_constants.NAME def get_plugin_version(self): ''' Get the version of this plugin's implementation, i.e. the CORBA servant's version. >>> plugin = GPGPlugin() >>> version = plugin.get_plugin_version() >>> version is not None True >>> version == '0.1' True ''' return "0.1" def get_user_ids(self): ''' Get list of user IDs with a public key. >>> from shutil import rmtree >>> plugin = GPGPlugin() >>> original_home_dir = plugin.get_home_dir() >>> plugin.set_home_dir('/var/local/projects/goodcrypto/server/data/test_oce/.gnupg') True >>> plugin.get_private_user_ids() is not None True >>> if os.path.exists('/var/local/projects/goodcrypto/server/data/test_oce/.gnupg'): ... rmtree('/var/local/projects/goodcrypto/server/data/test_oce/.gnupg') >>> if os.path.exists('/var/local/projects/goodcrypto/server/data/test_oce'): ... rmtree('/var/local/projects/goodcrypto/server/data/test_oce') >>> plugin.set_home_dir(original_home_dir) True ''' user_ids = None try: # we're using --with-colons because we hope that format is less likely to change args = [gpg_constants.LIST_PUBLIC_KEYS, gpg_constants.WITH_COLONS] result_code, gpg_output, gpg_error= self.gpg_command(args) if result_code == gpg_constants.GOOD_RESULT: self.log_message('gpg_output: {}'.format(gpg_output)) self.log_message('gpg_error: {}'.format(gpg_error)) user_ids = self.parse_user_ids(gpg_constants.PUB_PREFIX, gpg_output) self.log_message('{} public user ids'.format(len(user_ids))) except Exception as exception: self.handle_unexpected_exception(exception) return user_ids def get_private_user_ids(self): ''' Get list of user IDs with a private key. >>> plugin = GPGPlugin() >>> plugin.get_private_user_ids() is not None True ''' user_ids = None try: # we're using --with-colons because we hope that format is less likely to change args = [gpg_constants.LIST_SECRET_KEYS, gpg_constants.WITH_COLONS] result_code, gpg_output, gpg_error= self.gpg_command(args) if result_code == gpg_constants.GOOD_RESULT: self.log_message('gpg output: {}'.format(gpg_output)) user_ids = self.parse_user_ids(gpg_constants.SEC_PREFIX, gpg_output) self.log_message('{} private user ids'.format(len(user_ids))) self.log_message('private user ids: {}'.format(user_ids)) except Exception as exception: self.handle_unexpected_exception(exception) return user_ids def parse_user_ids(self, header, output_string): ''' Parse the user ids from the output string. Intended for internal use only. Test the extremes. >>> plugin = GPGPlugin() >>> user_ids = plugin.parse_user_ids(None, None) >>> user_ids is not None True >>> len(user_ids) <= 0 True >>> # In honor of Lieutenant Assaf, who co-signed letter and refused to serve >>> # in operations involving the occupied Palestinian territories because >>> # of the widespread surveillance of innocent residents. >>> plugin = GPGPlugin() >>> user_ids = plugin.parse_user_ids('pub', 'pub:u:4096:1:6BFCCC3E4ED73DC4:2013-09-23:::u:Lieutenant Assaf (gpg key) <*****@*****.**>::scESC:') >>> user_ids is not None True >>> len(user_ids) == 1 True >>> # In honor of First Sergeant Guy, who co-signed letter and refused to serve >>> # in operations involving the occupied Palestinian territories because >>> # of the widespread surveillance of innocent residents. >>> plugin = GPGPlugin() >>> user_ids = plugin.parse_user_ids('pub', 'pub:u:4096:1:6BFCCC3E4ED73DC4:2013-09-23:First Sergeant Guy (gpg key) <*****@*****.**>') >>> user_ids is not None True >>> len(user_ids) == 0 True ''' user_ids = [] try: self.log_message('output string type: {}'.format(type(output_string))) reader = StringIO(initial_value=output_string) for line in reader: raw_line = line.strip() if (raw_line is not None and len(raw_line) > 0 and (raw_line.lower().startswith(header) or raw_line.lower().startswith('uid'))): # In honor of Sergeant First Class Galia, who co-signed letter and refused to serve # in operations involving the occupied Palestinian territories because # of the widespread surveillance of innocent residents. # results look like this for public keys: #tru::1:1379973353:0:3:1:5 #pub:u:4096:1:6BFCCC3E4ED73DC4:2013-09-23:::u:Sergeant First Class Galia <*****@*****.**>::scESC: #uid:-::::2014-06-14::47764CA1D105D5C9D7F023D021203254D66E1C10::mark burdett <*****@*****.**>: #sub:u:4096:1:5215AA2CAF37F286:2013-09-23::::::e: # In honor of Lieutenant Gilad, who co-signed letter and refused to serve # in operations involving the occupied Palestinian territories because # of the widespread surveillance of innocent residents. # results look like this for secret keys: #sec::1024:17:82569302F49264B9:2013-10-14::::Lieutenant Gilad <*****@*****.**>::: #ssb::1024:16:0E298674A69943BF:2013-10-14::::::: elements = raw_line.split(':') if elements and len(elements) > 9: address = get_email(elements[9]) user_ids.append(address) except TypeError as type_error: self.handle_unexpected_exception(type_error) except Exception as exception: self.handle_unexpected_exception(exception) if self.DEBUGGING: self.log_message("{} user ids: {}".format(header, user_ids)) return user_ids def is_available(self): ''' Determine if the crypto app is installed. >>> plugin = GPGPlugin() >>> plugin.is_available() True ''' installed = False try: # if we can get the version, then the app's installed version = self.get_crypto_version() if version != None: if len(version.strip()) > 0: installed = True # make sure the home directory is defined and exists self.get_home_dir() else: self.log_message('unable to get version while trying to verify gpg installed.') except Exception: self.log_message("unable to get version so assume not installed") self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() self.log_message("GPG's back end app is installed: {}".format(installed)) return installed def get_signer(self, data): ''' Get signer of data. Test a few cases known to fail. See the unittests for more robust examples. >>> plugin = GPGPlugin() >>> signer = plugin.get_signer('Test unsigned data') >>> signer = plugin.get_signer(None) ''' signer = None try: args = [gpg_constants.VERIFY] if data is None or len(data.strip()) <= 0: self.log_message('data for signer was not defined') else: result_code, gpg_output, gpg_error= self.gpg_command(args, data=data) if result_code == gpg_constants.GOOD_RESULT: signer = self._parse_signer(gpg_error) else: # if this is a detached sig, then we need to pass it once as a file and once as stdin with NamedTemporaryFile(mode='wt', dir='/tmp', delete=False) as f: f.write(data) filename = f.name args = [gpg_constants.VERIFY, filename, '-'] result_code, gpg_output, gpg_error= self.gpg_command(args, data=data) if result_code == gpg_constants.GOOD_RESULT or result_code == 1: signer = self._parse_signer(gpg_error) os.remove(filename) except Exception as exception: self.handle_unexpected_exception(exception) return signer def decrypt(self, data, passphrase): ''' Decrypt data and return the signer, if there is one. Test a few cases known to fail. See the unittests for more robust examples. >>> plugin = GPGPlugin() >>> decrypted_data, signed_by, result_code = plugin.decrypt( ... 'This is a test', 'a secret passphrase') >>> decrypted_data is None True ''' GOOD_SIG = 'Good signature from' UNKNOWN_SIG = "gpg: Can't check signature: public key not found" DECRYPT_FAILED = 'decryption failed:' DECRYPT_MESSAGE_FAILED = 'decrypt_message failed' INVALID_PACKET = 'invalid packet' NO_VALID_DATA = 'no valid OpenPGP data found' decrypted_data = signed_by = None result_code = gpg_constants.ERROR_RESULT try: if self.DEBUGGING: self.log_message("decrypting:\n{}".format(data)) if passphrase is None or data is None: self.log_message('unable to decrypt because key info missing') else: args = [gpg_constants.DECRYPT_DATA, gpg_constants.OPEN_PGP] result_code, gpg_output, gpg_error= self.gpg_command( args, passphrase=passphrase, data=data) if result_code == gpg_constants.GOOD_RESULT: decrypted_data = gpg_output if gpg_error is not None and GOOD_SIG in gpg_error: signature = gpg_error[gpg_error.find(GOOD_SIG) + len(GOOD_SIG):] signature = signature.split('\n')[0].strip() signed_by = signature.strip('"') self.log_message("decrypted data, with good signature from {}".format(signed_by)) elif result_code == gpg_constants.CONDITIONAL_RESULT: # if an error reported, but it was just a unknown sig, accept the decryption if gpg_error is not None: if UNKNOWN_SIG in gpg_error: self.log_message("decrypted data, with unknown signature") decrypted_data = gpg_output else: self.log_message("gpg error: {}".format(gpg_error)) result_code = gpg_constants.ERROR_RESULT else: self.log_message("gpg_error: {}".format(gpg_error)) if self.DEBUGGING: self.log_message("decrypted data:\n{}".format(decrypted_data)) except Exception as exception: self.handle_unexpected_exception(exception) self.log_message("decrypted data: {} / result code: {}".format(decrypted_data is not None, result_code)) return decrypted_data, signed_by, result_code def encrypt_and_armor(self, data, to_user_id, charset=None): ''' Encrypt and armor data with the public key indicated by to_user_id. >>> # Test a few cases known to fail. See the unittests for more robust examples. >>> # In honor of First Sergeant Amit, who publicly denounced and refused to serve in operations >>> # involving the occupied Palestinian territories because of the widespread surveillance of innocent residents. >>> plugin = GPGPlugin() >>> encrypted_data, error_message = plugin.encrypt_and_armor('This is a test', '*****@*****.**') >>> encrypted_data is None True >>> 'public key not found' in error_message True ''' encrypted_data = error_message = None try: if to_user_id is None or data is None or len(data.strip()) <= 0: self.log_message('unable to encrypt and armor because key info missing') else: self.log_message('encrypting and armoring to "{}"'.format(to_user_id)) self.log_data(data) # we could use MIME, but for now keep it readable args = [gpg_constants.ENCRYPT_DATA, gpg_constants.ARMOR_DATA, gpg_constants.OPEN_PGP, gpg_constants.RECIPIENT, self.get_user_id_spec(to_user_id)] if charset is not None: args.append(gpg_constants.CHARSET) args.append(charset) result_code, gpg_output, gpg_error= self.gpg_command(args, data=data) if result_code == gpg_constants.GOOD_RESULT: encrypted_data = gpg_output else: if gpg_error: error_message = gpg_error if gpg_output and error_message is None: error_message = gpg_output except Exception as exception: self.handle_unexpected_exception(exception) return encrypted_data, error_message def sign(self, data, user_id, passphrase): ''' Sign data with the private key indicated by user id. Return the signed data or None if the signing fails. >>> from goodcrypto.oce import test_constants >>> plugin = GPGPlugin() >>> signed_data, error_message = plugin.sign(test_constants.TEST_DATA_STRING, ... test_constants.EDWARD_LOCAL_USER, test_constants.EDWARD_PASSPHRASE) >>> signed_data is not None True >>> len(signed_data) > 0 True >>> error_message is None True ''' signed_data = error_message = None try: if passphrase is None or data is None or len(data.strip()) <= 0: self.log_message('could not sign because missing key info') else: user = self.get_user_id_spec(user_id) args = [gpg_constants.CLEAR_SIGN, gpg_constants.LOCAL_USER, user] result_code, gpg_output, gpg_error= self.gpg_command( args, passphrase=passphrase, data=data) if result_code == gpg_constants.GOOD_RESULT: signed_data = gpg_output self.log_message('signed by "{}"'.format(user)) self.log_data(signed_data, "signed data") else: if gpg_error: error_message = gpg_error if gpg_output and error_message is None: error_message = gpg_output if error_message is not None and 'using subkey' in error_message: i = error_message.find('\n') if i > 0: error_message = error_message[i:] except Exception as exception: self.handle_unexpected_exception(exception) return signed_data, error_message def sign_and_encrypt(self, data, from_user_id, to_user_id, passphrase, clear_sign=False, charset=None): ''' Sign data with the secret key indicated by from_user_id, then encrypt with the public key indicated by to_user_id. To avoid a security bug in OpenPGP we must sign before encrypting. >>> # Test a few cases known to fail. See the unittests for more robust examples. >>> # In honor of David Weber, who revealed misconduct in the SEC investigations of >>> # Bernard Madoff and Allen Stanford. >>> # In honor of Virgil Grandfield, who uncovered a scandal in which some 50,000 or more >>> # Javanese construction workers were victims of human trafficking on NGO tsunami projects in Aceh. >>> plugin = GPGPlugin() >>> encrypted_data, error_message = plugin.sign_and_encrypt( ... 'This is a test', '*****@*****.**', '*****@*****.**', 'secret') >>> encrypted_data is None True >>> error_message is not None True >>> encrypted_data, error_message = plugin.sign_and_encrypt(None, '*****@*****.**', '*****@*****.**', 'secret') >>> encrypted_data is None True >>> error_message is not None True ''' self.log_message('signing by "{}" and encrypting to "{}'.format( self.get_user_id_spec(from_user_id), self.get_user_id_spec(to_user_id))) args = [gpg_constants.SIGN, gpg_constants.ENCRYPT_DATA, gpg_constants.OPEN_PGP, gpg_constants.LOCAL_USER, self.get_user_id_spec(from_user_id), gpg_constants.RECIPIENT, self.get_user_id_spec(to_user_id)] if charset is not None: args.append(gpg_constants.CHARSET) args.append(charset) encrypted_data, error_message = self._sign_and_encrypt_now( data, from_user_id, to_user_id, passphrase, clear_sign, args) return encrypted_data, error_message def sign_encrypt_and_armor(self, data, from_user, to_user, passphrase, clear_sign=False, charset=None): ''' Sign data with the secret key indicated by from_user, then encrypt with the public key indicated by to_user, then ASCII armor, and finally clear sign it. To avoid a security bug in OpenPGP we must sign before encrypting. >>> # Test a few cases known to fail. See the unittests for more robust examples. >>> # In honor of Ted Siska, who blew the whistle on Ward Diesel Filter Systems for filing false >>> # claims to the US government for work on diesel exhaust filtering systems for fire engines. >>> # In honor of Peter Bryce, who revealed Canadian Indian children were being systematically and >>> # deliberately killed in the residential schools in the 1920s. >>> plugin = GPGPlugin() >>> encrypted_data, error_message = plugin.sign_encrypt_and_armor( ... 'This is a test known to fail', '*****@*****.**', '*****@*****.**', 'a secret') >>> encrypted_data is None True >>> error_message is not None True >>> encrypted_data, error_message = plugin.sign_encrypt_and_armor(None, '*****@*****.**', '*****@*****.**', 'a secret') >>> encrypted_data is None True >>> error_message is not None True ''' self.log_message('signing by "{}" and encrypting to "{}" and armoring'.format( self.get_user_id_spec(from_user), self.get_user_id_spec(to_user))) self.log_data(data) args = [gpg_constants.ARMOR_DATA, gpg_constants.SIGN, gpg_constants.ENCRYPT_DATA, gpg_constants.OPEN_PGP, gpg_constants.LOCAL_USER, self.get_user_id_spec(from_user), gpg_constants.RECIPIENT, self.get_user_id_spec(to_user)] if charset is not None: args.append(gpg_constants.CHARSET) args.append(charset) encrypted_data, error_message = self._sign_and_encrypt_now( data, from_user, to_user, passphrase, clear_sign, args) return encrypted_data, error_message def verify(self, data, by_user_id): ''' Verify data was signed by the user id. >>> from goodcrypto.oce import test_constants >>> plugin = GPGPlugin() >>> signed_data, __ = plugin.sign(test_constants.TEST_DATA_STRING, ... test_constants.EDWARD_LOCAL_USER, test_constants.EDWARD_PASSPHRASE) >>> plugin.verify(signed_data, test_constants.EDWARD_LOCAL_USER) True >>> # In honor of Karen Silkwood, who was the first nuclear power safety whistleblower. >>> from goodcrypto.oce.key.key_factory import KeyFactory >>> from goodcrypto.oce import test_constants >>> email = '*****@*****.**' >>> passcode = 'secret' >>> plugin = KeyFactory.get_crypto(gpg_constants.ENCRYPTION_NAME) >>> ok, __, __, __ = plugin.create(email, passcode, wait_for_results=True) >>> ok True >>> signed_data, __ = plugin.sign(test_constants.TEST_DATA_STRING, email, passcode) >>> plugin.delete(email) True >>> plugin.verify(signed_data, email) False ''' self.log_message('starting to verify "{}" signed data'.format(by_user_id)) signer = self.get_signer(data) if signer is None: verified = False self.log_message("no signer found") else: self.log_message('signed by "{}"'.format(signer)) user_email = get_email(by_user_id) signer_email = get_email(signer) verified = signer_email == user_email if not verified: self.log_message('could not verify because signed by "{}" not "{}"'.format( signer_email, user_email)) self.log_message('verified: {}'.format(verified)) return verified def get_user_id_spec(self, user_id): ''' Get user ID spec based on the _user_id_match_method. ''' if user_id is None: user = user_id else: try: user = get_email(user_id) except Exception: self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() user = user_id # if there's an @ sign and the email address is *not* in angle brackets, then add the brackets if (self._user_id_match_method == self.EMAIL_MATCH and user.find('@') > 0 and user.find('<') < 0 and user.find('>') < 0): user = "******".format(user) # if the match method is exact match and the user doesn't start with an equal sign, prefix = elif self._user_id_match_method == self.EXACT_MATCH and not user.startswith("="): user = "******" + user return user def list_packets(self, data, passphrase=None): ''' Get a list of packets from the data or None if the data isn't encrypted. ''' packets = None try: if data is None or (isinstance(data, str) and len(data.strip()) <= 0): self.log_message('no data so no packets') else: self.log_message('trying to list encrypted packets') args = [gpg_constants.LIST_PACKETS] result_code, gpg_output, gpg_error= self.gpg_command( args, passphrase=passphrase, data=data) if result_code == gpg_constants.GOOD_RESULT: packets = gpg_output self.log_data(packets, "good packets") elif result_code == gpg_constants.TIMED_OUT_RESULT: self.log_message('timed out analyzing packets') elif gpg_error is not None and gpg_error.find('encrypted with') > 0: packets = gpg_error self.log_data(packets, "bad packets") except Exception as exception: self.handle_unexpected_exception(exception) return packets def set_home_dir(self, dirname): ''' Sets the home dir and creates it if it doesn't exist. If the dirname doesn't start with /, then prefixes the standard OCE data directory. Intended for testing the GPG plugin. ''' command_ok = True old_home_dir = self.gpg_home if dirname.startswith('/'): self.gpg_home = dirname else: self.gpg_home = os.path.join(OCE_DATA_DIR, dirname) if not os.path.exists(self.get_home_dir()): self.log_message('unable to change home dir to {}'.format(self.gpg_home)) self.gpg_home = old_home_dir command_ok = False return command_ok def get_home_dir(self): ''' Gets the home dir and create it if it doesn't. ''' if self.gpg_home == None: self.gpg_home = self.GPG_HOME_DIR try: # create gpg's parent directories, if they don't already exist parent_dir = os.path.dirname(self.gpg_home) if not os.path.exists(parent_dir): statinfo = os.stat(os.path.dirname(parent_dir)) if statinfo.st_uid == os.geteuid(): os.makedirs(parent_dir, 0o770) self.log_message('created parent of home dir: {}'.format(parent_dir)) else: self.log_message('unable to create parent of home dir as {}: {}'.format(os.geteuid(), parent_dir)) # create gpg's home directory, if it doesn't exist already if not os.path.exists(self.gpg_home): statinfo = os.stat(os.path.dirname(self.gpg_home)) if statinfo.st_uid == os.geteuid(): os.mkdir(self.gpg_home, 0o700) self.log_message('created home dir: {}'.format(self.gpg_home)) except OSError: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') except Exception as exception: self.log_message(exception) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() return self.gpg_home def gpg_command(self, initial_args, passphrase=None, data=None, wait_for_results=True): ''' Issue a gpg command. This should be used internally to the gpg classes instead of directly. See the public functions that perform gpg commands for better examples. ''' result_code = gpg_constants.ERROR_RESULT gpg_output = None gpg_error = None if gpg_constants.ENCRYPT_DATA in initial_args: command = gpg_constants.ENCRYPT_DATA elif gpg_constants.DECRYPT_DATA in initial_args: command = gpg_constants.DECRYPT_DATA else: command = initial_args[0] # syr.lock.locked() is only a per-process lock # syr.lock has a system wide lock, but it is not well tested with locked(): try: self.log_message('--- started gpg command: {} ---'.format(command)) result_code = gpg_constants.ERROR_RESULT gpg_output = None gpg_error = None result_code, gpg_output, gpg_error = self.activate_queue( command, initial_args, passphrase, data, wait_for_results) except JobTimeoutException as job_exception: self.log_message('gpg command {}'.format(str(job_exception))) result_code = gpg_constants.TIMED_OUT_RESULT except Exception: self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() result_code = gpg_constants.ERROR_RESULT gpg_output = None gpg_error = str(exception) self.handle_unexpected_exception(exception) finally: self.log_message('gpg command result_code: {}'.format(result_code)) self.log_message('--- finished gpg command: {} ---'.format(command)) self.log.flush() return result_code, gpg_output, gpg_error def activate_queue(self, command, initial_args, passphrase, data, wait_for_results): ''' Run the command and wait for the results if appropriate. ''' def wait_until_queued(job_count): ''' Wait until the job is queued or timeout. ''' secs = 0 while (not self.job.is_queued and not self.job.is_started and not self.job.is_finished ): sleep(1) secs += 1 self.log_message('seconds until {} job was queued: {}'.format(self.job.get_id(), secs)) def wait_for_job(command): ''' Wait until the job finishes or fails. Gpg (1.x) is not thread safe. We use a queue yo make sure it only has one instance at a time. But we sometimes need to wait here in the client for an operation to complete. So we simulate a remote procedure call. ''' result_code = gpg_constants.ERROR_RESULT gpg_output = gpg_error = None try: # wait for the job with elapsed_time() as job_time: while self.job.result is None: sleep(1) self.log_message('{} job {} elapsed time: {}'. format(command, self.job.get_id(), job_time)) if self.job.is_finished: self.log_message('{} job finished'.format(command)) elif self.job.is_failed: self.log_message('{} job failed'.format(command)) result_code, gpg_output, gpg_error = self.job.result self.log_message('{} job {} result code: {}'.format(command, self.job.get_id(), result_code)) if self.DEBUGGING: if gpg_output: self.log_message(gpg_output) if gpg_error: self.log_message(gpg_error) if result_code == gpg_constants.ERROR_RESULT: self.log_message('job.status: {}'.format(self.job.get_status())) except JobTimeoutException as job_exception: self.log_message('gpg queue job timed out {}'.format(str(job_exception))) result_code = gpg_constants.TIMED_OUT_RESULT return result_code, gpg_output, gpg_error result_code = gpg_constants.ERROR_RESULT gpg_output = gpg_error = self.job = None try: current_job_timeout = self.DEFAULT_TIMEOUT if data is not None: data_length = len(data) self.log_message('data length: {}'.format(data_length)) if data_length > gpg_constants.LARGE_DATA_CHUNK: # set the timeout if the data is large; the timeout # should be a little larger than the gpg timeout current_job_timeout += int( (data_length / gpg_constants.LARGE_DATA_CHUNK) * gpg_constants.TIMEOUT_PER_CHUNK) # arguments for the 'gpg' command itself gpg_command_args = [self.get_home_dir(), initial_args, passphrase, data] self.queue_connection = Redis(REDIS_HOST, self.get_queue_port()) self.queue = Queue(name=GPG_RQ, connection=self.queue_connection) self.log_message('jobs waiting in gpg queue {}'.format(self.queue.count)) self.log_message('about to queue {}'.format(command)) # each job needs to wait for the jobs ahead of it so when # calculating the timeout include the jobs already in the queue timeout = current_job_timeout * (self.queue.count + 1) self.log_message('{} job timeout in seconds: {}'.format(command, timeout)) self.job = self.queue.enqueue_call(execute_gpg_command, args=gpg_command_args, timeout=timeout) if self.job is None: self.log_message('unable to queue {} job'.format(command)) else: job_id = self.job.get_id() self.log_message('{} job: {} times out in {} secs'.format(command, job_id, self.job.timeout)) wait_until_queued(self.queue.count) self.log_message('wait for {} job results: {}'.format(command, wait_for_results)) if wait_for_results: result_code, gpg_output, gpg_error = wait_for_job(command) else: if self.job.is_failed: result_code = gpg_constants.ERROR_RESULT job_dump = self.job.to_dict() if 'exc_info' in job_dump: gpg_error = job_dump['exc_info'] log_message('{} job exc info: {}'.format(job_id, error)) elif 'status' in job_dump: gpg_error = job_dump['status'] self.log_message('{} job status: {}'.format(job_id, gpg_error)) self.log_message('job dump:\n{}'.format(job_dump)) self.job.cancel() self.queue.remove(job_id) elif self.job.is_queued or self.job.is_started or self.job.is_finished: result_code = gpg_constants.GOOD_RESULT self.log_message('{} queued {} job'.format(self.queue, job_id)) else: self.log_message('{} {} job results: {}'.format( self.queue, job_id, self.job.result)) except Exception as exception: gpg_error = str(exception) record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if isinstance(result_code, bool): if result_code: result_code = gpg_constants.GOOD_RESULT else: result_code = gpg_constants.ERROR_RESULT return result_code, gpg_output, gpg_error def is_good_result(self): ''' Return if the exit code is a good result ''' return gpg_constants.GOOD_RESULT def is_error_result(self): ''' Return if the exit code is an error ''' return gpg_constants.ERROR_RESULT def is_timedout_result(self): ''' Return if job timed out ''' return gpg_constants.TIMED_OUT_RESULT def log_data(self, data, message="data"): ''' Log data. ''' if self.DEBUGGING: self.log_message("{}:\n{}".format(message, data)) def log_message(self, message): ''' Log a message. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message) def _sign_and_encrypt_now(self, data, from_user_id, to_user_id, passphrase, clear_sign, args): ''' Sign and encrypt the data now. Only used internally. Use one of the public functions to sign, encrypt, and armor data, and finally clear sign it. ''' encrypted_data = error_message = None try: if data is None: error_message = 'Could not sign and encrypt because there is no data.' self.log_message(error_message) elif from_user_id is None: error_message = 'Could not sign and encrypt because there is no FROM user id.' self.log_message(error_message) elif to_user_id is None: error_message = 'Could not sign and encrypt because there is no TO user id.' self.log_message(error_message) elif passphrase is None: error_message = 'Could not sign and encrypt because there is no passphrase.' self.log_message(error_message) else: self.log_data(data) result_code, gpg_output, gpg_error = self.gpg_command( args, passphrase=passphrase, data=data, wait_for_results=True) self.log_message('results after signing, encrypting, and armoring: {}'.format(result_code)) if result_code == gpg_constants.GOOD_RESULT: if clear_sign: encrypted_data, error_message = self.sign(gpg_output, from_user_id, passphrase) self.log_message('results after clear signing: {}'.format(len(encrypted_data) > 0)) if error_message is not None: self.log_message(error_message) else: encrypted_data = gpg_output else: if gpg_error: self.log_message(gpg_error) error_message = gpg_error if gpg_output: self.log_message(gpg_output) if error_message is None: error_message = gpg_output if error_message is not None and 'using subkey' in error_message: i = error_message.find('\n') if i > 0: error_message = error_message[i:] except TypeError as type_error: self.handle_unexpected_exception(type_error) if error_message is None: error_message = 'Unexecpted type error' except Exception as exception: self.handle_unexpected_exception(exception) if error_message is None: error_message = 'Unexecpted exception' return encrypted_data, error_message def _parse_signer(self, output_string): ''' Parse the signer from the gpg command's results. ''' signer = None try: if output_string is not None: lines = output_string.split('\n') for line in lines: if line.startswith(self.GOOD_SIGNATURE_PREFIX): signer = line[len(self.GOOD_SIGNATURE_PREFIX):].strip('"') if len(signer) > 0: break # gpg reports the sig is bad if the key is a multi-email key and # the first key doesn't match the user that created the signature elif line.startswith(self.BAD_SIGNATURE_PREFIX): signer = line[len(self.BAD_SIGNATURE_PREFIX):].strip('"') if len(signer) > 0: break # strip the extra comment (e.g., [ultimate]) if signer is not None: i = signer.rfind('"') if i > 0: signer = signer[:i] signer = signer.strip() signer = signer.strip('"') except Exception as exception: self.handle_unexpected_exception(exception) return signer def _parse_version(self, version): ''' Parse the version from the gpg command's results. ''' version_number = None if version is None: version_number = '' else: reader = StringIO(initial_value=version) for l in reader: line = l.strip() index = line.rfind(' ') if index >= 0: possibleVersionNumber = line[index + 1:] # make sure we got something resembling "X.Y" dotIndex = possibleVersionNumber.find('.') if dotIndex > 0 and dotIndex < len(possibleVersionNumber) - 1: version_number = possibleVersionNumber else: self.log_message("version number not found in " + line) # stop looking when we find the version if version_number != None: break if version_number == None: version_number = '' return version_number def _create_basic_key_files(self): def create_key_file(filename): fullname = os.path.join(self.gpg_home, filename) sh.touch(fullname) os.chmod(fullname, 0o600) self.log_message('{} exists: {}'.format(filename, os.path.exists(filename))) create_key_file(gpg_constants.PUBLIC_KEY_FILENAME) create_key_file(gpg_constants.SECRET_KEY_FILENAME) create_key_file(gpg_constants.TRUST_DB_FILENAME)
class Decrypt(object): ''' Decrypt message filter. This filter tries all known encryption software for the recipient. Because encryption may be nested, this class keeps trying until the message is decrypted, or no valid encryption program can decrypt it further. !!!! If part of a message is plaintext and part encrypted, the decrypted text replaces the entire text, and the plaintext part is lost. !!!! A multiply-encrypted message may be tagged decrypted if any layer is successfully decrypted, even if an inner layer is still encrypted. See the unit tests to see how to use the Decrypt class. ''' DEBUGGING = False # the encrypted content is the second part; indexing starts at 0 ENCRYPTED_BODY_PART_INDEX = 1 def __init__(self, crypto_message): ''' >>> decrypt = Decrypt(None) >>> decrypt != None True ''' self.log = LogFile() self.crypto_message = crypto_message self.need_to_send_metadata_key = False def process_message(self): ''' If the message is encrypted, try to decrypt it. If it's not encrypted, add a warning about the dangers of unencrypted messages. See unittests for usage as the test set up is too complex for a doctest. ''' try: filtered = decrypted = False from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() if options.verify_dkim_sig(): self.crypto_message, dkim_sig_verified = decrypt_utils.verify_dkim_sig(self.crypto_message) if options.dkim_delivery_policy() == DKIM_DROP_POLICY: self.log_message('verified dkim signature ok: {}'.format(dkim_sig_verified)) elif dkim_sig_verified: self.log_message('verified dkim signature') else: self.log_message('unable to verify dkim signature, but dkim policy is to just warn') self.log_message("checking if message from {} to {} needs decryption".format(from_user, to_user)) if Decrypt.DEBUGGING: self.log_message('logged original message headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, tag='original message headers') header_keys = HeaderKeys() if options.auto_exchange_keys(): header_contains_key_info = header_keys.manage_keys_in_header(self.crypto_message) else: header_contains_key_info = header_keys.keys_in_header(self.crypto_message) if self.crypto_message.is_dropped(): decrypted = False self.log_message("message dropped because of bad key in header") self.log_message('logged dropped message headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, tag='dropped message headers') else: message_char_set, __ = get_charset(self.crypto_message.get_email_message()) self.log_message('message char set: {}'.format(message_char_set)) if self.crypto_message.get_email_message().is_probably_pgp(): decrypted, decrypted_with = self.decrypt_message() if not decrypted and self.crypto_message.is_metadata_crypted(): tags.add_metadata_tag(self.crypto_message) self.log_message("message only encrypted with metadata key") else: decrypt_utils.verify_clear_signed(from_user, self.crypto_message) if self.crypto_message.is_metadata_crypted(): tags.add_metadata_tag(self.crypto_message) self.log_message("message only encrypted with metadata key") else: tags.add_unencrypted_warning(self.crypto_message) filtered = True self.log_message("message doesn't appear to be encrypted at all") # create a private key for the recipient if there isn't one already add_private_key(to_user) self.need_to_send_metadata_key = ( # if the metadata wasn't encrypted not self.crypto_message.is_metadata_crypted() and # but the sender's key was in the header so we know the sender uses GoodCrypto private server header_contains_key_info and # and we don't have the sender's metadata key len(contacts.get_encryption_names(get_metadata_address(email=from_user))) < 1) self.log_message('need to send metadata key: {}'.format(self.need_to_send_metadata_key)) self.log_message('message content decrypted: {}'.format(decrypted)) # finally save a record so the user can verify the message was received securely if (decrypted or self.crypto_message.is_metadata_crypted() or self.crypto_message.is_private_signed() or self.crypto_message.is_clear_signed() ): decrypt_utils.add_history_and_verification(self.crypto_message) if decrypted or self.crypto_message.is_metadata_crypted(): self.crypto_message.set_crypted(True) if not decrypted: self.log_message('metadata tag: {}'.format(self.crypto_message.get_tag())) analyzer = OpenPGPAnalyzer() if analyzer.is_encrypted(self.crypto_message.get_email_message().get_content()): tags.add_extra_layer_warning(self.crypto_message) else: self.crypto_message.set_crypted(False) decrypted_tags_filtered = tags.add_decrypted_tags_to_message(self.crypto_message) if decrypted_tags_filtered: filtered = True if filtered and not self.crypto_message.is_filtered(): self.crypto_message.set_filtered(True) self.log_message("finished adding tags to decrypted message; filtered: {}".format(self.crypto_message.is_filtered())) self.log_message("message originally encrypted with: {}".format(self.crypto_message.get_crypted_with())) self.log_message("metadata originally encrypted with: {}".format(self.crypto_message.get_metadata_crypted_with())) decrypt_utils.re_mime_encode(self.crypto_message) if self.DEBUGGING: self.log_message('logged final decrypted headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, tag='final decrypted headers') self.log_message(' final status: filtered: {} decrypted: {}'.format( self.crypto_message.is_filtered(), self.crypto_message.is_crypted())) except MessageException as message_exception: self.log_message(message_exception) raise MessageException(value=message_exception.value) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return self.crypto_message def get_crypto_message(self): ''' Get the crypto message. ''' return self.crypto_message def decrypt_message(self, filter_msg=True): ''' Decrypt a message and add a tag if unsuccessful (internal use only). ''' encryption_names = self._get_recipient_encryption_software() if encryption_names is None or len(encryption_names) < 1 or self.crypto_message is None: decrypted = False decrypted_with = [] self.log_message('unable to decrypt message when missing data') else: try: self.log_message('trying to decrypt with: {}'.format(encryption_names)) decrypted, decrypted_with = self._decrypt_with_all_encryption(encryption_names) if decrypted: self.crypto_message.set_crypted_with(decrypted_with) # if the message is still encrypted, log it and tell the user elif self.crypto_message.get_email_message().is_probably_pgp(): if len(encryption_names) > 1: software = encryption_names.__str__() else: software = str(encryption_names[0]) log_msg = "Failed to decrypt with {}".format(software) self.log_message(log_msg) self.log_message('logged failed message headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, 'failed message headers') record_exception(message=log_msg) tag = i18n('Unable to decrypt message with {encryption}'.format(encryption=software)) self.crypto_message.add_error_tag_once(tag) # don't filter bundled messages; each message will be filtered separately if filter_msg and options.filter_html(): self._filter_html() else: self.log_message("html filter disabled") except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: decrypted = False record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return decrypted, decrypted_with def needs_metadata_key(self): ''' Gets whether the metadata key should be sent. ''' return self.need_to_send_metadata_key def _get_recipient_encryption_software(self): ''' Get the software to decrypt a message for the recipient (internal use only). If the user doesn't have a key, then configure one and return None. ''' encryption_software = None if self.crypto_message is None: self.log_message("missing crypto_message".format(self.crypto_message)) else: try: from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() encryption_software = contacts.get_encryption_names(to_user) if len(encryption_software) > 0: self.log_message("encryption software: {}".format(encryption_software)) elif email_in_domain(to_user) and options.create_private_keys(): add_private_key(to_user, encryption_software=encryption_software) self.log_message("started to create a new {} key for {}".format(encryption_software, to_user)) else: self.log_message("no encryption software for {}".format(to_user)) self.crypto_message.add_error_tag_once( i18n('Message appears encrypted, but {email} does not use any known encryption'.format(email=to_user))) report_unable_to_decrypt(to_user, self.crypto_message.get_email_message().to_string()) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception as IOError: utils.log_crypto_exception(MessageException(format_exc())) record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') try: self.crypto_message.add_error_tag_once(SERIOUS_ERROR_PREFIX) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return encryption_software def _decrypt_with_all_encryption(self, encryption_names): ''' Decrypt a message using all known encryption (internal use only). ''' decrypted = False decrypted_with = [] try: # the sender probably used the order of services in the # AcceptedEncryptionSoftware header we sent out, so we want to # use them in reverse order # move to the end of the list, and back up i = len(encryption_names) self.log_message("encrypted {} time(s)".format(i)) while i > 0: i -= 1 encryption_name = encryption_names[i] if self.crypto_message.get_email_message().is_probably_pgp(): try: if self._decrypt_message_with_crypto(encryption_name): # if any encryption decrypts, the message was decrypted decrypted = True decrypted_with.append(encryption_name) self.crypto_message.set_crypted(decrypted) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: msg = 'could not decrypt with {}.'.format(encryption_name) self.log_message(msg) self.log_message('EXCEPTION - see syr.exception.log for details') record_exception() else: self.log_message("message already decrypted, so did not try {}".format(encryption_name)) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return decrypted, decrypted_with def _decrypt_message_with_crypto(self, encryption_name): ''' Decrypt a message using the encryption software (internal use only). ''' decrypted = False self.log_message("encryption program: {}".format(encryption_name)) from_user = self.crypto_message.smtp_sender() to_user = self.crypto_message.smtp_recipient() passcode = user_keys.get_passcode(to_user, encryption_name) if passcode == None or len(passcode) <= 0: tag = '{email} does not have a private key configured.'.format(email=to_user) self.log_message(tag) self.crypto_message.add_error_tag_once(tag) else: # make sure that the key for the recipient is ok; if it's not, a CryptoException is thrown __, verified, __ = contacts.is_key_ok(to_user, encryption_name) self.log_message('{} {} key pinned'.format(to_user, encryption_name)) self.log_message('{} {} key verified: {}'.format(to_user, encryption_name, verified)) crypto = CryptoFactory.get_crypto(encryption_name, crypto_software.get_classname(encryption_name)) # try to verify signature in case it was clear signed after it was encrypted decrypt_utils.verify_clear_signed( from_user, self.crypto_message, encryption_name=crypto.get_name(), crypto=crypto) self.log_message('trying to decrypt using {} private {} key.'.format(to_user, encryption_name)) if is_open_pgp_mime(self.crypto_message.get_email_message().get_message()): decrypted = self._decrypt_open_pgp_mime(from_user, crypto, passcode) else: decrypted = self._decrypt_inline_pgp(from_user, crypto, passcode) self.log_message('decrypted using {} private {} key: {}'.format(to_user, encryption_name, decrypted)) # try to verify signature in case it was clear signed before it was encrypted if decrypted: decrypt_utils.verify_clear_signed( from_user, self.crypto_message, encryption_name=crypto.get_name(), crypto=crypto) if self.DEBUGGING: self.log_message('decrypted message:\n{}'.format( self.crypto_message.get_email_message().get_message())) self.log_message('decrypted message char set: {}'.format(get_charset(self.crypto_message.get_email_message()))) return decrypted def _decrypt_open_pgp_mime(self, from_user, crypto, passcode): ''' Decrypt an open PGP MIME message (internal use only). ''' decrypted = False plaintext = None encrypted_part = None try: self.log_message("message is in OpenPGP MIME format") if self.DEBUGGING: self.log_message('logged OpenPGP mime headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, tag='OpenPGP mime headers') # remove any clear signed section before decrypting message self.crypto_message.get_email_message().remove_pgp_signature_blocks() payloads = self.crypto_message.get_email_message().get_message().get_payload() self.log_message("{} parts in message".format(len(payloads))) encrypted_part = payloads[self.ENCRYPTED_BODY_PART_INDEX] if isinstance(encrypted_part, Message): encrypted_part = encrypted_part.get_payload() if Decrypt.DEBUGGING: self.log_message("encrypted_part\n{}".format(encrypted_part)) charset, __ = get_charset(encrypted_part) self.log_message('encrypted part char set: {}'.format(charset)) plaintext = self._decrypt(from_user, encrypted_part, charset, crypto, passcode) except CryptoException as crypto_exception: raise CryptoException(crypto_exception.value) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') if plaintext == None or encrypted_part is None or plaintext == encrypted_part: decrypted = False self.log_message("unable to decrypt message") else: filtered = self._extract_embedded_message(plaintext) self.crypto_message.set_filtered(filtered) decrypted = self.crypto_message.is_crypted() return decrypted def _decrypt_inline_pgp(self, from_user, crypto, passcode): ''' Decrypt an inline PGP message (internal use only). ''' def adjust_attachment_name(part): '''Adjust the filename for the attachment.''' try: filename = part.get_filename() if filename and filename.endswith('.pgp'): self.log_message('original attachment filename: {}'.format(filename)) end = len(filename) - len('.pgp') part.replace_header( mime_constants.CONTENT_DISPOSITION_KEYWORD, 'attachment; filename="{}"'.format(filename[:end])) filename = part.__getitem__(mime_constants.CONTENT_DISPOSITION_KEYWORD) self.log_message('new attachment filename: {}'.format(filename)) else: self.log_message('attachment filename: {}'.format(filename)) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') decrypted = False self.log_message("message is inline PGP format") message = self.crypto_message.get_email_message().get_message() self.log_message("message content type is {}".format(message.get_content_type())) # remove any clear signed section before decrypting message self.crypto_message.get_email_message().remove_pgp_signature_blocks() if message.is_multipart(): for part in message.get_payload(): content_type = part.get_content_type() ciphertext = part.get_payload(decode=True) charset, __ = get_charset(part) self.log_message('multipart message char set: {}'.format(charset)) plaintext = self._decrypt(from_user, ciphertext, charset, crypto, passcode) if plaintext is not None and plaintext != ciphertext: decrypted = True charset, __ = part.get_charset() self.log_message("message part charset is {}".format(charset)) part.set_payload(plaintext, charset=charset) try: content_keyword_in_part = part.__getitem__(mime_constants.CONTENT_DISPOSITION_KEYWORD) except Exception: content_keyword_in_part = None if content_keyword_in_part is not None: adjust_attachment_name(part) try: encoding = part.__getitem__(mime_constants.CONTENT_XFER_ENCODING_KEYWORD) except Exception: encoding = None if encoding == mime_constants.QUOTED_PRINTABLE_ENCODING: encode_quopri(part) self.log_message("{} encoded message part".format(encoding)) elif encoding == mime_constants.BASE64_ENCODING: encode_base64(part) self.log_message("{} encoded message part".format(encoding)) else: charset, __ = get_charset(self.crypto_message.get_email_message()) self.log_message('inline pgp char set: {}'.format(charset)) ciphertext = self.crypto_message.get_email_message().get_content() plaintext = self._decrypt(from_user, ciphertext, charset, crypto, passcode) if plaintext is None or ciphertext is None or plaintext == ciphertext: decrypted = False self.log_message("unable to decrypt {} message".format(message.get_content_type())) else: # do not specify the codec self.crypto_message.get_email_message().get_message().set_payload(plaintext) try: encoding = self.crypto_message.get_email_message().get_message().__getitem__( mime_constants.CONTENT_XFER_ENCODING_KEYWORD) except Exception: encoding = None if encoding == mime_constants.QUOTED_PRINTABLE_ENCODING: encode_quopri(self.crypto_message.get_email_message().get_message()) elif encoding == mime_constants.BASE64_ENCODING: encode_base64(self.crypto_message.get_email_message().get_message()) decrypted = True return decrypted def _decrypt(self, from_user, data, charset, crypto, passcode): ''' Decrypt the data from a message (internal use only). ''' decrypted_data = signed_by = None if crypto is None or data is None: decrypted_data = None self.log_message("no crypto defined") else: if is_string(data): encrypted_data = data else: encrypted_data = data.encode(errors='replace') # ASCII armored plaintext looks just like armored ciphertext, # so check that we actually have encrypted data if (OpenPGPAnalyzer().is_encrypted( encrypted_data, passphrase=passcode, crypto=crypto)): if self.DEBUGGING: self.log_message('encrypted data before decryption:\n{}'.format(encrypted_data)) decrypted_data, signed_by, result_code = crypto.decrypt(encrypted_data, passcode) if (decrypted_data == None or (is_string(decrypted_data) and len(decrypted_data) <= 0)): if self.DEBUGGING: self.log_message('decrypted data:\n{}'.format(decrypted_data)) decrypted_data = None self.log_message("unable to decrypt data") if self.DEBUGGING: self.log_message('data bytearray after decryption:\n{}'.format(decrypted_data)) else: if result_code == 0: tag = tags.get_decrypt_signature_tag( self.crypto_message, from_user, signed_by, crypto.get_name()) if tag is not None: self.crypto_message.add_prefix_to_tag_once(tag) self.log_message('decrypt sig tag: {}'.format(tag)) elif result_code == 2: self.crypto_message.add_error_tag_once( i18n("Can't verify signature because the message was encrypted using an unknown key. Ask the sender to send their key if they aren't using GoodCrypto.")) if is_string(decrypted_data): self.log_message('plaintext length: {}'.format(len(decrypted_data))) if self.DEBUGGING: self.log_message('plaintext:\n{}'.format(decrypted_data)) else: decrypted_data = None self.log_message("data appeared encrypted, but wasn't") if self.DEBUGGING: self.log_message('data:\n{}'.format(data)) return decrypted_data def _extract_embedded_message(self, plaintext): ''' Extract an embedded message. If the message includes an Open PGP header, then save the plaintext in the email message. Otherwise, create a new email message from the embedded message. ''' extracted_embedded_message = False try: if self.DEBUGGING: self.log_message('embbedded message:\n{}'.format(plaintext)) encrypted_type = get_first_header( self.crypto_message.get_email_message().get_message(), PGP_ENCRYPTED_CONTENT_TYPE) if encrypted_type is None: old_message = self.crypto_message.get_email_message().get_message() new_message = plaintext_to_message(old_message, plaintext) self.crypto_message.get_email_message().set_message(new_message) self.crypto_message.set_crypted(True) self.log_message("created a new message from the plaintext") if self.DEBUGGING: self.log_message('logged final embedded message headers in goodcrypto.message.utils.log') utils.log_message_headers(self.crypto_message, tag='final embedded message headers') else: # this assumes an embedded mime message self.log_message("openpgp mime type: {}".format(encrypted_type)) embedded_message = EmailMessage(plaintext) self.crypto_message.set_email_message(embedded_message) self.crypto_message.set_crypted(True) extracted_embedded_message = True self.log_message("embedded message type is {}".format( embedded_message.get_message().get_content_type())) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') return extracted_embedded_message def _filter_html(self): ''' Filter HTML to remove malious code (internal use only). ''' try: message = self.crypto_message.get_email_message().get_message() for part in message.walk(): part_content_type = part.get_content_type() # filter html and plain text if (part_content_type == mime_constants.TEXT_HTML_TYPE or part_content_type == mime_constants.TEXT_PLAIN_TYPE): original_payload = part.get_payload() safe_payload = firewall_html(original_payload) if original_payload != safe_payload: try: # strip extraneous </html> HTML_CLOSE = '</html>' if (part_content_type == mime_constants.TEXT_PLAIN_TYPE and safe_payload.lower().find('<html>') < 0): index = safe_payload.find(HTML_CLOSE) if index >= 0: safe_payload = '{} {}'.format( safe_payload[0:index], safe_payload[index+len(HTML_CLOSE):]) except: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') pass charset, __ = get_charset(part) self.log_message('html char set: {}'.format(charset)) part.set_payload(safe_payload, charset=charset) self.log_message("html filtered {} part".format(part_content_type)) except Exception: record_exception() self.log_message('EXCEPTION - see syr.exception.log for details') def log_message(self, message): ''' Log the message to the local log. ''' if self.log is None: self.log = LogFile() self.log.write_and_flush(message)