Esempio n. 1
0
def go():
    logger.debug_log('s3decryptor.go: start')

    s3_conn = S3Connection(config['aws_access_key_id'],
                           config['aws_secret_access_key'])
    bucket = s3_conn.get_bucket(config['s3_bucket_name'])

    # Note that `_bucket_iterator` throttles itself if/when there are no
    # available objects in the bucket.
    for encrypted_info_json in _bucket_iterator(bucket):
        logger.debug_log('s3decryptor.go: processing item')

        # In theory, all bucket items should be usable by us, but there's
        # always the possibility that a user (or attacker) is messing with us.
        try:
            encrypted_info = json.loads(encrypted_info_json)

            diagnostic_info = decryptor.decrypt(encrypted_info)

            diagnostic_info = diagnostic_info.strip()

            # HACK: PyYaml only supports YAML 1.1, which is not a true superset
            # of JSON. Therefore it can (and does) throw errors on some Android
            # feedback. We will try to load using JSON first.
            # TODO: Get rid of all YAML feedback and remove it from here.
            try:
                diagnostic_info = json.loads(diagnostic_info)
                logger.debug_log('s3decryptor.go: loaded JSON')
            except:
                diagnostic_info = yaml.safe_load(diagnostic_info)
                logger.debug_log('s3decryptor.go: loaded YAML')

            # Modifies diagnostic_info
            utils.convert_psinet_values(config, diagnostic_info)

            if not utils.is_diagnostic_info_sane(diagnostic_info):
                # Something is wrong. Skip and continue.
                continue

            # Modifies diagnostic_info
            datatransformer.transform(diagnostic_info)

            # Store the diagnostic info
            record_id = datastore.insert_diagnostic_info(diagnostic_info)

            if _should_email_data(diagnostic_info):
                logger.debug_log('s3decryptor.go: should email')
                # Record in the DB that the diagnostic info should be emailed
                datastore.insert_email_diagnostic_info(record_id, None, None)

            # Store an autoresponder entry for this diagnostic info
            datastore.insert_autoresponder_entry(None, record_id)

            logger.log('decrypted diagnostic data')

        except decryptor.DecryptorException as e:
            logger.exception()
            logger.error(str(e))
            try:
                # Something bad happened while decrypting. Report it via email.
                sender.send(config['decryptedEmailRecipient'],
                            config['emailUsername'],
                            u'S3Decryptor: bad object', encrypted_info_json,
                            None)  # no html body
            except smtplib.SMTPException as e:
                logger.exception()
                logger.error(str(e))

        # yaml.constructor.ConstructorError was being thown when a YAML value
        # consisted of just string "=". Probably due to this PyYAML bug:
        # http://pyyaml.org/ticket/140
        except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
            # Try the next attachment/message
            logger.exception()
            logger.error(str(e))

    logger.debug_log('s3decryptor.go: end')
Esempio n. 2
0
def go():
    logger.debug_log('maildecryptor.go start')

    emailgetter = EmailGetter(config['popServer'],
                              config['popPort'],
                              config['emailUsername'],
                              config['emailPassword'])

    # Retrieve and process email.
    # Note that `emailgetter.get` throttles itself if/when there are no emails
    # immediately available.
    for msg in emailgetter.get():
        logger.debug_log('maildecryptor.go: msg has %d attachments' % len(msg['attachments']))

        diagnostic_info = None

        #
        # First try to process attachments.
        #
        for attachment in msg['attachments']:
            # Not all attachments will be in our format, so expect exceptions.
            try:
                encrypted_info = attachment.getvalue()

                encrypted_info = json.loads(encrypted_info)

                diagnostic_info = decryptor.decrypt(encrypted_info)

                diagnostic_info = diagnostic_info.strip()

                diagnostic_info = _load_yaml(diagnostic_info)

                # Modifies diagnostic_info
                utils.convert_psinet_values(config, diagnostic_info)

                if not utils.is_diagnostic_info_sane(diagnostic_info):
                    # Something is wrong. Skip and continue.
                    continue

                # Modifies diagnostic_info
                datatransformer.transform(diagnostic_info)

                logger.log('email attachment decrypted')
                break

            except decryptor.DecryptorException as e:
                # Something bad happened while decrypting. Report it via email.
                logger.exception()
                try:
                    sender.send(config['decryptedEmailRecipient'],
                                config['emailUsername'],
                                u'Re: %s' % (msg['subject'] or ''),
                                'Decrypt failed: %s' % e,
                                msg['msgobj']['Message-ID'])
                except smtplib.SMTPException as e:
                    # Something went wrong with the sending of the response. Log it.
                    logger.exception()
                    logger.error(str(e))

            except (ValueError, TypeError) as e:
                # Try the next attachment/message
                logger.exception()
                logger.error(str(e))

        #
        # Store what info we have
        #

        email_info = _get_email_info(msg)
        diagnostic_info_record_id = None

        if diagnostic_info:
            # Add the user's email information to diagnostic_info.
            # This will allow us to later auto-respond, or act as a
            # remailer between the user and the Psiphon support team.
            diagnostic_info['EmailInfo'] = email_info

            # Store the diagnostic info
            diagnostic_info_record_id = datastore.insert_diagnostic_info(diagnostic_info)

            # Store the association between the diagnostic info and the email
            datastore.insert_email_diagnostic_info(diagnostic_info_record_id,
                                                   msg['msgobj']['Message-ID'],
                                                   msg['subject'])

        # Store autoresponder info regardless of whether there was a diagnostic info
        datastore.insert_autoresponder_entry(email_info, diagnostic_info_record_id)

    logger.debug_log('maildecryptor.go end')
Esempio n. 3
0
def _process_work_items(work_queue):
    '''
    This runs in the multiprocessing forks to do the actual work. It is a long-lived loop.
    '''
    while True:
        if terminate:
            logger.debug_log('got terminate; stopping work')
            break

        # In theory, all bucket items should be usable by us, but there's
        # always the possibility that a user (or attacker) is messing with us.
        try:
            logger.debug_log('_process_work_items: dequeueing work item')
            # This blocks if the queue is empty
            encrypted_info_json = work_queue.get()
            logger.debug_log('_process_work_items: dequeued work item')

            logger.debug_log('_process_work_items: processing item')

            diagnostic_info = None

            encrypted_info = json.loads(encrypted_info_json)

            diagnostic_info = decryptor.decrypt(encrypted_info)
            if not diagnostic_info:
                logger.error('diagnostic_info decrypted empty')
                # Also throw, so we get an email about it
                raise Exception('diagnostic_info decrypted empty')

            diagnostic_info = diagnostic_info.strip()
            if not diagnostic_info:
                logger.error('diagnostic_info stripped empty')
                # Also throw, so we get an email about it
                raise Exception('diagnostic_info stripped empty')

            # HACK: PyYaml only supports YAML 1.1, which is not a true superset
            # of JSON. Therefore it can (and does) throw errors on some Android
            # feedback. We will try to load using JSON first.
            # TODO: Get rid of all YAML feedback and remove it from here.
            try:
                diagnostic_info = json.loads(diagnostic_info)
                logger.debug_log('_process_work_items: loaded JSON')
            except:
                diagnostic_info = yaml.safe_load(diagnostic_info)
                logger.debug_log('_process_work_items: loaded YAML')

            if not diagnostic_info:
                logger.error('diagnostic_info unmarshalled empty')
                # Also throw, so we get an email about it
                raise Exception('diagnostic_info unmarshalled empty')

            logger.log('feedback id: %s' %
                       diagnostic_info.get('Metadata', {}).get('id'))

            # Modifies diagnostic_info
            utils.convert_psinet_values(config, diagnostic_info)

            if not utils.is_diagnostic_info_sane(diagnostic_info):
                # Something is wrong. Skip and continue.
                continue

            # Modifies diagnostic_info
            redactor.redact_sensitive_values(diagnostic_info)

            # Modifies diagnostic_info
            datatransformer.transform(diagnostic_info)

            # Store the diagnostic info
            record_id = datastore.insert_diagnostic_info(diagnostic_info)
            if record_id is None:
                # An error occurred or diagnostic info was a duplicate.
                continue

            if _should_email_data(diagnostic_info):
                logger.debug_log('_process_work_items: should email')
                # Record in the DB that the diagnostic info should be emailed
                datastore.insert_email_diagnostic_info(record_id, None, None)

            # Store an autoresponder entry for this diagnostic info
            datastore.insert_autoresponder_entry(None, record_id)

            logger.debug_log('decrypted diagnostic data')

        except decryptor.DecryptorException as e:
            logger.exception()
            logger.error(str(e))
            try:
                # Something bad happened while decrypting. Report it via email.
                sender.send(config['decryptedEmailRecipient'],
                            config['emailUsername'],
                            u'S3Decryptor: bad object', encrypted_info_json,
                            None)  # no html body
            except smtplib.SMTPException as e:
                logger.exception()
                logger.error(str(e))

        # yaml.constructor.ConstructorError was being thown when a YAML value
        # consisted of just string "=". Probably due to this PyYAML bug:
        # http://pyyaml.org/ticket/140
        except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
            # Try the next attachment/message
            logger.exception()
            logger.error(str(e))

        except Exception as e:
            try:
                # Something bad happened while decrypting. Report it via email.
                sender.send(config['decryptedEmailRecipient'],
                            config['emailUsername'],
                            u'S3Decryptor: unhandled exception',
                            str(e) + '\n---\n' + str(diagnostic_info),
                            None)  # no html body
            except smtplib.SMTPException as e:
                logger.exception()
                logger.error(str(e))
            raise

    logger.debug_log('_process_work_items: done')
def go():
    logger.debug_log('maildecryptor.go start')

    emailgetter = EmailGetter(config['popServer'], config['popPort'],
                              config['emailUsername'], config['emailPassword'])

    # Retrieve and process email.
    # Note that `emailgetter.get` throttles itself if/when there are no emails
    # immediately available.
    for msg in emailgetter.get():
        logger.debug_log('maildecryptor.go: msg has %d attachments' %
                         len(msg['attachments']))

        diagnostic_info = None

        #
        # First try to process attachments.
        #
        for attachment in msg['attachments']:
            # Not all attachments will be in our format, so expect exceptions.
            try:
                encrypted_info = attachment.getvalue()

                encrypted_info = json.loads(encrypted_info)

                diagnostic_info = decryptor.decrypt(encrypted_info)

                diagnostic_info = diagnostic_info.strip()

                diagnostic_info = _load_yaml(diagnostic_info)

                # Modifies diagnostic_info
                utils.convert_psinet_values(config, diagnostic_info)

                if not utils.is_diagnostic_info_sane(diagnostic_info):
                    # Something is wrong. Skip and continue.
                    continue

                # Modifies diagnostic_info
                datatransformer.transform(diagnostic_info)

                logger.log('email attachment decrypted')
                break

            except decryptor.DecryptorException as e:
                # Something bad happened while decrypting. Report it via email.
                logger.exception()
                try:
                    sender.send(config['decryptedEmailRecipient'],
                                config['emailUsername'],
                                u'Re: %s' % (msg['subject'] or ''),
                                'Decrypt failed: %s' % e,
                                msg['msgobj']['Message-ID'])
                except smtplib.SMTPException as e:
                    # Something went wrong with the sending of the response. Log it.
                    logger.exception()
                    logger.error(str(e))

            except (ValueError, TypeError) as e:
                # Try the next attachment/message
                logger.exception()
                logger.error(str(e))

        #
        # Store what info we have
        #

        email_info = _get_email_info(msg)
        diagnostic_info_record_id = None

        if diagnostic_info:
            # Add the user's email information to diagnostic_info.
            # This will allow us to later auto-respond, or act as a
            # remailer between the user and the Psiphon support team.
            diagnostic_info['EmailInfo'] = email_info

            # Store the diagnostic info
            diagnostic_info_record_id = datastore.insert_diagnostic_info(
                diagnostic_info)

            # Store the association between the diagnostic info and the email
            datastore.insert_email_diagnostic_info(diagnostic_info_record_id,
                                                   msg['msgobj']['Message-ID'],
                                                   msg['subject'])

        # Store autoresponder info regardless of whether there was a diagnostic info
        datastore.insert_autoresponder_entry(email_info,
                                             diagnostic_info_record_id)

    logger.debug_log('maildecryptor.go end')
Esempio n. 5
0
def go():
    logger.debug_log('s3decryptor.go: start')

    s3_conn = S3Connection(config['aws_access_key_id'], config['aws_secret_access_key'])
    bucket = s3_conn.get_bucket(config['s3_bucket_name'])

    # Note that `_bucket_iterator` throttles itself if/when there are no
    # available objects in the bucket.
    for encrypted_info_json in _bucket_iterator(bucket):
        logger.debug_log('s3decryptor.go: processing item')

        # In theory, all bucket items should be usable by us, but there's
        # always the possibility that a user (or attacker) is messing with us.
        try:
            encrypted_info = json.loads(encrypted_info_json)

            diagnostic_info = decryptor.decrypt(encrypted_info)

            diagnostic_info = diagnostic_info.strip()

            # HACK: PyYaml only supports YAML 1.1, which is not a true superset
            # of JSON. Therefore it can (and does) throw errors on some Android
            # feedback. We will try to load using JSON first.
            # TODO: Get rid of all YAML feedback and remove it from here.
            try:
                diagnostic_info = json.loads(diagnostic_info)
                logger.debug_log('s3decryptor.go: loaded JSON')
            except:
                diagnostic_info = yaml.safe_load(diagnostic_info)
                logger.debug_log('s3decryptor.go: loaded YAML')

            # Modifies diagnostic_info
            utils.convert_psinet_values(config, diagnostic_info)

            if not utils.is_diagnostic_info_sane(diagnostic_info):
                # Something is wrong. Skip and continue.
                continue

            # Modifies diagnostic_info
            datatransformer.transform(diagnostic_info)

            # Store the diagnostic info
            record_id = datastore.insert_diagnostic_info(diagnostic_info)

            if _should_email_data(diagnostic_info):
                logger.debug_log('s3decryptor.go: should email')
                # Record in the DB that the diagnostic info should be emailed
                datastore.insert_email_diagnostic_info(record_id, None, None)

            # Store an autoresponder entry for this diagnostic info
            datastore.insert_autoresponder_entry(None, record_id)

            logger.log('decrypted diagnostic data')

        except decryptor.DecryptorException as e:
            logger.exception()
            logger.error(str(e))
            try:
                # Something bad happened while decrypting. Report it via email.
                sender.send(config['decryptedEmailRecipient'],
                            config['emailUsername'],
                            u'S3Decryptor: bad object',
                            encrypted_info_json,
                            None)  # no html body
            except smtplib.SMTPException as e:
                logger.exception()
                logger.error(str(e))

        # yaml.constructor.ConstructorError was being thown when a YAML value
        # consisted of just string "=". Probably due to this PyYAML bug:
        # http://pyyaml.org/ticket/140
        except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
            # Try the next attachment/message
            logger.exception()
            logger.error(str(e))

    logger.debug_log('s3decryptor.go: end')