def go():
logger.debug_log('s3decryptor.go: start')
s3_conn = S3Connection(config['aws_access_key_id'],
config['aws_secret_access_key'])
bucket = s3_conn.get_bucket(config['s3_bucket_name'])
# Note that `_bucket_iterator` throttles itself if/when there are no
# available objects in the bucket.
for encrypted_info_json in _bucket_iterator(bucket):
logger.debug_log('s3decryptor.go: processing item')
# In theory, all bucket items should be usable by us, but there's
# always the possibility that a user (or attacker) is messing with us.
try:
encrypted_info = json.loads(encrypted_info_json)
diagnostic_info = decryptor.decrypt(encrypted_info)
diagnostic_info = diagnostic_info.strip()
# HACK: PyYaml only supports YAML 1.1, which is not a true superset
# of JSON. Therefore it can (and does) throw errors on some Android
# feedback. We will try to load using JSON first.
# TODO: Get rid of all YAML feedback and remove it from here.
try:
diagnostic_info = json.loads(diagnostic_info)
logger.debug_log('s3decryptor.go: loaded JSON')
except:
diagnostic_info = yaml.safe_load(diagnostic_info)
logger.debug_log('s3decryptor.go: loaded YAML')
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
# Store the diagnostic info
record_id = datastore.insert_diagnostic_info(diagnostic_info)
if _should_email_data(diagnostic_info):
logger.debug_log('s3decryptor.go: should email')
# Record in the DB that the diagnostic info should be emailed
datastore.insert_email_diagnostic_info(record_id, None, None)
# Store an autoresponder entry for this diagnostic info
datastore.insert_autoresponder_entry(None, record_id)
logger.log('decrypted diagnostic data')
except decryptor.DecryptorException as e:
logger.exception()
logger.error(str(e))
try:
# Something bad happened while decrypting. Report it via email.
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'S3Decryptor: bad object', encrypted_info_json,
None) # no html body
except smtplib.SMTPException as e:
logger.exception()
logger.error(str(e))
# yaml.constructor.ConstructorError was being thown when a YAML value
# consisted of just string "=". Probably due to this PyYAML bug:
# http://pyyaml.org/ticket/140
except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
logger.debug_log('s3decryptor.go: end')
def go():
logger.debug_log('maildecryptor.go start')
emailgetter = EmailGetter(config['popServer'],
config['popPort'],
config['emailUsername'],
config['emailPassword'])
# Retrieve and process email.
# Note that `emailgetter.get` throttles itself if/when there are no emails
# immediately available.
for msg in emailgetter.get():
logger.debug_log('maildecryptor.go: msg has %d attachments' % len(msg['attachments']))
diagnostic_info = None
#
# First try to process attachments.
#
for attachment in msg['attachments']:
# Not all attachments will be in our format, so expect exceptions.
try:
encrypted_info = attachment.getvalue()
encrypted_info = json.loads(encrypted_info)
diagnostic_info = decryptor.decrypt(encrypted_info)
diagnostic_info = diagnostic_info.strip()
diagnostic_info = _load_yaml(diagnostic_info)
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
logger.log('email attachment decrypted')
break
except decryptor.DecryptorException as e:
# Something bad happened while decrypting. Report it via email.
logger.exception()
try:
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'Re: %s' % (msg['subject'] or ''),
'Decrypt failed: %s' % e,
msg['msgobj']['Message-ID'])
except smtplib.SMTPException as e:
# Something went wrong with the sending of the response. Log it.
logger.exception()
logger.error(str(e))
except (ValueError, TypeError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
#
# Store what info we have
#
email_info = _get_email_info(msg)
diagnostic_info_record_id = None
if diagnostic_info:
# Add the user's email information to diagnostic_info.
# This will allow us to later auto-respond, or act as a
# remailer between the user and the Psiphon support team.
diagnostic_info['EmailInfo'] = email_info
# Store the diagnostic info
diagnostic_info_record_id = datastore.insert_diagnostic_info(diagnostic_info)
# Store the association between the diagnostic info and the email
datastore.insert_email_diagnostic_info(diagnostic_info_record_id,
msg['msgobj']['Message-ID'],
msg['subject'])
# Store autoresponder info regardless of whether there was a diagnostic info
datastore.insert_autoresponder_entry(email_info, diagnostic_info_record_id)
logger.debug_log('maildecryptor.go end')
def _process_work_items(work_queue):
'''
This runs in the multiprocessing forks to do the actual work. It is a long-lived loop.
'''
while True:
if terminate:
logger.debug_log('got terminate; stopping work')
break
# In theory, all bucket items should be usable by us, but there's
# always the possibility that a user (or attacker) is messing with us.
try:
logger.debug_log('_process_work_items: dequeueing work item')
# This blocks if the queue is empty
encrypted_info_json = work_queue.get()
logger.debug_log('_process_work_items: dequeued work item')
logger.debug_log('_process_work_items: processing item')
diagnostic_info = None
encrypted_info = json.loads(encrypted_info_json)
diagnostic_info = decryptor.decrypt(encrypted_info)
if not diagnostic_info:
logger.error('diagnostic_info decrypted empty')
# Also throw, so we get an email about it
raise Exception('diagnostic_info decrypted empty')
diagnostic_info = diagnostic_info.strip()
if not diagnostic_info:
logger.error('diagnostic_info stripped empty')
# Also throw, so we get an email about it
raise Exception('diagnostic_info stripped empty')
# HACK: PyYaml only supports YAML 1.1, which is not a true superset
# of JSON. Therefore it can (and does) throw errors on some Android
# feedback. We will try to load using JSON first.
# TODO: Get rid of all YAML feedback and remove it from here.
try:
diagnostic_info = json.loads(diagnostic_info)
logger.debug_log('_process_work_items: loaded JSON')
except:
diagnostic_info = yaml.safe_load(diagnostic_info)
logger.debug_log('_process_work_items: loaded YAML')
if not diagnostic_info:
logger.error('diagnostic_info unmarshalled empty')
# Also throw, so we get an email about it
raise Exception('diagnostic_info unmarshalled empty')
logger.log('feedback id: %s' %
diagnostic_info.get('Metadata', {}).get('id'))
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
redactor.redact_sensitive_values(diagnostic_info)
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
# Store the diagnostic info
record_id = datastore.insert_diagnostic_info(diagnostic_info)
if record_id is None:
# An error occurred or diagnostic info was a duplicate.
continue
if _should_email_data(diagnostic_info):
logger.debug_log('_process_work_items: should email')
# Record in the DB that the diagnostic info should be emailed
datastore.insert_email_diagnostic_info(record_id, None, None)
# Store an autoresponder entry for this diagnostic info
datastore.insert_autoresponder_entry(None, record_id)
logger.debug_log('decrypted diagnostic data')
except decryptor.DecryptorException as e:
logger.exception()
logger.error(str(e))
try:
# Something bad happened while decrypting. Report it via email.
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'S3Decryptor: bad object', encrypted_info_json,
None) # no html body
except smtplib.SMTPException as e:
logger.exception()
logger.error(str(e))
# yaml.constructor.ConstructorError was being thown when a YAML value
# consisted of just string "=". Probably due to this PyYAML bug:
# http://pyyaml.org/ticket/140
except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
except Exception as e:
try:
# Something bad happened while decrypting. Report it via email.
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'S3Decryptor: unhandled exception',
str(e) + '\n---\n' + str(diagnostic_info),
None) # no html body
except smtplib.SMTPException as e:
logger.exception()
logger.error(str(e))
raise
logger.debug_log('_process_work_items: done')
def go():
logger.debug_log('maildecryptor.go start')
emailgetter = EmailGetter(config['popServer'], config['popPort'],
config['emailUsername'], config['emailPassword'])
# Retrieve and process email.
# Note that `emailgetter.get` throttles itself if/when there are no emails
# immediately available.
for msg in emailgetter.get():
logger.debug_log('maildecryptor.go: msg has %d attachments' %
len(msg['attachments']))
diagnostic_info = None
#
# First try to process attachments.
#
for attachment in msg['attachments']:
# Not all attachments will be in our format, so expect exceptions.
try:
encrypted_info = attachment.getvalue()
encrypted_info = json.loads(encrypted_info)
diagnostic_info = decryptor.decrypt(encrypted_info)
diagnostic_info = diagnostic_info.strip()
diagnostic_info = _load_yaml(diagnostic_info)
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
logger.log('email attachment decrypted')
break
except decryptor.DecryptorException as e:
# Something bad happened while decrypting. Report it via email.
logger.exception()
try:
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'Re: %s' % (msg['subject'] or ''),
'Decrypt failed: %s' % e,
msg['msgobj']['Message-ID'])
except smtplib.SMTPException as e:
# Something went wrong with the sending of the response. Log it.
logger.exception()
logger.error(str(e))
except (ValueError, TypeError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
#
# Store what info we have
#
email_info = _get_email_info(msg)
diagnostic_info_record_id = None
if diagnostic_info:
# Add the user's email information to diagnostic_info.
# This will allow us to later auto-respond, or act as a
# remailer between the user and the Psiphon support team.
diagnostic_info['EmailInfo'] = email_info
# Store the diagnostic info
diagnostic_info_record_id = datastore.insert_diagnostic_info(
diagnostic_info)
# Store the association between the diagnostic info and the email
datastore.insert_email_diagnostic_info(diagnostic_info_record_id,
msg['msgobj']['Message-ID'],
msg['subject'])
# Store autoresponder info regardless of whether there was a diagnostic info
datastore.insert_autoresponder_entry(email_info,
diagnostic_info_record_id)
logger.debug_log('maildecryptor.go end')
def go():
logger.debug_log('s3decryptor.go: start')
s3_conn = S3Connection(config['aws_access_key_id'], config['aws_secret_access_key'])
bucket = s3_conn.get_bucket(config['s3_bucket_name'])
# Note that `_bucket_iterator` throttles itself if/when there are no
# available objects in the bucket.
for encrypted_info_json in _bucket_iterator(bucket):
logger.debug_log('s3decryptor.go: processing item')
# In theory, all bucket items should be usable by us, but there's
# always the possibility that a user (or attacker) is messing with us.
try:
encrypted_info = json.loads(encrypted_info_json)
diagnostic_info = decryptor.decrypt(encrypted_info)
diagnostic_info = diagnostic_info.strip()
# HACK: PyYaml only supports YAML 1.1, which is not a true superset
# of JSON. Therefore it can (and does) throw errors on some Android
# feedback. We will try to load using JSON first.
# TODO: Get rid of all YAML feedback and remove it from here.
try:
diagnostic_info = json.loads(diagnostic_info)
logger.debug_log('s3decryptor.go: loaded JSON')
except:
diagnostic_info = yaml.safe_load(diagnostic_info)
logger.debug_log('s3decryptor.go: loaded YAML')
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
# Store the diagnostic info
record_id = datastore.insert_diagnostic_info(diagnostic_info)
if _should_email_data(diagnostic_info):
logger.debug_log('s3decryptor.go: should email')
# Record in the DB that the diagnostic info should be emailed
datastore.insert_email_diagnostic_info(record_id, None, None)
# Store an autoresponder entry for this diagnostic info
datastore.insert_autoresponder_entry(None, record_id)
logger.log('decrypted diagnostic data')
except decryptor.DecryptorException as e:
logger.exception()
logger.error(str(e))
try:
# Something bad happened while decrypting. Report it via email.
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'S3Decryptor: bad object',
encrypted_info_json,
None) # no html body
except smtplib.SMTPException as e:
logger.exception()
logger.error(str(e))
# yaml.constructor.ConstructorError was being thown when a YAML value
# consisted of just string "=". Probably due to this PyYAML bug:
# http://pyyaml.org/ticket/140
except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
logger.debug_log('s3decryptor.go: end')