def _load_yaml(yaml_string):
# TODO: Rip the backwards-compatibility out of this at some later date.
# JSON is supposed to be a subset of YAML: en.wikipedia.org/wiki/YAML#JSON
# So we switched the client-side encoding of the diagnostic data from YAML
# to JSON (and got huge performance improvements), and we were still able
# to decode it the same way. But... it turns out that, at least in Python's
# YAML implementation, there is some JSON that's not valid YAML. I.e.:
# >>> x = json.loads('{"key": "hi\\/there"}')
# ... print x
# {u'key': u'hi/there'}
# >>> x = yaml.load('{"key": "hi\\/there"}')
# ... print x
# <big stack trace>
# found unknown escape character '/'
# in "<string>", line 1, column 13:
# {"key": "hi\/there"}
#
# So we're going to try loading `yaml_string` as JSON first, and then fall
# back to YAML if that fails.
logger.debug_log('maildecryptor._load_yaml start')
try:
obj = json.loads(yaml_string)
except:
yaml_docs = []
for yaml_doc in yaml.safe_load_all(yaml_string):
yaml_docs.append(yaml_doc)
obj = _upgrade_old_object(yaml_docs)
logger.debug_log('maildecryptor._load_yaml end')
return obj
def go():
'''
Spawns the worker subprocesses and sends data to them.
'''
logger.debug_log('go: start')
# Set up the multiprocessing
worker_manager = multiprocessing.Manager()
# We only want to pull items out of S3 as we process them, so the queue needs to be
# limited to the number of worker processes.
work_queue = worker_manager.Queue(maxsize=config['numProcesses'])
# Spin up the workers
worker_pool = multiprocessing.Pool(processes=config['numProcesses'])
[
worker_pool.apply_async(_process_work_items, (work_queue, ))
for i in range(config['numProcesses'])
]
# Retrieve and process email-to-diagnostic-info records.
# Note that `_email_diagnostic_info_records` throttles itself if/when
# there are no records immediately available.
for email_diagnostic_info in _email_diagnostic_info_records_iterator():
if terminate:
logger.debug_log('go: got terminate; closing work_queue')
work_queue.close()
break
logger.debug_log('go: enqueueing work item')
# This blocks if the queue is full
work_queue.put(email_diagnostic_info)
logger.debug_log('go: enqueued work item')
logger.debug_log('go: done')
def flask_fdelete():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
if not token or not file_path:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_file_path = os.path.join(login, file_path)
exists = full_file_path in db_user2files.lrange(login, 0, -1)
if not exists:
data = {MESSAGE_KEY: "The file for deleting does not exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_FOUND)
for node_ip in db_file2nodes.lrange(full_file_path, 0, -1):
res = request_node(node_ip, '/fdelete', {FULL_PATH_KEY: full_file_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fdelete")
db_node2files.lrem(node_ip, 0, full_file_path)
db_user2files.lrem(login, 0, full_file_path)
db_file2nodes.delete(full_file_path)
db_file2size.delete(full_file_path)
data = {MESSAGE_KEY: f"Successfully deleted the file"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def _load_yaml(yaml_string):
# TODO: Rip the backwards-compatibility out of this at some later date.
# JSON is supposed to be a subset of YAML: en.wikipedia.org/wiki/YAML#JSON
# So we switched the client-side encoding of the diagnostic data from YAML
# to JSON (and got huge performance improvements), and we were still able
# to decode it the same way. But... it turns out that, at least in Python's
# YAML implementation, there is some JSON that's not valid YAML. I.e.:
# >>> x = json.loads('{"key": "hi\\/there"}')
# ... print x
# {u'key': u'hi/there'}
# >>> x = yaml.load('{"key": "hi\\/there"}')
# ... print x
# <big stack trace>
# found unknown escape character '/'
# in "<string>", line 1, column 13:
# {"key": "hi\/there"}
#
# So we're going to try loading `yaml_string` as JSON first, and then fall
# back to YAML if that fails.
logger.debug_log('maildecryptor._load_yaml start')
try:
obj = json.loads(yaml_string)
except:
yaml_docs = []
for yaml_doc in yaml.safe_load_all(yaml_string):
yaml_docs.append(yaml_doc)
obj = _upgrade_old_object(yaml_docs)
logger.debug_log('maildecryptor._load_yaml end')
return obj
def request_node(ip, url, data):
try:
res = https_client.request('POST', f"http://{ip}{url}", fields=data)
return res
except Exception as e:
debug_log(f"Requesting node failed. Error: '{e}'")
return None
def _check_and_add_address_blacklist(address):
'''
Returns True if the address is blacklisted, otherwise inserts it in the DB
and returns False.
'''
logger.debug_log('_check_and_add_address_blacklist: enter')
# We need to normalize, otherwise we could get fooled by the fact that
# "*****@*****.**" is the same as "*****@*****.**".
# We're going to be fairly draconian and normalize down to just alpha-numerics.
match = _address_splitter.match(address)
if not match:
# Someone is messing with us
raise ValueError('invalid email address: match failed')
# Discard the '+'' part
name, _, domain = match.groups()
# Get rid of non-alphanumerics
name = _address_name_normalize_regex.sub('', name)
if not name:
# Someone is messing with us
raise ValueError('invalid email address: name part is invalid')
normalized_address = '%s@%s' % (name, domain)
blacklisted = datastore.check_and_add_response_address_blacklist(normalized_address)
logger.debug_log('_check_and_add_address_blacklist: exiting with blacklisted=%s' % blacklisted)
return blacklisted
def _bucket_iterator(bucket):
logger.debug_log('s3decryptor._bucket_iterator start')
while True:
for key in bucket.list():
logger.debug_log('s3decryptor._bucket_iterator: %s' % key)
contents = None
# Do basic sanity checks before trying to download the object
if _is_bucket_item_sane(key):
logger.debug_log(
's3decryptor._bucket_iterator: good item found, yielding')
contents = key.get_contents_as_string()
# Make sure to delete the key *before* proceeding, so we don't
# try to re-process if there's an error.
bucket.delete_key(key)
if contents:
yield contents
logger.debug_log(
's3decryptor._bucket_iterator: no item found, sleeping')
time.sleep(_SLEEP_TIME_SECS)
logger.debug_log('s3decryptor._bucket_iterator end')
def _bucket_iterator(bucket):
logger.debug_log('s3decryptor._bucket_iterator start')
while True:
for key in bucket.list():
logger.debug_log('s3decryptor._bucket_iterator: %s' % key)
contents = None
# Do basic sanity checks before trying to download the object
if _is_bucket_item_sane(key):
logger.debug_log('s3decryptor._bucket_iterator: good item found, yielding')
contents = key.get_contents_as_string()
# Make sure to delete the key *before* proceeding, so we don't
# try to re-process if there's an error.
bucket.delete_key(key)
if contents:
yield contents
logger.debug_log('s3decryptor._bucket_iterator: no item found, sleeping')
time.sleep(_SLEEP_TIME_SECS)
logger.debug_log('s3decryptor._bucket_iterator end')
def flask_fcreate():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
if not full_file_path:
data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
# create new empty file
try:
# create all subdirs
all_dirs = full_file_path.split("/")[:-1]
dir_path = ROOT
for directory in all_dirs:
dir_path = os.path.join(dir_path, directory)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# create new file there
with open(get_path(full_file_path), "w"):
pass
except OSError:
debug_log(f"Creation of the file {full_file_path} failed")
data = {MESSAGE_KEY: "Failed during file creation."}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {MESSAGE_KEY: "Success"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_replicate():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
target_node_ip = flask.request.form.get(key=NODE_IP_KEY,
default=None,
type=str)
if not full_file_path or not target_node_ip:
data = {
MESSAGE_KEY:
f"Missing required parameters: `{FULL_PATH_KEY}`, `{NODE_IP_KEY}`"
}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
file_path = get_path(full_file_path)
try:
with open(file_path, "rb") as f:
file_data = str(base64.encodebytes(f.read()), 'utf-8')
except OSError as e:
debug_log(e.strerror)
data = {MESSAGE_KEY: "Error on storage server"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {FULL_PATH_KEY: full_file_path, FILE_BYTES: file_data}
request_node(target_node_ip, '/save_replication', data)
return flask.make_response(flask.jsonify({}), HTTPStatus.OK)
def _check_and_add_address_blacklist(address):
'''
Returns True if the address is blacklisted, otherwise inserts it in the DB
and returns False.
'''
logger.debug_log('_check_and_add_address_blacklist: enter')
# We need to normalize, otherwise we could get fooled by the fact that
# "*****@*****.**" is the same as "*****@*****.**".
# We're going to be fairly draconian and normalize down to just alpha-numerics.
match = _address_splitter.match(address)
if not match:
# Someone is messing with us
raise ValueError('invalid email address: match failed')
# Discard the '+'' part
name, _, domain = match.groups()
# Get rid of non-alphanumerics
name = _address_name_normalize_regex.sub('', name)
if not name:
# Someone is messing with us
raise ValueError('invalid email address: name part is invalid')
normalized_address = '%s@%s' % (name, domain)
blacklisted = datastore.check_and_add_response_address_blacklist(normalized_address)
logger.debug_log('_check_and_add_address_blacklist: exiting with blacklisted=%s' % blacklisted)
return blacklisted
def flask_uploaded():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str)
file_size = flask.request.form.get(key=FILE_SIZE_KEY, default=None, type=str)
if not full_file_path or not file_size:
data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_SIZE_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = full_file_path.split('/')[0]
db_file2size.set(full_file_path, file_size)
if full_file_path not in db_user2files.lrange(login, 0, -1):
db_user2files.lpush(login, full_file_path)
source_node_ip = flask.request.environ.get('HTTP_X_REAL_IP', flask.request.remote_addr)
nodes_with_obsolete_files = db_file2nodes.lrange(full_file_path, 0, -1)
if source_node_ip in nodes_with_obsolete_files:
nodes_with_obsolete_files.remove(source_node_ip)
else:
db_node2files.lpush(source_node_ip, full_file_path)
db_file2nodes.lpush(full_file_path, source_node_ip)
for node_ip in nodes_with_obsolete_files:
res = request_node(node_ip, '/fdelete', {FULL_PATH_KEY: full_file_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fdelete")
db_node2files.lrem(node_ip, 0, full_file_path)
db_file2nodes.lrem(full_file_path, 0, node_ip)
replicate(full_file_path=full_file_path)
data = {MESSAGE_KEY: 'OK, uploaded.'}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def tell_naming_node_im_born():
debug_log("Wait 7 sec...")
sleep(7)
pub_ip = requests.request(
"GET", "http://169.254.169.254/latest/meta-data/public-ipv4").text
debug_log(f"Going to tell that I'm born, pub_ip = {pub_ip}")
request_node(NAMENODE_IP, '/new_node', {'pub': pub_ip})
def flask_save_replication():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
file_bytes = flask.request.form.get(key=FILE_BYTES, default=None, type=str)
if not full_file_path or not file_bytes:
data = {
MESSAGE_KEY:
f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_BYTES}`"
}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
file_path = get_path(full_file_path)
file_bytes = base64.decodebytes(file_bytes.encode())
try:
with open(file_path, 'wb') as f:
f.write(file_bytes)
except OSError as e:
debug_log(e.strerror)
data = {MESSAGE_KEY: "Error on the server"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {MESSAGE_KEY: "OK"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_init():
login = flask.request.form.get(key=LOGIN_KEY, default=None, type=str)
if not login:
data = {MESSAGE_KEY: f"Missing required parameters: `{LOGIN_KEY}`"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
user_folder = get_path(login)
if os.path.exists(user_folder):
# rm user's folder if exists
try:
shutil.rmtree(user_folder)
except OSError:
debug_log(f"Deletion of the directory {user_folder} failed")
data = {
MESSAGE_KEY:
"The folder is already exists. Couldn't delete it."
}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
# create user's folder
try:
os.mkdir(user_folder)
except OSError:
debug_log(f"Creation of the directory {user_folder} failed")
data = {MESSAGE_KEY: "Failed during folder creation."}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {MESSAGE_KEY: "Success"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def ping():
try:
st = statvfs(ROOT)
congestion = f'{(st.f_blocks - st.f_bavail) / st.f_blocks:.2f}'
except OSError as e:
debug_log(f"statvfs failed {e}")
congestion = 0
data = {CONGESTION_KEY: congestion}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_ddir():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
dir_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
force = flask.request.form.get(key=FORCE_KEY, default=None, type=str)
if not token or dir_path is None or force is None:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`, `{FORCE_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
dir_path = dir_path[:-1] if dir_path and dir_path[-1] == '/' else dir_path
if not dir_path:
data = {MESSAGE_KEY: "Can't delete the root directory"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_dir_path = os.path.join(login, dir_path)
if full_dir_path not in db_user2folders.lrange(login, 0, -1):
data = {MESSAGE_KEY: "The folder for deleting does not exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_FOUND)
inner_files_path_list = [full_path for full_path in db_user2files.lrange(login, 0, -1) if
full_path.startswith(full_dir_path + '/')]
inner_folders_path_list = [full_path for full_path in db_user2folders.lrange(login, 0, -1)
if full_path.startswith(full_dir_path + '/')]
if force == 'False' and (inner_files_path_list or inner_folders_path_list):
data = {MESSAGE_KEY: 'The directory contains files. `force=true` to delete'}
return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_ACCEPTABLE)
db_user2folders.lrem(login, 0, full_dir_path)
for inner_dir in inner_folders_path_list:
db_user2folders.lrem(login, 0, inner_dir)
nodes_ip = set()
for inner_file in inner_files_path_list:
db_file2size.delete(inner_file)
db_user2files.lrem(login, 0, inner_file)
for node_ip in db_file2nodes.lrange(inner_file, 0, -1):
db_file2nodes.lrem(inner_file, 0, node_ip)
db_node2files.lrem(node_ip, 0, inner_file)
nodes_ip.add(node_ip)
for node_ip in list(nodes_ip):
res = request_node(node_ip, '/ddir', {FULL_PATH_KEY: full_dir_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /ddir ({full_dir_path})")
data = {MESSAGE_KEY: 'Successfully deleted the directory'}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def __init__(self, DATABASE):
if os.path.exists(DATABASE):
self.db_path = DATABASE
self.connection = sqlite3.connect(self.db_path)
self.cursor = self.connection.cursor()
logger.debug_log("Database connection successful.",
logger.level.debug)
else:
logger.debug_log("Database connection failed.", logger.level.error)
def flask_new_node():
new_node_ip = flask.request.environ.get('HTTP_X_REAL_IP', flask.request.remote_addr)
pub_ip = flask.request.form.get(key='pub', default=None, type=str)
if new_node_ip in db_congestion.keys():
debug_log(f"!!!!!!!ERROR!!!!!!!! {new_node_ip} calls /new_node but it's already in the db")
else:
debug_log(f"{new_node_ip} calls /new_node and added to db. Pub_ip = {pub_ip}")
db_congestion.set(new_node_ip, 0)
db_pub.set(new_node_ip, pub_ip)
return flask.make_response(flask.jsonify({}), HTTPStatus.OK)
def _is_bucket_item_sane(key):
logger.debug_log('s3decryptor._is_bucket_item_sane start')
if key.size < _BUCKET_ITEM_MIN_SIZE or key.size > int(config['s3ObjectMaxSize']):
err = 'item not sane size: %d' % key.size
logger.error(err)
return False
logger.debug_log('s3decryptor._is_bucket_item_sane end')
return True
def _is_bucket_item_sane(key):
logger.debug_log('s3decryptor._is_bucket_item_sane start')
if key.size < _BUCKET_ITEM_MIN_SIZE or key.size > int(
config['s3ObjectMaxSize']):
err = 'item not sane size: %d' % key.size
logger.error(err)
return False
logger.debug_log('s3decryptor._is_bucket_item_sane end')
return True
def _analyze_diagnostic_info(diagnostic_info, reply_info):
'''
Determines what response should be sent based on `diagnostic_info` content.
Returns a list of response IDs.
Returns None if no response should be sent.
'''
logger.debug_log('_analyze_diagnostic_info: enter')
responses = None
AddressParts = collections.namedtuple('AddressParts',
['name', 'plus', 'domain'])
# Get our local email address parts
match = _address_splitter.match(config['emailUsername'])
local_email_parts = AddressParts(*match.groups())
# We'll break apart the "to" address (if applicable)
to_parts = AddressParts(None, None, None)
if reply_info.to:
match = _address_splitter.match(reply_info.to)
if not match:
# Someone is messing with us
raise ValueError('invalid email address: to-address match failed')
to_parts = AddressParts(*match.groups())
# TODO: more flexible rules, not so hard-coded
if utils.coalesce(
diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'isPlayStoreBuild']):
# No download links in Play Store email
responses = ['generic_info']
elif to_parts.name == local_email_parts.name and \
to_parts.domain == local_email_parts.domain:
# E.g., [email protected], [email protected]
responses = [
'download_new_version_links',
# Disabling attachment responses for now. Not sure if
# it's a good idea. Note that it needs to be tested.
# 'download_new_version_attachments',
]
elif to_parts.domain == local_email_parts.domain:
# E.g., *@psiphon.ca
responses = ['generic_info']
elif not reply_info.to and \
utils.coalesce(diagnostic_info, ('Metadata', 'platform'), utils.string_types):
# Windows S3 feedback
responses = ['download_new_version_links']
logger.debug_log('_analyze_diagnostic_info: exit: %s' % responses)
return responses
def _get_email_reply_info(autoresponder_info):
"""Returns None if no reply info found, otherwise an instance of _ReplyInfo.
Note that this function also validates the email address.
"""
logger.debug_log('_get_email_reply_info: enter')
email_info = autoresponder_info.get('email_info')
diagnostic_info = autoresponder_info.get('diagnostic_info')
reply_info = None
if email_info:
reply_info = _ReplyInfo(email_info.get('address'),
email_info.get('message_id'),
email_info.get('subject'),
email_info.get('to'))
elif utils.coalesce(diagnostic_info, 'EmailInfo', required_types=dict):
reply_info = _ReplyInfo(diagnostic_info['EmailInfo'].get('address'),
diagnostic_info['EmailInfo'].get('message_id'),
diagnostic_info['EmailInfo'].get('subject'),
diagnostic_info['EmailInfo'].get('to'))
elif utils.coalesce(diagnostic_info, 'Feedback', required_types=dict):
reply_info = _ReplyInfo(diagnostic_info['Feedback'].get('email'), None,
None, None)
if not reply_info:
logger.debug_log('_get_email_reply_info: no/bad reply_info, exiting')
return None
# Sometimes the recorded address looks like "<*****@*****.**>"
if reply_info.address:
reply_info.address = reply_info.address.strip('<>')
if not reply_info.address:
logger.debug_log(
'_get_email_reply_info: no/bad reply_info.address, exiting')
return None
if reply_info.to:
reply_info.to = reply_info.to.strip('<>')
validator = email_validator.EmailValidator(fix=True, lookup_dns='mx')
try:
fixed_address = validator.validate_or_raise(reply_info.address)
reply_info.address = fixed_address
except:
logger.debug_log(
'_get_email_reply_info: address validator raised, exiting')
return None
logger.debug_log('_get_email_reply_info: exit')
return reply_info
def _upgrade_old_object(yaml_docs):
'''
The diagnostic info stuff was released for Android before versioning was
added.
Returns the appropriately modified YAML dict.
'''
logger.debug_log('maildecryptor._upgrade_old_object start')
# The non-versioned YAML used multiple docs, so that's the main test
if len(yaml_docs) == 1:
return yaml_docs.pop()
# Our old YAML had '.' in some key names, which is illegal.
for path, val in utils.objwalk(yaml_docs):
if type(path[-1]) == str and path[-1].find('.') >= 0:
utils.rename_key_in_obj_at_path(yaml_docs,
path,
path[-1].replace('.', '__'))
# Our old YAML format was multiple YAML docs in a single string. We'll
# convert that to the new format.
obj = {}
# Old YAML had no Metadata section
metadata = {}
metadata['platform'] = 'android'
metadata['version'] = 1
metadata['id'] = binascii.hexlify(os.urandom(8))
obj['Metadata'] = metadata
idx = 0
obj['SystemInformation'] = yaml_docs[idx]
idx += 1
obj['ServerResponseCheck'] = yaml_docs[idx]
idx += 1
# The presence of DiagnosticHistory was optional once upon a time
if len(yaml_docs) > 3:
obj['DiagnosticHistory'] = yaml_docs[idx]
idx += 1
else:
obj['DiagnosticHistory'] = []
obj['StatusHistory'] = yaml_docs[idx]
idx += 1
logger.debug_log('maildecryptor._upgrade_old_object end')
return obj
def _upgrade_old_object(yaml_docs):
'''
The diagnostic info stuff was released for Android before versioning was
added.
Returns the appropriately modified YAML dict.
'''
logger.debug_log('maildecryptor._upgrade_old_object start')
# The non-versioned YAML used multiple docs, so that's the main test
if len(yaml_docs) == 1:
return yaml_docs.pop()
# Our old YAML had '.' in some key names, which is illegal.
for path, val in utils.objwalk(yaml_docs):
if isinstance(path[-1],
utils.string_types) and path[-1].find('.') >= 0:
utils.rename_key_in_obj_at_path(yaml_docs, path,
path[-1].replace('.', '__'))
# Our old YAML format was multiple YAML docs in a single string. We'll
# convert that to the new format.
obj = {}
# Old YAML had no Metadata section
metadata = {}
metadata['platform'] = 'android'
metadata['version'] = 1
metadata['id'] = binascii.hexlify(os.urandom(8))
obj['Metadata'] = metadata
idx = 0
obj['SystemInformation'] = yaml_docs[idx]
idx += 1
obj['ServerResponseCheck'] = yaml_docs[idx]
idx += 1
# The presence of DiagnosticHistory was optional once upon a time
if len(yaml_docs) > 3:
obj['DiagnosticHistory'] = yaml_docs[idx]
idx += 1
else:
obj['DiagnosticHistory'] = []
obj['StatusHistory'] = yaml_docs[idx]
idx += 1
logger.debug_log('maildecryptor._upgrade_old_object end')
return obj
def _html_to_text(html):
'''
Convert given `html` to plain text.
'''
logger.debug_log('_html_to_text: enter')
h2t = html2text.HTML2Text()
h2t.body_width = 0
txt = h2t.handle(html)
logger.debug_log('_html_to_text: returning text length %d' % len(txt))
return txt
def flask_fmove():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
file_destination_path = flask.request.form.get(key=PATH_DESTINATION_KEY, default=None, type=str)
if not token or not file_path or file_destination_path is None:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`, `{PATH_DESTINATION_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
if not os.path.basename(file_destination_path):
data = {MESSAGE_KEY: "The destination file name is empty"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_file_path = os.path.join(login, file_path)
full_file_destination_path = os.path.join(login, file_destination_path)
if full_file_path not in db_user2files.lrange(login, 0, -1):
data = {MESSAGE_KEY: "The source file doesn't exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
if os.path.dirname(full_file_destination_path) not in db_user2folders.lrange(login, 0, -1):
data = {MESSAGE_KEY: "Can't move the file. Destination folder doesn't exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
for node_ip in db_file2nodes.lrange(full_file_path, 0, -1):
res = request_node(node_ip, '/fmove', {FULL_PATH_KEY: full_file_path,
FULL_PATH_DESTINATION_KEY: full_file_destination_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fmove")
else:
db_node2files.lrem(node_ip, 0, full_file_path)
db_node2files.lpush(node_ip, full_file_destination_path)
db_file2nodes.lrem(full_file_path, 0, node_ip)
db_file2nodes.lpush(full_file_destination_path, node_ip)
db_user2files.lrem(login, 0, full_file_path)
db_user2files.lpush(login, full_file_destination_path)
db_file2size.set(full_file_destination_path, db_file2size.get(full_file_path))
db_file2size.delete(full_file_path)
data = {MESSAGE_KEY: f"Successfully moved the file"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def _analyze_diagnostic_info(diagnostic_info, reply_info):
'''
Determines what response should be sent based on `diagnostic_info` content.
Returns a list of response IDs.
Returns None if no response should be sent.
'''
logger.debug_log('_analyze_diagnostic_info: enter')
responses = None
AddressParts = collections.namedtuple('AddressParts', ['name', 'plus', 'domain'])
# Get our local email address parts
match = _address_splitter.match(config['emailUsername'])
local_email_parts = AddressParts(*match.groups())
# We'll break apart the "to" address (if applicable)
to_parts = AddressParts(None, None, None)
if reply_info.to:
match = _address_splitter.match(reply_info.to)
if not match:
# Someone is messing with us
raise ValueError('invalid email address: to-address match failed')
to_parts = AddressParts(*match.groups())
# TODO: more flexible rules, not so hard-coded
if utils.coalesce(diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'isPlayStoreBuild']):
# No download links in Play Store email
responses = ['generic_info']
elif to_parts.name == local_email_parts.name and \
to_parts.domain == local_email_parts.domain:
# E.g., [email protected], [email protected]
responses = ['download_new_version_links',
# Disabling attachment responses for now. Not sure if
# it's a good idea. Note that it needs to be tested.
# 'download_new_version_attachments',
]
elif to_parts.domain == local_email_parts.domain:
# E.g., *@psiphon.ca
responses = ['generic_info']
elif not reply_info.to and \
utils.coalesce(diagnostic_info, ('Metadata', 'platform'), utils.string_types):
# Windows S3 feedback
responses = ['download_new_version_links']
logger.debug_log('_analyze_diagnostic_info: exit: %s' % responses)
return responses
def _html_to_text(html):
'''
Convert given `html` to plain text.
'''
logger.debug_log('_html_to_text: enter')
h2t = html2text.HTML2Text()
h2t.body_width = 0
txt = h2t.handle(html)
logger.debug_log('_html_to_text: returning text length %d' % len(txt))
return txt
def _get_email_reply_info(autoresponder_info):
"""Returns None if no reply info found, otherwise an instance of _ReplyInfo.
Note that this function also validates the email address.
"""
logger.debug_log('_get_email_reply_info: enter')
email_info = autoresponder_info.get('email_info')
diagnostic_info = autoresponder_info.get('diagnostic_info')
reply_info = None
if email_info:
reply_info = _ReplyInfo(email_info.get('address'),
email_info.get('message_id'),
email_info.get('subject'),
email_info.get('to'))
elif utils.coalesce(diagnostic_info, 'EmailInfo', required_types=dict):
reply_info = _ReplyInfo(diagnostic_info['EmailInfo'].get('address'),
diagnostic_info['EmailInfo'].get('message_id'),
diagnostic_info['EmailInfo'].get('subject'),
diagnostic_info['EmailInfo'].get('to'))
elif utils.coalesce(diagnostic_info, 'Feedback', required_types=dict):
reply_info = _ReplyInfo(diagnostic_info['Feedback'].get('email'),
None, None, None)
if not reply_info:
logger.debug_log('_get_email_reply_info: no/bad reply_info, exiting')
return None
# Sometimes the recorded address looks like "<*****@*****.**>"
if reply_info.address:
reply_info.address = reply_info.address.strip('<>')
if not reply_info.address:
logger.debug_log('_get_email_reply_info: no/bad reply_info.address, exiting')
return None
if reply_info.to:
reply_info.to = reply_info.to.strip('<>')
validator = email_validator.EmailValidator(fix=True, lookup_dns='mx')
try:
fixed_address = validator.validate_or_raise(reply_info.address)
reply_info.address = fixed_address
except:
logger.debug_log('_get_email_reply_info: address validator raised, exiting')
return None
logger.debug_log('_get_email_reply_info: exit')
return reply_info
def _get_lang_id_from_diagnostic_info(diagnostic_info):
'''
Derive the lanague from `diagnostic_info` and return its ID/code.
Returns `None` if the language can't be determined.
'''
logger.debug_log('_get_lang_id_from_diagnostic_info: enter')
lang_id = None
# There can be different -- and better or worse -- ways of determining the
# user's language depending on platform, the type of feedback, and so on.
# Windows, with feedback message
lang_id = lang_id or utils.coalesce(
diagnostic_info, ['Feedback', 'Message', 'text_lang_code'],
required_types=utils.string_types)
if lang_id and lang_id.find('INDETERMINATE') >= 0:
lang_id = None
# All Windows feedback
lang_id = lang_id or utils.coalesce(diagnostic_info, [
'DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LocaleInfo',
'language_code'
],
required_types=utils.string_types)
# All Windows feedback
lang_id = lang_id or utils.coalesce(diagnostic_info, [
'DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LanguageInfo',
'language_code'
],
required_types=utils.string_types)
# Android, from email
lang_id = lang_id or utils.coalesce(
diagnostic_info, ['EmailInfo', 'body', 'text_lang_code'],
required_types=utils.string_types)
if lang_id and lang_id.find('INDETERMINATE') >= 0:
lang_id = None
# Android, from system language
lang_id = lang_id or utils.coalesce(
diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'language'],
required_types=utils.string_types)
logger.debug_log(
'_get_lang_id_from_diagnostic_info: exiting with lang_id=%s' % lang_id)
return lang_id
def ping_nodes():
for node_ip in db_congestion.keys():
if node_ip not in non_responsive_count:
non_responsive_count[node_ip] = 0
res = request_node(node_ip, '/ping', {})
res = get_dict_from_response(res)
if res is None:
non_responsive_count[node_ip] += 1
debug_log(f"Node {node_ip} did not response {non_responsive_count[node_ip]} times")
if non_responsive_count[node_ip] == MAX_PING_TRIES:
remove_node(node_ip)
del non_responsive_count[node_ip]
else:
debug_log(f"Pinging node {node_ip} - {res}")
db_congestion.set(node_ip, res[CONGESTION_KEY])
def _autoresponder_record_iter():
logger.debug_log('_autoresponder_record_iter: enter')
while True:
for rec in datastore.get_autoresponder_iterator():
logger.debug_log('_autoresponder_record_iter: %s' % repr(rec))
if rec.get('diagnostic_info_record_id'):
rec['diagnostic_info'] = datastore.find_diagnostic_info(rec.get('diagnostic_info_record_id'))
logger.debug_log('_autoresponder_record_iter: yielding rec: %s' % rec['_id'])
yield rec
logger.debug_log('_diagnostic_record_iter: sleeping')
time.sleep(_SLEEP_TIME_SECS)
logger.debug_log('_autoresponder_record_iter: exit')
def _autoresponder_record_iter():
logger.debug_log('_autoresponder_record_iter: enter')
while True:
for rec in datastore.get_autoresponder_iterator():
logger.debug_log('_autoresponder_record_iter: %s' % repr(rec))
if rec.get('diagnostic_info_record_id'):
rec['diagnostic_info'] = datastore.find_diagnostic_info(rec.get('diagnostic_info_record_id'))
logger.debug_log('_autoresponder_record_iter: yielding rec: %s' % rec['_id'])
yield rec
logger.debug_log('_diagnostic_record_iter: sleeping')
time.sleep(_SLEEP_TIME_SECS)
logger.debug_log('_autoresponder_record_iter: exit')
def change_active_database(self, PATH):
if not os.path.exists(PATH):
return
print("Catastrophic failure: Database path non-existent")
# check if a current connection is already open
if self.connection:
self.connection.close()
# TODO(jamie): Implement backup of current database file
# create connection to new database
self.connection = sqlite3.connect(PATH)
self.cursor = self.connection.cursor()
if not self.connection:
logger.debug_log("Database connection failed.", logger.level.error)
else:
logger.debug_log("New database [{}] opened".format(PATH),
logger.level.debug)
def _email_diagnostic_info_records_iterator():
'''
Generator for obtaining email_diagnostic_info records.
'''
while True:
# Every hour or so, pymongo throws a CursorNotFound error. This is due to the
# the cursor being idle more than 10 minutes. It generally doesn't take us that
# long to send an email, but each time a request to mongodb is made, there's a
# batch of records returned. Processing all of the records in that batch _does_
# take more than 10 minutes. So when we try to get another batch the cursor is dead.
# To address this, we could decrease the batch size or increase the cursor lifetime.
# More details: https://stackoverflow.com/a/24200795/729729
logger.debug_log('fresh cursor')
for rec in datastore.get_email_diagnostic_info_iterator():
yield rec
time.sleep(_SLEEP_TIME_SECS)
def _get_lang_id_from_diagnostic_info(diagnostic_info):
'''
Derive the lanague from `diagnostic_info` and return its ID/code.
Returns `None` if the language can't be determined.
'''
logger.debug_log('_get_lang_id_from_diagnostic_info: enter')
lang_id = None
# There can be different -- and better or worse -- ways of determining the
# user's language depending on platform, the type of feedback, and so on.
# Windows, with feedback message
lang_id = lang_id or utils.coalesce(diagnostic_info,
['Feedback', 'Message', 'text_lang_code'],
required_types=utils.string_types)
if lang_id and lang_id.find('INDETERMINATE') >= 0:
lang_id = None
# All Windows feedback
lang_id = lang_id or utils.coalesce(diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LocaleInfo', 'language_code'],
required_types=utils.string_types)
# All Windows feedback
lang_id = lang_id or utils.coalesce(diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LanguageInfo', 'language_code'],
required_types=utils.string_types)
# Android, from email
lang_id = lang_id or utils.coalesce(diagnostic_info,
['EmailInfo', 'body', 'text_lang_code'],
required_types=utils.string_types)
if lang_id and lang_id.find('INDETERMINATE') >= 0:
lang_id = None
# Android, from system language
lang_id = lang_id or utils.coalesce(diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'language'],
required_types=utils.string_types)
logger.debug_log('_get_lang_id_from_diagnostic_info: exiting with lang_id=%s' % lang_id)
return lang_id
def flask_fwrite():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
file_bytes = flask.request.form.get(key=FILE_BYTES, default=None, type=str)
if not full_file_path or file_bytes is None:
data = {
MESSAGE_KEY:
f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_BYTES}`"
}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
file_path = get_path(full_file_path)
file_bytes = base64.decodebytes(file_bytes.encode())
try:
all_dirs = full_file_path.split("/")[:-1]
dir_path = ROOT
for directory in all_dirs:
dir_path = os.path.join(dir_path, directory)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# writeing to file
with open(file_path, 'wb') as f:
f.write(file_bytes)
except OSError as e:
debug_log(e.strerror)
data = {MESSAGE_KEY: "Error on the server"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {FULL_PATH_KEY: full_file_path, FILE_SIZE_KEY: len(file_bytes)}
res = request_node(NAMENODE_IP, '/uploaded', data)
if res.status != HTTPStatus.OK:
data = {MESSAGE_KEY: get_dict_from_response(res)[MESSAGE_KEY]}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
data = {MESSAGE_KEY: "OK"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_ddir():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
if not full_file_path:
data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
file_path = get_path(full_file_path)
try:
shutil.rmtree(file_path)
except OSError:
debug_log(f"Delete the directory {file_path} failed")
data = {MESSAGE_KEY: "Failed during dir delete."}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {MESSAGE_KEY: "Success"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def _get_email_info(msg):
logger.debug_log('maildecryptor._get_email_info start')
subject_translation = translation.translate(config['googleApiServers'],
config['googleApiKey'],
msg['subject'])
subject = dict(text=msg['subject'],
text_lang_code=subject_translation[0],
text_lang_name=subject_translation[1],
text_translated=subject_translation[2])
body_translation = translation.translate(config['googleApiServers'],
config['googleApiKey'],
msg['body'])
body = dict(text=msg['body'],
text_lang_code=body_translation[0],
text_lang_name=body_translation[1],
text_translated=body_translation[2],
html=msg['html'])
raw_address = msg['from'] or msg['msgobj'].get('Return-Path')
stripped_address = None
if raw_address:
match = _email_stripper_regex.match(raw_address)
if not match:
logger.error('when stripping email address failed to match: %s' %
str(raw_address))
return None
stripped_address = match.group(2)
email_info = dict(address=stripped_address,
to=msg['to'],
message_id=msg['msgobj']['Message-ID'],
subject=subject,
body=body)
logger.debug_log('maildecryptor._get_email_info end')
return email_info
def _get_email_info(msg):
logger.debug_log('maildecryptor._get_email_info start')
subject_translation = translation.translate(config['googleApiServers'],
config['googleApiKey'],
msg['subject'])
subject = dict(text=msg['subject'],
text_lang_code=subject_translation[0],
text_lang_name=subject_translation[1],
text_translated=subject_translation[2])
body_translation = translation.translate(config['googleApiServers'],
config['googleApiKey'],
msg['body'])
body = dict(text=msg['body'],
text_lang_code=body_translation[0],
text_lang_name=body_translation[1],
text_translated=body_translation[2],
html=msg['html'])
raw_address = msg['from'] or msg['msgobj'].get('Return-Path')
stripped_address = None
if raw_address:
match = _email_stripper_regex.match(raw_address)
if not match:
logger.error('when stripping email address failed to match: %s' % str(raw_address))
return None
stripped_address = match.group(2)
email_info = dict(address=stripped_address,
to=msg['to'],
message_id=msg['msgobj']['Message-ID'],
subject=subject,
body=body)
logger.debug_log('maildecryptor._get_email_info end')
return email_info
def _render_email(data):
logger.debug_log('_render_email: enter')
global _template
if not _template:
_template = Template(filename='templates/feedback_response.mako',
default_filters=['unicode', 'h', 'decode.utf8'],
input_encoding='utf-8', output_encoding='utf-8',
lookup=TemplateLookup(directories=['.']))
logger.debug_log('_render_email: template loaded')
rendered = _template.render(data=data)
# CSS in email HTML must be inline
rendered = pynliner.fromString(rendered)
logger.debug_log('_render_email: exiting with len(rendered)=%d' % len(rendered))
return rendered
def go():
logger.debug_log('s3decryptor.go: start')
s3_conn = S3Connection(config['aws_access_key_id'], config['aws_secret_access_key'])
bucket = s3_conn.get_bucket(config['s3_bucket_name'])
# Note that `_bucket_iterator` throttles itself if/when there are no
# available objects in the bucket.
for encrypted_info_json in _bucket_iterator(bucket):
logger.debug_log('s3decryptor.go: processing item')
# In theory, all bucket items should be usable by us, but there's
# always the possibility that a user (or attacker) is messing with us.
try:
encrypted_info = json.loads(encrypted_info_json)
diagnostic_info = decryptor.decrypt(encrypted_info)
diagnostic_info = diagnostic_info.strip()
# HACK: PyYaml only supports YAML 1.1, which is not a true superset
# of JSON. Therefore it can (and does) throw errors on some Android
# feedback. We will try to load using JSON first.
# TODO: Get rid of all YAML feedback and remove it from here.
try:
diagnostic_info = json.loads(diagnostic_info)
logger.debug_log('s3decryptor.go: loaded JSON')
except:
diagnostic_info = yaml.safe_load(diagnostic_info)
logger.debug_log('s3decryptor.go: loaded YAML')
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
# Store the diagnostic info
record_id = datastore.insert_diagnostic_info(diagnostic_info)
if _should_email_data(diagnostic_info):
logger.debug_log('s3decryptor.go: should email')
# Record in the DB that the diagnostic info should be emailed
datastore.insert_email_diagnostic_info(record_id, None, None)
# Store an autoresponder entry for this diagnostic info
datastore.insert_autoresponder_entry(None, record_id)
logger.log('decrypted diagnostic data')
except decryptor.DecryptorException as e:
logger.exception()
logger.error(str(e))
try:
# Something bad happened while decrypting. Report it via email.
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'S3Decryptor: bad object',
encrypted_info_json,
None) # no html body
except smtplib.SMTPException as e:
logger.exception()
logger.error(str(e))
# yaml.constructor.ConstructorError was being thown when a YAML value
# consisted of just string "=". Probably due to this PyYAML bug:
# http://pyyaml.org/ticket/140
except (ValueError, TypeError, yaml.constructor.ConstructorError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
logger.debug_log('s3decryptor.go: end')
def go():
logger.debug_log('go: enter')
# Note that `_diagnostic_record_iter` throttles itself if/when there are
# no records to process.
for autoresponder_info in _autoresponder_record_iter():
diagnostic_info = autoresponder_info.get('diagnostic_info')
email_info = autoresponder_info.get('email_info')
logger.debug_log('go: got autoresponder record')
# For now we don't do any interesting processing/analysis and we just
# respond to every feedback with an exhortation to upgrade.
reply_info = _get_email_reply_info(autoresponder_info)
if not reply_info or not reply_info.address:
# If we don't have any reply info, we can't reply
logger.debug_log('go: no reply_info or address')
continue
# Check if the address is blacklisted
if _check_and_add_address_blacklist(reply_info.address):
logger.debug_log('go: blacklisted')
continue
responses = _analyze_diagnostic_info(diagnostic_info, reply_info)
if not responses:
logger.debug_log('go: no response')
continue
logger.log('Sending feedback response')
for response_id in responses:
response_content = _get_response_content(response_id, diagnostic_info)
if not response_content:
logger.debug_log('go: no response_content')
continue
# The original diagnostic info may have originated from an email,
# in which case we have a subject to reply to. Or it may have have
# originated from an uploaded data package, in which case we need
# set our own subject.
if type(reply_info.subject) is dict:
subject = u'Re: %s' % reply_info.subject.get('text', '')
else:
subject = response_content['subject']
try:
sender.send_response(reply_info.address,
config['reponseEmailAddress'],
subject,
response_content['body_text'],
response_content['body_html'],
reply_info.message_id,
response_content['attachments'])
except Exception as e:
logger.debug_log('go: send_response excepted')
logger.exception()
logger.error(str(e))
def _get_response_content(response_id, diagnostic_info):
"""Gets the response for the given response_id. diagnostic_info will be
used to determine language and some content, but may be None.
Returns a dict of the form:
{
subject: <subject text>,
body_text: <body text>,
body_html: <rich html body>,
attachments: <attachments list, may be None>
}
Returns None if no response content can be derived.
"""
logger.debug_log('_get_response_content: enter')
sponsor_name = utils.coalesce(diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'SPONSOR_ID'],
required_types=utils.string_types)
prop_channel_name = utils.coalesce(diagnostic_info,
['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'PROPAGATION_CHANNEL_ID'],
required_types=utils.string_types)
# Use default values if we couldn't get good user-specific values
sponsor_name = sponsor_name or config['defaultSponsorName']
prop_channel_name = prop_channel_name or config['defaultPropagationChannelName']
lang_id = _get_lang_id_from_diagnostic_info(diagnostic_info)
# lang_id may be None, if the language could not be determined
# Read in all translations HTML
response_translations = []
for root, _, files in os.walk(_RESPONSES_DIR):
for name in files:
lang, ext = os.path.splitext(name)
if ext != '.html':
continue
if lang == 'master':
lang = 'en'
with open(os.path.join(root, name)) as translation_file:
translation = translation_file.read().decode('utf-8')
# Strip leading and trailing whitespace so that we don't get extra
# text elements in our BeautifulSoup
translation = translation.strip()
response_translations.append((lang, translation.strip()))
# Reorder the array according to the detected language and _TOP_LANGS
def lang_sorter(item):
lang, _ = item
rank = 999
try:
if lang == lang_id:
rank = -1
else:
rank = _TOP_LANGS.index(lang)
except ValueError:
pass
return rank
response_translations.sort(key=lang_sorter)
# Gather the info we'll need for formatting the email
bucketname, email_address = psi_ops_helpers.get_bucket_name_and_email_address(sponsor_name, prop_channel_name)
# Use default values if we couldn't get good user-specific values
if not bucketname or not email_address:
default_bucketname, default_email_address = \
psi_ops_helpers.get_bucket_name_and_email_address(config['defaultSponsorName'],
config['defaultPropagationChannelName'])
bucketname = bucketname or default_bucketname
email_address = email_address or default_email_address
# If, despite our best efforts, we still don't have a bucketname and
# email address, just bail.
if not bucketname or not email_address:
logger.debug_log('_get_response_content: exiting due to no bucketname or address')
return None
# Collect the translations of the specific response we're sending
subject = None
bodies = []
for lang_id, html in response_translations:
soup = BeautifulSoup(html)
if not subject:
subject = soup.find(id='default_response_subject')
if subject:
# Strip outer element
subject = u''.join(unicode(elem) for elem in subject.contents).strip()
body = soup.find(id=response_id)
if body:
# Strip outer element
body = u''.join(unicode(elem) for elem in body.contents).strip()
# The user might be using a language for which there isn't a
# download page. Fall back to English if that's the case.
home_page_url = psi_ops_helpers.get_s3_bucket_home_page_url(
bucketname,
lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')
download_page_url = psi_ops_helpers.get_s3_bucket_download_page_url(
bucketname,
lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')
faq_page_url = psi_ops_helpers.get_s3_bucket_faq_url(
bucketname,
lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')
# We're using numbers rather than more readable names here because
# they're less likely to be accidentally modified by translators
# (we think).
format_dict = {
'0': email_address,
'1': download_page_url,
'2': home_page_url,
'3': faq_page_url
}
body = unicode(body) % format_dict
bodies.append(body)
# Render the email body from the Mako template
body_html = _render_email({
'lang_id': lang_id,
'response_id': response_id,
'responses': bodies
})
# Get attachments.
# This depends on which response we're returning.
attachments = None
if response_id == 'download_new_version_links':
pass
elif response_id == 'download_new_version_attachments':
fp_windows = aws_helpers.get_s3_attachment('attachments',
bucketname,
psi_ops_helpers.DOWNLOAD_SITE_WINDOWS_BUILD_FILENAME)
fp_android = aws_helpers.get_s3_attachment('attachments',
bucketname,
psi_ops_helpers.DOWNLOAD_SITE_ANDROID_BUILD_FILENAME)
attachments = [(fp_windows, psi_ops_helpers.EMAIL_RESPONDER_WINDOWS_ATTACHMENT_FILENAME),
(fp_android, psi_ops_helpers.EMAIL_RESPONDER_ANDROID_ATTACHMENT_FILENAME)]
else:
pass
logger.debug_log('_get_response_content: exit')
return {
'subject': subject,
'body_text': _html_to_text(body_html),
'body_html': body_html,
'attachments': attachments
}
import logger
log1 = logger.debug_log()
log2 = logger.debug_log()
log1.info("log1 log")
log2.info("log2 log")
def go():
logger.debug_log('maildecryptor.go start')
emailgetter = EmailGetter(config['popServer'],
config['popPort'],
config['emailUsername'],
config['emailPassword'])
# Retrieve and process email.
# Note that `emailgetter.get` throttles itself if/when there are no emails
# immediately available.
for msg in emailgetter.get():
logger.debug_log('maildecryptor.go: msg has %d attachments' % len(msg['attachments']))
diagnostic_info = None
#
# First try to process attachments.
#
for attachment in msg['attachments']:
# Not all attachments will be in our format, so expect exceptions.
try:
encrypted_info = attachment.getvalue()
encrypted_info = json.loads(encrypted_info)
diagnostic_info = decryptor.decrypt(encrypted_info)
diagnostic_info = diagnostic_info.strip()
diagnostic_info = _load_yaml(diagnostic_info)
# Modifies diagnostic_info
utils.convert_psinet_values(config, diagnostic_info)
if not utils.is_diagnostic_info_sane(diagnostic_info):
# Something is wrong. Skip and continue.
continue
# Modifies diagnostic_info
datatransformer.transform(diagnostic_info)
logger.log('email attachment decrypted')
break
except decryptor.DecryptorException as e:
# Something bad happened while decrypting. Report it via email.
logger.exception()
try:
sender.send(config['decryptedEmailRecipient'],
config['emailUsername'],
u'Re: %s' % (msg['subject'] or ''),
'Decrypt failed: %s' % e,
msg['msgobj']['Message-ID'])
except smtplib.SMTPException as e:
# Something went wrong with the sending of the response. Log it.
logger.exception()
logger.error(str(e))
except (ValueError, TypeError) as e:
# Try the next attachment/message
logger.exception()
logger.error(str(e))
#
# Store what info we have
#
email_info = _get_email_info(msg)
diagnostic_info_record_id = None
if diagnostic_info:
# Add the user's email information to diagnostic_info.
# This will allow us to later auto-respond, or act as a
# remailer between the user and the Psiphon support team.
diagnostic_info['EmailInfo'] = email_info
# Store the diagnostic info
diagnostic_info_record_id = datastore.insert_diagnostic_info(diagnostic_info)
# Store the association between the diagnostic info and the email
datastore.insert_email_diagnostic_info(diagnostic_info_record_id,
msg['msgobj']['Message-ID'],
msg['subject'])
# Store autoresponder info regardless of whether there was a diagnostic info
datastore.insert_autoresponder_entry(email_info, diagnostic_info_record_id)
logger.debug_log('maildecryptor.go end')
