def _load_yaml(yaml_string): # TODO: Rip the backwards-compatibility out of this at some later date. # JSON is supposed to be a subset of YAML: en.wikipedia.org/wiki/YAML#JSON # So we switched the client-side encoding of the diagnostic data from YAML # to JSON (and got huge performance improvements), and we were still able # to decode it the same way. But... it turns out that, at least in Python's # YAML implementation, there is some JSON that's not valid YAML. I.e.: # >>> x = json.loads('{"key": "hi\\/there"}') # ... print x # {u'key': u'hi/there'} # >>> x = yaml.load('{"key": "hi\\/there"}') # ... print x # <big stack trace> # found unknown escape character '/' # in "<string>", line 1, column 13: # {"key": "hi\/there"} # # So we're going to try loading `yaml_string` as JSON first, and then fall # back to YAML if that fails. logger.debug_log('maildecryptor._load_yaml start') try: obj = json.loads(yaml_string) except: yaml_docs = [] for yaml_doc in yaml.safe_load_all(yaml_string): yaml_docs.append(yaml_doc) obj = _upgrade_old_object(yaml_docs) logger.debug_log('maildecryptor._load_yaml end') return obj
def go(): ''' Spawns the worker subprocesses and sends data to them. ''' logger.debug_log('go: start') # Set up the multiprocessing worker_manager = multiprocessing.Manager() # We only want to pull items out of S3 as we process them, so the queue needs to be # limited to the number of worker processes. work_queue = worker_manager.Queue(maxsize=config['numProcesses']) # Spin up the workers worker_pool = multiprocessing.Pool(processes=config['numProcesses']) [ worker_pool.apply_async(_process_work_items, (work_queue, )) for i in range(config['numProcesses']) ] # Retrieve and process email-to-diagnostic-info records. # Note that `_email_diagnostic_info_records` throttles itself if/when # there are no records immediately available. for email_diagnostic_info in _email_diagnostic_info_records_iterator(): if terminate: logger.debug_log('go: got terminate; closing work_queue') work_queue.close() break logger.debug_log('go: enqueueing work item') # This blocks if the queue is full work_queue.put(email_diagnostic_info) logger.debug_log('go: enqueued work item') logger.debug_log('go: done')
def flask_fdelete(): token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str) file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str) if not token or not file_path: data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) login = decode_auth_token(token) if not login or (type(login) == str and login not in db_auth.keys()): data = {MESSAGE_KEY: "The token is invalid or has expired"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) full_file_path = os.path.join(login, file_path) exists = full_file_path in db_user2files.lrange(login, 0, -1) if not exists: data = {MESSAGE_KEY: "The file for deleting does not exist"} return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_FOUND) for node_ip in db_file2nodes.lrange(full_file_path, 0, -1): res = request_node(node_ip, '/fdelete', {FULL_PATH_KEY: full_file_path}) res = get_dict_from_response(res) if res is None: debug_log(f"Node {node_ip} did not response on /fdelete") db_node2files.lrem(node_ip, 0, full_file_path) db_user2files.lrem(login, 0, full_file_path) db_file2nodes.delete(full_file_path) db_file2size.delete(full_file_path) data = {MESSAGE_KEY: f"Successfully deleted the file"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def request_node(ip, url, data): try: res = https_client.request('POST', f"http://{ip}{url}", fields=data) return res except Exception as e: debug_log(f"Requesting node failed. Error: '{e}'") return None
def _check_and_add_address_blacklist(address): ''' Returns True if the address is blacklisted, otherwise inserts it in the DB and returns False. ''' logger.debug_log('_check_and_add_address_blacklist: enter') # We need to normalize, otherwise we could get fooled by the fact that # "*****@*****.**" is the same as "*****@*****.**". # We're going to be fairly draconian and normalize down to just alpha-numerics. match = _address_splitter.match(address) if not match: # Someone is messing with us raise ValueError('invalid email address: match failed') # Discard the '+'' part name, _, domain = match.groups() # Get rid of non-alphanumerics name = _address_name_normalize_regex.sub('', name) if not name: # Someone is messing with us raise ValueError('invalid email address: name part is invalid') normalized_address = '%s@%s' % (name, domain) blacklisted = datastore.check_and_add_response_address_blacklist(normalized_address) logger.debug_log('_check_and_add_address_blacklist: exiting with blacklisted=%s' % blacklisted) return blacklisted
def _bucket_iterator(bucket): logger.debug_log('s3decryptor._bucket_iterator start') while True: for key in bucket.list(): logger.debug_log('s3decryptor._bucket_iterator: %s' % key) contents = None # Do basic sanity checks before trying to download the object if _is_bucket_item_sane(key): logger.debug_log( 's3decryptor._bucket_iterator: good item found, yielding') contents = key.get_contents_as_string() # Make sure to delete the key *before* proceeding, so we don't # try to re-process if there's an error. bucket.delete_key(key) if contents: yield contents logger.debug_log( 's3decryptor._bucket_iterator: no item found, sleeping') time.sleep(_SLEEP_TIME_SECS) logger.debug_log('s3decryptor._bucket_iterator end')
def _bucket_iterator(bucket): logger.debug_log('s3decryptor._bucket_iterator start') while True: for key in bucket.list(): logger.debug_log('s3decryptor._bucket_iterator: %s' % key) contents = None # Do basic sanity checks before trying to download the object if _is_bucket_item_sane(key): logger.debug_log('s3decryptor._bucket_iterator: good item found, yielding') contents = key.get_contents_as_string() # Make sure to delete the key *before* proceeding, so we don't # try to re-process if there's an error. bucket.delete_key(key) if contents: yield contents logger.debug_log('s3decryptor._bucket_iterator: no item found, sleeping') time.sleep(_SLEEP_TIME_SECS) logger.debug_log('s3decryptor._bucket_iterator end')
def flask_fcreate(): full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str) if not full_file_path: data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) # create new empty file try: # create all subdirs all_dirs = full_file_path.split("/")[:-1] dir_path = ROOT for directory in all_dirs: dir_path = os.path.join(dir_path, directory) if not os.path.exists(dir_path): os.mkdir(dir_path) # create new file there with open(get_path(full_file_path), "w"): pass except OSError: debug_log(f"Creation of the file {full_file_path} failed") data = {MESSAGE_KEY: "Failed during file creation."} return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) data = {MESSAGE_KEY: "Success"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_replicate(): full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str) target_node_ip = flask.request.form.get(key=NODE_IP_KEY, default=None, type=str) if not full_file_path or not target_node_ip: data = { MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`, `{NODE_IP_KEY}`" } return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) file_path = get_path(full_file_path) try: with open(file_path, "rb") as f: file_data = str(base64.encodebytes(f.read()), 'utf-8') except OSError as e: debug_log(e.strerror) data = {MESSAGE_KEY: "Error on storage server"} return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) data = {FULL_PATH_KEY: full_file_path, FILE_BYTES: file_data} request_node(target_node_ip, '/save_replication', data) return flask.make_response(flask.jsonify({}), HTTPStatus.OK)
def flask_uploaded(): full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str) file_size = flask.request.form.get(key=FILE_SIZE_KEY, default=None, type=str) if not full_file_path or not file_size: data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_SIZE_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) login = full_file_path.split('/')[0] db_file2size.set(full_file_path, file_size) if full_file_path not in db_user2files.lrange(login, 0, -1): db_user2files.lpush(login, full_file_path) source_node_ip = flask.request.environ.get('HTTP_X_REAL_IP', flask.request.remote_addr) nodes_with_obsolete_files = db_file2nodes.lrange(full_file_path, 0, -1) if source_node_ip in nodes_with_obsolete_files: nodes_with_obsolete_files.remove(source_node_ip) else: db_node2files.lpush(source_node_ip, full_file_path) db_file2nodes.lpush(full_file_path, source_node_ip) for node_ip in nodes_with_obsolete_files: res = request_node(node_ip, '/fdelete', {FULL_PATH_KEY: full_file_path}) res = get_dict_from_response(res) if res is None: debug_log(f"Node {node_ip} did not response on /fdelete") db_node2files.lrem(node_ip, 0, full_file_path) db_file2nodes.lrem(full_file_path, 0, node_ip) replicate(full_file_path=full_file_path) data = {MESSAGE_KEY: 'OK, uploaded.'} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def tell_naming_node_im_born(): debug_log("Wait 7 sec...") sleep(7) pub_ip = requests.request( "GET", "http://169.254.169.254/latest/meta-data/public-ipv4").text debug_log(f"Going to tell that I'm born, pub_ip = {pub_ip}") request_node(NAMENODE_IP, '/new_node', {'pub': pub_ip})
def flask_save_replication(): full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str) file_bytes = flask.request.form.get(key=FILE_BYTES, default=None, type=str) if not full_file_path or not file_bytes: data = { MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_BYTES}`" } return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) file_path = get_path(full_file_path) file_bytes = base64.decodebytes(file_bytes.encode()) try: with open(file_path, 'wb') as f: f.write(file_bytes) except OSError as e: debug_log(e.strerror) data = {MESSAGE_KEY: "Error on the server"} return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) data = {MESSAGE_KEY: "OK"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_init(): login = flask.request.form.get(key=LOGIN_KEY, default=None, type=str) if not login: data = {MESSAGE_KEY: f"Missing required parameters: `{LOGIN_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) user_folder = get_path(login) if os.path.exists(user_folder): # rm user's folder if exists try: shutil.rmtree(user_folder) except OSError: debug_log(f"Deletion of the directory {user_folder} failed") data = { MESSAGE_KEY: "The folder is already exists. Couldn't delete it." } return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) # create user's folder try: os.mkdir(user_folder) except OSError: debug_log(f"Creation of the directory {user_folder} failed") data = {MESSAGE_KEY: "Failed during folder creation."} return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) data = {MESSAGE_KEY: "Success"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def ping(): try: st = statvfs(ROOT) congestion = f'{(st.f_blocks - st.f_bavail) / st.f_blocks:.2f}' except OSError as e: debug_log(f"statvfs failed {e}") congestion = 0 data = {CONGESTION_KEY: congestion} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_ddir(): token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str) dir_path = flask.request.form.get(key=PATH_KEY, default=None, type=str) force = flask.request.form.get(key=FORCE_KEY, default=None, type=str) if not token or dir_path is None or force is None: data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`, `{FORCE_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) login = decode_auth_token(token) if not login or (type(login) == str and login not in db_auth.keys()): data = {MESSAGE_KEY: "The token is invalid or has expired"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) dir_path = dir_path[:-1] if dir_path and dir_path[-1] == '/' else dir_path if not dir_path: data = {MESSAGE_KEY: "Can't delete the root directory"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) full_dir_path = os.path.join(login, dir_path) if full_dir_path not in db_user2folders.lrange(login, 0, -1): data = {MESSAGE_KEY: "The folder for deleting does not exist"} return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_FOUND) inner_files_path_list = [full_path for full_path in db_user2files.lrange(login, 0, -1) if full_path.startswith(full_dir_path + '/')] inner_folders_path_list = [full_path for full_path in db_user2folders.lrange(login, 0, -1) if full_path.startswith(full_dir_path + '/')] if force == 'False' and (inner_files_path_list or inner_folders_path_list): data = {MESSAGE_KEY: 'The directory contains files. `force=true` to delete'} return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_ACCEPTABLE) db_user2folders.lrem(login, 0, full_dir_path) for inner_dir in inner_folders_path_list: db_user2folders.lrem(login, 0, inner_dir) nodes_ip = set() for inner_file in inner_files_path_list: db_file2size.delete(inner_file) db_user2files.lrem(login, 0, inner_file) for node_ip in db_file2nodes.lrange(inner_file, 0, -1): db_file2nodes.lrem(inner_file, 0, node_ip) db_node2files.lrem(node_ip, 0, inner_file) nodes_ip.add(node_ip) for node_ip in list(nodes_ip): res = request_node(node_ip, '/ddir', {FULL_PATH_KEY: full_dir_path}) res = get_dict_from_response(res) if res is None: debug_log(f"Node {node_ip} did not response on /ddir ({full_dir_path})") data = {MESSAGE_KEY: 'Successfully deleted the directory'} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def __init__(self, DATABASE): if os.path.exists(DATABASE): self.db_path = DATABASE self.connection = sqlite3.connect(self.db_path) self.cursor = self.connection.cursor() logger.debug_log("Database connection successful.", logger.level.debug) else: logger.debug_log("Database connection failed.", logger.level.error)
def flask_new_node(): new_node_ip = flask.request.environ.get('HTTP_X_REAL_IP', flask.request.remote_addr) pub_ip = flask.request.form.get(key='pub', default=None, type=str) if new_node_ip in db_congestion.keys(): debug_log(f"!!!!!!!ERROR!!!!!!!! {new_node_ip} calls /new_node but it's already in the db") else: debug_log(f"{new_node_ip} calls /new_node and added to db. Pub_ip = {pub_ip}") db_congestion.set(new_node_ip, 0) db_pub.set(new_node_ip, pub_ip) return flask.make_response(flask.jsonify({}), HTTPStatus.OK)
def _is_bucket_item_sane(key): logger.debug_log('s3decryptor._is_bucket_item_sane start') if key.size < _BUCKET_ITEM_MIN_SIZE or key.size > int(config['s3ObjectMaxSize']): err = 'item not sane size: %d' % key.size logger.error(err) return False logger.debug_log('s3decryptor._is_bucket_item_sane end') return True
def _is_bucket_item_sane(key): logger.debug_log('s3decryptor._is_bucket_item_sane start') if key.size < _BUCKET_ITEM_MIN_SIZE or key.size > int( config['s3ObjectMaxSize']): err = 'item not sane size: %d' % key.size logger.error(err) return False logger.debug_log('s3decryptor._is_bucket_item_sane end') return True
def _analyze_diagnostic_info(diagnostic_info, reply_info): ''' Determines what response should be sent based on `diagnostic_info` content. Returns a list of response IDs. Returns None if no response should be sent. ''' logger.debug_log('_analyze_diagnostic_info: enter') responses = None AddressParts = collections.namedtuple('AddressParts', ['name', 'plus', 'domain']) # Get our local email address parts match = _address_splitter.match(config['emailUsername']) local_email_parts = AddressParts(*match.groups()) # We'll break apart the "to" address (if applicable) to_parts = AddressParts(None, None, None) if reply_info.to: match = _address_splitter.match(reply_info.to) if not match: # Someone is messing with us raise ValueError('invalid email address: to-address match failed') to_parts = AddressParts(*match.groups()) # TODO: more flexible rules, not so hard-coded if utils.coalesce( diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'isPlayStoreBuild']): # No download links in Play Store email responses = ['generic_info'] elif to_parts.name == local_email_parts.name and \ to_parts.domain == local_email_parts.domain: # E.g., [email protected], [email protected] responses = [ 'download_new_version_links', # Disabling attachment responses for now. Not sure if # it's a good idea. Note that it needs to be tested. # 'download_new_version_attachments', ] elif to_parts.domain == local_email_parts.domain: # E.g., *@psiphon.ca responses = ['generic_info'] elif not reply_info.to and \ utils.coalesce(diagnostic_info, ('Metadata', 'platform'), utils.string_types): # Windows S3 feedback responses = ['download_new_version_links'] logger.debug_log('_analyze_diagnostic_info: exit: %s' % responses) return responses
def _get_email_reply_info(autoresponder_info): """Returns None if no reply info found, otherwise an instance of _ReplyInfo. Note that this function also validates the email address. """ logger.debug_log('_get_email_reply_info: enter') email_info = autoresponder_info.get('email_info') diagnostic_info = autoresponder_info.get('diagnostic_info') reply_info = None if email_info: reply_info = _ReplyInfo(email_info.get('address'), email_info.get('message_id'), email_info.get('subject'), email_info.get('to')) elif utils.coalesce(diagnostic_info, 'EmailInfo', required_types=dict): reply_info = _ReplyInfo(diagnostic_info['EmailInfo'].get('address'), diagnostic_info['EmailInfo'].get('message_id'), diagnostic_info['EmailInfo'].get('subject'), diagnostic_info['EmailInfo'].get('to')) elif utils.coalesce(diagnostic_info, 'Feedback', required_types=dict): reply_info = _ReplyInfo(diagnostic_info['Feedback'].get('email'), None, None, None) if not reply_info: logger.debug_log('_get_email_reply_info: no/bad reply_info, exiting') return None # Sometimes the recorded address looks like "<*****@*****.**>" if reply_info.address: reply_info.address = reply_info.address.strip('<>') if not reply_info.address: logger.debug_log( '_get_email_reply_info: no/bad reply_info.address, exiting') return None if reply_info.to: reply_info.to = reply_info.to.strip('<>') validator = email_validator.EmailValidator(fix=True, lookup_dns='mx') try: fixed_address = validator.validate_or_raise(reply_info.address) reply_info.address = fixed_address except: logger.debug_log( '_get_email_reply_info: address validator raised, exiting') return None logger.debug_log('_get_email_reply_info: exit') return reply_info
def _upgrade_old_object(yaml_docs): ''' The diagnostic info stuff was released for Android before versioning was added. Returns the appropriately modified YAML dict. ''' logger.debug_log('maildecryptor._upgrade_old_object start') # The non-versioned YAML used multiple docs, so that's the main test if len(yaml_docs) == 1: return yaml_docs.pop() # Our old YAML had '.' in some key names, which is illegal. for path, val in utils.objwalk(yaml_docs): if type(path[-1]) == str and path[-1].find('.') >= 0: utils.rename_key_in_obj_at_path(yaml_docs, path, path[-1].replace('.', '__')) # Our old YAML format was multiple YAML docs in a single string. We'll # convert that to the new format. obj = {} # Old YAML had no Metadata section metadata = {} metadata['platform'] = 'android' metadata['version'] = 1 metadata['id'] = binascii.hexlify(os.urandom(8)) obj['Metadata'] = metadata idx = 0 obj['SystemInformation'] = yaml_docs[idx] idx += 1 obj['ServerResponseCheck'] = yaml_docs[idx] idx += 1 # The presence of DiagnosticHistory was optional once upon a time if len(yaml_docs) > 3: obj['DiagnosticHistory'] = yaml_docs[idx] idx += 1 else: obj['DiagnosticHistory'] = [] obj['StatusHistory'] = yaml_docs[idx] idx += 1 logger.debug_log('maildecryptor._upgrade_old_object end') return obj
def _upgrade_old_object(yaml_docs): ''' The diagnostic info stuff was released for Android before versioning was added. Returns the appropriately modified YAML dict. ''' logger.debug_log('maildecryptor._upgrade_old_object start') # The non-versioned YAML used multiple docs, so that's the main test if len(yaml_docs) == 1: return yaml_docs.pop() # Our old YAML had '.' in some key names, which is illegal. for path, val in utils.objwalk(yaml_docs): if isinstance(path[-1], utils.string_types) and path[-1].find('.') >= 0: utils.rename_key_in_obj_at_path(yaml_docs, path, path[-1].replace('.', '__')) # Our old YAML format was multiple YAML docs in a single string. We'll # convert that to the new format. obj = {} # Old YAML had no Metadata section metadata = {} metadata['platform'] = 'android' metadata['version'] = 1 metadata['id'] = binascii.hexlify(os.urandom(8)) obj['Metadata'] = metadata idx = 0 obj['SystemInformation'] = yaml_docs[idx] idx += 1 obj['ServerResponseCheck'] = yaml_docs[idx] idx += 1 # The presence of DiagnosticHistory was optional once upon a time if len(yaml_docs) > 3: obj['DiagnosticHistory'] = yaml_docs[idx] idx += 1 else: obj['DiagnosticHistory'] = [] obj['StatusHistory'] = yaml_docs[idx] idx += 1 logger.debug_log('maildecryptor._upgrade_old_object end') return obj
def _html_to_text(html): ''' Convert given `html` to plain text. ''' logger.debug_log('_html_to_text: enter') h2t = html2text.HTML2Text() h2t.body_width = 0 txt = h2t.handle(html) logger.debug_log('_html_to_text: returning text length %d' % len(txt)) return txt
def flask_fmove(): token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str) file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str) file_destination_path = flask.request.form.get(key=PATH_DESTINATION_KEY, default=None, type=str) if not token or not file_path or file_destination_path is None: data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`, `{PATH_DESTINATION_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) login = decode_auth_token(token) if not login or (type(login) == str and login not in db_auth.keys()): data = {MESSAGE_KEY: "The token is invalid or has expired"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) if not os.path.basename(file_destination_path): data = {MESSAGE_KEY: "The destination file name is empty"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) full_file_path = os.path.join(login, file_path) full_file_destination_path = os.path.join(login, file_destination_path) if full_file_path not in db_user2files.lrange(login, 0, -1): data = {MESSAGE_KEY: "The source file doesn't exist"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) if os.path.dirname(full_file_destination_path) not in db_user2folders.lrange(login, 0, -1): data = {MESSAGE_KEY: "Can't move the file. Destination folder doesn't exist"} return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN) for node_ip in db_file2nodes.lrange(full_file_path, 0, -1): res = request_node(node_ip, '/fmove', {FULL_PATH_KEY: full_file_path, FULL_PATH_DESTINATION_KEY: full_file_destination_path}) res = get_dict_from_response(res) if res is None: debug_log(f"Node {node_ip} did not response on /fmove") else: db_node2files.lrem(node_ip, 0, full_file_path) db_node2files.lpush(node_ip, full_file_destination_path) db_file2nodes.lrem(full_file_path, 0, node_ip) db_file2nodes.lpush(full_file_destination_path, node_ip) db_user2files.lrem(login, 0, full_file_path) db_user2files.lpush(login, full_file_destination_path) db_file2size.set(full_file_destination_path, db_file2size.get(full_file_path)) db_file2size.delete(full_file_path) data = {MESSAGE_KEY: f"Successfully moved the file"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def _analyze_diagnostic_info(diagnostic_info, reply_info): ''' Determines what response should be sent based on `diagnostic_info` content. Returns a list of response IDs. Returns None if no response should be sent. ''' logger.debug_log('_analyze_diagnostic_info: enter') responses = None AddressParts = collections.namedtuple('AddressParts', ['name', 'plus', 'domain']) # Get our local email address parts match = _address_splitter.match(config['emailUsername']) local_email_parts = AddressParts(*match.groups()) # We'll break apart the "to" address (if applicable) to_parts = AddressParts(None, None, None) if reply_info.to: match = _address_splitter.match(reply_info.to) if not match: # Someone is messing with us raise ValueError('invalid email address: to-address match failed') to_parts = AddressParts(*match.groups()) # TODO: more flexible rules, not so hard-coded if utils.coalesce(diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'isPlayStoreBuild']): # No download links in Play Store email responses = ['generic_info'] elif to_parts.name == local_email_parts.name and \ to_parts.domain == local_email_parts.domain: # E.g., [email protected], [email protected] responses = ['download_new_version_links', # Disabling attachment responses for now. Not sure if # it's a good idea. Note that it needs to be tested. # 'download_new_version_attachments', ] elif to_parts.domain == local_email_parts.domain: # E.g., *@psiphon.ca responses = ['generic_info'] elif not reply_info.to and \ utils.coalesce(diagnostic_info, ('Metadata', 'platform'), utils.string_types): # Windows S3 feedback responses = ['download_new_version_links'] logger.debug_log('_analyze_diagnostic_info: exit: %s' % responses) return responses
def _get_email_reply_info(autoresponder_info): """Returns None if no reply info found, otherwise an instance of _ReplyInfo. Note that this function also validates the email address. """ logger.debug_log('_get_email_reply_info: enter') email_info = autoresponder_info.get('email_info') diagnostic_info = autoresponder_info.get('diagnostic_info') reply_info = None if email_info: reply_info = _ReplyInfo(email_info.get('address'), email_info.get('message_id'), email_info.get('subject'), email_info.get('to')) elif utils.coalesce(diagnostic_info, 'EmailInfo', required_types=dict): reply_info = _ReplyInfo(diagnostic_info['EmailInfo'].get('address'), diagnostic_info['EmailInfo'].get('message_id'), diagnostic_info['EmailInfo'].get('subject'), diagnostic_info['EmailInfo'].get('to')) elif utils.coalesce(diagnostic_info, 'Feedback', required_types=dict): reply_info = _ReplyInfo(diagnostic_info['Feedback'].get('email'), None, None, None) if not reply_info: logger.debug_log('_get_email_reply_info: no/bad reply_info, exiting') return None # Sometimes the recorded address looks like "<*****@*****.**>" if reply_info.address: reply_info.address = reply_info.address.strip('<>') if not reply_info.address: logger.debug_log('_get_email_reply_info: no/bad reply_info.address, exiting') return None if reply_info.to: reply_info.to = reply_info.to.strip('<>') validator = email_validator.EmailValidator(fix=True, lookup_dns='mx') try: fixed_address = validator.validate_or_raise(reply_info.address) reply_info.address = fixed_address except: logger.debug_log('_get_email_reply_info: address validator raised, exiting') return None logger.debug_log('_get_email_reply_info: exit') return reply_info
def _get_lang_id_from_diagnostic_info(diagnostic_info): ''' Derive the lanague from `diagnostic_info` and return its ID/code. Returns `None` if the language can't be determined. ''' logger.debug_log('_get_lang_id_from_diagnostic_info: enter') lang_id = None # There can be different -- and better or worse -- ways of determining the # user's language depending on platform, the type of feedback, and so on. # Windows, with feedback message lang_id = lang_id or utils.coalesce( diagnostic_info, ['Feedback', 'Message', 'text_lang_code'], required_types=utils.string_types) if lang_id and lang_id.find('INDETERMINATE') >= 0: lang_id = None # All Windows feedback lang_id = lang_id or utils.coalesce(diagnostic_info, [ 'DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LocaleInfo', 'language_code' ], required_types=utils.string_types) # All Windows feedback lang_id = lang_id or utils.coalesce(diagnostic_info, [ 'DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LanguageInfo', 'language_code' ], required_types=utils.string_types) # Android, from email lang_id = lang_id or utils.coalesce( diagnostic_info, ['EmailInfo', 'body', 'text_lang_code'], required_types=utils.string_types) if lang_id and lang_id.find('INDETERMINATE') >= 0: lang_id = None # Android, from system language lang_id = lang_id or utils.coalesce( diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'language'], required_types=utils.string_types) logger.debug_log( '_get_lang_id_from_diagnostic_info: exiting with lang_id=%s' % lang_id) return lang_id
def ping_nodes(): for node_ip in db_congestion.keys(): if node_ip not in non_responsive_count: non_responsive_count[node_ip] = 0 res = request_node(node_ip, '/ping', {}) res = get_dict_from_response(res) if res is None: non_responsive_count[node_ip] += 1 debug_log(f"Node {node_ip} did not response {non_responsive_count[node_ip]} times") if non_responsive_count[node_ip] == MAX_PING_TRIES: remove_node(node_ip) del non_responsive_count[node_ip] else: debug_log(f"Pinging node {node_ip} - {res}") db_congestion.set(node_ip, res[CONGESTION_KEY])
def _autoresponder_record_iter(): logger.debug_log('_autoresponder_record_iter: enter') while True: for rec in datastore.get_autoresponder_iterator(): logger.debug_log('_autoresponder_record_iter: %s' % repr(rec)) if rec.get('diagnostic_info_record_id'): rec['diagnostic_info'] = datastore.find_diagnostic_info(rec.get('diagnostic_info_record_id')) logger.debug_log('_autoresponder_record_iter: yielding rec: %s' % rec['_id']) yield rec logger.debug_log('_diagnostic_record_iter: sleeping') time.sleep(_SLEEP_TIME_SECS) logger.debug_log('_autoresponder_record_iter: exit')
def change_active_database(self, PATH): if not os.path.exists(PATH): return print("Catastrophic failure: Database path non-existent") # check if a current connection is already open if self.connection: self.connection.close() # TODO(jamie): Implement backup of current database file # create connection to new database self.connection = sqlite3.connect(PATH) self.cursor = self.connection.cursor() if not self.connection: logger.debug_log("Database connection failed.", logger.level.error) else: logger.debug_log("New database [{}] opened".format(PATH), logger.level.debug)
def _email_diagnostic_info_records_iterator(): ''' Generator for obtaining email_diagnostic_info records. ''' while True: # Every hour or so, pymongo throws a CursorNotFound error. This is due to the # the cursor being idle more than 10 minutes. It generally doesn't take us that # long to send an email, but each time a request to mongodb is made, there's a # batch of records returned. Processing all of the records in that batch _does_ # take more than 10 minutes. So when we try to get another batch the cursor is dead. # To address this, we could decrease the batch size or increase the cursor lifetime. # More details: https://stackoverflow.com/a/24200795/729729 logger.debug_log('fresh cursor') for rec in datastore.get_email_diagnostic_info_iterator(): yield rec time.sleep(_SLEEP_TIME_SECS)
def _get_lang_id_from_diagnostic_info(diagnostic_info): ''' Derive the lanague from `diagnostic_info` and return its ID/code. Returns `None` if the language can't be determined. ''' logger.debug_log('_get_lang_id_from_diagnostic_info: enter') lang_id = None # There can be different -- and better or worse -- ways of determining the # user's language depending on platform, the type of feedback, and so on. # Windows, with feedback message lang_id = lang_id or utils.coalesce(diagnostic_info, ['Feedback', 'Message', 'text_lang_code'], required_types=utils.string_types) if lang_id and lang_id.find('INDETERMINATE') >= 0: lang_id = None # All Windows feedback lang_id = lang_id or utils.coalesce(diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LocaleInfo', 'language_code'], required_types=utils.string_types) # All Windows feedback lang_id = lang_id or utils.coalesce(diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LanguageInfo', 'language_code'], required_types=utils.string_types) # Android, from email lang_id = lang_id or utils.coalesce(diagnostic_info, ['EmailInfo', 'body', 'text_lang_code'], required_types=utils.string_types) if lang_id and lang_id.find('INDETERMINATE') >= 0: lang_id = None # Android, from system language lang_id = lang_id or utils.coalesce(diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'language'], required_types=utils.string_types) logger.debug_log('_get_lang_id_from_diagnostic_info: exiting with lang_id=%s' % lang_id) return lang_id
def flask_fwrite(): full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str) file_bytes = flask.request.form.get(key=FILE_BYTES, default=None, type=str) if not full_file_path or file_bytes is None: data = { MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_BYTES}`" } return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) file_path = get_path(full_file_path) file_bytes = base64.decodebytes(file_bytes.encode()) try: all_dirs = full_file_path.split("/")[:-1] dir_path = ROOT for directory in all_dirs: dir_path = os.path.join(dir_path, directory) if not os.path.exists(dir_path): os.mkdir(dir_path) # writeing to file with open(file_path, 'wb') as f: f.write(file_bytes) except OSError as e: debug_log(e.strerror) data = {MESSAGE_KEY: "Error on the server"} return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) data = {FULL_PATH_KEY: full_file_path, FILE_SIZE_KEY: len(file_bytes)} res = request_node(NAMENODE_IP, '/uploaded', data) if res.status != HTTPStatus.OK: data = {MESSAGE_KEY: get_dict_from_response(res)[MESSAGE_KEY]} return flask.make_response(flask.jsonify(data), HTTPStatus.OK) data = {MESSAGE_KEY: "OK"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def flask_ddir(): full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str) if not full_file_path: data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`"} return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY) file_path = get_path(full_file_path) try: shutil.rmtree(file_path) except OSError: debug_log(f"Delete the directory {file_path} failed") data = {MESSAGE_KEY: "Failed during dir delete."} return flask.make_response(flask.jsonify(data), HTTPStatus.INTERNAL_SERVER_ERROR) data = {MESSAGE_KEY: "Success"} return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def _get_email_info(msg): logger.debug_log('maildecryptor._get_email_info start') subject_translation = translation.translate(config['googleApiServers'], config['googleApiKey'], msg['subject']) subject = dict(text=msg['subject'], text_lang_code=subject_translation[0], text_lang_name=subject_translation[1], text_translated=subject_translation[2]) body_translation = translation.translate(config['googleApiServers'], config['googleApiKey'], msg['body']) body = dict(text=msg['body'], text_lang_code=body_translation[0], text_lang_name=body_translation[1], text_translated=body_translation[2], html=msg['html']) raw_address = msg['from'] or msg['msgobj'].get('Return-Path') stripped_address = None if raw_address: match = _email_stripper_regex.match(raw_address) if not match: logger.error('when stripping email address failed to match: %s' % str(raw_address)) return None stripped_address = match.group(2) email_info = dict(address=stripped_address, to=msg['to'], message_id=msg['msgobj']['Message-ID'], subject=subject, body=body) logger.debug_log('maildecryptor._get_email_info end') return email_info
def _render_email(data): logger.debug_log('_render_email: enter') global _template if not _template: _template = Template(filename='templates/feedback_response.mako', default_filters=['unicode', 'h', 'decode.utf8'], input_encoding='utf-8', output_encoding='utf-8', lookup=TemplateLookup(directories=['.'])) logger.debug_log('_render_email: template loaded') rendered = _template.render(data=data) # CSS in email HTML must be inline rendered = pynliner.fromString(rendered) logger.debug_log('_render_email: exiting with len(rendered)=%d' % len(rendered)) return rendered
def go(): logger.debug_log('s3decryptor.go: start') s3_conn = S3Connection(config['aws_access_key_id'], config['aws_secret_access_key']) bucket = s3_conn.get_bucket(config['s3_bucket_name']) # Note that `_bucket_iterator` throttles itself if/when there are no # available objects in the bucket. for encrypted_info_json in _bucket_iterator(bucket): logger.debug_log('s3decryptor.go: processing item') # In theory, all bucket items should be usable by us, but there's # always the possibility that a user (or attacker) is messing with us. try: encrypted_info = json.loads(encrypted_info_json) diagnostic_info = decryptor.decrypt(encrypted_info) diagnostic_info = diagnostic_info.strip() # HACK: PyYaml only supports YAML 1.1, which is not a true superset # of JSON. Therefore it can (and does) throw errors on some Android # feedback. We will try to load using JSON first. # TODO: Get rid of all YAML feedback and remove it from here. try: diagnostic_info = json.loads(diagnostic_info) logger.debug_log('s3decryptor.go: loaded JSON') except: diagnostic_info = yaml.safe_load(diagnostic_info) logger.debug_log('s3decryptor.go: loaded YAML') # Modifies diagnostic_info utils.convert_psinet_values(config, diagnostic_info) if not utils.is_diagnostic_info_sane(diagnostic_info): # Something is wrong. Skip and continue. continue # Modifies diagnostic_info datatransformer.transform(diagnostic_info) # Store the diagnostic info record_id = datastore.insert_diagnostic_info(diagnostic_info) if _should_email_data(diagnostic_info): logger.debug_log('s3decryptor.go: should email') # Record in the DB that the diagnostic info should be emailed datastore.insert_email_diagnostic_info(record_id, None, None) # Store an autoresponder entry for this diagnostic info datastore.insert_autoresponder_entry(None, record_id) logger.log('decrypted diagnostic data') except decryptor.DecryptorException as e: logger.exception() logger.error(str(e)) try: # Something bad happened while decrypting. Report it via email. sender.send(config['decryptedEmailRecipient'], config['emailUsername'], u'S3Decryptor: bad object', encrypted_info_json, None) # no html body except smtplib.SMTPException as e: logger.exception() logger.error(str(e)) # yaml.constructor.ConstructorError was being thown when a YAML value # consisted of just string "=". Probably due to this PyYAML bug: # http://pyyaml.org/ticket/140 except (ValueError, TypeError, yaml.constructor.ConstructorError) as e: # Try the next attachment/message logger.exception() logger.error(str(e)) logger.debug_log('s3decryptor.go: end')
def go(): logger.debug_log('go: enter') # Note that `_diagnostic_record_iter` throttles itself if/when there are # no records to process. for autoresponder_info in _autoresponder_record_iter(): diagnostic_info = autoresponder_info.get('diagnostic_info') email_info = autoresponder_info.get('email_info') logger.debug_log('go: got autoresponder record') # For now we don't do any interesting processing/analysis and we just # respond to every feedback with an exhortation to upgrade. reply_info = _get_email_reply_info(autoresponder_info) if not reply_info or not reply_info.address: # If we don't have any reply info, we can't reply logger.debug_log('go: no reply_info or address') continue # Check if the address is blacklisted if _check_and_add_address_blacklist(reply_info.address): logger.debug_log('go: blacklisted') continue responses = _analyze_diagnostic_info(diagnostic_info, reply_info) if not responses: logger.debug_log('go: no response') continue logger.log('Sending feedback response') for response_id in responses: response_content = _get_response_content(response_id, diagnostic_info) if not response_content: logger.debug_log('go: no response_content') continue # The original diagnostic info may have originated from an email, # in which case we have a subject to reply to. Or it may have have # originated from an uploaded data package, in which case we need # set our own subject. if type(reply_info.subject) is dict: subject = u'Re: %s' % reply_info.subject.get('text', '') else: subject = response_content['subject'] try: sender.send_response(reply_info.address, config['reponseEmailAddress'], subject, response_content['body_text'], response_content['body_html'], reply_info.message_id, response_content['attachments']) except Exception as e: logger.debug_log('go: send_response excepted') logger.exception() logger.error(str(e))
def _get_response_content(response_id, diagnostic_info): """Gets the response for the given response_id. diagnostic_info will be used to determine language and some content, but may be None. Returns a dict of the form: { subject: <subject text>, body_text: <body text>, body_html: <rich html body>, attachments: <attachments list, may be None> } Returns None if no response content can be derived. """ logger.debug_log('_get_response_content: enter') sponsor_name = utils.coalesce(diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'SPONSOR_ID'], required_types=utils.string_types) prop_channel_name = utils.coalesce(diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'PROPAGATION_CHANNEL_ID'], required_types=utils.string_types) # Use default values if we couldn't get good user-specific values sponsor_name = sponsor_name or config['defaultSponsorName'] prop_channel_name = prop_channel_name or config['defaultPropagationChannelName'] lang_id = _get_lang_id_from_diagnostic_info(diagnostic_info) # lang_id may be None, if the language could not be determined # Read in all translations HTML response_translations = [] for root, _, files in os.walk(_RESPONSES_DIR): for name in files: lang, ext = os.path.splitext(name) if ext != '.html': continue if lang == 'master': lang = 'en' with open(os.path.join(root, name)) as translation_file: translation = translation_file.read().decode('utf-8') # Strip leading and trailing whitespace so that we don't get extra # text elements in our BeautifulSoup translation = translation.strip() response_translations.append((lang, translation.strip())) # Reorder the array according to the detected language and _TOP_LANGS def lang_sorter(item): lang, _ = item rank = 999 try: if lang == lang_id: rank = -1 else: rank = _TOP_LANGS.index(lang) except ValueError: pass return rank response_translations.sort(key=lang_sorter) # Gather the info we'll need for formatting the email bucketname, email_address = psi_ops_helpers.get_bucket_name_and_email_address(sponsor_name, prop_channel_name) # Use default values if we couldn't get good user-specific values if not bucketname or not email_address: default_bucketname, default_email_address = \ psi_ops_helpers.get_bucket_name_and_email_address(config['defaultSponsorName'], config['defaultPropagationChannelName']) bucketname = bucketname or default_bucketname email_address = email_address or default_email_address # If, despite our best efforts, we still don't have a bucketname and # email address, just bail. if not bucketname or not email_address: logger.debug_log('_get_response_content: exiting due to no bucketname or address') return None # Collect the translations of the specific response we're sending subject = None bodies = [] for lang_id, html in response_translations: soup = BeautifulSoup(html) if not subject: subject = soup.find(id='default_response_subject') if subject: # Strip outer element subject = u''.join(unicode(elem) for elem in subject.contents).strip() body = soup.find(id=response_id) if body: # Strip outer element body = u''.join(unicode(elem) for elem in body.contents).strip() # The user might be using a language for which there isn't a # download page. Fall back to English if that's the case. home_page_url = psi_ops_helpers.get_s3_bucket_home_page_url( bucketname, lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en') download_page_url = psi_ops_helpers.get_s3_bucket_download_page_url( bucketname, lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en') faq_page_url = psi_ops_helpers.get_s3_bucket_faq_url( bucketname, lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en') # We're using numbers rather than more readable names here because # they're less likely to be accidentally modified by translators # (we think). format_dict = { '0': email_address, '1': download_page_url, '2': home_page_url, '3': faq_page_url } body = unicode(body) % format_dict bodies.append(body) # Render the email body from the Mako template body_html = _render_email({ 'lang_id': lang_id, 'response_id': response_id, 'responses': bodies }) # Get attachments. # This depends on which response we're returning. attachments = None if response_id == 'download_new_version_links': pass elif response_id == 'download_new_version_attachments': fp_windows = aws_helpers.get_s3_attachment('attachments', bucketname, psi_ops_helpers.DOWNLOAD_SITE_WINDOWS_BUILD_FILENAME) fp_android = aws_helpers.get_s3_attachment('attachments', bucketname, psi_ops_helpers.DOWNLOAD_SITE_ANDROID_BUILD_FILENAME) attachments = [(fp_windows, psi_ops_helpers.EMAIL_RESPONDER_WINDOWS_ATTACHMENT_FILENAME), (fp_android, psi_ops_helpers.EMAIL_RESPONDER_ANDROID_ATTACHMENT_FILENAME)] else: pass logger.debug_log('_get_response_content: exit') return { 'subject': subject, 'body_text': _html_to_text(body_html), 'body_html': body_html, 'attachments': attachments }
import logger log1 = logger.debug_log() log2 = logger.debug_log() log1.info("log1 log") log2.info("log2 log")
def go(): logger.debug_log('maildecryptor.go start') emailgetter = EmailGetter(config['popServer'], config['popPort'], config['emailUsername'], config['emailPassword']) # Retrieve and process email. # Note that `emailgetter.get` throttles itself if/when there are no emails # immediately available. for msg in emailgetter.get(): logger.debug_log('maildecryptor.go: msg has %d attachments' % len(msg['attachments'])) diagnostic_info = None # # First try to process attachments. # for attachment in msg['attachments']: # Not all attachments will be in our format, so expect exceptions. try: encrypted_info = attachment.getvalue() encrypted_info = json.loads(encrypted_info) diagnostic_info = decryptor.decrypt(encrypted_info) diagnostic_info = diagnostic_info.strip() diagnostic_info = _load_yaml(diagnostic_info) # Modifies diagnostic_info utils.convert_psinet_values(config, diagnostic_info) if not utils.is_diagnostic_info_sane(diagnostic_info): # Something is wrong. Skip and continue. continue # Modifies diagnostic_info datatransformer.transform(diagnostic_info) logger.log('email attachment decrypted') break except decryptor.DecryptorException as e: # Something bad happened while decrypting. Report it via email. logger.exception() try: sender.send(config['decryptedEmailRecipient'], config['emailUsername'], u'Re: %s' % (msg['subject'] or ''), 'Decrypt failed: %s' % e, msg['msgobj']['Message-ID']) except smtplib.SMTPException as e: # Something went wrong with the sending of the response. Log it. logger.exception() logger.error(str(e)) except (ValueError, TypeError) as e: # Try the next attachment/message logger.exception() logger.error(str(e)) # # Store what info we have # email_info = _get_email_info(msg) diagnostic_info_record_id = None if diagnostic_info: # Add the user's email information to diagnostic_info. # This will allow us to later auto-respond, or act as a # remailer between the user and the Psiphon support team. diagnostic_info['EmailInfo'] = email_info # Store the diagnostic info diagnostic_info_record_id = datastore.insert_diagnostic_info(diagnostic_info) # Store the association between the diagnostic info and the email datastore.insert_email_diagnostic_info(diagnostic_info_record_id, msg['msgobj']['Message-ID'], msg['subject']) # Store autoresponder info regardless of whether there was a diagnostic info datastore.insert_autoresponder_entry(email_info, diagnostic_info_record_id) logger.debug_log('maildecryptor.go end')