def test_token_create_and_get_upload_expire(self): file_list = [] token_collection = [] for i in xrange(20): st = Token('submission') token_collection.append(st) for t in token_collection: token = TokenList.get(t.id) yield self.emulate_file_upload(token, 3) for f in token.uploaded_files: self.assertTrue(os.path.exists(f['encrypted_path'])) file_list.append(f['encrypted_path']) TokenList.reactor.pump([1] * TokenList.get_timeout()) for t in token_collection: self.assertRaises(errors.TokenFailure, TokenList.get, t.id) for f in file_list: self.assertFalse(os.path.exists(f))
def put(self, token_id): """ Parameter: token_id Request: SubmissionDesc Response: SubmissionDesc PUT finalize the submission """ @transact def put_transact(store, token, request): status = db_create_submission(store, token, request, self.request.language) receipt = db_create_whistleblower_tip(store, status) status.update({'receipt': receipt}) return status request = self.validate_message(self.request.body, requests.SubmissionDesc) # the .get method raise an exception if the token is invalid token = TokenList.get(token_id) if not token.context_associated == request['context_id']: raise errors.InvalidInputFormat("Token context unaligned with REST url") token.validate(request) status = yield put_transact(token, request) TokenList.delete(token_id) self.set_status(202) # Updated, also if submission if effectively created (201) self.finish(status)
def get(self, token_id): """ Parameter: internaltip_id Request: Unknown Response: Unknown Errors: TokenFailure """ TokenList.get(token_id) self.set_status(204) # We currently do not implement file resume self.finish()
def post(self, token_id): """ Errors: TokenFailure """ token = TokenList.get(token_id) log.debug("file upload with token associated: %s" % token) uploaded_file = self.get_file_upload() if uploaded_file is None: return uploaded_file['body'].avoid_delete() uploaded_file['body'].close() dst = os.path.join(Settings.submission_path, os.path.basename(uploaded_file['path'])) directory_traversal_check(Settings.submission_path, dst) uploaded_file = yield threads.deferToThread( write_upload_encrypted_to_disk, uploaded_file, dst) uploaded_file['date'] = datetime_now() uploaded_file['submission'] = True token.associate_file(uploaded_file)
def test_token_create_and_get_upload_expire(self): file_list = [] token_collection = [] for i in xrange(20): st = Token('submission') token_collection.append(st) for t in token_collection: token = TokenList.get(t.id) yield self.emulate_file_upload(token, 3) for f in token.uploaded_files: self.assertTrue(os.path.exists(f['encrypted_path'])) file_list.append(f['encrypted_path']) token.expire() self.assertRaises( errors.TokenFailure, TokenList.get, t.id ) for f in file_list: self.assertFalse(os.path.exists(f))
def handle_file_upload(self, token_id): token = TokenList.get(token_id) log.debug("file upload with token associated: %s" % token) uploaded_file = self.get_file_upload() if uploaded_file is None: return uploaded_file['body'].avoid_delete() uploaded_file['body'].close() try: # dump_file_fs return the new filepath inside the dictionary uploaded_file = yield threads.deferToThread( dump_file_fs, uploaded_file) uploaded_file['creation_date'] = datetime_now() uploaded_file['submission'] = True token.associate_file(uploaded_file) serialize_memory_file(uploaded_file) except: log.err("Unable to save file in filesystem: %s" % excep) raise errors.InternalServerError("Unable to accept files")
def test_token_create_and_get_upload_expire(self): # This is at the beginning event.EventTrackQueue.reset() file_list = [] token_collection = [] for i in xrange(20): st = Token("submission", context_id="ignored") st.set_difficulty(TestToken.shared_alarm_obj.get_token_difficulty()) token_collection.append(st) for t in token_collection: token = TokenList.get(t.id) difficulty = {"human_captcha": True, "graph_captcha": False, "proof_of_work": False} token.set_difficulty(difficulty) self.assertRaises(errors.TokenFailure, token.validate, {"human_captcha_answer": 0}) yield self.emulate_file_upload(token, 3) for f in token.uploaded_files: self.assertTrue(os.path.exists(f["encrypted_path"])) file_list.append(f["encrypted_path"]) token.expire() self.assertRaises(errors.TokenFailure, TokenList.get, t.id) for f in file_list: self.assertFalse(os.path.exists(f))
def init_environment(self): os.umask(0o77) self.settings.eval_paths() self.create_directories() self.cleaning_dead_files() self.tokens = TokenList(self.settings.tmp_path)
def post(self, token_id): """ Parameter: internaltip_id Request: Unknown Response: Unknown Errors: TokenFailure """ token = TokenList.get(token_id) log.debug("file upload with token associated: %s" % token) uploaded_file = self.get_file_upload() if uploaded_file is None: return uploaded_file['body'].avoid_delete() uploaded_file['body'].close() try: dst = os.path.join(GLSettings.submission_path, os.path.basename(uploaded_file['path'])) directory_traversal_check(GLSettings.submission_path, dst) uploaded_file = yield threads.deferToThread(write_upload_encrypted_to_disk, uploaded_file, dst) uploaded_file['date'] = datetime_now() uploaded_file['submission'] = True token.associate_file(uploaded_file) except Exception as excep: log.err("Unable to save file in filesystem: %s" % excep) raise errors.InternalServerError("Unable to accept files") self.set_status(201) # Created
def post(self, token_id): """ Parameter: internaltip_id Request: Unknown Response: Unknown Errors: TokenFailure """ token = TokenList.get(token_id) log.debug("file upload with token associated: %s" % token) uploaded_file = self.get_file_upload() if uploaded_file is None: return uploaded_file['body'].avoid_delete() uploaded_file['body'].close() try: dst = os.path.join(GLSettings.submission_path, os.path.basename(uploaded_file['path'])) directory_traversal_check(GLSettings.submission_path, dst) uploaded_file = yield threads.deferToThread( write_upload_encrypted_to_disk, uploaded_file, dst) uploaded_file['date'] = datetime_now() uploaded_file['submission'] = True token.associate_file(uploaded_file) except Exception as excep: log.err("Unable to save file in filesystem: %s" % excep) raise errors.InternalServerError("Unable to accept files")
def test_token_validate(self): # This is at the beginning anomaly.EventTrackQueue.reset() token = Token('submission', context_id='ignored') difficulty = { 'human_captcha': True, 'graph_captcha': False, 'proof_of_work': False, } token.set_difficulty(difficulty) token = TokenList.get(token.token_id) token.human_captcha = { 'answer': 1 } token.remaining_allowed_attempts = 1 # validate with right value: OK token.validate({'human_captcha_answer': 1}) # validate with wrong value: FAIL self.assertRaises( errors.TokenFailure, token.validate, {'human_captcha_answer': 0} ) # validate with right value but with no additional # attemps available: FAIL self.assertRaises( errors.TokenFailure, token.validate, {'human_captcha_answer': 1} )
def test_tokens_garbage_collected(self): self.assertTrue(len(TokenList) == 0) for i in range(100): Token('submission') self.test_reactor.advance(TokenList.get_timeout()+1) self.assertTrue(len(TokenList) == 0)
def test_tokens_garbage_collected(self): self.assertTrue(len(TokenList) == 0) for i in range(100): Token('submission') self.test_reactor.advance(TokenList.get_timeout() + 1) self.assertTrue(len(TokenList) == 0)
def put(self, token_id): request = self.validate_message(self.request.content.read(), requests.TokenAnswerDesc) token = TokenList.get(token_id) if token is None or self.request.tid != token.tid: raise errors.InvalidAuthentication token.update(request) return token.serialize()
def put(self, token_id): """ Finalize the submission """ request = self.validate_message(self.request.content.read(), requests.SubmissionDesc) # The get and use method will raise if the token is invalid token = TokenList.get(token_id) token.use() submission = create_submission(self.request.tid, request, token.uploaded_files, self.request.client_using_tor) # Delete the token only when a valid submission has been stored in the DB TokenList.delete(token_id) return submission
def put(self, token_id): """ Parameter: token_id Request: TokenAnswerDesc Response: TokenDesc """ request = self.validate_message(self.request.body, requests.TokenAnswerDesc) token = TokenList.get(token_id) token.update(request) self.set_status(202) # Updated self.finish(token.serialize())
def put(self, token_id): """ Parameter: token_id Request: TokenAnswerDesc Response: TokenDesc """ request = self.validate_message(self.request.content.read(), requests.TokenAnswerDesc) token = TokenList.get(token_id) token.update(request) return token.serialize()
def put(self, token_id): """ Parameter: token_id Request: SubmissionDesc Response: SubmissionDesc PUT finalize the submission """ request = self.validate_message(self.request.body, requests.SubmissionDesc) # The get and use method will raise if the token is invalid token = TokenList.get(token_id) token.use() submission = yield create_submission(request, token.uploaded_files, self.check_tor2web(), self.request.language) # Delete the token only when a valid submission has been stored in the DB TokenList.delete(token_id) self.set_status(202) # Updated, also if submission if effectively created (201) self.write(submission)
def post(self, token_id): """ Parameter: internaltip_id Request: Unknown Response: Unknown Errors: TokenFailure """ token = TokenList.get(token_id) log.debug("file upload with token associated: %s" % token) yield self.handle_file_upload(token) self.set_status(201) # Created self.finish()
def put(self, token_id): """ Parameter: token_id Request: TokenAnswerDesc Response: TokenDesc """ request = self.validate_message(self.request.content.read(), requests.TokenAnswerDesc) token = TokenList.get(token_id) if not token.update(request): raise errors.TokenFailure('failed challenge') return token.serialize()
def put(self, token_id): """ Parameter: token_id Request: TokenAnswerDesc Response: TokenDesc """ request = self.validate_message(self.request.body, requests.TokenAnswerDesc) token = TokenList.get(token_id) if not token.update(request): raise errors.TokenFailure('failed challenge') self.set_status(202) # Updated self.write(token.serialize())
def test_token_create_and_get_upload_expire(self): file_list = [] token_collection = [] for _ in range(20): st = Token(1, 'submission') token_collection.append(st) for t in token_collection: token = TokenList.get(t.id) self.emulate_file_upload(token, 3) for f in token.uploaded_files: self.assertTrue(os.path.exists(f['path'])) file_list.append(f['path']) self.test_reactor.advance(TokenList.get_timeout() + 1) for t in token_collection: self.assertRaises(errors.TokenFailure, TokenList.get, t.id) for f in file_list: self.assertFalse(os.path.exists(f))
def test_proof_of_work_right_answer(self): token = Token('submission') difficulty = { 'human_captcha': False, 'proof_of_work': False } token.generate_token_challenge(difficulty) token = TokenList.get(token.id) # Note, this solution works with two '00' at the end, if the # difficulty changes, also this dummy value has to. token.proof_of_work = {'question': "7GJ4Sl37AEnP10Zk9p7q"} # validate with right value: OK self.assertTrue(token.update({'proof_of_work_answer': 0}))
def test_proof_of_work_right_answer(self): token = Token('submission') difficulty = { 'human_captcha': False, 'graph_captcha': False, 'proof_of_work': False } token.generate_token_challenge(difficulty) token = TokenList.get(token.id) # Note, this solution works with two '00' at the end, if the # difficulty changes, also this dummy value has to. token.proof_of_work = { 'question': "7GJ4Sl37AEnP10Zk9p7q" } # validate with right value: OK self.assertTrue(token.update({'proof_of_work_answer': 0}))
def test_token_create_and_get_upload_expire(self): # This is at the beginning event.EventTrackQueue.reset() file_list = [] token_collection = [] for i in xrange(20): st = Token('submission', context_id='ignored') st.set_difficulty( TestToken.shared_alarm_obj.get_token_difficulty()) token_collection.append(st) for t in token_collection: token = TokenList.get(t.id) difficulty = { 'human_captcha': True, 'graph_captcha': False, 'proof_of_work': False, } token.set_difficulty(difficulty) self.assertRaises(errors.TokenFailure, token.validate, {'human_captcha_answer': 0}) yield self.emulate_file_upload(token, 3) for f in token.uploaded_files: self.assertTrue(os.path.exists(f['encrypted_path'])) file_list.append(f['encrypted_path']) token.expire() self.assertRaises(errors.TokenFailure, TokenList.get, t.id) for f in file_list: self.assertFalse(os.path.exists(f))
def test_token_validate(self): # This is at the beginning event.EventTrackQueue.reset() token = Token("submission", context_id="ignored") difficulty = {"human_captcha": True, "graph_captcha": False, "proof_of_work": False} token.set_difficulty(difficulty) token = TokenList.get(token.token_id) token.human_captcha = {"answer": 1} token.remaining_allowed_attempts = 1 # validate with right value: OK token.validate({"human_captcha_answer": 1}) # validate with wrong value: FAIL self.assertRaises(errors.TokenFailure, token.validate, {"human_captcha_answer": 0}) # validate with right value but with no additional # attemps available: FAIL self.assertRaises(errors.TokenFailure, token.validate, {"human_captcha_answer": 1})
def handle_file_upload(self, token_id): token = TokenList.get(token_id) log.debug("file upload with token associated: %s" % token) uploaded_file = self.get_file_upload() if uploaded_file is None: return uploaded_file['body'].avoid_delete() uploaded_file['body'].close() try: # dump_file_fs return the new filepath inside the dictionary uploaded_file = yield threads.deferToThread(dump_file_fs, uploaded_file) uploaded_file['creation_date'] = datetime_now() token.associate_file(uploaded_file) serialize_memory_file(uploaded_file) except Exception as excep: log.err("Unable to save file in filesystem: %s" % excep) raise errors.InternalServerError("Unable to accept files")
def test_proof_of_work_right_answer(self): # This is at the beginning event.EventTrackQueue.reset() token = Token('submission') difficulty = { 'human_captcha': False, 'graph_captcha': False, 'proof_of_work': False } token.generate_token_challenge(difficulty) token = TokenList.get(token.id) # Note, this solution works with two '00' at the end, if the # difficulty changes, also this dummy value has to. token.proof_of_work = { 'question': "7GJ4Sl37AEnP10Zk9p7q" } # validate with right value: OK self.assertFalse(token.update({'proof_of_work_answer': 26})) # verify that the challenge is marked as solved self.assertFalse(token.proof_of_work)
def test_proof_of_work_right_answer(self): # This is at the beginning event.EventTrackQueue.reset() token = Token('submission') difficulty = { 'human_captcha': False, 'graph_captcha': False, 'proof_of_work': False } token.generate_token_challenge(difficulty) token = TokenList.get(token.id) # Note, this solution works with two '00' at the end, if the # difficulty changes, also this dummy value has to. token.proof_of_work = {'question': "7GJ4Sl37AEnP10Zk9p7q"} # validate with right value: OK self.assertFalse(token.update({'proof_of_work_answer': 26})) # verify that the challenge is marked as solved self.assertFalse(token.proof_of_work)
def post(self, token_id): token = TokenList.get(token_id) self.uploaded_file['submission'] = True token.associate_file(self.uploaded_file)
def db_create_submission(store, token_id, request, t2w, language): # the .get method raise an exception if the token is invalid token = TokenList.get(token_id) if not token.context_associated == request['context_id']: raise errors.InvalidInputFormat("Token context does not match the one specified in submission payload") token.validate(request) TokenList.delete(token_id) answers = request['answers'] context = store.find(Context, Context.id == token.context_associated).one() if not context: # this can happen only if the context is removed # between submission POST and PUT.. :) that's why is better just # ignore this check, take che cached and wait the reference below fault log.err("Context requested: [%s] not found!" % token.context_associated) raise errors.ContextIdNotFound submission = InternalTip() submission.expiration_date = utc_future_date(seconds=context.tip_timetolive) submission.context_id = context.id submission.creation_date = datetime_now() # Tor2Web is spot in the handler and passed here, is done to keep track of the # security level adopted by the whistleblower submission.tor2web = t2w try: questionnaire = db_get_context_steps(store, context.id, GLSettings.memory_copy.default_language) questionnaire_hash = sha256(json.dumps(questionnaire)) submission.questionnaire_hash = questionnaire_hash submission.preview = extract_answers_preview(questionnaire, answers) store.add(submission) db_archive_questionnaire_schema(store, submission) db_save_questionnaire_answers(store, submission, answers) except Exception as excep: log.err("Submission create: fields validation fail: %s" % excep) raise excep try: import_receivers(store, submission, request['receivers']) except Exception as excep: log.err("Submission create: receivers import fail: %s" % excep) raise excep try: for filedesc in token.uploaded_files: associated_f = InternalFile() associated_f.name = filedesc['filename'] associated_f.description = "" associated_f.content_type = filedesc['content_type'] associated_f.size = filedesc['body_len'] associated_f.internaltip_id = submission.id associated_f.file_path = filedesc['encrypted_path'] store.add(associated_f) log.debug("=> file associated %s|%s (%d bytes)" % ( associated_f.name, associated_f.content_type, associated_f.size)) except Exception as excep: log.err("Unable to create a DB entry for file! %s" % excep) raise excep receipt = db_create_whistleblower_tip(store, submission) submission_dict = wb_serialize_internaltip(store, submission) submission_dict.update({'receipt': receipt}) return submission_dict
def db_create_submission(store, token_id, request, t2w, language): # the .get method raise an exception if the token is invalid token = TokenList.get(token_id) token.use() answers = request['answers'] context = store.find(models.Context, models.Context.id == request['context_id']).one() if not context: raise errors.ContextIdNotFound submission = models.InternalTip() submission.progressive = db_assign_submission_progressive(store) submission.expiration_date = utc_future_date(seconds=context.tip_timetolive) # this is get from the client as it the only possibility possible # that would fit with the end to end submission. # the score is only an indicator and not a critical information so we can accept to # be fooled by the malicious user. submission.total_score = request['total_score'] # The use of Tor2Web is detected by the basehandler and the status forwared here; # The status is used to keep track of the security level adopted by the whistleblower submission.tor2web = t2w submission.context_id = context.id submission.enable_two_way_comments = context.enable_two_way_comments submission.enable_two_way_messages = context.enable_two_way_messages submission.enable_attachments = context.enable_attachments submission.enable_whistleblower_identity = context.questionnaire.enable_whistleblower_identity if submission.enable_whistleblower_identity and request['identity_provided']: submission.identity_provided = True submission.identity_provided_date = datetime_now() try: questionnaire = db_get_context_steps(store, context.id, None) questionnaire_hash = unicode(sha256(json.dumps(questionnaire))) submission.questionnaire_hash = questionnaire_hash submission.preview = extract_answers_preview(questionnaire, answers) store.add(submission) db_archive_questionnaire_schema(store, questionnaire, questionnaire_hash) db_save_questionnaire_answers(store, submission.id, answers) except Exception as excep: log.err("Submission create: fields validation fail: %s" % excep) raise excep try: import_receivers(store, submission, request['receivers']) except Exception as excep: log.err("Submission create: receivers import fail: %s" % excep) raise excep try: for filedesc in token.uploaded_files: new_file = models.InternalFile() new_file.name = filedesc['filename'] new_file.description = "" new_file.content_type = filedesc['content_type'] new_file.size = filedesc['body_len'] new_file.internaltip_id = submission.id new_file.submission = filedesc['submission'] new_file.file_path = filedesc['encrypted_path'] store.add(new_file) log.debug("=> file associated %s|%s (%d bytes)" % ( new_file.name, new_file.content_type, new_file.size)) except Exception as excep: log.err("Submission create: unable to create db entry for files: %s" % excep) raise excep receipt, wbtip = db_create_whistleblower_tip(store, submission) submission_dict = serialize_usertip(store, wbtip, language) submission_dict.update({'receipt': receipt}) return submission_dict
def db_create_submission(store, token_id, request, t2w, language): # the .get method raise an exception if the token is invalid token = TokenList.get(token_id) if not token.context_associated == request['context_id']: raise errors.InvalidInputFormat( "Token context does not match the one specified in submission payload" ) token.validate(request) TokenList.delete(token_id) answers = request['answers'] context = store.find(Context, Context.id == token.context_associated).one() if not context: # this can happen only if the context is removed # between submission POST and PUT.. :) that's why is better just # ignore this check, take che cached and wait the reference below fault log.err("Context requested: [%s] not found!" % token.context_associated) raise errors.ContextIdNotFound submission = InternalTip() submission.expiration_date = utc_future_date( seconds=context.tip_timetolive) submission.context_id = context.id submission.creation_date = datetime_now() # Tor2Web is spot in the handler and passed here, is done to keep track of the # security level adopted by the whistleblower submission.tor2web = t2w try: questionnaire = db_get_context_steps( store, context.id, GLSettings.memory_copy.default_language) questionnaire_hash = sha256(json.dumps(questionnaire)) submission.questionnaire_hash = questionnaire_hash submission.preview = extract_answers_preview(questionnaire, answers) store.add(submission) db_archive_questionnaire_schema(store, submission) db_save_questionnaire_answers(store, submission, answers) except Exception as excep: log.err("Submission create: fields validation fail: %s" % excep) raise excep try: import_receivers(store, submission, request['receivers']) except Exception as excep: log.err("Submission create: receivers import fail: %s" % excep) raise excep try: for filedesc in token.uploaded_files: associated_f = InternalFile() associated_f.name = filedesc['filename'] associated_f.description = "" associated_f.content_type = filedesc['content_type'] associated_f.size = filedesc['body_len'] associated_f.internaltip_id = submission.id associated_f.file_path = filedesc['encrypted_path'] store.add(associated_f) log.debug("=> file associated %s|%s (%d bytes)" % (associated_f.name, associated_f.content_type, associated_f.size)) except Exception as excep: log.err("Unable to create a DB entry for file! %s" % excep) raise excep receipt = db_create_whistleblower_tip(store, submission) submission_dict = wb_serialize_internaltip(store, submission) submission_dict.update({'receipt': receipt}) return submission_dict
def db_create_submission(store, token_id, request, t2w, language): # the .get method raise an exception if the token is invalid token = TokenList.get(token_id) token.use() answers = request['answers'] context = store.find(models.Context, models.Context.id == request['context_id']).one() if not context: raise errors.ContextIdNotFound submission = models.InternalTip() submission.progressive = db_assign_submission_progressive(store) submission.expiration_date = utc_future_date(seconds=context.tip_timetolive) # The use of Tor2Web is detected by the basehandler and the status forwared here; # The status is used to keep track of the security level adopted by the whistleblower submission.tor2web = t2w submission.context_id = context.id submission.enable_two_way_comments = context.enable_two_way_comments submission.enable_two_way_messages = context.enable_two_way_messages submission.enable_attachments = context.enable_attachments submission.enable_whistleblower_identity = context.enable_whistleblower_identity if context.enable_whistleblower_identity and request['identity_provided']: submission.identity_provided = True submission.identity_provided_date = datetime_now() try: questionnaire = db_get_context_steps(store, context.id, None) questionnaire_hash = unicode(sha256(json.dumps(questionnaire))) submission.questionnaire_hash = questionnaire_hash submission.preview = extract_answers_preview(questionnaire, answers) store.add(submission) db_archive_questionnaire_schema(store, questionnaire, questionnaire_hash) db_save_questionnaire_answers(store, submission.id, answers) except Exception as excep: log.err("Submission create: fields validation fail: %s" % excep) raise excep try: import_receivers(store, submission, request['receivers']) except Exception as excep: log.err("Submission create: receivers import fail: %s" % excep) raise excep try: for filedesc in token.uploaded_files: associated_f = models.InternalFile() associated_f.name = filedesc['filename'] associated_f.description = "" associated_f.content_type = filedesc['content_type'] associated_f.size = filedesc['body_len'] associated_f.internaltip_id = submission.id associated_f.file_path = filedesc['encrypted_path'] store.add(associated_f) log.debug("=> file associated %s|%s (%d bytes)" % ( associated_f.name, associated_f.content_type, associated_f.size)) except Exception as excep: log.err("Submission create: unable to create db entry for files: %s" % excep) raise excep receipt, wbtip = db_create_whistleblower_tip(store, submission) submission_dict = serialize_usertip(store, wbtip, language) submission_dict.update({'receipt': receipt}) return submission_dict
def setUp(self): yield helpers.TestGL.setUp(self) TokenList.clear() self.pollute_events() yield Alarm.compute_activity_level()