def db_postpone_expiration_date(session, tid, itip):
context = session.query(models.Context).filter(models.Context.id == itip.context_id, models.Context.tid == tid).one()
if context.tip_timetolive > -1:
itip.expiration_date = get_expiration(context.tip_timetolive)
else:
itip.expiration_date = datetime_never()
def db_postpone_expiration_date(session, tid, itip):
context = session.query(models.Context).filter(models.Context.id == itip.context_id, models.Context.tid == tid).one()
if context.tip_timetolive > 0:
itip.expiration_date = get_expiration(context.tip_timetolive)
else:
itip.expiration_date = datetime_never()
def db_postpone_expiration_date(store, itip):
context = store.find((models.Context), id=itip.context_id).one()
if context.tip_timetolive > -1:
itip.expiration_date = get_expiration(context.tip_timetolive)
else:
itip.expiration_date = datetime_never()
def db_postpone_expiration(session, itip):
"""
Transaction for postponing the expiration of a submission
:param session: An ORM session
:param itip: A submission model to be postponed
"""
context = session.query(
models.Context).filter(models.Context.id == itip.context_id).one()
if context.tip_timetolive > 0:
itip.expiration_date = get_expiration(context.tip_timetolive)
else:
itip.expiration_date = datetime_never()
def test_put_postpone_never(self):
yield self.force_itip_expiration()
yield self.set_contexts_timetolive(-1)
rtip_descs = yield self.get_rtips()
for rtip_desc in rtip_descs:
self.assertTrue(rtip_desc['expiration_date'] == datetime_null())
operation = {'operation': 'postpone_expiration', 'args': {}}
handler = self.request(operation,
role='receiver',
user_id=rtip_desc['receiver_id'])
yield handler.put(rtip_desc['id'])
self.assertEqual(handler.request.code, 200)
rtip_descs = yield self.get_rtips()
for rtip_desc in rtip_descs:
self.assertTrue(rtip_desc['expiration_date'] == datetime_never())
def db_create_submission(session, tid, request, uploaded_files, client_using_tor):
answers = request['answers']
context, questionnaire = session.query(models.Context, models.Questionnaire) \
.filter(models.Context.id == request['context_id'],
models.Questionnaire.id == models.Context.questionnaire_id,
models.Questionnaire.tid.in_(set([1, tid]))).one_or_none()
if not context:
raise errors.ModelNotFound(models.Context)
steps = db_get_questionnaire(session, tid, questionnaire.id, None)['steps']
questionnaire_hash = text_type(sha256(json.dumps(steps)))
db_archive_questionnaire_schema(session, steps, questionnaire_hash)
submission = models.InternalTip()
submission.tid = tid
submission.status = db_get_id_for_system_status(session, tid, 'new')
submission.progressive = db_assign_submission_progressive(session, tid)
if context.tip_timetolive > 0:
submission.expiration_date = get_expiration(context.tip_timetolive)
else:
submission.expiration_date = datetime_never()
# this is get from the client as it the only possibility possible
# that would fit with the end to end submission.
# the score is only an indicator and not a critical information so we can accept to
# be fooled by the malicious user.
submission.total_score = request['total_score']
# The status https is used to keep track of the security level adopted by the whistleblower
submission.https = not client_using_tor
submission.context_id = context.id
submission.enable_two_way_comments = context.enable_two_way_comments
submission.enable_two_way_messages = context.enable_two_way_messages
submission.enable_attachments = context.enable_attachments
whistleblower_identity = session.query(models.Field) \
.filter(models.Field.template_id == u'whistleblower_identity',
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id == context.questionnaire_id).one_or_none()
submission.enable_whistleblower_identity = whistleblower_identity is not None
if submission.enable_whistleblower_identity and request['identity_provided']:
submission.identity_provided = True
submission.identity_provided_date = datetime_now()
submission.questionnaire_hash = questionnaire_hash
submission.preview = extract_answers_preview(steps, answers)
session.add(submission)
session.flush()
receipt = text_type(generateRandomReceipt())
wbtip = models.WhistleblowerTip()
wbtip.id = submission.id
wbtip.tid = submission.tid
wbtip.receipt_hash = hash_password(receipt, State.tenant_cache[tid].receipt_salt)
session.add(wbtip)
db_save_questionnaire_answers(session, tid, submission.id, answers)
for filedesc in uploaded_files:
new_file = models.InternalFile()
new_file.tid = tid
new_file.name = filedesc['name']
new_file.description = ""
new_file.content_type = filedesc['type']
new_file.size = filedesc['size']
new_file.internaltip_id = submission.id
new_file.submission = filedesc['submission']
new_file.filename = filedesc['filename']
session.add(new_file)
log.debug("=> file associated %s|%s (%d bytes)",
new_file.name, new_file.content_type, new_file.size)
if context.maximum_selectable_receivers > 0 and \
len(request['receivers']) > context.maximum_selectable_receivers:
raise errors.InputValidationError("selected an invalid number of recipients")
rtips_count = 0
for receiver, user in session.query(models.Receiver, models.User) \
.filter(models.Receiver.id.in_(request['receivers']),
models.ReceiverContext.receiver_id == models.Receiver.id,
models.ReceiverContext.context_id == context.id,
models.User.id == models.Receiver.id,
models.UserTenant.user_id == models.User.id,
models.UserTenant.tenant_id == tid):
if user.pgp_key_public or State.tenant_cache[tid].allow_unencrypted:
db_create_receivertip(session, receiver, submission)
rtips_count += 1
if rtips_count == 0:
raise errors.InputValidationError("need at least one recipient")
log.debug("The finalized submission had created %d models.ReceiverTip(s)", rtips_count)
return {'receipt': receipt}
def test_is_expired(self):
self.assertTrue(utility.is_expired(utility.datetime_null()))
self.assertTrue(utility.is_expired(utility.datetime_now()))
self.assertFalse(utility.is_expired(utility.datetime_never()))
def set_expiration_of_all_rtips_to_unlimited(session):
session.query(models.InternalTip).update({'expiration_date': datetime_never()})
def set_expiration_of_all_rtips_to_unlimited(store):
store.find(models.InternalTip).set(expiration_date = datetime_never())
def db_create_submission(session, tid, request, token, client_using_tor):
answers = request['answers']
context, questionnaire = session.query(models.Context, models.Questionnaire) \
.filter(models.Context.id == request['context_id'],
models.Questionnaire.id == models.Context.questionnaire_id,
models.Questionnaire.tid.in_(set([1, tid]))).one_or_none()
if not context:
raise errors.ModelNotFound(models.Context)
steps = db_get_questionnaire(session, tid, questionnaire.id, None)['steps']
questionnaire_hash = db_archive_questionnaire_schema(session, steps)
itip = models.InternalTip()
itip.tid = tid
itip.status = db_get_id_for_system_status(session, tid, u'new')
itip.progressive = db_assign_submission_progressive(session, tid)
itip.additional_questionnaire_id = context.additional_questionnaire_id
if context.tip_timetolive > 0:
itip.expiration_date = get_expiration(context.tip_timetolive)
else:
itip.expiration_date = datetime_never()
# this is get from the client as it the only possibility possible
# that would fit with the end to end submission.
# the score is only an indicator and not a critical information so we can accept to
# be fooled by the malicious user.
itip.total_score = request['total_score']
# The status https is used to keep track of the security level adopted by the whistleblower
itip.https = not client_using_tor
itip.context_id = context.id
itip.enable_two_way_comments = context.enable_two_way_comments
itip.enable_two_way_messages = context.enable_two_way_messages
itip.enable_attachments = context.enable_attachments
x = session.query(models.Field, models.FieldAttr.value) \
.filter(models.Field.template_id == u'whistleblower_identity',
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id == context.questionnaire_id,
models.FieldAttr.field_id == models.Field.id,
models.FieldAttr.name == u'visibility_subject_to_authorization').one_or_none()
whistleblower_identity = None
can_access_whistleblower_identity = True
if x:
whistleblower_identity = x[0]
can_access_whistleblower_identity = not x[1]
itip.enable_whistleblower_identity = whistleblower_identity is not None
itip.preview = extract_answers_preview(steps, answers)
session.add(itip)
session.flush()
receipt = GCE.generate_receipt()
receipt_salt = State.tenant_cache[tid].receipt_salt
wbtip = models.WhistleblowerTip()
wbtip.id = itip.id
wbtip.tid = tid
wbtip.hash_alg = GCE.HASH
wbtip.receipt_hash = GCE.hash_password(receipt, receipt_salt)
crypto_is_available = State.tenant_cache[1].encryption
if crypto_is_available:
users_count = session.query(models.User) \
.filter(models.Receiver.id.in_(request['receivers']),
models.ReceiverContext.receiver_id == models.Receiver.id,
models.ReceiverContext.context_id == context.id,
models.User.id == models.Receiver.id,
models.User.crypto_prv_key != b'',
models.UserTenant.user_id == models.User.id,
models.UserTenant.tenant_id == tid).count()
crypto_is_available = users_count == len(request['receivers'])
if crypto_is_available:
crypto_tip_prv_key, itip.crypto_tip_pub_key = GCE.generate_keypair()
wb_key = GCE.derive_key(receipt.encode(), receipt_salt)
wb_prv_key, wb_pub_key = GCE.generate_keypair()
wbtip.crypto_prv_key = GCE.symmetric_encrypt(wb_key, wb_prv_key)
wbtip.crypto_pub_key = wb_pub_key
wbtip.crypto_tip_prv_key = GCE.asymmetric_encrypt(
wb_pub_key, crypto_tip_prv_key)
if itip.enable_whistleblower_identity and request[
'identity_provided'] and answers[whistleblower_identity.id]:
wbi = answers[whistleblower_identity.id][0]
answers[whistleblower_identity.id] = ''
if crypto_is_available:
wbi = base64.b64encode(
GCE.asymmetric_encrypt(itip.crypto_tip_pub_key,
json.dumps(wbi).encode())).decode()
db_set_internaltip_data(session, itip.id, 'identity_provided', True,
False)
db_set_internaltip_data(session, itip.id, 'whistleblower_identity',
wbi, crypto_is_available)
if crypto_is_available:
answers = base64.b64encode(
GCE.asymmetric_encrypt(itip.crypto_tip_pub_key,
json.dumps(answers).encode())).decode()
else:
db_save_questionnaire_answers(session, tid, itip.id, answers)
db_set_internaltip_answers(session, itip.id, questionnaire_hash, answers,
crypto_is_available)
session.add(wbtip)
for filedesc in token.uploaded_files:
new_file = models.InternalFile()
new_file.tid = tid
new_file.encrypted = crypto_is_available
new_file.name = filedesc['name']
new_file.description = ""
new_file.content_type = filedesc['type']
new_file.size = filedesc['size']
new_file.internaltip_id = itip.id
new_file.submission = filedesc['submission']
new_file.filename = filedesc['filename']
session.add(new_file)
log.debug("=> file associated %s|%s (%d bytes)", new_file.name,
new_file.content_type, new_file.size)
if context.maximum_selectable_receivers > 0 and \
len(request['receivers']) > context.maximum_selectable_receivers:
raise errors.InputValidationError(
"selected an invalid number of recipients")
rtips_count = 0
for receiver, user in session.query(models.Receiver, models.User) \
.filter(models.Receiver.id.in_(request['receivers']),
models.ReceiverContext.receiver_id == models.Receiver.id,
models.ReceiverContext.context_id == context.id,
models.User.id == models.Receiver.id,
models.UserTenant.user_id == models.User.id,
models.UserTenant.tenant_id == tid):
if not crypto_is_available and not user.pgp_key_public and not State.tenant_cache[
tid].allow_unencrypted:
continue
_tip_key = b''
if crypto_is_available:
_tip_key = GCE.asymmetric_encrypt(user.crypto_pub_key,
crypto_tip_prv_key)
db_create_receivertip(session, receiver, itip,
can_access_whistleblower_identity, _tip_key)
rtips_count += 1
if not rtips_count:
raise errors.InputValidationError("need at least one recipient")
log.debug("The finalized submission had created %d models.ReceiverTip(s)",
rtips_count)
return {'receipt': receipt}
def set_expiration_of_all_rtips_to_unlimited(store):
for rtip in store.find(models.ReceiverTip):
rtip.internaltip.expiration_date = datetime_never()
def db_create_submission(store, request, uploaded_files, client_using_tor):
answers = request['answers']
context, questionnaire = store.find(
(models.Context, models.Questionnaire),
models.Context.id == request['context_id'],
models.Questionnaire.id == models.Context.questionnaire_id).one()
if not context:
raise errors.ModelNotFound(models.Context)
steps = db_get_questionnaire(store, questionnaire.id, None)['steps']
questionnaire_hash = unicode(sha256(json.dumps(steps)))
submission = models.InternalTip()
submission.progressive = db_assign_submission_progressive(store)
if context.tip_timetolive > -1:
submission.expiration_date = get_expiration(context.tip_timetolive)
else:
submission.expiration_date = datetime_never()
# this is get from the client as it the only possibility possible
# that would fit with the end to end submission.
# the score is only an indicator and not a critical information so we can accept to
# be fooled by the malicious user.
submission.total_score = request['total_score']
# The status tor2web is used to keep track of the security level adopted by the whistleblower
submission.tor2web = not client_using_tor
submission.context_id = context.id
submission.enable_two_way_comments = context.enable_two_way_comments
submission.enable_two_way_messages = context.enable_two_way_messages
submission.enable_attachments = context.enable_attachments
submission.enable_whistleblower_identity = questionnaire.enable_whistleblower_identity
if submission.enable_whistleblower_identity and request[
'identity_provided']:
submission.identity_provided = True
submission.identity_provided_date = datetime_now()
submission.questionnaire_hash = questionnaire_hash
submission.preview = extract_answers_preview(steps, answers)
store.add(submission)
db_archive_questionnaire_schema(store, steps, questionnaire_hash)
db_save_questionnaire_answers(store, submission.id, answers)
for filedesc in uploaded_files:
new_file = models.InternalFile()
new_file.name = filedesc['name']
new_file.description = ""
new_file.content_type = filedesc['type']
new_file.size = filedesc['size']
new_file.internaltip_id = submission.id
new_file.submission = filedesc['submission']
new_file.file_path = filedesc['path']
store.add(new_file)
log.debug("=> file associated %s|%s (%d bytes)", new_file.name,
new_file.content_type, new_file.size)
receipt = db_create_whistleblowertip(store, submission)
if context.maximum_selectable_receivers > 0 and \
len(request['receivers']) > context.maximum_selectable_receivers:
raise errors.SubmissionValidationFailure(
"selected an invalid number of recipients")
rtips_count = 0
for receiver, user, in store.find(
(models.Receiver, models.User),
In(models.Receiver.id, request['receivers']),
models.ReceiverContext.receiver_id == models.Receiver.id,
models.ReceiverContext.context_id == context.id,
models.User.id == models.Receiver.id):
if user.pgp_key_public or GLSettings.memory_copy.allow_unencrypted:
db_create_receivertip(store, receiver, submission)
rtips_count += 1
if rtips_count == 0:
raise errors.SubmissionValidationFailure("need at least one recipient")
log.debug("The finalized submission had created %d models.ReceiverTip(s)",
rtips_count)
return {'receipt': receipt}
def db_create_submission(store, request, uploaded_files, t2w, language):
answers = request['answers']
context = store.find(models.Context, models.Context.id == request['context_id']).one()
if not context:
raise errors.ContextIdNotFound
submission = models.InternalTip()
submission.progressive = db_assign_submission_progressive(store)
if context.tip_timetolive > -1:
submission.expiration_date = utc_future_date(days=context.tip_timetolive)
else:
submission.expiration_date = datetime_never()
# this is get from the client as it the only possibility possible
# that would fit with the end to end submission.
# the score is only an indicator and not a critical information so we can accept to
# be fooled by the malicious user.
submission.total_score = request['total_score']
# The use of Tor2Web is detected by the basehandler and the status forwared here;
# The status is used to keep track of the security level adopted by the whistleblower
submission.tor2web = t2w
submission.context_id = context.id
submission.enable_two_way_comments = context.enable_two_way_comments
submission.enable_two_way_messages = context.enable_two_way_messages
submission.enable_attachments = context.enable_attachments
submission.enable_whistleblower_identity = context.questionnaire.enable_whistleblower_identity
if submission.enable_whistleblower_identity and request['identity_provided']:
submission.identity_provided = True
submission.identity_provided_date = datetime_now()
try:
questionnaire = db_get_context_steps(store, context.id, None)
questionnaire_hash = unicode(sha256(json.dumps(questionnaire)))
submission.questionnaire_hash = questionnaire_hash
submission.preview = extract_answers_preview(questionnaire, answers)
store.add(submission)
db_archive_questionnaire_schema(store, questionnaire, questionnaire_hash)
db_save_questionnaire_answers(store, submission.id, answers)
except Exception as excep:
log.err("Submission create: fields validation fail: %s" % excep)
raise excep
try:
for filedesc in uploaded_files:
new_file = models.InternalFile()
new_file.name = filedesc['name']
new_file.description = ""
new_file.content_type = filedesc['type']
new_file.size = filedesc['size']
new_file.internaltip_id = submission.id
new_file.submission = filedesc['submission']
new_file.file_path = filedesc['path']
store.add(new_file)
log.debug("=> file associated %s|%s (%d bytes)" % (
new_file.name, new_file.content_type, new_file.size))
except Exception as excep:
log.err("Submission create: unable to create db entry for files: %s" % excep)
raise excep
receipt, wbtip = db_create_whistleblowertip(store, submission)
if submission.context.maximum_selectable_receivers > 0 and \
len(request['receivers']) > submission.context.maximum_selectable_receivers:
raise errors.SubmissionValidationFailure("provided an invalid number of receivers")
rtips = []
for receiver in store.find(models.Receiver, In(models.Receiver.id, request['receivers'])):
if submission.context not in receiver.contexts:
continue
if not GLSettings.memory_copy.allow_unencrypted and len(receiver.user.pgp_key_public) == 0:
continue
rtips.append(db_create_receivertip(store, receiver, submission))
if len(rtips) == 0:
raise errors.SubmissionValidationFailure("needed almost one receiver")
log.debug("The finalized submission had created %d models.ReceiverTip(s)" % len(rtips))
submission_dict = serialize_usertip(store, wbtip, language)
submission_dict.update({'receipt': receipt})
return submission_dict
def db_create_submission(store, request, uploaded_files, client_using_tor,
language):
answers = request['answers']
context = store.find(models.Context,
models.Context.id == request['context_id']).one()
if not context:
raise errors.ContextIdNotFound
submission = models.InternalTip()
submission.progressive = db_assign_submission_progressive(store)
if context.tip_timetolive > -1:
submission.expiration_date = get_expiration(context.tip_timetolive)
else:
submission.expiration_date = datetime_never()
# this is get from the client as it the only possibility possible
# that would fit with the end to end submission.
# the score is only an indicator and not a critical information so we can accept to
# be fooled by the malicious user.
submission.total_score = request['total_score']
# The status tor2web is used to keep track of the security level adopted by the whistleblower
submission.tor2web = not client_using_tor
submission.context_id = context.id
submission.enable_two_way_comments = context.enable_two_way_comments
submission.enable_two_way_messages = context.enable_two_way_messages
submission.enable_attachments = context.enable_attachments
submission.enable_whistleblower_identity = context.questionnaire.enable_whistleblower_identity
if submission.enable_whistleblower_identity and request[
'identity_provided']:
submission.identity_provided = True
submission.identity_provided_date = datetime_now()
try:
questionnaire = db_get_context_steps(store, context.id, None)
questionnaire_hash = unicode(sha256(json.dumps(questionnaire)))
submission.questionnaire_hash = questionnaire_hash
submission.preview = extract_answers_preview(questionnaire, answers)
store.add(submission)
db_archive_questionnaire_schema(store, questionnaire,
questionnaire_hash)
db_save_questionnaire_answers(store, submission.id, answers)
except Exception as excep:
log.err("Submission create: fields validation fail: %s" % excep)
raise excep
try:
for filedesc in uploaded_files:
new_file = models.InternalFile()
new_file.name = filedesc['name']
new_file.description = ""
new_file.content_type = filedesc['type']
new_file.size = filedesc['size']
new_file.internaltip_id = submission.id
new_file.submission = filedesc['submission']
new_file.file_path = filedesc['path']
store.add(new_file)
log.debug("=> file associated %s|%s (%d bytes)" %
(new_file.name, new_file.content_type, new_file.size))
except Exception as excep:
log.err("Submission create: unable to create db entry for files: %s" %
excep)
raise excep
receipt, wbtip = db_create_whistleblowertip(store, submission)
if submission.context.maximum_selectable_receivers > 0 and \
len(request['receivers']) > submission.context.maximum_selectable_receivers:
raise errors.SubmissionValidationFailure(
"selected an invalid number of recipients")
rtips = []
for receiver in store.find(models.Receiver,
In(models.Receiver.id, request['receivers'])):
if submission.context not in receiver.contexts:
continue
if not GLSettings.memory_copy.allow_unencrypted and len(
receiver.user.pgp_key_public) == 0:
continue
rtips.append(db_create_receivertip(store, receiver, submission))
if len(rtips) == 0:
raise errors.SubmissionValidationFailure("need at least one recipient")
log.debug("The finalized submission had created %d models.ReceiverTip(s)" %
len(rtips))
submission_dict = serialize_usertip(store, wbtip, language)
submission_dict.update({'receipt': receipt})
return submission_dict
def check_for_expiring_submissions(self, store):
threshold = datetime_now() + timedelta(
hours=GLSettings.memory_copy.notif.tip_expiration_threshold)
receivers = store.find(models.Receiver)
for receiver in receivers:
rtips = store.find(
models.ReceiverTip,
models.ReceiverTip.internaltip_id == models.InternalTip.id,
models.InternalTip.expiration_date < threshold,
models.ReceiverTip.receiver_id == models.Receiver.id,
models.Receiver.id == receiver.id)
if rtips.count() == 0:
continue
user = receiver.user
language = user.language
node_desc = db_admin_serialize_node(store, language)
notification_desc = db_get_notification(store, language)
receiver_desc = admin_serialize_receiver(store, receiver, language)
if rtips.count() == 1:
rtip = rtips[0]
tip_desc = serialize_rtip(store, rtip, user.language)
context_desc = admin_serialize_context(
store, rtip.internaltip.context, language)
data = {
'type': u'tip_expiration',
'node': node_desc,
'context': context_desc,
'receiver': receiver_desc,
'notification': notification_desc,
'tip': tip_desc
}
else:
tips_desc = []
earliest_expiration_date = datetime_never()
for rtip in rtips:
if rtip.internaltip.expiration_date < earliest_expiration_date:
earliest_expiration_date = rtip.internaltip.expiration_date
tips_desc.append(serialize_rtip(store, rtip,
user.language))
data = {
'type':
u'tip_expiration_summary',
'node':
node_desc,
'notification':
notification_desc,
'receiver':
receiver_desc,
'expiring_submission_count':
rtips.count(),
'earliest_expiration_date':
datetime_to_ISO8601(earliest_expiration_date)
}
subject, body = Templating().get_mail_subject_and_body(data)
mail = models.Mail({
'address': receiver_desc['mail_address'],
'subject': subject,
'body': body
})
store.add(mail)
def set_expiration_of_all_rtips_to_unlimited(session):
session.query(models.InternalTip).update(
{'expiration_date': datetime_never()})
def test_is_expired(self):
self.assertTrue(utility.is_expired(utility.datetime_null()))
self.assertTrue(utility.is_expired(utility.datetime_now()))
self.assertFalse(utility.is_expired(utility.datetime_never()))
