def get_tip_export(store, user_id, rtip_id):
rtip = db_access_rtip(store, user_id, rtip_id)
files = []
for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id):
rf.downloads += 1
file_dict = serialize_receiver_file(rf)
file_dict['name'] = 'files/' + file_dict['name']
files.append(copy.deepcopy(file_dict))
receiver = rtip.receiver
user_language = receiver.user.language
export_dict = {
'type': u'export_template',
'node': db_admin_serialize_node(store, user_language),
'notification': db_get_notification(store, user_language),
'tip': serialize_rtip(store, rtip, user_language),
'context': admin_serialize_context(store, rtip.internaltip.context, user_language),
'receiver': admin_serialize_receiver(receiver, user_language),
'comments': db_get_comment_list(rtip),
'messages': db_get_message_list(rtip),
'files': []
}
export_template = Templating().format_template(export_dict['notification']['export_template'], export_dict).encode('utf-8')
export_dict['files'].append({'buf': export_template, 'name': "data.txt"})
return export_dict
def get_receiver_from_rtip(store, rtip_id, language):
rtip = store.find(ReceiverTip, ReceiverTip.id == rtip_id).one()
if not rtip:
raise errors.TipIdNotFound
return receiver.admin_serialize_receiver(rtip.receiver, language)
def check_for_expiring_submissions(self, store):
threshold = datetime_now() + timedelta(GLSettings.memory_copy.notif.tip_expiration_threshold)
for rtip in store.find(models.ReceiverTip, models.ReceiverTip.internaltip_id == models.InternalTip.id,
models.InternalTip.expiration_date < threshold):
user = rtip.receiver.user
language = user.language
node_desc = db_admin_serialize_node(store, language)
notification_desc = db_get_notification(store, language)
context_desc = admin_serialize_context(store, rtip.internaltip.context, language)
receiver_desc = admin_serialize_receiver(rtip.receiver, language)
tip_desc = serialize_rtip(store, rtip, user.language)
data = {
'type': u'tip_expiration',
'node': node_desc,
'context': context_desc,
'receiver': receiver_desc,
'notification': notification_desc,
'tip': tip_desc
}
subject, body = Templating().get_mail_subject_and_body(data)
mail = models.Mail({
'address': data['receiver']['mail_address'],
'subject': subject,
'body': body
})
store.add(mail)
def get_tip_export(store, user_id, rtip_id, language):
rtip = db_access_rtip(store, user_id, rtip_id)
receiver = rtip.receiver
export_dict = {
'type': u'export_template',
'node': db_admin_serialize_node(store, language),
'notification': db_get_notification(store, language),
'tip': serialize_rtip(store, rtip, language),
'context': admin_serialize_context(store, rtip.internaltip.context, language),
'receiver': admin_serialize_receiver(receiver, language),
'comments': db_get_comment_list(rtip),
'messages': db_get_message_list(rtip),
'files': []
}
export_template = Templating().format_template(export_dict['notification']['export_template'], export_dict).encode('utf-8')
export_dict['files'].append({'buf': export_template, 'name': "data.txt"})
for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id):
rf.downloads += 1
file_dict = serialize_receiver_file(rf)
file_dict['name'] = 'files/' + file_dict['name']
export_dict['files'].append(copy.deepcopy(file_dict))
return export_dict
def serialize_obj(self, store, key, obj, language):
obj_id = obj.id
cache_key = key + '-' + obj_id + '-' + language
cache_obj = None
if cache_key not in self.cache:
if key == 'tip':
itip = store.find(models.InternalTip, id=obj.internaltip_id).one()
cache_obj = serialize_rtip(store, obj, itip, language)
elif key == 'context':
cache_obj = admin_serialize_context(store, obj, language)
elif key == 'receiver':
user = store.find(models.User, id=obj.id).one()
cache_obj = admin_serialize_receiver(store, obj, user, language)
elif key == 'message':
cache_obj = serialize_message(store, obj)
elif key == 'comment':
cache_obj = serialize_comment(store, obj)
elif key == 'file':
cache_obj = models.serializers.serialize_ifile(store, obj)
self.cache[cache_key] = cache_obj
return self.cache[cache_key]
def check_for_expiring_submissions(self, store):
threshold = datetime_now() - timedelta(
GLSettings.memory_copy.tip_expiration_threshold)
for itip in store.find(models.InternalTip,
models.InternalTip.expiration_date < threshold):
for rtip in itip.receivertips:
user = rtip.receiver.user
language = user.language
node_desc = db_admin_serialize_node(store, language)
notification_desc = db_get_notification(store, language)
context_desc = admin_serialize_context(
store, rtip.internaltip.context, language)
receiver_desc = admin_serialize_receiver(
rtip.receiver, language)
tip_desc = serialize_rtip(store, rtip, user.language)
data = {
'type': u'tip_expiration',
'node': node_desc,
'context': context_desc,
'receiver': receiver_desc,
'notification': notification_desc,
'tip': tip_desc
}
subject, body = Templating().get_mail_subject_and_body(data)
mail = models.Mail({
'address': data['receiver']['mail_address'],
'subject': subject,
'body': body
})
store.add(mail)
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
for ifile in store.find(InternalFile, InternalFile.new == True):
if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX:
ifile.new = False
error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
continue
elif ifile.processing_attempts >= 1:
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts += 1
for rtip in ifile.internaltip.receivertips:
receiverfile = ReceiverFile()
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtip.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
# https://github.com/globaleaks/GlobaLeaks/issues/444
# avoid to mark the receiverfile as new if it is part of a submission
# this way we avoid to send unuseful messages
receiverfile.new = False if ifile.submission else True
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(rtip.receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def get_receiver_from_rtip(store, rtip_id, language):
rtip = store.find(ReceiverTip, ReceiverTip.id == rtip_id).one()
if not rtip:
raise errors.TipIdNotFound
return receiver.admin_serialize_receiver(rtip.receiver, language)
def process_ReceiverTip(self, store, rtip, data):
language = rtip.receiver.user.language
data['tip'] = serialize_rtip(store, rtip, language)
data['context'] = admin_serialize_context(store, rtip.internaltip.context, language)
data['receiver'] = admin_serialize_receiver(rtip.receiver, language)
self.process_mail_creation(store, data)
def pgp_validation_check(self, store):
expired_or_expiring = []
for rcvr in store.find(models.Receiver):
if rcvr.user.pgp_key_public and rcvr.user.pgp_key_expiration != datetime_null():
if rcvr.user.pgp_key_expiration < datetime_now():
expired_or_expiring.append(admin_serialize_receiver(rcvr, GLSettings.memory_copy.default_language))
if GLSettings.memory_copy.allow_unencrypted:
# The PGP key status should be downgraded only if the node
# accept non PGP mails/files to be sent/stored.
# If the node wont accept this the pgp key status
# will remain enabled and mail won't be sent by regular flow.
rcvr.user.pgp_key_status = u"disabled"
elif rcvr.user.pgp_key_expiration < datetime_now() - timedelta(days=15):
expired_or_expiring.append(admin_serialize_receiver(rcvr, GLSettings.memory_copy.default_language))
return expired_or_expiring
def get_tip_export(store, user_id, rtip_id, language):
rtip = db_access_rtip(store, user_id, rtip_id)
receiver = rtip.receiver
rtip_dict = serialize_rtip(store, rtip, language)
export_dict = {
'type':
u'export_template',
'node':
db_admin_serialize_node(store, language),
'notification':
db_get_notification(store, language),
'tip':
serialize_rtip(store, rtip, language),
'context':
admin_serialize_context(store, rtip.internaltip.context, language),
'receiver':
admin_serialize_receiver(store, receiver, language),
'comments':
rtip_dict['comments'],
'messages':
rtip_dict['messages'],
'files': []
}
export_template = Templating().format_template(
export_dict['notification']['export_template'],
export_dict).encode('utf-8')
export_template = msdos_encode(export_template)
export_dict['files'].append({'buf': export_template, 'name': "data.txt"})
for rf in store.find(models.ReceiverFile,
models.ReceiverFile.receivertip_id == rtip_id):
rf.downloads += 1
file_dict = models.serializers.serialize_rfile(rf)
file_dict['name'] = 'files/' + file_dict['name']
export_dict['files'].append(file_dict)
rtips_ids = [rt.id for rt in rtip.internaltip.receivertips]
wfs = store.find(models.WhistleblowerFile,
In(models.WhistleblowerFile.receivertip_id, rtips_ids))
for wf in wfs:
file_dict = models.serializers.serialize_wbfile(wf)
file_dict['name'] = 'files_from_recipients/' + file_dict['name']
export_dict['files'].append(file_dict)
return export_dict
def process_Message(self, store, message, data):
# if the message is destinated to the whistleblower no mail should be sent
if message.type == u"receiver":
return
language = message.receivertip.receiver.user.language
data['message'] = serialize_message(message)
data['tip'] = serialize_rtip(store, message.receivertip, language)
data['context'] = admin_serialize_context(store, message.receivertip.internaltip.context, language)
data['receiver'] = admin_serialize_receiver(message.receivertip.receiver, language)
self.process_mail_creation(store, data)
def pgp_validation_check(self, store):
expired_or_expiring = []
for rcvr in store.find(models.Receiver):
if rcvr.user.pgp_key_public and rcvr.user.pgp_key_expiration != datetime_null(
):
if rcvr.user.pgp_key_expiration < datetime_now():
expired_or_expiring.append(
admin_serialize_receiver(
rcvr, GLSettings.memory_copy.default_language))
if GLSettings.memory_copy.allow_unencrypted:
# The PGP key status should be downgraded only if the node
# accept non PGP mails/files to be sent/stored.
# If the node wont accept this the pgp key status
# will remain enabled and mail won't be sent by regular flow.
rcvr.user.pgp_key_status = u'disabled'
elif rcvr.user.pgp_key_expiration < datetime_now() - timedelta(
days=15):
expired_or_expiring.append(
admin_serialize_receiver(
rcvr, GLSettings.memory_copy.default_language))
return expired_or_expiring
def process_Comment(self, store, comment, data):
for rtip in comment.internaltip.receivertips:
if comment.type == u'receiver' and comment.author == rtip.receiver.user.name:
continue
language = rtip.receiver.user.language
dataX = copy.deepcopy(data)
dataX['comment'] = serialize_comment(comment)
dataX['tip'] = serialize_rtip(store, rtip, language)
dataX['context'] = admin_serialize_context(store, comment.internaltip.context, language)
dataX['receiver'] = admin_serialize_receiver(rtip.receiver, language)
self.process_mail_creation(store, dataX)
def get_tip_export(store, user_id, rtip_id, language):
rtip = db_access_rtip(store, user_id, rtip_id)
receiver = rtip.receiver
rtip_dict = serialize_rtip(store, rtip, language)
export_dict = {
'type': u'export_template',
'node': db_admin_serialize_node(store, language),
'notification': db_get_notification(store, language),
'tip': serialize_rtip(store, rtip, language),
'context': admin_serialize_context(store, rtip.internaltip.context, language),
'receiver': admin_serialize_receiver(receiver, language),
'comments': rtip_dict['comments'],
'messages': rtip_dict['messages'],
'files': []
}
export_template = Templating().format_template(export_dict['notification']['export_template'], export_dict).encode('utf-8')
export_template = msdos_encode(export_template)
export_dict['files'].append({'buf': export_template, 'name': "data.txt"})
for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id):
rf.downloads += 1
file_dict = models.serializers.serialize_rfile(rf)
file_dict['name'] = 'files/' + file_dict['name']
export_dict['files'].append(copy.deepcopy(file_dict))
rtips_ids = [rt.id for rt in rtip.internaltip.receivertips]
wfs = store.find(models.WhistleblowerFile, In(models.WhistleblowerFile.receivertip_id, rtips_ids))
for wf in wfs:
file_dict = models.serializers.serialize_wbfile(wf)
file_dict['name'] = 'files_from_recipients/' + file_dict['name']
export_dict['files'].append(copy.deepcopy(file_dict))
return export_dict
def serialize_content(store, cache, key, obj, language):
obj_id = obj.id
cache_key = key + obj_id + language
if cache_key not in cache:
if key == 'tip':
cache_obj = serialize_rtip(store, obj, language)
elif key == 'context':
cache_obj = admin_serialize_context(store, obj, language)
elif key == 'receiver':
cache_obj = admin_serialize_receiver(obj, language)
elif key == 'message':
cache_obj = serialize_message(obj)
elif key == 'comment':
cache_obj = serialize_comment(obj)
elif key == 'file':
cache_obj = serialize_internalfile(obj)
cache[cache_key] = cache_obj
return copy.deepcopy(cache[cache_key])
def serialize_content(store, cache, key, obj, language):
obj_id = obj.id
cache_key = key + obj_id + language
if cache_key not in cache:
if key == 'tip':
cache_obj = serialize_rtip(store, obj, language)
elif key == 'context':
cache_obj = admin_serialize_context(store, obj, language)
elif key == 'receiver':
cache_obj = admin_serialize_receiver(obj, language)
elif key == 'message':
cache_obj = serialize_message(obj)
elif key == 'comment':
cache_obj = serialize_comment(obj)
elif key == 'file':
cache_obj = serialize_internalfile(obj)
cache[cache_key] = cache_obj
return copy.deepcopy(cache[cache_key])
def serialize_obj(self, store, key, obj, language):
obj_id = obj.id
cache_key = key + '-' + obj_id + '-' + language
if cache_key not in self.cache:
if key == 'tip':
cache_obj = serialize_rtip(store, obj, language)
elif key == 'context':
cache_obj = admin_serialize_context(store, obj, language)
elif key == 'receiver':
cache_obj = admin_serialize_receiver(obj, language)
elif key == 'message':
cache_obj = serialize_message(obj)
elif key == 'comment':
cache_obj = serialize_comment(obj)
elif key == 'file':
cache_obj = serialize_internalfile(obj)
self.cache[cache_key] = cache_obj
return self.cache[cache_key]
def serialize_obj(self, store, key, obj, language):
obj_id = obj.id
cache_key = key + '-' + obj_id + '-' + language
if cache_key not in self.cache:
if key == 'tip':
cache_obj = serialize_rtip(store, obj, language)
elif key == 'context':
cache_obj = admin_serialize_context(store, obj, language)
elif key == 'receiver':
cache_obj = admin_serialize_receiver(obj, language)
elif key == 'message':
cache_obj = serialize_message(obj)
elif key == 'comment':
cache_obj = serialize_comment(obj)
elif key == 'file':
cache_obj = serialize_internalfile(obj)
self.cache[cache_key] = cache_obj
return self.cache[cache_key]
def get_file_collection(store, user_id, rtip_id):
rtip = db_access_rtip(store, user_id, rtip_id)
archive_dict = {'files': [], 'file_counter': 0, 'total_size': 0}
file_list = []
for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id):
rf.downloads += 1
file_list.append(serialize_receiver_file(rf))
archive_dict['file_counter'] += 1
archive_dict['total_size'] += rf.size
archive_dict['files'].append(serialize_receiver_file(rf))
receiver = rtip.receiver
user_language = receiver.user.language
data = {
'type': u'archive_description',
'node': db_admin_serialize_node(store, user_language),
'notification': db_get_notification(store, user_language),
'tip': serialize_rtip(store, rtip, user_language),
'context': admin_serialize_context(store, rtip.internaltip.context, user_language),
'receiver': admin_serialize_receiver(receiver, user_language),
'archive': archive_dict
}
archive_description = Templating().format_template(data['notification']['archive_description'], data).encode('utf-8')
file_list.append(
{
'buf': archive_description,
'name': "COLLECTION_INFO.txt"
}
)
return file_list
def check_for_expiring_submissions(self, store):
threshold = datetime_now() + timedelta(
hours=GLSettings.memory_copy.notif.tip_expiration_threshold)
receivers = store.find(models.Receiver)
for receiver in receivers:
rtips = store.find(
models.ReceiverTip,
models.ReceiverTip.internaltip_id == models.InternalTip.id,
models.InternalTip.expiration_date < threshold,
models.ReceiverTip.receiver_id == models.Receiver.id,
models.Receiver.id == receiver.id)
if rtips.count() == 0:
continue
user = receiver.user
language = user.language
node_desc = db_admin_serialize_node(store, language)
notification_desc = db_get_notification(store, language)
receiver_desc = admin_serialize_receiver(store, receiver, language)
if rtips.count() == 1:
rtip = rtips[0]
tip_desc = serialize_rtip(store, rtip, user.language)
context_desc = admin_serialize_context(
store, rtip.internaltip.context, language)
data = {
'type': u'tip_expiration',
'node': node_desc,
'context': context_desc,
'receiver': receiver_desc,
'notification': notification_desc,
'tip': tip_desc
}
else:
tips_desc = []
earliest_expiration_date = datetime_never()
for rtip in rtips:
if rtip.internaltip.expiration_date < earliest_expiration_date:
earliest_expiration_date = rtip.internaltip.expiration_date
tips_desc.append(serialize_rtip(store, rtip,
user.language))
data = {
'type':
u'tip_expiration_summary',
'node':
node_desc,
'notification':
notification_desc,
'receiver':
receiver_desc,
'expiring_submission_count':
rtips.count(),
'earliest_expiration_date':
datetime_to_ISO8601(earliest_expiration_date)
}
subject, body = Templating().get_mail_subject_and_body(data)
mail = models.Mail({
'address': receiver_desc['mail_address'],
'subject': subject,
'body': body
})
store.add(mail)
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug("Delivery iterating over %d InternalFile from a queue of %d" % (
GLSettings.jobs_operation_limit, ifilescnt))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX:
ifile.new = False
error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif ifile.processing_attempts >= 1:
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
