def get_tip_export(store, user_id, rtip_id): rtip = db_access_rtip(store, user_id, rtip_id) files = [] for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id): rf.downloads += 1 file_dict = serialize_receiver_file(rf) file_dict['name'] = 'files/' + file_dict['name'] files.append(copy.deepcopy(file_dict)) receiver = rtip.receiver user_language = receiver.user.language export_dict = { 'type': u'export_template', 'node': db_admin_serialize_node(store, user_language), 'notification': db_get_notification(store, user_language), 'tip': serialize_rtip(store, rtip, user_language), 'context': admin_serialize_context(store, rtip.internaltip.context, user_language), 'receiver': admin_serialize_receiver(receiver, user_language), 'comments': db_get_comment_list(rtip), 'messages': db_get_message_list(rtip), 'files': [] } export_template = Templating().format_template(export_dict['notification']['export_template'], export_dict).encode('utf-8') export_dict['files'].append({'buf': export_template, 'name': "data.txt"}) return export_dict
def get_receiver_from_rtip(store, rtip_id, language): rtip = store.find(ReceiverTip, ReceiverTip.id == rtip_id).one() if not rtip: raise errors.TipIdNotFound return receiver.admin_serialize_receiver(rtip.receiver, language)
def check_for_expiring_submissions(self, store): threshold = datetime_now() + timedelta(GLSettings.memory_copy.notif.tip_expiration_threshold) for rtip in store.find(models.ReceiverTip, models.ReceiverTip.internaltip_id == models.InternalTip.id, models.InternalTip.expiration_date < threshold): user = rtip.receiver.user language = user.language node_desc = db_admin_serialize_node(store, language) notification_desc = db_get_notification(store, language) context_desc = admin_serialize_context(store, rtip.internaltip.context, language) receiver_desc = admin_serialize_receiver(rtip.receiver, language) tip_desc = serialize_rtip(store, rtip, user.language) data = { 'type': u'tip_expiration', 'node': node_desc, 'context': context_desc, 'receiver': receiver_desc, 'notification': notification_desc, 'tip': tip_desc } subject, body = Templating().get_mail_subject_and_body(data) mail = models.Mail({ 'address': data['receiver']['mail_address'], 'subject': subject, 'body': body }) store.add(mail)
def get_tip_export(store, user_id, rtip_id, language): rtip = db_access_rtip(store, user_id, rtip_id) receiver = rtip.receiver export_dict = { 'type': u'export_template', 'node': db_admin_serialize_node(store, language), 'notification': db_get_notification(store, language), 'tip': serialize_rtip(store, rtip, language), 'context': admin_serialize_context(store, rtip.internaltip.context, language), 'receiver': admin_serialize_receiver(receiver, language), 'comments': db_get_comment_list(rtip), 'messages': db_get_message_list(rtip), 'files': [] } export_template = Templating().format_template(export_dict['notification']['export_template'], export_dict).encode('utf-8') export_dict['files'].append({'buf': export_template, 'name': "data.txt"}) for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id): rf.downloads += 1 file_dict = serialize_receiver_file(rf) file_dict['name'] = 'files/' + file_dict['name'] export_dict['files'].append(copy.deepcopy(file_dict)) return export_dict
def serialize_obj(self, store, key, obj, language): obj_id = obj.id cache_key = key + '-' + obj_id + '-' + language cache_obj = None if cache_key not in self.cache: if key == 'tip': itip = store.find(models.InternalTip, id=obj.internaltip_id).one() cache_obj = serialize_rtip(store, obj, itip, language) elif key == 'context': cache_obj = admin_serialize_context(store, obj, language) elif key == 'receiver': user = store.find(models.User, id=obj.id).one() cache_obj = admin_serialize_receiver(store, obj, user, language) elif key == 'message': cache_obj = serialize_message(store, obj) elif key == 'comment': cache_obj = serialize_comment(store, obj) elif key == 'file': cache_obj = models.serializers.serialize_ifile(store, obj) self.cache[cache_key] = cache_obj return self.cache[cache_key]
def check_for_expiring_submissions(self, store): threshold = datetime_now() - timedelta( GLSettings.memory_copy.tip_expiration_threshold) for itip in store.find(models.InternalTip, models.InternalTip.expiration_date < threshold): for rtip in itip.receivertips: user = rtip.receiver.user language = user.language node_desc = db_admin_serialize_node(store, language) notification_desc = db_get_notification(store, language) context_desc = admin_serialize_context( store, rtip.internaltip.context, language) receiver_desc = admin_serialize_receiver( rtip.receiver, language) tip_desc = serialize_rtip(store, rtip, user.language) data = { 'type': u'tip_expiration', 'node': node_desc, 'context': context_desc, 'receiver': receiver_desc, 'notification': notification_desc, 'tip': tip_desc } subject, body = Templating().get_mail_subject_and_body(data) mail = models.Mail({ 'address': data['receiver']['mail_address'], 'subject': subject, 'body': body }) store.add(mail)
def receiverfile_planning(store): """ This function roll over the InternalFile uploaded, extract a path, id and receivers associated, one entry for each combination. representing the ReceiverFile that need to be created. """ receiverfiles_maps = {} for ifile in store.find(InternalFile, InternalFile.new == True): if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX: ifile.new = False error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \ (ifile.id, INTERNALFILES_HANDLE_RETRY_MAX) log.err(error) continue elif ifile.processing_attempts >= 1: log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) if ifile.processing_attempts: log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) ifile.processing_attempts += 1 for rtip in ifile.internaltip.receivertips: receiverfile = ReceiverFile() receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtip.id receiverfile.file_path = ifile.file_path receiverfile.size = ifile.size receiverfile.status = u'processing' # https://github.com/globaleaks/GlobaLeaks/issues/444 # avoid to mark the receiverfile as new if it is part of a submission # this way we avoid to send unuseful messages receiverfile.new = False if ifile.submission else True store.add(receiverfile) if ifile.id not in receiverfiles_maps: receiverfiles_maps[ifile.id] = { 'plaintext_file_needed': False, 'ifile_id': ifile.id, 'ifile_path': ifile.file_path, 'ifile_size': ifile.size, 'rfiles': [] } receiverfiles_maps[ifile.id]['rfiles'].append({ 'id': receiverfile.id, 'status': u'processing', 'path': ifile.file_path, 'size': ifile.size, 'receiver': admin_serialize_receiver(rtip.receiver, GLSettings.memory_copy.default_language) }) return receiverfiles_maps
def get_receiver_from_rtip(store, rtip_id, language): rtip = store.find(ReceiverTip, ReceiverTip.id == rtip_id).one() if not rtip: raise errors.TipIdNotFound return receiver.admin_serialize_receiver(rtip.receiver, language)
def process_ReceiverTip(self, store, rtip, data): language = rtip.receiver.user.language data['tip'] = serialize_rtip(store, rtip, language) data['context'] = admin_serialize_context(store, rtip.internaltip.context, language) data['receiver'] = admin_serialize_receiver(rtip.receiver, language) self.process_mail_creation(store, data)
def pgp_validation_check(self, store): expired_or_expiring = [] for rcvr in store.find(models.Receiver): if rcvr.user.pgp_key_public and rcvr.user.pgp_key_expiration != datetime_null(): if rcvr.user.pgp_key_expiration < datetime_now(): expired_or_expiring.append(admin_serialize_receiver(rcvr, GLSettings.memory_copy.default_language)) if GLSettings.memory_copy.allow_unencrypted: # The PGP key status should be downgraded only if the node # accept non PGP mails/files to be sent/stored. # If the node wont accept this the pgp key status # will remain enabled and mail won't be sent by regular flow. rcvr.user.pgp_key_status = u"disabled" elif rcvr.user.pgp_key_expiration < datetime_now() - timedelta(days=15): expired_or_expiring.append(admin_serialize_receiver(rcvr, GLSettings.memory_copy.default_language)) return expired_or_expiring
def get_tip_export(store, user_id, rtip_id, language): rtip = db_access_rtip(store, user_id, rtip_id) receiver = rtip.receiver rtip_dict = serialize_rtip(store, rtip, language) export_dict = { 'type': u'export_template', 'node': db_admin_serialize_node(store, language), 'notification': db_get_notification(store, language), 'tip': serialize_rtip(store, rtip, language), 'context': admin_serialize_context(store, rtip.internaltip.context, language), 'receiver': admin_serialize_receiver(store, receiver, language), 'comments': rtip_dict['comments'], 'messages': rtip_dict['messages'], 'files': [] } export_template = Templating().format_template( export_dict['notification']['export_template'], export_dict).encode('utf-8') export_template = msdos_encode(export_template) export_dict['files'].append({'buf': export_template, 'name': "data.txt"}) for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id): rf.downloads += 1 file_dict = models.serializers.serialize_rfile(rf) file_dict['name'] = 'files/' + file_dict['name'] export_dict['files'].append(file_dict) rtips_ids = [rt.id for rt in rtip.internaltip.receivertips] wfs = store.find(models.WhistleblowerFile, In(models.WhistleblowerFile.receivertip_id, rtips_ids)) for wf in wfs: file_dict = models.serializers.serialize_wbfile(wf) file_dict['name'] = 'files_from_recipients/' + file_dict['name'] export_dict['files'].append(file_dict) return export_dict
def process_Message(self, store, message, data): # if the message is destinated to the whistleblower no mail should be sent if message.type == u"receiver": return language = message.receivertip.receiver.user.language data['message'] = serialize_message(message) data['tip'] = serialize_rtip(store, message.receivertip, language) data['context'] = admin_serialize_context(store, message.receivertip.internaltip.context, language) data['receiver'] = admin_serialize_receiver(message.receivertip.receiver, language) self.process_mail_creation(store, data)
def pgp_validation_check(self, store): expired_or_expiring = [] for rcvr in store.find(models.Receiver): if rcvr.user.pgp_key_public and rcvr.user.pgp_key_expiration != datetime_null( ): if rcvr.user.pgp_key_expiration < datetime_now(): expired_or_expiring.append( admin_serialize_receiver( rcvr, GLSettings.memory_copy.default_language)) if GLSettings.memory_copy.allow_unencrypted: # The PGP key status should be downgraded only if the node # accept non PGP mails/files to be sent/stored. # If the node wont accept this the pgp key status # will remain enabled and mail won't be sent by regular flow. rcvr.user.pgp_key_status = u'disabled' elif rcvr.user.pgp_key_expiration < datetime_now() - timedelta( days=15): expired_or_expiring.append( admin_serialize_receiver( rcvr, GLSettings.memory_copy.default_language)) return expired_or_expiring
def process_Comment(self, store, comment, data): for rtip in comment.internaltip.receivertips: if comment.type == u'receiver' and comment.author == rtip.receiver.user.name: continue language = rtip.receiver.user.language dataX = copy.deepcopy(data) dataX['comment'] = serialize_comment(comment) dataX['tip'] = serialize_rtip(store, rtip, language) dataX['context'] = admin_serialize_context(store, comment.internaltip.context, language) dataX['receiver'] = admin_serialize_receiver(rtip.receiver, language) self.process_mail_creation(store, dataX)
def get_tip_export(store, user_id, rtip_id, language): rtip = db_access_rtip(store, user_id, rtip_id) receiver = rtip.receiver rtip_dict = serialize_rtip(store, rtip, language) export_dict = { 'type': u'export_template', 'node': db_admin_serialize_node(store, language), 'notification': db_get_notification(store, language), 'tip': serialize_rtip(store, rtip, language), 'context': admin_serialize_context(store, rtip.internaltip.context, language), 'receiver': admin_serialize_receiver(receiver, language), 'comments': rtip_dict['comments'], 'messages': rtip_dict['messages'], 'files': [] } export_template = Templating().format_template(export_dict['notification']['export_template'], export_dict).encode('utf-8') export_template = msdos_encode(export_template) export_dict['files'].append({'buf': export_template, 'name': "data.txt"}) for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id): rf.downloads += 1 file_dict = models.serializers.serialize_rfile(rf) file_dict['name'] = 'files/' + file_dict['name'] export_dict['files'].append(copy.deepcopy(file_dict)) rtips_ids = [rt.id for rt in rtip.internaltip.receivertips] wfs = store.find(models.WhistleblowerFile, In(models.WhistleblowerFile.receivertip_id, rtips_ids)) for wf in wfs: file_dict = models.serializers.serialize_wbfile(wf) file_dict['name'] = 'files_from_recipients/' + file_dict['name'] export_dict['files'].append(copy.deepcopy(file_dict)) return export_dict
def serialize_content(store, cache, key, obj, language): obj_id = obj.id cache_key = key + obj_id + language if cache_key not in cache: if key == 'tip': cache_obj = serialize_rtip(store, obj, language) elif key == 'context': cache_obj = admin_serialize_context(store, obj, language) elif key == 'receiver': cache_obj = admin_serialize_receiver(obj, language) elif key == 'message': cache_obj = serialize_message(obj) elif key == 'comment': cache_obj = serialize_comment(obj) elif key == 'file': cache_obj = serialize_internalfile(obj) cache[cache_key] = cache_obj return copy.deepcopy(cache[cache_key])
def serialize_content(store, cache, key, obj, language): obj_id = obj.id cache_key = key + obj_id + language if cache_key not in cache: if key == 'tip': cache_obj = serialize_rtip(store, obj, language) elif key == 'context': cache_obj = admin_serialize_context(store, obj, language) elif key == 'receiver': cache_obj = admin_serialize_receiver(obj, language) elif key == 'message': cache_obj = serialize_message(obj) elif key == 'comment': cache_obj = serialize_comment(obj) elif key == 'file': cache_obj = serialize_internalfile(obj) cache[cache_key] = cache_obj return copy.deepcopy(cache[cache_key])
def serialize_obj(self, store, key, obj, language): obj_id = obj.id cache_key = key + '-' + obj_id + '-' + language if cache_key not in self.cache: if key == 'tip': cache_obj = serialize_rtip(store, obj, language) elif key == 'context': cache_obj = admin_serialize_context(store, obj, language) elif key == 'receiver': cache_obj = admin_serialize_receiver(obj, language) elif key == 'message': cache_obj = serialize_message(obj) elif key == 'comment': cache_obj = serialize_comment(obj) elif key == 'file': cache_obj = serialize_internalfile(obj) self.cache[cache_key] = cache_obj return self.cache[cache_key]
def serialize_obj(self, store, key, obj, language): obj_id = obj.id cache_key = key + '-' + obj_id + '-' + language if cache_key not in self.cache: if key == 'tip': cache_obj = serialize_rtip(store, obj, language) elif key == 'context': cache_obj = admin_serialize_context(store, obj, language) elif key == 'receiver': cache_obj = admin_serialize_receiver(obj, language) elif key == 'message': cache_obj = serialize_message(obj) elif key == 'comment': cache_obj = serialize_comment(obj) elif key == 'file': cache_obj = serialize_internalfile(obj) self.cache[cache_key] = cache_obj return self.cache[cache_key]
def get_file_collection(store, user_id, rtip_id): rtip = db_access_rtip(store, user_id, rtip_id) archive_dict = {'files': [], 'file_counter': 0, 'total_size': 0} file_list = [] for rf in store.find(models.ReceiverFile, models.ReceiverFile.receivertip_id == rtip_id): rf.downloads += 1 file_list.append(serialize_receiver_file(rf)) archive_dict['file_counter'] += 1 archive_dict['total_size'] += rf.size archive_dict['files'].append(serialize_receiver_file(rf)) receiver = rtip.receiver user_language = receiver.user.language data = { 'type': u'archive_description', 'node': db_admin_serialize_node(store, user_language), 'notification': db_get_notification(store, user_language), 'tip': serialize_rtip(store, rtip, user_language), 'context': admin_serialize_context(store, rtip.internaltip.context, user_language), 'receiver': admin_serialize_receiver(receiver, user_language), 'archive': archive_dict } archive_description = Templating().format_template(data['notification']['archive_description'], data).encode('utf-8') file_list.append( { 'buf': archive_description, 'name': "COLLECTION_INFO.txt" } ) return file_list
def check_for_expiring_submissions(self, store): threshold = datetime_now() + timedelta( hours=GLSettings.memory_copy.notif.tip_expiration_threshold) receivers = store.find(models.Receiver) for receiver in receivers: rtips = store.find( models.ReceiverTip, models.ReceiverTip.internaltip_id == models.InternalTip.id, models.InternalTip.expiration_date < threshold, models.ReceiverTip.receiver_id == models.Receiver.id, models.Receiver.id == receiver.id) if rtips.count() == 0: continue user = receiver.user language = user.language node_desc = db_admin_serialize_node(store, language) notification_desc = db_get_notification(store, language) receiver_desc = admin_serialize_receiver(store, receiver, language) if rtips.count() == 1: rtip = rtips[0] tip_desc = serialize_rtip(store, rtip, user.language) context_desc = admin_serialize_context( store, rtip.internaltip.context, language) data = { 'type': u'tip_expiration', 'node': node_desc, 'context': context_desc, 'receiver': receiver_desc, 'notification': notification_desc, 'tip': tip_desc } else: tips_desc = [] earliest_expiration_date = datetime_never() for rtip in rtips: if rtip.internaltip.expiration_date < earliest_expiration_date: earliest_expiration_date = rtip.internaltip.expiration_date tips_desc.append(serialize_rtip(store, rtip, user.language)) data = { 'type': u'tip_expiration_summary', 'node': node_desc, 'notification': notification_desc, 'receiver': receiver_desc, 'expiring_submission_count': rtips.count(), 'earliest_expiration_date': datetime_to_ISO8601(earliest_expiration_date) } subject, body = Templating().get_mail_subject_and_body(data) mail = models.Mail({ 'address': receiver_desc['mail_address'], 'subject': subject, 'body': body }) store.add(mail)
def receiverfile_planning(store): """ This function roll over the InternalFile uploaded, extract a path, id and receivers associated, one entry for each combination. representing the ReceiverFile that need to be created. """ receiverfiles_maps = {} ifilescnt = store.find(InternalFile, InternalFile.new == True).count() ifiles = store.find(InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit] if ifilescnt > GLSettings.jobs_operation_limit: log.debug("Delivery iterating over %d InternalFile from a queue of %d" % ( GLSettings.jobs_operation_limit, ifilescnt)) elif ifilescnt: log.debug("Delivery iterating over %d InternalFile" % ifilescnt) else: pass # 0 files to be processed for ifile in ifiles: if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX: ifile.new = False error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \ (ifile.id, INTERNALFILES_HANDLE_RETRY_MAX) log.err(error) send_exception_email(error) continue elif ifile.processing_attempts >= 1: log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) if ifile.processing_attempts: log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) ifile.processing_attempts = ifile.processing_attempts + 1 for receiver in ifile.internaltip.receivers: rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver.id).one() receiverfile = ReceiverFile() receiverfile.receiver_id = receiver.id receiverfile.internaltip_id = ifile.internaltip_id receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtrf.id receiverfile.file_path = ifile.file_path receiverfile.size = ifile.size receiverfile.status = u'processing' store.add(receiverfile) if ifile.id not in receiverfiles_maps: receiverfiles_maps[ifile.id] = { 'plaintext_file_needed': False, 'ifile_id': ifile.id, 'ifile_path': ifile.file_path, 'ifile_size': ifile.size, 'rfiles': [] } receiverfiles_maps[ifile.id]['rfiles'].append({ 'id': receiverfile.id, 'status': u'processing', 'path': ifile.file_path, 'size': ifile.size, 'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language) }) return receiverfiles_maps