def pgp_validation_check(self, store):
node_desc = db_admin_serialize_node(store, 'en')
expired_or_expiring = []
rcvrs = store.find(Receiver)
for rcvr in rcvrs:
if rcvr.pgp_key_public and rcvr.pgp_key_expiration != datetime_null(
):
if rcvr.pgp_key_expiration < datetime_now():
expired_or_expiring.append(
admin_serialize_receiver(
rcvr, GLSettings.memory_copy.default_language))
if node_desc['allow_unencrypted']:
# The PGP key status should be downgraded only if the node
# accept non PGP mails/files to be sent/stored.
# If the node wont accept this the pgp key status
# will remain enabled and mail won't be sent by regular flow.
rcvr.pgp_key_status = u'disabled'
elif rcvr.pgp_key_expiration < datetime_now() - timedelta(
days=15):
expired_or_expiring.append(
admin_serialize_receiver(
rcvr, GLSettings.memory_copy.default_language))
return expired_or_expiring
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
ifilesmap = {}
files = store.find(InternalFile, InternalFile.new == True)
for filex in files:
for receiver in filex.internaltip.receivers:
if filex.file_path not in ifilesmap:
ifilesmap[filex.file_path] = list()
receiver_desc = admin_serialize_receiver(receiver, GLSetting.memory_copy.language)
map_info = {
'receiver' : receiver_desc,
'path' : filex.file_path,
'size' : filex.size,
'status' : u'reference'
}
# AS KEY, file path is used to keep track of the original
# path, because shall be renamed in .plaintext (in the unlucky case
# of receivers without PGP)
# AS FIELD, it can be replaced with a dedicated PGP encrypted path
ifilesmap[filex.file_path].append(map_info)
filex.new = False
return ifilesmap
def get_receiver_from_rtip(store, rtip_id, language):
rtip = store.find(ReceiverTip, ReceiverTip.id == rtip_id).one()
if not rtip:
log.err("Download of a Zip file without ReceiverTip associated!")
raise errors.TipIdNotFound
return admin.admin_serialize_receiver(rtip.receiver, language)
def get_receiver_from_rtip(store, rtip_id):
rtip = store.find(ReceiverTip, ReceiverTip.id == rtip_id).one()
if not rtip:
log.err("Download of a Zip file without ReceiverTip associated!")
raise errors.TipIdNotFound
return admin.admin_serialize_receiver(rtip.receiver, GLSetting.memory_copy.default_language)
def pgp_validation_check(self, store):
node_desc = db_admin_serialize_node(store, 'en')
expired_or_expiring = []
rcvrs = store.find(Receiver)
for rcvr in rcvrs:
if rcvr.pgp_key_public and rcvr.pgp_key_expiration != datetime_null():
if rcvr.pgp_key_expiration < datetime_now():
expired_or_expiring.append(admin_serialize_receiver(rcvr, GLSetting.memory_copy.language))
if node_desc['allow_unencrypted']:
# The PGP key status should be downgraded only if the node
# accept non PGP mails/files to be sent/stored.
# If the node wont accept this the pgp key status
# will remain enabled and mail won't be sent by regular flow.
rcvr.pgp_key_status = u'disabled'
elif rcvr.pgp_key_expiration < datetime_now() - timedelta(days=15):
expired_or_expiring.append(admin_serialize_receiver(rcvr, GLSetting.memory_copy.language))
return expired_or_expiring
def import_receiver(self, receiver):
self.language = receiver.user.language
if self.trigger == 'Message':
self.template_type = u'message'
elif self.trigger == 'Tip':
self.template_type = u'tip'
elif self.trigger == 'Comment':
self.template_type = u'comment'
elif self.trigger == 'File':
self.template_type = u'file'
elif self.trigger == 'ExpiringTip':
self.template_type = u'upcoming_tip_expiration'
else:
raise Exception("self.trigger of unexpected kind ? %s" % self.trigger)
receiver_desc = admin.admin_serialize_receiver(receiver, self.language)
return (receiver.tip_notification, receiver_desc)
def import_receiver(self, receiver):
self.receiver_desc = admin.admin_serialize_receiver(receiver, self.language)
if self.trigger == 'Message':
self.template_type = u'encrypted_message' if \
receiver.gpg_key_status == u'enabled' else u'plaintext_message'
return receiver.message_notification
elif self.trigger == 'Tip':
self.template_type = u'encrypted_tip' if \
receiver.gpg_key_status == u'enabled' else u'plaintext_tip'
return receiver.tip_notification
elif self.trigger == 'Comment':
self.template_type = u'encrypted_comment' if \
receiver.gpg_key_status == u'enabled' else u'plaintext_comment'
return receiver.comment_notification
elif self.trigger == 'File':
self.template_type = u'encrypted_file' if \
receiver.gpg_key_status == u'enabled' else u'plaintext_file'
return receiver.file_notification
else:
raise Exception("self.trigger of unexpected kind ? %s" % self.trigger)
def import_receiver(self, receiver):
self.language = receiver.user.language
if self.trigger == 'Message':
self.template_type = u'encrypted_message' if \
receiver.pgp_key_status == u'enabled' else u'plaintext_message'
elif self.trigger == 'Tip':
self.template_type = u'encrypted_tip' if \
receiver.pgp_key_status == u'enabled' else u'plaintext_tip'
elif self.trigger == 'Comment':
self.template_type = u'encrypted_comment' if \
receiver.pgp_key_status == u'enabled' else u'plaintext_comment'
elif self.trigger == 'File':
self.template_type = u'encrypted_file' if \
receiver.pgp_key_status == u'enabled' else u'plaintext_file'
elif self.trigger == 'ExpiringTip':
self.template_type = u'upcoming_tip_expiration'
else:
raise Exception("self.trigger of unexpected kind ? %s" % self.trigger)
receiver_desc = admin.admin_serialize_receiver(receiver, self.language)
return (receiver.tip_notification, receiver_desc)
def create_file_notification_events(self, store):
"""
Creates events for performing notification of newly added files..
Returns:
events: a list of tuples containing ((receiverfile_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_rfiles = store.find(models.ReceiverFile,
models.ReceiverFile.mark == models.ReceiverFile._marker[0]
)
node_desc = admin.admin_serialize_node(store.find(models.Node).one(), GLSetting.memory_copy.default_language)
if not_notified_rfiles.count():
log.debug("Receiverfiles found to be notified: %d" % not_notified_rfiles.count() )
for rfile in not_notified_rfiles:
if not rfile.internalfile:
log.err("(file_notification) Integrity check failure (InternalFile)")
continue
file_desc = serialize_internalfile(rfile.internalfile)
if not rfile.internalfile or \
not rfile.internalfile.internaltip or \
not rfile.internalfile.internaltip.context:
log.err("(file_notification) Integrity check failure (File+Tip)")
continue
context_desc = admin.admin_serialize_context(rfile.internalfile.internaltip.context,
GLSetting.memory_copy.default_language)
receiver_desc = admin.admin_serialize_receiver(rfile.receiver, GLSetting.memory_copy.default_language)
if not receiver_desc.has_key('notification_fields') or \
not rfile.receiver.notification_fields.has_key('mail_address'):
log.err("Receiver %s lack of email address!" % rfile.receiver.user.name)
continue
# check if the receiver has the File notification enabled or not
if not rfile.receiver.file_notification:
log.debug("Receiver %s has file notification disabled: %s skipped" % (
rfile.receiver.user.username, rfile.internalfile.name ))
rfile.mark = models.ReceiverFile._marker[3] # 'disabled'
store.commit()
continue
# by ticket https://github.com/globaleaks/GlobaLeaks/issues/444
# send notification of file only if notification of tip is already on send status
if rfile.receiver_tip.mark == models.ReceiverTip._marker[0]: # 'not notified'
rfile.mark = models.ReceiverFile._marker[4] # 'skipped'
log.debug("Skipped notification of %s (for %s) because Tip not yet notified" %
(rfile.internalfile.name, rfile.receiver.name) )
store.commit()
continue
event = Event(type=u'file', trigger='File',
notification_settings=self.notification_settings,
trigger_info=file_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append(((unicode(rfile.id), unicode(rfile.receiver.id)), event))
return events
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(
InternalFile,
InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug(
"Delivery iterating over %d InternalFile from a Queue of %d" %
(GLSettings.jobs_operation_limit, ifilescnt))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if (ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX):
ifile.new = False
error = "Failed to handle receiverfiles creation for ifilee %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif (ifile.processing_attempts >= 1):
log.err(
"Failed to handle receiverfiles creation for ifile %s (retry %d/%d)"
% (ifile.id, ifile.processing_attempts,
INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug(
"Starting handling receiverfiles creation for ifile %s retry %d/%d"
% (ifile.id, ifile.processing_attempts,
INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(
ReceiverTip,
ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id':
receiverfile.id,
'status':
u'processing',
'path':
ifile.file_path,
'size':
ifile.size,
'receiver':
admin_serialize_receiver(
receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def transact_dummy_whatever(store, receiver_id, mock_request):
receiver = store.find(Receiver, Receiver.id == receiver_id).one()
gpg_options_parse(receiver, mock_request)
return admin_serialize_receiver(receiver)
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
REMIND: (keyword) escalation escalate pertinence vote
here need to be updated whenever an escalation is implemented.
checking of status and marker and recipients
"""
try:
files = store.find(InternalFile, InternalFile.mark == u'not processed')
except Exception as excep:
log.err("Unable to find InternalFile in scheduler! %s" % str(excep))
return []
ifilesmap = {}
for filex in files:
if not filex.internaltip:
log.err("Integrity failure: the file %s"\
"has not an InternalTip assigned (path: %s)" %
(filex.name, filex.file_path) )
try:
os.remove(os.path.join(GLSetting.submission_path, filex.file_path))
except OSError as excep:
log.err("Unable to remove %s in integrity fixing routine: %s" %
(filex.file_path, excep.strerror) )
key_id = os.path.basename(filex.file_path).split('.')[0]
keypath = os.path.join(GLSetting.ramdisk_path, ("%s%s" % (GLSetting.AES_keyfile_prefix, key_id)))
try:
os.remove(keypath)
except OSError as excep:
log.err("Unable to delete keyfile %s: %s" % (keypath, excep.strerror))
continue
# here we select the file which deserve to be processed.
# They need to be:
# From a Tip in (Tip = 'finalize' or 'first' )
# From an InternalFile (File = 'ready')
# Tips may have two statuses both valid.
# if these conditions are met the InternalFile(s) is/are marked as 'locked',
# Whenever a delivery scheduler run, do not touch 'locked' file, and if 'locked' file
# appears in the Admin interface of file overview, this mean that something is broken.
if (filex.internaltip.mark == u'finalize' or \
filex.internaltip.mark == u'first') and \
(filex.mark == u'not processed'):
filex.mark = u'locked'
else:
continue
try:
for receiver in filex.internaltip.receivers:
if not ifilesmap.has_key(filex.file_path):
ifilesmap[filex.file_path] = list()
receiver_desc = admin_serialize_receiver(receiver, GLSetting.memory_copy.default_language)
map_info = {
'receiver' : receiver_desc,
'path' : filex.file_path,
'size' : filex.size,
'status' : u'reference'
}
# this may seem apparently redounded, but is not!
# AS KEY, file path is used to keep track of the original
# path, because shall be renamed in .plaintext (in the unlucky case
# of receivers without PGP)
# AS FIELD, it can be replaced with a dedicated PGP encrypted path
ifilesmap[filex.file_path].append(map_info)
except Exception as excep:
log.debug("Invalid Storm operation in checking for PGP cap: %s" % excep)
continue
return ifilesmap
def create_message_notification_events(self, store, notification_counter):
"""
Creates events for performing notification of newly added messages.
Returns:
events: a list of tuples containing ((message_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
language = GLSetting.memory_copy.default_language
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_messages = store.find(models.Message,
models.Message.mark == models.Message._marker[0]
)
node_desc = admin.db_admin_serialize_node(store, language)
if not_notified_messages.count():
log.debug("Messages found to be notified: %d" % not_notified_messages.count() )
for message in not_notified_messages:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug("Notification counter has reached the suggested limit: %d (messages)" %
notification_counter)
store.remove(message)
break
if message.receivertip is None:
log.err("Message %s has ReceiverTip broken reference" % message.id)
message.mark = models.Message._marker[2] # 'unable to notify'
continue
tip_desc = serialize_receivertip(message.receivertip)
receiver = models.Receiver.get(store, message.receivertip.receiver_id)
if not receiver:
log.err("Message %s do not find receiver!?" % message.id)
if not receiver.mail_address:
log.err("Receiver %s lack of email address!" % receiver.name)
continue
receiver_desc = admin.admin_serialize_receiver(receiver, language)
log.debug("Messages receiver: %s" % message.receivertip.receiver.name)
context = message.receivertip.internaltip.context
if not context:
log.err("Reference chain fail!")
continue
context_desc = admin.admin_serialize_context(store, context, language)
message_desc = rtip.receiver_serialize_message(message)
message.mark = u'notified' # models.Message._marker[1]
if message.type == u"receiver":
log.debug("Receiver is the Author (%s): skipped" % receiver.user.username)
continue
# check if the receiver has the Message notification enabled or not
if not receiver.message_notification:
log.debug("Receiver %s has message notification disabled: skipped [source: %s]" % (
receiver.user.username, message.author))
continue
if receiver_desc['gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_message'
else:
template_type = u'plaintext_message'
notification_settings = self._get_notification_settings(store, receiver_desc['language'])
event = Event(type=template_type, trigger='Message',
notification_settings=notification_settings,
trigger_info=message_desc,
trigger_parent=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
steps_info = admin.db_get_context_steps(store,
context_desc['id'],
language),
plugin=plugin)
events.append(((unicode(message.id), unicode(receiver.id)), event))
return events, notification_counter
def create_tip_notification_events(self, store, notification_counter):
"""
This transaction will return all a list of tuples containing the tips
for which the notification event has not been run.
Returns:
events: a list of tuples containing (tip_id, an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
# settings.notification_plugins contain a list of supported plugin
# at the moment only 1. so [0] is used. but different context/receiver
# may use different code-plugin:
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_tips = store.find(
models.ReceiverTip,
models.ReceiverTip.mark == models.ReceiverTip._marker[0])
node_desc = admin.db_admin_serialize_node(
store, GLSetting.memory_copy.default_language)
if not_notified_tips.count():
log.debug("Receiver Tips found to be notified: %d" %
not_notified_tips.count())
for receiver_tip in not_notified_tips:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug(
"Notification counter has reached the suggested limit: %d (tip)"
% notification_counter)
break
if not receiver_tip.internaltip or not receiver_tip.internaltip.context:
log.err(
"(tip_notification) Integrity failure: missing InternalTip|Context"
)
continue
context_desc = admin.admin_serialize_context(
receiver_tip.internaltip.context,
GLSetting.memory_copy.default_language)
receiver_desc = admin.admin_serialize_receiver(
receiver_tip.receiver, GLSetting.memory_copy.default_language)
if not receiver_desc.has_key('mail_address'):
log.err("Receiver %s lack of email address!" %
receiver_tip.receiver.name)
continue
# check if the receiver has the Tip notification enabled or not
if not receiver_desc['tip_notification']:
log.debug("Receiver %s has tip notification disabled" %
receiver_tip.receiver.user.username)
receiver_tip.mark = models.ReceiverTip._marker[3] # 'disabled'
store.commit()
continue
tip_desc = serialize_receivertip(receiver_tip)
if receiver_desc[
'gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_tip'
else:
template_type = u'plaintext_tip'
event = Event(type=template_type,
trigger='Tip',
notification_settings=self.notification_settings,
trigger_info=tip_desc,
trigger_parent=None,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append((unicode(receiver_tip.id), event))
return events, notification_counter
def create_file_notification_events(self, store, notification_counter):
"""
Creates events for performing notification of newly added files..
Returns:
events: a list of tuples containing ((receiverfile_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_rfiles = store.find(
models.ReceiverFile,
models.ReceiverFile.mark == models.ReceiverFile._marker[0])
node_desc = admin.db_admin_serialize_node(
store, GLSetting.memory_copy.default_language)
if not_notified_rfiles.count():
log.debug("Receiverfiles found to be notified: %d" %
not_notified_rfiles.count())
for rfile in not_notified_rfiles:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug(
"Notification counter has reached the suggested limit: %d (files)"
% notification_counter)
break
if not rfile.internalfile:
log.err(
"(file_notification) Integrity check failure (InternalFile)"
)
continue
file_desc = serialize_internalfile(rfile.internalfile)
if not rfile.internalfile or \
not rfile.internalfile.internaltip or \
not rfile.internalfile.internaltip.context:
log.err(
"(file_notification) Integrity check failure (File+Tip)")
continue
context_desc = admin.admin_serialize_context(
rfile.internalfile.internaltip.context,
GLSetting.memory_copy.default_language)
receiver_desc = admin.admin_serialize_receiver(
rfile.receiver, GLSetting.memory_copy.default_language)
if not receiver_desc.has_key('mail_address'):
log.err("Receiver %s lack of email address!" %
rfile.receiver.user.name)
continue
# check if the receiver has the File notification enabled or not
if not rfile.receiver.file_notification:
log.debug(
"Receiver %s has file notification disabled: %s skipped" %
(rfile.receiver.user.username, rfile.internalfile.name))
rfile.mark = models.ReceiverFile._marker[3] # 'disabled'
store.commit()
continue
# by ticket https://github.com/globaleaks/GlobaLeaks/issues/444
# send notification of file only if notification of tip is already on send status
if rfile.receiver_tip.mark == models.ReceiverTip._marker[
0]: # 'not notified'
rfile.mark = models.ReceiverFile._marker[4] # 'skipped'
log.debug(
"Skipped notification of %s (for %s) because Tip not yet notified"
% (rfile.internalfile.name, rfile.receiver.name))
store.commit()
continue
tip_desc = serialize_receivertip(rfile.receiver_tip)
if receiver_desc[
'gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_file'
else:
template_type = u'plaintext_file'
event = Event(type=template_type,
trigger='File',
notification_settings=self.notification_settings,
trigger_info=file_desc,
trigger_parent=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append(
((unicode(rfile.id), unicode(rfile.receiver.id)), event))
return events, notification_counter
def create_comment_notification_events(self, store, notification_counter):
"""
Creates events for performing notification of newly added comments.
Returns:
events: a list of tuples containing ((comment_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_comments = store.find(
models.Comment, models.Comment.mark == models.Comment._marker[0])
node_desc = admin.db_admin_serialize_node(
store, GLSetting.memory_copy.default_language)
if not_notified_comments.count():
log.debug("Comments found to be notified: %d" %
not_notified_comments.count())
for comment in not_notified_comments:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug(
"Notification counter has reached the suggested limit: %d (comment)"
% notification_counter)
break
if comment.internaltip is None or comment.internaltip.receivers is None:
log.err(
"Comment %s has internaltip or receivers broken reference"
% comment.id)
comment.mark = models.Comment._marker[2] # 'unable to notify'
continue
# for every comment, iter on the associated receiver
log.debug("Comments receiver: %d" %
comment.internaltip.receivers.count())
comment_desc = rtip.receiver_serialize_comment(comment)
if not comment.internaltip.context:
log.err(
"(comment_notification) Integrity check failure Context")
continue
context_desc = admin.admin_serialize_context(
comment.internaltip.context,
GLSetting.memory_copy.default_language)
# XXX BUG! All notification is marked as correctly send,
# This can't be managed by callback, and can't be managed by actual DB design
comment.mark = models.Comment._marker[1] # 'notified'
for receiver in comment.internaltip.receivers:
receiver_desc = admin.admin_serialize_receiver(
receiver, GLSetting.memory_copy.default_language)
if not receiver_desc.has_key('mail_address'):
log.err("Receiver %s lack of email address!" %
receiver.name)
continue
# if the comment author is the one to be notified: skip the notification
# ----- BUG, remind,
# if two receiver has the same name, and one has notification disabled
# also the homonymous would get the notification dropped.
if comment.type == models.Comment._types[
0] and comment.author == receiver.name:
log.debug("Receiver is the Author (%s): skipped" %
receiver.user.username)
continue
# check if the receiver has the Comment notification enabled or not
if not receiver.comment_notification:
log.debug(
"Receiver %s has comment notification disabled: skipped [source: %s]"
% (receiver.user.username, comment.author))
continue
receivertip = store.find(
models.ReceiverTip,
(models.ReceiverTip.internaltip_id
== comment.internaltip_id, models.ReceiverTip.receiver_id
== receiver.id)).one()
tip_desc = serialize_receivertip(receivertip)
if receiver_desc[
'gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_comment'
else:
template_type = u'plaintext_comment'
event = Event(type=template_type,
trigger='Comment',
notification_settings=self.notification_settings,
trigger_info=comment_desc,
trigger_parent=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append(
((unicode(comment.id), unicode(receiver.id)), event))
return events, notification_counter
def create_message_notification_events(self, store, notification_counter):
"""
Creates events for performing notification of newly added messages.
Returns:
events: a list of tuples containing ((message_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_messages = store.find(
models.Message, models.Message.mark == models.Message._marker[0])
node_desc = admin.db_admin_serialize_node(
store, GLSetting.memory_copy.default_language)
if not_notified_messages.count():
log.debug("Messages found to be notified: %d" %
not_notified_messages.count())
for message in not_notified_messages:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug(
"Notification counter has reached the suggested limit: %d (messages)"
% notification_counter)
break
if message.receivertip is None:
log.err("Message %s has ReceiverTip broken reference" %
message.id)
message.mark = models.Message._marker[2] # 'unable to notify'
continue
tip_desc = serialize_receivertip(message.receivertip)
receiver = store.find(
Receiver,
Receiver.id == message.receivertip.receiver_id).one()
if not receiver:
log.err("Message %s do not find receiver!?" % message.id)
if not receiver.mail_address:
log.err("Receiver %s lack of email address!" % receiver.name)
continue
receiver_desc = admin.admin_serialize_receiver(
receiver, GLSetting.memory_copy.default_language)
log.debug("Messages receiver: %s" %
message.receivertip.receiver.name)
context = message.receivertip.internaltip.context
if not context:
log.err("Reference chain fail!")
continue
context_desc = admin.admin_serialize_context(
context, GLSetting.memory_copy.default_language)
message_desc = rtip.receiver_serialize_message(message)
message.mark = u'notified' # models.Message._marker[1]
if message.type == u"receiver":
log.debug("Receiver is the Author (%s): skipped" %
receiver.user.username)
continue
# check if the receiver has the Message notification enabled or not
if not receiver.message_notification:
log.debug(
"Receiver %s has message notification disabled: skipped [source: %s]"
% (receiver.user.username, message.author))
continue
if receiver_desc[
'gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_message'
else:
template_type = u'plaintext_message'
event = Event(type=template_type,
trigger='Message',
notification_settings=self.notification_settings,
trigger_info=message_desc,
trigger_parent=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append(((unicode(message.id), unicode(receiver.id)), event))
return events, notification_counter
def create_comment_notification_events(self, store, notification_counter):
"""
Creates events for performing notification of newly added comments.
Returns:
events: a list of tuples containing ((comment_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
language = GLSetting.memory_copy.default_language
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_comments = store.find(models.Comment,
models.Comment.mark == models.Comment._marker[0]
)
node_desc = admin.db_admin_serialize_node(store, language)
if not_notified_comments.count():
log.debug("Comments found to be notified: %d" % not_notified_comments.count() )
for comment in not_notified_comments:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug("Notification counter has reached the suggested limit: %d (comment)" %
notification_counter)
store.remove(comment)
break
if comment.internaltip is None or comment.internaltip.receivers is None:
log.err("Comment %s has internaltip or receivers broken reference" % comment.id)
comment.mark = models.Comment._marker[2] # 'unable to notify'
continue
# for every comment, iter on the associated receiver
log.debug("Comments receiver: %d" % comment.internaltip.receivers.count())
comment_desc = rtip.receiver_serialize_comment(comment)
if not comment.internaltip.context:
log.err("(comment_notification) Integrity check failure Context")
store.remove(comment)
continue
context_desc = admin.admin_serialize_context(store, comment.internaltip.context, language)
# XXX BUG! All notification is marked as correctly send,
# This can't be managed by callback, and can't be managed by actual DB design
comment.mark = models.Comment._marker[1] # 'notified'
for receiver in comment.internaltip.receivers:
receiver_desc = admin.admin_serialize_receiver(receiver, language)
if not receiver_desc.has_key('mail_address'):
log.err("Receiver %s lack of email address!" % receiver.name)
continue
# if the comment author is the one to be notified: skip the notification
# ----- BUG, remind,
# if two receiver has the same name, and one has notification disabled
# also the homonymous would get the notification dropped.
if comment.type == models.Comment._types[0] and comment.author == receiver.name:
log.debug("Receiver is the Author (%s): skipped" % receiver.user.username)
continue
# check if the receiver has the Comment notification enabled or not
if not receiver.comment_notification:
log.debug("Receiver %s has comment notification disabled: skipped [source: %s]" % (
receiver.user.username, comment.author))
continue
receivertip = store.find(models.ReceiverTip,
(models.ReceiverTip.internaltip_id == comment.internaltip_id,
models.ReceiverTip.receiver_id == receiver.id)).one()
tip_desc = serialize_receivertip(receivertip)
if receiver_desc['gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_comment'
else:
template_type = u'plaintext_comment'
notification_settings = self._get_notification_settings(store, receiver_desc['language'])
event = Event(type=template_type, trigger='Comment',
notification_settings=notification_settings,
trigger_info=comment_desc,
trigger_parent=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
steps_info = admin.db_get_context_steps(store,
context_desc['id'],
language),
plugin=plugin)
events.append(((unicode(comment.id), unicode(receiver.id)), event))
return events, notification_counter
def create_file_notification_events(self, store, notification_counter):
"""
Creates events for performing notification of newly added files..
Returns:
events: a list of tuples containing ((receiverfile_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
language = GLSetting.memory_copy.default_language
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_rfiles = store.find(models.ReceiverFile,
models.ReceiverFile.mark == models.ReceiverFile._marker[0]
)
node_desc = admin.db_admin_serialize_node(store, language)
if not_notified_rfiles.count():
log.debug("Receiverfiles found to be notified: %d" % not_notified_rfiles.count() )
for rfile in not_notified_rfiles:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug("Notification counter has reached the suggested limit: %d (files)" %
notification_counter)
store.remove(rfile)
break
if not rfile.internalfile:
log.err("(file_notification) Integrity check failure (InternalFile)")
store.remove(rfile)
continue
file_desc = serialize_internalfile(rfile.internalfile)
if not rfile.internalfile.internaltip or \
not rfile.internalfile.internaltip.context:
log.err("(file_notification) Integrity check failure (InternalTip/Context)")
store.remove(rfile)
continue
context_desc = admin.admin_serialize_context(store,
rfile.internalfile.internaltip.context,
language)
receiver_desc = admin.admin_serialize_receiver(rfile.receiver, language)
if not receiver_desc.has_key('mail_address'):
log.err("Receiver %s lack of email address!" % rfile.receiver.user.name)
continue
# check if the receiver has the File notification enabled or not
if not rfile.receiver.file_notification:
log.debug("Receiver %s has file notification disabled: %s skipped" % (
rfile.receiver.user.username, rfile.internalfile.name ))
rfile.mark = models.ReceiverFile._marker[3] # 'disabled'
store.commit()
continue
# by ticket https://github.com/globaleaks/GlobaLeaks/issues/444
# send notification of file only if notification of tip is already on send status
if rfile.receiver_tip.mark == models.ReceiverTip._marker[0]: # 'not notified'
rfile.mark = models.ReceiverFile._marker[4] # 'skipped'
log.debug("Skipped notification of %s (for %s) because Tip not yet notified" %
(rfile.internalfile.name, rfile.receiver.name) )
store.commit()
continue
tip_desc = serialize_receivertip(rfile.receiver_tip)
if receiver_desc['gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_file'
else:
template_type = u'plaintext_file'
notification_settings = self._get_notification_settings(store, receiver_desc['language'])
event = Event(type=template_type, trigger='File',
notification_settings=notification_settings,
trigger_info=file_desc,
trigger_parent=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
steps_info = admin.db_get_context_steps(store,
context_desc['id'],
language),
plugin=plugin)
events.append(((unicode(rfile.id), unicode(rfile.receiver.id)), event))
return events, notification_counter
def create_comment_notification_events(self, store):
"""
Creates events for performing notification of newly added comments.
Returns:
events: a list of tuples containing ((comment_id, receiver_id), an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_comments = store.find(models.Comment,
models.Comment.mark == models.Comment._marker[0]
)
node_desc = admin.admin_serialize_node(store.find(models.Node).one(), GLSetting.memory_copy.default_language)
if not_notified_comments.count():
log.debug("Comments found to be notified: %d" % not_notified_comments.count() )
for comment in not_notified_comments:
if comment.internaltip is None or comment.internaltip.receivers is None:
log.err("Comment %s has internaltip or receivers broken reference" % comment.id)
comment.mark = models.Comment._marker[2] # 'unable to notify'
continue
# for every comment, iter on the associated receiver
log.debug("Comments receiver: %d" % comment.internaltip.receivers.count())
comment_desc = tip.serialize_comment(comment)
if not comment.internaltip.context:
log.err("(comment_notification) Integrity check failure Context")
continue
context_desc = admin.admin_serialize_context(comment.internaltip.context, GLSetting.memory_copy.default_language)
# XXX BUG! All notification is marked as correctly send,
# This can't be managed by callback, and can't be managed by actual DB design
comment.mark = models.Comment._marker[1] # 'notified'
for receiver in comment.internaltip.receivers:
receiver_desc = admin.admin_serialize_receiver(receiver, GLSetting.memory_copy.default_language)
if not receiver_desc.has_key('notification_fields') or\
not receiver.notification_fields.has_key('mail_address'):
log.err("Receiver %s lack of email address!" % receiver.name)
continue
# if the comment author is the one to be notified: skip the notification
# ----- BUG, remind,
# if two receiver has the same name, and one has notification disabled
# also the homonymous would get the notification dropped.
if comment._types == models.Comment._types[0] and comment.author == receiver.name:
log.debug("Receiver is the Author (%s): skipped" % receiver.user.username)
continue
# check if the receiver has the Comment notification enabled or not
if not receiver.comment_notification:
log.debug("Receiver %s has comment notification disabled: skipped [source: %s]" % (
receiver.user.username, comment.author))
continue
event = Event(type=u'comment', trigger='Comment',
notification_settings=self.notification_settings,
trigger_info=comment_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append(((unicode(comment.id), unicode(receiver.id)), event))
return events
def create_tip_notification_events(self, store, notification_counter):
"""
This transaction will return all a list of tuples containing the tips
for which the notification event has not been run.
Returns:
events: a list of tuples containing (tip_id, an instance of
:class:`globaleaks.plugins.base.Event`).
"""
language = GLSetting.memory_copy.default_language
events = []
# settings.notification_plugins contain a list of supported plugin
# at the moment only 1. so [0] is used. but different context/receiver
# may use different code-plugin:
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_tips = store.find(models.ReceiverTip,
models.ReceiverTip.mark == models.ReceiverTip._marker[0]
)
node_desc = admin.db_admin_serialize_node(store, language)
if not_notified_tips.count():
log.debug("Receiver Tips found to be notified: %d" % not_notified_tips.count() )
for receiver_tip in not_notified_tips:
notification_counter += 1
if notification_counter >= GLSetting.notification_limit:
log.debug("Notification counter has reached the suggested limit: %d (tip)" %
notification_counter)
break
if not receiver_tip.internaltip or not receiver_tip.internaltip.context:
log.err("(tip_notification) Integrity failure: missing (InternalTip/Context)")
store.remove(receiver_tip)
continue
context_desc = admin.admin_serialize_context(store, receiver_tip.internaltip.context, language)
receiver_desc = admin.admin_serialize_receiver(receiver_tip.receiver, language)
if not receiver_desc.has_key('mail_address'):
log.err("Receiver %s lack of email address!" % receiver_tip.receiver.name)
continue
# check if the receiver has the Tip notification enabled or not
if not receiver_desc['tip_notification']:
log.debug("Receiver %s has tip notification disabled" % receiver_tip.receiver.user.username)
receiver_tip.mark = models.ReceiverTip._marker[3] # 'disabled'
store.commit()
continue
tip_desc = serialize_receivertip(receiver_tip)
if receiver_desc['gpg_key_status'] == u'Enabled': # Receiver._gpg_types[1]
template_type = u'encrypted_tip'
else:
template_type = u'plaintext_tip'
notification_settings = self._get_notification_settings(store, receiver_desc['language'])
event = Event(type=template_type, trigger='Tip',
notification_settings=notification_settings,
trigger_info=tip_desc,
trigger_parent=None,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
steps_info = admin.db_get_context_steps(store,
context_desc['id'],
language),
plugin=plugin)
events.append((unicode(receiver_tip.id), event))
return events, notification_counter
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug("Delivery iterating over %d InternalFile from a Queue of %d" % (
GLSettings.jobs_operation_limit, ifilescnt ))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if (ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX):
ifile.new = False
error = "Failed to handle receiverfiles creation for ifilee %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif (ifile.processing_attempts >= 1):
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
REMIND: (keyword) esclation escalate pertinence vote
here need to be updated whenever an escalation is implemented.
checking of status and marker and recipients
"""
try:
files = store.find(InternalFile, InternalFile.mark == InternalFile._marker[0])
except Exception as excep:
log.err("Unable to find InternalFile in scheduler! %s" % str(excep))
return []
rfileslist = []
for filex in files:
if not filex.internaltip:
log.err("Integrity failure: the file %s of %s"\
"has not an InternalTip assigned (path: %s)" %
(filex.name, pretty_date_time(filex.creation_date), filex.file_path) )
try:
store.remove(filex)
except Exception as excep:
log.err("Unable to remove InternalFile in scheduler! %s" % str(excep))
continue
try:
os.unlink( os.path.join(GLSetting.submission_path, filex.file_path) )
except OSError as excep:
log.err("Unable to remove %s in integrity fixing routine: %s" %
(filex.file_path, excep.strerror) )
continue
# here we select the file which deserve to be processed.
# They need to be:
# From a Tip in (Tip = 'finalize' or 'first' )
# From an InternalFile (File = 'ready')
# Tips may have two statuses both valid.
# if these conditions are met the InternalFile(s) is/are marked as 'locked',
# Whenever a delivery scheduler run, do not touch 'locked' file, and if 'locked' file
# appears in the Admin interface of file overview, this mean that something is broken.
if (filex.internaltip.mark == InternalTip._marker[1] or \
filex.internaltip.mark == InternalTip._marker[2]) and \
(filex.mark == InternalFile._marker[0]):
filex.mark = InternalFile._marker[1] # 'locked'
else:
continue
try:
for receiver in filex.internaltip.receivers:
receiver_desc = admin_serialize_receiver(receiver, GLSetting.memory_copy.default_language)
if receiver_desc['gpg_key_status'] == Receiver._gpg_types[1] and receiver_desc['gpg_enable_files']:
rfileslist.append([ filex.id,
ReceiverFile._status_list[2], # encrypted
filex.file_path, filex.size, receiver_desc ])
else:
rfileslist.append([ filex.id,
ReceiverFile._status_list[1], # reference
filex.file_path, filex.size, receiver_desc ])
except Exception as excep:
log.debug("Invalid Storm operation in checking for GPG cap: %s" % excep)
continue
return rfileslist
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
try:
files = store.find(InternalFile, InternalFile.mark == u'not processed')
except Exception as excep:
log.err("Unable to find InternalFile in scheduler! %s" % str(excep))
return []
ifilesmap = {}
for filex in files:
if not filex.internaltip:
log.err("Integrity failure: the file %s"\
"has not an InternalTip assigned (path: %s)" %
(filex.name, filex.file_path) )
try:
os.remove(
os.path.join(GLSetting.submission_path, filex.file_path))
except OSError as excep:
log.err("Unable to remove %s in integrity fixing routine: %s" %
(filex.file_path, excep.strerror))
key_id = os.path.basename(filex.file_path).split('.')[0]
keypath = os.path.join(GLSetting.ramdisk_path,
("%s%s" %
(GLSetting.AES_keyfile_prefix, key_id)))
try:
os.remove(keypath)
except OSError as excep:
log.err("Unable to delete keyfile %s: %s" %
(keypath, excep.strerror))
# if the file is not associated to any tip it should be
# removed to avoid infinite loop
store.remove(filex)
continue
# here we select the file which deserve to be processed.
# They need to be:
# From a Tip in (Tip = 'finalize' or 'first' )
# From an InternalFile (File = 'ready')
# Tips may have two statuses both valid.
# if these conditions are met the InternalFile(s) is/are marked as 'locked',
# Whenever a delivery scheduler run, do not touch 'locked' file, and if 'locked' file
# appears in the Admin interface of file overview, this mean that something is broken.
if (filex.internaltip.mark == u'finalize' or \
filex.internaltip.mark == u'first') and \
(filex.mark == u'not processed'):
filex.mark = u'locked'
else:
continue
try:
for receiver in filex.internaltip.receivers:
if filex.file_path not in ifilesmap:
ifilesmap[filex.file_path] = list()
receiver_desc = admin_serialize_receiver(
receiver, GLSetting.memory_copy.language)
map_info = {
'receiver': receiver_desc,
'path': filex.file_path,
'size': filex.size,
'status': u'reference'
}
# this may seem apparently redounded, but is not!
# AS KEY, file path is used to keep track of the original
# path, because shall be renamed in .plaintext (in the unlucky case
# of receivers without PGP)
# AS FIELD, it can be replaced with a dedicated PGP encrypted path
ifilesmap[filex.file_path].append(map_info)
except Exception as excep:
log.debug("Invalid Storm operation in checking for PGP cap: %s" %
excep)
continue
return ifilesmap
def create_tip_notification_events(self, store):
"""
This transaction will return all a list of tuples containing the tips
for which the notification event has not been run.
Returns:
events: a list of tuples containing (tip_id, an instance of
:class:`globaleaks.plugins.base.Event`).
"""
events = []
# settings.notification_plugins contain a list of supported plugin
# at the moment only 1. so [0] is used. but different context/receiver
# may use different code-plugin:
cplugin = GLSetting.notification_plugins[0]
plugin = getattr(notification, cplugin)()
not_notified_tips = store.find(models.ReceiverTip,
models.ReceiverTip.mark == models.ReceiverTip._marker[0]
)
node_desc = admin.admin_serialize_node(store.find(models.Node).one(),
GLSetting.memory_copy.default_language)
if not_notified_tips.count():
log.debug("Receiver Tips found to be notified: %d" % not_notified_tips.count() )
for rtip in not_notified_tips:
if not rtip.internaltip or not rtip.internaltip.context:
log.err("(tip_notification) Integrity failure: missing InternalTip|Context")
continue
context_desc = admin.admin_serialize_context(rtip.internaltip.context, GLSetting.memory_copy.default_language)
receiver_desc = admin.admin_serialize_receiver(rtip.receiver, GLSetting.memory_copy.default_language)
if not receiver_desc.has_key('notification_fields') or\
not rtip.receiver.notification_fields.has_key('mail_address'):
log.err("Receiver %s lack of email address!" % rtip.receiver.name)
continue
# check if the receiver has the Tip notification enabled or not
if not receiver_desc['tip_notification']:
log.debug("Receiver %s has tip notification disabled" % rtip.receiver.user.username)
rtip.mark = models.ReceiverTip._marker[3] # 'disabled'
store.commit()
continue
tip_desc = serialize_receivertip(rtip)
event = Event(type=u'tip', trigger='Tip',
notification_settings=self.notification_settings,
trigger_info=tip_desc,
node_info=node_desc,
receiver_info=receiver_desc,
context_info=context_desc,
plugin=plugin)
events.append((unicode(rtip.id), event))
return events
