def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
for ifile in store.find(InternalFile, InternalFile.new == True):
if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX:
ifile.new = False
error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
continue
elif ifile.processing_attempts >= 1:
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts += 1
for rtip in ifile.internaltip.receivertips:
receiverfile = ReceiverFile()
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtip.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
# https://github.com/globaleaks/GlobaLeaks/issues/444
# avoid to mark the receiverfile as new if it is part of a submission
# this way we avoid to send unuseful messages
receiverfile.new = False if ifile.submission else True
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(rtip.receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def receiverfile_create(store, if_path, recv_path, status, recv_size, receiver_desc):
try:
ifile = store.find(InternalFile, InternalFile.file_path == unicode(if_path)).one()
if not ifile:
log.err("InternalFile with path %s not found !?" % if_path)
raise Exception("This is bad!")
log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)"
% (receiver_desc['name'], ifile.name, recv_size, status ) )
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver_desc['id']).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver_desc['id']
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = unicode(recv_path)
receiverfile.size = recv_size
receiverfile.status = unicode(status)
store.add(receiverfile)
except Exception as excep:
log.err("Error when saving ReceiverFile %s for '%s': %s" %
(if_path, receiver_desc['name'], excep.message))
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(
InternalFile,
InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug(
"Delivery iterating over %d InternalFile from a Queue of %d" %
(GLSettings.jobs_operation_limit, ifilescnt))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if (ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX):
ifile.new = False
error = "Failed to handle receiverfiles creation for ifilee %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif (ifile.processing_attempts >= 1):
log.err(
"Failed to handle receiverfiles creation for ifile %s (retry %d/%d)"
% (ifile.id, ifile.processing_attempts,
INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug(
"Starting handling receiverfiles creation for ifile %s retry %d/%d"
% (ifile.id, ifile.processing_attempts,
INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(
ReceiverTip,
ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id':
receiverfile.id,
'status':
u'processing',
'path':
ifile.file_path,
'size':
ifile.size,
'receiver':
admin_serialize_receiver(
receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug("Delivery iterating over %d InternalFile from a queue of %d" % (
GLSettings.jobs_operation_limit, ifilescnt))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX:
ifile.new = False
error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif ifile.processing_attempts >= 1:
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
