def receiverfile_create(store, if_path, recv_path, status, recv_size, receiver_desc):
try:
ifile = store.find(InternalFile, InternalFile.file_path == unicode(if_path)).one()
if not ifile:
log.err("InternalFile with path %s not found !?" % if_path)
raise Exception("This is bad!")
log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)"
% (receiver_desc['name'], ifile.name, recv_size, status ) )
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver_desc['id']).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver_desc['id']
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = unicode(recv_path)
receiverfile.size = recv_size
receiverfile.status = unicode(status)
store.add(receiverfile)
except Exception as excep:
log.err("Error when saving ReceiverFile %s for '%s': %s" %
(if_path, receiver_desc['name'], excep.message))
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
for ifile in store.find(InternalFile, InternalFile.new == True):
if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX:
ifile.new = False
error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
continue
elif ifile.processing_attempts >= 1:
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts += 1
for rtip in ifile.internaltip.receivertips:
receiverfile = ReceiverFile()
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtip.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
# https://github.com/globaleaks/GlobaLeaks/issues/444
# avoid to mark the receiverfile as new if it is part of a submission
# this way we avoid to send unuseful messages
receiverfile.new = False if ifile.submission else True
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(rtip.receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def receiverfile_create(store, fid, status, fpath, flen, receiver_desc):
assert type(1) == type(flen)
assert isinstance(receiver_desc, dict)
assert os.path.isfile(os.path.join(GLSetting.submission_path, fpath))
try:
ifile = store.find(InternalFile, InternalFile.id == unicode(fid)).one()
log.debug("ReceiverFile creation for user %s, file %s (%d bytes %s)"
% (receiver_desc['name'], ifile.name, flen, status ) )
receiverfile = ReceiverFile()
receiverfile.downloads = 0
receiverfile.receiver_id = receiver_desc['receiver_gus']
receiverfile.internalfile_id = ifile.id
receiverfile.internaltip_id = ifile.internaltip_id
# Receiver Tip reference
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver_desc['receiver_gus']).one()
receiverfile.receiver_tip_id = rtrf.id
# inherit by previous operation and checks
receiverfile.file_path = fpath
receiverfile.size = flen
receiverfile.status = status
receiverfile.mark = ReceiverFile._marker[0] # not notified
store.add(receiverfile)
store.commit()
# to avoid eventual file leakage or inconsistency, now is
# loaded the object to verify reference
test = store.find(ReceiverFile, ReceiverFile.internalfile_id == fid,
ReceiverFile.receiver_id == receiver_desc['receiver_gus']).one()
# assert over context, filesystem and receiver state
assert test.internaltip.context_id, "Context broken"
assert os.path.isfile(
os.path.join(GLSetting.submission_path,
test.file_path)), "FS broken (r)"
assert os.path.isfile(
os.path.join(GLSetting.submission_path,
test.internalfile.file_path)), "FS broken (i)"
assert test.receiver.user.state != u'disabled', "User broken"
return serialize_receiverfile(receiverfile)
except Exception as excep:
log.err("Error when saving ReceiverFile %s for %s: %s" % (
fpath, receiver_desc['name'], str(excep) ))
return []
def receiverfile_create(store, if_path, recv_path, status, recv_size,
receiver_desc):
assert type(1) == type(recv_size)
assert isinstance(receiver_desc, dict)
assert os.path.isfile(os.path.join(GLSetting.submission_path, if_path))
try:
ifile = store.find(InternalFile,
InternalFile.file_path == unicode(if_path)).one()
if not ifile:
log.err("InternalFile with path %s not found !?" % if_path)
raise Exception("This is bad!")
log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)" %
(receiver_desc['name'], ifile.name, recv_size, status))
receiverfile = ReceiverFile()
receiverfile.downloads = 0
receiverfile.receiver_id = receiver_desc['id']
receiverfile.internalfile_id = ifile.id
receiverfile.internaltip_id = ifile.internaltip_id
# Receiver Tip reference
rtrf = store.find(
ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver_desc['id']).one()
receiverfile.receiver_tip_id = rtrf.id
# inherited by previous operation and checks
receiverfile.file_path = unicode(recv_path)
receiverfile.size = recv_size
receiverfile.status = unicode(status)
receiverfile.mark = u'not notified'
store.add(receiverfile)
return serialize_receiverfile(receiverfile)
except Exception as excep:
log.err("Error when saving ReceiverFile %s for '%s': %s" %
(if_path, receiver_desc['name'], excep.message))
return []
def receiverfile_create(store, if_path, recv_path, status, recv_size, receiver_desc):
assert type(1) == type(recv_size)
assert isinstance(receiver_desc, dict)
assert os.path.isfile(os.path.join(GLSetting.submission_path, if_path))
try:
ifile = store.find(InternalFile, InternalFile.file_path == unicode(if_path)).one()
if not ifile:
log.err("InternalFile with path %s not found !?" % if_path)
raise Exception("This is bad!")
log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)"
% (receiver_desc['name'], ifile.name, recv_size, status ) )
receiverfile = ReceiverFile()
receiverfile.downloads = 0
receiverfile.receiver_id = receiver_desc['id']
receiverfile.internalfile_id = ifile.id
receiverfile.internaltip_id = ifile.internaltip_id
# Receiver Tip reference
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver_desc['id']).one()
receiverfile.receiver_tip_id = rtrf.id
# inherited by previous operation and checks
receiverfile.file_path = unicode(recv_path)
receiverfile.size = recv_size
receiverfile.status = unicode(status)
receiverfile.mark = u'not notified'
store.add(receiverfile)
return serialize_receiverfile(receiverfile)
except Exception as excep:
log.err("Error when saving ReceiverFile %s for '%s': %s" % (
if_path, receiver_desc['name'], excep.message))
return []
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(
InternalFile,
InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug(
"Delivery iterating over %d InternalFile from a Queue of %d" %
(GLSettings.jobs_operation_limit, ifilescnt))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if (ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX):
ifile.new = False
error = "Failed to handle receiverfiles creation for ifilee %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif (ifile.processing_attempts >= 1):
log.err(
"Failed to handle receiverfiles creation for ifile %s (retry %d/%d)"
% (ifile.id, ifile.processing_attempts,
INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug(
"Starting handling receiverfiles creation for ifile %s retry %d/%d"
% (ifile.id, ifile.processing_attempts,
INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(
ReceiverTip,
ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id':
receiverfile.id,
'status':
u'processing',
'path':
ifile.file_path,
'size':
ifile.size,
'receiver':
admin_serialize_receiver(
receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps
def receiverfile_planning(store):
"""
This function roll over the InternalFile uploaded, extract a path, id and
receivers associated, one entry for each combination. representing the
ReceiverFile that need to be created.
"""
receiverfiles_maps = {}
ifilescnt = store.find(InternalFile, InternalFile.new == True).count()
ifiles = store.find(InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit]
if ifilescnt > GLSettings.jobs_operation_limit:
log.debug("Delivery iterating over %d InternalFile from a queue of %d" % (
GLSettings.jobs_operation_limit, ifilescnt))
elif ifilescnt:
log.debug("Delivery iterating over %d InternalFile" % ifilescnt)
else:
pass # 0 files to be processed
for ifile in ifiles:
if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX:
ifile.new = False
error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \
(ifile.id, INTERNALFILES_HANDLE_RETRY_MAX)
log.err(error)
send_exception_email(error)
continue
elif ifile.processing_attempts >= 1:
log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
if ifile.processing_attempts:
log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" %
(ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX))
ifile.processing_attempts = ifile.processing_attempts + 1
for receiver in ifile.internaltip.receivers:
rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id,
ReceiverTip.receiver_id == receiver.id).one()
receiverfile = ReceiverFile()
receiverfile.receiver_id = receiver.id
receiverfile.internaltip_id = ifile.internaltip_id
receiverfile.internalfile_id = ifile.id
receiverfile.receivertip_id = rtrf.id
receiverfile.file_path = ifile.file_path
receiverfile.size = ifile.size
receiverfile.status = u'processing'
store.add(receiverfile)
if ifile.id not in receiverfiles_maps:
receiverfiles_maps[ifile.id] = {
'plaintext_file_needed': False,
'ifile_id': ifile.id,
'ifile_path': ifile.file_path,
'ifile_size': ifile.size,
'rfiles': []
}
receiverfiles_maps[ifile.id]['rfiles'].append({
'id': receiverfile.id,
'status': u'processing',
'path': ifile.file_path,
'size': ifile.size,
'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language)
})
return receiverfiles_maps