def receiverfile_create(store, if_path, recv_path, status, recv_size, receiver_desc): try: ifile = store.find(InternalFile, InternalFile.file_path == unicode(if_path)).one() if not ifile: log.err("InternalFile with path %s not found !?" % if_path) raise Exception("This is bad!") log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)" % (receiver_desc['name'], ifile.name, recv_size, status ) ) rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver_desc['id']).one() receiverfile = ReceiverFile() receiverfile.receiver_id = receiver_desc['id'] receiverfile.internaltip_id = ifile.internaltip_id receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtrf.id receiverfile.file_path = unicode(recv_path) receiverfile.size = recv_size receiverfile.status = unicode(status) store.add(receiverfile) except Exception as excep: log.err("Error when saving ReceiverFile %s for '%s': %s" % (if_path, receiver_desc['name'], excep.message))
def receiverfile_create(store, fid, status, fpath, flen, receiver_desc): assert type(1) == type(flen) assert isinstance(receiver_desc, dict) assert os.path.isfile(os.path.join(GLSetting.submission_path, fpath)) try: ifile = store.find(InternalFile, InternalFile.id == unicode(fid)).one() log.debug("ReceiverFile creation for user %s, file %s (%d bytes %s)" % (receiver_desc['name'], ifile.name, flen, status ) ) receiverfile = ReceiverFile() receiverfile.downloads = 0 receiverfile.receiver_id = receiver_desc['receiver_gus'] receiverfile.internalfile_id = ifile.id receiverfile.internaltip_id = ifile.internaltip_id # Receiver Tip reference rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver_desc['receiver_gus']).one() receiverfile.receiver_tip_id = rtrf.id # inherit by previous operation and checks receiverfile.file_path = fpath receiverfile.size = flen receiverfile.status = status receiverfile.mark = ReceiverFile._marker[0] # not notified store.add(receiverfile) store.commit() # to avoid eventual file leakage or inconsistency, now is # loaded the object to verify reference test = store.find(ReceiverFile, ReceiverFile.internalfile_id == fid, ReceiverFile.receiver_id == receiver_desc['receiver_gus']).one() # assert over context, filesystem and receiver state assert test.internaltip.context_id, "Context broken" assert os.path.isfile( os.path.join(GLSetting.submission_path, test.file_path)), "FS broken (r)" assert os.path.isfile( os.path.join(GLSetting.submission_path, test.internalfile.file_path)), "FS broken (i)" assert test.receiver.user.state != u'disabled', "User broken" return serialize_receiverfile(receiverfile) except Exception as excep: log.err("Error when saving ReceiverFile %s for %s: %s" % ( fpath, receiver_desc['name'], str(excep) )) return []
def receiverfile_create(store, if_path, recv_path, status, recv_size, receiver_desc): assert type(1) == type(recv_size) assert isinstance(receiver_desc, dict) assert os.path.isfile(os.path.join(GLSetting.submission_path, if_path)) try: ifile = store.find(InternalFile, InternalFile.file_path == unicode(if_path)).one() if not ifile: log.err("InternalFile with path %s not found !?" % if_path) raise Exception("This is bad!") log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)" % (receiver_desc['name'], ifile.name, recv_size, status)) receiverfile = ReceiverFile() receiverfile.downloads = 0 receiverfile.receiver_id = receiver_desc['id'] receiverfile.internalfile_id = ifile.id receiverfile.internaltip_id = ifile.internaltip_id # Receiver Tip reference rtrf = store.find( ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver_desc['id']).one() receiverfile.receiver_tip_id = rtrf.id # inherited by previous operation and checks receiverfile.file_path = unicode(recv_path) receiverfile.size = recv_size receiverfile.status = unicode(status) receiverfile.mark = u'not notified' store.add(receiverfile) return serialize_receiverfile(receiverfile) except Exception as excep: log.err("Error when saving ReceiverFile %s for '%s': %s" % (if_path, receiver_desc['name'], excep.message)) return []
def receiverfile_create(store, if_path, recv_path, status, recv_size, receiver_desc): assert type(1) == type(recv_size) assert isinstance(receiver_desc, dict) assert os.path.isfile(os.path.join(GLSetting.submission_path, if_path)) try: ifile = store.find(InternalFile, InternalFile.file_path == unicode(if_path)).one() if not ifile: log.err("InternalFile with path %s not found !?" % if_path) raise Exception("This is bad!") log.debug("ReceiverFile creation for user %s, '%s' bytes %d = %s)" % (receiver_desc['name'], ifile.name, recv_size, status ) ) receiverfile = ReceiverFile() receiverfile.downloads = 0 receiverfile.receiver_id = receiver_desc['id'] receiverfile.internalfile_id = ifile.id receiverfile.internaltip_id = ifile.internaltip_id # Receiver Tip reference rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver_desc['id']).one() receiverfile.receiver_tip_id = rtrf.id # inherited by previous operation and checks receiverfile.file_path = unicode(recv_path) receiverfile.size = recv_size receiverfile.status = unicode(status) receiverfile.mark = u'not notified' store.add(receiverfile) return serialize_receiverfile(receiverfile) except Exception as excep: log.err("Error when saving ReceiverFile %s for '%s': %s" % ( if_path, receiver_desc['name'], excep.message)) return []
def receiverfile_planning(store): """ This function roll over the InternalFile uploaded, extract a path, id and receivers associated, one entry for each combination. representing the ReceiverFile that need to be created. """ receiverfiles_maps = {} for ifile in store.find(InternalFile, InternalFile.new == True): if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX: ifile.new = False error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \ (ifile.id, INTERNALFILES_HANDLE_RETRY_MAX) log.err(error) send_exception_email(error) continue elif ifile.processing_attempts >= 1: log.err( "Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) if ifile.processing_attempts: log.debug( "Starting handling receiverfiles creation for ifile %s retry %d/%d" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) ifile.processing_attempts += 1 for receiver in ifile.internaltip.receivers: rtrf = store.find( ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver.id).one() receiverfile = ReceiverFile() receiverfile.receiver_id = receiver.id receiverfile.internaltip_id = ifile.internaltip_id receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtrf.id receiverfile.file_path = ifile.file_path receiverfile.size = ifile.size receiverfile.status = u'processing' # https://github.com/globaleaks/GlobaLeaks/issues/444 # avoid to mark the receiverfile as new if it is part of a submission # this way we avoid to send unuseful messages receiverfile.new = False if ifile.submission else True store.add(receiverfile) if ifile.id not in receiverfiles_maps: receiverfiles_maps[ifile.id] = { 'plaintext_file_needed': False, 'ifile_id': ifile.id, 'ifile_path': ifile.file_path, 'ifile_size': ifile.size, 'rfiles': [] } receiverfiles_maps[ifile.id]['rfiles'].append({ 'id': receiverfile.id, 'status': u'processing', 'path': ifile.file_path, 'size': ifile.size, 'receiver': admin_serialize_receiver( receiver, GLSettings.memory_copy.default_language) }) return receiverfiles_maps
def receiverfile_planning(store): """ This function roll over the InternalFile uploaded, extract a path, id and receivers associated, one entry for each combination. representing the ReceiverFile that need to be created. """ receiverfiles_maps = {} ifilescnt = store.find(InternalFile, InternalFile.new == True).count() ifiles = store.find( InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit] if ifilescnt > GLSettings.jobs_operation_limit: log.debug( "Delivery iterating over %d InternalFile from a Queue of %d" % (GLSettings.jobs_operation_limit, ifilescnt)) elif ifilescnt: log.debug("Delivery iterating over %d InternalFile" % ifilescnt) else: pass # 0 files to be processed for ifile in ifiles: if (ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX): ifile.new = False error = "Failed to handle receiverfiles creation for ifilee %s (%d retries)" % \ (ifile.id, INTERNALFILES_HANDLE_RETRY_MAX) log.err(error) send_exception_email(error) continue elif (ifile.processing_attempts >= 1): log.err( "Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) if ifile.processing_attempts: log.debug( "Starting handling receiverfiles creation for ifile %s retry %d/%d" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) ifile.processing_attempts = ifile.processing_attempts + 1 for receiver in ifile.internaltip.receivers: rtrf = store.find( ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver.id).one() receiverfile = ReceiverFile() receiverfile.receiver_id = receiver.id receiverfile.internaltip_id = ifile.internaltip_id receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtrf.id receiverfile.file_path = ifile.file_path receiverfile.size = ifile.size receiverfile.status = u'processing' store.add(receiverfile) if ifile.id not in receiverfiles_maps: receiverfiles_maps[ifile.id] = { 'plaintext_file_needed': False, 'ifile_id': ifile.id, 'ifile_path': ifile.file_path, 'ifile_size': ifile.size, 'rfiles': [] } receiverfiles_maps[ifile.id]['rfiles'].append({ 'id': receiverfile.id, 'status': u'processing', 'path': ifile.file_path, 'size': ifile.size, 'receiver': admin_serialize_receiver( receiver, GLSettings.memory_copy.default_language) }) return receiverfiles_maps
def receiverfile_planning(store): """ This function roll over the InternalFile uploaded, extract a path, id and receivers associated, one entry for each combination. representing the ReceiverFile that need to be created. """ receiverfiles_maps = {} ifilescnt = store.find(InternalFile, InternalFile.new == True).count() ifiles = store.find(InternalFile, InternalFile.new == True)[:GLSettings.jobs_operation_limit] if ifilescnt > GLSettings.jobs_operation_limit: log.debug("Delivery iterating over %d InternalFile from a queue of %d" % ( GLSettings.jobs_operation_limit, ifilescnt)) elif ifilescnt: log.debug("Delivery iterating over %d InternalFile" % ifilescnt) else: pass # 0 files to be processed for ifile in ifiles: if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX: ifile.new = False error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \ (ifile.id, INTERNALFILES_HANDLE_RETRY_MAX) log.err(error) send_exception_email(error) continue elif ifile.processing_attempts >= 1: log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) if ifile.processing_attempts: log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) ifile.processing_attempts = ifile.processing_attempts + 1 for receiver in ifile.internaltip.receivers: rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver.id).one() receiverfile = ReceiverFile() receiverfile.receiver_id = receiver.id receiverfile.internaltip_id = ifile.internaltip_id receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtrf.id receiverfile.file_path = ifile.file_path receiverfile.size = ifile.size receiverfile.status = u'processing' store.add(receiverfile) if ifile.id not in receiverfiles_maps: receiverfiles_maps[ifile.id] = { 'plaintext_file_needed': False, 'ifile_id': ifile.id, 'ifile_path': ifile.file_path, 'ifile_size': ifile.size, 'rfiles': [] } receiverfiles_maps[ifile.id]['rfiles'].append({ 'id': receiverfile.id, 'status': u'processing', 'path': ifile.file_path, 'size': ifile.size, 'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language) }) return receiverfiles_maps
def receiverfile_planning(store): """ This function roll over the InternalFile uploaded, extract a path, id and receivers associated, one entry for each combination. representing the ReceiverFile that need to be created. """ receiverfiles_maps = {} for ifile in store.find(InternalFile, InternalFile.new == True): if ifile.processing_attempts >= INTERNALFILES_HANDLE_RETRY_MAX: ifile.new = False error = "Failed to handle receiverfiles creation for ifile %s (%d retries)" % \ (ifile.id, INTERNALFILES_HANDLE_RETRY_MAX) log.err(error) send_exception_email(error) continue elif ifile.processing_attempts >= 1: log.err("Failed to handle receiverfiles creation for ifile %s (retry %d/%d)" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) if ifile.processing_attempts: log.debug("Starting handling receiverfiles creation for ifile %s retry %d/%d" % (ifile.id, ifile.processing_attempts, INTERNALFILES_HANDLE_RETRY_MAX)) ifile.processing_attempts += 1 for receiver in ifile.internaltip.receivers: rtrf = store.find(ReceiverTip, ReceiverTip.internaltip_id == ifile.internaltip_id, ReceiverTip.receiver_id == receiver.id).one() receiverfile = ReceiverFile() receiverfile.receiver_id = receiver.id receiverfile.internaltip_id = ifile.internaltip_id receiverfile.internalfile_id = ifile.id receiverfile.receivertip_id = rtrf.id receiverfile.file_path = ifile.file_path receiverfile.size = ifile.size receiverfile.status = u'processing' # https://github.com/globaleaks/GlobaLeaks/issues/444 # avoid to mark the receiverfile as new if it is part of a submission # this way we avoid to send unuseful messages receiverfile.new = False if ifile.submission else True store.add(receiverfile) if ifile.id not in receiverfiles_maps: receiverfiles_maps[ifile.id] = { 'plaintext_file_needed': False, 'ifile_id': ifile.id, 'ifile_path': ifile.file_path, 'ifile_size': ifile.size, 'rfiles': [] } receiverfiles_maps[ifile.id]['rfiles'].append({ 'id': receiverfile.id, 'status': u'processing', 'path': ifile.file_path, 'size': ifile.size, 'receiver': admin_serialize_receiver(receiver, GLSettings.memory_copy.default_language) }) return receiverfiles_maps