def package_data(all_files_fetched, diss_instructions): # process files fetched for file_path in all_files_fetched: filename = os.path.basename(file_path) request_id_list = Database.get_id_list_by_filename(filename) # no reference => file is an orphan if request_id_list == []: Tools.remove_file(file_path, "orphan file", LOGGER) continue # purge requestId_list or req_id that are not in # diss_instructions keys. That is to prevent trying to find # an instruction file related to a file that has been processed # by a previous request request_id_list = [item for item in request_id_list if item in diss_instructions.keys()] LOGGER.info("Processing downloaded file %s linked to " "requests %s", file_path, request_id_list) # instantiate a DiffMetManager object that connects the file # to its single or multiple requests and instructioons diff_manager = DiffMetManager(request_id_list, file_path, diss_instructions) # rename files according to regex renaming_ok = diff_manager.rename() # package the archive if renaming_ok: diff_manager.compile_archive() else: msg = ("Dissemination failed for requests %s because user settings " "regex resulted in incorrect filename for difmet" % request_id_list) LOGGER.error(msg) for req_id in request_id_list: Database.update_field_by_query("requestStatus", REQ_STATUS.failed, **dict(fullrequestId=req_id)) Database.update_field_by_query("message", msg, **dict(fullrequestId=req_id))