class ArchivingMission(AbstractObject):
def __init__(self, archiving_domain, project, id=None):
super(ArchivingMission, self).__init__()
if id is None:
self.id = self.generate_new_arch_mssn_id()
else:
self.id = id
# ensure_exists the link between itself and it's domain
self.archiving_domain = archiving_domain
self.app = self.archiving_domain.app
# ensure_exists the link between itself and it's project (from each end)
self.project = project
self.project.arch_mssns.append(self)
arch_mssn_folder_name_hidden = ".%s" % self.id # Make folder name start with a dot TODO: What's this?
arch_mssn_folder_name = self.id # Make folder name start with a dot TODO: What's this?
self.folder = ArchivingMissionFolder(self, os.path.join(project.drop_folder.path, arch_mssn_folder_name_hidden))
self.uploadcache_proj_folder = ArchivingMissionFolder(self, os.path.join(project.uploadcache_proj_folder.path, arch_mssn_folder_name))
self.confirmation_files_folder = ArchivingMissionFolder(self, os.path.join(project.confirmation_files_folder.path, arch_mssn_folder_name))
self.file_chunks = []
self.lock = None
self.lock_file = File(os.path.join(self.folder.path, "archiving_in_progress.lock"))
self.lock_file_uploadcache = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_in_progress.lock"))
self.state = "undefined"
self.state_file_new = File(os.path.join(self.folder.path, "archiving_state.new"))
self.state_file_failed_prepare_for_upload1 = File(os.path.join(self.folder.path, "archiving_state.failed_prepare_for_upload"))
self.state_file_failed_prepare_for_upload2 = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_state.failed_prepare_for_upload"))
self.state_file_prepared_for_upload1 = File(os.path.join(self.folder.path, "archiving_state.prepared_for_upload"))
self.state_file_prepared_for_upload2 = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_state.prepared_for_upload"))
self.state_file_failed_upload = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_state.failed_upload"))
self.state_file_uploaded = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_state.uploaded"))
self.state_file_failed_create_confirm_files = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_state.failed_create_confirm_files"))
self.state_file_created_confirm_files = File(os.path.join(self.uploadcache_proj_folder.path, "archiving_state.created_confirm_files"))
# ------------------------- MAIN ACTIONS -----------------------------
def prepare_for_upload(self):
if self.state not in ["new", "failed_prepare_for_upload"]:
self.log.info("Found existing arch_mssn %s, but had not status new or failed_prepare_for_upload" % self.id)
else:
self.log.info("Now preparing %s in project %s for upload ..." % (self.id, self.project.name))
# Some assertions
self.uploadcache_proj_folder.ensure_exists()
File(self.app.config.options["tarexcludepatternsfile"]).assert_exists()
self.partition_files_in_chunks()
for file_chunk in self.file_chunks:
file_chunk.create_as_tar_file_in_upload_cache()
file_chunk.create_meta_file()
if self.app.config.options["create_par2_files"] == "True": # TODO: Verify correct behaviour!
file_chunk.create_par2_files()
self.set_upload_cache_owner_to_upload_user() # Must be run as root!
def upload(self):
'''Upload all files that are ready for upload'''
files_to_upload = []
meta_files = self.uploadcache_proj_folder.get_files_matching_pattern(".*\.tar\.meta")
for meta_file in meta_files:
self.log.debug("Found meta meta_file: %s (path: %s) " % (meta_file.name, meta_file.path))
tar_file_basename = utils.rchop(meta_file.name, ".tar.meta")
files_to_upload.extend(self.uploadcache_proj_folder.get_files_matching_pattern("%s(\.tar\.split\d{5}|\.vol\d{3}\+\d{2}\.par2)?" % tar_file_basename))
if len(files_to_upload) == 0:
self.log.warn("No files to upload, for archiving mission %s in project %s" % (self.name, self.project.path))
else:
for file in files_to_upload:
swestore_path = self.create_swestore_path(self, file)
if not self.swestore_file_is_uploaded(swestore_path):
try:
self.upload_file(file, swestore_path)
except:
self.log.error("Failed to upload file %s" % file.path)
raise
else:
self.log.info("File already uploaded, so skipping: %s" % swestore_path)
def create_confirm_files(self):
confirm_folder_path = os.path.join(self.project.folder.path, self.app.config.options["confirm_files_path_rel"], self.id)
confirm_folder = Folder(confirm_folder_path)
confirm_folder.ensure_exists()
file_names_in_upload_cache = self.uploadcache_proj_folder.list_files()
file_names = self.get_strings_matching_pattern_from_list(".*\.meta", file_names_in_upload_cache)
for file_name in file_names:
src_path = os.path.join(self.uploadcache_proj_folder.path, file_name)
dest_path = os.path.join(confirm_folder.path, file_name)
try:
shutil.copy(src_path, dest_path)
self.log.info("Successfully moved file %s to %s" % (src_path, dest_path))
except:
self.log.error("Could not move file %s to %s" % (src_path, dest_path))
raise
# --------------------- HELPER METHODS -----------------------------------
def partition_files_in_chunks(self):
'''Divide the files/folders depending on their size into approximately
equally sized chunks. If one single file/folder exceeds the maximum chunk
size, the chunk is told to be split (which happens physically later on).'''
files_and_folders = self.get_file_list_sorted_by_size()
file_chunk = FileChunk(self)
for file in files_and_folders:
file_size = file.get_size()
rem_size = file_chunk.get_remaining_size()
if (file_size >= rem_size) and not file_chunk.is_empty():
# Store away current file chunk, and start filling a new one
self.file_chunks.append(file_chunk)
file_chunk = FileChunk(self)
file_chunk.add_file(file)
self.file_chunks.append(file_chunk)
def set_upload_cache_owner_to_upload_user(self):
'''Change owner of the uploadcache folder for the current project to a user
which has the required credentials for uploading the files to SweStore'''
uid = int(self.archiving_domain.app.config.options["swestoreuploaduid"])
gid = int(self.archiving_domain.app.config.options["swestoreuploadgid"])
try:
os.chown(self.uploadcache_proj_folder.path, uid, gid)
except:
self.log.error("Failed to chown folder to user with uid %d and gid %d. Are you really running as root?" % (uid, gid))
raise
# ----------------------- UPLOAD HELPER METHODS ------------------------------
def create_swestore_path(self, arch_mssn, file_to_upload):
swestore_path = os.path.join(self.app.config.options["swestorebasepath"], arch_mssn.project.name, arch_mssn.id, file_to_upload.name)
return swestore_path
def swestore_file_is_uploaded(self, swestore_path):
check_exist_cmd = ["ngls", swestore_path]
(stdout, stderr, returncode) = OSUtils().exec_command(check_exist_cmd, exception_on_nonzero_return_code=False)
if returncode != 0:
if re.match(".*No such file or directory.*", stderr, re.MULTILINE|re.DOTALL):
return False
else:
raise Exception("Listing of file did not succeed! Have you ran grid-proxy-init?")
else:
return True
def upload_file(self, file, path_on_swestore):
self.log.info("Uploading '%s' to '%s'" % (file.path, path_on_swestore))
upload_cmd = ["ngcp", "-r", "5", "-d", str(self.app.config.options["ngcp_debug_level"]), file.path, path_on_swestore]
retries_left = int(self.app.config.options["uploadretries"])
while retries_left > 0:
try:
OSUtils().exec_command(upload_cmd)
self.log.info("Upload successful!")
retries_left = 0 # Break the loop
except:
retries_left -= 1
if retries_left > 0:
self.log.warn("Failed uploading, trying again %d times ..." % retries_left)
else:
raise
# ----------------- CREATE CONFIRM FILES HELPER METHODS ------------------------
def get_strings_matching_pattern_from_list(self, pattern, list):
result_list = []
for item in list:
if re.match(pattern, item) and item not in result_list:
result_list.append(item)
return result_list
# ------------------- SOMEWHAT GENERIC HELPER METHODS ------------------------
def get_file_list_sorted_by_size(self):
'''Get a list of files and folders sorted per size, so that it can be used
to partition file in suitable sized chunks for each tar archive'''
files = self.folder.get_files_matching_pattern(self.app.config.options["allowedfilename_pattern"], self.app.config.options["excludedfilename_pattern"], recursive=True, only_empty_folders=True)
if len(files) == 0:
errmsg = "No files for arch mssn folder: %s" % self.folder.path
self.log.error(errmsg)
raise Exception(errmsg)
files_sorted_by_size = sorted(files, key=lambda file: file.get_size())
return files_sorted_by_size
def get_folders_in_folder_matching_pattern(self, folder, pattern, antipattern=""):
'''Method used as a more secure alternative to "glob". It returns all files in
the specified folders (except files in subfolders), whose filenames match the
specified regex pattern'''
resultfiles = []
try:
# List all files in the directory
files = os.listdir(folder)
for file in files:
# If they match the pattern ....
if (re.match(pattern, file) and antipattern=="") or (re.match(pattern, file) and not re.match(antipattern, file)):
filepath = os.path.join(folder, file)
# Then add them to the resulting list ...
resultfiles.append(filepath)
except Exception:
self.logger.error("Could not get files in folder %s, matching pattern %s (antipattern %s)" % (folder, pattern, antipattern))
raise
return resultfiles
def get_no_of_file_chunks(self):
return len(self.file_chunks)
def generate_new_arch_mssn_id(self):
time_part = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
arch_mssn_id = "arch_mssn-%s" % time_part
return arch_mssn_id
def detect_state_from_file_system(self):
if self.state_file_failed_create_confirm_files.exists():
self.set_state_failed_create_confirm_files()
elif self.state_file_created_confirm_files.exists():
self.set_state_created_confirm_files()
elif self.state_file_failed_upload.exists():
self.set_state_failed_upload()
elif self.state_file_uploaded.exists():
self.set_state_uploaded()
elif self.archiving_domain.app.action in ["prepare"] and self.state_file_failed_prepare_for_upload1.exists():
self.set_state_failed_prepare_for_upload()
elif self.state_file_failed_prepare_for_upload2.exists():
self.set_state_failed_prepare_for_upload()
elif self.archiving_domain.app.action in ["prepare"] and self.state_file_prepared_for_upload1.exists():
self.log.info("State file exists: %s" % self.state_file_prepared_for_upload1.path)
self.set_state_prepared_for_upload()
elif self.state_file_prepared_for_upload2.exists():
self.log.info("State file exists: %s" % self.state_file_prepared_for_upload2.path)
self.set_state_prepared_for_upload()
elif self.state_file_new.exists():
self.set_state_new()
def set_state_new(self):
self.state = "new"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
self.state_file_new.ensure_exists()
def set_state_prepared_for_upload(self):
self.state = "prepared_for_upload"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
# We can't be sure that the original drophere folder does exist, so we better
# check that first
if self.archiving_domain.app.action in ["prepare"]:
if self.folder.exists():
self.state_file_prepared_for_upload1.ensure_exists()
else:
self.log.warn("Archiving mission folder missing in project's drophere folder: %s" % self.folder.path)
if self.uploadcache_proj_folder.exists():
self.state_file_prepared_for_upload2.ensure_exists()
else:
self.log.warn("Archiving mission folder missing in project's upload cache folder: %s" % self.uploadcache_proj_folder.path)
def set_state_failed_prepare_for_upload(self):
self.state = "failed_prepare_for_upload"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
# We can't be sure that the original drophere folder does exist, so we better
# check that first
if self.archiving_domain.app.action in ["prepare"]:
if self.folder.exists():
self.state_file_failed_prepare_for_upload1.ensure_exists()
else:
self.log.warn("Archiving mission folder missing in project's drophere folder: %s" % self.folder.path)
if self.uploadcache_proj_folder.exists():
self.state_file_failed_prepare_for_upload2.ensure_exists()
else:
self.log.warn("Archiving mission folder missing in project's upload cache folder: %s" % self.uploadcache_proj_folder.path)
def set_state_uploaded(self):
self.state = "uploaded"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
self.state_file_uploaded.ensure_exists()
def set_state_failed_upload(self):
self.state = "failed_upload"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
self.state_file_failed_upload.ensure_exists()
def set_state_created_confirm_files(self):
self.state = "created_confirm_files"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
self.state_file_created_confirm_files.ensure_exists()
def set_state_failed_create_confirm_files(self):
self.state = "failed_create_confirm_files"
self.log.debug("Set state of archiving mission %s to: %s" % (self.id, self.state))
self.ensure_delete_all_state_files()
self.state_file_failed_create_confirm_files.ensure_exists()
def ensure_delete_all_state_files(self):
action = self.archiving_domain.app.action
if action in ["prepare", "createconfirmfiles"]:
self.state_file_new.ensure_delete()
self.state_file_failed_prepare_for_upload1.ensure_delete()
self.state_file_failed_prepare_for_upload2.ensure_delete()
self.state_file_prepared_for_upload1.ensure_delete()
self.state_file_prepared_for_upload2.ensure_delete()
self.state_file_failed_upload.ensure_delete()
self.state_file_uploaded.ensure_delete()
self.state_file_failed_create_confirm_files.ensure_delete()
self.state_file_created_confirm_files.ensure_delete()
elif action == "upload":
self.state_file_prepared_for_upload2.ensure_delete()
self.state_file_failed_upload.ensure_delete()
self.state_file_uploaded.ensure_delete()
def lock_main_folder(self):
'''Get an archiving_domain lock for the main folder (located under the project's drop folder)'''
self.lock_file.ensure_exists()
self.lock = ApplicationLock(self.lock_file)
# ------------------- LOCK FILE METHODS ------------------------------
def unlock_main_folder(self):
if self.lock is not None:
self.lock.unlock()
if self.lock_file.exists():
self.lock_file.delete()
def lock_uploadcache_folder(self):
'''Get an archiving_domain lock for the main folder (located under the project's drop folder)'''
self.log.debug("Trying to lock upload cache folder for %s in project %s ..." % (self.id, self.project.name))
self.lock_file_uploadcache.ensure_exists()
self.lock_uploadcache = ApplicationLock(self.lock_file_uploadcache)
self.log.debug("Succeded to lock upload cache folder for %s in project %s ..." % (self.id, self.project.name))
def unlock_uploadcache_folder(self):
if self.lock_uploadcache is not None:
self.lock_uploadcache.unlock()
if self.lock_file_uploadcache.exists():
self.lock_file_uploadcache.delete()
def has_unlocked_main_folder(self):
return self.folder.exists() and not self.lock_file.exists()
def has_unlocked_uploadcache_folder(self):
return self.uploadcache_proj_folder.exists() and not self.lock_file_uploadcache.exists()
def serve(request: bytes) -> str:
""" Handle the peer request
Parameters:
request - the list containing the request parameters
Returns:
str - the response
"""
command = request[0:4].decode('UTF-8')
if command == "LOGI":
if len(request) != 64:
return "0" * 16
ip = request[4:59].decode('UTF-8')
port = request[59:64].decode('UTF-8')
try:
conn = database.get_connection(db_file)
conn.row_factory = database.sqlite3.Row
except database.Error as e:
print(f'Error: {e}')
return "0" * 16
try:
peer = peer_repository.find_by_ip(conn, ip)
# if the peer didn't already logged in
if peer is None:
session_id = str(uuid.uuid4().hex[:16].upper())
peer = peer_repository.find(conn, session_id)
# while the generated session_id exists
while peer is not None:
session_id = str(uuid.uuid4().hex[:16].upper())
peer = peer_repository.find(conn, session_id)
peer = Peer(session_id, ip, port)
peer.insert(conn)
conn.commit()
conn.close()
except database.Error as e:
conn.close()
print(f'Error: {e}')
return "0" * 16
return "ALGI" + peer.session_id
elif command == "ADDF":
if len(request) != 152:
return "Invalid request. Usage is: ADDF<your_session_id><file_md5><filename>"
session_id = request[4:20].decode('UTF-8')
md5 = request[20:52].decode('UTF-8')
name = request[52:152].decode('UTF-8').lower()
try:
conn = database.get_connection(db_file)
conn.row_factory = database.sqlite3.Row
except database.Error as e:
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
try:
peer = peer_repository.find(conn, session_id)
if peer is None:
conn.close()
return "Unauthorized: your SessionID is invalid"
file = file_repository.find(conn, md5)
if file is None:
file = File(md5, name, 0)
file.insert(conn)
file_repository.add_owner(conn, md5, session_id)
else:
file.file_name = name
file.update(conn)
if not file_repository.peer_has_file(conn, session_id, md5):
file_repository.add_owner(conn, md5, session_id)
num_copies = file_repository.get_copies(conn, md5)
conn.commit()
conn.close()
except database.Error as e:
conn.rollback()
conn.close()
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
return "AADD" + str(num_copies).zfill(3)
elif command == "DELF":
if len(request) != 52:
return "Invalid request. Usage is: DELF<your_session_id><file_md5>"
session_id = request[4:20].decode('UTF-8')
md5 = request[20:52].decode('UTF-8')
try:
conn = database.get_connection(db_file)
conn.row_factory = database.sqlite3.Row
except database.Error as e:
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
try:
peer = peer_repository.find(conn, session_id)
if peer is None:
conn.close()
return "Unauthorized: your SessionID is invalid"
if not file_repository.peer_has_file(conn, session_id, md5):
conn.close()
return "ADEL999"
peer_repository.file_unlink(conn, session_id, md5)
copy = file_repository.get_copies(conn, md5)
if copy == 0:
file = file_repository.find(conn, md5)
file.delete(conn)
conn.commit()
conn.close()
except database.Error as e:
conn.rollback()
conn.close()
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
return "ADEL" + str(copy).zfill(3)
elif command == "FIND":
if len(request) != 40:
return "Invalid command. Usage is: FIND<your_session_id><query_string>"
session_id = request[4:20].decode('UTF-8')
query = request[20:40].decode('UTF-8').lower().lstrip().rstrip()
if query != '*':
query = '%' + query + '%'
try:
conn = database.get_connection(db_file)
conn.row_factory = database.sqlite3.Row
except database.Error as e:
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
try:
peer = peer_repository.find(conn, session_id)
if peer is None:
conn.close()
return "Unauthorized: your SessionID is invalid"
total_file = file_repository.get_files_count_by_querystring(conn, query)
if total_file == 0:
return 'AFIN' + str(total_file).zfill(3)
result = str(total_file).zfill(3)
file_list = file_repository.get_files_with_copy_amount_by_querystring(conn, query)
for file_row in file_list:
file_md5 = file_row['file_md5']
file_name = file_row['file_name']
copies = file_row['copies']
result = result + file_md5 + file_name + str(copies).zfill(3)
peer_list = peer_repository.get_peers_by_file(conn, file_md5)
for peer_row in peer_list:
peer_ip = peer_row['ip']
peer_port = peer_row['port']
result = result + peer_ip + peer_port
conn.commit()
conn.close()
except database.Error as e:
conn.rollback()
conn.close()
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
return "AFIN" + result
elif command == "DREG":
if len(request) != 52:
return "Invalid request. Usage is: DREG<your_session_id><file_md5>"
session_id = request[4:20].decode('UTF-8')
md5 = request[20:52].decode('UTF-8')
try:
conn = database.get_connection(db_file)
conn.row_factory = database.sqlite3.Row
except database.Error as e:
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
try:
peer = peer_repository.find(conn, session_id)
if peer is None:
conn.close()
return "Unauthorized: your SessionID is invalid"
file = file_repository.find(conn, md5)
if file is None:
return "File not found."
file.download_count += 1
file.update(conn)
conn.commit()
conn.close()
except database.Error as e:
conn.rollback()
conn.close()
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
return "ADRE" + str(file.download_count).zfill(5)
elif command == "LOGO":
if len(request) != 20:
return "Invalid request. Usage is: LOGO<your_session_id>"
session_id = request[4:20].decode('UTF-8')
try:
conn = database.get_connection(db_file)
conn.row_factory = database.sqlite3.Row
except database.Error as e:
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
try:
peer = peer_repository.find(conn, session_id)
if peer is None:
conn.close()
return "Unauthorized: your SessionID is invalid"
deleted = file_repository.delete_peer_files(conn, session_id)
peer.delete(conn)
conn.commit()
conn.close()
except database.Error as e:
conn.rollback()
conn.close()
print(f'Error: {e}')
return "The server has encountered an error while trying to serve the request."
return "ALGO" + str(deleted).zfill(3)
else:
return "Command \'" + request.decode('UTF-8') + "\' is invalid, try again."
