def FAILED_completed(backend_object, config): """Do the tasks for a FAILED request: unlock the original files """ storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # these occur during a PUT or MIGRATE request fail_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(stage=MigrationRequest.FAILED) & Q(migration__storage__storage=storage_id) ) for fr in fail_reqs: try: # we want to use the locked FAILED requests to restore the original # permissions on the files if fr.locked: unlock_original_files(backend_object, fr, config) fr.unlock() # transition to FAILED_COMPLETED fr.stage = MigrationRequest.FAILED_COMPLETED fr.save() # log logging.info(( "Transition: request ID: {} external ID: {}: FAILED->FAILED_COMPLETED" ).format(fr.pk, fr.migration.external_id)) except Exception as e: logging.error("FAILED: error in FAILED_completed {}".format(str(e))) # look for FAILED_COMPLETED requests storage_id = StorageQuota.get_storage_index(backend_object.get_id()) fail_cmpl_reqs = MigrationRequest.objects.filter( Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.FAILED_COMPLETED) ) now = datetime.datetime.utcnow() num_days = datetime.timedelta(days=config["COMPLETED_REQUEST_DAYS"]) # for fr in fail_cmpl_reqs: if not fr: continue try: # remove the request if the requisite time has elapsed if (now - fr.date).days > num_days.days: logging.info("FAILED_COMPLETED: deleting request {}".format(fr.pk)) fr.delete() else: fr.unlock() except Exception as e: logging.error("FAILED_COMPLETED: error in FAILED_completed {}".format(str(e)))
def get_completed_deletes(backend_object): """Get all the completed deletes for the ObjectStore""" # avoiding a circular dependency from jdma_control.models import MigrationRequest, Migration, StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("objectstore") # get the decrypt key key = AES_tools.AES_read_key(settings.ENCRYPT_KEY_FILE) # list of completed DELETEs to return completed_DELETEs = [] # now loop over the PUT requests del_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.DELETE)) & Q(stage=MigrationRequest.DELETING) & Q(migration__storage__storage=storage_id)) for dr in del_reqs: # decrypt the credentials credentials = AES_tools.AES_decrypt_dict(key, dr.credentials) try: # create a connection to the object store s3c = boto3.client( "s3", endpoint_url=backend_object.OS_Settings["S3_ENDPOINT"], aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key']) # if the bucket has been deleted then the deletion has completed buckets = s3c.list_buckets() if ('Buckets' not in buckets or dr.migration.external_id not in buckets['Buckets']): completed_DELETEs.append(dr.migration.external_id) except Exception as e: raise Exception(e) return completed_DELETEs
def lock_put_migrations(backend_object, config): """Lock the directories that are going to be put to external storage. Also build the MigrationFiles entries from walking the directories This is to ensure that the user doesn't write any more data to them while the external storage write is ongoing. """ # get the storage id for the backend storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # get the list of PUT requests pr = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(locked=False) & Q(stage=MigrationRequest.PUT_START) & Q(migration__storage__storage=storage_id) ).first() # .first() returns None when no requests that match the filter are found if not pr: return # lock the Migration to prevent other processes acting upon it if not pr.lock(): return # carry out the lock migration try: lock_put_migration(pr, config) pr.unlock() except Exception as e: pr.unlock() mark_migration_failed(pr, str(e), e, True)
def monitor_put(completed_PUTs, backend_object): """Monitor the PUTs and MIGRATES and transition from PUTTING to VERIFY_PENDING (or FAILED)""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # now loop over the PUT requests pr_objs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) # & Q(locked=False) & Q(stage=MigrationRequest.PUTTING) & Q(migration__stage=Migration.PUTTING) & Q(migration__storage__storage=storage_id) ) for pr in pr_objs: # This is the standard locking code. See functions in "jdma_lock" for full # details if not pr: return # if not pr.lock(): # return pr.lock() ### # check whether it's in the completed_PUTs if pr.migration.external_id in completed_PUTs: # if it is then migrate to VERIFY_PENDING pr.stage = MigrationRequest.VERIFY_PENDING # reset the last_archive - needed for verify_get pr.last_archive = 0 pr.save() logging.info(( "Transition: request ID: {} external ID {} PUTTING->VERIFY_PENDING" ).format(pr.pk, pr.migration.external_id)) pr.unlock()
def monitor_get(completed_GETs, backend_object): """Monitor the GETs and transition from GETTING to ON_DISK (or FAILED)""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) gr_objs = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.GET) # & Q(locked=False) & Q(stage=MigrationRequest.GETTING) & Q(migration__storage__storage=storage_id) ) for gr in gr_objs: # This is the standard locking code. See functions in "jdma_lock" for full # details if not gr: return # if not gr.lock(): # return gr.lock() ### if gr.transfer_id in completed_GETs: # There may be multiple completed_GETs with external_id as Migrations # can be downloaded by multiple MigrationRequests # The only way to check is to make sure all the files in the # original migration are present in the target_dir gr.stage = MigrationRequest.GET_UNPACKING # reset the last archive counter gr.last_archive = 0 gr.save() logging.info(( "Transition: request ID: {} GETTING->GET_UNPACKING" ).format(gr.pk)) gr.unlock()
def monitor_delete(completed_DELETEs, backend_object): """Monitor the DELETEs and transition from DELETING to DELETE_TIDY""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) dr_objs = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.DELETE) # & Q(locked=False) & Q(stage=MigrationRequest.DELETING) & Q(migration__storage__storage=storage_id) ) for dr in dr_objs: # This is the standard locking code. See functions in "jdma_lock" for full # details if not dr: return # if not dr.lock(): # return dr.lock() ### if dr.migration.external_id in completed_DELETEs: dr.stage = MigrationRequest.DELETE_TIDY logging.info(( "Transition: request ID: {} external ID: {} DELETING->DELETE_TIDY" ).format(dr.pk, dr.migration.external_id)) # reset the last archive counter dr.last_archive = 0 dr.save() dr.unlock()
def monitor_verify(completed_GETs, backend_object): """Monitor the VERIFYs and transition from VERIFY_GETTING to VERIFYING""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) vr_objs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) # & Q(locked=False) & Q(stage=MigrationRequest.VERIFY_GETTING) & Q(migration__storage__storage=storage_id) ) for vr in vr_objs: # This is the standard locking code. See functions in "jdma_lock" for full # details if not vr: return # if not vr.lock(): # return vr.lock() ### if vr.transfer_id in completed_GETs: vr.stage = MigrationRequest.VERIFYING logging.info(( "Transition: request ID: {} external ID: {} VERIFY_GETTING->VERIFYING" ).format(vr.pk, vr.transfer_id)) # reset the last archive counter vr.last_archive = 0 vr.save() vr.unlock()
def verify(backend_object, credentials, pr): """Start the verification process. Transition from VERIFY_PENDING->VERIFY_GETTING and create the target directory. Download the batch from the backend storage to the target directory """ try: # open a connection to the backend. Creating the connection can account # for a significant portion of the run time. So we only do it once! global connection_pool conn = connection_pool.find_or_create_connection( backend_object, mig_req=pr, credentials=credentials, mode="download", uid="VERIFY") # Transition pr.stage = MigrationRequest.VERIFY_GETTING pr.save() logging.info(( "Transition: request ID: {} external ID: {} VERIFY_PENDING->VERIFY_GETTING" ).format(pr.pk, pr.migration.external_id)) # get the name of the verification directory target_dir = get_verify_dir(backend_object, pr) # create the target directory if it doesn't exist os.makedirs(target_dir, exist_ok=True) # for verify, we want to get the whole batch # get the archive set archive_set = pr.migration.migrationarchive_set.order_by('pk') # add all the files in the archive to a file_list for downloading file_list = [] for archive in archive_set: # add files in this archive to those already added if archive.packed: archive_files = [archive.get_archive_name()] else: archive_files = archive.get_file_names()['FILE'] file_list.extend(archive_files) logging.debug(("Downloading files to verify: {} from {}").format( file_list, backend_object.get_name())) backend_object.download_files(conn, pr, file_list=file_list, target_dir=target_dir) connection_pool.close_connection(backend_object, pr, credentials, mode="download", uid="VERIFY") except Exception as e: storage_name = StorageQuota.get_storage_name( pr.migration.storage.storage) error_string = ("Failed to download for verify the migration: {} " "on external storage: {}. Exception: {}").format( pr.migration.pk, storage_name, str(e)) raise Exception(error_string)
def GET_completed(backend_object, config): """Do the tasks for a completed GET request: send a notification email delete the request """ storage_id = StorageQuota.get_storage_index(backend_object.get_id()) gr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.GET) & Q(locked=False) & Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.GET_COMPLETED) ).first() now = datetime.datetime.utcnow() num_days = datetime.timedelta(days=config["COMPLETED_REQUEST_DAYS"]) if not gr: return if not gr.lock(): return try: # remove the request if the requisite time has elapsed if (now - gr.date).days > num_days.days: logging.info("GET: deleting GET request {}".format(gr.pk)) gr.delete() else: gr.unlock() except Exception as e: logging.error("GET: error in GET_completed {}".format(str(e)))
def GET_tidy(backend_object, config): """Do the clean up tasks for a completed GET request""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # these occur during a PUT or MIGRATE request gr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.GET) & Q(locked=False) & Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.GET_TIDY) ).first() if not gr: return if not gr.lock(): return try: # remove the temporary archive files (tarfiles) remove_archive_files(backend_object, gr) # update the request to GET_COMPLETED gr.stage = MigrationRequest.GET_COMPLETED logging.info(( "Transition: request ID: {} external ID: {}: GET_TIDY->GET_COMPLETED" ).format(gr.pk, gr.migration.external_id)) gr.migration.save() gr.save() # send a notification email that the gets have completed send_get_notification_email(backend_object, gr) except Exception as e: logging.error("GET: error in GET_tidy {}".format(str(e))) gr.unlock()
def get_completed_puts(backend_object): """Get all the completed puts for the ObjectStore""" # avoiding a circular dependency from jdma_control.models import MigrationRequest, Migration, StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("objectstore") # get the decrypt key key = AES_tools.AES_read_key(settings.ENCRYPT_KEY_FILE) # list of completed PUTs to return completed_PUTs = [] # now loop over the PUT requests put_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(stage=MigrationRequest.PUTTING) & Q(migration__stage=Migration.PUTTING) & Q(migration__storage__storage=storage_id)) for pr in put_reqs: # decrypt the credentials credentials = AES_tools.AES_decrypt_dict(key, pr.credentials) try: # create a connection to the object store s3c = boto3.client( "s3", endpoint_url=backend_object.OS_Settings["S3_ENDPOINT"], aws_access_key_id=credentials['access_key'], aws_secret_access_key=credentials['secret_key']) # loop over each archive in the migration archive_set = pr.migration.migrationarchive_set.order_by('pk') # counter for number of uploaded archives n_up_arch = 0 for archive in archive_set: # get the list of files for this archive if archive.packed: file_list = [archive.get_archive_name()] else: file_list = archive.get_file_names()['FILE'] n_files = 0 for file_path in file_list: # object name is the file_path, without any prefix try: if s3c.head_object(Bucket=pr.migration.external_id, Key=file_path): n_files += 1 except: pass # check if all files uploaded and then inc archive if n_files == len(file_list): n_up_arch += 1 if n_up_arch == pr.migration.migrationarchive_set.count(): completed_PUTs.append(pr.migration.external_id) except Exception as e: raise Exception(e) return completed_PUTs
def get_completed_puts(backend_object): """Get all the completed puts for the FTP backend""" # avoiding a circular dependency from jdma_control.models import MigrationRequest, Migration, StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("ftp") # get the decrypt key key = AES_tools.AES_read_key(settings.ENCRYPT_KEY_FILE) # list of completed PUTs to return completed_PUTs = [] # now loop over the PUT requests put_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(stage=MigrationRequest.PUTTING) & Q(migration__stage=Migration.PUTTING) & Q(migration__storage__storage=storage_id)) for pr in put_reqs: # decrypt the credentials credentials = AES_tools.AES_decrypt_dict(key, pr.credentials) try: ftp = ftplib.FTP(host=backend_object.FTP_Settings["FTP_ENDPOINT"], user=credentials['username'], passwd=credentials['password']) # loop over each archive in the migration archive_set = pr.migration.migrationarchive_set.order_by('pk') # counter for number of uploaded archives n_up_arch = 0 for archive in archive_set: # get the list of files for this archive file_list = archive.get_file_names()['FILE'] n_files = 0 for file_path in file_list['FILE']: # object name is the file_path, without the gws prefix object_name = (pr.migration.external_id + "/" + file_path) # enforce switch to binary (images here, but that doesn't # matter) ftp.voidcmd('TYPE I') try: fsize = ftp.size(object_name) if fsize is not None: n_files += 1 except: pass # check if all files uploaded and then inc archive if n_files == len(file_list): n_up_arch += 1 if n_up_arch == pr.migration.migrationarchive_set.count(): completed_PUTs.append(pr.migration.external_id) ftp.quit() except Exception as e: raise Exception(e) return completed_PUTs
def get_completed_puts(backend_object): """Get all the completed puts for the Elastic Tape""" # avoiding a circular dependency from jdma_control.models import MigrationRequest, Migration, StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("elastictape") # list of completed PUTs to return completed_PUTs = [] ET_Settings = backend_object.ET_Settings # now loop over the PUT requests put_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(stage=MigrationRequest.PUTTING) & Q(migration__stage=Migration.PUTTING) & Q(migration__storage__storage=storage_id)) for pr in put_reqs: if pr.migration.external_id is None: continue # form the url and get the response, parse the document using bs4 holdings_url = "{}?batch={}".format( ET_Settings["ET_INPUT_BATCH_SUMMARY_URL"], pr.migration.external_id) sleep(0.1) # 100 ms delay to avoid overloading the server r = requests.get(holdings_url) if r.status_code == 200: bs = BeautifulSoup(r.content, "xml") else: # log error rather than raising exception logging.error("Error in ET monitor:{} is unreachable".format( str(holdings_url))) continue # get the 2nd table - 1st is just a heading table table = bs.find_all("table")[1] if len(table) == 0: continue # get the first row rows = table.find_all("tr") if len(rows) < 2: continue row_1 = table.find_all("tr")[1] # the status is the first column cols = row_1.find_all("td") if len(cols) < 3: continue transfer_id = cols[0].get_text() status = cols[0].get_text() # check for completion if status in ["SYNCED", "TAPED"]: completed_PUTs.append(pr.migration.external_id) return completed_PUTs
def create_user_gws_quotas(data): # Create the User, GroupWorkspace and StorageQuota from each line of the # data storageid = StorageQuota.get_storage_index("objectstore") for line in data: if len(line) == 4: # create user entry new_gws = create_user_entry(line) # create the new storage quota create_quota_entry(storageid, new_gws, 32 * 10**12, 0)
def DELETE_completed(backend_object, config): """Do the tasks for a completed DELETE request: send a notification email delete the request delete the migration delete any associated requests (PUT, MIGRATE or GETs) """ storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # these occur during a PUT or MIGRATE request dr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.DELETE) & Q(locked=False) & Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.DELETE_COMPLETED) ).first() now = datetime.datetime.utcnow() num_days = datetime.timedelta(days=config["COMPLETED_REQUEST_DAYS"]) if not dr: return if not dr.lock(): return try: # remove the request if the requisite time has elapsed if (now - dr.date).days > num_days.days: # get the associated PUT or MIGRATE requests - there should only # be one other_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE) | Q(request_type=MigrationRequest.GET)) & Q(migration=dr.migration) & Q(migration__storage__storage=storage_id) ) for otr in other_reqs: logging.info(( "DELETE: deleting request {} associated with DELETE request {}." ).format(otr.pk, dr.pk)) otr.delete() logging.info("DELETE: deleting DELETE request {}".format(dr.pk)) dr.delete() # delete the migration logging.info(( "DELETE: deleting migration {} associated with DELETE request {}." ).format(dr.migration.pk, dr.pk)) dr.migration.delete() # we are done! else: dr.unlock() except Exception as e: logging.error("DELETE: error in DELETE_completed {}".format(str(e)))
def user_has_put_quota(self, conn): """Check the remaining quota for the user in the workspace. We just check the database here, i.e. check that we are not over quota. When jdma_lock calculates the file sizes we can check the quota again and flag the transfer as FAILED if it goes over the quota. """ from jdma_control.models import StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("objectstore") storage_quota = StorageQuota.objects.filter( storage=storage_id, workspace__workspace=conn.jdma_workspace)[0] return storage_quota.quota_used < storage_quota.quota_size
def get_transfers(backend_object, key): """Work through the state machine to download batches from the external storage""" # get the storage id for the backend object storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # get the GET requests which are queued (GET_PENDING) for this backend gr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.GET) & Q(locked=False) & Q(migration__storage__storage=storage_id) #& ~Q(user__name="n1280run") # NRM 21/04/2022 - blocking this user name for now until disk cleared & Q(stage__in=[ MigrationRequest.GET_PENDING, MigrationRequest.GETTING, MigrationRequest.GET_RESTORE, ])).first() # .first() returns None when no requests that match the filter are found if not gr: return # lock the Migration to prevent other processes acting upon it if not gr.lock(): return # determine the credentials for the user - decrypt if necessary if gr.credentials != {}: credentials = AES_tools.AES_decrypt_dict(key, gr.credentials) else: credentials = {} if gr.stage == MigrationRequest.GET_PENDING: # we might have to do something here, like create a download batch # for elastic tape. Also create the directory and transition the # state try: download(backend_object, credentials, gr) except Exception as e: # Something went wrong, set FAILED and failure_reason mark_migration_failed(gr, str(e), e, upload_mig=False) elif gr.stage == MigrationRequest.GETTING: pass elif gr.stage == MigrationRequest.GET_RESTORE: # restore the file permissions try: restore_owner_and_group_on_get(backend_object, gr) except Exception as e: mark_migration_failed(gr, str(e), e, upload_mig=False) gr.unlock()
def put_transfers(backend_object, key): """Work through the state machine to upload batches to the external storage""" # get the storage id for the backend object storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # Get the first non-locked PUT request for this backend. # This involves resolving two foreign keys pr = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(locked=False) & Q(migration__storage__storage=storage_id) & Q(stage__in=[ MigrationRequest.PUT_PENDING, MigrationRequest.VERIFY_PENDING, ])).first() # .first() returns None when no requests that match the filter are found if not pr: return # lock the Migration to prevent other processes acting upon it if not pr.lock(): return # determine the credentials for the user - decrypt if necessary if pr.credentials != {}: credentials = AES_tools.AES_decrypt_dict(key, pr.credentials) else: credentials = {} # Check whether data is being put to external storage if pr.stage == MigrationRequest.PUT_PENDING: # create the batch on this instance, next time the script is run # the archives will be created as tarfiles try: upload(backend_object, credentials, pr) except Exception as e: # Something went wrong, set FAILED and failure_reason mark_migration_failed(pr, str(e), e) # check if data is now on external storage and should be pulled # back for verification elif pr.stage == MigrationRequest.VERIFY_PENDING: # pull back the data from the external storage try: verify(backend_object, credentials, pr) except Exception as e: # Something went wrong, set FAILED and failure_reason mark_migration_failed(pr, str(e), e) # unlock pr.unlock()
def PUT_tidy(backend_object, config): """Do the clean up tasks for a completed PUT or MIGRATE request""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # these occur during a PUT or MIGRATE request pr = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(locked=False) & Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.PUT_TIDY) ).first() if not pr: return # check locked if not pr.lock(): return try: # remove the temporary staged archive files remove_archive_files(backend_object, pr) # remove the verification files remove_verification_files(backend_object, pr) # only remove the original files for a MIGRATE if pr.request_type == MigrationRequest.MIGRATE: remove_original_files(backend_object, pr, config) else: # otherwise unlock them (restore uids, gids and permissions) unlock_original_files(backend_object, pr, config) # set to completed and last archive to 0 # pr will be deleted next time jdma_tidy is invoked pr.stage = MigrationRequest.PUT_COMPLETED logging.info("Transition: deleting PUT request {}".format(pr.pk)) pr.migration.stage = Migration.ON_STORAGE logging.info(( "Transition: request ID: {} external ID: {}: PUT_TIDY->PUT_COMPLETED, PUTTING->ON_STORAGE" ).format(pr.pk, pr.migration.external_id)) pr.last_archive = 0 pr.migration.save() pr.save() # send a notification email that the puts have completed send_put_notification_email(backend_object, pr) # update the amount of quota the migration has used update_storage_quota(backend_object, pr.migration, update="add") except Exception as e: raise Exception(e) logging.error("Error in PUT_tidy {}".format(str(e))) pr.unlock()
def create_user_gws_quotas(data, config): # Create the User, GroupWorkspace and StorageQuota from each line of the # data storageid = StorageQuota.get_storage_index("elastictape") for line in data: if len(line) == 4: # create the user entry using the above script new_gws = create_user_entry(line) # get the quota and quota used quota, quota_used = get_et_quota_used(config["ET_QUOTA_URL"], line[0]) # create the new storage quota and assign the workspace create_quota_entry(storageid, new_gws, int(line[2]), quota_used) # sleep for 100ms to prevent server getting overloaded sleep(0.1)
def get_completed_deletes(backend_object): """Get all the completed deletes for the Elastic Tape""" # avoiding a circular dependency from jdma_control.models import MigrationRequest, Migration, StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("elastictape") ET_Settings = backend_object.ET_Settings # list of completed DELETEs to return completed_DELETEs = [] # now loop over the PUT requests del_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.DELETE)) & Q(stage=MigrationRequest.DELETING) & Q(migration__storage__storage=storage_id)) for dr in del_reqs: # assume deleted deleted = True # get a list of synced batches for this workspace and user holdings_url = "{}?workspace={};caller={};level=batch".format( ET_Settings["ET_HOLDINGS_URL"], dr.migration.workspace.workspace, dr.migration.user.name) # use requests to fetch the URL sleep(0.1) # 100 ms delay to avoid overloading the server r = requests.get(holdings_url) if r.status_code == 200: bs = BeautifulSoup(r.content, "xml") else: logging.error("Error in ET monitor:{} is unreachable".format( str(holdings_url))) continue # if the dr.migration.external_id is not in the list of batches # then the delete has completed batches = bs.select("batch") for b in batches: batch_id = b.find("batch_id").text.strip() if batch_id == dr.migration.external_id: deleted = False if deleted: # it's been deleted so add to the returned list of completed DELETEs completed_DELETEs.append(dr.migration.external_id) return completed_DELETEs
def remove_verification_files(backend_object, pr): """Remove those temporary files that have been created in the verification step""" storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # get the directory that the temporary files are in batch_id = pr.migration.external_id # get the temporary directory verify_dir = get_verify_dir(backend_object, pr) # remove the directory if os.path.isdir(verify_dir): try: shutil.rmtree(verify_dir) logging.debug("Deleting verify directory " + verify_dir) except Exception as e: logging.error(( "Could not delete verify directory {} : {}" ).format(verify_dir, str(e))) else: logging.debug("Cannot find verify directory " + verify_dir)
def verify_files(backend_object, config): """Verify the files that have been uploaded to external storage and then downloaded back to a temporary directory.""" # get the storage id for the backend object storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # these are part of a PUT request - get the list of PUT request pr = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(locked=False) & Q(stage=MigrationRequest.VERIFYING) & Q(migration__storage__storage=storage_id)).first() if not pr: return # lock the Migration to prevent other processes acting upon it if not pr.lock(): return verify(backend_object, pr, config) pr.unlock()
def lock_delete_migrations(backend_object): # get the storage id for the backend storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # get the list of GET requests dr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.DELETE) & Q(locked=False) & Q(stage=MigrationRequest.DELETE_START) & Q(migration__storage__storage=storage_id) ).first() if not dr: return if not dr.lock(): return try: lock_delete_migration(backend_object, dr) dr.unlock() except Exception as e: dr.unlock() mark_migration_failed(dr, str(e), e, False)
def user_has_put_quota(self, conn): """Check the remaining quota for the user in the workspace. We just check the database here, i.e. check that we are not over quota. When jdma_lock calculates the file sizes we can check the quota again and flag the transfer as FAILED if it goes over the quota. """ from jdma_control.models import StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("elastictape") storage_quota = StorageQuota.objects.filter( storage=storage_id, workspace__workspace=conn.jdma_workspace)[0] jdma_quota_remaining = storage_quota.quota_size - storage_quota.quota_used # get the quota from the elastic tape feed et_quota_remaining = workspace_quota_remaining( conn.jdma_user, conn.jdma_workspace.workspace, self.ET_Settings, ) return (jdma_quota_remaining > 0) & (et_quota_remaining > 0)
def get_unpacking(backend_object, config): """Unpack the ArchiveFiles from a TarFile to a target directory""" # get the storage id for the backend object storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # Get the GET requests for this backend which are in the PACKING stage. gr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.GET) & Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.GET_UNPACKING)).first() if not gr: return try: if not gr.lock(): return unpack_request(gr, get_download_dir(backend_object, gr), config) gr.unlock() except Exception as e: error_string = ("Could not unpack request for batch: {}: {}").format( str(gr.migration.external_id), str(e)) logging.error(error_string) mark_migration_failed(gr, error_string, e, upload_mig=True)
def lock_delete_migration(backend_object, dr): # lock this migration request as well # find the associated PUT, MIGRATE and GET migration requests and lock # them storage_id = StorageQuota.get_storage_index(backend_object.get_id()) other_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE) | Q(request_type=MigrationRequest.GET)) & Q(migration=dr.migration) & Q(migration__storage__storage=storage_id) ) # lock the associated migration(s) for otr in other_reqs: otr.lock() # transition to DELETE_PENDING dr.stage = MigrationRequest.DELETE_PENDING dr.save() logging.info("DELETE: Locked migration: {}".format(dr.migration.pk)) logging.info(( "Transition: request ID: {} DELETE_START->DELETE_PENDING" ).format(dr.pk))
def get_completed_deletes(backend_object): """Get all the completed deletes for the ObjectStore""" # avoiding a circular dependency from jdma_control.models import MigrationRequest, Migration, StorageQuota # get the storage id storage_id = StorageQuota.get_storage_index("ftp") # get the decrypt key key = AES_tools.AES_read_key(settings.ENCRYPT_KEY_FILE) # list of completed DELETEs to return completed_DELETEs = [] # now loop over the PUT requests del_reqs = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.DELETE)) & Q(stage=MigrationRequest.DELETING) & Q(migration__storage__storage=storage_id)) for dr in del_reqs: # decrypt the credentials credentials = AES_tools.AES_decrypt_dict(key, dr.credentials) try: # create a connection to the object store ftp = ftplib.FTP(host=backend_object.FTP_Settings["FTP_ENDPOINT"], user=credentials['username'], passwd=credentials['password']) # if the external_id directory has been deleted then the # deletion has completed dir_list = ftp.mlsd("/") found = False for d in dir_list: # check if directory and groupworkspace name is in directory if d[1]['type'] == 'dir' and dr.migration.external_id in d[0]: found = True break if not found: completed_DELETEs.append(dr.migration.external_id) except Exception as e: raise Exception(e) return completed_DELETEs
def lock_get_migrations(backend_object): """Lock the directories that the targets for recovering data from external storage. This is to ensure that there aren't any filename conflicts. """ # get the storage id for the backend storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # get the list of GET requests gr = MigrationRequest.objects.filter( Q(request_type=MigrationRequest.GET) & Q(locked=False) & Q(stage=MigrationRequest.GET_START) & Q(migration__storage__storage=storage_id) ).first() if not gr: return if not gr.lock(): return try: lock_get_migration(gr) gr.unlock() except Exception as e: gr.unlock() mark_migration_failed(gr, str(e), e, False)
def put_packing(backend_object, config): """Pack the ArchiveFiles into a TarFile in the ARCHIVE_STAGING_DIR for this backend""" # get the storage id for the backend object storage_id = StorageQuota.get_storage_index(backend_object.get_id()) # Get the PUT requests for this backend which are in the PACKING stage pr = MigrationRequest.objects.filter( (Q(request_type=MigrationRequest.PUT) | Q(request_type=MigrationRequest.MIGRATE)) & Q(locked=False) & Q(migration__storage__storage=storage_id) & Q(stage=MigrationRequest.PUT_PACKING)).first() if not pr: return try: if not pr.lock(): return pack_request(pr, backend_object.ARCHIVE_STAGING_DIR, config) pr.unlock() except Exception as e: error_string = ("Could not pack archive for batch: {}: {}").format( pr.migration.get_id(), str(e)) # mark the migration as failed mark_migration_failed(pr, error_string, e)