def list_bad_replicas_status(state=BadFilesStatus.BAD, rse=None, younger_than=None, older_than=None, limit=None, list_pfns=False): """ List the bad file replicas history states. Method used by the rucio-ui. :param state: The state of the file (SUSPICIOUS or BAD). :param rse: The RSE name. :param younger_than: datetime object to select bad replicas younger than this date. :param older_than: datetime object to select bad replicas older than this date. :param limit: The maximum number of replicas returned. """ rse_id = None if rse is not None: rse_id = get_rse_id(rse=rse) replicas = replica.list_bad_replicas_status(state=state, rse_id=rse_id, younger_than=younger_than, older_than=older_than, limit=limit, list_pfns=list_pfns) return [api_update_return_dict(r) for r in replicas]
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.replica_client = ReplicaClient() # Using two test RSEs self.rse4suspicious = 'MOCK_SUSPICIOUS' self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo) self.rse4recovery = 'MOCK_RECOVERY' self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo) self.scope = 'mock' self.internal_scope = InternalScope(self.scope, **self.vo) # For testing, we create 3 files and upload them to Rucio to two test RSEs. self.tmp_file1 = file_generator() self.tmp_file2 = file_generator() self.tmp_file3 = file_generator() self.tmp_file4 = file_generator() self.tmp_file5 = file_generator() self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE} for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5]] for rse in [self.rse4suspicious, self.rse4recovery]: cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5) exitcode, out, err = execute(cmd) # checking if Rucio upload went OK assert exitcode == 0 # Set fictional datatypes set_metadata(self.internal_scope, path.basename(self.tmp_file4), 'datatype', 'testtypedeclarebad') set_metadata(self.internal_scope, path.basename(self.tmp_file5), 'datatype', 'testtypenopolicy') # Allow for the RSEs to be affected by the suspicious file recovery daemon add_rse_attribute(self.rse4suspicious_id, "enable_suspicious_file_recovery", True) add_rse_attribute(self.rse4recovery_id, "enable_suspicious_file_recovery", True) # removing physical files from /tmp location - keeping only their DB info remove(self.tmp_file1) remove(self.tmp_file2) remove(self.tmp_file3) remove(self.tmp_file4) remove(self.tmp_file5) # Gather replica info replicalist = list_replicas(dids=self.listdids) # Changing the replica statuses as follows: # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious (available) testtypedeclarebad # tmp_file5 unavailable suspicious (available) testtypenopolicy # ---------------------------------------------------------------------------------------------------------------------------------- for replica in replicalist: suspicious_pfns = replica['rses'][self.rse4suspicious_id] for i in range(3): print("Declaring suspicious file replica: " + suspicious_pfns[0]) self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.') sleep(1) if replica['name'] == path.basename(self.tmp_file2): print("Declaring bad file replica: " + suspicious_pfns[0]) self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason') if replica['name'] == path.basename(self.tmp_file3): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE) if replica['name'] == path.basename(self.tmp_file4): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file4), ReplicaState.UNAVAILABLE) if replica['name'] == path.basename(self.tmp_file5): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file5), ReplicaState.UNAVAILABLE) # Gather replica info after setting initial replica statuses replicalist = list_replicas(dids=self.listdids) # Checking if the status changes were effective for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file4): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file5): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if only self.tmp_file2 is declared as 'BAD' self.from_date = datetime.now() - timedelta(days=1) bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_replica_recoverer(self): """ REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs. setUp function (above) is supposed to run first (nose does this automatically): - uploads 6 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS') - prepares their statuses to be as follows: # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious (available) testtypedeclare_bad # tmp_file5 unavailable suspicious (available) testtypenopolicy # ---------------------------------------------------------------------------------------------------------------------------------- - Explaination: Suspicious replicas that are the last remaining copy (unavailable on MOCK_RECOVERY) are handeled differently depending by their metadata "datatype". RAW files have the poilcy to be ignored. testtype_declare_bad files are of a fictional type that has the policy of being declared bad. testtype_nopolicy files are of a fictional type that doesn't have a policy specified, meaning they should be ignored by default. Runs the Test: - running suspicious_replica_recoverer Concluding: - checks that tmp_file1 and tmp_file4 were declared as 'BAD' on 'MOCK_SUSPICIOUS' """ # Run replica recoverer once try: run(once=True, younger_than=1, nattempts=2, limit_suspicious_files_on_rse=5) except KeyboardInterrupt: stop() # Checking the outcome: # we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious + bad (unavailable) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious + bad (unvailable) test_type_declare_bad # tmp_file5 unavailable suspicious (available) test_type_ignore # ---------------------------------------------------------------------------------------------------------------------------------- # Gather replica info after replica_recoverer has run. replicalist = list_replicas(dids=self.listdids) for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file4): # The key 'state' only exists if the replica is available on at least one RSE. It shouldn't exist for tmp_file4. assert ('states' in replica) is False if replica['name'] == path.basename(self.tmp_file5): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if replicas declared as 'BAD' bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_replica_recoverer(self): """ REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs. setUp function (above) is supposed to run first (nose does this automatically): - uploads 3 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS') - prepares their statuses to be as follows: # -------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) # -------------------------------------------------------------------------------------------- Runs the Test: - running suspicious_replica_recoverer Concluding: - checks that the only change made is that tmp_file1 was declared as 'BAD on 'MOCK_SUSPICIOUS' """ # Run replica recoverer once try: run(once=True, younger_than=1, nattempts=2, rse_expression='MOCK_SUSPICIOUS') except KeyboardInterrupt: stop() # Checking the outcome: # we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # tmp_file1 available suspicious + bad (unavailable) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) # -------------------------------------------------------------------------------------------- # Gather replica info after replica_recoverer has run. replicalist = list_replicas(dids=self.listdids) for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if replicas declared as 'BAD' bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_add_and_delete_bad_replicas(rse_factory, mock_scope, root_account, did_client, vo): """ REPLICA (CORE): Add bad replicas and delete them""" # Adding replicas to deterministic RSE nbfiles = 5 rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True, vo=vo) files = [{ 'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] client_files = [{ 'scope': file_['scope'].external, 'name': file_['name'] } for file_ in files] add_replicas(rse_id=rse1_id, files=files, account=root_account, ignore_availability=True) tmp_dsn = 'dataset_%s' % generate_uuid() did_client.add_dataset(scope=mock_scope.external, name=tmp_dsn) did_client.add_files_to_dataset(mock_scope.external, name=tmp_dsn, files=client_files, rse=rse1) # Declare replica bad replicas = [] for replica in list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE } for f in files], schemes=['srm']): replicas.extend(replica['rses'][rse1_id]) r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account) assert r == {} # Check state of bad replicas list_bad_rep = [{ 'scope': rep['scope'].external, 'name': rep['name'] } for rep in list_bad_replicas_status( state=BadFilesStatus.BAD, rse_id=rse1_id, vo=vo)] for rep in client_files: assert rep in list_bad_rep assert [ rep for rep in list_bad_replicas_status( state=BadFilesStatus.DELETED, rse_id=rse1_id, vo=vo) ] == [] # Now delete the dataset delete_dids([{ 'scope': mock_scope, 'name': tmp_dsn, 'did_type': DIDType.DATASET, 'purge_replicas': True }], account=root_account) assert [ rep for rep in list_bad_replicas_status( state=BadFilesStatus.BAD, rse_id=rse1_id, vo=vo) ] == [] list_deleted_rep = [{ 'scope': rep['scope'].external, 'name': rep['name'] } for rep in list_bad_replicas_status( state=BadFilesStatus.DELETED, rse_id=rse1_id, vo=vo)] for rep in client_files: assert rep in list_deleted_rep
def setUp(self): self.replica_client = ReplicaClient() # Using two test RSEs self.rse4suspicious = 'MOCK_SUSPICIOUS' self.rse4recovery = 'MOCK_RECOVERY' self.scope = 'mock' # For testing, we create 3 files and upload them to Rucio to two test RSEs. self.tmp_file1 = file_generator() self.tmp_file2 = file_generator() self.tmp_file3 = file_generator() self.listdids = [{ 'scope': self.scope, 'name': path.basename(self.tmp_file1), 'type': DIDType.FILE }, { 'scope': self.scope, 'name': path.basename(self.tmp_file2), 'type': DIDType.FILE }, { 'scope': self.scope, 'name': path.basename(self.tmp_file3), 'type': DIDType.FILE }] for rse in [self.rse4suspicious, self.rse4recovery]: cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format( rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3) exitcode, out, err = execute(cmd) # checking if Rucio upload went OK assert_true(exitcode == 0) # removing physical files from /tmp location - keeping only their DB info remove(self.tmp_file1) remove(self.tmp_file2) remove(self.tmp_file3) # Gather replica info replicalist = list_replicas(dids=self.listdids) # Changing the replica statuses as follows: # -------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) # -------------------------------------------------------------------------------------------- for replica in replicalist: for i in range(3): print("Declaring suspicious file replica: " + replica['rses'][self.rse4suspicious][0]) self.replica_client.declare_suspicious_file_replicas([ replica['rses'][self.rse4suspicious][0], ], 'This is a good reason.') sleep(1) if replica['name'] == path.basename(self.tmp_file2): print("Declaring bad file replica: " + replica['rses'][self.rse4suspicious][0]) self.replica_client.declare_bad_file_replicas([ replica['rses'][self.rse4suspicious][0], ], 'This is a good reason') if replica['name'] == path.basename(self.tmp_file3): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery][0]) update_replica_state(self.rse4recovery, self.scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE) # Gather replica info after setting initial replica statuses replicalist = list_replicas(dids=self.listdids) # Checking if the status changes were effective for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1): assert_true( replica['states'][self.rse4suspicious] == 'AVAILABLE') assert_true( replica['states'][self.rse4recovery] == 'AVAILABLE') if replica['name'] == path.basename(self.tmp_file2): assert_true( (self.rse4suspicious in replica['states']) is False) assert_true( replica['states'][self.rse4recovery] == 'AVAILABLE') if replica['name'] == path.basename(self.tmp_file3): assert_true( replica['states'][self.rse4suspicious] == 'AVAILABLE') assert_true((self.rse4recovery in replica['states']) is False) # Checking if only self.tmp_file2 is declared as 'BAD' self.from_date = datetime.now() - timedelta(days=1) bad_replicas_list = list_bad_replicas_status( rse=self.rse4suspicious, younger_than=self.from_date) bad_checklist = [(badf['name'], badf['rse'], badf['state']) for badf in bad_replicas_list] assert_true((path.basename(self.tmp_file2), self.rse4suspicious, BadFilesStatus.BAD) in bad_checklist) assert_true((path.basename(self.tmp_file1), self.rse4suspicious, BadFilesStatus.BAD) not in bad_checklist) assert_true((path.basename(self.tmp_file3), self.rse4suspicious, BadFilesStatus.BAD) not in bad_checklist) bad_replicas_list = list_bad_replicas_status( rse=self.rse4recovery, younger_than=self.from_date) bad_checklist = [(badf['name'], badf['rse'], badf['state']) for badf in bad_replicas_list] assert_true((path.basename(self.tmp_file1), self.rse4recovery, BadFilesStatus.BAD) not in bad_checklist) assert_true((path.basename(self.tmp_file2), self.rse4recovery, BadFilesStatus.BAD) not in bad_checklist) assert_true((path.basename(self.tmp_file3), self.rse4recovery, BadFilesStatus.BAD) not in bad_checklist)