class TestReplicaRecoverer(unittest.TestCase): def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.replica_client = ReplicaClient() # Using two test RSEs self.rse4suspicious = 'MOCK_SUSPICIOUS' self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo) self.rse4recovery = 'MOCK_RECOVERY' self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo) self.scope = 'mock' self.internal_scope = InternalScope(self.scope, **self.vo) # For testing, we create 3 files and upload them to Rucio to two test RSEs. self.tmp_file1 = file_generator() self.tmp_file2 = file_generator() self.tmp_file3 = file_generator() self.tmp_file4 = file_generator() self.tmp_file5 = file_generator() self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE} for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5]] for rse in [self.rse4suspicious, self.rse4recovery]: cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5) exitcode, out, err = execute(cmd) # checking if Rucio upload went OK assert exitcode == 0 # Set fictional datatypes set_metadata(self.internal_scope, path.basename(self.tmp_file4), 'datatype', 'testtypedeclarebad') set_metadata(self.internal_scope, path.basename(self.tmp_file5), 'datatype', 'testtypenopolicy') # Allow for the RSEs to be affected by the suspicious file recovery daemon add_rse_attribute(self.rse4suspicious_id, "enable_suspicious_file_recovery", True) add_rse_attribute(self.rse4recovery_id, "enable_suspicious_file_recovery", True) # removing physical files from /tmp location - keeping only their DB info remove(self.tmp_file1) remove(self.tmp_file2) remove(self.tmp_file3) remove(self.tmp_file4) remove(self.tmp_file5) # Gather replica info replicalist = list_replicas(dids=self.listdids) # Changing the replica statuses as follows: # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious (available) testtypedeclarebad # tmp_file5 unavailable suspicious (available) testtypenopolicy # ---------------------------------------------------------------------------------------------------------------------------------- for replica in replicalist: suspicious_pfns = replica['rses'][self.rse4suspicious_id] for i in range(3): print("Declaring suspicious file replica: " + suspicious_pfns[0]) self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.') sleep(1) if replica['name'] == path.basename(self.tmp_file2): print("Declaring bad file replica: " + suspicious_pfns[0]) self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason') if replica['name'] == path.basename(self.tmp_file3): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE) if replica['name'] == path.basename(self.tmp_file4): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file4), ReplicaState.UNAVAILABLE) if replica['name'] == path.basename(self.tmp_file5): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file5), ReplicaState.UNAVAILABLE) # Gather replica info after setting initial replica statuses replicalist = list_replicas(dids=self.listdids) # Checking if the status changes were effective for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file4): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file5): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if only self.tmp_file2 is declared as 'BAD' self.from_date = datetime.now() - timedelta(days=1) bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist # On purpose not checking for status to be declared 'SUSPICIOUS' on MOCK_SUSPICIOUS. # The only existing function (to date) gathering info about 'SUSPICIOUS' replicas # is used (currently only) from the tested replica_recoverer itself. def test_replica_recoverer(self): """ REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs. setUp function (above) is supposed to run first (nose does this automatically): - uploads 6 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS') - prepares their statuses to be as follows: # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious (available) testtypedeclare_bad # tmp_file5 unavailable suspicious (available) testtypenopolicy # ---------------------------------------------------------------------------------------------------------------------------------- - Explaination: Suspicious replicas that are the last remaining copy (unavailable on MOCK_RECOVERY) are handeled differently depending by their metadata "datatype". RAW files have the poilcy to be ignored. testtype_declare_bad files are of a fictional type that has the policy of being declared bad. testtype_nopolicy files are of a fictional type that doesn't have a policy specified, meaning they should be ignored by default. Runs the Test: - running suspicious_replica_recoverer Concluding: - checks that tmp_file1 and tmp_file4 were declared as 'BAD' on 'MOCK_SUSPICIOUS' """ # Run replica recoverer once try: run(once=True, younger_than=1, nattempts=2, limit_suspicious_files_on_rse=5) except KeyboardInterrupt: stop() # Checking the outcome: # we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious + bad (unavailable) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious + bad (unvailable) test_type_declare_bad # tmp_file5 unavailable suspicious (available) test_type_ignore # ---------------------------------------------------------------------------------------------------------------------------------- # Gather replica info after replica_recoverer has run. replicalist = list_replicas(dids=self.listdids) for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file4): # The key 'state' only exists if the replica is available on at least one RSE. It shouldn't exist for tmp_file4. assert ('states' in replica) is False if replica['name'] == path.basename(self.tmp_file5): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if replicas declared as 'BAD' bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
class TestReplicaClients: def setup(self): self.replica_client = ReplicaClient() self.did_client = DIDClient() def test_add_list_bad_replicas(self): """ REPLICA (CLIENT): Add bad replicas""" tmp_scope = 'mock' nbfiles = 5 # Adding replicas to deterministic RSE files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] rse_info = rsemgr.get_rse_info('MOCK') rse_id1 = rse_info['id'] self.replica_client.add_replicas(rse='MOCK', files=files) # Listing replicas on deterministic RSE replicas, list_rep = [], [] for replica in self.replica_client.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK']) list_rep.append(replica) r = self.replica_client.declare_bad_file_replicas( replicas, 'This is a good reason') assert_equal(r, {}) bad_replicas = list_bad_replicas() nbbadrep = 0 for rep in list_rep: for badrep in bad_replicas: if badrep['rse_id'] == rse_id1: if badrep['scope'] == rep['scope'] and badrep[ 'name'] == rep['name']: nbbadrep += 1 assert_equal(len(replicas), nbbadrep) # Run necromancer once necromancer_run(threads=1, bulk=10000, once=True) # Try to attach a lost file tmp_dsn = 'dataset_%s' % generate_uuid() self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn) with assert_raises(UnsupportedOperation): self.did_client.add_files_to_dataset(tmp_scope, name=tmp_dsn, files=files, rse='MOCK') # Adding replicas to non-deterministic RSE files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': { 'events': 10 } } for _ in range(nbfiles)] rse_info = rsemgr.get_rse_info('MOCK2') rse_id2 = rse_info['id'] self.replica_client.add_replicas(rse='MOCK2', files=files) # Listing replicas on non-deterministic RSE replicas, list_rep = [], [] for replica in self.replica_client.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK2']) list_rep.append(replica) print(replicas, list_rep) r = self.replica_client.declare_bad_file_replicas( replicas, 'This is a good reason') print(r) assert_equal(r, {}) bad_replicas = list_bad_replicas() nbbadrep = 0 for rep in list_rep: for badrep in bad_replicas: if badrep['rse_id'] == rse_id2: if badrep['scope'] == rep['scope'] and badrep[ 'name'] == rep['name']: nbbadrep += 1 assert_equal(len(replicas), nbbadrep) # Now adding non-existing bad replicas files = [ 'srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), ] r = self.replica_client.declare_bad_file_replicas( files, 'This is a good reason') output = ['%s Unknown replica' % rep for rep in files] assert_equal(r, {'MOCK2': output}) def test_add_suspicious_replicas(self): """ REPLICA (CLIENT): Add suspicious replicas""" tmp_scope = 'mock' nbfiles = 5 # Adding replicas to deterministic RSE files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files) # Listing replicas on deterministic RSE replicas = [] list_rep = [] for replica in self.replica_client.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK']) list_rep.append(replica) r = self.replica_client.declare_suspicious_file_replicas( replicas, 'This is a good reason') assert_equal(r, {}) # Adding replicas to non-deterministic RSE files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK2', files=files) # Listing replicas on non-deterministic RSE replicas = [] list_rep = [] for replica in self.replica_client.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK2']) list_rep.append(replica) r = self.replica_client.declare_suspicious_file_replicas( replicas, 'This is a good reason') assert_equal(r, {}) # Now adding non-existing bad replicas files = [ 'srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), ] r = self.replica_client.declare_suspicious_file_replicas( files, 'This is a good reason') output = ['%s Unknown replica' % rep for rep in files] assert_equal(r, {'MOCK2': output}) def test_bad_replica_methods_for_UI(self): """ REPLICA (REST): Test the listing of bad and suspicious replicas """ mw = [] headers1 = { 'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******' } result = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(result.status, 200) token = str(result.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Auth-Token': str(token)} data = dumps({}) result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(result.status, 200) tot_files = [] for line in result.body.split('\n'): if line != '': tot_files.append(dumps(line)) nb_tot_files = len(tot_files) data = dumps({'state': 'B'}) result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(result.status, 200) tot_bad_files = [] for line in result.body.split('\n'): if line != '': tot_bad_files.append(dumps(line)) nb_tot_bad_files1 = len(tot_bad_files) data = dumps({'state': 'S', 'list_pfns': 'True'}) result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(result.status, 200) tot_suspicious_files = [] for line in result.body.split('\n'): if line != '': tot_suspicious_files.append(dumps(line)) nb_tot_suspicious_files = len(tot_suspicious_files) data = dumps({'state': 'T', 'list_pfns': 'True'}) result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(result.status, 200) tot_temporary_unavailable_files = [] for line in result.body.split('\n'): if line != '': tot_temporary_unavailable_files.append(dumps(line)) nb_tot_temporary_unavailable_files = len( tot_temporary_unavailable_files) assert_equal( nb_tot_files, nb_tot_bad_files1 + nb_tot_suspicious_files + nb_tot_temporary_unavailable_files) tomorrow = datetime.utcnow() + timedelta(days=1) data = dumps({'state': 'B', 'younger_than': tomorrow.isoformat()}) result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(result.status, 200) tot_bad_files = [] for line in result.body.split('\n'): if line != '': tot_bad_files.append(dumps(line)) nb_tot_bad_files = len(tot_bad_files) assert_equal(nb_tot_bad_files, 0) data = dumps({}) result = TestApp(rep_app.wsgifunc(*mw)).get('/bad/summary', headers=headers2, params=data, expect_errors=True) assert_equal(result.status, 200) nb_tot_bad_files2 = 0 for line in result.body.split('\n'): if line != '': line = loads(line) nb_tot_bad_files2 += int(line.get('BAD', 0)) assert_equal(nb_tot_bad_files1, nb_tot_bad_files2) def test_add_list_replicas(self): """ REPLICA (CLIENT): Add, change state and list file replicas """ tmp_scope = 'mock' nbfiles = 5 files1 = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files1) files2 = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK3', files=files2) replicas = [ r for r in self.replica_client.list_replicas( dids=[{ 'scope': i['scope'], 'name': i['name'] } for i in files1]) ] assert_equal(len(replicas), len(files1)) replicas = [ r for r in self.replica_client.list_replicas( dids=[{ 'scope': i['scope'], 'name': i['name'] } for i in files2], schemes=['file']) ] assert_equal(len(replicas), 5) replicas = [ r for r in self.replica_client.list_replicas( dids=[{ 'scope': i['scope'], 'name': i['name'] } for i in files2], schemes=['srm']) ] assert_equal(len(replicas), 5) files3 = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'state': 'U', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK3', files=files3) replicas = [ r for r in self.replica_client.list_replicas( dids=[{ 'scope': i['scope'], 'name': i['name'] } for i in files3], schemes=['file']) ] for i in range(nbfiles): assert_equal(replicas[i]['rses'], {}) files4 = [] for file in files3: file['state'] = 'A' files4.append(file) self.replica_client.update_replicas_states('MOCK3', files=files4) replicas = [ r for r in self.replica_client.list_replicas( dids=[{ 'scope': i['scope'], 'name': i['name'] } for i in files3], schemes=['file'], unavailable=True) ] assert_equal(len(replicas), 5) for i in range(nbfiles): assert_in('MOCK3', replicas[i]['rses']) def test_delete_replicas(self): """ REPLICA (CLIENT): Add and delete file replicas """ tmp_scope = 'mock' nbfiles = 5 files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files) with assert_raises(AccessDenied): self.replica_client.delete_replicas(rse='MOCK', files=files) # replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files])] # assert_equal(len(replicas), 0) def test_add_temporary_unavailable_pfns(self): """ REPLICA (CLIENT): Add temporary unavailable PFNs""" tmp_scope = 'mock' nbfiles = 5 # Adding replicas to deterministic RSE files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files) # Listing replicas on deterministic RSE list_rep = [] for replica in self.replica_client.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in files], schemes=['srm'], unavailable=True): pfn = replica['pfns'].keys()[0] list_rep.append(pfn) # Submit bad PFNs now = datetime.utcnow() reason_str = generate_uuid() self.replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='TEMPORARY_UNAVAILABLE', expires_at=now.isoformat()) result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None) bad_pfns = {} for res in result: bad_pfns[res['pfn']] = (res['state'], res['reason'], res['expires_at']) for pfn in list_rep: pfn = str(clean_surls([pfn])[0]) assert_in(pfn, bad_pfns) assert_equal(str(bad_pfns[pfn][0]), 'TEMPORARY_UNAVAILABLE') assert_equal(bad_pfns[pfn][1], reason_str) # Submit with wrong state with assert_raises(RucioException): self.replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='BADSTATE', expires_at=now.isoformat()) # Run minos once minos_run(threads=1, bulk=10000, once=True) result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None) pfns = [res['pfn'] for res in result] res_pfns = [] for replica in list_rep: if replica in pfns: res_pfns.append(replica) assert_equal(res_pfns, []) # Check the state in the replica table for did in files: rep = get_replicas_state(scope=did['scope'], name=did['name']) assert_equal(str(rep.keys()[0]), 'TEMPORARY_UNAVAILABLE') rep = [] for did in files: did['state'] = ReplicaState.from_sym('TEMPORARY_UNAVAILABLE') rep.append(did) # Run the minos expiration minos_temp_run(threads=1, once=True) # Check the state in the replica table for did in files: rep = get_replicas_state(scope=did['scope'], name=did['name']) assert_equal(str(rep.keys()[0]), 'AVAILABLE') def test_set_tombstone(self): """ REPLICA (CLIENT): set tombstone on replica """ # Set tombstone on one replica rse = 'MOCK4' scope = 'mock' user = '******' name = generate_uuid() add_replica(rse, scope, name, 4, user) assert_equal(get_replica(rse, scope, name)['tombstone'], None) self.replica_client.set_tombstone([{ 'rse': rse, 'scope': scope, 'name': name }]) assert_equal(get_replica(rse, scope, name)['tombstone'], OBSOLETE) # Set tombstone on locked replica name = generate_uuid() add_replica(rse, scope, name, 4, user) RuleClient().add_replication_rule([{ 'name': name, 'scope': scope }], 1, rse, locked=True) with assert_raises(ReplicaIsLocked): self.replica_client.set_tombstone([{ 'rse': rse, 'scope': scope, 'name': name }]) # Set tombstone on not found replica name = generate_uuid() with assert_raises(ReplicaNotFound): self.replica_client.set_tombstone([{ 'rse': rse, 'scope': scope, 'name': name }])
class TestReplicaRecoverer(unittest.TestCase): def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} else: self.vo = {} self.replica_client = ReplicaClient() # Using two test RSEs self.rse4suspicious = 'MOCK_SUSPICIOUS' self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo) self.rse4recovery = 'MOCK_RECOVERY' self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo) self.scope = 'mock' self.internal_scope = InternalScope(self.scope, **self.vo) # For testing, we create 3 files and upload them to Rucio to two test RSEs. self.tmp_file1 = file_generator() self.tmp_file2 = file_generator() self.tmp_file3 = file_generator() self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE} for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3]] for rse in [self.rse4suspicious, self.rse4recovery]: cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3) exitcode, out, err = execute(cmd) # checking if Rucio upload went OK assert exitcode == 0 # removing physical files from /tmp location - keeping only their DB info remove(self.tmp_file1) remove(self.tmp_file2) remove(self.tmp_file3) # Gather replica info replicalist = list_replicas(dids=self.listdids) # Changing the replica statuses as follows: # -------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) # -------------------------------------------------------------------------------------------- for replica in replicalist: suspicious_pfns = replica['rses'][self.rse4suspicious_id] for i in range(3): print("Declaring suspicious file replica: " + suspicious_pfns[0]) self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.') sleep(1) if replica['name'] == path.basename(self.tmp_file2): print("Declaring bad file replica: " + suspicious_pfns[0]) self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason') if replica['name'] == path.basename(self.tmp_file3): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE) # Gather replica info after setting initial replica statuses replicalist = list_replicas(dids=self.listdids) # Checking if the status changes were effective for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if only self.tmp_file2 is declared as 'BAD' self.from_date = datetime.now() - timedelta(days=1) bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist # On purpose not checking for status to be declared 'SUSPICIOUS' on MOCK_SUSPICIOUS. # The only existing function (to date) gathering info about 'SUSPICIOUS' replicas # is used (currently only) from the tested replica_recoverer itself. def test_replica_recoverer(self): """ REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs. setUp function (above) is supposed to run first (nose does this automatically): - uploads 3 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS') - prepares their statuses to be as follows: # -------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) # -------------------------------------------------------------------------------------------- Runs the Test: - running suspicious_replica_recoverer Concluding: - checks that the only change made is that tmp_file1 was declared as 'BAD on 'MOCK_SUSPICIOUS' """ # Run replica recoverer once try: run(once=True, younger_than=1, nattempts=2, rse_expression='MOCK_SUSPICIOUS') except KeyboardInterrupt: stop() # Checking the outcome: # we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS # -------------------------------------------------------------------------------------------- # tmp_file1 available suspicious + bad (unavailable) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) # -------------------------------------------------------------------------------------------- # Gather replica info after replica_recoverer has run. replicalist = list_replicas(dids=self.listdids) for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if replicas declared as 'BAD' bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
class TestReplicaClients: def setup(self): self.replica_client = ReplicaClient() self.did_client = DIDClient() def test_add_list_bad_replicas(self): """ REPLICA (CLIENT): Add bad replicas""" tmp_scope = 'mock' nbfiles = 5 # Adding replicas to deterministic RSE files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)] rse_info = rsemgr.get_rse_info('MOCK') rse_id1 = rse_info['id'] self.replica_client.add_replicas(rse='MOCK', files=files) # Listing replicas on deterministic RSE replicas, list_rep = [], [] for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK']) list_rep.append(replica) r = self.replica_client.declare_bad_file_replicas(replicas, 'This is a good reason') assert_equal(r, {}) bad_replicas = list_bad_replicas() nbbadrep = 0 for rep in list_rep: for badrep in bad_replicas: if badrep['rse_id'] == rse_id1: if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']: nbbadrep += 1 assert_equal(len(replicas), nbbadrep) # Run necromancer once run(threads=1, bulk=10000, once=True) # Try to attach a lost file tmp_dsn = 'dataset_%s' % generate_uuid() self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn) with assert_raises(UnsupportedOperation): self.did_client.add_files_to_dataset(tmp_scope, name=tmp_dsn, files=files, rse='MOCK') # Adding replicas to non-deterministic RSE files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': {'events': 10}} for i in range(nbfiles)] rse_info = rsemgr.get_rse_info('MOCK2') rse_id2 = rse_info['id'] self.replica_client.add_replicas(rse='MOCK2', files=files) # Listing replicas on non-deterministic RSE replicas, list_rep = [], [] for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK2']) list_rep.append(replica) print(replicas, list_rep) r = self.replica_client.declare_bad_file_replicas(replicas, 'This is a good reason') print(r) assert_equal(r, {}) bad_replicas = list_bad_replicas() nbbadrep = 0 for rep in list_rep: for badrep in bad_replicas: if badrep['rse_id'] == rse_id2: if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']: nbbadrep += 1 assert_equal(len(replicas), nbbadrep) # Now adding non-existing bad replicas files = ['srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), ] r = self.replica_client.declare_bad_file_replicas(files, 'This is a good reason') output = ['%s Unknown replica' % rep for rep in files] assert_equal(r, {'MOCK2': output}) def test_add_suspicious_replicas(self): """ REPLICA (CLIENT): Add suspicious replicas""" tmp_scope = 'mock' nbfiles = 5 # Adding replicas to deterministic RSE files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files) # Listing replicas on deterministic RSE replicas = [] list_rep = [] for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK']) list_rep.append(replica) r = self.replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason') assert_equal(r, {}) # Adding replicas to non-deterministic RSE files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'pfn': 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), 'meta': {'events': 10}} for i in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK2', files=files) # Listing replicas on non-deterministic RSE replicas = [] list_rep = [] for replica in self.replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], unavailable=True): replicas.extend(replica['rses']['MOCK2']) list_rep.append(replica) r = self.replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason') assert_equal(r, {}) # Now adding non-existing bad replicas files = ['srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' % (tmp_scope, generate_uuid()), ] r = self.replica_client.declare_suspicious_file_replicas(files, 'This is a good reason') output = ['%s Unknown replica' % rep for rep in files] assert_equal(r, {'MOCK2': output}) def test_bad_replica_methods_for_UI(self): """ REPLICA (REST): Test the listing of bad and suspicious replicas """ mw = [] headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(r1.status, 200) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Auth-Token': str(token)} data = dumps({}) r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(r2.status, 200) tot_files = [] for line in r2.body.split('\n'): if line != '': tot_files.append(dumps(line)) nb_tot_files = len(tot_files) data = dumps({'state': 'B'}) r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(r2.status, 200) tot_bad_files = [] for line in r2.body.split('\n'): if line != '': tot_bad_files.append(dumps(line)) nb_tot_bad_files1 = len(tot_bad_files) data = dumps({'state': 'S', 'list_pfns': 'True'}) r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(r2.status, 200) tot_suspicious_files = [] for line in r2.body.split('\n'): if line != '': tot_suspicious_files.append(dumps(line)) nb_tot_suspicious_files = len(tot_suspicious_files) assert_equal(nb_tot_files, nb_tot_bad_files1 + nb_tot_suspicious_files) tomorrow = datetime.utcnow() + timedelta(days=1) data = dumps({'state': 'B', 'younger_than': tomorrow.isoformat()}) r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/states', headers=headers2, params=data, expect_errors=True) assert_equal(r2.status, 200) tot_bad_files = [] for line in r2.body.split('\n'): if line != '': tot_bad_files.append(dumps(line)) nb_tot_bad_files = len(tot_bad_files) assert_equal(nb_tot_bad_files, 0) data = dumps({}) r2 = TestApp(rep_app.wsgifunc(*mw)).get('/bad/summary', headers=headers2, params=data, expect_errors=True) assert_equal(r2.status, 200) nb_tot_bad_files2 = 0 for line in r2.body.split('\n'): if line != '': line = loads(line) nb_tot_bad_files2 += int(line['BAD']) assert_equal(nb_tot_bad_files1, nb_tot_bad_files2) def test_add_list_replicas(self): """ REPLICA (CLIENT): Add, change state and list file replicas """ tmp_scope = 'mock' nbfiles = 5 files1 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files1) files2 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK3', files=files2) replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files1])] assert_equal(len(replicas), len(files1)) replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files2], schemes=['file'])] assert_equal(len(replicas), 5) replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files2], schemes=['srm'])] assert_equal(len(replicas), 5) files3 = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'state': 'U', 'meta': {'events': 10}} for i in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK3', files=files3) replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files3], schemes=['file'])] for i in range(nbfiles): assert_equal(replicas[i]['rses'], {}) files4 = [] for file in files3: file['state'] = 'A' files4.append(file) self.replica_client.update_replicas_states('MOCK3', files=files4) replicas = [r for r in self.replica_client.list_replicas(dids=[{'scope': i['scope'], 'name': i['name']} for i in files3], schemes=['file'], unavailable=True)] assert_equal(len(replicas), 5) for i in range(nbfiles): assert_in('MOCK3', replicas[i]['rses']) def test_delete_replicas(self): """ REPLICA (CLIENT): Add and delete file replicas """ tmp_scope = 'mock' nbfiles = 5 files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for i in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files) with assert_raises(AccessDenied): self.replica_client.delete_replicas(rse='MOCK', files=files)