def test_replica_recoverer(self):
""" REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs.
setUp function (above) is supposed to run first
(nose does this automatically):
- uploads 6 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS')
- prepares their statuses to be as follows:
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious (available) testtypedeclare_bad
# tmp_file5 unavailable suspicious (available) testtypenopolicy
# ----------------------------------------------------------------------------------------------------------------------------------
- Explaination: Suspicious replicas that are the last remaining copy (unavailable on MOCK_RECOVERY) are handeled differently depending
by their metadata "datatype". RAW files have the poilcy to be ignored. testtype_declare_bad files are of a fictional
type that has the policy of being declared bad. testtype_nopolicy files are of a fictional type that doesn't have a
policy specified, meaning they should be ignored by default.
Runs the Test:
- running suspicious_replica_recoverer
Concluding:
- checks that tmp_file1 and tmp_file4 were declared as 'BAD' on 'MOCK_SUSPICIOUS'
"""
# Run replica recoverer once
try:
run(once=True, younger_than=1, nattempts=2, limit_suspicious_files_on_rse=5)
except KeyboardInterrupt:
stop()
# Checking the outcome:
# we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious + bad (unavailable)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious + bad (unvailable) test_type_declare_bad
# tmp_file5 unavailable suspicious (available) test_type_ignore
# ----------------------------------------------------------------------------------------------------------------------------------
# Gather replica info after replica_recoverer has run.
replicalist = list_replicas(dids=self.listdids)
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file4):
# The key 'state' only exists if the replica is available on at least one RSE. It shouldn't exist for tmp_file4.
assert ('states' in replica) is False
if replica['name'] == path.basename(self.tmp_file5):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if replicas declared as 'BAD'
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_replica_recoverer(self):
""" REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs.
setUp function (above) is supposed to run first
(nose does this automatically):
- uploads 3 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS')
- prepares their statuses to be as follows:
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
Runs the Test:
- running suspicious_replica_recoverer
Concluding:
- checks that the only change made is that tmp_file1 was declared as 'BAD on 'MOCK_SUSPICIOUS'
"""
# Run replica recoverer once
try:
run(once=True, younger_than=1, nattempts=2, rse_expression='MOCK_SUSPICIOUS')
except KeyboardInterrupt:
stop()
# Checking the outcome:
# we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious + bad (unavailable)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
# Gather replica info after replica_recoverer has run.
replicalist = list_replicas(dids=self.listdids)
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if replicas declared as 'BAD'
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist