def list_bad_replicas_status(state=BadFilesStatus.BAD,
rse=None,
younger_than=None,
older_than=None,
limit=None,
list_pfns=False):
"""
List the bad file replicas history states. Method used by the rucio-ui.
:param state: The state of the file (SUSPICIOUS or BAD).
:param rse: The RSE name.
:param younger_than: datetime object to select bad replicas younger than this date.
:param older_than: datetime object to select bad replicas older than this date.
:param limit: The maximum number of replicas returned.
"""
rse_id = None
if rse is not None:
rse_id = get_rse_id(rse=rse)
replicas = replica.list_bad_replicas_status(state=state,
rse_id=rse_id,
younger_than=younger_than,
older_than=older_than,
limit=limit,
list_pfns=list_pfns)
return [api_update_return_dict(r) for r in replicas]
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.replica_client = ReplicaClient()
# Using two test RSEs
self.rse4suspicious = 'MOCK_SUSPICIOUS'
self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo)
self.rse4recovery = 'MOCK_RECOVERY'
self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo)
self.scope = 'mock'
self.internal_scope = InternalScope(self.scope, **self.vo)
# For testing, we create 3 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.tmp_file4 = file_generator()
self.tmp_file5 = file_generator()
self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE}
for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5]]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5)
exitcode, out, err = execute(cmd)
# checking if Rucio upload went OK
assert exitcode == 0
# Set fictional datatypes
set_metadata(self.internal_scope, path.basename(self.tmp_file4), 'datatype', 'testtypedeclarebad')
set_metadata(self.internal_scope, path.basename(self.tmp_file5), 'datatype', 'testtypenopolicy')
# Allow for the RSEs to be affected by the suspicious file recovery daemon
add_rse_attribute(self.rse4suspicious_id, "enable_suspicious_file_recovery", True)
add_rse_attribute(self.rse4recovery_id, "enable_suspicious_file_recovery", True)
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
remove(self.tmp_file4)
remove(self.tmp_file5)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious (available) testtypedeclarebad
# tmp_file5 unavailable suspicious (available) testtypenopolicy
# ----------------------------------------------------------------------------------------------------------------------------------
for replica in replicalist:
suspicious_pfns = replica['rses'][self.rse4suspicious_id]
for i in range(3):
print("Declaring suspicious file replica: " + suspicious_pfns[0])
self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " + suspicious_pfns[0])
self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file4):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file4), ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file5):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file5), ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file4):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file5):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_replica_recoverer(self):
""" REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs.
setUp function (above) is supposed to run first
(nose does this automatically):
- uploads 6 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS')
- prepares their statuses to be as follows:
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious (available) testtypedeclare_bad
# tmp_file5 unavailable suspicious (available) testtypenopolicy
# ----------------------------------------------------------------------------------------------------------------------------------
- Explaination: Suspicious replicas that are the last remaining copy (unavailable on MOCK_RECOVERY) are handeled differently depending
by their metadata "datatype". RAW files have the poilcy to be ignored. testtype_declare_bad files are of a fictional
type that has the policy of being declared bad. testtype_nopolicy files are of a fictional type that doesn't have a
policy specified, meaning they should be ignored by default.
Runs the Test:
- running suspicious_replica_recoverer
Concluding:
- checks that tmp_file1 and tmp_file4 were declared as 'BAD' on 'MOCK_SUSPICIOUS'
"""
# Run replica recoverer once
try:
run(once=True, younger_than=1, nattempts=2, limit_suspicious_files_on_rse=5)
except KeyboardInterrupt:
stop()
# Checking the outcome:
# we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious + bad (unavailable)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious + bad (unvailable) test_type_declare_bad
# tmp_file5 unavailable suspicious (available) test_type_ignore
# ----------------------------------------------------------------------------------------------------------------------------------
# Gather replica info after replica_recoverer has run.
replicalist = list_replicas(dids=self.listdids)
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file4):
# The key 'state' only exists if the replica is available on at least one RSE. It shouldn't exist for tmp_file4.
assert ('states' in replica) is False
if replica['name'] == path.basename(self.tmp_file5):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if replicas declared as 'BAD'
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_replica_recoverer(self):
""" REPLICA RECOVERER: Testing declaration of suspicious replicas as bad if they are found available on other RSEs.
setUp function (above) is supposed to run first
(nose does this automatically):
- uploads 3 test files to two test RSEs ('MOCK_RECOVERY', 'MOCK_SUSPICIOUS')
- prepares their statuses to be as follows:
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
Runs the Test:
- running suspicious_replica_recoverer
Concluding:
- checks that the only change made is that tmp_file1 was declared as 'BAD on 'MOCK_SUSPICIOUS'
"""
# Run replica recoverer once
try:
run(once=True, younger_than=1, nattempts=2, rse_expression='MOCK_SUSPICIOUS')
except KeyboardInterrupt:
stop()
# Checking the outcome:
# we expect to see only one change, i.e. tmp_file1 declared as bad on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious + bad (unavailable)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
# Gather replica info after replica_recoverer has run.
replicalist = list_replicas(dids=self.listdids)
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1) or replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if replicas declared as 'BAD'
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def test_add_and_delete_bad_replicas(rse_factory, mock_scope, root_account,
did_client, vo):
""" REPLICA (CORE): Add bad replicas and delete them"""
# Adding replicas to deterministic RSE
nbfiles = 5
rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True, vo=vo)
files = [{
'scope': mock_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
client_files = [{
'scope': file_['scope'].external,
'name': file_['name']
} for file_ in files]
add_replicas(rse_id=rse1_id,
files=files,
account=root_account,
ignore_availability=True)
tmp_dsn = 'dataset_%s' % generate_uuid()
did_client.add_dataset(scope=mock_scope.external, name=tmp_dsn)
did_client.add_files_to_dataset(mock_scope.external,
name=tmp_dsn,
files=client_files,
rse=rse1)
# Declare replica bad
replicas = []
for replica in list_replicas(dids=[{
'scope': f['scope'],
'name': f['name'],
'type': DIDType.FILE
} for f in files],
schemes=['srm']):
replicas.extend(replica['rses'][rse1_id])
r = declare_bad_file_replicas(replicas, 'This is a good reason',
root_account)
assert r == {}
# Check state of bad replicas
list_bad_rep = [{
'scope': rep['scope'].external,
'name': rep['name']
} for rep in list_bad_replicas_status(
state=BadFilesStatus.BAD, rse_id=rse1_id, vo=vo)]
for rep in client_files:
assert rep in list_bad_rep
assert [
rep for rep in list_bad_replicas_status(
state=BadFilesStatus.DELETED, rse_id=rse1_id, vo=vo)
] == []
# Now delete the dataset
delete_dids([{
'scope': mock_scope,
'name': tmp_dsn,
'did_type': DIDType.DATASET,
'purge_replicas': True
}],
account=root_account)
assert [
rep for rep in list_bad_replicas_status(
state=BadFilesStatus.BAD, rse_id=rse1_id, vo=vo)
] == []
list_deleted_rep = [{
'scope': rep['scope'].external,
'name': rep['name']
} for rep in list_bad_replicas_status(
state=BadFilesStatus.DELETED, rse_id=rse1_id, vo=vo)]
for rep in client_files:
assert rep in list_deleted_rep
def setUp(self):
self.replica_client = ReplicaClient()
# Using two test RSEs
self.rse4suspicious = 'MOCK_SUSPICIOUS'
self.rse4recovery = 'MOCK_RECOVERY'
self.scope = 'mock'
# For testing, we create 3 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.listdids = [{
'scope': self.scope,
'name': path.basename(self.tmp_file1),
'type': DIDType.FILE
}, {
'scope': self.scope,
'name': path.basename(self.tmp_file2),
'type': DIDType.FILE
}, {
'scope': self.scope,
'name': path.basename(self.tmp_file3),
'type': DIDType.FILE
}]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4}'.format(
rse, self.scope, self.tmp_file1, self.tmp_file2,
self.tmp_file3)
exitcode, out, err = execute(cmd)
# checking if Rucio upload went OK
assert_true(exitcode == 0)
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# --------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS
# --------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available)
# --------------------------------------------------------------------------------------------
for replica in replicalist:
for i in range(3):
print("Declaring suspicious file replica: " +
replica['rses'][self.rse4suspicious][0])
self.replica_client.declare_suspicious_file_replicas([
replica['rses'][self.rse4suspicious][0],
], 'This is a good reason.')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " +
replica['rses'][self.rse4suspicious][0])
self.replica_client.declare_bad_file_replicas([
replica['rses'][self.rse4suspicious][0],
], 'This is a good reason')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " +
replica['rses'][self.rse4recovery][0])
update_replica_state(self.rse4recovery, self.scope,
path.basename(self.tmp_file3),
ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert_true(
replica['states'][self.rse4suspicious] == 'AVAILABLE')
assert_true(
replica['states'][self.rse4recovery] == 'AVAILABLE')
if replica['name'] == path.basename(self.tmp_file2):
assert_true(
(self.rse4suspicious in replica['states']) is False)
assert_true(
replica['states'][self.rse4recovery] == 'AVAILABLE')
if replica['name'] == path.basename(self.tmp_file3):
assert_true(
replica['states'][self.rse4suspicious] == 'AVAILABLE')
assert_true((self.rse4recovery in replica['states']) is False)
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(
rse=self.rse4suspicious, younger_than=self.from_date)
bad_checklist = [(badf['name'], badf['rse'], badf['state'])
for badf in bad_replicas_list]
assert_true((path.basename(self.tmp_file2), self.rse4suspicious,
BadFilesStatus.BAD) in bad_checklist)
assert_true((path.basename(self.tmp_file1), self.rse4suspicious,
BadFilesStatus.BAD) not in bad_checklist)
assert_true((path.basename(self.tmp_file3), self.rse4suspicious,
BadFilesStatus.BAD) not in bad_checklist)
bad_replicas_list = list_bad_replicas_status(
rse=self.rse4recovery, younger_than=self.from_date)
bad_checklist = [(badf['name'], badf['rse'], badf['state'])
for badf in bad_replicas_list]
assert_true((path.basename(self.tmp_file1), self.rse4recovery,
BadFilesStatus.BAD) not in bad_checklist)
assert_true((path.basename(self.tmp_file2), self.rse4recovery,
BadFilesStatus.BAD) not in bad_checklist)
assert_true((path.basename(self.tmp_file3), self.rse4recovery,
BadFilesStatus.BAD) not in bad_checklist)
