def test_get_bad_replicas_backlog(rse_factory, mock_scope, root_account, file_config_mock):
""" REPLICA (CORE): Check the behaviour of the necromancer in case of backlog on an RSE"""
# Run necromancer once
necromancer_run(threads=1, bulk=10000, once=True)
nbfiles1 = 100
nbfiles2 = 20
# Adding replicas to deterministic RSE
rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
_, rse2_id = rse_factory.make_srm_rse(deterministic=True)
# Create bad replicas on rse1
files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles1)]
add_replicas(rse_id=rse1_id, files=files, account=root_account, ignore_availability=True)
replicas = []
list_rep = []
for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
replicas.extend(replica['rses'][rse1_id])
list_rep.append({'scope': replica['scope'], 'name': replica['name'], 'rse': rse1, 'rse_id': rse1_id})
res = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
assert res == {}
result = get_bad_replicas_backlog(force_refresh=True)
assert rse1_id in result
assert result[rse1_id] == nbfiles1
# Create more bad replicas on rse2
files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles2)]
add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True)
repl = []
for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
repl.extend(replica['rses'][rse2_id])
res = declare_bad_file_replicas(repl, 'This is a good reason', root_account)
assert res == {}
# List bad replicas on rse1
bad_replicas = list_bad_replicas(rses=[{'id': rse1_id}])
assert len(bad_replicas) == nbfiles1
for rep in bad_replicas:
assert rep in list_rep
# Run necromancer once, all the files on RSE2 should be gone, 80 files should stay on RSE1
get_bad_replicas_backlog(force_refresh=True)
necromancer_run(threads=1, bulk=20, once=True)
bad_replicas = list_bad_replicas(rses=[{'id': rse1_id}, {'id': rse2_id}])
assert len(bad_replicas) == 80
for rep in bad_replicas:
assert rep['rse_id'] == rse1_id
def test_add_list_bad_replicas(self):
""" REPLICA (CLIENT): Add bad replicas"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK')
rse_id1 = rse_info['id']
self.replica_client.add_replicas(rse='MOCK', files=files)
# Listing replicas on deterministic RSE
replicas, list_rep = [], []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
replicas.extend(replica['rses']['MOCK'])
list_rep.append(replica)
r = self.replica_client.declare_bad_file_replicas(
replicas, 'This is a good reason')
assert_equal(r, {})
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse_id1:
if badrep['scope'] == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert_equal(len(replicas), nbbadrep)
# Run necromancer once
necromancer_run(threads=1, bulk=10000, once=True)
# Try to attach a lost file
tmp_dsn = 'dataset_%s' % generate_uuid()
self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
with assert_raises(UnsupportedOperation):
self.did_client.add_files_to_dataset(tmp_scope,
name=tmp_dsn,
files=files,
rse='MOCK')
# Adding replicas to non-deterministic RSE
files = [{
'scope':
tmp_scope,
'name':
'file_%s' % generate_uuid(),
'bytes':
1,
'adler32':
'0cc737eb',
'pfn':
'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s'
% (tmp_scope, generate_uuid()),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
rse_info = rsemgr.get_rse_info('MOCK2')
rse_id2 = rse_info['id']
self.replica_client.add_replicas(rse='MOCK2', files=files)
# Listing replicas on non-deterministic RSE
replicas, list_rep = [], []
for replica in self.replica_client.list_replicas(dids=[{
'scope':
f['scope'],
'name':
f['name']
} for f in files],
schemes=['srm'],
unavailable=True):
replicas.extend(replica['rses']['MOCK2'])
list_rep.append(replica)
print(replicas, list_rep)
r = self.replica_client.declare_bad_file_replicas(
replicas, 'This is a good reason')
print(r)
assert_equal(r, {})
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse_id2:
if badrep['scope'] == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert_equal(len(replicas), nbbadrep)
# Now adding non-existing bad replicas
files = [
'srm://mock2.com/rucio/tmpdisk/rucio_tests/%s/%s' %
(tmp_scope, generate_uuid()),
]
r = self.replica_client.declare_bad_file_replicas(
files, 'This is a good reason')
output = ['%s Unknown replica' % rep for rep in files]
assert_equal(r, {'MOCK2': output})
def test_client_add_list_bad_replicas(rse_factory, replica_client, did_client):
""" REPLICA (CLIENT): Add bad replicas"""
tmp_scope = 'mock'
nbfiles = 5
# Adding replicas to deterministic RSE
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
replica_client.add_replicas(rse=rse1, files=files)
# Listing replicas on deterministic RSE
replicas, list_rep = [], []
for replica in replica_client.list_replicas(dids=[{
'scope': f['scope'],
'name': f['name']
} for f in files],
schemes=['srm'],
all_states=True):
replicas.extend(replica['rses'][rse1])
list_rep.append(replica)
r = replica_client.declare_bad_file_replicas(replicas,
'This is a good reason')
assert r == {}
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse1_id:
if badrep['scope'].external == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert len(replicas) == nbbadrep
# Run necromancer once
necromancer_run(threads=1, bulk=10000, once=True)
# Try to attach a lost file
tmp_dsn = 'dataset_%s' % generate_uuid()
did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
with pytest.raises(UnsupportedOperation):
did_client.add_files_to_dataset(tmp_scope,
name=tmp_dsn,
files=files,
rse=rse1)
# Adding replicas to non-deterministic RSE
rse2, rse2_id = rse_factory.make_srm_rse(deterministic=False)
files = [{
'scope':
tmp_scope,
'name':
'file_%s' % generate_uuid(),
'bytes':
1,
'adler32':
'0cc737eb',
'pfn':
'srm://%s.cern.ch/srm/managerv2?SFN=/test_%s/%s/%s' %
(rse2_id, rse2_id, tmp_scope, generate_uuid()),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
replica_client.add_replicas(rse=rse2, files=files)
# Listing replicas on non-deterministic RSE
replicas, list_rep = [], []
for replica in replica_client.list_replicas(dids=[{
'scope': f['scope'],
'name': f['name']
} for f in files],
schemes=['srm'],
all_states=True):
replicas.extend(replica['rses'][rse2])
list_rep.append(replica)
r = replica_client.declare_bad_file_replicas(replicas,
'This is a good reason')
assert r == {}
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse2_id:
if badrep['scope'].external == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert len(replicas) == nbbadrep
# Now adding non-existing bad replicas
files = [
'srm://%s.cern.ch/test_%s/%s/%s' %
(rse2_id, rse2_id, tmp_scope, generate_uuid()),
]
r = replica_client.declare_bad_file_replicas(files,
'This is a good reason')
output = ['%s Unknown replica' % rep for rep in files]
assert r == {rse2: output}
# Now test adding bad_replicas with a list of replicas instead of PFNs
# Adding replicas to deterministic RSE
rse3, rse3_id = rse_factory.make_srm_rse(deterministic=True)
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
replica_client.add_replicas(rse=rse3, files=files)
list_rep = [{
'scope': file_['scope'],
'name': file_['name'],
'rse': rse3
} for file_ in files]
# Listing replicas on deterministic RSE
replicas = []
for replica in replica_client.list_replicas(dids=[{
'scope': f['scope'],
'name': f['name']
} for f in files],
schemes=['srm'],
all_states=True):
replicas.extend(replica['rses'][rse3])
r = replica_client.declare_bad_file_replicas(list_rep,
'This is a good reason')
assert r == {}
bad_replicas = list_bad_replicas()
nbbadrep = 0
for rep in list_rep:
for badrep in bad_replicas:
if badrep['rse_id'] == rse3_id:
if badrep['scope'].external == rep['scope'] and badrep[
'name'] == rep['name']:
nbbadrep += 1
assert len(replicas) == nbbadrep
# InvalidType is raised if list_rep contains a mixture of replicas and PFNs
list_rep.extend([
'srm://%s.cern.ch/test_%s/%s/%s' %
(rse2_id, rse2_id, tmp_scope, generate_uuid()),
])
with pytest.raises(InvalidType):
r = replica_client.declare_bad_file_replicas(list_rep,
'This is a good reason')