def test_client_add_temporary_unavailable_pfns(rse_factory, mock_scope, replica_client): """ REPLICA (CLIENT): Add temporary unavailable PFNs""" rse, rse_id = rse_factory.make_posix_rse() nbfiles = 5 # Adding replicas to deterministic RSE files = [{'scope': mock_scope.external, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)] replica_client.add_replicas(rse=rse, files=files) # Listing replicas on deterministic RSE list_rep = [] for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['file'], all_states=True): pfn = list(replica['pfns'].keys())[0] list_rep.append(pfn) # Submit bad PFNs now = datetime.utcnow() reason_str = generate_uuid() replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='TEMPORARY_UNAVAILABLE', expires_at=now.isoformat()) result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None) bad_pfns = {} for res in result: bad_pfns[res['pfn']] = (res['state'], res['reason'], res['expires_at']) for pfn in list_rep: pfn = str(clean_surls([pfn])[0]) assert pfn in bad_pfns assert bad_pfns[pfn][0] == BadPFNStatus.TEMPORARY_UNAVAILABLE assert bad_pfns[pfn][1] == reason_str # Submit with wrong state with pytest.raises(RucioException): replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='BADSTATE', expires_at=now.isoformat()) # Run minos once minos_run(threads=1, bulk=10000, once=True) result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None) pfns = [res['pfn'] for res in result if res['pfn'] in bad_pfns] res_pfns = [] for replica in list_rep: if replica in pfns: res_pfns.append(replica) assert res_pfns == [] # Check the state in the replica table for did in files: rep = get_replicas_state(scope=mock_scope, name=did['name']) assert list(rep.keys())[0] == ReplicaState.TEMPORARY_UNAVAILABLE rep = [] for did in files: did['state'] = ReplicaState.TEMPORARY_UNAVAILABLE rep.append(did) # Run the minos expiration minos_temp_run(threads=1, once=True) # Check the state in the replica table for did in files: rep = get_replicas_state(scope=mock_scope, name=did['name']) assert list(rep.keys())[0] == ReplicaState.AVAILABLE
def test_add_temporary_unavailable_pfns(self): """ REPLICA (CLIENT): Add temporary unavailable PFNs""" tmp_scope = 'mock' nbfiles = 5 # Adding replicas to deterministic RSE files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] self.replica_client.add_replicas(rse='MOCK', files=files) # Listing replicas on deterministic RSE list_rep = [] for replica in self.replica_client.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in files], schemes=['srm'], unavailable=True): pfn = replica['pfns'].keys()[0] list_rep.append(pfn) # Submit bad PFNs now = datetime.utcnow() reason_str = generate_uuid() self.replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='TEMPORARY_UNAVAILABLE', expires_at=now.isoformat()) result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None) bad_pfns = {} for res in result: bad_pfns[res['pfn']] = (res['state'], res['reason'], res['expires_at']) for pfn in list_rep: pfn = str(clean_surls([pfn])[0]) assert_in(pfn, bad_pfns) assert_equal(str(bad_pfns[pfn][0]), 'TEMPORARY_UNAVAILABLE') assert_equal(bad_pfns[pfn][1], reason_str) # Submit with wrong state with assert_raises(RucioException): self.replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='BADSTATE', expires_at=now.isoformat()) # Run minos once minos_run(threads=1, bulk=10000, once=True) result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None) pfns = [res['pfn'] for res in result] res_pfns = [] for replica in list_rep: if replica in pfns: res_pfns.append(replica) assert_equal(res_pfns, []) # Check the state in the replica table for did in files: rep = get_replicas_state(scope=did['scope'], name=did['name']) assert_equal(str(rep.keys()[0]), 'TEMPORARY_UNAVAILABLE') rep = [] for did in files: did['state'] = ReplicaState.from_sym('TEMPORARY_UNAVAILABLE') rep.append(did) # Run the minos expiration minos_temp_run(threads=1, once=True) # Check the state in the replica table for did in files: rep = get_replicas_state(scope=did['scope'], name=did['name']) assert_equal(str(rep.keys()[0]), 'AVAILABLE')
def test_add_list_bad_replicas(rse_factory, mock_scope, root_account): """ REPLICA (CORE): Add bad replicas and list them""" nbfiles = 5 # Adding replicas to deterministic RSE _, rse1_id = rse_factory.make_srm_rse(deterministic=True) files = [{ 'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': { 'events': 10 } } for _ in range(nbfiles)] add_replicas(rse_id=rse1_id, files=files, account=root_account, ignore_availability=True) # Listing replicas on deterministic RSE replicas = [] list_rep = [] for replica in list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE } for f in files], schemes=['srm']): replicas.extend(replica['rses'][rse1_id]) list_rep.append(replica) r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account) assert r == {} bad_replicas = list_bad_replicas() nbbadrep = 0 for rep in list_rep: for badrep in bad_replicas: if badrep['rse_id'] == rse1_id: if badrep['scope'] == rep['scope'] and badrep['name'] == rep[ 'name']: nbbadrep += 1 assert len(replicas) == nbbadrep # Adding replicas to non-deterministic RSE _, rse2_id = rse_factory.make_srm_rse(deterministic=False) files = [{ 'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test_%s/%s/%s' % (rse2_id, rse2_id, mock_scope, generate_uuid()), 'meta': { 'events': 10 } } for _ in range(nbfiles)] add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True) # Listing replicas on non-deterministic RSE replicas = [] list_rep = [] for replica in list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE } for f in files], schemes=['srm']): replicas.extend(replica['rses'][rse2_id]) list_rep.append(replica) r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account) assert r == {} bad_replicas = list_bad_replicas() nbbadrep = 0 for rep in list_rep: for badrep in bad_replicas: if badrep['rse_id'] == rse2_id: if badrep['scope'] == rep['scope'] and badrep['name'] == rep[ 'name']: nbbadrep += 1 assert len(replicas) == nbbadrep # Try adding replicas already declared bad r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account) output = ['%s Unknown replica' % rep for rep in replicas] assert list(r.keys()) == [rse2_id] list1 = r[rse2_id] list1.sort() list2 = ['%s Already declared' % clean_surls([rep])[0] for rep in replicas] list2.sort() assert list1 == list2 # Now adding non-existing bad replicas files = [ 'srm://%s.cern.ch/test_%s/%s/%s' % (rse2_id, rse2_id, mock_scope, generate_uuid()), ] r = declare_bad_file_replicas(files, 'This is a good reason', root_account) output = ['%s Unknown replica' % rep for rep in files] assert r == {rse2_id: output}