def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" nb_files = 30 file_size = 2147483648 # 2G rse_id = rse_core.get_rse_id(rse='MOCK') for i in range(nb_files): replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip'), name='lfn' + generate_uuid(), bytes=file_size, account=InternalAccount('root'), adler32=None, md5=None) rse_core.set_rse_usage(rse_id=rse_id, source='srm', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) reaper(once=True, rses=[], include_rses='MOCK', exclude_rses=[]) reaper(once=True, rses=[], include_rses='MOCK', exclude_rses=[])
def test_reaper_multi_vo(vo): """ REAPER2 (DAEMON): Test the reaper2 daemon with multiple vo.""" new_vo = __setup_new_vo() _, [scope_tst, scope_new] = __setup_scopes_for_vos(vo, new_vo) nb_files = 250 file_size = 200 # 2G rse1_name, rse1_id, dids1 = __add_test_rse_and_replicas( vo=vo, scope=scope_tst, rse_name=rse_name_generator(), names=['lfn' + generate_uuid() for _ in range(nb_files)], file_size=file_size) rse2_name, rse2_id, dids2 = __add_test_rse_and_replicas( vo=new_vo, scope=scope_new, rse_name=rse_name_generator(), names=['lfn' + generate_uuid() for _ in range(nb_files)], file_size=file_size) rse_core.set_rse_limits(rse_id=rse1_id, name='MinFreeSpace', value=50 * file_size) rse_core.set_rse_limits(rse_id=rse2_id, name='MinFreeSpace', value=50 * file_size) # Check we reap all VOs by default REGION.invalidate() rse_core.set_rse_usage(rse_id=rse1_id, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_usage(rse_id=rse2_id, source='storage', used=nb_files * file_size, free=1) both_rses = '%s|%s' % (rse1_name, rse2_name) reaper(once=True, rses=[], include_rses=both_rses, exclude_rses=None) reaper(once=True, rses=[], include_rses=both_rses, exclude_rses=None) assert len( list(replica_core.list_replicas(dids=dids1, rse_expression=both_rses))) == 200 assert len( list(replica_core.list_replicas(dids=dids2, rse_expression=both_rses))) == 200
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} else: vo = {} rse_name = rse_name_generator() rse_id = rse_core.add_rse(rse_name, **vo) mock_protocol = {'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}} rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) nb_files = 30 file_size = 2147483648 # 2G file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if vo: reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[]) reaper(once=True, rses=[], include_rses='vo=%s&(%s)' % (vo['vo'], rse_name), exclude_rses=[]) else: reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))) == nb_files - 5
def test_reaper(vo): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" scope = InternalScope('data13_hip', vo=vo) nb_files = 250 file_size = 200 # 2G rse_name, rse_id, dids = __add_test_rse_and_replicas( vo=vo, scope=scope, rse_name=rse_name_generator(), names=['lfn' + generate_uuid() for _ in range(nb_files)], file_size=file_size) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size) assert len( list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == nb_files # Check first if the reaper does not delete anything if no space is needed REGION.invalidate() rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) assert len( list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == nb_files # Now put it over threshold and delete REGION.invalidate() rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) assert len(list(replica_core.list_replicas( dids, rse_expression=rse_name))) == 200
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = { 'vo': config_get('client', 'vo', raise_exception=False, default='tst') } new_vo = {'vo': 'new'} if not vo_core.vo_exists(**new_vo): vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo) if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)): scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo)) nb_rses = 2 else: vo = {} new_vo = {} nb_rses = 1 mock_protocol = { 'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } } nb_files = 30 file_size = 2147483648 # 2G rse_names = [] all_file_names = [] for j in range(nb_rses): rse_name = rse_name_generator() rse_names.append(rse_name) rse_id = rse_core.add_rse(rse_name, **vo) rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) if new_vo: rse_id_new = rse_core.add_rse(rse_name, **new_vo) rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol) file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) if new_vo: replica_core.add_replica( rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **new_vo), adler32=None, md5=None) all_file_names.append(file_names) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if new_vo: rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10) if not vo: reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_name))) == nb_files - 5 else: # Check we reap all VOs by default reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_names[0]))) == nb_files - 5 assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **new_vo), 'name': n } for n in all_file_names[0]], rse_expression=rse_names[0]))) == nb_files - 5 # Check we don't affect a second VO that isn't specified reaper(once=True, rses=[], include_rses=rse_names[1], exclude_rses=[], vos=['new']) reaper(once=True, rses=[], include_rses=rse_names[1], exclude_rses=[], vos=['new']) assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **vo), 'name': n } for n in all_file_names[1]], rse_expression=rse_names[1]))), nb_files assert len( list( replica_core.list_replicas( dids=[{ 'scope': InternalScope('data13_hip', **new_vo), 'name': n } for n in all_file_names[1]], rse_expression=rse_names[1]))), nb_files - 5
def test_archive_removal_impact_on_constituents(rse_factory, did_factory, mock_scope, root_account): rse_name, rse_id = rse_factory.make_mock_rse() scope = mock_scope account = root_account # Create an 2 archives and 4 files: # - One only exists in the first archive # - One in both, plus another replica, which is not in an archive # - One in both, plus another replica, which is not in an archive; and this replica has expired # - One in both, plus another replica, which is not in an archive; and this replica has expired; but a replication rule exists on this second replica # Also add these files to datasets, one of which will be removed at the end nb_constituents = 4 nb_c_outside_archive = nb_constituents - 1 constituent_size = 2000 archive_size = 1000 uuid = str(generate_uuid()) constituents = [{ 'scope': scope, 'name': 'lfn.%s.%d' % (uuid, i) } for i in range(nb_constituents)] did_factory.register_dids(constituents) c_first_archive_only, c_with_replica, c_with_expired_replica, c_with_replica_and_rule = constituents replica_core.add_replica(rse_id=rse_id, account=account, bytes=constituent_size, **c_with_replica) replica_core.add_replica(rse_id=rse_id, account=account, bytes=constituent_size, tombstone=datetime.utcnow() - timedelta(days=1), **c_with_expired_replica) replica_core.add_replica(rse_id=rse_id, account=account, bytes=constituent_size, tombstone=datetime.utcnow() - timedelta(days=1), **c_with_replica_and_rule) rule_core.add_rule(dids=[c_with_replica_and_rule], account=account, copies=1, rse_expression=rse_name, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) archive1, archive2 = [{ 'scope': scope, 'name': 'archive_%s.%d.zip' % (uuid, i) } for i in range(2)] replica_core.add_replica(rse_id=rse_id, bytes=archive_size, account=account, **archive1) replica_core.add_replica(rse_id=rse_id, bytes=archive_size, account=account, **archive2) did_core.attach_dids(dids=[{ 'scope': c['scope'], 'name': c['name'], 'bytes': constituent_size } for c in constituents], account=account, **archive1) did_core.attach_dids(dids=[ { 'scope': c['scope'], 'name': c['name'], 'bytes': constituent_size } for c in [c_with_replica, c_with_expired_replica, c_with_replica_and_rule] ], account=account, **archive2) dataset1, dataset2 = [{ 'scope': scope, 'name': 'dataset_%s.%i' % (uuid, i) } for i in range(2)] did_core.add_did(type='DATASET', account=account, **dataset1) did_core.attach_dids(dids=constituents, account=account, **dataset1) did_core.add_did(type='DATASET', account=account, **dataset2) did_core.attach_dids(dids=[c_first_archive_only, c_with_expired_replica], account=account, **dataset2) @read_session def __get_archive_contents_history_count(archive, session=None): return session.query(ConstituentAssociationHistory).filter_by( **archive).count() # Run reaper the first time. # the expired non-archive replica of c_with_expired_replica must be removed, # but the did must not be remove and it must still remain in the dataset because # it still has the replica from inside the archive assert replica_core.get_replica(rse_id=rse_id, **c_with_expired_replica) REGION.invalidate() rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=2 * archive_size + nb_c_outside_archive * constituent_size) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=2 * archive_size + nb_c_outside_archive * constituent_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) for did in constituents + [archive1, archive2]: assert did_core.get_did(**did) for did in [archive1, archive2, c_with_replica, c_with_replica_and_rule]: assert replica_core.get_replica(rse_id=rse_id, **did) with pytest.raises(ReplicaNotFound): # The replica is only on the archive, not on the constituent replica_core.get_replica(rse_id=rse_id, **c_first_archive_only) with pytest.raises(ReplicaNotFound): # The replica outside the archive was removed by reaper nb_c_outside_archive -= 1 replica_core.get_replica(rse_id=rse_id, **c_with_expired_replica) # Compared to get_replica, list_replicas resolves archives, must return replicas for all files assert len(list(replica_core.list_replicas(dids=constituents))) == 4 assert len(list(did_core.list_content(**dataset1))) == 4 assert len(list(did_core.list_archive_content(**archive1))) == 4 assert len(list(did_core.list_archive_content(**archive2))) == 3 assert __get_archive_contents_history_count(archive1) == 0 assert __get_archive_contents_history_count(archive2) == 0 # Expire the first archive and run reaper again # the archive will be removed; and c_first_archive_only must be removed from datasets # and from the did table. replica_core.set_tombstone(rse_id=rse_id, tombstone=datetime.utcnow() - timedelta(days=1), **archive1) REGION.invalidate() rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=2 * archive_size + nb_c_outside_archive * constituent_size) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=2 * archive_size + nb_c_outside_archive * constituent_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**archive1) with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**c_first_archive_only) assert len(list(replica_core.list_replicas(dids=constituents))) == 3 assert len(list(did_core.list_content(**dataset1))) == 3 assert len(list(did_core.list_archive_content(**archive1))) == 0 assert len(list(did_core.list_archive_content(**archive2))) == 3 assert __get_archive_contents_history_count(archive1) == 4 assert __get_archive_contents_history_count(archive2) == 0 # Expire the second archive replica and run reaper another time # c_with_expired_replica is removed because its external replica got removed at previous step # and it exist only inside the archive now. # If not open, Dataset2 will be removed because it will be empty. did_core.set_status(open=False, **dataset2) replica_core.set_tombstone(rse_id=rse_id, tombstone=datetime.utcnow() - timedelta(days=1), **archive2) REGION.invalidate() rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=archive_size + nb_c_outside_archive * constituent_size) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=archive_size + nb_c_outside_archive * constituent_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) # The archive must be removed with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**archive2) # The DIDs which only existed in the archive are also removed with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**c_first_archive_only) with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**c_with_expired_replica) # If the DID has a non-expired replica outside the archive without rules on it, the DID is not removed assert did_core.get_did(**c_with_replica) # If the DID has an expired replica outside the archive, but has rules on that replica, the DID is not removed assert did_core.get_did(**c_with_replica_and_rule) assert len(list(replica_core.list_replicas(dids=constituents))) == 2 assert len(list(did_core.list_content(**dataset1))) == 2 with pytest.raises(DataIdentifierNotFound): did_core.get_did(**dataset2) assert len(list(did_core.list_content(**dataset2))) == 0 assert len(list(did_core.list_archive_content(**archive2))) == 0 assert __get_archive_contents_history_count(archive1) == 4 assert __get_archive_contents_history_count(archive2) == 3
def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that intermediate replicas created by the submitter are protected from deletion even if their tombstone is set to epoch. After successful transfers, intermediate replicas with default (epoch) tombstone must be removed. The others must be left intact. """ src_rse1_name = 'XRD1' src_rse1_id = rse_core.get_rse_id(rse=src_rse1_name, vo=vo) src_rse2_name = 'XRD2' src_rse2_id = rse_core.get_rse_id(rse=src_rse2_name, vo=vo) jump_rse_name = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse_name, vo=vo) dst_rse_name = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse_name, vo=vo) all_rses = [src_rse1_id, src_rse2_id, jump_rse_id, dst_rse_id] did = did_factory.upload_test_file(src_rse1_name) # Copy replica to a second source. To avoid the special case of having a unique last replica, which could be handled in a special (more careful) way rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=src_rse2_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) replica = __wait_for_replica_transfer(dst_rse_id=src_rse2_id, **did) assert replica['state'] == ReplicaState.AVAILABLE rse_core.set_rse_limits(rse_id=jump_rse_id, name='MinFreeSpace', value=1) rse_core.set_rse_usage(rse_id=jump_rse_id, source='storage', used=1, free=0) try: rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Submit transfers to FTS # Ensure a replica was created on the intermediary host submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) request = request_core.get_request_by_did(rse_id=jump_rse_id, **did) assert request['state'] == RequestState.SUBMITTED replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.COPYING # The intermediate replica is protected by its state (Copying) rucio.daemons.reaper.reaper2.REGION.invalidate() reaper(once=True, rses=[], include_rses=jump_rse_name, exclude_rses=None) replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.COPYING # Wait for the intermediate replica to become ready replica = __wait_for_replica_transfer(dst_rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE # The intermediate replica is protected by an entry in the sources table # Reaper must not remove this replica, even if it has an obsolete tombstone # # TODO: Uncomment following lines # rucio.daemons.reaper.reaper2.REGION.invalidate() # reaper(once=True, rses=[], include_rses=jump_rse_name, exclude_rses=None) # replica = replica_core.get_replica(rse_id=jump_rse_id, **did) # assert replica # FTS fails the second transfer, so run submitter again to copy from jump rse to destination rse submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) # Wait for the destination replica to become ready replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE rucio.daemons.reaper.reaper2.REGION.invalidate() reaper(once=True, rses=[], include_rses='test_container_xrd=True', exclude_rses=None) # TODO: reaper must delete this replica. It is not a source anymore. replica_core.get_replica(rse_id=jump_rse_id, **did) finally: @transactional_session def _cleanup_all_usage_and_limits(rse_id, session=None): session.query(models.RSELimit).filter_by(rse_id=rse_id).delete() session.query(models.RSEUsage).filter_by(rse_id=rse_id, source='storage').delete() _cleanup_all_usage_and_limits(rse_id=jump_rse_id)
def test_reaper(): """ REAPER2 (DAEMON): Test the reaper2 daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} new_vo = {'vo': 'new'} if not vo_core.vo_exists(**new_vo): vo_core.add_vo(description='Test', email='*****@*****.**', **new_vo) if not scope_core.check_scope(InternalScope('data13_hip', **new_vo)): scope_core.add_scope(InternalScope('data13_hip', **new_vo), InternalAccount('root', **new_vo)) nb_rses = 2 else: vo = {} new_vo = {} nb_rses = 1 mock_protocol = {'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}} nb_files = 250 file_size = 200 # 2G rse_names = [] all_file_names = [] for j in range(nb_rses): rse_name = rse_name_generator() rse_names.append(rse_name) rse_id = rse_core.add_rse(rse_name, **vo) rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) if new_vo: rse_id_new = rse_core.add_rse(rse_name, **new_vo) rse_core.add_protocol(rse_id=rse_id_new, parameter=mock_protocol) file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) if new_vo: replica_core.add_replica(rse_id=rse_id_new, scope=InternalScope('data13_hip', **new_vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **new_vo), adler32=None, md5=None) all_file_names.append(file_names) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size) # rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) if new_vo: rse_core.set_rse_usage(rse_id=rse_id_new, source='storage', used=nb_files * file_size, free=1) rse_core.set_rse_limits(rse_id=rse_id_new, name='MinFreeSpace', value=50 * file_size) # rse_core.set_rse_limits(rse_id=rse_id_new, name='MaxBeingDeletedFiles', value=10) from rucio.daemons.reaper.reaper2 import REGION REGION.invalidate() if not vo: assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == nb_files # Check first if the reaper does not delete anything if no space is needed rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == nb_files # Now put it over threshold and delete rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=1) from rucio.daemons.reaper.reaper2 import REGION REGION.invalidate() reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_name))) == 200 else: # Check we reap all VOs by default reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) reaper(once=True, rses=[], include_rses=rse_names[0], exclude_rses=[]) assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_names[0]))) == 200 assert len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **new_vo), 'name': n} for n in all_file_names[0]], rse_expression=rse_names[0]))) == 200