def test_cache_add_replicas_file_not_exist(self): """ RucioCache(Func): Test rucio cache add replica with not existed file(failed) """ ret = cache_add_replicas(self.rse_exist_volatile, self.files_noExist, self.account, self.lifetime) assert_equal(ret, DID_NOT_FOUND) for file in self.files_noExist: try: replica.get_replica(self.rse_exist_volatile, file['scope'], file['name']) assert False except Exception, e: if "No row was found" in str(e): assert True else: assert False
def test_cache_add_replicas_file_meta_wrong(self): """ RucioCache(Func): Test rucio cache add replica with wrong meta data(failed) """ ret = cache_add_replicas(self.rse_exist_volatile, self.files_exist_wrong_meta, self.account, self.lifetime) assert_equal(ret, META_MISMATCH) for file in self.files_exist_wrong_meta: try: replica.get_replica(self.rse_exist_volatile, file['scope'], file['name']) assert False except Exception, e: if "No row was found" in str(e): assert True else: assert False
def test_cache_add_delete_replicas(self): """ RucioCache(Func): Test rucio cache add and delete replicas(success) """ ret = cache_add_replicas(self.rse_exist_volatile, self.files_exist, self.account, self.lifetime) assert_equal(ret, 0) for file in self.files_exist: reps = replica.get_replica(self.rse_exist_volatile, file['scope'], file['name']) assert_equal(len(reps) > 0, True) ret = cache_delete_replicas(self.rse_exist_volatile, self.files_exist, self.account) assert_equal(ret, 0) for file in self.files_exist: try: reps = replica.get_replica(self.rse_exist_volatile, file['scope'], file['name']) assert False except Exception, e: if "No row was found" in str(e): assert True else: assert False
def _get_path_nondeterministic_server(self, scope, name): """ Provides the path of a replica for non-detemernisic sites. Will be assigned to get path by the __init__ method if neccessary. """ r = replica.get_replica(rse=self.rse['rse'], scope=scope, name=name, rse_id=self.rse['id']) if 'path' in r and r['path'] is not None: path = r['path'] elif 'state' in r and (r['state'] is None or r['state'] == 'UNAVAILABLE'): raise exception.ReplicaUnAvailable('Missing path information and state is UNAVAILABLE for replica %s:%s on none-determinstic storage named %s' % (scope, name, self.rse['rse'])) else: raise exception.ReplicaNotFound('Missing path information for replica %s:%s on none-determinstic storage named %s' % (scope, name, self.rse['rse'])) if path.startswith('/'): path = path[1:] if path.endswith('/'): path = path[:-1] return path
def test_add_rule_with_purge(self): """ REPLICATION RULE (CORE): Add a replication rule with purge setting""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, purge_replicas=True)[0] delete_rule(rule_id) # Check if the Locks are created properly for file in files: replica = get_replica(rse=self.rse4, scope=file['scope'], name=file['name']) assert(replica['tombstone'] == OBSOLETE)
def test_update_lock_counter(self): """ RSE (CORE): Test the update of a replica lock counter """ rse = 'MOCK' tmp_scope = 'mock' tmp_file = 'file_%s' % generate_uuid() add_replica(rse=rse, scope=tmp_scope, name=tmp_file, bytes=1L, adler32='0cc737eb', account='jdoe') values = (1, 1, 1, -1, -1, -1, 1, 1, -1) tombstones = (True, True, True, True, True, False, True, True, True) lock_counters = (1, 2, 3, 2, 1, 0, 1, 2, 1) for value, tombstone, lock_counter in zip(values, tombstones, lock_counters): status = update_replica_lock_counter(rse=rse, scope=tmp_scope, name=tmp_file, value=value) assert_equal(status, True) replica = get_replica(rse=rse, scope=tmp_scope, name=tmp_file) assert_equal(replica['tombstone'] is None, tombstone) assert_equal(lock_counter, replica['lock_cnt'])
def test_multihop_sources_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that multihop transfers are handled and intermediate request correctly created """ src_rse_name, src_rse_id = rse_factory.make_posix_rse() _, jump_rse1_id = rse_factory.make_posix_rse() _, jump_rse2_id = rse_factory.make_posix_rse() _, jump_rse3_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id] all_rses = jump_rses + [src_rse_id, dst_rse_id] for rse_id in jump_rses: rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True) rse_tombstone_delay = 3600 rse_multihop_tombstone_delay = 12 * 3600 default_multihop_tombstone_delay = 24 * 3600 # if both attributes are set, the multihop one will take precedence rse_core.add_rse_attribute(jump_rse1_id, 'tombstone_delay', rse_tombstone_delay) rse_core.add_rse_attribute(jump_rse1_id, 'multihop_tombstone_delay', rse_multihop_tombstone_delay) # if multihop delay not set, it's the default multihop takes precedence. Not normal tombstone delay. rse_core.add_rse_attribute(jump_rse2_id, 'tombstone_delay', rse_tombstone_delay) core_config.set(section='transfers', option='multihop_tombstone_delay', value=default_multihop_tombstone_delay) # if multihop delay is set to 0, the replica will have no tombstone rse_core.add_rse_attribute(jump_rse3_id, 'multihop_tombstone_delay', 0) distance_core.add_distance(src_rse_id, jump_rse1_id, ranking=10) distance_core.add_distance(jump_rse1_id, jump_rse2_id, ranking=10) distance_core.add_distance(jump_rse2_id, jump_rse3_id, ranking=10) distance_core.add_distance(jump_rse3_id, dst_rse_id, ranking=10) did = did_factory.upload_test_file(src_rse_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None) # Ensure that each intermediate request was correctly created for rse_id in jump_rses: assert request_core.get_request_by_did(rse_id=rse_id, **did) @read_session def __ensure_source_exists(rse_id, scope, name, session=None): return session.query(Source). \ filter(Source.rse_id == rse_id). \ filter(Source.scope == scope). \ filter(Source.name == name). \ one() # Ensure that sources where created for transfers for rse_id in jump_rses + [src_rse_id]: __ensure_source_exists(rse_id, **did) # Ensure the tombstone is correctly set on intermediate replicas expected_tombstone = datetime.utcnow() + timedelta( seconds=rse_multihop_tombstone_delay) replica = replica_core.get_replica(jump_rse1_id, **did) assert expected_tombstone - timedelta(minutes=5) < replica[ 'tombstone'] < expected_tombstone + timedelta(minutes=5) expected_tombstone = datetime.utcnow() + timedelta( seconds=default_multihop_tombstone_delay) replica = replica_core.get_replica(jump_rse2_id, **did) assert expected_tombstone - timedelta(minutes=5) < replica[ 'tombstone'] < expected_tombstone + timedelta(minutes=5) replica = replica_core.get_replica(jump_rse3_id, **did) assert replica['tombstone'] is None
def test_archive_removal_impact_on_constituents(rse_factory, did_factory, mock_scope, root_account, caches_mock): [cache_region] = caches_mock rse_name, rse_id = rse_factory.make_mock_rse() scope = mock_scope account = root_account # Create an 2 archives and 4 files: # - One only exists in the first archive # - One in both, plus another replica, which is not in an archive # - One in both, plus another replica, which is not in an archive; and this replica has expired # - One in both, plus another replica, which is not in an archive; and this replica has expired; but a replication rule exists on this second replica # Also add these files to datasets, one of which will be removed at the end nb_constituents = 4 nb_c_outside_archive = nb_constituents - 1 constituent_size = 2000 archive_size = 1000 uuid = str(generate_uuid()) constituents = [{ 'scope': scope, 'name': 'lfn.%s.%d' % (uuid, i) } for i in range(nb_constituents)] did_factory.register_dids(constituents) c_first_archive_only, c_with_replica, c_with_expired_replica, c_with_replica_and_rule = constituents replica_core.add_replica(rse_id=rse_id, account=account, bytes_=constituent_size, **c_with_replica) replica_core.add_replica(rse_id=rse_id, account=account, bytes_=constituent_size, tombstone=datetime.utcnow() - timedelta(days=1), **c_with_expired_replica) replica_core.add_replica(rse_id=rse_id, account=account, bytes_=constituent_size, tombstone=datetime.utcnow() - timedelta(days=1), **c_with_replica_and_rule) rule_core.add_rule(dids=[c_with_replica_and_rule], account=account, copies=1, rse_expression=rse_name, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) archive1, archive2 = [{ 'scope': scope, 'name': 'archive_%s.%d.zip' % (uuid, i) } for i in range(2)] replica_core.add_replica(rse_id=rse_id, bytes_=archive_size, account=account, **archive1) replica_core.add_replica(rse_id=rse_id, bytes_=archive_size, account=account, **archive2) did_core.attach_dids(dids=[{ 'scope': c['scope'], 'name': c['name'], 'bytes': constituent_size } for c in constituents], account=account, **archive1) did_core.attach_dids(dids=[ { 'scope': c['scope'], 'name': c['name'], 'bytes': constituent_size } for c in [c_with_replica, c_with_expired_replica, c_with_replica_and_rule] ], account=account, **archive2) dataset1, dataset2 = [{ 'scope': scope, 'name': 'dataset_%s.%i' % (uuid, i) } for i in range(2)] did_core.add_did(did_type='DATASET', account=account, **dataset1) did_core.attach_dids(dids=constituents, account=account, **dataset1) did_core.add_did(did_type='DATASET', account=account, **dataset2) did_core.attach_dids(dids=[c_first_archive_only, c_with_expired_replica], account=account, **dataset2) @read_session def __get_archive_contents_history_count(archive, session=None): return session.query(ConstituentAssociationHistory).filter_by( **archive).count() # Run reaper the first time. # the expired non-archive replica of c_with_expired_replica must be removed, # but the did must not be remove and it must still remain in the dataset because # it still has the replica from inside the archive assert replica_core.get_replica(rse_id=rse_id, **c_with_expired_replica) cache_region.invalidate() rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=2 * archive_size + nb_c_outside_archive * constituent_size) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=2 * archive_size + nb_c_outside_archive * constituent_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) for did in constituents + [archive1, archive2]: assert did_core.get_did(**did) for did in [archive1, archive2, c_with_replica, c_with_replica_and_rule]: assert replica_core.get_replica(rse_id=rse_id, **did) with pytest.raises(ReplicaNotFound): # The replica is only on the archive, not on the constituent replica_core.get_replica(rse_id=rse_id, **c_first_archive_only) with pytest.raises(ReplicaNotFound): # The replica outside the archive was removed by reaper nb_c_outside_archive -= 1 replica_core.get_replica(rse_id=rse_id, **c_with_expired_replica) # Compared to get_replica, list_replicas resolves archives, must return replicas for all files assert len(list(replica_core.list_replicas(dids=constituents))) == 4 assert len(list(did_core.list_content(**dataset1))) == 4 assert len(list(did_core.list_archive_content(**archive1))) == 4 assert len(list(did_core.list_archive_content(**archive2))) == 3 assert __get_archive_contents_history_count(archive1) == 0 assert __get_archive_contents_history_count(archive2) == 0 # Expire the first archive and run reaper again # the archive will be removed; and c_first_archive_only must be removed from datasets # and from the did table. replica_core.set_tombstone(rse_id=rse_id, tombstone=datetime.utcnow() - timedelta(days=1), **archive1) cache_region.invalidate() rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=2 * archive_size + nb_c_outside_archive * constituent_size) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=2 * archive_size + nb_c_outside_archive * constituent_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**archive1) with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**c_first_archive_only) assert len(list(replica_core.list_replicas(dids=constituents))) == 3 assert len(list(did_core.list_content(**dataset1))) == 3 assert len(list(did_core.list_archive_content(**archive1))) == 0 assert len(list(did_core.list_archive_content(**archive2))) == 3 assert __get_archive_contents_history_count(archive1) == 4 assert __get_archive_contents_history_count(archive2) == 0 # Expire the second archive replica and run reaper another time # c_with_expired_replica is removed because its external replica got removed at previous step # and it exist only inside the archive now. # If not open, Dataset2 will be removed because it will be empty. did_core.set_status(open=False, **dataset2) replica_core.set_tombstone(rse_id=rse_id, tombstone=datetime.utcnow() - timedelta(days=1), **archive2) cache_region.invalidate() rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=archive_size + nb_c_outside_archive * constituent_size) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=archive_size + nb_c_outside_archive * constituent_size, free=1) reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None) # The archive must be removed with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**archive2) # The DIDs which only existed in the archive are also removed with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**c_first_archive_only) with pytest.raises(DataIdentifierNotFound): assert did_core.get_did(**c_with_expired_replica) # If the DID has a non-expired replica outside the archive without rules on it, the DID is not removed assert did_core.get_did(**c_with_replica) # If the DID has an expired replica outside the archive, but has rules on that replica, the DID is not removed assert did_core.get_did(**c_with_replica_and_rule) assert len(list(replica_core.list_replicas(dids=constituents))) == 2 assert len(list(did_core.list_content(**dataset1))) == 2 with pytest.raises(DataIdentifierNotFound): did_core.get_did(**dataset2) assert len(list(did_core.list_content(**dataset2))) == 0 assert len(list(did_core.list_archive_content(**archive2))) == 0 assert __get_archive_contents_history_count(archive1) == 4 assert __get_archive_contents_history_count(archive2) == 3
def test_to_repair_a_rule_with_NONE_grouping_whose_transfer_failed(self): """ JUDGE REPAIRER: Test to repair a rule with 1 failed transfer (lock)""" rule_repairer(once=True) # Clean out the repairer scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse4_id, bytes_=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=1, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0] failed_rse_id = get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id assert (get_replica( scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['state'] == ReplicaState.COPYING) assert (get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['lock_cnt'] == 1) successful_transfer( scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) successful_transfer( scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) failed_transfer( scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) assert (rule_id == get_rule(rule_id)['id'].replace('-', '').lower()) assert (RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) assert (RuleState.REPLICATING == get_rule(rule_id)['state']) assert (get_replica( scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['state'] == ReplicaState.UNAVAILABLE) assert (get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['lock_cnt'] == 0)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK', **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for _ in range(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for _ in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account=root) # arbitrary keys do not work without JSON support (sqlite, Oracle < 12) if json_implemented(): # Add generic metadata on did set_metadata(tmp_scope, dsns1[0]['name'], "test_key", "test_value") replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for _ in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files add_rules(dids=dsns1, rules=[{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is not None
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ if get_policy() != 'atlas': LOG.info("Skipping atlas-specific test") return tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for _ in range(nbdatasets)] add_dids(dids=dsns2, account=root) replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for _ in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert (get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is None) for dsn in dsns2: assert (get_did(scope=InternalScope('archive', **self.vo), name=dsn['name'])['name'] == dsn['name']) assert (len([ x for x in list_rules( filters={ 'scope': InternalScope('archive', **self.vo), 'name': dsn['name'] }) ]) == 1)
session = get_session() lost_requests = get_lost_requests(input_file_name) for scope, name, rse_name in lost_requests: # print rse_name, scope, name if rse_name not in rse_ids: rse_id = rse_core.get_rse_id(rse_name, session=session) rse_ids[rse_name] = rse_id else: rse_id = rse_ids[rse_name] try: replica = replica_core.get_replica(rse=rse_name, scope=scope, name=name, rse_id=rse_id, session=session) except NoResultFound: continue if replica['state'] != ReplicaState.AVAILABLE: continue if rse_name not in protocols: rse_info = rsemgr.get_rse_info(rse_name, session=session) protocols[rse_name] = rsemgr.create_protocol(rse_info, 'write', 'srm,gsiftp') lfn = { 'scope': replica['scope'], 'name': replica['name'], 'path': replica['path']
add_rules(dids=dsns1, rules=[{ 'account': 'jdoe', 'copies': 1, 'rse_expression': 'MOCK', 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal( get_replica(scope=replica['scope'], name=replica['name'], rse='MOCK')['tombstone'], None) def test_list_expired_dids_with_locked_rules(self): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules""" tmp_scope = 'mock' # Add quota set_account_limit('jdoe', get_rse_id('MOCK'), -1) dsn = { 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type':
def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account, core_config_mock, caches_mock): """ Ensure that intermediate replicas created by the submitter are protected from deletion even if their tombstone is set to epoch. After successful transfers, intermediate replicas with default (epoch) tombstone must be removed. The others must be left intact. """ src_rse1_name = 'XRD1' src_rse1_id = rse_core.get_rse_id(rse=src_rse1_name, vo=vo) src_rse2_name = 'XRD2' src_rse2_id = rse_core.get_rse_id(rse=src_rse2_name, vo=vo) jump_rse_name = 'XRD3' jump_rse_id = rse_core.get_rse_id(rse=jump_rse_name, vo=vo) dst_rse_name = 'XRD4' dst_rse_id = rse_core.get_rse_id(rse=dst_rse_name, vo=vo) all_rses = [src_rse1_id, src_rse2_id, jump_rse_id, dst_rse_id] did = did_factory.upload_test_file(src_rse1_name) # Copy replica to a second source. To avoid the special case of having a unique last replica, which could be handled in a special (more careful) way rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=src_rse2_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) replica = __wait_for_replica_transfer(dst_rse_id=src_rse2_id, **did) assert replica['state'] == ReplicaState.AVAILABLE rse_core.set_rse_limits(rse_id=jump_rse_id, name='MinFreeSpace', value=1) rse_core.set_rse_usage(rse_id=jump_rse_id, source='storage', used=1, free=0) try: rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Submit transfers to FTS # Ensure a replica was created on the intermediary host with epoch tombstone submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) request = request_core.get_request_by_did(rse_id=jump_rse_id, **did) assert request['state'] == RequestState.SUBMITTED replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica['tombstone'] == datetime(year=1970, month=1, day=1) assert replica['state'] == ReplicaState.COPYING # The intermediate replica is protected by its state (Copying) rucio.daemons.reaper.reaper.REGION.invalidate() reaper(once=True, rses=[], include_rses=jump_rse_name, exclude_rses=None) replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.COPYING # Wait for the intermediate replica to become ready replica = __wait_for_replica_transfer(dst_rse_id=jump_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE # The intermediate replica is protected by an entry in the sources table # Reaper must not remove this replica, even if it has an obsolete tombstone rucio.daemons.reaper.reaper.REGION.invalidate() reaper(once=True, rses=[], include_rses=jump_rse_name, exclude_rses=None) replica = replica_core.get_replica(rse_id=jump_rse_id, **did) assert replica # FTS fails the second transfer, so run submitter again to copy from jump rse to destination rse submitter(once=True, rses=[{ 'id': rse_id } for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None) # Wait for the destination replica to become ready replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did) assert replica['state'] == ReplicaState.AVAILABLE rucio.daemons.reaper.reaper.REGION.invalidate() reaper(once=True, rses=[], include_rses='test_container_xrd=True', exclude_rses=None) with pytest.raises(ReplicaNotFound): replica_core.get_replica(rse_id=jump_rse_id, **did) finally: @transactional_session def _cleanup_all_usage_and_limits(rse_id, session=None): session.query(models.RSELimit).filter_by(rse_id=rse_id).delete() session.query(models.RSEUsage).filter_by( rse_id=rse_id, source='storage').delete() _cleanup_all_usage_and_limits(rse_id=jump_rse_id)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = InternalScope('mock') jdoe = InternalAccount('jdoe') root = InternalAccount('root') nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK') set_account_limit(jdoe, rse_id, -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for i in range(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account=root) replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files add_rules(dids=dsns1, rules=[{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal( get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_removal_all_replicas2(rse_factory, root_account, mock_scope, core_config_mock, caches_mock): """ UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """ rse1, rse1_id = rse_factory.make_posix_rse() rse2, rse2_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() set_local_account_limit(root_account, rse1_id, -1) set_local_account_limit(root_account, rse2_id, -1) nbdatasets = 1 nbfiles = 5 dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1 } for _ in range(nbdatasets)] add_dids(dids=dsns1, account=root_account) replicas = list() for dsn in dsns1: files = [{ 'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb' } for _ in range(nbfiles)] attach_dids(scope=mock_scope, name=dsn['name'], rse_id=rse1_id, dids=files, account=root_account) add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True) replicas += files add_rules(dids=dsns1, rules=[{ 'account': root_account, 'copies': 1, 'rse_expression': rse1, 'grouping': 'DATASET' }]) add_rules(dids=dsns1, rules=[{ 'account': root_account, 'copies': 1, 'rse_expression': rse2, 'grouping': 'DATASET', 'lifetime': -86400 }]) # Clean the rules on MOCK2. Replicas are tombstoned with non Epoch rule_cleaner(once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] is not None undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse1_id)['tombstone'] == datetime(year=1970, month=1, day=1) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] == datetime(year=1970, month=1, day=1)
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() add_rse(rse) set_account_limit('jdoe', get_rse_id(rse), -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in range(nbdatasets)] add_dids(dids=dsns2, account='root') replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root') replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert (get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None) for dsn in dsns2: assert (get_did(scope='archive', name=dsn['name'])['name'] == dsn['name']) assert (len([ x for x in list_rules(filters={ 'scope': 'archive', 'name': dsn['name'] }) ]) == 1)