def test_list_expired_dids_with_locked_rules(self): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules""" tmp_scope = InternalScope('mock') jdoe = InternalAccount('jdoe') root = InternalAccount('root') # Add quota set_local_account_limit(jdoe, get_rse_id('MOCK'), -1) dsn = { 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': 'MOCK', 'locked': True, 'grouping': 'DATASET' }] } add_dids(dids=[dsn], account=root) for did in list_expired_dids(limit=1000): assert (did['scope'] != dsn['scope'] and did['name'] != dsn['name'])
def test_clean_and_get_collection_replica_updates(self): """ REPLICA (CORE): Get cleaned update requests for collection replicas. """ dataset_name_with_collection_replica = 'dataset_with_rse%s' % generate_uuid() dataset_name_without_collection_replica = 'dataset_without_rse%s' % generate_uuid() add_dids(dids=[{'name': dataset_name_without_collection_replica, 'scope': self.scope, 'type': constants.DIDType.DATASET}, {'name': dataset_name_with_collection_replica, 'scope': self.scope, 'type': constants.DIDType.DATASET}], account=self.account, session=self.db_session) self.db_session.query(models.UpdatedCollectionReplica).delete() # pylint: disable=no-member self.db_session.commit() # pylint: disable=no-member # setup test data - 4 without corresponding replica, 4 duplicates and 2 correct models.CollectionReplica(rse_id=self.rse_id, scope=self.scope, bytes=10, length=0, available_replicas_cnt=0, state=constants.ReplicaState.AVAILABLE, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET)\ .save(session=self.db_session) models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session) cleaned_collection_replica_updates = get_cleaned_updated_collection_replicas(total_workers=0, worker_number=0, session=self.db_session) assert len(cleaned_collection_replica_updates) == 2 for update_request in cleaned_collection_replica_updates: update_request = self.db_session.query(models.UpdatedCollectionReplica).filter_by(id=update_request['id']).one() # pylint: disable=no-member assert update_request.scope == self.scope assert update_request.name in (dataset_name_with_collection_replica, dataset_name_without_collection_replica)
def add_dids(dids, issuer): """ Bulk Add did. :param dids: A list of dids. :param issuer: The issuer account. """ for d in dids: if 'rse' in d: rse_id = None if d['rse'] is not None: rse_id = get_rse_id(rse=d['rse']) d['rse_id'] = rse_id kwargs = {'issuer': issuer, 'dids': dids} if not rucio.api.permission.has_permission( issuer=issuer, action='add_dids', kwargs=kwargs): raise rucio.common.exception.AccessDenied( 'Account %s can not bulk add data identifier' % (issuer)) issuer = InternalAccount(issuer) for d in dids: d['scope'] = InternalScope(d['scope']) if 'dids' in d.keys(): for child in d['dids']: child['scope'] = InternalScope(child['scope']) return did.add_dids(dids, account=issuer)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 set_account_limit('jdoe', get_rse_id('MOCK'), -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for i in xrange(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': 'MOCK', 'grouping': 'DATASET' }] } for i in xrange(nbdatasets)] add_dids(dids=dsns1 + dsns2, account='root') replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in xrange(nbfiles)] attach_dids(scope=tmp_scope,
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() add_rse(rse) set_account_limit('jdoe', get_rse_id(rse), -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in xrange(nbdatasets)] add_dids(dids=dsns2, account='root') replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in xrange(nbfiles)]
def add_dids(dids, issuer): """ Bulk Add did. :param dids: A list of dids. :param issuer: The issuer account. """ kwargs = {'issuer': issuer, 'dids': dids} if not rucio.api.permission.has_permission(issuer=issuer, action='add_dids', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not bulk add data identifier' % (issuer)) return did.add_dids(dids, account=issuer)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK') set_account_limit('jdoe', rse_id, -1) dsns1 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1} for i in range(nbdatasets)] dsns2 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for i in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account='root') replicas = list() for dsn in dsns1 + dsns2: files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account='root') replicas += files add_rules(dids=dsns1, rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal(get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() add_rse(rse) set_account_limit('jdoe', get_rse_id(rse), -1) dsns2 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for i in range(nbdatasets)] add_dids(dids=dsns2, account='root') replicas = list() for dsn in dsns2: files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root') replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None) for dsn in dsns2: assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name']) assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
def add_dids(dids, issuer): """ Bulk Add did. :param dids: A list of dids. :param issuer: The issuer account. """ for d in dids: if 'rse' in d.keys(): d.update({'rse_id': get_rse_id(rse=d.get('rse'))}) kwargs = {'issuer': issuer, 'dids': dids} if not rucio.api.permission.has_permission( issuer=issuer, action='add_dids', kwargs=kwargs): raise rucio.common.exception.AccessDenied( 'Account %s can not bulk add data identifier' % (issuer)) return did.add_dids(dids, account=issuer)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = InternalScope('mock') jdoe = InternalAccount('jdoe') root = InternalAccount('root') nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK') set_local_account_limit(jdoe, rse_id, -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for i in range(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account=root) # Add generic metadata on did test_metadata = {"test_key": "test_value"} try: add_did_meta(tmp_scope, dsns1[0]['name'], test_metadata) except NotImplementedError: # add_did_meta is not Implemented for Oracle < 12 pass replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files add_rules(dids=dsns1, rules=[{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal( get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK', **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for _ in range(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for _ in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account=root) # arbitrary keys do not work without JSON support (sqlite, Oracle < 12) if json_implemented(): # Add generic metadata on did set_metadata(tmp_scope, dsns1[0]['name'], "test_key", "test_value") replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for _ in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files add_rules(dids=dsns1, rules=[{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is not None
def test_removal_all_replicas2(rse_factory, root_account, mock_scope, core_config_mock, caches_mock): """ UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """ rse1, rse1_id = rse_factory.make_posix_rse() rse2, rse2_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() set_local_account_limit(root_account, rse1_id, -1) set_local_account_limit(root_account, rse2_id, -1) nbdatasets = 1 nbfiles = 5 dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1 } for _ in range(nbdatasets)] add_dids(dids=dsns1, account=root_account) replicas = list() for dsn in dsns1: files = [{ 'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb' } for _ in range(nbfiles)] attach_dids(scope=mock_scope, name=dsn['name'], rse_id=rse1_id, dids=files, account=root_account) add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True) replicas += files add_rules(dids=dsns1, rules=[{ 'account': root_account, 'copies': 1, 'rse_expression': rse1, 'grouping': 'DATASET' }]) add_rules(dids=dsns1, rules=[{ 'account': root_account, 'copies': 1, 'rse_expression': rse2, 'grouping': 'DATASET', 'lifetime': -86400 }]) # Clean the rules on MOCK2. Replicas are tombstoned with non Epoch rule_cleaner(once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] is not None undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse1_id)['tombstone'] == datetime(year=1970, month=1, day=1) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] == datetime(year=1970, month=1, day=1)
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ if get_policy() != 'atlas': LOG.info("Skipping atlas-specific test") return tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for _ in range(nbdatasets)] add_dids(dids=dsns2, account=root) replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for _ in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert (get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is None) for dsn in dsns2: assert (get_did(scope=InternalScope('archive', **self.vo), name=dsn['name'])['name'] == dsn['name']) assert (len([ x for x in list_rules( filters={ 'scope': InternalScope('archive', **self.vo), 'name': dsn['name'] }) ]) == 1)
def test_update_collection_replica(self): """ REPLICA (CORE): Update collection replicas from update requests. """ file_size = 2 files = [{ 'name': 'file_%s' % generate_uuid(), 'scope': self.scope, 'bytes': file_size } for i in range(0, 2)] dataset_name = 'dataset_test_%s' % generate_uuid() add_replicas(rse_id=self.rse_id, files=files, account=self.account, session=self.db_session) add_dids([{ 'scope': self.scope, 'name': dataset_name, 'type': constants.DIDType.DATASET }], account=self.account, session=self.db_session) attach_dids(scope=self.scope, name=dataset_name, dids=files, account=self.account, session=self.db_session) models.CollectionReplica(rse_id=self.rse_id, scope=self.scope, state=constants.ReplicaState.AVAILABLE, name=dataset_name, did_type=constants.DIDType.DATASET, bytes=len(files) * file_size, length=len(files), available_replicas_cnt=0)\ .save(session=self.db_session) # Update request with rse id # First update -> dataset replica should be available models.UpdatedCollectionReplica( rse_id=self.rse_id, scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter_by( rse_id=self.rse_id, scope=self.scope, name=dataset_name).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter_by( id=update_request.id).first() # pylint: disable=no-member assert update_request is None dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE # Delete one file replica -> dataset replica should be unavailable delete_replicas(rse_id=self.rse_id, files=[files[0]], session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter_by( rse_id=self.rse_id, scope=self.scope, name=dataset_name).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.UNAVAILABLE # Add one file replica -> dataset replica should be available again add_replicas(rse_id=self.rse_id, files=[files[0]], account=self.account, session=self.db_session) attach_dids(scope=self.scope, name=dataset_name, dids=[files[0]], account=self.account, session=self.db_session) models.UpdatedCollectionReplica( rse_id=self.rse_id, scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter_by( rse_id=self.rse_id, scope=self.scope, name=dataset_name).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE # Old behaviour, open empty datasets are not deleted # Delete all file replicas -> dataset replica should be deleted delete_replicas(rse_id=self.rse_id, files=files, session=self.db_session) with pytest.raises(NoResultFound): update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) # Update request without rse_id - using two replicas per file -> total 4 replicas dataset_name = 'dataset_test_%s' % generate_uuid() add_dids([{ 'scope': self.scope, 'name': dataset_name, 'type': constants.DIDType.DATASET }], account=self.account, session=self.db_session) add_replicas(rse_id=self.rse_id, files=files, account=self.account, session=self.db_session) add_replicas(rse_id=self.rse2_id, files=files, account=self.account, session=self.db_session) attach_dids(scope=self.scope, name=dataset_name, dids=files, account=self.account, session=self.db_session) models.CollectionReplica( rse_id=self.rse_id, scope=self.scope, name=dataset_name, state=constants.ReplicaState.UNAVAILABLE, did_type=constants.DIDType.DATASET, bytes=len(files) * file_size, length=len(files)).save(session=self.db_session) models.CollectionReplica( rse_id=self.rse2_id, scope=self.scope, name=dataset_name, state=constants.ReplicaState.UNAVAILABLE, did_type=constants.DIDType.DATASET, bytes=len(files) * file_size, length=len(files)).save(session=self.db_session) # First update -> replicas should be available models.UpdatedCollectionReplica( scope=self.scope, name=dataset_name).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter_by( scope=self.scope, name=dataset_name).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) for dataset_replica in self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name).all(): # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE # Delete first replica on first RSE -> replica on first RSE should be unavailable, replica on second RSE should be still available delete_replicas(rse_id=self.rse_id, files=[files[0]], session=self.db_session) models.UpdatedCollectionReplica( scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) # delete_replica creates also update object but with rse_id -> extra filter for rse_id is NULL update_request = self.db_session.query( models.UpdatedCollectionReplica).filter( models.UpdatedCollectionReplica.scope == self.scope, models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.UNAVAILABLE dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse2_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE # Set the state of the first replica on the second RSE to UNAVAILABLE -> both replicass should be unavailable file_replica = self.db_session.query( models.RSEFileAssociation).filter_by(rse_id=self.rse2_id, scope=self.scope, name=files[0]['name']).one() # pylint: disable=no-member file_replica.state = constants.ReplicaState.UNAVAILABLE models.UpdatedCollectionReplica( scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter( models.UpdatedCollectionReplica.scope == self.scope, models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.UNAVAILABLE dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse2_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.UNAVAILABLE # Delete first replica on second RSE -> file is not longer part of dataset -> both replicas should be available delete_replicas(rse_id=self.rse2_id, files=[files[0]], session=self.db_session) models.UpdatedCollectionReplica( scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter( models.UpdatedCollectionReplica.scope == self.scope, models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == (len(files) - 1) * file_size assert dataset_replica['length'] == len(files) - 1 assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.AVAILABLE dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse2_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == (len(files) - 1) * file_size assert dataset_replica['length'] == len(files) - 1 assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.AVAILABLE # Add first replica to the first RSE -> first replicas should be available add_replicas(rse_id=self.rse_id, files=[files[0]], account=self.account, session=self.db_session) attach_dids(scope=self.scope, name=dataset_name, dids=[files[0]], account=self.account, session=self.db_session) models.UpdatedCollectionReplica( scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter( models.UpdatedCollectionReplica.scope == self.scope, models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse2_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == (len(files) - 1) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) - 1 assert dataset_replica['state'] == ReplicaState.UNAVAILABLE # Add first replica to the second RSE -> both replicas should be available again add_replicas(rse_id=self.rse2_id, files=[files[0]], account=self.account, session=self.db_session) models.UpdatedCollectionReplica( scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET).save(session=self.db_session) update_request = self.db_session.query( models.UpdatedCollectionReplica).filter( models.UpdatedCollectionReplica.scope == self.scope, models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member update_collection_replica(update_request=update_request.to_dict(), session=self.db_session) dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE dataset_replica = self.db_session.query( models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name, rse_id=self.rse2_id).one() # pylint: disable=no-member assert dataset_replica['bytes'] == len(files) * file_size assert dataset_replica['length'] == len(files) assert dataset_replica['available_bytes'] == len(files) * file_size assert dataset_replica['available_replicas_cnt'] == len(files) assert dataset_replica['state'] == ReplicaState.AVAILABLE