def test_list_expired_dids_with_locked_rules(self):
""" UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules"""
tmp_scope = InternalScope('mock')
jdoe = InternalAccount('jdoe')
root = InternalAccount('root')
# Add quota
set_local_account_limit(jdoe, get_rse_id('MOCK'), -1)
dsn = {
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': 'MOCK',
'locked': True,
'grouping': 'DATASET'
}]
}
add_dids(dids=[dsn], account=root)
for did in list_expired_dids(limit=1000):
assert (did['scope'] != dsn['scope']
and did['name'] != dsn['name'])
def test_clean_and_get_collection_replica_updates(self):
""" REPLICA (CORE): Get cleaned update requests for collection replicas. """
dataset_name_with_collection_replica = 'dataset_with_rse%s' % generate_uuid()
dataset_name_without_collection_replica = 'dataset_without_rse%s' % generate_uuid()
add_dids(dids=[{'name': dataset_name_without_collection_replica, 'scope': self.scope, 'type': constants.DIDType.DATASET},
{'name': dataset_name_with_collection_replica, 'scope': self.scope, 'type': constants.DIDType.DATASET}], account=self.account, session=self.db_session)
self.db_session.query(models.UpdatedCollectionReplica).delete() # pylint: disable=no-member
self.db_session.commit() # pylint: disable=no-member
# setup test data - 4 without corresponding replica, 4 duplicates and 2 correct
models.CollectionReplica(rse_id=self.rse_id, scope=self.scope, bytes=10, length=0, available_replicas_cnt=0, state=constants.ReplicaState.AVAILABLE, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET)\
.save(session=self.db_session)
models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(rse_id=self.rse_id, scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_without_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
models.UpdatedCollectionReplica(scope=self.scope, name=dataset_name_with_collection_replica, did_type=constants.DIDType.DATASET).save(session=self.db_session)
cleaned_collection_replica_updates = get_cleaned_updated_collection_replicas(total_workers=0, worker_number=0, session=self.db_session)
assert len(cleaned_collection_replica_updates) == 2
for update_request in cleaned_collection_replica_updates:
update_request = self.db_session.query(models.UpdatedCollectionReplica).filter_by(id=update_request['id']).one() # pylint: disable=no-member
assert update_request.scope == self.scope
assert update_request.name in (dataset_name_with_collection_replica, dataset_name_without_collection_replica)
def add_dids(dids, issuer):
"""
Bulk Add did.
:param dids: A list of dids.
:param issuer: The issuer account.
"""
for d in dids:
if 'rse' in d:
rse_id = None
if d['rse'] is not None:
rse_id = get_rse_id(rse=d['rse'])
d['rse_id'] = rse_id
kwargs = {'issuer': issuer, 'dids': dids}
if not rucio.api.permission.has_permission(
issuer=issuer, action='add_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not bulk add data identifier' % (issuer))
issuer = InternalAccount(issuer)
for d in dids:
d['scope'] = InternalScope(d['scope'])
if 'dids' in d.keys():
for child in d['dids']:
child['scope'] = InternalScope(child['scope'])
return did.add_dids(dids, account=issuer)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
set_account_limit('jdoe', get_rse_id('MOCK'), -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for i in xrange(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': 'MOCK',
'grouping': 'DATASET'
}]
} for i in xrange(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account='root')
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1L,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
attach_dids(scope=tmp_scope,
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': 'jdoe',
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in xrange(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1L,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in xrange(nbfiles)]
def add_dids(dids, issuer):
"""
Bulk Add did.
:param dids: A list of dids.
:param issuer: The issuer account.
"""
kwargs = {'issuer': issuer, 'dids': dids}
if not rucio.api.permission.has_permission(issuer=issuer, action='add_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not bulk add data identifier' % (issuer))
return did.add_dids(dids, account=issuer)
def add_dids(dids, issuer):
"""
Bulk Add did.
:param dids: A list of dids.
:param issuer: The issuer account.
"""
kwargs = {'issuer': issuer, 'dids': dids}
if not rucio.api.permission.has_permission(issuer=issuer, action='add_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not bulk add data identifier' % (issuer))
return did.add_dids(dids, account=issuer)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK')
set_account_limit('jdoe', rse_id, -1)
dsns1 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1} for i in range(nbdatasets)]
dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1,
'rules': [{'account': 'jdoe', 'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'}]} for i in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account='root')
replicas = list()
for dsn in dsns1 + dsns2:
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(),
'bytes': 1, 'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account='root')
replicas += files
add_rules(dids=dsns1, rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
tmp_scope = 'mock'
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
add_rse(rse)
set_account_limit('jdoe', get_rse_id(rse), -1)
dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1,
'rules': [{'account': 'jdoe', 'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'}]} for i in range(nbdatasets)]
add_dids(dids=dsns2, account='root')
replicas = list()
for dsn in dsns2:
files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1,
'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root')
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None)
for dsn in dsns2:
assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name'])
assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
def add_dids(dids, issuer):
"""
Bulk Add did.
:param dids: A list of dids.
:param issuer: The issuer account.
"""
for d in dids:
if 'rse' in d.keys():
d.update({'rse_id': get_rse_id(rse=d.get('rse'))})
kwargs = {'issuer': issuer, 'dids': dids}
if not rucio.api.permission.has_permission(
issuer=issuer, action='add_dids', kwargs=kwargs):
raise rucio.common.exception.AccessDenied(
'Account %s can not bulk add data identifier' % (issuer))
return did.add_dids(dids, account=issuer)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = InternalScope('mock')
jdoe = InternalAccount('jdoe')
root = InternalAccount('root')
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK')
set_local_account_limit(jdoe, rse_id, -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for i in range(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account=root)
# Add generic metadata on did
test_metadata = {"test_key": "test_value"}
try:
add_did_meta(tmp_scope, dsns1[0]['name'], test_metadata)
except NotImplementedError:
# add_did_meta is not Implemented for Oracle < 12
pass
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert_not_equal(
get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'], None)
def test_undertaker(self):
""" UNDERTAKER (CORE): Test the undertaker. """
tmp_scope = InternalScope('mock', **self.vo)
jdoe = InternalAccount('jdoe', **self.vo)
root = InternalAccount('root', **self.vo)
nbdatasets = 5
nbfiles = 5
rse = 'MOCK'
rse_id = get_rse_id('MOCK', **self.vo)
set_local_account_limit(jdoe, rse_id, -1)
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': tmp_scope,
'type': 'DATASET',
'lifetime': -1
} for _ in range(nbdatasets)]
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for _ in range(nbdatasets)]
add_dids(dids=dsns1 + dsns2, account=root)
# arbitrary keys do not work without JSON support (sqlite, Oracle < 12)
if json_implemented():
# Add generic metadata on did
set_metadata(tmp_scope, dsns1[0]['name'], "test_key", "test_value")
replicas = list()
for dsn in dsns1 + dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}])
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'] is not None
def test_removal_all_replicas2(rse_factory, root_account, mock_scope,
core_config_mock, caches_mock):
""" UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """
rse1, rse1_id = rse_factory.make_posix_rse()
rse2, rse2_id = rse_factory.make_posix_rse()
dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
set_local_account_limit(root_account, rse1_id, -1)
set_local_account_limit(root_account, rse2_id, -1)
nbdatasets = 1
nbfiles = 5
dsns1 = [{
'name': 'dsn_%s' % generate_uuid(),
'scope': mock_scope,
'type': 'DATASET',
'lifetime': -1
} for _ in range(nbdatasets)]
add_dids(dids=dsns1, account=root_account)
replicas = list()
for dsn in dsns1:
files = [{
'scope': mock_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb'
} for _ in range(nbfiles)]
attach_dids(scope=mock_scope,
name=dsn['name'],
rse_id=rse1_id,
dids=files,
account=root_account)
add_replicas(rse_id=rse2_id,
files=files,
account=root_account,
ignore_availability=True)
replicas += files
add_rules(dids=dsns1,
rules=[{
'account': root_account,
'copies': 1,
'rse_expression': rse1,
'grouping': 'DATASET'
}])
add_rules(dids=dsns1,
rules=[{
'account': root_account,
'copies': 1,
'rse_expression': rse2,
'grouping': 'DATASET',
'lifetime': -86400
}])
# Clean the rules on MOCK2. Replicas are tombstoned with non Epoch
rule_cleaner(once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse2_id)['tombstone'] is not None
undertaker(worker_number=1, total_workers=1, once=True)
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse1_id)['tombstone'] == datetime(year=1970,
month=1,
day=1)
for replica in replicas:
assert get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse2_id)['tombstone'] == datetime(year=1970,
month=1,
day=1)
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
if get_policy() != 'atlas':
LOG.info("Skipping atlas-specific test")
return
tmp_scope = InternalScope('mock', **self.vo)
jdoe = InternalAccount('jdoe', **self.vo)
root = InternalAccount('root', **self.vo)
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
rse_id = add_rse(rse, **self.vo)
set_local_account_limit(jdoe, rse_id, -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for _ in range(nbdatasets)]
add_dids(dids=dsns2, account=root)
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for _ in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert (get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'] is None)
for dsn in dsns2:
assert (get_did(scope=InternalScope('archive', **self.vo),
name=dsn['name'])['name'] == dsn['name'])
assert (len([
x for x in list_rules(
filters={
'scope': InternalScope('archive', **self.vo),
'name': dsn['name']
})
]) == 1)
def test_update_collection_replica(self):
""" REPLICA (CORE): Update collection replicas from update requests. """
file_size = 2
files = [{
'name': 'file_%s' % generate_uuid(),
'scope': self.scope,
'bytes': file_size
} for i in range(0, 2)]
dataset_name = 'dataset_test_%s' % generate_uuid()
add_replicas(rse_id=self.rse_id,
files=files,
account=self.account,
session=self.db_session)
add_dids([{
'scope': self.scope,
'name': dataset_name,
'type': constants.DIDType.DATASET
}],
account=self.account,
session=self.db_session)
attach_dids(scope=self.scope,
name=dataset_name,
dids=files,
account=self.account,
session=self.db_session)
models.CollectionReplica(rse_id=self.rse_id, scope=self.scope, state=constants.ReplicaState.AVAILABLE, name=dataset_name, did_type=constants.DIDType.DATASET, bytes=len(files) * file_size, length=len(files), available_replicas_cnt=0)\
.save(session=self.db_session)
# Update request with rse id
# First update -> dataset replica should be available
models.UpdatedCollectionReplica(
rse_id=self.rse_id,
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter_by(
rse_id=self.rse_id, scope=self.scope, name=dataset_name).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter_by(
id=update_request.id).first() # pylint: disable=no-member
assert update_request is None
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
# Delete one file replica -> dataset replica should be unavailable
delete_replicas(rse_id=self.rse_id,
files=[files[0]],
session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter_by(
rse_id=self.rse_id, scope=self.scope, name=dataset_name).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.UNAVAILABLE
# Add one file replica -> dataset replica should be available again
add_replicas(rse_id=self.rse_id,
files=[files[0]],
account=self.account,
session=self.db_session)
attach_dids(scope=self.scope,
name=dataset_name,
dids=[files[0]],
account=self.account,
session=self.db_session)
models.UpdatedCollectionReplica(
rse_id=self.rse_id,
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter_by(
rse_id=self.rse_id, scope=self.scope, name=dataset_name).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
# Old behaviour, open empty datasets are not deleted
# Delete all file replicas -> dataset replica should be deleted
delete_replicas(rse_id=self.rse_id,
files=files,
session=self.db_session)
with pytest.raises(NoResultFound):
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
# Update request without rse_id - using two replicas per file -> total 4 replicas
dataset_name = 'dataset_test_%s' % generate_uuid()
add_dids([{
'scope': self.scope,
'name': dataset_name,
'type': constants.DIDType.DATASET
}],
account=self.account,
session=self.db_session)
add_replicas(rse_id=self.rse_id,
files=files,
account=self.account,
session=self.db_session)
add_replicas(rse_id=self.rse2_id,
files=files,
account=self.account,
session=self.db_session)
attach_dids(scope=self.scope,
name=dataset_name,
dids=files,
account=self.account,
session=self.db_session)
models.CollectionReplica(
rse_id=self.rse_id,
scope=self.scope,
name=dataset_name,
state=constants.ReplicaState.UNAVAILABLE,
did_type=constants.DIDType.DATASET,
bytes=len(files) * file_size,
length=len(files)).save(session=self.db_session)
models.CollectionReplica(
rse_id=self.rse2_id,
scope=self.scope,
name=dataset_name,
state=constants.ReplicaState.UNAVAILABLE,
did_type=constants.DIDType.DATASET,
bytes=len(files) * file_size,
length=len(files)).save(session=self.db_session)
# First update -> replicas should be available
models.UpdatedCollectionReplica(
scope=self.scope, name=dataset_name).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter_by(
scope=self.scope, name=dataset_name).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
for dataset_replica in self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name).all(): # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
# Delete first replica on first RSE -> replica on first RSE should be unavailable, replica on second RSE should be still available
delete_replicas(rse_id=self.rse_id,
files=[files[0]],
session=self.db_session)
models.UpdatedCollectionReplica(
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
# delete_replica creates also update object but with rse_id -> extra filter for rse_id is NULL
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter(
models.UpdatedCollectionReplica.scope == self.scope,
models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member
models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.UNAVAILABLE
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse2_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
# Set the state of the first replica on the second RSE to UNAVAILABLE -> both replicass should be unavailable
file_replica = self.db_session.query(
models.RSEFileAssociation).filter_by(rse_id=self.rse2_id,
scope=self.scope,
name=files[0]['name']).one() # pylint: disable=no-member
file_replica.state = constants.ReplicaState.UNAVAILABLE
models.UpdatedCollectionReplica(
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter(
models.UpdatedCollectionReplica.scope == self.scope,
models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member
models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.UNAVAILABLE
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse2_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.UNAVAILABLE
# Delete first replica on second RSE -> file is not longer part of dataset -> both replicas should be available
delete_replicas(rse_id=self.rse2_id,
files=[files[0]],
session=self.db_session)
models.UpdatedCollectionReplica(
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter(
models.UpdatedCollectionReplica.scope == self.scope,
models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member
models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == (len(files) - 1) * file_size
assert dataset_replica['length'] == len(files) - 1
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.AVAILABLE
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse2_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == (len(files) - 1) * file_size
assert dataset_replica['length'] == len(files) - 1
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.AVAILABLE
# Add first replica to the first RSE -> first replicas should be available
add_replicas(rse_id=self.rse_id,
files=[files[0]],
account=self.account,
session=self.db_session)
attach_dids(scope=self.scope,
name=dataset_name,
dids=[files[0]],
account=self.account,
session=self.db_session)
models.UpdatedCollectionReplica(
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter(
models.UpdatedCollectionReplica.scope == self.scope,
models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member
models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse2_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == (len(files) -
1) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files) - 1
assert dataset_replica['state'] == ReplicaState.UNAVAILABLE
# Add first replica to the second RSE -> both replicas should be available again
add_replicas(rse_id=self.rse2_id,
files=[files[0]],
account=self.account,
session=self.db_session)
models.UpdatedCollectionReplica(
scope=self.scope,
name=dataset_name,
did_type=constants.DIDType.DATASET).save(session=self.db_session)
update_request = self.db_session.query(
models.UpdatedCollectionReplica).filter(
models.UpdatedCollectionReplica.scope == self.scope,
models.UpdatedCollectionReplica.name == dataset_name, # pylint: disable=no-member
models.UpdatedCollectionReplica.rse_id.is_(None)).one() # pylint: disable=no-member
update_collection_replica(update_request=update_request.to_dict(),
session=self.db_session)
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
dataset_replica = self.db_session.query(
models.CollectionReplica).filter_by(scope=self.scope,
name=dataset_name,
rse_id=self.rse2_id).one() # pylint: disable=no-member
assert dataset_replica['bytes'] == len(files) * file_size
assert dataset_replica['length'] == len(files)
assert dataset_replica['available_bytes'] == len(files) * file_size
assert dataset_replica['available_replicas_cnt'] == len(files)
assert dataset_replica['state'] == ReplicaState.AVAILABLE
