def test_add_rules_datasets_none(self): """ REPLICATION RULE (CORE): Add replication rules to multiple datasets, NONE Grouping""" scope = 'mock' files1 = create_files(3, scope, self.rse4) dataset1 = 'dataset_' + str(uuid()) add_did(scope, dataset1, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset1, files1, 'jdoe') files2 = create_files(3, scope, self.rse4) dataset2 = 'dataset_' + str(uuid()) add_did(scope, dataset2, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset2, files2, 'jdoe') # Add the rules to both DS add_rules(dids=[{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}], rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': self.T1, 'grouping': 'NONE', 'weight': None, 'lifetime': None, 'locked': False, 'subscription_id': None}, {'account': 'root', 'copies': 1, 'rse_expression': self.T1, 'grouping': 'NONE', 'weight': 'fakeweight', 'lifetime': None, 'locked': False, 'subscription_id': None}]) # Check if the Locks are created properly for file in files1: rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])] assert(rse_locks[0] == rse_locks[1]) for file in files2: rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])] assert(rse_locks[0] == rse_locks[1])
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK') set_account_limit('jdoe', rse_id, -1) dsns1 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1} for i in range(nbdatasets)] dsns2 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for i in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account='root') replicas = list() for dsn in dsns1 + dsns2: files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account='root') replicas += files add_rules(dids=dsns1, rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal(get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = InternalScope('mock') jdoe = InternalAccount('jdoe') root = InternalAccount('root') nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK') set_local_account_limit(jdoe, rse_id, -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for i in range(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account=root) # Add generic metadata on did test_metadata = {"test_key": "test_value"} try: add_did_meta(tmp_scope, dsns1[0]['name'], test_metadata) except NotImplementedError: # add_did_meta is not Implemented for Oracle < 12 pass replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files add_rules(dids=dsns1, rules=[{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal( get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK', **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for _ in range(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for _ in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account=root) # arbitrary keys do not work without JSON support (sqlite, Oracle < 12) if json_implemented(): # Add generic metadata on did set_metadata(tmp_scope, dsns1[0]['name'], "test_key", "test_value") replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for _ in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files add_rules(dids=dsns1, rules=[{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is not None
def test_removal_all_replicas2(rse_factory, root_account, mock_scope, core_config_mock, caches_mock): """ UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """ rse1, rse1_id = rse_factory.make_posix_rse() rse2, rse2_id = rse_factory.make_posix_rse() dst_rse_name, dst_rse_id = rse_factory.make_posix_rse() set_local_account_limit(root_account, rse1_id, -1) set_local_account_limit(root_account, rse2_id, -1) nbdatasets = 1 nbfiles = 5 dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1 } for _ in range(nbdatasets)] add_dids(dids=dsns1, account=root_account) replicas = list() for dsn in dsns1: files = [{ 'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb' } for _ in range(nbfiles)] attach_dids(scope=mock_scope, name=dsn['name'], rse_id=rse1_id, dids=files, account=root_account) add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True) replicas += files add_rules(dids=dsns1, rules=[{ 'account': root_account, 'copies': 1, 'rse_expression': rse1, 'grouping': 'DATASET' }]) add_rules(dids=dsns1, rules=[{ 'account': root_account, 'copies': 1, 'rse_expression': rse2, 'grouping': 'DATASET', 'lifetime': -86400 }]) # Clean the rules on MOCK2. Replicas are tombstoned with non Epoch rule_cleaner(once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] is not None undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse1_id)['tombstone'] == datetime(year=1970, month=1, day=1) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] == datetime(year=1970, month=1, day=1)
'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in xrange(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse='MOCK', dids=files, account='root') replicas += files add_rules(dids=dsns1, rules=[{ 'account': 'jdoe', 'copies': 1, 'rse_expression': 'MOCK', 'grouping': 'DATASET' }]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal( get_replica(scope=replica['scope'], name=replica['name'], rse='MOCK')['tombstone'], None) def test_list_expired_dids_with_locked_rules(self): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules"""