def test_list_dataset_replicas(self): """ REPLICA (CLIENT): List dataset replicas.""" replica_client = ReplicaClient() rule_client = RuleClient() did_client = DIDClient() scope = 'mock' dataset = 'dataset_' + str(generate_uuid()) did_client.add_dataset(scope=scope, name=dataset) rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='root', copies=1, rse_expression='MOCK', grouping='DATASET') replicas = [r for r in replica_client.list_dataset_replicas(scope=scope, name=dataset)] assert len(replicas) == 1
def test_list_dataset_replicas_archive(self): """ REPLICA (CLIENT): List dataset replicas with archives. """ replica_client = ReplicaClient() did_client = DIDClient() rule_client = RuleClient() scope = 'mock' rse = 'APERTURE_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) add_protocol(rse_id=rse_id, parameter={ 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) rse2 = 'BLACKMESA_%s' % rse_name_generator() rse2_id = add_rse(rse2, **self.vo) add_protocol(rse_id=rse2_id, parameter={ 'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//underground/facility', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) # register archive archive = { 'scope': scope, 'name': 'another.%s.zip' % generate_uuid(), 'type': 'FILE', 'bytes': 2596, 'adler32': 'deedbeaf' } replica_client.add_replicas(rse=rse, files=[archive]) replica_client.add_replicas(rse=rse2, files=[archive]) archived_files = [{ 'scope': scope, 'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 4322, 'adler32': 'deaddead' } for i in range(2)] replica_client.add_replicas(rse=rse2, files=archived_files) did_client.add_files_to_archive(scope=scope, name=archive['name'], files=archived_files) dataset_name = 'find_me.' + str(generate_uuid()) did_client.add_dataset(scope=scope, name=dataset_name) did_client.attach_dids(scope=scope, name=dataset_name, dids=archived_files) rule_client.add_replication_rule(dids=[{ 'scope': scope, 'name': dataset_name }], account='root', copies=1, rse_expression=rse, grouping='DATASET') res = [ r for r in replica_client.list_dataset_replicas(scope=scope, name=dataset_name) ] assert len(res) == 1 assert res[0]['state'] == 'UNAVAILABLE' res = [ r for r in replica_client.list_dataset_replicas( scope=scope, name=dataset_name, deep=True) ] assert len(res) == 3 assert res[0]['state'] == 'AVAILABLE' assert res[1]['state'] == 'AVAILABLE' assert res[2]['state'] == 'AVAILABLE' del_rse(rse_id)
class TestAbacusCollectionReplica(): def setUp(self): self.account = 'root' self.scope = 'mock' self.rule_client = RuleClient() self.did_client = DIDClient() self.replica_client = ReplicaClient() self.upload_client = UploadClient() self.file_sizes = 2 self.dataset = 'dataset_%s' % generate_uuid() self.rse = 'MOCK5' self.rse_id = get_rse_id(rse=self.rse) def tearDown(self): undertaker.run(once=True) cleaner.run(once=True) reaper.run(once=True, rses=[self.rse], greedy=True) def test_abacus_collection_replica(self): """ ABACUS (COLLECTION REPLICA): Test update of collection replica. """ self.files = [{'did_scope': self.scope, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for i in range(0, 2)] self.did_client.add_did(self.scope, self.dataset, DIDType.DATASET, lifetime=-1) self.upload_client.upload(self.files) self.did_client.attach_dids(scope=self.scope, name=self.dataset, dids=[{'name': file['did_name'], 'scope': file['did_scope']} for file in self.files]) self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1) [os.remove(file['path']) for file in self.files] # Check dataset replica after rule creation - initial data dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)][0] assert_equal(dataset_replica['bytes'], 0) assert_equal(dataset_replica['length'], 0) assert_equal(dataset_replica['available_bytes'], 0) assert_equal(dataset_replica['available_length'], 0) assert_equal(str(dataset_replica['state']), 'UNAVAILABLE') # Run Abacus collection_replica.run(once=True) # Check dataset replica after abacus - abacus should update the collection_replica table from updated_col_rep dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)][0] assert_equal(dataset_replica['bytes'], len(self.files) * self.file_sizes) assert_equal(dataset_replica['length'], len(self.files)) assert_equal(dataset_replica['available_bytes'], len(self.files) * self.file_sizes) assert_equal(dataset_replica['available_length'], len(self.files)) assert_equal(str(dataset_replica['state']), 'AVAILABLE') # Delete one file -> collection replica should be unavailable cleaner.run(once=True) delete_replicas(rse_id=self.rse_id, files=[{'name': self.files[0]['did_name'], 'scope': InternalScope(self.files[0]['did_scope'])}]) self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1) collection_replica.run(once=True) dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)][0] assert_equal(dataset_replica['length'], len(self.files)) assert_equal(dataset_replica['bytes'], len(self.files) * self.file_sizes) assert_equal(dataset_replica['available_length'], len(self.files) - 1) assert_equal(dataset_replica['available_bytes'], (len(self.files) - 1) * self.file_sizes) assert_equal(str(dataset_replica['state']), 'UNAVAILABLE') # Delete all files -> collection replica should be deleted cleaner.run(once=True) reaper.run(once=True, rses=[self.rse], greedy=True) self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1) collection_replica.run(once=True) dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)] assert_equal(len(dataset_replica), 0)
class TestAbacusCollectionReplica(unittest.TestCase): def setUp(self): self.account = 'root' self.scope = 'mock' self.rse = 'MOCK5' self.file_sizes = 2 self.dataset = 'dataset_%s' % generate_uuid() self.rule_client = RuleClient() self.did_client = DIDClient() self.replica_client = ReplicaClient() self.upload_client = UploadClient() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = { 'vo': config_get('client', 'vo', raise_exception=False, default='tst') } else: self.vo = {} self.rse_id = get_rse_id(rse=self.rse, **self.vo) def tearDown(self): undertaker.run(once=True) cleaner.run(once=True) if self.vo: reaper.run(once=True, include_rses='vo=%s&(%s)' % (self.vo['vo'], self.rse), greedy=True) else: reaper.run(once=True, include_rses=self.rse, greedy=True) def test_abacus_collection_replica(self): """ ABACUS (COLLECTION REPLICA): Test update of collection replica. """ self.files = [{ 'did_scope': self.scope, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1 } for i in range(0, 2)] self.did_client.add_did(self.scope, self.dataset, DIDType.DATASET, lifetime=-1) self.upload_client.upload(self.files) self.did_client.attach_dids(scope=self.scope, name=self.dataset, dids=[{ 'name': file['did_name'], 'scope': file['did_scope'] } for file in self.files]) self.rule_client.add_replication_rule([{ 'scope': self.scope, 'name': self.dataset }], 1, self.rse, lifetime=-1) [os.remove(file['path']) for file in self.files] # Check dataset replica after rule creation - initial data dataset_replica = [ replica for replica in self.replica_client.list_dataset_replicas( self.scope, self.dataset) ][0] assert dataset_replica['bytes'] == 0 assert dataset_replica['length'] == 0 assert dataset_replica['available_bytes'] == 0 assert dataset_replica['available_length'] == 0 assert str(dataset_replica['state']) == 'UNAVAILABLE' # Run Abacus collection_replica.run(once=True) # Check dataset replica after abacus - abacus should update the collection_replica table from updated_col_rep dataset_replica = [ replica for replica in self.replica_client.list_dataset_replicas( self.scope, self.dataset) ][0] assert dataset_replica['bytes'] == len(self.files) * self.file_sizes assert dataset_replica['length'] == len(self.files) assert dataset_replica['available_bytes'] == len( self.files) * self.file_sizes assert dataset_replica['available_length'] == len(self.files) assert str(dataset_replica['state']) == 'AVAILABLE' # Delete one file -> collection replica should be unavailable cleaner.run(once=True) delete_replicas(rse_id=self.rse_id, files=[{ 'name': self.files[0]['did_name'], 'scope': InternalScope(self.files[0]['did_scope'], **self.vo) }]) self.rule_client.add_replication_rule([{ 'scope': self.scope, 'name': self.dataset }], 1, self.rse, lifetime=-1) collection_replica.run(once=True) dataset_replica = [ replica for replica in self.replica_client.list_dataset_replicas( self.scope, self.dataset) ][0] assert dataset_replica['length'] == len(self.files) assert dataset_replica['bytes'] == len(self.files) * self.file_sizes assert dataset_replica['available_length'] == len(self.files) - 1 assert dataset_replica['available_bytes'] == (len(self.files) - 1) * self.file_sizes assert str(dataset_replica['state']) == 'UNAVAILABLE' # Delete all files -> collection replica should be deleted cleaner.run(once=True) if self.vo: reaper.run(once=True, include_rses='vo=%s&(%s)' % (self.vo['vo'], self.rse), greedy=True) else: reaper.run(once=True, include_rses=self.rse, greedy=True) self.rule_client.add_replication_rule([{ 'scope': self.scope, 'name': self.dataset }], 1, self.rse, lifetime=-1) collection_replica.run(once=True) dataset_replica = [ replica for replica in self.replica_client.list_dataset_replicas( self.scope, self.dataset) ] assert len(dataset_replica) == 0