def test_api_did(self):
""" DID (API): Test external representation of DIDs """
# add some dids
ext_parent = did_name_generator('container')
ext_child = did_name_generator('dataset')
add_did(self.scope_name,
ext_parent,
'container',
issuer='root',
account=self.account_name,
**self.vo)
add_did(self.scope_name,
ext_child,
'dataset',
issuer='root',
account=self.account_name,
**self.vo)
attachment = {
'scope':
self.scope_name,
'name':
ext_parent,
'dids': [{
'scope': self.scope_name,
'name': ext_child,
'type': 'DATASET'
}]
}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
# test scope_list
out = scope_list(self.scope_name, recursive=True, **self.vo)
out = list(out)
assert 0 != len(out)
parent_found = False
for did in out:
assert did['scope'] == self.scope_name
if did['parent'] is not None:
parent_found = True
assert did['parent']['scope'] == self.scope_name
assert parent_found
# test get_did
add_did_to_followed(self.scope_name, ext_parent, self.account_name,
**self.vo)
out = get_users_following_did(ext_parent, self.scope_name, **self.vo)
out = list(out)
assert 0 != len(out)
for user in out:
assert user['user'] == self.account_name
def test_dirac_addfile(rse_factory, did_factory, root_account, did_client, dirac_client, rse_client, replica_client):
""" DIRAC (CLIENT): Test the functionality of the addfile method """
nbfiles = 5
rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
rse_client.add_rse_attribute(rse=rse1, key='ANY', value='True')
# Create replicas on rse1 using addfile
lfns = [{'lfn': did_name_generator('file'), 'rse': rse1, 'bytes': 1, 'adler32': '0cc737eb', 'guid': generate_uuid()} for _ in range(nbfiles)]
files = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn']} for lfn in lfns]
reps = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn'], 'rse': rse1} for lfn in lfns]
dirac_client.add_files(lfns=lfns, ignore_availability=False)
replicas = [rep for rep in replica_client.list_replicas(dids=files)]
for replica in replicas:
assert {'scope': replica['scope'], 'name': replica['name'], 'rse': list(replica['rses'].keys())[0]} in reps
# Check the existence of all parents from the files
for lfn in lfns:
directories = lfn['lfn'].split('/')
for cnt, directory in enumerate(directories):
parent = "/".join(directories[0:cnt])
child = "/".join(directories[0:cnt + 1])
if parent != '':
parent_scope, parent_name = extract_scope(parent, [])
children = [did['name'] for did in did_client.list_content(parent_scope, parent_name)]
assert child in children
# Check that the default rules are created
for lfn in lfns:
# Check dataset rule
directory = "/".join(lfn['lfn'].split('/')[:-1])
scope, name = extract_scope(directory, [])
rules = [rule for rule in did_client.list_did_rules(scope, name)]
assert len(rules) == 1
assert rules[0]['rse_expression'] == 'ANY=true'
assert rules[0]['expires_at'] is None
# Check file rule
scope, name = extract_scope(lfn['lfn'], [])
rules = [rule for rule in did_client.list_did_rules(scope, name)]
assert len(rules) == 1
assert rules[0]['rse_expression'] == rse1
assert datetime.now() - rules[0]['expires_at'] < timedelta(hours=24)
def test_api_subscription(self):
""" SUBSCRIPTION (API): Test external representation of subscriptions """
sub = 'ext_' + generate_uuid()
did = did_name_generator('file')
new_acc_name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
new_scope_name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
add_account(new_acc_name, 'USER', '*****@*****.**', 'root',
**self.new_vo)
add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10,
'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10,
'root', **self.new_vo)
add_did(new_scope_name,
did,
'DATASET',
'root',
account=new_acc_name,
rse=self.rse3_name,
**self.new_vo)
sub_id = add_subscription(sub, new_acc_name, {
'account': [new_acc_name],
'scope': [new_scope_name]
}, [{
'copies': 1,
'rse_expression': self.rse3_name,
'weight': 0,
'activity': 'Functional Test',
'source_replica_expression': self.rse4_name
}], '', False, 0, 0, 3, 'root', **self.new_vo)
add_replication_rule(dids=[{
'scope': new_scope_name,
'name': did
}],
copies=1,
rse_expression=self.rse3_name,
weight=None,
lifetime=180,
grouping='DATASET',
account=new_acc_name,
locked=False,
subscription_id=sub_id,
source_replica_expression=self.rse4_name,
activity='Functional Test',
notify=None,
purge_replicas=False,
ignore_availability=False,
comment='',
ask_approval=False,
asynchronous=False,
delay_injection=None,
priority=0,
split_container=False,
meta='',
issuer='root',
**self.new_vo)
out = list_subscriptions(sub, **self.new_vo)
out = list(out)
assert 0 != len(out)
assert sub_id in [o['id'] for o in out]
for o in out:
if o['id'] == sub_id:
assert o['account'] == new_acc_name
rules = loads(o['replication_rules'])[0]
assert rules['rse_expression'] == self.rse3_name
assert rules['source_replica_expression'] == self.rse4_name
fil = loads(o['filter'])
assert fil['account'] == [new_acc_name]
assert fil['scope'] == [new_scope_name]
out = list_subscription_rule_states(sub, **self.new_vo)
out = list(out)
assert 0 != len(out)
for o in out:
assert o.account == new_acc_name
out = get_subscription_by_id(sub_id, **self.new_vo)
assert out['account'] == new_acc_name
rules = loads(out['replication_rules'])[0]
assert rules['rse_expression'] == self.rse3_name
assert rules['source_replica_expression'] == self.rse4_name
fil = loads(out['filter'])
assert fil['account'] == [new_acc_name]
assert fil['scope'] == [new_scope_name]
def test_api_rse(self):
""" RSE (API): Test external representation of RSEs """
out = api_rse.get_rse(self.rse_name, **self.vo)
assert out['rse'] == self.rse_name
assert out['id'] == self.rse_id
out = api_rse.list_rses(**self.new_vo)
out = list(out)
assert 0 != len(out)
rse_ids = [rse['id'] for rse in out]
assert self.rse3_id in rse_ids
assert self.rse4_id in rse_ids
for rse in out:
assert 'rse' in rse
if rse['id'] == self.rse3_id:
assert rse['rse'] == self.rse3_name
elif rse['id'] == self.rse4_id:
assert rse['rse'] == self.rse4_name
key = "KEY_" + generate_uuid()
api_rse.add_rse_attribute(self.rse_name,
key,
1,
issuer='root',
**self.vo)
out = api_rse.get_rses_with_attribute(key)
out = list(out)
assert 0 != len(out)
for rse in out:
assert rse['rse'] == self.rse_name
out = api_rse.get_rse_protocols(self.rse_name,
issuer='root',
**self.vo)
assert out['rse'] == self.rse_name
# add some account and RSE counters
rse_mock = 'MOCK4'
rse_mock_id = get_rse_id(rse_mock, **self.vo)
account_counter.del_counter(rse_id=rse_mock_id, account=self.account)
account_counter.add_counter(rse_id=rse_mock_id, account=self.account)
account_counter.increase(rse_id=rse_mock_id,
account=self.account,
files=1,
bytes_=10)
account_counter.update_account_counter(self.account, rse_mock_id)
did = did_name_generator('file')
add_did(self.scope_name,
did,
'DATASET',
'root',
account=self.account_name,
rse=rse_mock,
**self.vo)
abacus_rse.run(once=True)
out = api_rse.get_rse_usage(rse_mock,
per_account=True,
issuer='root',
**self.vo)
assert rse_mock_id in [o['rse_id'] for o in out]
for usage in out:
if usage['rse_id'] == rse_mock_id:
assert usage['rse'] == rse_mock
accounts = [u['account'] for u in usage['account_usages']]
assert self.account_name in accounts
if self.multi_vo:
assert self.account.internal not in accounts
# clean up files
cleaner.run(once=True)
if self.multi_vo:
reaper.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock),
greedy=True)
else:
reaper.run(once=True, include_rses=rse_mock, greedy=True)
abacus_rse.run(once=True)
out = api_rse.parse_rse_expression(
'%s|%s' % (self.rse_name, self.rse2_name), **self.vo)
assert self.rse_name in out
assert self.rse2_name in out
assert self.rse_id not in out
assert self.rse2_id not in out
def test_api_request(self):
""" REQUEST (API): Test external representation of requests """
did = did_name_generator('dataset')
add_did(self.scope_name,
did,
'dataset',
issuer='root',
account=self.account_name,
rse=self.rse_name,
**self.vo)
requests = [{
'dest_rse_id': self.rse2_id,
'source_rse_id': self.rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': did,
'scope': self.scope_name,
'account': self.account_name,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now(),
'attributes': {
'activity': 'Functional Test',
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
reqs = queue_requests(
requests, issuer='root',
**self.vo) # this does not pass in the source rse
reqs = list(reqs)
assert 0 != len(reqs)
for r in reqs:
assert r['scope'] == self.scope_name
assert r['account'] == self.account_name
assert r['source_rse'] == self.rse_name
assert r['dest_rse'] == self.rse2_name
out = get_request_by_did(self.scope_name,
did,
self.rse2_name,
issuer='root',
**self.vo)
assert out['scope'] == self.scope_name
assert out['account'] == self.account_name
assert out['dest_rse'] == self.rse2_name
assert out['source_rse'] == self.rse_name
out = list_requests([self.rse_name], [self.rse2_name],
[constants.RequestState.QUEUED],
issuer='root',
**self.vo)
out = list(out)
assert 0 != len(out)
assert self.scope_name in [req['scope'] for req in out]
for req in out:
if req['scope'] == self.scope_name:
assert req['scope'] == self.scope_name
assert req['account'] == self.account_name
assert req['dest_rse'] == self.rse2_name
assert req['source_rse'] == self.rse_name
def test_api_replica(self):
""" REPLICA (API): Test external representation of replicas """
did = did_name_generator('file')
did_parent = did_name_generator('dataset')
pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (
self.scope_name, generate_uuid())
add_replicas(self.rse2_name,
files=[{
'scope': self.scope_name,
'name': did,
'bytes': 100,
'pfn': pfn
}],
issuer='root',
**self.vo)
add_did(self.scope_name,
did_parent,
'dataset',
issuer='root',
account=self.account_name,
**self.vo)
attachment = {
'scope': self.scope_name,
'name': did_parent,
'dids': [{
'scope': self.scope_name,
'name': did
}]
}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
out = get_did_from_pfns([pfn], self.rse2_name, **self.vo)
out = list(out)
assert 0 != len(out)
did_found = False
for p in out:
for key in p:
if p[key]['name'] == did:
did_found = True
assert self.scope_name == p[key]['scope']
assert did_found
out = list_replicas(dids=[{
'scope': self.scope_name,
'name': did
}],
resolve_parents=True,
**self.vo)
out = list(out)
assert 0 != len(out)
parents_found = False
for rep in out:
assert rep['scope'] == self.scope_name
if 'parents' in rep:
parents_found = True
for parent in rep['parents']:
assert self.scope_name in parent
if self.multi_vo:
assert self.scope.internal not in parent
assert parents_found