def test_update_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers and update the flag new """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
dids = []
for i in xrange(0, 5):
d = {
'scope': tmp_scope,
'name': '%s-%i' % (tmp_dsn, i),
'did_type': DIDType.DATASET
}
did.add_did(scope=tmp_scope,
name='%s-%i' % (tmp_dsn, i),
type='DATASET',
issuer='root')
dids.append(d)
st = did.set_new_dids(dids, None)
assert_true(st)
with assert_raises(DataIdentifierNotFound):
did.set_new_dids([{
'scope': 'dummyscope',
'name': 'dummyname',
'did_type': DIDType.DATASET
}], None)
def test_api_did(self):
""" DID (API): Test external representation of DIDs """
# add some dids
add_did(self.scope_name, 'ext_parent', 'container', issuer='root', account=self.account_name, **self.vo)
add_did(self.scope_name, 'ext_child', 'dataset', issuer='root', account=self.account_name, **self.vo)
attachment = {'scope': self.scope_name, 'name': 'ext_parent',
'dids': [{'scope': self.scope_name, 'name': 'ext_child', 'type': 'DATASET'}]}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
# test scope_list
out = scope_list(self.scope_name, recursive=True, **self.vo)
out = list(out)
assert_not_equal(0, len(out))
parent_found = False
for did in out:
assert_equal(did['scope'], self.scope_name)
if did['parent'] is not None:
parent_found = True
assert_equal(did['parent']['scope'], self.scope_name)
assert_true(parent_found)
# test get_did
add_did_to_followed(self.scope_name, 'ext_parent', self.account_name, **self.vo)
out = get_users_following_did('ext_parent', self.scope_name, **self.vo)
out = list(out)
assert_not_equal(0, len(out))
for user in out:
assert_equal(user['user'], self.account_name)
def register_container(self):
self.container_exists = False
if self.is_at_pnn and self.dry_run:
logging.info('Dry Run: Create container %s in scope %s.',
self.container, self.scope)
self.container_exists = True
return self.container_exists
try:
get_did(scope=self.scope, name=self.container)
monitor.record_counter('cms_sync.container_exists')
self.container_exists = True
logging.info('Found container %s', self.container)
except DataIdentifierNotFound:
if self.is_at_pnn:
try:
logging.info('Create container %s in scope %s.',
self.container, self.scope)
add_did(scope=self.scope,
name=self.container,
type='CONTAINER',
issuer=self.account,
lifetime=self.lifetime)
monitor.record_counter('cms_sync.container_created')
self.container_exists = True
logging.info('Created container %s in scope %s.',
self.container, self.scope)
except DataIdentifierAlreadyExists:
logging.warning('Container was created in the meanwhile')
monitor.record_counter('cms_sync.container_collision')
self.container_exists = True
else:
logging.warning('Container was not at PNN')
return self.container_exists
def register_block(self):
"""
Register the dataset (if there is a replica at the pnn) and attach to container
:dry: Dry run. Default false.
"""
# FIXME: The logic here could use some improvement as we try to create a block even if it exists already
try:
get_did(scope=self.scope, name=self.block_name)
self.block_exists = True
monitor.record_counter('cms_sync.dataset_exists')
except DataIdentifierNotFound:
self.block_exists = False
if self.is_at_pnn and self.dry_run:
logging.info('Dry Run: Create dataset %s in scope %s.',
self.block_name, self.scope)
self.block_exists = True
elif self.is_at_pnn:
logging.info('Create block %s in scope %s.', self.block_name,
self.scope)
try:
if not self.block_exists:
add_did(scope=self.scope,
name=self.block_name,
type='DATASET',
issuer=self.account,
lifetime=self.lifetime)
monitor.record_counter('cms_sync.dataset_created')
except DataIdentifierAlreadyExists:
logging.warning('Attempt to add %s:%s failed, already exists.',
self.scope, self.block_name)
monitor.record_counter('cms_sync.dataset_collision')
try:
attach_dids(scope=self.scope,
name=self.container,
attachment={
'dids': [{
'scope': self.scope,
'name': self.block_name
}]
},
issuer=self.account)
except DuplicateContent:
logging.warning(
'Attempt to add %s:%s to %s failed, already exists.',
self.scope, self.block_name, self.container)
except DataIdentifierNotFound:
logging.error(
'Attempt to add %s:%s to %s failed. Container does not exist.',
self.scope, self.block_name, self.container)
return False
self.block_exists = True
else:
logging.warning('Block %s was not at PNN', self.block_name)
return self.block_exists
def POST(self, scope, name):
"""
Create a new data identifier.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:param scope: Create the data identifier within this scope.
:param name: Create the data identifier with this name.
"""
statuses, meta, rules, lifetime, dids, rse = {}, {}, [], None, [], None
try:
json_data = loads(data())
type = json_data['type']
if 'statuses' in json_data:
statuses = json_data['statuses']
if 'meta' in json_data:
meta = json_data['meta']
if 'rules' in json_data:
rules = json_data['rules']
if 'lifetime' in json_data:
lifetime = json_data['lifetime']
if 'dids' in json_data:
dids = json_data['dids']
if 'rse' in json_data:
rse = json_data['rse']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
except KeyError as error:
raise generate_http_error(400, 'ValueError', str(error))
try:
add_did(scope=scope, name=name, type=type, statuses=statuses, meta=meta, rules=rules, lifetime=lifetime, dids=dids, rse=rse, issuer=ctx.env.get('issuer'))
except DataIdentifierNotFound as error:
raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
except DuplicateContent as error:
raise generate_http_error(409, 'DuplicateContent', error.args[0])
except DataIdentifierAlreadyExists as error:
raise generate_http_error(409, 'DataIdentifierAlreadyExists', error.args[0])
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except UnsupportedOperation as error:
raise generate_http_error(409, 'UnsupportedOperation', error.args[0])
except DatabaseException as error:
raise generate_http_error(500, 'DatabaseException', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created()
def register_and_attach_did(self, scope='cms', name=None, did_type=None, parent_did=None):
existed = False
created = False
attached = False
already_attached = False
try:
get_did(scope=scope, name=name)
existed = True
logging.info('DID existed: %s %s:%s', did_type, scope, name)
except DataIdentifierNotFound:
try:
add_did(scope=scope, name=name, type=did_type,
issuer=self.account, lifetime=self.lifetime)
created = True
logging.info('Created DID: %s %s:%s', did_type, scope, name)
except:
logging.critical('Attempt to add %s:%s failed. Unknown', scope, name)
logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created,
attached)
except:
logging.critical('Attempt to get %s:%s failed. Unknown', scope, name)
logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created,
attached)
if parent_did:
try:
attach_dids(scope=scope, name=parent_did,
attachment={'dids': [{'scope': scope, 'name': name}]}, issuer=self.account)
logging.info('Attached %s to %s', name, parent_did)
attached = True
except DuplicateContent:
logging.warning('Attempt to add %s:%s to %s failed, already exists.',
self.scope, self.block_name, self.container)
attached = True
already_attached = True
except DataIdentifierNotFound:
logging.critical('Attempt to add %s to %s failed. Parent does not exist.',
self.block_name, self.container)
logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed,
created,
attached)
except:
logging.critical('Attempt to attach %s to %s failed. Unknown', name, parent_did)
logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed,
created,
attached)
self.block_exists = True
return existed, created, attached, already_attached
def test_api_subscription(self):
""" SUBSCRIPTION (API): Test external representation of subscriptions """
sub = 'ext_' + generate_uuid()
did = 'ext_' + generate_uuid()
new_acc_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
new_scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
add_account(new_acc_name, 'USER', '*****@*****.**', 'root', **self.new_vo)
add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10, 'root', **self.new_vo)
add_did(new_scope_name, did, 'DATASET', 'root', account=new_acc_name, rse=self.rse3_name, **self.new_vo)
sub_id = add_subscription(sub, new_acc_name, {'account': [new_acc_name], 'scope': [new_scope_name]},
[{'copies': 1, 'rse_expression': self.rse3_name, 'weight': 0, 'activity': 'User Subscriptions',
'source_replica_expression': self.rse4_name}],
'', False, 0, 0, 3, 'root', **self.new_vo)
add_replication_rule(dids=[{'scope': new_scope_name, 'name': did}], copies=1, rse_expression=self.rse3_name, weight=None,
lifetime=180, grouping='DATASET', account=new_acc_name, locked=False, subscription_id=sub_id,
source_replica_expression=self.rse4_name, activity='User Subscriptions', notify=None,
purge_replicas=False, ignore_availability=False, comment='', ask_approval=False, asynchronous=False,
priority=0, split_container=False, meta='', issuer='root', **self.new_vo)
out = list_subscriptions(sub, **self.new_vo)
out = list(out)
assert_not_equal(0, len(out))
assert_in(sub_id, [o['id'] for o in out])
for o in out:
if o['id'] == sub_id:
assert_equal(o['account'], new_acc_name)
rules = loads(o['replication_rules'])[0]
assert_equal(rules['rse_expression'], self.rse3_name)
assert_equal(rules['source_replica_expression'], self.rse4_name)
fil = loads(o['filter'])
assert_equal(fil['account'], [new_acc_name])
assert_equal(fil['scope'], [new_scope_name])
out = list_subscription_rule_states(sub, **self.new_vo)
out = list(out)
assert_not_equal(0, len(out))
for o in out:
assert_equal(o.account, new_acc_name)
out = get_subscription_by_id(sub_id, **self.new_vo)
assert_equal(out['account'], new_acc_name)
rules = loads(out['replication_rules'])[0]
assert_equal(rules['rse_expression'], self.rse3_name)
assert_equal(rules['source_replica_expression'], self.rse4_name)
fil = loads(out['filter'])
assert_equal(fil['account'], [new_acc_name])
assert_equal(fil['scope'], [new_scope_name])
def test_api_request(self):
""" REQUEST (API): Test external representation of requests """
did = generate_uuid()
add_did(self.scope_name, did, 'dataset', issuer='root', account=self.account_name, rse=self.rse_name, **self.vo)
requests = [{
'dest_rse_id': self.rse2_id,
'source_rse_id': self.rse_id,
'request_type': constants.RequestType.TRANSFER,
'request_id': generate_uuid(),
'name': did,
'scope': self.scope_name,
'account': self.account_name,
'rule_id': generate_uuid(),
'retry_count': 1,
'requested_at': datetime.now(),
'attributes': {
'activity': 'User Subscription',
'bytes': 10,
'md5': '',
'adler32': ''
}
}]
reqs = queue_requests(requests, issuer='root', **self.vo) # this does not pass in the source rse
reqs = list(reqs)
assert_not_equal(0, len(reqs))
for r in reqs:
assert_equal(r['scope'], self.scope_name)
assert_equal(r['account'], self.account_name)
assert_equal(r['source_rse'], self.rse_name)
assert_equal(r['dest_rse'], self.rse2_name)
out = get_request_by_did(self.scope_name, did, self.rse2_name, issuer='root', **self.vo)
assert_equal(out['scope'], self.scope_name)
assert_equal(out['account'], self.account_name)
assert_equal(out['dest_rse'], self.rse2_name)
assert_equal(out['source_rse'], self.rse_name)
out = list_requests([self.rse_name], [self.rse2_name], [constants.RequestState.QUEUED], issuer='root', **self.vo)
out = list(out)
assert_not_equal(0, len(out))
assert_in(self.scope_name, [req['scope'] for req in out])
for req in out:
if req['scope'] == self.scope_name:
assert_equal(req['scope'], self.scope_name)
assert_equal(req['account'], self.account_name)
assert_equal(req['dest_rse'], self.rse2_name)
assert_equal(req['source_rse'], self.rse_name)
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
for i in xrange(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
for i in did.list_new_dids('DATASET'):
assert_not_equal(i, {})
assert_equal(str(i['did_type']), 'DATASET')
break
for i in did.list_new_dids():
assert_not_equal(i, {})
break
def test_update_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers and update the flag new """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
dids = []
for i in xrange(0, 5):
d = {'scope': tmp_scope, 'name': '%s-%i' % (tmp_dsn, i), 'did_type': DIDType.DATASET}
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
dids.append(d)
st = did.set_new_dids(dids, None)
assert_true(st)
with assert_raises(DataIdentifierNotFound):
did.set_new_dids([{'scope': 'dummyscope', 'name': 'dummyname', 'did_type': DIDType.DATASET}], None)
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe', **self.vo)
for i in range(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root', **self.vo)
for i in did.list_new_dids('DATASET', **self.vo):
assert i != {}
assert str(i['did_type']) == 'DATASET'
break
for i in did.list_new_dids(**self.vo):
assert i != {}
break
def test_api_did(self):
""" DID (API): Test external representation of DIDs """
# add some dids
ext_parent = did_name_generator('container')
ext_child = did_name_generator('dataset')
add_did(self.scope_name,
ext_parent,
'container',
issuer='root',
account=self.account_name,
**self.vo)
add_did(self.scope_name,
ext_child,
'dataset',
issuer='root',
account=self.account_name,
**self.vo)
attachment = {
'scope':
self.scope_name,
'name':
ext_parent,
'dids': [{
'scope': self.scope_name,
'name': ext_child,
'type': 'DATASET'
}]
}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
# test scope_list
out = scope_list(self.scope_name, recursive=True, **self.vo)
out = list(out)
assert 0 != len(out)
parent_found = False
for did in out:
assert did['scope'] == self.scope_name
if did['parent'] is not None:
parent_found = True
assert did['parent']['scope'] == self.scope_name
assert parent_found
# test get_did
add_did_to_followed(self.scope_name, ext_parent, self.account_name,
**self.vo)
out = get_users_following_did(ext_parent, self.scope_name, **self.vo)
out = list(out)
assert 0 != len(out)
for user in out:
assert user['user'] == self.account_name
def test_list_new_dids(self):
""" DATA IDENTIFIERS (API): List new identifiers """
tmp_scope = scope_name_generator()
tmp_dsn = 'dsn_%s' % generate_uuid()
scope.add_scope(tmp_scope, 'jdoe', 'jdoe')
for i in range(0, 5):
did.add_did(scope=tmp_scope, name='%s-%i' % (tmp_dsn, i), type='DATASET', issuer='root')
for i in did.list_new_dids('DATASET'):
assert_not_equal(i, {})
assert_equal(str(i['did_type']), 'DATASET')
break
for i in did.list_new_dids():
assert_not_equal(i, {})
break
def test_dids_at_different_vos(self):
""" MULTI VO (CLIENT): Test that dids from 2nd vo don't interfere """
scope_uuid = str(generate_uuid()).lower()[:16]
scope = 'shr_%s' % scope_uuid
add_scope(scope, 'root', 'root', **self.vo)
add_scope(scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst = 'tstset_%s' % did_uuid
new = 'newset_%s' % did_uuid
shr = 'shrset_%s' % did_uuid
did_client.add_did(scope, tst, 'DATASET')
did_client.add_did(scope, shr, 'DATASET')
add_did(scope, new, 'DATASET', 'root', **self.new_vo)
add_did(scope, shr, 'DATASET', 'root', **self.new_vo)
did_list_tst = list(did_client.list_dids(scope, {}))
did_list_new = list(list_dids(scope, {}, **self.new_vo))
assert_true(tst in did_list_tst)
assert_false(new in did_list_tst)
assert_true(shr in did_list_tst)
assert_false(tst in did_list_new)
assert_true(new in did_list_new)
assert_true(shr in did_list_new)
def test_api_replica(self):
""" REPLICA (API): Test external representation of replicas """
did = 'ext_' + str(generate_uuid())
pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (self.scope_name, generate_uuid())
add_replicas(self.rse2_name, files=[{'scope': self.scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', **self.vo)
add_did(self.scope_name, 'ext_parent_2', 'dataset', issuer='root', account=self.account_name, **self.vo)
attachment = {'scope': self.scope_name, 'name': 'ext_parent_2',
'dids': [{'scope': self.scope_name, 'name': did}]}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
out = get_did_from_pfns([pfn], self.rse2_name, **self.vo)
out = list(out)
assert_not_equal(0, len(out))
did_found = False
for p in out:
for key in p:
if p[key]['name'] == did:
did_found = True
assert_equal(self.scope_name, p[key]['scope'])
assert_true(did_found)
out = list_replicas(dids=[{'scope': self.scope_name, 'name': did}], resolve_parents=True, **self.vo)
out = list(out)
assert_not_equal(0, len(out))
parents_found = False
for rep in out:
assert_equal(rep['scope'], self.scope_name)
if 'parents' in rep:
parents_found = True
for parent in rep['parents']:
assert_in(self.scope_name, parent)
if self.multi_vo:
assert_not_in(self.scope.internal, parent)
assert_true(parents_found)
def post(self, scope_name):
"""
Create a new data identifier.
.. :quickref: DIDs; Create a new DID.
.. sourcecode:: http
POST /dids/scope1/container1 HTTP/1.1
Host: rucio.cern.ch
{"type": "CONTAINER", "lifetime": 86400},
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:reqheader Accept: application/json
:param scope_name: data identifier (scope)/(name).
:<json string type: the new DID type
:<json dict statuses: Dictionary with statuses, e.g. {'monotonic':True}
:<json dict meta: Dictionary with metadata, e.g. {'length':1234}
:<json dict rules: Replication rules associated with the did. e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]
:<json int lifetime: DID's liftime in seconds.
:<json list dids: The content.
:<json string rse: The RSE name when registering replicas.
:status 201: new DIDs created
:status 401: Invalid Auth Token
:status 409: DID already exists
"""
try:
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
parameters = json_parameters()
type_param = param_get(parameters, 'type')
try:
add_did(
scope=scope,
name=name,
type=type_param,
statuses=param_get(parameters, 'statuses', default={}),
meta=param_get(parameters, 'meta', default={}),
rules=param_get(parameters, 'rules', default=[]),
lifetime=param_get(parameters, 'lifetime', default=None),
dids=param_get(parameters, 'dids', default=[]),
rse=param_get(parameters, 'rse', default=None),
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
)
except (InvalidObject, InvalidPath) as error:
return generate_http_error_flask(400, error)
except (DataIdentifierNotFound, ScopeNotFound) as error:
return generate_http_error_flask(404, error)
except (DuplicateContent, DataIdentifierAlreadyExists, UnsupportedOperation) as error:
return generate_http_error_flask(409, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
return 'Created', 201
if 'dids' in json_data:
dids = json_data['dids']
if 'rse' in json_data:
rse = json_data['rse']
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
except KeyError, error:
return generate_http_error_flask(400, 'ValueError', str(error))
try:
add_did(scope=scope,
name=name,
type=type,
statuses=statuses,
meta=meta,
rules=rules,
lifetime=lifetime,
dids=dids,
rse=rse,
issuer=request.environ.get('issuer'))
except DataIdentifierNotFound, error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0][0])
except DuplicateContent, error:
return generate_http_error_flask(409, 'DuplicateContent',
error.args[0][0])
except DataIdentifierAlreadyExists, error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0][0])
except AccessDenied, error:
create_root_account()
add_account_identity('/CN=docker client',
'x509',
'root',
'*****@*****.**',
issuer="root")
# Create a user called jdoe
add_account('jdoe', 'USER', 'test', 'root')
# Add 2 scopes
add_scope('user.jdoe', 'jdoe', 'root')
add_scope('tests', 'root', 'root')
# Create a test dataset for jdoe
add_did('user.jdoe', 'test1', 'DATASET', 'root', account='jdoe')
# Create 2 sites into the /tmp partition
os.mkdir('/tmp/SITE2_DISK')
os.mkdir('/tmp/SITE1_DISK')
params = {
'scheme': 'file',
'prefix': '/tmp/SITE1_DISK/',
'impl': 'rucio.rse.protocols.posix.Default',
'domains': {
"lan": {
"read": 1,
"write": 1,
"delete": 1
},
def test_api_replica(self):
""" REPLICA (API): Test external representation of replicas """
did = did_name_generator('file')
did_parent = did_name_generator('dataset')
pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (
self.scope_name, generate_uuid())
add_replicas(self.rse2_name,
files=[{
'scope': self.scope_name,
'name': did,
'bytes': 100,
'pfn': pfn
}],
issuer='root',
**self.vo)
add_did(self.scope_name,
did_parent,
'dataset',
issuer='root',
account=self.account_name,
**self.vo)
attachment = {
'scope': self.scope_name,
'name': did_parent,
'dids': [{
'scope': self.scope_name,
'name': did
}]
}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
out = get_did_from_pfns([pfn], self.rse2_name, **self.vo)
out = list(out)
assert 0 != len(out)
did_found = False
for p in out:
for key in p:
if p[key]['name'] == did:
did_found = True
assert self.scope_name == p[key]['scope']
assert did_found
out = list_replicas(dids=[{
'scope': self.scope_name,
'name': did
}],
resolve_parents=True,
**self.vo)
out = list(out)
assert 0 != len(out)
parents_found = False
for rep in out:
assert rep['scope'] == self.scope_name
if 'parents' in rep:
parents_found = True
for parent in rep['parents']:
assert self.scope_name in parent
if self.multi_vo:
assert self.scope.internal not in parent
assert parents_found
def make_block(mock_request, did_metadata, rse, n_files):
# Open a new block
blockname = mock_request["outputdataset"] + "#" + str(uuid.uuid1())
# client.add_dataset(scope="cms", name=blockname, meta=did_metadata)
add_did(scope="cms",
name=blockname,
type="DATASET",
issuer="wma_prod",
account="wma_prod",
meta=did_metadata)
# Here we place a rule with a lifetime so that the block will be deleted.
# The file source replicas will not go away until a rule is placed on them
# and subsequently expired: the transition lock count 1 -> 0 sets a "tombstone"
# which marks the replica as candidate for deletion.
# The container rule will still collect the output before these files are removed
client.add_replication_rule(
dids=[{
"scope": "cms",
"name": blockname
}],
copies=1,
rse_expression=rse,
comment="Production block protection",
meta=json.dumps({
# Here it may be useful to save some metadata that ties the block to the workflow
"workflow": "workflow_name",
"agent": "agent_name",
}),
# Activity is probably optional as the rule should not cause any transfer requests
activity="Production Output",
# The lifetime is somewhat arbitrary, as even if the rule expires, any replicas that
# are needed for transfer (to satsify the container rule) will not be removed until
# the transfers complete, so there is no race condition.
# *update* since TaskChains may force these blocks to stay at a site for indefinite period,
# we let WMAgent hold onto the rule and release it when they deem appropriate
# lifetime=datetime.timedelta(days=7).total_seconds(),
)
# For the container rule to be updated, we have to attach the block to the container
client.add_datasets_to_container(
scope="cms",
name=mock_request["outputdataset"],
dsns=[{
"scope": "cms",
"name": blockname
}],
)
# Inject some file DIDs and source replicas into an RSE
lfns = []
for _ in range(n_files):
lfn = "/store/mc/{campaign}/{pd}/AODSIM/{ps}/{uuid}.root".format(
campaign=mock_request["campaign"],
pd=mock_request["primarydataset"],
ps=mock_request["processingstring"],
uuid=uuid.uuid1(),
)
# This creates a DID and adds replica in one call
client.add_replica(
rse=rse,
scope="cms",
name=lfn,
bytes=int(random.gauss(3e9, 2e8)),
adler32="12341234",
)
lfns.append({"scope": "cms", "name": lfn})
# Add the files to the block
client.attach_dids(scope="cms", name=blockname, dids=lfns)
# Here we close the block, saying no more files will be added
client.close(scope="cms", name=blockname)
def post(self, scope, name):
"""
Create a new data identifier.
.. :quickref: DIDs; Create a new DID.
.. sourcecode:: http
POST /dids/scope1/container1 HTTP/1.1
Host: rucio.cern.ch
{"type": "CONTAINER", "lifetime": 86400},
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:reqheader Accept: application/json
:param scope: data identifier scope.
:param name: data identifier name.
:<json string type: the new DID type
:<json dict statuses: Dictionary with statuses, e.g. {'monotonic':True}
:<json dict meta: Dictionary with metadata, e.g. {'length':1234}
:<json dict rules: Replication rules associated with the did. e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]
:<json int lifetime: DID's liftime in seconds.
:<json list dids: The content.
:<json string rse: The RSE name when registering replicas.
:status 201: new DIDs created
:status 401: Invalid Auth Token
:status 409: DID already exists
:status 500: Database Exception
"""
statuses, meta, rules, lifetime, dids, rse = {}, {}, [], None, [], None
try:
json_data = loads(request.data)
type = json_data['type']
if 'statuses' in json_data:
statuses = json_data['statuses']
if 'meta' in json_data:
meta = json_data['meta']
if 'rules' in json_data:
rules = json_data['rules']
if 'lifetime' in json_data:
lifetime = json_data['lifetime']
if 'dids' in json_data:
dids = json_data['dids']
if 'rse' in json_data:
rse = json_data['rse']
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
except KeyError as error:
return generate_http_error_flask(400, 'ValueError', str(error))
try:
add_did(scope=scope,
name=name,
type=type,
statuses=statuses,
meta=meta,
rules=rules,
lifetime=lifetime,
dids=dids,
rse=rse,
issuer=request.environ.get('issuer'))
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except DuplicateContent as error:
return generate_http_error_flask(409, 'DuplicateContent',
error.args[0])
except DataIdentifierAlreadyExists as error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except DatabaseException as error:
return generate_http_error_flask(500, 'DatabaseException',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print format_exc()
return error, 500
return "Created", 201
def test_subscriptions_at_different_vos(self):
""" MULTI VO (CLIENT): Test that subscriptions from 2nd vo don't interfere """
account_client = AccountClient()
usr_uuid = str(generate_uuid()).lower()[:16]
shr_acc = 'shr-%s' % usr_uuid
account_client.add_account(shr_acc, 'USER', '*****@*****.**')
add_account(shr_acc, 'USER', '*****@*****.**', 'root', **self.new_vo)
scope_client = ScopeClient()
scope_uuid = str(generate_uuid()).lower()[:16]
tst_scope = 'tst_%s' % scope_uuid
new_scope = 'new_%s' % scope_uuid
scope_client.add_scope('root', tst_scope)
add_scope(new_scope, 'root', 'root', **self.new_vo)
did_client = DIDClient()
did_uuid = str(generate_uuid()).lower()[:16]
tst_did = 'tstset_%s' % did_uuid
new_did = 'newset_%s' % did_uuid
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst_rse1 = 'TST1_%s' % rse_str
tst_rse2 = 'TST2_%s' % rse_str
new_rse1 = 'NEW1_%s' % rse_str
new_rse2 = 'NEW2_%s' % rse_str
rse_client.add_rse(tst_rse1)
rse_client.add_rse(tst_rse2)
add_rse(new_rse1, 'root', **self.new_vo)
add_rse(new_rse2, 'root', **self.new_vo)
acc_lim_client = AccountLimitClient()
acc_lim_client.set_local_account_limit(shr_acc, tst_rse1, 10)
acc_lim_client.set_local_account_limit(shr_acc, tst_rse2, 10)
set_local_account_limit(shr_acc, new_rse1, 10, 'root', **self.new_vo)
set_local_account_limit(shr_acc, new_rse2, 10, 'root', **self.new_vo)
did_client.add_did(tst_scope, tst_did, 'DATASET', rse=tst_rse1)
add_did(new_scope, new_did, 'DATASET', 'root', rse=new_rse1, **self.new_vo)
sub_client = SubscriptionClient()
sub_str = generate_uuid()
tst_sub = 'tstsub_' + sub_str
new_sub = 'newsub_' + sub_str
shr_sub = 'shrsub_' + sub_str
tst_sub_id = sub_client.add_subscription(tst_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
shr_tst_sub_id = sub_client.add_subscription(shr_sub, shr_acc, {'scope': [tst_scope]},
[{'copies': 1, 'rse_expression': tst_rse2, 'weight': 0,
'activity': 'User Subscriptions'}],
'', None, 0, 0)
new_sub_id = add_subscription(new_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
shr_new_sub_id = add_subscription(shr_sub, shr_acc, {'scope': [new_scope]},
[{'copies': 1, 'rse_expression': new_rse2, 'weight': 0, 'activity': 'User Subscriptions'}],
'', False, 0, 0, 3, 'root', **self.new_vo)
tst_subs = [s['id'] for s in sub_client.list_subscriptions()]
assert_in(tst_sub_id, tst_subs)
assert_in(shr_tst_sub_id, tst_subs)
assert_not_in(new_sub_id, tst_subs)
assert_not_in(shr_new_sub_id, tst_subs)
new_subs = [s['id'] for s in list_subscriptions(**self.new_vo)]
assert_in(new_sub_id, new_subs)
assert_in(shr_new_sub_id, new_subs)
assert_not_in(tst_sub_id, new_subs)
assert_not_in(shr_tst_sub_id, new_subs)
shr_tst_subs = [s['id'] for s in sub_client.list_subscriptions(name=shr_sub)]
assert_in(shr_tst_sub_id, shr_tst_subs)
assert_not_in(shr_new_sub_id, shr_tst_subs)
shr_new_subs = [s['id'] for s in list_subscriptions(name=shr_sub, **self.new_vo)]
assert_in(shr_new_sub_id, shr_new_subs)
assert_not_in(shr_tst_sub_id, shr_new_subs)
acc_tst_subs = [s['id'] for s in sub_client.list_subscriptions(account=shr_acc)]
assert_in(tst_sub_id, acc_tst_subs)
assert_in(shr_tst_sub_id, acc_tst_subs)
assert_not_in(new_sub_id, acc_tst_subs)
assert_not_in(shr_new_sub_id, acc_tst_subs)
acc_new_subs = [s['id'] for s in list_subscriptions(account=shr_acc, **self.new_vo)]
assert_in(new_sub_id, acc_new_subs)
assert_in(shr_new_sub_id, acc_new_subs)
assert_not_in(tst_sub_id, acc_new_subs)
assert_not_in(shr_tst_sub_id, acc_new_subs)
def test_api_rse(self):
""" RSE (API): Test external representation of RSEs """
out = api_rse.get_rse(self.rse_name, **self.vo)
assert out['rse'] == self.rse_name
assert out['id'] == self.rse_id
out = api_rse.list_rses(**self.new_vo)
out = list(out)
assert 0 != len(out)
rse_ids = [rse['id'] for rse in out]
assert self.rse3_id in rse_ids
assert self.rse4_id in rse_ids
for rse in out:
assert 'rse' in rse
if rse['id'] == self.rse3_id:
assert rse['rse'] == self.rse3_name
elif rse['id'] == self.rse4_id:
assert rse['rse'] == self.rse4_name
key = "KEY_" + generate_uuid()
api_rse.add_rse_attribute(self.rse_name,
key,
1,
issuer='root',
**self.vo)
out = api_rse.get_rses_with_attribute(key)
out = list(out)
assert 0 != len(out)
for rse in out:
assert rse['rse'] == self.rse_name
out = api_rse.get_rse_protocols(self.rse_name,
issuer='root',
**self.vo)
assert out['rse'] == self.rse_name
# add some account and RSE counters
rse_mock = 'MOCK4'
rse_mock_id = get_rse_id(rse_mock, **self.vo)
account_counter.del_counter(rse_id=rse_mock_id, account=self.account)
account_counter.add_counter(rse_id=rse_mock_id, account=self.account)
account_counter.increase(rse_id=rse_mock_id,
account=self.account,
files=1,
bytes=10)
account_counter.update_account_counter(self.account, rse_mock_id)
did = 'file_' + generate_uuid()
add_did(self.scope_name,
did,
'DATASET',
'root',
account=self.account_name,
rse=rse_mock,
**self.vo)
abacus_rse.run(once=True)
out = api_rse.get_rse_usage(rse_mock,
per_account=True,
issuer='root',
**self.vo)
assert rse_mock_id in [o['rse_id'] for o in out]
for usage in out:
if usage['rse_id'] == rse_mock_id:
assert usage['rse'] == rse_mock
accounts = [u['account'] for u in usage['account_usages']]
assert self.account_name in accounts
if self.multi_vo:
assert self.account.internal not in accounts
# clean up files
cleaner.run(once=True)
if self.multi_vo:
reaper.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock),
greedy=True)
else:
reaper.run(once=True, include_rses=rse_mock, greedy=True)
abacus_rse.run(once=True)
out = api_rse.parse_rse_expression(
'%s|%s' % (self.rse_name, self.rse2_name), **self.vo)
assert self.rse_name in out
assert self.rse2_name in out
assert self.rse_id not in out
assert self.rse2_id not in out
