def test_create_existing_subscription(self):
""" SUBSCRIPTION (API): Test the creation of a existing subscription """
subscription_name = uuid()
add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern':
'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
\.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'},
replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern':
'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
\.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'},
replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
def POST(self, account, name):
"""
Create a new subscription.
HTTP Success:
201 Created
HTTP Error:
400 Bad Request
401 Unauthorized
409 Conflict
500 Internal Error
"""
dry_run = 0
json_data = data()
try:
params = loads(json_data)
filter = params['filter']
replication_rules = params['replication_rules']
comments = params['comments']
lifetime = params['lifetime']
retroactive = params['retroactive']
dry_run = params['dry_run']
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
subscription_id = add_subscription(name=name, account=account, filter=filter, replication_rules=replication_rules, comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run)
except SubscriptionDuplicate as e:
raise generate_http_error(409, 'SubscriptionDuplicate', e.args[0][0])
except RucioException, e:
raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
for r in list_subscription_rule_states(account='root', name=subscription_name):
assert_equal(r[3], 2)
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = 'mock_' + uuid()[:8]
add_scope(tmp_scope, 'root')
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
add_rse(site_a)
add_rse(site_b)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
type=DIDType.DATASET, account='root')
subscription_name = uuid()
id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'},
replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery')
subscriptions = list_subscriptions(name=subscription_name, account='root')
# workaround until add_subscription returns the id
id = None
for s in subscriptions:
id = s['id']
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id)
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
assert_equal(r1.status, 200)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True)
for line in r2.body.split('\n'):
print line
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """
subscription_name = uuid()
result = add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern':
'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
\.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'},
replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
with assert_raises(TypeError):
result = update_subscription(name=subscription_name, account='root', filter='toto')
with assert_raises(InvalidObject):
result = update_subscription(name=subscription_name, account='root', filter={'project': 'toto'})
result = update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]})
assert_equal(result, None)
result = list_subscriptions(name=subscription_name, account='root')
sub = []
for r in result:
sub.append(r)
assert_equal(len(sub), 1)
assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
from rucio.api.subscription import list_subscriptions, add_subscription, update_subscription, list_subscription_rule_states, get_subscription_by_id
from rucio.client.subscriptionclient import SubscriptionClient
from rucio.common.utils import generate_uuid as uuid
subscription_name = uuid()
result = add_subscription(name=subscription_name, account='root', filter={'project': ['pass4'], 'account': 'root'}, replication_rules=[(1, 'EOS00_AMS02DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
def test_create_and_update_and_list_subscription(self):
""" SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """
subscription_name = uuid()
with pytest.raises(InvalidObject):
result = add_subscription(name=subscription_name,
account='root',
filter={
'project': self.projects,
'datatype': [
'AOD',
],
'excluded_pattern': self.pattern1,
'account': [
'tier0',
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
'MOCK|MOCK2',
'copies':
2,
'activity':
'noactivity'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='This is a comment',
issuer='root',
**self.vo)
result = add_subscription(name=subscription_name,
account='root',
filter={
'project': self.projects,
'datatype': [
'AOD',
],
'excluded_pattern': self.pattern1,
'account': [
'tier0',
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
'MOCK|MOCK2',
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='This is a comment',
issuer='root',
**self.vo)
with pytest.raises(TypeError):
result = update_subscription(name=subscription_name,
account='root',
metadata={'filter': 'toto'},
issuer='root',
**self.vo)
with pytest.raises(InvalidObject):
result = update_subscription(
name=subscription_name,
account='root',
metadata={'filter': {
'project': 'toto'
}},
issuer='root',
**self.vo)
result = update_subscription(
name=subscription_name,
account='root',
metadata={'filter': {
'project': [
'toto',
]
}},
issuer='root',
**self.vo)
assert result is None
result = list_subscriptions(name=subscription_name,
account='root',
**self.vo)
sub = []
for res in result:
sub.append(res)
assert len(sub) == 1
assert loads(sub[0]['filter'])['project'][0] == 'toto'
def test_list_rules_states(vo, rest_client, auth_token):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
root = InternalAccount('root', vo=vo)
add_scope(tmp_scope, root)
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a, vo=vo)
site_b_id = add_rse(site_b, vo=vo)
# Add quota
set_local_account_limit(root, site_a_id, -1)
set_local_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name,
account='root',
filter={
'account': [
'root',
],
'scope': [
tmp_scope.external,
]
},
replication_rules=[{
'lifetime': 86400,
'rse_expression': 'MOCK|MOCK2',
'copies': 2,
'activity': 'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='We want a shrubbery',
issuer='root',
vo=vo)
# Add two rules
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=site_a,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=site_b,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
response = rest_client.get('/subscriptions/%s/%s/Rules/States' %
('root', subscription_name),
headers=headers(auth(auth_token)))
assert response.status_code == 200
rulestates = None
for line in response.get_data(as_text=True).split('\n'):
if line:
rulestates = loads(line)
if rulestates[1] == subscription_name:
break
assert rulestates is not None
assert rulestates[3] == 2
def test_list_rules_states(self):
""" SUBSCRIPTION (API): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
root = InternalAccount('root', **self.vo)
add_scope(tmp_scope, root)
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a, **self.vo)
site_b_id = add_rse(site_b, **self.vo)
# Add quota
set_local_account_limit(root, site_a_id, -1)
set_local_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name,
account='root',
filter={
'account': [
'root',
],
'scope': [
tmp_scope.external,
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
'MOCK|MOCK2',
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='This is a comment',
issuer='root',
**self.vo)
# Add two rules
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=site_a,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=site_b,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
for rule in list_subscription_rule_states(account='root',
name=subscription_name,
**self.vo):
assert rule[3] == 2
def test_api_subscription(self):
""" SUBSCRIPTION (API): Test external representation of subscriptions """
sub = 'ext_' + generate_uuid()
did = did_name_generator('file')
new_acc_name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
new_scope_name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
add_account(new_acc_name, 'USER', '*****@*****.**', 'root',
**self.new_vo)
add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10,
'root', **self.new_vo)
api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10,
'root', **self.new_vo)
add_did(new_scope_name,
did,
'DATASET',
'root',
account=new_acc_name,
rse=self.rse3_name,
**self.new_vo)
sub_id = add_subscription(sub, new_acc_name, {
'account': [new_acc_name],
'scope': [new_scope_name]
}, [{
'copies': 1,
'rse_expression': self.rse3_name,
'weight': 0,
'activity': 'Functional Test',
'source_replica_expression': self.rse4_name
}], '', False, 0, 0, 3, 'root', **self.new_vo)
add_replication_rule(dids=[{
'scope': new_scope_name,
'name': did
}],
copies=1,
rse_expression=self.rse3_name,
weight=None,
lifetime=180,
grouping='DATASET',
account=new_acc_name,
locked=False,
subscription_id=sub_id,
source_replica_expression=self.rse4_name,
activity='Functional Test',
notify=None,
purge_replicas=False,
ignore_availability=False,
comment='',
ask_approval=False,
asynchronous=False,
delay_injection=None,
priority=0,
split_container=False,
meta='',
issuer='root',
**self.new_vo)
out = list_subscriptions(sub, **self.new_vo)
out = list(out)
assert 0 != len(out)
assert sub_id in [o['id'] for o in out]
for o in out:
if o['id'] == sub_id:
assert o['account'] == new_acc_name
rules = loads(o['replication_rules'])[0]
assert rules['rse_expression'] == self.rse3_name
assert rules['source_replica_expression'] == self.rse4_name
fil = loads(o['filter'])
assert fil['account'] == [new_acc_name]
assert fil['scope'] == [new_scope_name]
out = list_subscription_rule_states(sub, **self.new_vo)
out = list(out)
assert 0 != len(out)
for o in out:
assert o.account == new_acc_name
out = get_subscription_by_id(sub_id, **self.new_vo)
assert out['account'] == new_acc_name
rules = loads(out['replication_rules'])[0]
assert rules['rse_expression'] == self.rse3_name
assert rules['source_replica_expression'] == self.rse4_name
fil = loads(out['filter'])
assert fil['account'] == [new_acc_name]
assert fil['scope'] == [new_scope_name]
def test_list_rules_states(self):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo)
root = InternalAccount('root', **self.vo)
add_scope(tmp_scope, root)
mw = []
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a, **self.vo)
site_b_id = add_rse(site_b, **self.vo)
# Add quota
set_account_limit(root, site_a_id, -1)
set_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name,
account='root',
filter={
'account': [
'root',
],
'scope': [
tmp_scope.external,
]
},
replication_rules=[{
'lifetime':
86400,
'rse_expression':
'MOCK|MOCK2',
'copies':
2,
'activity':
'Data Brokering'
}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='We want a shrubbery',
issuer='root',
**self.vo)
# Add two rules
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=site_a,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
add_rule(dids=[{
'scope': tmp_scope,
'name': dsn
}],
account=root,
copies=1,
rse_expression=site_b,
grouping='NONE',
weight=None,
lifetime=None,
locked=False,
subscription_id=subid)
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
headers1.update(self.vo_header)
res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
assert_equal(res1.status, 200)
token = str(res1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Auth-Token': str(token)}
res2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' %
('root', subscription_name),
headers=headers2,
expect_errors=True)
for line in res2.body.split('\n'):
print(line)
rs = loads(line)
if rs[1] == subscription_name:
break
assert_equal(rs[3], 2)
def post(self, account, name):
"""
Create a new subscription.
.. :quickref: Subscription; Create a subscription.
:param account: The account name.
:param name: The subscription name.
:status 201: Created.
:status 400: Cannot decode json parameter list.
:status 401: Invalid Auth Token.
:status 404: Subscription Not Found.
:status 500: Internal Error.
:returns: ID if newly created subscription.
"""
dry_run = 0
json_data = request.data
try:
params = loads(json_data)
params = params['options']
filter = params['filter']
replication_rules = params['replication_rules']
comments = params['comments']
lifetime = params['lifetime']
retroactive = params['retroactive']
dry_run = params['dry_run']
priority = params.get('priority', 3) or 3
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
subscription_id = add_subscription(
name=name,
account=account,
filter=filter,
replication_rules=replication_rules,
comments=comments,
lifetime=lifetime,
retroactive=retroactive,
dry_run=dry_run,
priority=priority,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except (InvalidObject, TypeError) as error:
return generate_http_error_flask(400, 'InvalidObject',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except SubscriptionDuplicate as error:
return generate_http_error_flask(409, 'SubscriptionDuplicate',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
return Response(subscription_id, status=201)
