def test_create_existing_subscription(self): """ SUBSCRIPTION (API): Test the creation of a existing subscription """ subscription_name = uuid() add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern': '(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \ \.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'}, replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment') add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern': '(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \ \.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'}, replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
def POST(self, account, name): """ Create a new subscription. HTTP Success: 201 Created HTTP Error: 400 Bad Request 401 Unauthorized 409 Conflict 500 Internal Error """ dry_run = 0 json_data = data() try: params = loads(json_data) filter = params['filter'] replication_rules = params['replication_rules'] comments = params['comments'] lifetime = params['lifetime'] retroactive = params['retroactive'] dry_run = params['dry_run'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: subscription_id = add_subscription(name=name, account=account, filter=filter, replication_rules=replication_rules, comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run) except SubscriptionDuplicate as e: raise generate_http_error(409, 'SubscriptionDuplicate', e.args[0][0]) except RucioException, e: raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def test_list_rules_states(self): """ SUBSCRIPTION (API): Test listing of rule states for subscription """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() add_rse(site_a) add_rse(site_b) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subscription_name = uuid() id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'}, replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment') subscriptions = list_subscriptions(name=subscription_name, account='root') # workaround until add_subscription returns the id id = None for s in subscriptions: id = s['id'] # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) for r in list_subscription_rule_states(account='root', name=subscription_name): assert_equal(r[3], 2)
def test_list_rules_states(self): """ SUBSCRIPTION (REST): Test listing of rule states for subscription """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') mw = [] site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() add_rse(site_a) add_rse(site_b) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subscription_name = uuid() id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'}, replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery') subscriptions = list_subscriptions(name=subscription_name, account='root') # workaround until add_subscription returns the id id = None for s in subscriptions: id = s['id'] # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(r1.status, 200) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Auth-Token': str(token)} r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True) for line in r2.body.split('\n'): print line rs = loads(line) if rs[1] == subscription_name: break assert_equal(rs[3], 2)
def test_create_and_update_and_list_subscription(self): """ SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """ subscription_name = uuid() result = add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern': '(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \ \.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'}, replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment') with assert_raises(TypeError): result = update_subscription(name=subscription_name, account='root', filter='toto') with assert_raises(InvalidObject): result = update_subscription(name=subscription_name, account='root', filter={'project': 'toto'}) result = update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]}) assert_equal(result, None) result = list_subscriptions(name=subscription_name, account='root') sub = [] for r in result: sub.append(r) assert_equal(len(sub), 1) assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
from rucio.api.subscription import list_subscriptions, add_subscription, update_subscription, list_subscription_rule_states, get_subscription_by_id from rucio.client.subscriptionclient import SubscriptionClient from rucio.common.utils import generate_uuid as uuid subscription_name = uuid() result = add_subscription(name=subscription_name, account='root', filter={'project': ['pass4'], 'account': 'root'}, replication_rules=[(1, 'EOS00_AMS02DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment')
def test_create_and_update_and_list_subscription(self): """ SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """ subscription_name = uuid() with pytest.raises(InvalidObject): result = add_subscription(name=subscription_name, account='root', filter={ 'project': self.projects, 'datatype': [ 'AOD', ], 'excluded_pattern': self.pattern1, 'account': [ 'tier0', ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'noactivity' }], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root', **self.vo) result = add_subscription(name=subscription_name, account='root', filter={ 'project': self.projects, 'datatype': [ 'AOD', ], 'excluded_pattern': self.pattern1, 'account': [ 'tier0', ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering' }], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root', **self.vo) with pytest.raises(TypeError): result = update_subscription(name=subscription_name, account='root', metadata={'filter': 'toto'}, issuer='root', **self.vo) with pytest.raises(InvalidObject): result = update_subscription( name=subscription_name, account='root', metadata={'filter': { 'project': 'toto' }}, issuer='root', **self.vo) result = update_subscription( name=subscription_name, account='root', metadata={'filter': { 'project': [ 'toto', ] }}, issuer='root', **self.vo) assert result is None result = list_subscriptions(name=subscription_name, account='root', **self.vo) sub = [] for res in result: sub.append(res) assert len(sub) == 1 assert loads(sub[0]['filter'])['project'][0] == 'toto'
def test_list_rules_states(vo, rest_client, auth_token): """ SUBSCRIPTION (REST): Test listing of rule states for subscription """ tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo) root = InternalAccount('root', vo=vo) add_scope(tmp_scope, root) site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() site_a_id = add_rse(site_a, vo=vo) site_b_id = add_rse(site_b, vo=vo) # Add quota set_local_account_limit(root, site_a_id, -1) set_local_account_limit(root, site_b_id, -1) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root) subscription_name = uuid() subid = add_subscription(name=subscription_name, account='root', filter={ 'account': [ 'root', ], 'scope': [ tmp_scope.external, ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering' }], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery', issuer='root', vo=vo) # Add two rules add_rule(dids=[{ 'scope': tmp_scope, 'name': dsn }], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) add_rule(dids=[{ 'scope': tmp_scope, 'name': dsn }], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) response = rest_client.get('/subscriptions/%s/%s/Rules/States' % ('root', subscription_name), headers=headers(auth(auth_token))) assert response.status_code == 200 rulestates = None for line in response.get_data(as_text=True).split('\n'): if line: rulestates = loads(line) if rulestates[1] == subscription_name: break assert rulestates is not None assert rulestates[3] == 2
def test_list_rules_states(self): """ SUBSCRIPTION (API): Test listing of rule states for subscription """ tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo) root = InternalAccount('root', **self.vo) add_scope(tmp_scope, root) site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() site_a_id = add_rse(site_a, **self.vo) site_b_id = add_rse(site_b, **self.vo) # Add quota set_local_account_limit(root, site_a_id, -1) set_local_account_limit(root, site_b_id, -1) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root) subscription_name = uuid() subid = add_subscription(name=subscription_name, account='root', filter={ 'account': [ 'root', ], 'scope': [ tmp_scope.external, ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering' }], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root', **self.vo) # Add two rules add_rule(dids=[{ 'scope': tmp_scope, 'name': dsn }], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) add_rule(dids=[{ 'scope': tmp_scope, 'name': dsn }], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) for rule in list_subscription_rule_states(account='root', name=subscription_name, **self.vo): assert rule[3] == 2
def test_api_subscription(self): """ SUBSCRIPTION (API): Test external representation of subscriptions """ sub = 'ext_' + generate_uuid() did = did_name_generator('file') new_acc_name = ''.join( random.choice(string.ascii_lowercase) for x in range(10)) new_scope_name = ''.join( random.choice(string.ascii_lowercase) for x in range(10)) add_account(new_acc_name, 'USER', '*****@*****.**', 'root', **self.new_vo) add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo) api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10, 'root', **self.new_vo) api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10, 'root', **self.new_vo) add_did(new_scope_name, did, 'DATASET', 'root', account=new_acc_name, rse=self.rse3_name, **self.new_vo) sub_id = add_subscription(sub, new_acc_name, { 'account': [new_acc_name], 'scope': [new_scope_name] }, [{ 'copies': 1, 'rse_expression': self.rse3_name, 'weight': 0, 'activity': 'Functional Test', 'source_replica_expression': self.rse4_name }], '', False, 0, 0, 3, 'root', **self.new_vo) add_replication_rule(dids=[{ 'scope': new_scope_name, 'name': did }], copies=1, rse_expression=self.rse3_name, weight=None, lifetime=180, grouping='DATASET', account=new_acc_name, locked=False, subscription_id=sub_id, source_replica_expression=self.rse4_name, activity='Functional Test', notify=None, purge_replicas=False, ignore_availability=False, comment='', ask_approval=False, asynchronous=False, delay_injection=None, priority=0, split_container=False, meta='', issuer='root', **self.new_vo) out = list_subscriptions(sub, **self.new_vo) out = list(out) assert 0 != len(out) assert sub_id in [o['id'] for o in out] for o in out: if o['id'] == sub_id: assert o['account'] == new_acc_name rules = loads(o['replication_rules'])[0] assert rules['rse_expression'] == self.rse3_name assert rules['source_replica_expression'] == self.rse4_name fil = loads(o['filter']) assert fil['account'] == [new_acc_name] assert fil['scope'] == [new_scope_name] out = list_subscription_rule_states(sub, **self.new_vo) out = list(out) assert 0 != len(out) for o in out: assert o.account == new_acc_name out = get_subscription_by_id(sub_id, **self.new_vo) assert out['account'] == new_acc_name rules = loads(out['replication_rules'])[0] assert rules['rse_expression'] == self.rse3_name assert rules['source_replica_expression'] == self.rse4_name fil = loads(out['filter']) assert fil['account'] == [new_acc_name] assert fil['scope'] == [new_scope_name]
def test_list_rules_states(self): """ SUBSCRIPTION (REST): Test listing of rule states for subscription """ tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo) root = InternalAccount('root', **self.vo) add_scope(tmp_scope, root) mw = [] site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() site_a_id = add_rse(site_a, **self.vo) site_b_id = add_rse(site_b, **self.vo) # Add quota set_account_limit(root, site_a_id, -1) set_account_limit(root, site_b_id, -1) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root) subscription_name = uuid() subid = add_subscription(name=subscription_name, account='root', filter={ 'account': [ 'root', ], 'scope': [ tmp_scope.external, ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering' }], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery', issuer='root', **self.vo) # Add two rules add_rule(dids=[{ 'scope': tmp_scope, 'name': dsn }], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) add_rule(dids=[{ 'scope': tmp_scope, 'name': dsn }], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) headers1 = { 'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******' } headers1.update(self.vo_header) res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(res1.status, 200) token = str(res1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Auth-Token': str(token)} res2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True) for line in res2.body.split('\n'): print(line) rs = loads(line) if rs[1] == subscription_name: break assert_equal(rs[3], 2)
def post(self, account, name): """ Create a new subscription. .. :quickref: Subscription; Create a subscription. :param account: The account name. :param name: The subscription name. :status 201: Created. :status 400: Cannot decode json parameter list. :status 401: Invalid Auth Token. :status 404: Subscription Not Found. :status 500: Internal Error. :returns: ID if newly created subscription. """ dry_run = 0 json_data = request.data try: params = loads(json_data) params = params['options'] filter = params['filter'] replication_rules = params['replication_rules'] comments = params['comments'] lifetime = params['lifetime'] retroactive = params['retroactive'] dry_run = params['dry_run'] priority = params.get('priority', 3) or 3 except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: subscription_id = add_subscription( name=name, account=account, filter=filter, replication_rules=replication_rules, comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) except (InvalidObject, TypeError) as error: return generate_http_error_flask(400, 'InvalidObject', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except SubscriptionDuplicate as error: return generate_http_error_flask(409, 'SubscriptionDuplicate', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return str(error), 500 return Response(subscription_id, status=201)