def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blacklisted(self): """ JUDGE REPAIRER: Test to repair a rule with only 1 rse whose site is blacklisted""" rse = rse_name_generator() rse_id = add_rse(rse) update_rse(rse_id, {'availability_write': False}) set_account_limit('jdoe', rse_id, -1) rule_repairer(once=True) # Clean out the repairer scope = 'mock' files = create_files(4, scope, self.rse4_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True, activity='DebugJudge')[0] assert(RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) # Stil assert STUCK because of ignore_availability: assert(RuleState.STUCK == get_rule(rule_id)['state']) region = make_region().configure('dogpile.cache.memcached', expiration_time=3600, arguments={'url': "127.0.0.1:11211", 'distributed_lock': True}) region.delete(sha256(rse).hexdigest()) update_rse(rse_id, {'availability_write': True}) rule_repairer(once=True) assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
def test_list_expired_dids_with_locked_rules(self): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules""" tmp_scope = 'mock' # Add quota set_account_limit('jdoe', get_rse_id('MOCK'), -1) dsn = { 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': 'MOCK', 'locked': True, 'grouping': 'DATASET' }] } add_dids(dids=[dsn], account='root') for did in list_expired_dids(limit=1000): assert (did['scope'] != dsn['scope'] and did['name'] != dsn['name'])
def generate_rse(endpoint, token): rse_name = 'RSE%s' % generate_uuid().upper() scheme = 'https' impl = 'rucio.rse.protocols.webdav.Default' if not endpoint.startswith('https://'): scheme = 'srm' impl = 'rucio.rse.protocols.srm.Default' tmp_proto = { 'impl': impl, 'scheme': scheme, 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}} rse_id = rse.add_rse(rse_name) tmp_proto['hostname'] = endpoint.split(':')[1][2:] tmp_proto['port'] = endpoint.split(':')[2].split('/')[0] tmp_proto['prefix'] = '/'.join([''] + endpoint.split(':')[2].split('/')[1:]) if scheme == 'srm': tmp_proto['extended_attributes'] = {'space_token': token, 'web_service_path': '/srm/managerv2?SFN='} rse.add_protocol(rse_id=rse_id, parameter=tmp_proto) rse.add_rse_attribute(rse_id=rse_id, key='fts', value='https://fts3-pilot.cern.ch:8446') account_limit.set_account_limit(account='root', rse_id=rsemanager.get_rse_info(rse_name)['id'], bytes=-1) return rsemanager.get_rse_info(rse_name)
def set_account_limit(account, rse, bytes, issuer): """ Set an account limit.. :param account: The account name. :param rse: The rse name. :param bytes: The limit in bytes. :param issuer: The issuer account_core. """ rse_id = get_rse_id(rse=rse) kwargs = {'account': account, 'rse': rse, 'rse_id': rse_id, 'bytes': bytes} if not rucio.api.permission.has_permission( issuer=issuer, action='set_account_limit', kwargs=kwargs): raise rucio.common.exception.AccessDenied( 'Account %s can not set account limits.' % (issuer)) account = InternalAccount(account) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound( 'Account %s does not exist' % (account)) account_limit_core.set_account_limit(account=account, rse_id=rse_id, bytes=bytes)
def test_add_did(self): """ DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample""" tmp_scope = 'mock' tmp_rse = 'MOCK' tmp_dsn = 'dsn_%s' % generate_uuid() set_account_limit('root', get_rse_id('MOCK'), -1) set_account_limit('root', get_rse_id('CERN-PROD_TZERO'), -1) # PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename dataset_meta = { 'project': 'data13_hip', 'run_number': 300000, 'stream_name': 'physics_CosmicCalo', 'prod_step': 'merge', 'datatype': 'NTUP_TRIG', 'version': 'f392_m927', } rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}] with assert_raises(ScopeNotFound): self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules) files = [ { 'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()), 'bytes': 724963570L, 'adler32': '0cc737eb' }, ]
def test_list_rules_states(self): """ SUBSCRIPTION (API): Test listing of rule states for subscription """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() add_rse(site_a) add_rse(site_b) # Add quota set_account_limit('root', get_rse_id(site_a), -1) set_account_limit('root', get_rse_id(site_b), -1) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subscription_name = uuid() subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope, ]}, replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root') # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) for rule in list_subscription_rule_states(account='root', name=subscription_name): assert_equal(rule[3], 2)
def test_listing_account_limit(self): """ ACCOUNT (CLIENTS): Test listing account limit """ account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id) account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=333) limit = self.client.get_account_limit(account=self.account, rse=self.rse1) assert_equal(limit, {self.rse1: 333}) account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id)
def test_delete_rule(self): """CLIENT(USER): rule deletion""" set_account_limit('root', get_rse_id(self.def_rse), -1) tmp_file1 = file_generator() # add files cmd = 'rucio upload --rse {0} --scope {1} {2}'.format( self.def_rse, self.user, tmp_file1) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add rse tmp_rse = rse_name_generator() cmd = 'rucio-admin rse add {0}'.format(tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out) set_account_limit('root', get_rse_id(tmp_rse), -1) # add rse atributes cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format( tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add rules cmd = "rucio add-rule {0}:{1} 1 'spacetoken=ATLASSCRATCHDISK'".format( self.user, tmp_file1[5:]) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(err) print(out) # get the rules for the file cmd = "rucio list-rules {0}:{1} | grep {0}:{1} | cut -f1 -d\ ".format( self.user, tmp_file1[5:]) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) (rule1, rule2) = out.split() # delete the rules for the file cmd = "rucio delete-rule {0}".format(rule1) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) cmd = "rucio delete-rule {0}".format(rule2) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # search for the file cmd = "rucio list-dids {0}:{1}".format(self.user, tmp_file1[5:]) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) nose.tools.assert_equal(5, len(out.splitlines()))
def test_set_account_limit(self): """ ACCOUNT_LIMIT (CORE): Setting account limit """ account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=100000) assert_equal( account_limit.get_account_limit(account=self.account, rse_id=self.rse1_id), 100000) assert_equal( account_limit.get_account_limit(account=self.account, rse_id=self.rse2_id), None)
def test_listing_account_limits(self): """ ACCOUNT (CLIENTS): Test listing account limits """ account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=12345) account_limit.set_account_limit(account=self.account, rse_id=self.rse2_id, bytes=12345) limits = self.client.get_account_limits(account=self.account) assert_in((self.rse1, 12345), limits.items()) assert_in((self.rse2, 12345), limits.items()) account_limit.delete_account_limit(account=self.account, rse_id=self.rse1_id) account_limit.delete_account_limit(account=self.account, rse_id=self.rse2_id)
def test_rule_add_fails_account_limit(self): """ REPLICATION RULE (CORE): Test if a rule fails correctly when account limit conflict""" scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') set_account_limit(account='jdoe', rse_id=self.rse1_id, bytes=5) assert_raises(InsufficientAccountLimit, add_rule, dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) delete_account_limit(account='jdoe', rse_id=self.rse1_id)
def setup(self): try: remove('/tmp/.rucio_root/auth_token_root') except OSError as e: if e.args[0] != 2: raise e self.marker = '$> ' self.host = config_get('client', 'rucio_host') self.auth_host = config_get('client', 'auth_host') self.user = '******' self.def_rse = 'MOCK4' self.did_client = DIDClient() self.replica_client = ReplicaClient() set_account_limit('root', get_rse_id(self.def_rse), -1)
def test_abacus_account(self): """ ABACUS (ACCOUNT): Test update of account usage """ self.session.query(models.UpdatedAccountCounter).delete() # pylint: disable=no-member self.session.query(models.AccountUsage).delete() # pylint: disable=no-member self.session.commit() # pylint: disable=no-member # Upload files -> account usage should increase self.files = [{ 'did_scope': self.scope.external, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1 } for i in range(0, 2)] self.upload_client.upload(self.files) [os.remove(file['path']) for file in self.files] account.run(once=True) account_usage = get_account_usage(account=self.account, rse_id=self.rse_id)[0] assert_equal(account_usage['bytes'], len(self.files) * self.file_sizes) assert_equal(account_usage['files'], len(self.files)) # Update and check the account history with the core method update_account_counter_history(account=self.account, rse_id=self.rse_id) usage_history = get_usage_history(rse_id=self.rse_id, account=self.account) assert_equal(usage_history[-1]['bytes'], len(self.files) * self.file_sizes) assert_equal(usage_history[-1]['files'], len(self.files)) # Check the account history with the client usage_history = self.account_client.get_account_usage_history( rse=self.rse, account=self.account.external) assert_equal(usage_history[-1]['bytes'], len(self.files) * self.file_sizes) assert_equal(usage_history[-1]['files'], len(self.files)) # Delete rules -> account usage should decrease cleaner.run(once=True) account.run(once=True) # set account limit because return value of get_account_usage differs if a limit is set or not set_account_limit(account=self.account, rse_id=self.rse_id, bytes=10) account_usages = get_account_usage(account=self.account, rse_id=self.rse_id)[0] assert_equal(account_usages['bytes'], 0) assert_equal(account_usages['files'], 0)
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 set_account_limit('jdoe', get_rse_id('MOCK'), -1) dsns1 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1 } for i in xrange(nbdatasets)] dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': 'MOCK', 'grouping': 'DATASET' }] } for i in xrange(nbdatasets)] add_dids(dids=dsns1 + dsns2, account='root') replicas = list() for dsn in dsns1 + dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in xrange(nbfiles)] attach_dids(scope=tmp_scope,
def test_list_rules_states(self): """ SUBSCRIPTION (REST): Test listing of rule states for subscription """ tmp_scope = InternalScope('mock_' + uuid()[:8]) root = InternalAccount('root') add_scope(tmp_scope, root) mw = [] site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() site_a_id = add_rse(site_a) site_b_id = add_rse(site_b) # Add quota set_account_limit(root, site_a_id, -1) set_account_limit(root, site_b_id, -1) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root) subscription_name = uuid() subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope.external, ]}, replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery', issuer='root') # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} res1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(res1.status, 200) token = str(res1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Auth-Token': str(token)} res2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True) for line in res2.body.split('\n'): print(line) rs = loads(line) if rs[1] == subscription_name: break assert_equal(rs[3], 2)
def set_account_limit(account, rse, bytes, issuer): """ Set an account limit.. :param account: The account name. :param rse: The rse name. :param bytes: The limit in bytes. :param issuer: The issuer account_core. """ kwargs = {'account': account, 'rse': rse, 'bytes': bytes} if not rucio.api.permission.has_permission(issuer=issuer, action='set_account_limit', kwargs=kwargs): raise rucio.common.exception.AccessDenied('Account %s can not set account limits.' % (issuer)) if not account_exists(account=account): raise rucio.common.exception.AccountNotFound('Account %s does not exist' % (account)) rse_id = get_rse_id(rse=rse) account_limit_core.set_account_limit(account=account, rse_id=rse_id, bytes=bytes)
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() add_rse(rse) set_account_limit('jdoe', get_rse_id(rse), -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in xrange(nbdatasets)] add_dids(dids=dsns2, account='root') replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1L, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in xrange(nbfiles)]
def test_undertaker(self): """ UNDERTAKER (CORE): Test the undertaker. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'MOCK' rse_id = get_rse_id('MOCK') set_account_limit('jdoe', rse_id, -1) dsns1 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1} for i in range(nbdatasets)] dsns2 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for i in range(nbdatasets)] add_dids(dids=dsns1 + dsns2, account='root') replicas = list() for dsn in dsns1 + dsns2: files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account='root') replicas += files add_rules(dids=dsns1, rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]) undertaker(worker_number=1, total_workers=1, once=True) undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert_not_equal(get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'], None)
class TestBinRucio(): def setup(self): try: remove('/tmp/.rucio_root/auth_token_root') except OSError, e: if e.args[0] != 2: raise e self.marker = '$> ' self.host = config_get('client', 'rucio_host') self.auth_host = config_get('client', 'auth_host') self.user = '******' self.def_rse = 'MOCK4' set_account_limit('root', get_rse_id(self.def_rse), -1)
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() add_rse(rse) set_account_limit('jdoe', get_rse_id(rse), -1) dsns2 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for i in range(nbdatasets)] add_dids(dids=dsns2, account='root') replicas = list() for dsn in dsns2: files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root') replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None) for dsn in dsns2: assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name']) assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
def setUpClass(cls): # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse_id(rse=cls.rse1) cls.rse3_id = get_rse_id(rse=cls.rse3) cls.rse4_id = get_rse_id(rse=cls.rse4) cls.rse5_id = get_rse_id(rse=cls.rse5) # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1_id, cls.T1, True) add_rse_attribute(cls.rse3_id, cls.T1, True) add_rse_attribute(cls.rse4_id, cls.T2, True) add_rse_attribute(cls.rse5_id, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1_id, "fakeweight", 10) add_rse_attribute(cls.rse3_id, "fakeweight", 0) add_rse_attribute(cls.rse4_id, "fakeweight", 0) add_rse_attribute(cls.rse5_id, "fakeweight", 0) # Add quota cls.jdoe = InternalAccount('jdoe') cls.root = InternalAccount('root') set_account_limit(cls.jdoe, cls.rse1_id, -1) set_account_limit(cls.jdoe, cls.rse3_id, -1) set_account_limit(cls.jdoe, cls.rse4_id, -1) set_account_limit(cls.jdoe, cls.rse5_id, -1) set_account_limit(cls.root, cls.rse1_id, -1) set_account_limit(cls.root, cls.rse3_id, -1) set_account_limit(cls.root, cls.rse4_id, -1) set_account_limit(cls.root, cls.rse5_id, -1)
def create_scope(scope): add_scope(scope, 'root', 'root') set_account_limit('root', get_rse_id('PSDM_DISK'), 100000000000)
def setUpClass(cls): # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse(cls.rse1).id cls.rse3_id = get_rse(cls.rse3).id cls.rse4_id = get_rse(cls.rse4).id cls.rse5_id = get_rse(cls.rse5).id # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1, cls.T1, True) add_rse_attribute(cls.rse3, cls.T1, True) add_rse_attribute(cls.rse4, cls.T2, True) add_rse_attribute(cls.rse5, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1, "fakeweight", 10) add_rse_attribute(cls.rse3, "fakeweight", 0) add_rse_attribute(cls.rse4, "fakeweight", 0) add_rse_attribute(cls.rse5, "fakeweight", 0) # Add quota set_account_limit('jdoe', cls.rse1_id, -1) set_account_limit('jdoe', cls.rse3_id, -1) set_account_limit('jdoe', cls.rse4_id, -1) set_account_limit('jdoe', cls.rse5_id, -1) set_account_limit('root', cls.rse1_id, -1) set_account_limit('root', cls.rse3_id, -1) set_account_limit('root', cls.rse4_id, -1) set_account_limit('root', cls.rse5_id, -1)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': 'tst'} else: cls.vo = {} # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo) cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo) cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo) cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo) # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1_id, cls.T1, True) add_rse_attribute(cls.rse3_id, cls.T1, True) add_rse_attribute(cls.rse4_id, cls.T2, True) add_rse_attribute(cls.rse5_id, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1_id, "fakeweight", 10) add_rse_attribute(cls.rse3_id, "fakeweight", 0) add_rse_attribute(cls.rse4_id, "fakeweight", 0) add_rse_attribute(cls.rse5_id, "fakeweight", 0) # Add quota cls.jdoe = InternalAccount('jdoe', **cls.vo) cls.root = InternalAccount('root', **cls.vo) set_account_limit(cls.jdoe, cls.rse1_id, -1) set_account_limit(cls.jdoe, cls.rse3_id, -1) set_account_limit(cls.jdoe, cls.rse4_id, -1) set_account_limit(cls.jdoe, cls.rse5_id, -1) set_account_limit(cls.root, cls.rse1_id, -1) set_account_limit(cls.root, cls.rse3_id, -1) set_account_limit(cls.root, cls.rse4_id, -1) set_account_limit(cls.root, cls.rse5_id, -1)
add_protocol('SITE1_DISK', params) add_rse_attribute(rse='SITE1_DISK', key='istape', value='False') params = { 'scheme': 'file', 'prefix': '/tmp/SITE2_DISK/', 'impl': 'rucio.rse.protocols.posix.Default', 'domains': { "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1 } } } add_rse('SITE2_DISK', 'root') add_protocol('SITE2_DISK', params) add_rse_attribute(rse='SITE2_DISK', key='istape', value='False') # Now set a quota for root and jdoe on the 2 RSEs set_account_limit('root', get_rse_id('SITE1_DISK'), 100000000000) set_account_limit('root', get_rse_id('SITE2_DISK'), 100000000000) set_account_limit('jdoe', get_rse_id('SITE1_DISK'), 1000000000) set_account_limit('jdoe', get_rse_id('SITE2_DISK'), 0)
def test_add_did(self): """ DATA IDENTIFIERS (CLIENT): Add, populate, list did content and create a sample""" tmp_scope = 'mock' tmp_rse = 'MOCK' tmp_dsn = 'dsn_%s' % generate_uuid() set_account_limit('root', get_rse_id('MOCK'), -1) set_account_limit('root', get_rse_id('CERN-PROD_TZERO'), -1) # PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename dataset_meta = {'project': 'data13_hip', 'run_number': 300000, 'stream_name': 'physics_CosmicCalo', 'prod_step': 'merge', 'datatype': 'NTUP_TRIG', 'version': 'f392_m927', } rules = [{'copies': 1, 'rse_expression': 'MOCK', 'account': 'root'}] with assert_raises(ScopeNotFound): self.did_client.add_dataset(scope='Nimportnawak', name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules) files = [{'scope': tmp_scope, 'name': 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb'}, ] with assert_raises(DataIdentifierNotFound): self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files) with assert_raises(DataIdentifierNotFound): self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files) files = [] for i in range(5): lfn = 'lfn.%(tmp_dsn)s.' % locals() + str(generate_uuid()) pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta # it doesn't work with mock: TBF # pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta pfn += '%(tmp_dsn)s/%(lfn)s' % locals() file_meta = {'guid': str(generate_uuid()), 'events': 10} files.append({'scope': tmp_scope, 'name': lfn, 'bytes': 724963570, 'adler32': '0cc737eb', 'pfn': pfn, 'meta': file_meta}) rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2), 'account': 'root'}] with assert_raises(InvalidPath): self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files, rse=tmp_rse) files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files] self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, statuses={'monotonic': True}, meta=dataset_meta, rules=rules, files=files_without_pfn, rse=tmp_rse) with assert_raises(DataIdentifierAlreadyExists): self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse) files = [] for i in range(5): lfn = '%(tmp_dsn)s.' % locals() + str(generate_uuid()) pfn = 'mock://localhost/tmp/rucio_rse/%(project)s/%(version)s/%(prod_step)s' % dataset_meta # it doesn't work with mock: TBF # pfn = 'srm://mock2.com:2880/pnfs/rucio/disk-only/scratchdisk/rucio_tests/%(project)s/%(version)s/%(prod_step)s' % dataset_meta pfn += '%(tmp_dsn)s/%(lfn)s' % locals() file_meta = {'guid': str(generate_uuid()), 'events': 100} files.append({'scope': tmp_scope, 'name': lfn, 'bytes': 724963570, 'adler32': '0cc737eb', 'pfn': pfn, 'meta': file_meta}) rules = [{'copies': 1, 'rse_expression': 'CERN-PROD_TZERO', 'lifetime': timedelta(days=2)}] with assert_raises(InvalidPath): self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files, rse=tmp_rse) files_without_pfn = [{'scope': i['scope'], 'name': i['name'], 'bytes': i['bytes'], 'adler32': i['adler32'], 'meta': i['meta']} for i in files] self.did_client.add_files_to_dataset(scope=tmp_scope, name=tmp_dsn, files=files_without_pfn, rse=tmp_rse) self.did_client.close(scope=tmp_scope, name=tmp_dsn) tmp_dsn_output = 'dsn_%s' % generate_uuid() self.did_client.create_did_sample(input_scope=tmp_scope, input_name=tmp_dsn, output_scope=tmp_scope, output_name=tmp_dsn_output, nbfiles=2) files = [f for f in self.did_client.list_files(scope=tmp_scope, name=tmp_dsn_output)] assert_equal(len(files), 2)
def test_create_rule(self): """CLIENT(USER): Rucio add rule""" tmp_file1 = file_generator() # add files cmd = 'rucio upload --rse {0} --scope {1} {2}'.format( self.def_rse, self.user, tmp_file1) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add rse tmp_rse = rse_name_generator() cmd = 'rucio-admin rse add {0}'.format(tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out) # add quota set_account_limit('root', get_rse_id(tmp_rse), -1) # add rse atributes cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format( tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add rse tmp_rse = rse_name_generator() cmd = 'rucio-admin rse add {0}'.format(tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add quota set_account_limit('root', get_rse_id(tmp_rse), -1) # add rse atributes cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format( tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add rse tmp_rse = rse_name_generator() cmd = 'rucio-admin rse add {0}'.format(tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add quota set_account_limit('root', get_rse_id(tmp_rse), -1) # add rse atributes cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format( tmp_rse) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) # add rules cmd = "rucio add-rule {0}:{1} 3 'spacetoken=ATLASSCRATCHDISK'".format( self.user, tmp_file1[5:]) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out) rule = out[:-1] # triming new line character # check if rule exist for the file cmd = "rucio list-rules {0}:{1}".format(self.user, tmp_file1[5:]) print(self.marker + cmd) exitcode, out, err = execute(cmd) print(out, err) nose.tools.assert_not_equal(re.search(rule, out), None)
def test_set_account_limit(self): """ ACCOUNT_LIMIT (CORE): Setting account limit """ account_limit.set_account_limit(account=self.account, rse_id=self.rse1_id, bytes=100000) assert_equal(account_limit.get_account_limit(account=self.account, rse_id=self.rse1_id), 100000) assert_equal(account_limit.get_account_limit(account=self.account, rse_id=self.rse2_id), float("Inf"))