def test_core_temporary_dids(): """ TMP DATA IDENTIFIERS (CORE): """ if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': get_vo()} else: vo = {} scope = InternalScope('mock', **vo) root = InternalAccount('root', **vo) temporary_dids = [] rse = 'MOCK' rse_id = get_rse_id(rse=rse, **vo) for _ in range(10): temporary_dids.append({'scope': scope, 'name': 'object_%s' % generate_uuid(), 'rse_id': rse_id, 'bytes': 1, 'path': None}) add_temporary_dids(dids=temporary_dids, account=root) compose(scope=scope, name='file_%s' % generate_uuid(), rse_id=rse_id, bytes=10, sources=temporary_dids, account=root, md5=None, adler32=None, pfn=None, meta={}, rules=[], parent_scope=None, parent_name=None) dids = list_expired_temporary_dids(rse_id=rse_id, limit=10) rowcount = delete_temporary_dids(dids=dids) assert rowcount == 10
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.tmp_scope = InternalScope('mock', **self.vo) self.root = InternalAccount('root', **self.vo)
def test_quarantined_replicas(): """ QUARANTINED REPLICA (CORE): Add, List and Delete quarantined replicas """ if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': get_vo()} else: vo = {} rse_id = get_rse_id(rse='MOCK', **vo) real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id, limit=10000) quarantined_replicas = len(real_replicas) + len(dark_replicas) nbreplicas = 5 replicas = [{ 'path': '/path/' + generate_uuid() } for _ in range(nbreplicas)] add_quarantined_replicas(rse_id=rse_id, replicas=replicas) real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id, limit=10000) assert quarantined_replicas + nbreplicas == len(dark_replicas) + len( real_replicas) delete_quarantined_replicas(rse_id=rse_id, replicas=replicas) real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id, limit=10000) assert quarantined_replicas == len(dark_replicas) + len(real_replicas)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} cls.multi_vo = True else: cls.vo = {} cls.multi_vo = False # Add test account cls.account = InternalAccount( ''.join(random.choice(string.ascii_uppercase) for x in range(10)), **cls.vo) add_account(account=cls.account, type_=AccountType.USER, email='*****@*****.**') # Add test RSE cls.rse1 = 'MOCK' cls.rse2 = 'MOCK2' cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo) cls.rse2_id = get_rse_id(rse=cls.rse2, **cls.vo) cls.db_session = session.get_session()
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} cls.projects = ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'] cls.pattern1 = r'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {}
def setUp(self): """ Setup the Test Case """ if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.account = InternalAccount(account_name_generator(), **self.vo) add_account(self.account, AccountType.USER, '*****@*****.**')
def setUp(self): """ Setup Test Case """ if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.usr = '******'
def setUpClass(cls): cls.upload_client = UploadClient() cls.session = get_session() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUp(self): """ Constructor.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.did_client = DIDClient()
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} @transactional_session def __cleanup_updated_dids(session=None): session.query(UpdatedDID).delete() __cleanup_updated_dids() # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo) cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo) cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo) cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo) # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1_id, cls.T1, True) add_rse_attribute(cls.rse3_id, cls.T1, True) add_rse_attribute(cls.rse4_id, cls.T2, True) add_rse_attribute(cls.rse5_id, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1_id, "fakeweight", 10) add_rse_attribute(cls.rse3_id, "fakeweight", 0) add_rse_attribute(cls.rse4_id, "fakeweight", 0) add_rse_attribute(cls.rse5_id, "fakeweight", 0) # Add quota cls.jdoe = InternalAccount('jdoe', **cls.vo) cls.root = InternalAccount('root', **cls.vo) set_local_account_limit(cls.jdoe, cls.rse1_id, -1) set_local_account_limit(cls.jdoe, cls.rse3_id, -1) set_local_account_limit(cls.jdoe, cls.rse4_id, -1) set_local_account_limit(cls.jdoe, cls.rse5_id, -1) set_local_account_limit(cls.root, cls.rse1_id, -1) set_local_account_limit(cls.root, cls.rse3_id, -1) set_local_account_limit(cls.root, cls.rse4_id, -1) set_local_account_limit(cls.root, cls.rse5_id, -1)
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.scopes = [ InternalScope(scope_name_generator(), **self.vo) for _ in range(5) ] self.jdoe = InternalAccount('jdoe', **self.vo)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} cls.rse_client = RSEClient() cls.tmp_rse_name = rse_name_generator() cls.rse_client.add_rse(cls.tmp_rse_name, vo=cls.vo) cls.tmp_rse = cls.rse_client.get_rse(cls.tmp_rse_name)['id']
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.scope = InternalScope('mock', **self.vo) self.rse = 'MOCK4' self.rse2 = 'MOCK3' self.account = InternalAccount('root', **self.vo) self.rse_id = get_rse_id(self.rse, **self.vo) self.rse2_id = get_rse_id(self.rse2, **self.vo) self.db_session = session.get_session()
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} cls.new_vo = {'vo': 'new'} cls.multi_vo = True if not vo_exists(**cls.new_vo): add_vo(description='Test', email='*****@*****.**', **cls.new_vo) else: cls.vo = {} cls.new_vo = {} cls.multi_vo = False # Add test account cls.account_name = ''.join( random.choice(string.ascii_lowercase) for x in range(10)) add_account(account=cls.account_name, type='user', email='*****@*****.**', issuer='root', **cls.vo) cls.account = InternalAccount(cls.account_name, **cls.vo) # Add test scope cls.scope_name = ''.join( random.choice(string.ascii_lowercase) for x in range(10)) add_scope(scope=cls.scope_name, account=cls.account_name, issuer='root', **cls.vo) cls.scope = InternalScope(cls.scope_name, **cls.vo) # Get test RSEs cls.rse_name = 'MOCK' cls.rse_id = get_rse_id(rse=cls.rse_name, **cls.vo) cls.rse2_name = 'MOCK2' cls.rse2_id = get_rse_id(rse=cls.rse2_name, **cls.vo) cls.rse3_name = rse_name_generator() cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', **cls.new_vo) cls.rse4_name = rse_name_generator() cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', **cls.new_vo) api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, **cls.new_vo)
def setUpClass(cls): cls.upload_client = UploadClient() cls.account_client = AccountClient() cls.session = get_session() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} cls.account = InternalAccount('root', **cls.vo) cls.scope = InternalScope('mock', **cls.vo) cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} self.filter = {'filter_': self.vo} else: self.vo = {} self.filter = {'filter_': {'vo': 'def'}} self.rse1 = rse_name_generator() self.rse2 = rse_name_generator() self.rse3 = rse_name_generator() self.rse4 = rse_name_generator() self.rse5 = rse_name_generator() self.rse1_id = rse.add_rse(self.rse1, **self.vo) self.rse2_id = rse.add_rse(self.rse2, **self.vo) self.rse3_id = rse.add_rse(self.rse3, **self.vo) self.rse4_id = rse.add_rse(self.rse4, **self.vo) self.rse5_id = rse.add_rse(self.rse5, **self.vo) # Add Attributes self.attribute = attribute_name_generator() rse.add_rse_attribute(self.rse1_id, self.attribute, "at") rse.add_rse_attribute(self.rse2_id, self.attribute, "de") rse.add_rse_attribute(self.rse3_id, self.attribute, "fr") rse.add_rse_attribute(self.rse4_id, self.attribute, "uk") rse.add_rse_attribute(self.rse5_id, self.attribute, "us") # Add numeric Attributes self.attribute_numeric = attribute_name_generator() rse.add_rse_attribute(self.rse1_id, self.attribute_numeric, 10) rse.add_rse_attribute(self.rse2_id, self.attribute_numeric, 20) rse.add_rse_attribute(self.rse3_id, self.attribute_numeric, 30) rse.add_rse_attribute(self.rse4_id, self.attribute_numeric, 40) rse.add_rse_attribute(self.rse5_id, self.attribute_numeric, 50) # Add Tags self.tag1 = tag_generator() self.tag2 = tag_generator() rse.add_rse_attribute(self.rse1_id, self.tag1, True) rse.add_rse_attribute(self.rse2_id, self.tag1, True) rse.add_rse_attribute(self.rse3_id, self.tag1, True) rse.add_rse_attribute(self.rse4_id, self.tag2, True) rse.add_rse_attribute(self.rse5_id, self.tag2, True)
def setUpClass(cls): cls.dataset = 'dataset_%s' % generate_uuid() cls.rule_client = RuleClient() cls.did_client = DIDClient() cls.replica_client = ReplicaClient() cls.upload_client = UploadClient() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} cls.rse_id = get_rse_id(rse=cls.rse, **cls.vo)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} cls.db_session = session.get_session() cls.dest_rse = 'MOCK' cls.dest_rse2 = 'MOCK2' cls.source_rse = 'MOCK4' cls.source_rse2 = 'MOCK5' cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo) cls.dest_rse_id2 = get_rse_id(cls.dest_rse2, **cls.vo) cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo) cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo)
def setUp(self): ''' INTERNAL TYPES: Setup the tests ''' if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.base = InternalType('test', **self.vo) self.same = InternalType('test', **self.vo) self.diff = InternalType('different', **self.vo) self.base_account = InternalAccount('test', **self.vo) self.base_scope = InternalScope('test', **self.vo)
def setUp(self): self.session = get_session() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.tmp_scope = InternalScope('mock', **self.vo) self.root = InternalAccount('root', **self.vo) self.mongo_meta = MongoDidMeta(host='mongo', port=27017, db='test_db', collection='test_collection')
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse_id(cls.rse1, **cls.vo) cls.rse3_id = get_rse_id(cls.rse3, **cls.vo) cls.rse4_id = get_rse_id(cls.rse4, **cls.vo) cls.rse5_id = get_rse_id(cls.rse5, **cls.vo) # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1_id, cls.T1, True) add_rse_attribute(cls.rse3_id, cls.T1, True) add_rse_attribute(cls.rse4_id, cls.T2, True) add_rse_attribute(cls.rse5_id, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1_id, "fakeweight", 10) add_rse_attribute(cls.rse3_id, "fakeweight", 0) add_rse_attribute(cls.rse4_id, "fakeweight", 0) add_rse_attribute(cls.rse5_id, "fakeweight", 0) # Add quota cls.jdoe = InternalAccount('jdoe', **cls.vo) cls.root = InternalAccount('root', **cls.vo) set_local_account_limit(cls.jdoe, cls.rse1_id, -1) set_local_account_limit(cls.jdoe, cls.rse3_id, -1) set_local_account_limit(cls.jdoe, cls.rse4_id, -1) set_local_account_limit(cls.jdoe, cls.rse5_id, -1) set_local_account_limit(cls.root, cls.rse1_id, -1) set_local_account_limit(cls.root, cls.rse3_id, -1) set_local_account_limit(cls.root, cls.rse4_id, -1) set_local_account_limit(cls.root, cls.rse5_id, -1)
def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): cls.vo = {'vo': get_vo()} else: cls.vo = {} cls.account = InternalAccount('jdoe', **cls.vo) cls.rse_1_name = 'MOCK4' cls.rse_2_name = 'MOCK5' cls.mock1_id = get_rse_id(cls.rse_1_name, **cls.vo) cls.mock2_id = get_rse_id(cls.rse_2_name, **cls.vo) cls.db_session = session.get_session() cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False} cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.root = InternalAccount('root', **self.vo) # add an S3 storage with a replica self.rc = client.ReplicaClient() self.rses3 = rse_name_generator() self.rses3_id = add_rse(self.rses3, **self.vo) add_protocol(self.rses3_id, {'scheme': 'https', 'hostname': 'fake-rucio.s3-eu-south-8.amazonaws.com', 'port': 443, 'prefix': '/', 'impl': 'rucio.rse.protocols.gfal.NoRename', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}}) add_rse_attribute(rse_id=self.rses3_id, key='sign_url', value='s3') add_rse_attribute(rse_id=self.rses3_id, key='fts', value='localhost') self.files3 = [{'scope': InternalScope('mock', **self.vo), 'name': 'file-on-aws', 'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 123}}] add_replicas(rse_id=self.rses3_id, files=self.files3, account=self.root) # add a non-S3 storage with a replica self.rsenons3 = rse_name_generator() self.rsenons3_id = add_rse(self.rsenons3, **self.vo) add_protocol(self.rsenons3_id, {'scheme': 'https', 'hostname': 'somestorage.ch', 'port': 1094, 'prefix': '/my/prefix', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}}) add_rse_attribute(rse_id=self.rsenons3_id, key='fts', value='localhost') self.filenons3 = [{'scope': InternalScope('mock', **self.vo), 'name': 'file-on-storage', 'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 321}}] add_replicas(rse_id=self.rsenons3_id, files=self.filenons3, account=self.root) # set the distance both ways add_distance(self.rses3_id, self.rsenons3_id, ranking=1, agis_distance=1, geoip_distance=1) add_distance(self.rsenons3_id, self.rses3_id, ranking=1, agis_distance=1, geoip_distance=1)
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.rc = client.ReplicaClient() self.rse1 = rse_name_generator() self.rse2 = rse_name_generator() self.rse1_id = add_rse(self.rse1, **self.vo) self.rse2_id = add_rse(self.rse2, **self.vo) add_protocol(self.rse1_id, {'scheme': 'https', 'hostname': 'storage.googleapis.com', 'port': 443, 'prefix': '/atlas-europe-west1/', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}}) add_protocol(self.rse2_id, {'scheme': 'https', 'hostname': 'storage.googleapis.com', 'port': 443, 'prefix': '/atlas-europe-east1/', 'impl': 'rucio.rse.protocols.gfal.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}}) # register some files there self.files = [{'scope': InternalScope('mock', **self.vo), 'name': 'file-on-gcs_%s' % i, 'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 666}} for i in range(0, 3)] root = InternalAccount('root', **self.vo) add_replicas(rse_id=self.rse1_id, files=self.files, account=root, ignore_availability=True) add_replicas(rse_id=self.rse2_id, files=self.files, account=root, ignore_availability=True)
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.rse1 = rse_name_generator() self.rse2 = rse_name_generator() self.rse3 = rse_name_generator() self.rse4 = rse_name_generator() self.rse5 = rse_name_generator() self.rse1_id = rse.add_rse(self.rse1, **self.vo) self.rse2_id = rse.add_rse(self.rse2, **self.vo) self.rse3_id = rse.add_rse(self.rse3, **self.vo) self.rse4_id = rse.add_rse(self.rse4, **self.vo) self.rse5_id = rse.add_rse(self.rse5, **self.vo) # Add Attributes self.attribute = attribute_name_generator() rse.add_rse_attribute(self.rse1_id, self.attribute, "at") rse.add_rse_attribute(self.rse2_id, self.attribute, "de") rse.add_rse_attribute(self.rse3_id, self.attribute, "fr") rse.add_rse_attribute(self.rse4_id, self.attribute, "uk") rse.add_rse_attribute(self.rse5_id, self.attribute, "us") # Add Tags self.tag1 = tag_generator() self.tag2 = tag_generator() rse.add_rse_attribute(self.rse1_id, self.tag1, True) rse.add_rse_attribute(self.rse2_id, self.tag1, True) rse.add_rse_attribute(self.rse3_id, self.tag1, True) rse.add_rse_attribute(self.rse4_id, self.tag2, True) rse.add_rse_attribute(self.rse5_id, self.tag2, True) self.rse_client = RSEClient()
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.accountstring = 'test_' + rndstr() self.accountstring = self.accountstring.lower() self.account = InternalAccount(self.accountstring, **self.vo) try: add_account(self.accountstring, 'USER', '*****@*****.**', 'root', **self.vo) except Duplicate: pass # create 2 sessions that expire in 5 min and 3 that expire 'now' save_oauth_session_params(self.account, 300) save_oauth_session_params(self.account, 300) save_oauth_session_params(self.account, 0) save_oauth_session_params(self.account, 0) save_oauth_session_params(self.account, 0) assert get_oauth_session_param_count(self.account) == 5 # assuming daemon looprate of 10 min # test cases for access tokens without any refresh token save_oidc_token(self.account, 0, 0, None, False, '0_to_be_deleted') save_oidc_token(self.account, 300, 0, None, False, '00_to_be_kept') save_oidc_token(self.account, 1000, 0, None, False, '000_to_be_kept') # test cases for access token with refresh token save_oidc_token(self.account, 0, 300, "1_at_inval_rt_val_refresh_False_" + rndstr(), False, "1_to_be_kept_no_refresh") save_oidc_token(self.account, 300, 300, "2_at_val_rt_val_refresh_False_" + rndstr(), False, "2_to_be_kept_no_refresh") save_oidc_token(self.account, 0, 0, "3_at_inval_rt_inval_refresh_False_" + rndstr(), False, "3_to_be_deleted") save_oidc_token(self.account, 300, 0, "4_at_val_rt_inval_refresh_False_" + rndstr(), False, "4_to_be_kept_no_refresh") save_oidc_token(self.account, 0, 1000, "5_at_inval_rt_longval_refresh_False_" + rndstr(), False, "5_to_be_kept_no_refresh") save_oidc_token(self.account, 1000, 1000, "6_at_longval_rt_longval_refresh_False_" + rndstr(), False, "6_to_be_kept_no_refresh") save_oidc_token(self.account, 1000, 0, "7_at_longval_rt_inval_refresh_False_" + rndstr(), False, "7_to_be_kept_no_refresh") save_oidc_token(self.account, 300, 1000, "8_at_val_rt_longval_refresh_False_" + rndstr(), False, "8_to_be_kept_no_refresh") save_oidc_token(self.account, 1000, 300, "9_at_longval_rt_val_refresh_False_" + rndstr(), False, "9_to_be_kept_no_refresh") save_oidc_token(self.account, 0, 300, "10_at_inval_rt_val_refresh_True_" + rndstr(), True, "10_original_refreshed_and_deleted") save_oidc_token(self.account, 300, 300, "11_at_val_rt_val_refresh_True_" + rndstr(), True, "11_to_be_kept_and_refreshed") save_oidc_token(self.account, 0, 0, "12_at_inval_rt_inval_refresh_True_" + rndstr(), True, "12_to_be_deleted") save_oidc_token(self.account, 300, 0, "13_at_val_rt_inval_refresh_True_" + rndstr(), True, "13_to_be_kept_no_refresh") save_oidc_token(self.account, 0, 1000, "14_at_inval_rt_longval_refresh_True_" + rndstr(), True, "14_original_refreshed_and_deleted") save_oidc_token(self.account, 1000, 1000, "15_at_longval_rt_longval_refresh_True_" + rndstr(), True, "15_to_be_kept_no_refresh") save_oidc_token(self.account, 1000, 0, "16_at_longval_rt_inval_refresh_True_" + rndstr(), True, "16_to_be_kept_no_refresh") save_oidc_token(self.account, 300, 1000, "17_at_val_rt_longval_refresh_True_" + rndstr(), True, "17_to_be_kept_and_refreshed") save_oidc_token(self.account, 1000, 300, "18_at_longval_rt_val_refresh_True_" + rndstr(), True, "18_to_be_kept_no_refresh") assert get_token_count(self.account) == 21 sleep(1)
def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': get_vo()} else: self.vo = {} self.replica_client = ReplicaClient() # Using two test RSEs self.rse4suspicious = 'MOCK_SUSPICIOUS' self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo) self.rse4recovery = 'MOCK_RECOVERY' self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo) self.scope = 'mock' self.internal_scope = InternalScope(self.scope, **self.vo) # For testing, we create 3 files and upload them to Rucio to two test RSEs. self.tmp_file1 = file_generator() self.tmp_file2 = file_generator() self.tmp_file3 = file_generator() self.tmp_file4 = file_generator() self.tmp_file5 = file_generator() self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE} for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5]] for rse in [self.rse4suspicious, self.rse4recovery]: cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5) exitcode, out, err = execute(cmd) # checking if Rucio upload went OK assert exitcode == 0 # Set fictional datatypes set_metadata(self.internal_scope, path.basename(self.tmp_file4), 'datatype', 'testtypedeclarebad') set_metadata(self.internal_scope, path.basename(self.tmp_file5), 'datatype', 'testtypenopolicy') # Allow for the RSEs to be affected by the suspicious file recovery daemon add_rse_attribute(self.rse4suspicious_id, "enable_suspicious_file_recovery", True) add_rse_attribute(self.rse4recovery_id, "enable_suspicious_file_recovery", True) # removing physical files from /tmp location - keeping only their DB info remove(self.tmp_file1) remove(self.tmp_file2) remove(self.tmp_file3) remove(self.tmp_file4) remove(self.tmp_file5) # Gather replica info replicalist = list_replicas(dids=self.listdids) # Changing the replica statuses as follows: # ---------------------------------------------------------------------------------------------------------------------------------- # Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype" # ---------------------------------------------------------------------------------------------------------------------------------- # tmp_file1 available suspicious (available) # tmp_file2 available suspicious + bad (unavailable) # tmp_file3 unavailable suspicious (available) RAW # tmp_file4 unavailable suspicious (available) testtypedeclarebad # tmp_file5 unavailable suspicious (available) testtypenopolicy # ---------------------------------------------------------------------------------------------------------------------------------- for replica in replicalist: suspicious_pfns = replica['rses'][self.rse4suspicious_id] for i in range(3): print("Declaring suspicious file replica: " + suspicious_pfns[0]) self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.') sleep(1) if replica['name'] == path.basename(self.tmp_file2): print("Declaring bad file replica: " + suspicious_pfns[0]) self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason') if replica['name'] == path.basename(self.tmp_file3): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE) if replica['name'] == path.basename(self.tmp_file4): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file4), ReplicaState.UNAVAILABLE) if replica['name'] == path.basename(self.tmp_file5): print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0]) update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file5), ReplicaState.UNAVAILABLE) # Gather replica info after setting initial replica statuses replicalist = list_replicas(dids=self.listdids) # Checking if the status changes were effective for replica in replicalist: if replica['name'] == path.basename(self.tmp_file1): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file2): assert (self.rse4suspicious_id in replica['states']) is False assert replica['states'][self.rse4recovery_id] == 'AVAILABLE' if replica['name'] == path.basename(self.tmp_file3): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file4): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False if replica['name'] == path.basename(self.tmp_file5): assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE' assert (self.rse4recovery_id in replica['states']) is False # Checking if only self.tmp_file2 is declared as 'BAD' self.from_date = datetime.now() - timedelta(days=1) bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo) bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list] assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def vo(): from rucio.tests.common_server import get_vo return get_vo()