def test_core_temporary_dids():
""" TMP DATA IDENTIFIERS (CORE): """
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': get_vo()}
else:
vo = {}
scope = InternalScope('mock', **vo)
root = InternalAccount('root', **vo)
temporary_dids = []
rse = 'MOCK'
rse_id = get_rse_id(rse=rse, **vo)
for _ in range(10):
temporary_dids.append({'scope': scope,
'name': 'object_%s' % generate_uuid(),
'rse_id': rse_id,
'bytes': 1,
'path': None})
add_temporary_dids(dids=temporary_dids, account=root)
compose(scope=scope, name='file_%s' % generate_uuid(), rse_id=rse_id,
bytes=10, sources=temporary_dids, account=root,
md5=None, adler32=None, pfn=None, meta={}, rules=[],
parent_scope=None, parent_name=None)
dids = list_expired_temporary_dids(rse_id=rse_id, limit=10)
rowcount = delete_temporary_dids(dids=dids)
assert rowcount == 10
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
def test_quarantined_replicas():
""" QUARANTINED REPLICA (CORE): Add, List and Delete quarantined replicas """
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {'vo': get_vo()}
else:
vo = {}
rse_id = get_rse_id(rse='MOCK', **vo)
real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
limit=10000)
quarantined_replicas = len(real_replicas) + len(dark_replicas)
nbreplicas = 5
replicas = [{
'path': '/path/' + generate_uuid()
} for _ in range(nbreplicas)]
add_quarantined_replicas(rse_id=rse_id, replicas=replicas)
real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
limit=10000)
assert quarantined_replicas + nbreplicas == len(dark_replicas) + len(
real_replicas)
delete_quarantined_replicas(rse_id=rse_id, replicas=replicas)
real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
limit=10000)
assert quarantined_replicas == len(dark_replicas) + len(real_replicas)
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.multi_vo = True
else:
cls.vo = {}
cls.multi_vo = False
# Add test account
cls.account = InternalAccount(
''.join(random.choice(string.ascii_uppercase) for x in range(10)),
**cls.vo)
add_account(account=cls.account,
type_=AccountType.USER,
email='*****@*****.**')
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse2_id = get_rse_id(rse=cls.rse2, **cls.vo)
cls.db_session = session.get_session()
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.projects = ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV']
cls.pattern1 = r'(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
def setUp(self):
""" Setup the Test Case """
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.account = InternalAccount(account_name_generator(), **self.vo)
add_account(self.account, AccountType.USER, '*****@*****.**')
def setUp(self):
""" Setup Test Case """
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.usr = '******'
def setUpClass(cls):
cls.upload_client = UploadClient()
cls.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUp(self):
""" Constructor."""
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.did_client = DIDClient()
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
@transactional_session
def __cleanup_updated_dids(session=None):
session.query(UpdatedDID).delete()
__cleanup_updated_dids()
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse_id(rse=cls.rse1, **cls.vo)
cls.rse3_id = get_rse_id(rse=cls.rse3, **cls.vo)
cls.rse4_id = get_rse_id(rse=cls.rse4, **cls.vo)
cls.rse5_id = get_rse_id(rse=cls.rse5, **cls.vo)
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1_id, cls.T1, True)
add_rse_attribute(cls.rse3_id, cls.T1, True)
add_rse_attribute(cls.rse4_id, cls.T2, True)
add_rse_attribute(cls.rse5_id, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1_id, "fakeweight", 10)
add_rse_attribute(cls.rse3_id, "fakeweight", 0)
add_rse_attribute(cls.rse4_id, "fakeweight", 0)
add_rse_attribute(cls.rse5_id, "fakeweight", 0)
# Add quota
cls.jdoe = InternalAccount('jdoe', **cls.vo)
cls.root = InternalAccount('root', **cls.vo)
set_local_account_limit(cls.jdoe, cls.rse1_id, -1)
set_local_account_limit(cls.jdoe, cls.rse3_id, -1)
set_local_account_limit(cls.jdoe, cls.rse4_id, -1)
set_local_account_limit(cls.jdoe, cls.rse5_id, -1)
set_local_account_limit(cls.root, cls.rse1_id, -1)
set_local_account_limit(cls.root, cls.rse3_id, -1)
set_local_account_limit(cls.root, cls.rse4_id, -1)
set_local_account_limit(cls.root, cls.rse5_id, -1)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.scopes = [
InternalScope(scope_name_generator(), **self.vo) for _ in range(5)
]
self.jdoe = InternalAccount('jdoe', **self.vo)
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.rse_client = RSEClient()
cls.tmp_rse_name = rse_name_generator()
cls.rse_client.add_rse(cls.tmp_rse_name, vo=cls.vo)
cls.tmp_rse = cls.rse_client.get_rse(cls.tmp_rse_name)['id']
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.scope = InternalScope('mock', **self.vo)
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = InternalAccount('root', **self.vo)
self.rse_id = get_rse_id(self.rse, **self.vo)
self.rse2_id = get_rse_id(self.rse2, **self.vo)
self.db_session = session.get_session()
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.new_vo = {'vo': 'new'}
cls.multi_vo = True
if not vo_exists(**cls.new_vo):
add_vo(description='Test',
email='*****@*****.**',
**cls.new_vo)
else:
cls.vo = {}
cls.new_vo = {}
cls.multi_vo = False
# Add test account
cls.account_name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
add_account(account=cls.account_name,
type='user',
email='*****@*****.**',
issuer='root',
**cls.vo)
cls.account = InternalAccount(cls.account_name, **cls.vo)
# Add test scope
cls.scope_name = ''.join(
random.choice(string.ascii_lowercase) for x in range(10))
add_scope(scope=cls.scope_name,
account=cls.account_name,
issuer='root',
**cls.vo)
cls.scope = InternalScope(cls.scope_name, **cls.vo)
# Get test RSEs
cls.rse_name = 'MOCK'
cls.rse_id = get_rse_id(rse=cls.rse_name, **cls.vo)
cls.rse2_name = 'MOCK2'
cls.rse2_id = get_rse_id(rse=cls.rse2_name, **cls.vo)
cls.rse3_name = rse_name_generator()
cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', **cls.new_vo)
cls.rse4_name = rse_name_generator()
cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', **cls.new_vo)
api_rse.add_distance(cls.rse3_name,
cls.rse4_name,
issuer='root',
distance=3,
**cls.new_vo)
def setUpClass(cls):
cls.upload_client = UploadClient()
cls.account_client = AccountClient()
cls.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.account = InternalAccount('root', **cls.vo)
cls.scope = InternalScope('mock', **cls.vo)
cls.rse_id = get_rse_id(cls.rse, session=cls.session, **cls.vo)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
self.filter = {'filter_': self.vo}
else:
self.vo = {}
self.filter = {'filter_': {'vo': 'def'}}
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1, **self.vo)
self.rse2_id = rse.add_rse(self.rse2, **self.vo)
self.rse3_id = rse.add_rse(self.rse3, **self.vo)
self.rse4_id = rse.add_rse(self.rse4, **self.vo)
self.rse5_id = rse.add_rse(self.rse5, **self.vo)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add numeric Attributes
self.attribute_numeric = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute_numeric, 10)
rse.add_rse_attribute(self.rse2_id, self.attribute_numeric, 20)
rse.add_rse_attribute(self.rse3_id, self.attribute_numeric, 30)
rse.add_rse_attribute(self.rse4_id, self.attribute_numeric, 40)
rse.add_rse_attribute(self.rse5_id, self.attribute_numeric, 50)
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
def setUpClass(cls):
cls.dataset = 'dataset_%s' % generate_uuid()
cls.rule_client = RuleClient()
cls.did_client = DIDClient()
cls.replica_client = ReplicaClient()
cls.upload_client = UploadClient()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
cls.rse_id = get_rse_id(rse=cls.rse, **cls.vo)
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.dest_rse2 = 'MOCK2'
cls.source_rse = 'MOCK4'
cls.source_rse2 = 'MOCK5'
cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo)
cls.dest_rse_id2 = get_rse_id(cls.dest_rse2, **cls.vo)
cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo)
cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo)
def setUp(self):
''' INTERNAL TYPES: Setup the tests '''
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.base = InternalType('test', **self.vo)
self.same = InternalType('test', **self.vo)
self.diff = InternalType('different', **self.vo)
self.base_account = InternalAccount('test', **self.vo)
self.base_scope = InternalScope('test', **self.vo)
def setUp(self):
self.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
self.mongo_meta = MongoDidMeta(host='mongo',
port=27017,
db='test_db',
collection='test_collection')
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse_id(cls.rse1, **cls.vo)
cls.rse3_id = get_rse_id(cls.rse3, **cls.vo)
cls.rse4_id = get_rse_id(cls.rse4, **cls.vo)
cls.rse5_id = get_rse_id(cls.rse5, **cls.vo)
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1_id, cls.T1, True)
add_rse_attribute(cls.rse3_id, cls.T1, True)
add_rse_attribute(cls.rse4_id, cls.T2, True)
add_rse_attribute(cls.rse5_id, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1_id, "fakeweight", 10)
add_rse_attribute(cls.rse3_id, "fakeweight", 0)
add_rse_attribute(cls.rse4_id, "fakeweight", 0)
add_rse_attribute(cls.rse5_id, "fakeweight", 0)
# Add quota
cls.jdoe = InternalAccount('jdoe', **cls.vo)
cls.root = InternalAccount('root', **cls.vo)
set_local_account_limit(cls.jdoe, cls.rse1_id, -1)
set_local_account_limit(cls.jdoe, cls.rse3_id, -1)
set_local_account_limit(cls.jdoe, cls.rse4_id, -1)
set_local_account_limit(cls.jdoe, cls.rse5_id, -1)
set_local_account_limit(cls.root, cls.rse1_id, -1)
set_local_account_limit(cls.root, cls.rse3_id, -1)
set_local_account_limit(cls.root, cls.rse4_id, -1)
set_local_account_limit(cls.root, cls.rse5_id, -1)
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.account = InternalAccount('jdoe', **cls.vo)
cls.rse_1_name = 'MOCK4'
cls.rse_2_name = 'MOCK5'
cls.mock1_id = get_rse_id(cls.rse_1_name, **cls.vo)
cls.mock2_id = get_rse_id(cls.rse_2_name, **cls.vo)
cls.db_session = session.get_session()
cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False}
cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.root = InternalAccount('root', **self.vo)
# add an S3 storage with a replica
self.rc = client.ReplicaClient()
self.rses3 = rse_name_generator()
self.rses3_id = add_rse(self.rses3, **self.vo)
add_protocol(self.rses3_id, {'scheme': 'https',
'hostname': 'fake-rucio.s3-eu-south-8.amazonaws.com',
'port': 443,
'prefix': '/',
'impl': 'rucio.rse.protocols.gfal.NoRename',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}})
add_rse_attribute(rse_id=self.rses3_id, key='sign_url', value='s3')
add_rse_attribute(rse_id=self.rses3_id, key='fts', value='localhost')
self.files3 = [{'scope': InternalScope('mock', **self.vo), 'name': 'file-on-aws',
'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 123}}]
add_replicas(rse_id=self.rses3_id, files=self.files3, account=self.root)
# add a non-S3 storage with a replica
self.rsenons3 = rse_name_generator()
self.rsenons3_id = add_rse(self.rsenons3, **self.vo)
add_protocol(self.rsenons3_id, {'scheme': 'https',
'hostname': 'somestorage.ch',
'port': 1094,
'prefix': '/my/prefix',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}})
add_rse_attribute(rse_id=self.rsenons3_id, key='fts', value='localhost')
self.filenons3 = [{'scope': InternalScope('mock', **self.vo), 'name': 'file-on-storage',
'bytes': 1234, 'adler32': 'deadbeef', 'meta': {'events': 321}}]
add_replicas(rse_id=self.rsenons3_id, files=self.filenons3, account=self.root)
# set the distance both ways
add_distance(self.rses3_id, self.rsenons3_id, ranking=1, agis_distance=1, geoip_distance=1)
add_distance(self.rsenons3_id, self.rses3_id, ranking=1, agis_distance=1, geoip_distance=1)
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.rc = client.ReplicaClient()
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse1_id = add_rse(self.rse1, **self.vo)
self.rse2_id = add_rse(self.rse2, **self.vo)
add_protocol(self.rse1_id, {'scheme': 'https',
'hostname': 'storage.googleapis.com',
'port': 443,
'prefix': '/atlas-europe-west1/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}})
add_protocol(self.rse2_id, {'scheme': 'https',
'hostname': 'storage.googleapis.com',
'port': 443,
'prefix': '/atlas-europe-east1/',
'impl': 'rucio.rse.protocols.gfal.Default',
'domains': {
'lan': {'read': 1, 'write': 1, 'delete': 1},
'wan': {'read': 1, 'write': 1, 'delete': 1, 'third_party_copy': 1}}})
# register some files there
self.files = [{'scope': InternalScope('mock', **self.vo),
'name': 'file-on-gcs_%s' % i,
'bytes': 1234,
'adler32': 'deadbeef',
'meta': {'events': 666}} for i in range(0, 3)]
root = InternalAccount('root', **self.vo)
add_replicas(rse_id=self.rse1_id,
files=self.files,
account=root,
ignore_availability=True)
add_replicas(rse_id=self.rse2_id,
files=self.files,
account=root,
ignore_availability=True)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.rse1 = rse_name_generator()
self.rse2 = rse_name_generator()
self.rse3 = rse_name_generator()
self.rse4 = rse_name_generator()
self.rse5 = rse_name_generator()
self.rse1_id = rse.add_rse(self.rse1, **self.vo)
self.rse2_id = rse.add_rse(self.rse2, **self.vo)
self.rse3_id = rse.add_rse(self.rse3, **self.vo)
self.rse4_id = rse.add_rse(self.rse4, **self.vo)
self.rse5_id = rse.add_rse(self.rse5, **self.vo)
# Add Attributes
self.attribute = attribute_name_generator()
rse.add_rse_attribute(self.rse1_id, self.attribute, "at")
rse.add_rse_attribute(self.rse2_id, self.attribute, "de")
rse.add_rse_attribute(self.rse3_id, self.attribute, "fr")
rse.add_rse_attribute(self.rse4_id, self.attribute, "uk")
rse.add_rse_attribute(self.rse5_id, self.attribute, "us")
# Add Tags
self.tag1 = tag_generator()
self.tag2 = tag_generator()
rse.add_rse_attribute(self.rse1_id, self.tag1, True)
rse.add_rse_attribute(self.rse2_id, self.tag1, True)
rse.add_rse_attribute(self.rse3_id, self.tag1, True)
rse.add_rse_attribute(self.rse4_id, self.tag2, True)
rse.add_rse_attribute(self.rse5_id, self.tag2, True)
self.rse_client = RSEClient()
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.accountstring = 'test_' + rndstr()
self.accountstring = self.accountstring.lower()
self.account = InternalAccount(self.accountstring, **self.vo)
try:
add_account(self.accountstring, 'USER', '*****@*****.**', 'root',
**self.vo)
except Duplicate:
pass
# create 2 sessions that expire in 5 min and 3 that expire 'now'
save_oauth_session_params(self.account, 300)
save_oauth_session_params(self.account, 300)
save_oauth_session_params(self.account, 0)
save_oauth_session_params(self.account, 0)
save_oauth_session_params(self.account, 0)
assert get_oauth_session_param_count(self.account) == 5
# assuming daemon looprate of 10 min
# test cases for access tokens without any refresh token
save_oidc_token(self.account, 0, 0, None, False, '0_to_be_deleted')
save_oidc_token(self.account, 300, 0, None, False, '00_to_be_kept')
save_oidc_token(self.account, 1000, 0, None, False, '000_to_be_kept')
# test cases for access token with refresh token
save_oidc_token(self.account, 0, 300,
"1_at_inval_rt_val_refresh_False_" + rndstr(), False,
"1_to_be_kept_no_refresh")
save_oidc_token(self.account, 300, 300,
"2_at_val_rt_val_refresh_False_" + rndstr(), False,
"2_to_be_kept_no_refresh")
save_oidc_token(self.account, 0, 0,
"3_at_inval_rt_inval_refresh_False_" + rndstr(), False,
"3_to_be_deleted")
save_oidc_token(self.account, 300, 0,
"4_at_val_rt_inval_refresh_False_" + rndstr(), False,
"4_to_be_kept_no_refresh")
save_oidc_token(self.account, 0, 1000,
"5_at_inval_rt_longval_refresh_False_" + rndstr(),
False, "5_to_be_kept_no_refresh")
save_oidc_token(self.account, 1000, 1000,
"6_at_longval_rt_longval_refresh_False_" + rndstr(),
False, "6_to_be_kept_no_refresh")
save_oidc_token(self.account, 1000, 0,
"7_at_longval_rt_inval_refresh_False_" + rndstr(),
False, "7_to_be_kept_no_refresh")
save_oidc_token(self.account, 300, 1000,
"8_at_val_rt_longval_refresh_False_" + rndstr(), False,
"8_to_be_kept_no_refresh")
save_oidc_token(self.account, 1000, 300,
"9_at_longval_rt_val_refresh_False_" + rndstr(), False,
"9_to_be_kept_no_refresh")
save_oidc_token(self.account, 0, 300,
"10_at_inval_rt_val_refresh_True_" + rndstr(), True,
"10_original_refreshed_and_deleted")
save_oidc_token(self.account, 300, 300,
"11_at_val_rt_val_refresh_True_" + rndstr(), True,
"11_to_be_kept_and_refreshed")
save_oidc_token(self.account, 0, 0,
"12_at_inval_rt_inval_refresh_True_" + rndstr(), True,
"12_to_be_deleted")
save_oidc_token(self.account, 300, 0,
"13_at_val_rt_inval_refresh_True_" + rndstr(), True,
"13_to_be_kept_no_refresh")
save_oidc_token(self.account, 0, 1000,
"14_at_inval_rt_longval_refresh_True_" + rndstr(),
True, "14_original_refreshed_and_deleted")
save_oidc_token(self.account, 1000, 1000,
"15_at_longval_rt_longval_refresh_True_" + rndstr(),
True, "15_to_be_kept_no_refresh")
save_oidc_token(self.account, 1000, 0,
"16_at_longval_rt_inval_refresh_True_" + rndstr(),
True, "16_to_be_kept_no_refresh")
save_oidc_token(self.account, 300, 1000,
"17_at_val_rt_longval_refresh_True_" + rndstr(), True,
"17_to_be_kept_and_refreshed")
save_oidc_token(self.account, 1000, 300,
"18_at_longval_rt_val_refresh_True_" + rndstr(), True,
"18_to_be_kept_no_refresh")
assert get_token_count(self.account) == 21
sleep(1)
def setUp(self):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.replica_client = ReplicaClient()
# Using two test RSEs
self.rse4suspicious = 'MOCK_SUSPICIOUS'
self.rse4suspicious_id = get_rse_id(self.rse4suspicious, **self.vo)
self.rse4recovery = 'MOCK_RECOVERY'
self.rse4recovery_id = get_rse_id(self.rse4recovery, **self.vo)
self.scope = 'mock'
self.internal_scope = InternalScope(self.scope, **self.vo)
# For testing, we create 3 files and upload them to Rucio to two test RSEs.
self.tmp_file1 = file_generator()
self.tmp_file2 = file_generator()
self.tmp_file3 = file_generator()
self.tmp_file4 = file_generator()
self.tmp_file5 = file_generator()
self.listdids = [{'scope': self.internal_scope, 'name': path.basename(f), 'type': DIDType.FILE}
for f in [self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5]]
for rse in [self.rse4suspicious, self.rse4recovery]:
cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} {5} {6}'.format(rse, self.scope, self.tmp_file1, self.tmp_file2, self.tmp_file3, self.tmp_file4, self.tmp_file5)
exitcode, out, err = execute(cmd)
# checking if Rucio upload went OK
assert exitcode == 0
# Set fictional datatypes
set_metadata(self.internal_scope, path.basename(self.tmp_file4), 'datatype', 'testtypedeclarebad')
set_metadata(self.internal_scope, path.basename(self.tmp_file5), 'datatype', 'testtypenopolicy')
# Allow for the RSEs to be affected by the suspicious file recovery daemon
add_rse_attribute(self.rse4suspicious_id, "enable_suspicious_file_recovery", True)
add_rse_attribute(self.rse4recovery_id, "enable_suspicious_file_recovery", True)
# removing physical files from /tmp location - keeping only their DB info
remove(self.tmp_file1)
remove(self.tmp_file2)
remove(self.tmp_file3)
remove(self.tmp_file4)
remove(self.tmp_file5)
# Gather replica info
replicalist = list_replicas(dids=self.listdids)
# Changing the replica statuses as follows:
# ----------------------------------------------------------------------------------------------------------------------------------
# Name State(s) declared on MOCK_RECOVERY State(s) declared on MOCK_SUSPICIOUS Metadata "datatype"
# ----------------------------------------------------------------------------------------------------------------------------------
# tmp_file1 available suspicious (available)
# tmp_file2 available suspicious + bad (unavailable)
# tmp_file3 unavailable suspicious (available) RAW
# tmp_file4 unavailable suspicious (available) testtypedeclarebad
# tmp_file5 unavailable suspicious (available) testtypenopolicy
# ----------------------------------------------------------------------------------------------------------------------------------
for replica in replicalist:
suspicious_pfns = replica['rses'][self.rse4suspicious_id]
for i in range(3):
print("Declaring suspicious file replica: " + suspicious_pfns[0])
self.replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'This is a good reason.')
sleep(1)
if replica['name'] == path.basename(self.tmp_file2):
print("Declaring bad file replica: " + suspicious_pfns[0])
self.replica_client.declare_bad_file_replicas([suspicious_pfns[0], ], 'This is a good reason')
if replica['name'] == path.basename(self.tmp_file3):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file3), ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file4):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file4), ReplicaState.UNAVAILABLE)
if replica['name'] == path.basename(self.tmp_file5):
print("Updating replica state as unavailable: " + replica['rses'][self.rse4recovery_id][0])
update_replica_state(self.rse4recovery_id, self.internal_scope, path.basename(self.tmp_file5), ReplicaState.UNAVAILABLE)
# Gather replica info after setting initial replica statuses
replicalist = list_replicas(dids=self.listdids)
# Checking if the status changes were effective
for replica in replicalist:
if replica['name'] == path.basename(self.tmp_file1):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file2):
assert (self.rse4suspicious_id in replica['states']) is False
assert replica['states'][self.rse4recovery_id] == 'AVAILABLE'
if replica['name'] == path.basename(self.tmp_file3):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file4):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
if replica['name'] == path.basename(self.tmp_file5):
assert replica['states'][self.rse4suspicious_id] == 'AVAILABLE'
assert (self.rse4recovery_id in replica['states']) is False
# Checking if only self.tmp_file2 is declared as 'BAD'
self.from_date = datetime.now() - timedelta(days=1)
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4suspicious_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4suspicious_id, BadFilesStatus.BAD) not in bad_checklist
bad_replicas_list = list_bad_replicas_status(rse_id=self.rse4recovery_id, younger_than=self.from_date, **self.vo)
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
assert (path.basename(self.tmp_file1), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file2), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file3), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file4), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
assert (path.basename(self.tmp_file5), self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
def vo():
from rucio.tests.common_server import get_vo
return get_vo()
