def test_get_auth_token_ssh_fail(self):
"""AUTHENTICATION (CORE): SSH RSA public key exchange (wrong signature)."""
root = InternalAccount('root')
try:
add_account_identity(PUBLIC_KEY, IdentityType.SSH, root, email='*****@*****.**')
except Duplicate:
pass # might already exist, can skip
signature = ssh_sign(PRIVATE_KEY, 'sign_something_else')
result = get_auth_token_ssh(account='root', signature=signature, appid='test', ip='127.0.0.1')
assert_is_none(result)
del_account_identity(PUBLIC_KEY, IdentityType.SSH, root)
def del_account_attribute(key, account, issuer, vo='def'):
"""
Delete an attribute to an account.
:param key: attribute key.
:param account: The account name.
:param issuer: The issuer account.
:param vo: The VO to act on.
"""
kwargs = {'account': account, 'key': key}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='del_attribute', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not delete attribute' % (issuer))
account = InternalAccount(account, vo=vo)
account_core.del_account_attribute(account, key)
def setUpClass(cls):
# Add test account
cls.account = InternalAccount(''.join(
random.choice(string.ascii_uppercase) for x in range(10)))
add_account(account=cls.account,
type=AccountType.USER,
email='*****@*****.**')
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse_id(rse=cls.rse1)
cls.rse2_id = get_rse_id(rse=cls.rse2)
cls.db_session = session.get_session()
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.scope = InternalScope('mock', **self.vo)
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = InternalAccount('root', **self.vo)
self.rse_id = get_rse_id(self.rse, **self.vo)
self.rse2_id = get_rse_id(self.rse2, **self.vo)
self.db_session = session.get_session()
def get_global_account_limit(account, rse_expression, vo='def'):
"""
Lists the limitation names/values for the specified account name and rse expression.
REST API: http://<host>:<port>/rucio/account/<account>/limits
:param account: The account name.
:param rse_expression: The rse expression.
:param vo: The VO to act on.
:returns: The account limit.
"""
account = InternalAccount(account, vo=vo)
return {rse_expression: account_limit_core.get_global_account_limit(account=account, rse_expression=rse_expression)}
def setup(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {
'vo':
config_get('client',
'vo',
raise_exception=False,
default='tst')
}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
def __add_test_rse_and_replicas(vo, scope, rse_name, names, file_size, epoch_tombstone=False):
rse_id = rse_core.add_rse(rse_name, vo=vo)
rse_core.add_protocol(rse_id=rse_id, parameter=__mock_protocol)
tombstone = datetime.utcnow() - timedelta(days=1)
if epoch_tombstone:
tombstone = datetime(year=1970, month=1, day=1)
dids = []
for file_name in names:
dids.append({'scope': scope, 'name': file_name})
replica_core.add_replica(rse_id=rse_id, scope=scope,
name=file_name, bytes_=file_size,
tombstone=tombstone,
account=InternalAccount('root', vo=vo), adler32=None, md5=None)
return rse_name, rse_id, dids
def setUp(self):
self.session = get_session()
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.tmp_scope = InternalScope('mock', **self.vo)
self.root = InternalAccount('root', **self.vo)
self.mongo_meta = MongoDidMeta(host='mongo',
port=27017,
db='test_db',
collection='test_collection')
def del_account_identity(identity_key, id_type, account, issuer):
"""
Removes a membership association between identity and account.
:param identity_key: The identity key name. For example x509 DN, or a username.
:param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
:param account: The account name.
:param issuer: The issuer account.
"""
kwargs = {'account': account}
if not permission.has_permission(issuer=issuer, action='del_account_identity', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not delete account identity' % (issuer))
account = InternalAccount(account)
return identity.del_account_identity(identity_key, IdentityType.from_sym(id_type), account)
def mock_request(db_session, vo, source_rse, dest_rse, file):
account = InternalAccount('root', vo=vo)
def teardown(req):
delete_replicas(rse_id=source_rse['id'], files=[file], session=req.db_session)
add_replicas(rse_id=source_rse['id'], files=[file], account=account, session=db_session)
with GeneratedRequest(
scope=file['scope'],
name=file['name'],
dest_rse_id=dest_rse['id'],
account=account,
db_session=db_session,
teardown_func=teardown,
) as rucio_request:
yield rucio_request.db_object
def import_data(data, issuer, vo='def'):
"""
Import data to add/update/delete records in Rucio.
:param data: data to be imported.
:param issuer: the issuer.
:param vo: the VO of the issuer.
"""
kwargs = {'issuer': issuer}
validate_schema(name='import', obj=data)
if not permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not import data' % issuer)
for account in data.get('accounts', []):
account['account'] = InternalAccount(account['account'], vo=vo)
return importer.import_data(data, vo=vo)
def get_local_account_limit(account, rse):
"""
Lists the limitation names/values for the specified account name and rse name.
REST API: http://<host>:<port>/rucio/account/<account>/limits
:param account: The account name.
:param rse: The rse name.
:returns: The account limit.
"""
account = InternalAccount(account)
rse_id = get_rse_id(rse=rse)
return {rse: account_limit_core.get_local_account_limit(account=account, rse_id=rse_id)}
def setUp(self):
''' INTERNAL TYPES: Setup the tests '''
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': get_vo()}
else:
self.vo = {}
self.base = InternalType('test', **self.vo)
self.same = InternalType('test', **self.vo)
self.diff = InternalType('different', **self.vo)
self.base_account = InternalAccount('test', **self.vo)
self.base_scope = InternalScope('test', **self.vo)
def del_account(account, issuer, vo='def'):
"""
Disables an account with the provided account name.
:param account: The account name.
:param issuer: The issuer account.
:param vo: The VO to act on.
"""
kwargs = {'account': account}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='del_account', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not delete account' % (issuer))
account = InternalAccount(account, vo=vo)
account_core.del_account(account)
def add_files(lfns, issuer, ignore_availability):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blacklisted sites.
"""
scopes = list_scopes()
dids = []
rses = {}
for lfn in lfns:
scope, name = extract_scope(lfn['lfn'], scopes)
dids.append({'scope': scope, 'name': name})
rse = lfn['rse']
if rse not in rses:
rse_id = get_rse_id(rse=rse)
rses[rse] = rse_id
lfn['rse_id'] = rses[rse]
# Check if the issuer can add dids and use skip_availabitlity
for rse in rses:
rse_id = rses[rse]
kwargs = {'rse': rse, 'rse_id': rse_id}
if not has_permission(
issuer=issuer, action='add_replicas', kwargs=kwargs):
raise AccessDenied('Account %s can not add file replicas on %s' %
(issuer, rse))
if not has_permission(issuer=issuer,
action='skip_availability_check',
kwargs=kwargs):
ignore_availability = False
# Check if the issuer can add the files
kwargs = {'issuer': issuer, 'dids': dids}
if not has_permission(issuer=issuer, action='add_dids', kwargs=kwargs):
raise AccessDenied('Account %s can not bulk add data identifier' %
(issuer))
issuer = InternalAccount(issuer)
dirac.add_files(lfns=lfns,
account=issuer,
ignore_availability=ignore_availability,
session=None)
def test_list_rules_states(vo, rest_client, auth_token):
""" SUBSCRIPTION (REST): Test listing of rule states for subscription """
tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
root = InternalAccount('root', vo=vo)
add_scope(tmp_scope, root)
site_a = 'RSE%s' % uuid().upper()
site_b = 'RSE%s' % uuid().upper()
site_a_id = add_rse(site_a, vo=vo)
site_b_id = add_rse(site_b, vo=vo)
# Add quota
set_local_account_limit(root, site_a_id, -1)
set_local_account_limit(root, site_b_id, -1)
# add a new dataset
dsn = 'dataset-%s' % uuid()
add_did(scope=tmp_scope, name=dsn,
did_type=DIDType.DATASET, account=root)
subscription_name = uuid()
subid = add_subscription(name=subscription_name,
account='root',
filter_={'account': ['root', ], 'scope': [tmp_scope.external, ]},
replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}],
lifetime=100000,
retroactive=0,
dry_run=0,
comments='We want a shrubbery',
issuer='root',
vo=vo)
# Add two rules
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid)
response = rest_client.get('/subscriptions/%s/%s/Rules/States' % ('root', subscription_name), headers=headers(auth(auth_token)))
assert response.status_code == 200
rulestates = None
for line in response.get_data(as_text=True).split('\n'):
if line:
rulestates = loads(line)
if rulestates[1] == subscription_name:
break
assert rulestates is not None
assert rulestates[3] == 2
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse_id(rse='MOCK', **self.vo)
account = InternalAccount('jdoe', **self.vo)
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert cnt == {'files': 0, 'bytes': 0}
count, sum_ = 0, 0
for i in range(10):
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes_=2.147e+9)
account_update(once=True)
count += 1
sum_ += 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert cnt == {'files': count, 'bytes': sum_}
for i in range(4):
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes_=2.147e+9)
account_update(once=True)
count -= 1
sum_ -= 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert cnt == {'files': count, 'bytes': sum_}
for i in range(5):
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes_=2.147e+9)
account_update(once=True)
count += 1
sum_ += 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert cnt == {'files': count, 'bytes': sum_}
for i in range(8):
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes_=2.147e+9)
account_update(once=True)
count -= 1
sum_ -= 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert cnt == {'files': count, 'bytes': sum_}
def generate_rse(endpoint, token):
rse_name = 'RSE%s' % generate_uuid().upper()
scheme = 'https'
impl = 'rucio.rse.protocols.webdav.Default'
if not endpoint.startswith('https://'):
scheme = 'srm'
impl = 'rucio.rse.protocols.srm.Default'
tmp_proto = {
'impl': impl,
'scheme': scheme,
'domains': {
'lan': {
'read': 1,
'write': 1,
'delete': 1
},
'wan': {
'read': 1,
'write': 1,
'delete': 1
}
}
}
rse_id = rse.add_rse(rse_name, vo='def')
tmp_proto['hostname'] = endpoint.split(':')[1][2:]
tmp_proto['port'] = endpoint.split(':')[2].split('/')[0]
tmp_proto['prefix'] = '/'.join([''] +
endpoint.split(':')[2].split('/')[1:])
if scheme == 'srm':
tmp_proto['extended_attributes'] = {
'space_token': token,
'web_service_path': '/srm/managerv2?SFN='
}
rse.add_protocol(rse_id=rse_id, parameter=tmp_proto)
rse.add_rse_attribute(rse_id=rse_id,
key='fts',
value='https://fts3-pilot.cern.ch:8446')
account_limit.set_account_limit(account=InternalAccount('root', vo='def'),
rse_id=rse_id,
bytes=-1)
return rsemanager.get_rse_info(rse_id=rse_id)
def declare_bad_file_replicas(pfns, reason, issuer, vo='def', session=None):
"""
Declare a list of bad replicas.
:param pfns: Either a list of PFNs (string) or a list of replicas {'scope': <scope>, 'name': <name>, 'rse_id': <rse_id>}.
:param reason: The reason of the loss.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
"""
kwargs = {}
rse_map = {}
if not permission.has_permission(issuer=issuer,
vo=vo,
action='declare_bad_file_replicas',
kwargs=kwargs,
session=session):
raise exception.AccessDenied(
'Account %s can not declare bad replicas' % (issuer))
issuer = InternalAccount(issuer, vo=vo)
type_ = type(pfns[0]) if len(pfns) > 0 else None
for pfn in pfns:
if not isinstance(pfn, type_):
raise exception.InvalidType(
'The PFNs must be either a list of string or list of dict')
if type_ == dict:
rse = pfn['rse']
if rse not in rse_map:
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
rse_map[rse] = rse_id
pfn['rse_id'] = rse_map[rse]
pfn['scope'] = InternalScope(pfn['scope'], vo=vo)
replicas = replica.declare_bad_file_replicas(pfns=pfns,
reason=reason,
issuer=issuer,
status=BadFilesStatus.BAD,
session=session)
for k in list(replicas):
try:
rse = get_rse_name(rse_id=k, session=session)
replicas[rse] = replicas.pop(k)
except exception.RSENotFound:
pass
return replicas
def list_subscription_rule_states(name=None, account=None):
"""Returns a list of with the number of rules per state for a subscription.
:param name: Name of the subscription
:param account: Account identifier
:param session: The database session in use.
:returns: List with tuple (account, name, state, count)
"""
if account is not None:
account = InternalAccount(account)
subs = subscription.list_subscription_rule_states(name, account)
for sub in subs:
# sub is an immutable KeyedTuple so return new KeyedTuple with edited entries
labels = sub._fields
d = sub._asdict()
d['account'] = d['account'].external
yield KeyedTuple([d[l] for l in labels], labels=labels)
def test_touch_dids_access_cnt(self):
""" DATA IDENTIFIERS (CORE): Increase dids access_cnt"""
tmp_scope = InternalScope('mock')
root = InternalAccount('root')
tmp_dsn1 = 'dsn_%s' % generate_uuid()
tmp_dsn2 = 'dsn_%s' % generate_uuid()
add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account=root)
add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account=root)
assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
for i in range(100):
touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET}])
assert_equal(100, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
def get_account_limits(account):
"""
Lists the limitation names/values for the specified account name.
REST API: http://<host>:<port>/rucio/account/<account>/limits
:param account: The account name.
:returns: The account limits.
"""
account = InternalAccount(account)
rse_instead_id = {}
for elem in account_limit_core.get_account_limits(account=account).items():
rse_instead_id[get_rse_name(rse_id=elem[0])] = elem[1]
return rse_instead_id
def remove_did_from_followed(scope, name, account, issuer, session=None, vo='def'):
"""
Mark a did as not followed
:param scope: The scope name.
:param name: The data identifier name.
:param account: The account owner.
:param session: The database session in use.
:param issuer: The issuer account
"""
kwargs = {'scope': scope, 'issuer': issuer}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='remove_did_from_followed', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not remove data identifiers from followed table' % (issuer))
scope = InternalScope(scope, vo=vo)
account = InternalAccount(account, vo=vo)
return did.remove_did_from_followed(scope=scope, name=name, account=account, session=session)
def test_abacus_collection_replica_cleanup(self):
""" ABACUS (COLLECTION REPLICA): Test if the cleanup procedure works correctly. """
collection_replica.run(once=True)
db_session = session.get_session()
rse1 = rse_name_generator()
rse_id1 = add_rse(rse1, **self.vo)
rse2 = rse_name_generator()
rse_id2 = add_rse(rse2, **self.vo)
scope = InternalScope('mock', **self.vo)
dataset = 'dataset_%s' % generate_uuid()
jdoe = InternalAccount('jdoe', **self.vo)
add_did(scope, dataset, DIDType.DATASET, jdoe)
models.CollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1,
state=ReplicaState.AVAILABLE,
bytes=1).save(session=db_session, flush=False)
models.CollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2,
state=ReplicaState.AVAILABLE,
bytes=1).save(session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id1).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope,
name=dataset,
rse_id=rse_id2).save(
session=db_session, flush=False)
models.UpdatedCollectionReplica(scope=scope, name=dataset,
rse_id=None).save(session=db_session,
flush=False)
db_session.commit()
assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3
def test_list_account_identities(self):
""" ACCOUNT (CORE): Test listing of account identities """
email = 'email'
identity = uuid()
identity_type = IdentityType.USERPASS
account = InternalAccount('root', **self.vo)
add_account_identity(identity,
identity_type,
account,
email,
password='******')
identities = list_identities(account)
assert {
'type': identity_type,
'identity': identity,
'email': email
} in identities
def setUpClass(cls):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
cls.vo = {'vo': get_vo()}
else:
cls.vo = {}
cls.account = InternalAccount('jdoe', **cls.vo)
cls.rse_1_name = 'MOCK4'
cls.rse_2_name = 'MOCK5'
cls.mock1_id = get_rse_id(cls.rse_1_name, **cls.vo)
cls.mock2_id = get_rse_id(cls.rse_2_name, **cls.vo)
cls.db_session = session.get_session()
cls.rse_1 = {'id': cls.mock1_id, 'staging_area': False}
cls.rse_2 = {'id': cls.mock2_id, 'staging_area': False}
def setUpClass(cls):
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
cls.vo = {'vo': 'tst'}
else:
cls.vo = {}
cls.db_session = session.get_session()
cls.dest_rse = 'MOCK'
cls.source_rse = 'MOCK4'
cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo)
cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo)
cls.scope = InternalScope('mock', **cls.vo)
cls.account = InternalAccount('root', **cls.vo)
cls.user_activity = 'User Subscription'
cls.all_activities = 'all_activities'
set_rse_transfer_limits(cls.dest_rse_id, cls.user_activity, max_transfers=1, session=cls.db_session)
set('throttler_release_strategy', 'dest_%s' % cls.dest_rse_id, 'fifo', session=cls.db_session)
def setUp(self):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
self.vo = {'vo': 'tst'}
else:
self.vo = {}
self.account = InternalAccount('root', **self.vo)
self.scope = InternalScope('mock', **self.vo)
self.upload_client = UploadClient()
self.account_client = AccountClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse_id(self.rse, **self.vo)
self.session = get_session()
def test_ssh_fail(self):
"""AUTHENTICATION (REST): SSH RSA public key exchange (wrong credentials)."""
root = InternalAccount('root')
try:
add_account_identity(PUBLIC_KEY, IdentityType.SSH, root, email='*****@*****.**')
except Duplicate:
pass # might already exist, can skip
signature = ssh_sign(PRIVATE_KEY, 'sign_something_else')
options = []
headers = {'X-Rucio-Account': 'root', 'X-Rucio-SSH-Signature': signature}
result = TestApp(APP.wsgifunc(*options)).get('/ssh', headers=headers, expect_errors=True)
assert_equal(result.status, 401)
del_account_identity(PUBLIC_KEY, IdentityType.SSH, root)
def update_account(account, key, value, issuer='root', vo='def'):
""" Update a property of an account_core.
:param account: Name of the account_core.
:param key: Account property like status.
:param value: Property value.
:param issuer: The issuer account
:param vo: The VO to act on.
"""
validate_schema(name='account', obj=account, vo=vo)
kwargs = {}
if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='update_account', kwargs=kwargs):
raise rucio.common.exception.AccessDenied('Account %s can not change %s of the account' % (issuer, key))
account = InternalAccount(account, vo=vo)
return account_core.update_account(account, key, value)
