def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = InternalScope('mock', **self.vo)
files = create_files(3, scope, self.rse1_id, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe)
attach_dids(scope, dataset, files, self.jdoe)
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
account_update(once=True)
account_counter_before = get_usage(self.rse1_id, self.jdoe)
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_usage(self.rse1_id, self.jdoe)
assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes'])
assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse_id(rse='MOCK', **self.vo)
account = InternalAccount('jdoe', **self.vo)
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': 0, 'bytes': 0})
count, sum = 0, 0
for i in range(10):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(4):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(5):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(8):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = get_usage(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
def get_usage(rse, account, issuer):
"""
Returns current values of the specified counter, or raises CounterNotFound if the counter does not exist.
:param rse: The RSE.
:param account: The account name.
:param issuer: The issuer account.
:returns: A dictionary with total and bytes.
"""
rse_id = get_rse_id(rse=rse)
return account_core.get_usage(rse_id, account)
def get_usage(rse, account, issuer, vo='def', session=None):
"""
Returns current values of the specified counter, or raises CounterNotFound if the counter does not exist.
:param rse: The RSE.
:param account: The account name.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: A dictionary with total and bytes.
"""
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
account = InternalAccount(account, vo=vo)
return account_core.get_usage(rse_id, account, session=session)
def __init__(self,
account,
rses,
weight,
copies,
ignore_account_limit=False,
session=None):
"""
Initialize the RSE Selector.
:param account: Account owning the rule.
:param rses: List of rse dictionaries.
:param weight: Weighting to use.
:param copies: Number of copies to create.
:param ignore_account_limit: Flag if the quota should be ignored.
:param session: DB Session in use.
:raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
"""
self.account = account
self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}]
self.copies = copies
if weight is not None:
for rse in rses:
attributes = list_rse_attributes(rse_id=rse['id'],
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
if weight not in attributes:
continue # The RSE does not have the required weight set, therefore it is ignored
try:
self.rses.append({
'rse_id': rse['id'],
'weight': float(attributes[weight]),
'mock_rse': attributes.get('mock', False),
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
except ValueError:
raise InvalidRuleWeight(
'The RSE \'%s\' has a non-number specified for the weight \'%s\''
% (rse['rse'], weight))
else:
for rse in rses:
mock_rse = has_rse_attribute(rse['id'],
'mock',
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
self.rses.append({
'rse_id': rse['id'],
'weight': 1,
'mock_rse': mock_rse,
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
if len(self.rses) < self.copies:
raise InsufficientTargetRSEs(
'Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)'
% (self.copies, len(self.rses)))
rses_with_enough_quota = []
if has_account_attribute(account=account, key='admin',
session=session) or ignore_account_limit:
for rse in self.rses:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
rses_with_enough_quota.append(rse)
else:
global_quota_limit = get_global_account_limits(account=account,
session=session)
all_rse_usages = {
usage['rse_id']: usage['bytes']
for usage in get_all_rse_usages_per_account(account=account,
session=session)
}
for rse in self.rses:
if rse['mock_rse']:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
rses_with_enough_quota.append(rse)
else:
# check local quota
local_quota_left = None
quota_limit = get_local_account_limit(account=account,
rse_id=rse['rse_id'],
session=session)
if quota_limit is None:
local_quota_left = 0
else:
local_quota_left = quota_limit - get_usage(
rse_id=rse['rse_id'],
account=account,
session=session)['bytes']
# check global quota
rse['global_quota_left'] = {}
all_global_quota_enough = True
for rse_expression, limit in global_quota_limit.items():
if rse['rse_id'] in limit['resolved_rse_ids']:
quota_limit = limit['limit']
global_quota_left = None
if quota_limit is None:
global_quota_left = 0
else:
rse_expression_usage = 0
for rse_id in limit['resolved_rse_ids']:
rse_expression_usage += all_rse_usages.get(
rse_id, 0)
global_quota_left = quota_limit - rse_expression_usage
if global_quota_left <= 0:
all_global_quota_enough = False
break
else:
rse['global_quota_left'][
rse_expression] = global_quota_left
if local_quota_left > 0 and all_global_quota_enough:
rse['quota_left'] = local_quota_left
space_limit = get_rse_limits(
name='MaxSpaceAvailable',
rse_id=rse['rse_id'],
session=session).get('MaxSpaceAvailable')
if space_limit is None or space_limit < 0:
rse['space_left'] = float('inf')
else:
rse['space_left'] = space_limit - get_rse_counter(
rse_id=rse['rse_id'], session=session)['bytes']
rses_with_enough_quota.append(rse)
self.rses = rses_with_enough_quota
if len(self.rses) < self.copies:
raise InsufficientAccountLimit(
'There is insufficient quota on any of the target RSE\'s to fullfill the operation.'
)
def __init__(self,
account,
rses,
weight,
copies,
ignore_account_limit=False,
session=None):
"""
Initialize the RSE Selector.
:param account: Account owning the rule.
:param rses: List of rse dictionaries.
:param weight: Weighting to use.
:param copies: Number of copies to create.
:param ignore_account_limit: Flag if the quota should be ignored.
:param session: DB Session in use.
:raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
"""
self.account = account
self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}]
self.copies = copies
if weight is not None:
for rse in rses:
attributes = list_rse_attributes(rse_id=rse['id'],
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
if weight not in attributes:
continue # The RSE does not have the required weight set, therefore it is ignored
try:
self.rses.append({
'rse_id': rse['id'],
'weight': float(attributes[weight]),
'mock_rse': attributes.get('mock', False),
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
except ValueError:
raise InvalidRuleWeight(
'The RSE \'%s\' has a non-number specified for the weight \'%s\''
% (rse['rse'], weight))
else:
for rse in rses:
mock_rse = has_rse_attribute(rse['id'],
'mock',
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
self.rses.append({
'rse_id': rse['id'],
'weight': 1,
'mock_rse': mock_rse,
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
if len(self.rses) < self.copies:
raise InsufficientTargetRSEs(
'Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)'
% (self.copies, len(self.rses)))
if has_account_attribute(account=account, key='admin',
session=session) or ignore_account_limit:
for rse in self.rses:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
else:
for rse in self.rses:
if rse['mock_rse']:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
else:
quota_limit = get_account_limit(account=account,
rse_id=rse['rse_id'],
session=session)
if quota_limit is None:
rse['quota_left'] = 0
else:
rse['quota_left'] = quota_limit - get_usage(
rse_id=rse['rse_id'],
account=account,
session=session)['bytes']
space_limit = get_rse_limits(
name='MaxSpaceAvailable',
rse_id=rse['rse_id'],
session=session).get('MaxSpaceAvailable')
if space_limit is None or space_limit < 0:
rse['space_left'] = float('inf')
else:
rse['space_left'] = space_limit - get_rse_counter(
rse_id=rse['rse_id'], session=session)['bytes']
self.rses = [rse for rse in self.rses if rse['quota_left'] > 0]
if len(self.rses) < self.copies:
raise InsufficientAccountLimit(
'There is insufficient quota on any of the target RSE\'s to fullfill the operation.'
)
