def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
account_update(once=True)
account_counter_before = get_counter(self.rse1_id, 'jdoe')
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_counter(self.rse1_id, 'jdoe')
assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes'])
assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse('MOCK').id
account = 'jdoe'
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': 0, 'bytes': 0})
count, sum = 0, 0
for i in range(10):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(4):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(5):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(8):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse('MOCK').id
account = 'jdoe'
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': 0, 'bytes': 0})
count, sum = 0, 0
for i in xrange(10):
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in xrange(4):
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in xrange(5):
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in xrange(8):
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
def test_account_counter_judge_evaluate_detach(self):
""" JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS"""
re_evaluator(once=True)
account_update(once=True)
scope = 'mock'
files = create_files(3, scope, self.rse1, bytes=100)
dataset = 'dataset_' + str(uuid())
add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
attach_dids(scope, dataset, files, 'jdoe')
# Add a first rule to the DS
add_rule(dids=[{
'scope': scope,
'name': dataset
}],
account='jdoe',
copies=1,
rse_expression=self.rse1,
grouping='ALL',
weight=None,
lifetime=None,
locked=False,
subscription_id=None)
account_update(once=True)
account_counter_before = get_counter(self.rse1_id, 'jdoe')
detach_dids(scope, dataset, [files[0]])
# Fake judge
re_evaluator(once=True)
account_update(once=True)
account_counter_after = get_counter(self.rse1_id, 'jdoe')
assert (account_counter_before['bytes'] -
100 == account_counter_after['bytes'])
assert (account_counter_before['files'] -
1 == account_counter_after['files'])
def __init__(self, account, rses, weight, copies, session=None):
"""
Initialize the RSE Selector.
:param account: Account owning the rule.
:param rses: List of rse dictionaries.
:param weight: Weighting to use.
:param copies: Number of copies to create.
:param session: DB Session in use.
:raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
"""
self.account = account
self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}]
self.copies = copies
if weight is not None:
for rse in rses:
attributes = list_rse_attributes(rse=None, rse_id=rse['id'], session=session)
if weight not in attributes:
continue # The RSE does not have the required weight set, therefore it is ignored
try:
self.rses.append({'rse_id': rse['id'],
'weight': float(attributes[weight]),
'staging_area': rse['staging_area']})
except ValueError:
raise InvalidRuleWeight('The RSE with id \'%s\' has a non-number specified for the weight \'%s\'' % (rse['id'], weight))
else:
for rse in rses:
self.rses.append({'rse_id': rse['id'],
'weight': 1,
'staging_area': rse['staging_area']})
if len(self.rses) < self.copies:
raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses)))
for rse in self.rses:
# TODO: Add RSE-space-left here!
rse['quota_left'] = get_account_limit(account=account, rse_id=rse['rse_id'], session=session)\
- get_counter(rse_id=rse['rse_id'], account=account, session=session)['bytes']
self.rses = [rse for rse in self.rses if rse['quota_left'] > 0]
if len(self.rses) < self.copies:
raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None):
"""
Initialize the RSE Selector.
:param account: Account owning the rule.
:param rses: List of rse dictionaries.
:param weight: Weighting to use.
:param copies: Number of copies to create.
:param ignore_account_limit: Flag if the quota should be ignored.
:param session: DB Session in use.
:raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
"""
self.account = account
self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}]
self.copies = copies
if weight is not None:
for rse in rses:
attributes = list_rse_attributes(rse=None, rse_id=rse['id'], session=session)
availability_write = True if rse.get('availability', 7) & 2 else False
if weight not in attributes:
continue # The RSE does not have the required weight set, therefore it is ignored
try:
self.rses.append({'rse_id': rse['id'],
'weight': float(attributes[weight]),
'mock_rse': attributes.get('mock', False),
'availability_write': availability_write,
'staging_area': rse['staging_area']})
except ValueError:
raise InvalidRuleWeight('The RSE with id \'%s\' has a non-number specified for the weight \'%s\'' % (rse['id'], weight))
else:
for rse in rses:
mock_rse = has_rse_attribute(rse['id'], 'mock', session=session)
availability_write = True if rse.get('availability', 7) & 2 else False
self.rses.append({'rse_id': rse['id'],
'weight': 1,
'mock_rse': mock_rse,
'availability_write': availability_write,
'staging_area': rse['staging_area']})
if len(self.rses) < self.copies:
raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses)))
if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit:
for rse in self.rses:
rse['quota_left'] = float('inf')
else:
for rse in self.rses:
if rse['mock_rse']:
rse['quota_left'] = float('inf')
else:
# TODO: Add RSE-space-left here!
limit = get_account_limit(account=account, rse_id=rse['rse_id'], session=session)
if limit is None:
rse['quota_left'] = 0
else:
rse['quota_left'] = limit - get_counter(rse_id=rse['rse_id'], account=account, session=session)['bytes']
self.rses = [rse for rse in self.rses if rse['quota_left'] > 0]
if len(self.rses) < self.copies:
raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
