def test_account_counter_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS""" re_evaluator(once=True) account_update(once=True) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) account_update(once=True) account_counter_before = get_counter(self.rse1_id, 'jdoe') detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) account_update(once=True) account_counter_after = get_counter(self.rse1_id, 'jdoe') assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes']) assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_inc_dec_get_counter(self): """ACCOUNT COUNTER (CORE): Increase, decrease and get counter """ account_update(once=True) rse_id = get_rse('MOCK').id account = 'jdoe' account_counter.del_counter(rse_id=rse_id, account=account) account_counter.add_counter(rse_id=rse_id, account=account) cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': 0, 'bytes': 0}) count, sum = 0, 0 for i in range(10): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(4): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(5): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(8): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum})
def test_inc_dec_get_counter(self): """ACCOUNT COUNTER (CORE): Increase, decrease and get counter """ account_update(once=True) rse_id = get_rse('MOCK').id account = 'jdoe' account_counter.del_counter(rse_id=rse_id, account=account) account_counter.add_counter(rse_id=rse_id, account=account) cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': 0, 'bytes': 0}) count, sum = 0, 0 for i in xrange(10): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in xrange(4): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in xrange(5): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in xrange(8): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum})
def test_account_counter_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS""" re_evaluator(once=True) account_update(once=True) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) account_update(once=True) account_counter_before = get_counter(self.rse1_id, 'jdoe') detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) account_update(once=True) account_counter_after = get_counter(self.rse1_id, 'jdoe') assert (account_counter_before['bytes'] - 100 == account_counter_after['bytes']) assert (account_counter_before['files'] - 1 == account_counter_after['files'])
def __init__(self, account, rses, weight, copies, session=None): """ Initialize the RSE Selector. :param account: Account owning the rule. :param rses: List of rse dictionaries. :param weight: Weighting to use. :param copies: Number of copies to create. :param session: DB Session in use. :raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs """ self.account = account self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}] self.copies = copies if weight is not None: for rse in rses: attributes = list_rse_attributes(rse=None, rse_id=rse['id'], session=session) if weight not in attributes: continue # The RSE does not have the required weight set, therefore it is ignored try: self.rses.append({'rse_id': rse['id'], 'weight': float(attributes[weight]), 'staging_area': rse['staging_area']}) except ValueError: raise InvalidRuleWeight('The RSE with id \'%s\' has a non-number specified for the weight \'%s\'' % (rse['id'], weight)) else: for rse in rses: self.rses.append({'rse_id': rse['id'], 'weight': 1, 'staging_area': rse['staging_area']}) if len(self.rses) < self.copies: raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses))) for rse in self.rses: # TODO: Add RSE-space-left here! rse['quota_left'] = get_account_limit(account=account, rse_id=rse['rse_id'], session=session)\ - get_counter(rse_id=rse['rse_id'], account=account, session=session)['bytes'] self.rses = [rse for rse in self.rses if rse['quota_left'] > 0] if len(self.rses) < self.copies: raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None): """ Initialize the RSE Selector. :param account: Account owning the rule. :param rses: List of rse dictionaries. :param weight: Weighting to use. :param copies: Number of copies to create. :param ignore_account_limit: Flag if the quota should be ignored. :param session: DB Session in use. :raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs """ self.account = account self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}] self.copies = copies if weight is not None: for rse in rses: attributes = list_rse_attributes(rse=None, rse_id=rse['id'], session=session) availability_write = True if rse.get('availability', 7) & 2 else False if weight not in attributes: continue # The RSE does not have the required weight set, therefore it is ignored try: self.rses.append({'rse_id': rse['id'], 'weight': float(attributes[weight]), 'mock_rse': attributes.get('mock', False), 'availability_write': availability_write, 'staging_area': rse['staging_area']}) except ValueError: raise InvalidRuleWeight('The RSE with id \'%s\' has a non-number specified for the weight \'%s\'' % (rse['id'], weight)) else: for rse in rses: mock_rse = has_rse_attribute(rse['id'], 'mock', session=session) availability_write = True if rse.get('availability', 7) & 2 else False self.rses.append({'rse_id': rse['id'], 'weight': 1, 'mock_rse': mock_rse, 'availability_write': availability_write, 'staging_area': rse['staging_area']}) if len(self.rses) < self.copies: raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses))) if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit: for rse in self.rses: rse['quota_left'] = float('inf') else: for rse in self.rses: if rse['mock_rse']: rse['quota_left'] = float('inf') else: # TODO: Add RSE-space-left here! limit = get_account_limit(account=account, rse_id=rse['rse_id'], session=session) if limit is None: rse['quota_left'] = 0 else: rse['quota_left'] = limit - get_counter(rse_id=rse['rse_id'], account=account, session=session)['bytes'] self.rses = [rse for rse in self.rses if rse['quota_left'] > 0] if len(self.rses) < self.copies: raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')