def select_rse(self, size, preferred_rse_ids, copies=0, blacklist=[], prioritize_order_over_weight=False): """ Select n RSEs to replicate data to. :param size: Size of the block being replicated. :param preferred_rse_ids: Ordered list of preferred rses. (If possible replicate to them) :param copies: Select this amount of copies, if 0 use the pre-defined rule value. :param blacklist: List of blacklisted rses. (Do not put replicas on these sites) :param prioritze_order_over_weight: Prioritize the order of the preferred_rse_ids list over the picking done by weight. :returns: List of (RSE_id, staging_area) tuples. :raises: InsufficientAccountLimit, InsufficientTargetRSEs """ result = [] rses = self.rses count = self.copies if copies == 0 else copies # Remove blacklisted rses if blacklist: rses = [rse for rse in self.rses if rse['rse_id'] not in blacklist] if len(rses) < count: raise InsufficientTargetRSEs( 'There are not enough target RSEs to fulfil the request at this time.' ) # Remove rses which do not have enough quota rses = [rse for rse in rses if rse['quota_left'] > size] if len(rses) < count: raise InsufficientAccountLimit( 'There is insufficient quota on any of the target RSE\'s to fullfill the operation.' ) for copy in range(count): # Remove rses already in the result set rses = [ rse for rse in rses if rse['rse_id'] not in [item[0] for item in result] ] rses_dict = {} for rse in rses: rses_dict[rse['rse_id']] = rse # Prioritize the preffered rses preferred_rses = [ rses_dict[rse_id] for rse_id in preferred_rse_ids if rse_id in rses_dict ] if prioritize_order_over_weight and preferred_rses: rse = (preferred_rses[0]['rse_id'], preferred_rses[0]['staging_area']) elif preferred_rses: rse = self.__choose_rse(preferred_rses) else: rse = self.__choose_rse(rses) result.append(rse) self.__update_quota(rse, size) return result
def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None): """ Initialize the RSE Selector. :param account: Account owning the rule. :param rses: List of rse dictionaries. :param weight: Weighting to use. :param copies: Number of copies to create. :param ignore_account_limit: Flag if the quota should be ignored. :param session: DB Session in use. :raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs """ self.account = account self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}] self.copies = copies if weight is not None: for rse in rses: attributes = list_rse_attributes(rse_id=rse['id'], session=session) availability_write = True if rse.get('availability', 7) & 2 else False if weight not in attributes: continue # The RSE does not have the required weight set, therefore it is ignored try: self.rses.append({ 'rse_id': rse['id'], 'weight': float(attributes[weight]), 'mock_rse': attributes.get('mock', False), 'availability_write': availability_write, 'staging_area': rse['staging_area'] }) except ValueError: raise InvalidRuleWeight( 'The RSE \'%s\' has a non-number specified for the weight \'%s\'' % (rse['rse'], weight)) else: for rse in rses: mock_rse = has_rse_attribute(rse['id'], 'mock', session=session) availability_write = True if rse.get('availability', 7) & 2 else False self.rses.append({ 'rse_id': rse['id'], 'weight': 1, 'mock_rse': mock_rse, 'availability_write': availability_write, 'staging_area': rse['staging_area'] }) if len(self.rses) < self.copies: raise InsufficientTargetRSEs( 'Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses))) rses_with_enough_quota = [] if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit: for rse in self.rses: rse['quota_left'] = float('inf') rse['space_left'] = float('inf') rses_with_enough_quota.append(rse) else: global_quota_limit = get_global_account_limits(account=account, session=session) all_rse_usages = { usage['rse_id']: usage['bytes'] for usage in get_all_rse_usages_per_account(account=account, session=session) } for rse in self.rses: if rse['mock_rse']: rse['quota_left'] = float('inf') rse['space_left'] = float('inf') rses_with_enough_quota.append(rse) else: # check local quota local_quota_left = None quota_limit = get_local_account_limit(account=account, rse_id=rse['rse_id'], session=session) if quota_limit is None: local_quota_left = 0 else: local_quota_left = quota_limit - get_usage( rse_id=rse['rse_id'], account=account, session=session)['bytes'] # check global quota rse['global_quota_left'] = {} all_global_quota_enough = True for rse_expression, limit in global_quota_limit.items(): if rse['rse_id'] in limit['resolved_rse_ids']: quota_limit = limit['limit'] global_quota_left = None if quota_limit is None: global_quota_left = 0 else: rse_expression_usage = 0 for rse_id in limit['resolved_rse_ids']: rse_expression_usage += all_rse_usages.get( rse_id, 0) global_quota_left = quota_limit - rse_expression_usage if global_quota_left <= 0: all_global_quota_enough = False break else: rse['global_quota_left'][ rse_expression] = global_quota_left if local_quota_left > 0 and all_global_quota_enough: rse['quota_left'] = local_quota_left space_limit = get_rse_limits( name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable') if space_limit is None or space_limit < 0: rse['space_left'] = float('inf') else: rse['space_left'] = space_limit - get_rse_counter( rse_id=rse['rse_id'], session=session)['bytes'] rses_with_enough_quota.append(rse) self.rses = rses_with_enough_quota if len(self.rses) < self.copies: raise InsufficientAccountLimit( 'There is insufficient quota on any of the target RSE\'s to fullfill the operation.' )
def select_rse(self, size, preferred_rse_ids, copies=0, blocklist=[], prioritize_order_over_weight=False, existing_rse_size=None): """ Select n RSEs to replicate data to. :param size: Size of the block being replicated. :param preferred_rse_ids: Ordered list of preferred rses. (If possible replicate to them) :param copies: Select this amount of copies, if 0 use the pre-defined rule value. :param blocklist: List of blocked rses. (Do not put replicas on these sites) :param prioritze_order_over_weight: Prioritize the order of the preferred_rse_ids list over the picking done by weight. :existing_rse_size: Dictionary of size of files already present at each rse :returns: List of (RSE_id, staging_area, availability_write) tuples. :raises: InsufficientAccountLimit, InsufficientTargetRSEs """ result = [] rses = self.rses count = self.copies if copies == 0 else copies # Remove blocklisted rses if blocklist: rses = [rse for rse in self.rses if rse['rse_id'] not in blocklist] if len(rses) < count: raise InsufficientTargetRSEs( 'There are not enough target RSEs to fulfil the request at this time.' ) # Remove rses which do not have enough space, accounting for the files already at each rse if existing_rse_size is None: existing_rse_size = {} rses = [ rse for rse in rses if rse['space_left'] >= size - existing_rse_size.get(rse['rse_id'], 0) ] if len(rses) < count: raise RSEOverQuota( 'There is insufficient space on any of the target RSE\'s to fullfill the operation.' ) # Remove rses which do not have enough local quota rses = [rse for rse in rses if rse['quota_left'] > size] if len(rses) < count: raise InsufficientAccountLimit( 'There is insufficient quota on any of the target RSE\'s to fullfill the operation.' ) # Remove rses which do not have enough global quota rses_with_enough_quota = [] for rse in rses: enough_global_quota = True for rse_expression in rse.get('global_quota_left', []): if rse['global_quota_left'][rse_expression] < size: enough_global_quota = False break if enough_global_quota: rses_with_enough_quota.append(rse) rses = rses_with_enough_quota if len(rses) < count: raise InsufficientAccountLimit( 'There is insufficient quota on any of the target RSE\'s to fullfill the operation.' ) for copy in range(count): # Remove rses already in the result set rses = [ rse for rse in rses if rse['rse_id'] not in [item[0] for item in result] ] rses_dict = {} for rse in rses: rses_dict[rse['rse_id']] = rse # Prioritize the preffered rses preferred_rses = [ rses_dict[rse_id] for rse_id in preferred_rse_ids if rse_id in rses_dict ] if prioritize_order_over_weight and preferred_rses: rse = (preferred_rses[0]['rse_id'], preferred_rses[0]['staging_area'], preferred_rses[0]['availability_write']) elif preferred_rses: rse = self.__choose_rse(preferred_rses) else: rse = self.__choose_rse(rses) result.append(rse) self.__update_quota(rse, size) return result
def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None): """ Initialize the RSE Selector. :param account: Account owning the rule. :param rses: List of rse dictionaries. :param weight: Weighting to use. :param copies: Number of copies to create. :param ignore_account_limit: Flag if the quota should be ignored. :param session: DB Session in use. :raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs """ self.account = account self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}] self.copies = copies if weight is not None: for rse in rses: attributes = list_rse_attributes(rse=None, rse_id=rse['id'], session=session) availability_write = True if rse.get('availability', 7) & 2 else False if weight not in attributes: continue # The RSE does not have the required weight set, therefore it is ignored try: self.rses.append({'rse_id': rse['id'], 'weight': float(attributes[weight]), 'mock_rse': attributes.get('mock', False), 'availability_write': availability_write, 'staging_area': rse['staging_area']}) except ValueError: raise InvalidRuleWeight('The RSE with id \'%s\' has a non-number specified for the weight \'%s\'' % (rse['id'], weight)) else: for rse in rses: mock_rse = has_rse_attribute(rse['id'], 'mock', session=session) availability_write = True if rse.get('availability', 7) & 2 else False self.rses.append({'rse_id': rse['id'], 'weight': 1, 'mock_rse': mock_rse, 'availability_write': availability_write, 'staging_area': rse['staging_area']}) if len(self.rses) < self.copies: raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses))) if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit: for rse in self.rses: rse['quota_left'] = float('inf') else: for rse in self.rses: if rse['mock_rse']: rse['quota_left'] = float('inf') else: # TODO: Add RSE-space-left here! limit = get_account_limit(account=account, rse_id=rse['rse_id'], session=session) if limit is None: rse['quota_left'] = 0 else: rse['quota_left'] = limit - get_counter(rse_id=rse['rse_id'], account=account, session=session)['bytes'] self.rses = [rse for rse in self.rses if rse['quota_left'] > 0] if len(self.rses) < self.copies: raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None): """ Select a new target RSE for a rebalanced rule. :param parent_rule rule that is rebalanced. :param current_rse_id: RSE of the source. :param rse_expression: RSE Expression of the source rule. :param subscription_id: Subscription ID of the source rule. :param rse_attributes: The attributes of the source rse. :param other_rses: Other RSEs with existing dataset replicas. :param exclude_expression: Exclude this rse_expression from being target_rses. :param force_expression: Force a specific rse_expression as target. :param session: The DB Session. :returns: New RSE expression. """ current_rse = get_rse_name(rse_id=current_rse_id) current_rse_expr = current_rse # if parent rule has a vo, enforce it vo = parent_rule['scope'].vo if exclude_expression: target_rse = '((%s)|(%s))' % (exclude_expression, current_rse_expr) else: target_rse = current_rse_expr list_target_rses = [ rse['rse'] for rse in parse_expression( expression=target_rse, filter_={'vo': vo}, session=session) ] list_target_rses.sort() rses = parse_expression(expression=rse_expression, filter_={'vo': vo}, session=session) # TODO: Implement subscription rebalancing if force_expression is not None: if parent_rule['grouping'] != RuleGrouping.NONE: rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter_={ 'vo': vo, 'availability_write': True }, session=session) else: # in order to avoid replication of the part of distributed dataset not present at rebalanced rse -> rses in force_expression # this will be extended with development of delayed rule rses = parse_expression( expression='((%s)|(%s))\\%s' % (force_expression, rse_expression, target_rse), filter_={ 'vo': vo, 'availability_write': True }, session=session) else: # force_expression is not set, RSEs will be selected as rse_expression\rse list_rses = [rse['rse'] for rse in rses] list_rses.sort() if list_rses == list_target_rses: raise InsufficientTargetRSEs( 'Not enough RSEs to rebalance rule %s' % parent_rule['id']) else: rses = parse_expression(expression='(%s)\\%s' % (rse_expression, target_rse), filter_={ 'vo': vo, 'availability_write': True }, session=session) rseselector = RSESelector(account=InternalAccount('root', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session) return get_rse_name([ rse_id for rse_id, _, _ in rseselector.select_rse( size=0, preferred_rse_ids=[], blocklist=other_rses) ][0], session=session)