def get_global_account_limits(account=None, session=None): """ Returns the global account limits for the account. :param account: Account to check the limit for. :param session: Database session in use. :return: Dict {'MOCK': {'resolved_rses': ['MOCK'], 'limit': 10, 'resolved_rse_ids': [123]}}. """ if account: global_account_limits = session.query( models.AccountGlobalLimit).filter_by(account=account).all() else: global_account_limits = session.query(models.AccountGlobalLimit).all() resolved_global_account_limits = {} for limit in global_account_limits: if account: resolved_rses = parse_expression(limit['rse_expression'], filter_={'vo': account.vo}, session=session) else: resolved_rses = parse_expression(limit['rse_expression'], session=session) limit_in_bytes = limit['bytes'] if limit_in_bytes == -1: limit_in_bytes = float('inf') resolved_global_account_limits[limit['rse_expression']] = { 'resolved_rses': [resolved_rse['rse'] for resolved_rse in resolved_rses], 'resolved_rse_ids': [resolved_rse['id'] for resolved_rse in resolved_rses], 'limit': limit_in_bytes } return resolved_global_account_limits
def test_list_on_availability(self): """ RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter""" rsewrite_name = rse_name_generator() rsenowrite_name = rse_name_generator() rsewrite_id = rse.add_rse(rsewrite_name, **self.vo) rsenowrite_id = rse.add_rse(rsenowrite_name, **self.vo) attribute = attribute_name_generator() rse.add_rse_attribute(rsewrite_id, attribute, "de") rse.add_rse_attribute(rsenowrite_id, attribute, "de") rse.update_rse(rsewrite_id, {'availability_write': True}) rse.update_rse(rsenowrite_id, {'availability_write': False}) value = sorted([ item['id'] for item in rse_expression_parser.parse_expression( "%s=de" % attribute, **self.filter) ]) expected = sorted([rsewrite_id, rsenowrite_id]) assert value == expected filters = self.filter filters['availability_write'] = True value = sorted([ item['id'] for item in rse_expression_parser.parse_expression( "%s=de" % attribute, filters) ]) expected = sorted([rsewrite_id]) assert value == expected filters['availability_write'] = False pytest.raises(RSEWriteBlocked, rse_expression_parser.parse_expression, "%s=de" % attribute, filters)
def test_numeric_operators(self): """ RSE_EXPRESSION_PARSER (CORE) Test RSE attributes with numeric operations """ value = [ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s<11" % self.attribute_numeric, **self.filter) ] assert value == [self.rse1_id] pytest.raises(InvalidRSEExpression, rse_expression_parser.parse_expression, "%s<9" % self.attribute_numeric, **self.filter) value = sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s<21" % self.attribute_numeric, **self.filter) ]) expected = sorted([self.rse1_id, self.rse2_id]) assert value == expected value = [ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s>49" % self.attribute_numeric, **self.filter) ] assert value == [self.rse5_id] pytest.raises(InvalidRSEExpression, rse_expression_parser.parse_expression, "%s>51" % self.attribute_numeric, **self.filter) value = sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s>30" % self.attribute_numeric, **self.filter) ]) expected = sorted([self.rse4_id, self.rse5_id]) assert value == expected
def test_numeric_operators(self): """ RSE_EXPRESSION_PARSER (CORE) Test RSE attributes with numeric operations """ assert_equal([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s<11" % self.attribute_numeric) ], [self.rse1_id]) assert_raises(InvalidRSEExpression, rse_expression_parser.parse_expression, "%s<9" % self.attribute_numeric) assert_equal( sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s<21" % self.attribute_numeric) ]), sorted([self.rse1_id, self.rse2_id])) assert_equal([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s>49" % self.attribute_numeric) ], [self.rse5_id]) assert_raises(InvalidRSEExpression, rse_expression_parser.parse_expression, "%s>51" % self.attribute_numeric) assert_equal( sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s>30" % self.attribute_numeric) ]), sorted([self.rse4_id, self.rse5_id]))
def test_list_on_availability(): """ RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter""" rsewrite_name = rse_name_generator() rsenowrite_name = rse_name_generator() rsewrite_id = rse.add_rse(rsewrite_name) rsenowrite_id = rse.add_rse(rsenowrite_name) attribute = attribute_name_generator() rse.add_rse_attribute(rsewrite_id, attribute, "de") rse.add_rse_attribute(rsenowrite_id, attribute, "de") rse.update_rse(rsewrite_id, {'availability_write': True}) rse.update_rse(rsenowrite_id, {'availability_write': False}) assert_equal( sorted([ item['id'] for item in rse_expression_parser.parse_expression("%s=de" % attribute) ]), sorted([rsewrite_id, rsenowrite_id])) assert_equal( sorted([ item['id'] for item in rse_expression_parser.parse_expression( "%s=de" % attribute, {'availability_write': True}) ]), sorted([rsewrite_id])) assert_raises(RSEBlacklisted, rse_expression_parser.parse_expression, "%s=de" % attribute, {'availability_write': False})
def get_rses_to_process(rses, include_rses, exclude_rses, vos): """ Return the list of RSEs to process based on rses, include_rses and exclude_rses :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :param vos: VOs on which to look for RSEs. Only used in multi-VO mode. If None, we either use all VOs if run from "def", :returns: A list of RSEs to process """ multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: if vos: logging.log(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.') vos = ['def'] else: if vos: invalid = set(vos) - set([v['vo'] for v in list_vos()]) if invalid: msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid])) raise VONotFound(msg) else: vos = [v['vo'] for v in list_vos()] logging.log(logging.INFO, 'Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos]))) cache_key = 'rses_to_process' if multi_vo: cache_key += '@%s' % '-'.join(vo for vo in vos) result = REGION.get(cache_key) if result is not NO_VALUE: return result all_rses = [] for vo in vos: all_rses.extend(list_rses(filters={'vo': vo})) if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] REGION.set(cache_key, rses) logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses])) return rses
def test_order_of_operations(self): """ RSE_EXPRESSION_PARSER (CORE) Test order of operations """ value = sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("%s\\%s|%s=fr" % (self.tag1, self.rse3, self.attribute), **self.filter)]) expected = sorted([self.rse1_id, self.rse2_id, self.rse3_id]) assert value == expected value = sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("%s\\(%s|%s=fr)" % (self.tag1, self.rse3, self.attribute), **self.filter)]) expected = sorted([self.rse1_id, self.rse2_id]) assert value == expected
def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None, vos=None, logger=logging.log): """ Get a list of rses for conveyor :param rses: List of rses (Single-VO only) :param include_rses: RSEs to include :param exclude_rses: RSEs to exclude :param vos: VOs on which to look for RSEs. Only used in multi-VO mode. If None, we either use all VOs if run from "def", or the current VO otherwise. :param logger: Optional decorated logger that can be passed from the calling daemons or servers. :return: List of working rses """ multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: if vos: logger(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.') vos = ['def'] else: if vos: invalid = set(vos) - set([v['vo'] for v in list_vos()]) if invalid: msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid])) raise VONotFound(msg) else: vos = [v['vo'] for v in list_vos()] logger(logging.INFO, 'This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos]))) working_rses = [] rses_list = [] for vo in vos: rses_list.extend(list_rses(filters={'vo': vo})) if rses: working_rses = [rse for rse in rses_list if rse['rse'] in rses] if include_rses: for vo in vos: try: parsed_rses = parse_expression(include_rses, filter={'vo': vo}, session=None) except InvalidRSEExpression: logger(logging.ERROR, "Invalid RSE exception %s to include RSEs", include_rses) else: for rse in parsed_rses: if rse not in working_rses: working_rses.append(rse) if not (rses or include_rses): working_rses = rses_list if exclude_rses: try: parsed_rses = parse_expression(exclude_rses, session=None) except InvalidRSEExpression as error: logger(logging.ERROR, "Invalid RSE exception %s to exclude RSEs: %s", exclude_rses, error) else: working_rses = [rse for rse in working_rses if rse not in parsed_rses] working_rses = [rsemgr.get_rse_info(rse_id=rse['id']) for rse in working_rses] return working_rses
def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None): """ Select a new target RSE for a rebalanced rule. :param parent_rule rule that is rebalanced. :param current_rse_id: RSE of the source. :param rse_expression: RSE Expression of the source rule. :param subscription_id: Subscription ID of the source rule. :param rse_attributes: The attributes of the source rse. :param other_rses: Other RSEs with existing dataset replicas. :param exclude_expression: Exclude this rse_expression from being target_rses. :param force_expression: Force a specific rse_expression as target. :param session: The DB Session :returns: New RSE expression """ if rse_attributes['type'] != 'DATADISK' and force_expression is None: print('WARNING: dest RSE(s) has to be provided with --force-expression for rebalancing of non-datadisk RSES.') raise InsufficientTargetRSEs current_rse = get_rse_name(rse_id=current_rse_id) current_rse_expr = current_rse # if parent rule has a vo, enforce it vo = parent_rule['scope'].vo if exclude_expression: target_rse = '(%s)\\%s' % (exclude_expression, current_rse_expr) else: target_rse = current_rse_expr rses = parse_expression(expression=rse_expression, filter={'vo': vo}, session=session) # TODO: dest rse selection should be configurable, there might be cases when tier is not defined, or concept of DATADISKS is not present. # if subscription_id: # pass # # get_subscription_by_id(subscription_id, session) if force_expression is not None: if parent_rule['grouping'] != RuleGrouping.NONE: rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter={'vo': vo, 'availability_write': True}, session=session) else: # in order to avoid replication of the part of distributed dataset not present at rabalanced rse -> rses in force_expression # this will be extended with development of delayed rule rses = parse_expression(expression='((%s)|(%s))\\%s' % (force_expression, rse_expression, target_rse), filter={'vo': vo, 'availability_write': True}, session=session) elif len(rses) > 1: # Just define the RSE Expression without the current_rse return '(%s)\\%s' % (rse_expression, target_rse) else: if rse_attributes['tier'] is True or int(rse_attributes['tier']) == 1: # Tier 1 should go to another Tier 1 expression = '(tier=1&type=DATADISK)\\{}'.format(target_rse) elif int(rse_attributes['tier']) == 2: # Tier 2 should go to another Tier 2 expression = '(tier=2&type=DATADISK)\\{}'.format(target_rse) elif int(rse_attributes['tier']) == 3: # Tier 3 will go to Tier 2, since we don't have enough t3s expression = '((tier=2&type=DATADISK)\\datapolicynucleus=1)\\{}'.format(target_rse) rses = parse_expression(expression=expression, filter={'vo': vo, 'availability_write': True}, session=session) rseselector = RSESelector(account=InternalAccount('ddmadmin', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session) return get_rse_name([rse_id for rse_id, _, _ in rseselector.select_rse(size=0, preferred_rse_ids=[], blocklist=other_rses)][0], session=session)
def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, delay_seconds=0, all_rses=False): """ Starts up the reaper threads. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param threads_per_worker: Total number of threads created by each worker. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs (Single-VO only). :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. """ logging.info('main: starting processes') all_rses = list_rses() if all_rses: rses = all_rses else: if rses: if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): logging.warning('Ignoring argument rses, this is only available in a single-vo setup. Please try an RSE Expression with include_rses if it is required.') rses = [] else: rses = [rse_core.get_rse_id(rse=rse) for rse in rses] rses = [rse for rse in rses if rse in all_rses] else: rses = all_rses if exclude_rses: excluded_rses = [rse['id'] for rse in parse_expression(exclude_rses)] rses = [rse for rse in rses if rse not in excluded_rses] if include_rses: included_rses = [rse['id'] for rse in parse_expression(include_rses)] rses = [rse for rse in rses if rse in included_rses] if not rses: logging.error('Dark Reaper: No RSEs found. Exiting.') return threads = [] for worker in range(total_workers): kwargs = {'worker_number': worker, 'total_workers': total_workers, 'rses': rses, 'once': once, 'chunk_size': chunk_size, 'scheme': scheme} threads.append(threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, Total_Workers: %s' % (worker, total_workers))) [t.start() for t in threads] while threads[0].is_alive(): [t.join(timeout=3.14) for t in threads]
def test_rses_at_different_vos(self): """ MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """ # Set up RSEs at two VOs rse_client = RSEClient() rse_str = ''.join(choice(ascii_uppercase) for x in range(10)) tst = 'TST_%s' % rse_str new = 'NEW_%s' % rse_str shr = 'SHR_%s' % rse_str rse_client.add_rse(tst) rse_client.add_rse(shr) add_rse(new, 'root', **self.new_vo) shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new' # Check the cached rse-id from each VO does not interfere shr_id_tst = get_rse_id(shr, **self.vo) shr_id_new = get_rse_id(shr, **self.new_vo) assert_equal(shr_id_new, shr_id_new_original) assert_not_equal(shr_id_new, shr_id_tst) # Check that when listing RSEs we only get RSEs for our VO rse_list_tst = [r['rse'] for r in rse_client.list_rses()] rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)] assert_true(tst in rse_list_tst) assert_false(new in rse_list_tst) assert_true(shr in rse_list_tst) assert_false(tst in rse_list_new) assert_true(new in rse_list_new) assert_true(shr in rse_list_new) # Check the cached attribute-value results do not interfere and only give results from the appropriate VO attribute_value = generate_uuid() add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo) rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo)) rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo)) rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo)) rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo)) assert_equal(len(rses_tst_1), 0) assert_not_equal(len(rses_new_1), 0) assert_equal(len(rses_tst_2), 0) assert_not_equal(len(rses_new_2), 0) # check parse_expression rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']}) rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']}) rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']}) with assert_raises(InvalidRSEExpression): parse_expression(tst, filter={'vo': self.new_vo['vo']}) assert_equal(len(rses_tst_3), 1) assert_equal(shr_id_tst, rses_tst_3[0]['id']) assert_equal(len(rses_tst_4), 1) assert_equal(tst, rses_tst_4[0]['rse']) assert_equal(len(rses_new_3), 1) assert_equal(shr_id_new, rses_new_3[0]['id'])
def test_order_of_operations(self): """ RSE_EXPRESSION_PARSER (CORE) Test order of operations """ assert_equal( sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s\\%s|%s=fr" % (self.tag1, self.rse3, self.attribute)) ]), sorted([self.rse1_id, self.rse2_id, self.rse3_id])) assert_equal( sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s\\(%s|%s=fr)" % (self.tag1, self.rse3, self.attribute)) ]), sorted([self.rse1_id, self.rse2_id]))
def run(total_workers=1, chunk_size=100, threads_per_worker=None, once=False, greedy=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, delay_seconds=0): """ Starts up the reaper threads. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param threads_per_worker: Total number of threads created by each worker. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. """ logging.info('main: starting processes') rses_list = rse_core.list_rses() if rses: rses = [rse for rse in rses_list if rse['rse'] in rses] else: rses = rses_list if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses])) threads = [] nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1.0 for worker in xrange(total_workers): for child in xrange(threads_per_worker or 1): kwargs = {'worker_number': worker, 'child_number': child + 1, 'total_children': threads_per_worker or 1, 'once': once, 'chunk_size': chunk_size, 'greedy': greedy, 'rses': rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker], 'delay_seconds': delay_seconds, 'scheme': scheme} threads.append(threading.Thread(target=reaper, kwargs=kwargs)) [t.start() for t in threads] while threads[0].is_alive(): [t.join(timeout=3.14) for t in threads]
def get_rses_to_process(rses, include_rses, exclude_rses): """ Return the list of RSEs to process based on rses, include_rses and exclude_rses :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :returns: A list of RSEs to process """ result = REGION.get('rses_to_process') if result is not NO_VALUE: return result all_rses = list_rses() if rses: if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): logging.warning( 'Ignoring argument rses, this is only available in a single-VO setup. Please try an RSE Expression with include_rses if it is required.' ) rses = all_rses else: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format( 's' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] REGION.set('rses_to_process', rses) logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses])) return rses
def test_tag_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """ assert_equal( sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression(self.tag1) ]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_all_rse(self): """ RSE_EXPRESSION_PARSER (CORE) Test reference on all RSE """ all_rses = rse.list_rses() assert_equal( sorted(rse_expression_parser.parse_expression("*"), key=lambda rse: rse['rse']), sorted(all_rses, key=lambda rse: rse['rse']))
def test_attribute_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """ assert_equal([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression("%s=uk" % self.attribute) ], [self.rse4_id])
def perm_add_rule(issuer, kwargs, session=None): """ Checks if an account can add a replication rule. :param issuer: Account identifier which issues the command. :param kwargs: List of arguments for the action. :param session: The DB session to use :returns: True if account is allowed, otherwise False """ rses = parse_expression(kwargs['rse_expression'], filter_={'vo': issuer.vo}, session=session) # Keep while sync is running so it can make rules on all RSEs if _is_root(issuer) and repr(kwargs['account']).startswith('sync_'): return True if isinstance(repr(issuer), basestring) and repr(issuer).startswith('sync_'): return True # Anyone can use _Temp RSEs if a lifetime is set and under a month all_temp = True for rse in rses: rse_attr = list_rse_attributes(rse_id=rse['id'], session=session) rse_type = rse_attr.get('cms_type', None) if rse_type not in ['temp']: all_temp = False if all_temp and kwargs['lifetime'] is not None and kwargs['lifetime'] < 31 * 24 * 60 * 60: return True if kwargs['account'] == issuer and not kwargs['locked']: return True if _is_root(issuer) or has_account_attribute(account=issuer, key='admin', session=session): return True return False
def __init__(self, datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas): self._fsc = FreeSpaceCollector() self._nmc = NetworkMetricsCollector() self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400) self._dc = DatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400) self._added_bytes = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_bytes_") self._added_files = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_files_") self._datatypes = datatypes.split(',') self._dest_rse_expr = dest_rse_expr self._max_bytes_hour = max_bytes_hour self._max_files_hour = max_files_hour self._max_bytes_hour_rse = max_bytes_hour_rse self._max_files_hour_rse = max_files_hour_rse self._min_popularity = min_popularity self._min_recent_requests = min_recent_requests self._max_replicas = max_replicas rses = parse_expression(self._dest_rse_expr) self._rses = {} self._sites = {} for rse in rses: rse_attrs = list_rse_attributes(rse['rse']) rse_attrs['rse'] = rse['rse'] self._rses[rse['rse']] = rse_attrs self._sites[rse_attrs['site']] = rse_attrs self._dst_penalties = {} self._src_penalties = {} self._print_params()
def perm_approve_rule(issuer, kwargs, session=None): """ Checks if an issuer can approve a replication rule. :param issuer: Account identifier which issues the command. :param kwargs: List of arguments for the action. :param session: The DB session to use :returns: True if account is allowed to call the API call, otherwise False """ if _is_root(issuer) or has_account_attribute( account=issuer, key='admin', session=session): return True rule = get_rule(rule_id=kwargs['rule_id']) rses = parse_expression(rule['rse_expression'], filter_={'vo': issuer.vo}, session=session) # Those in rule_approvers can approve the rule for rse in rses: rse_attr = list_rse_attributes(rse_id=rse['id'], session=session) rule_approvers = rse_attr.get('rule_approvers', None) if rule_approvers and issuer.external in rule_approvers.split(','): return True return False
def perm_del_rule(issuer, kwargs): """ Checks if an issuer can delete a replication rule. :param issuer: Account identifier which issues the command. :param kwargs: List of arguments for the action. :returns: True if account is allowed to call the API call, otherwise False """ if issuer == 'root' or issuer == 'ddmadmin': return True if get_rule(kwargs['rule_id'])['account'] == issuer: return True # Check if user is a country admin admin_in_country = [] for kv in list_account_attributes(account=issuer): if kv['key'].startswith('country-') and kv['value'] == 'admin': admin_in_country.append(kv['key'].partition('-')[2]) rule = get_rule(rule_id=kwargs['rule_id']) rses = parse_expression(rule['rse_expression']) if admin_in_country: for rse in rses: if list_rse_attributes(rse=None, rse_id=rse['id']).get('country') in admin_in_country: return True # DELETERS can approve the rule for rse in rses: rse_attr = list_rse_attributes(rse=None, rse_id=rse['id']) if rse_attr.get('rule_deleters'): if issuer in rse_attr.get('rule_deleters').split(','): return True return False
def perm_get_global_account_usage(issuer, kwargs): """ Checks if an account can get the account usage of an account. :param issuer: Account identifier which issues the command. :param kwargs: List of arguments for the action. :returns: True if account is allowed, otherwise False """ if _is_root(issuer) or has_account_attribute( account=issuer, key='admin') or kwargs.get('account') == issuer: return True # Check if user is a country admin for all involved countries admin_in_country = set() for kv in list_account_attributes(account=issuer): if kv['key'].startswith('country-') and kv['value'] == 'admin': admin_in_country.add(kv['key'].partition('-')[2]) resolved_rse_countries = { list_rse_attributes(rse_id=rse['rse_id']).get('country') for rse in parse_expression(kwargs['rse_exp'], filter={'vo': issuer.vo}) } if resolved_rse_countries.issubset(admin_in_country): return True return False
def perm_add_rule(issuer, kwargs): """ Checks if an account can add a replication rule. :param issuer: Account identifier which issues the command. :param kwargs: List of arguments for the action. :returns: True if account is allowed, otherwise False """ rses = parse_expression(kwargs['rse_expression']) # If all the RSEs matching the expression need approval, the rule cannot be created if not kwargs['ask_approval']: all_rses_need_approval = True for rse in rses: rse_attr = list_rse_attributes(rse_id=rse['id']) if rse_attr.get('requires_approval', False): all_rses_need_approval = False if not all_rses_need_approval: return False if kwargs['account'] == issuer and not kwargs['locked']: return True if _is_root(issuer) or has_account_attribute(account=issuer, key='admin'): return True return False
def test_intersect(self): """ RSE_EXPRESSION_PARSER (CORE) Test intersect operator """ value = [ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s&%s=uk" % (self.tag2, self.attribute), **self.filter) ] assert value == [self.rse4_id]
def perm_delete_global_account_limit(issuer, kwargs, session=None): """ Checks if an account can delete a global account limit. :param issuer: Account identifier which issues the command. :param kwargs: List of arguments for the action. :param session: The DB session to use :returns: True if account is allowed, otherwise False """ if _is_root(issuer) or has_account_attribute( account=issuer, key='admin', session=session): return True # Check if user is a country admin admin_in_country = set() for kv in list_account_attributes(account=issuer, session=session): if kv['key'].startswith('country-') and kv['value'] == 'admin': admin_in_country.add(kv['key'].partition('-')[2]) if admin_in_country: resolved_rse_countries = { list_rse_attributes(rse_id=rse['rse_id'], session=session).get('country') for rse in parse_expression(kwargs['rse_expression'], filter_={'vo': issuer.vo}, session=session) } if resolved_rse_countries.issubset(admin_in_country): return True return False
def test_simple_rse_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE reference """ value = [ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( self.rse1, **self.filter) ] assert value == [self.rse1_id]
def test_attribute_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """ value = [ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s=uk" % self.attribute, **self.filter) ] assert value == [self.rse4_id]
def test_tag_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """ value = sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( self.tag1, **self.filter) ]) expected = sorted([self.rse1_id, self.rse2_id, self.rse3_id]) assert value == expected
def test_parantheses(self): """ RSE_EXPRESSION_PARSER (CORE) Test parantheses """ value = sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "(%s)" % self.tag1, **self.filter) ]) expected = sorted([self.rse1_id, self.rse2_id, self.rse3_id]) assert value == expected
def test_complement(self): """ RSE_EXPRESSION_PARSER (CORE) Test complement operator """ value = sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "%s\\%s" % (self.tag1, self.rse3), **self.filter) ]) expected = sorted([self.rse1_id, self.rse2_id]) assert value == expected
def test_complicated_expression_3(self): """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 3""" value = sorted([ t_rse['id'] for t_rse in rse_expression_parser.parse_expression( "(*)&%s=at" % self.attribute, **self.filter) ]) expected = sorted([self.rse1_id]) assert value == expected
def parse_rse_expression(rse_expression): """ Parse an RSE expression and return the list of RSEs. :param rse_expression: The RSE expression. :returns: List of RSEs :raises: InvalidRSEExpression """ rses = parse_expression(rse_expression) return [rse['rse'] for rse in rses]
def test_list_rses_based_on_availability(self): """ RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter""" rseWRITE_name = rse_name_generator() rseNOWRITE_name = rse_name_generator() rseWRITE_id = rse.add_rse(rseWRITE_name) rseNOWRITE_id = rse.add_rse(rseNOWRITE_name) attribute = attribute_name_generator() rse.add_rse_attribute(rseWRITE_name, attribute, "de") rse.add_rse_attribute(rseNOWRITE_name, attribute, "de") rse.update_rse(rseWRITE_name, {'availability_write': True}) rse.update_rse(rseNOWRITE_name, {'availability_write': False}) assert_equal(sorted([item['id'] for item in rse_expression_parser.parse_expression("%s=de" % attribute)]), sorted([rseWRITE_id, rseNOWRITE_id])) assert_equal(sorted([item['id'] for item in rse_expression_parser.parse_expression("%s=de" % attribute, {'availability_write': True})]), sorted([rseWRITE_id])) assert_raises(InvalidRSEExpression, rse_expression_parser.parse_expression, "%s=de" % attribute, {'availability_write': False})
def get_rses(rses=None, include_rses=None, exclude_rses=None): working_rses = [] rses_list = rse_core.list_rses() if rses: working_rses = [rse for rse in rses_list if rse['rse'] in rses] if include_rses: try: parsed_rses = parse_expression(include_rses, session=None) except InvalidRSEExpression, e: logging.error("Invalid RSE exception %s to include RSEs" % (include_rses)) else: for rse in parsed_rses: if rse not in working_rses: working_rses.append(rse)
def site_selector(replicas, site): """ Return a list of replicas located on one site. :param replicas : A dict with RSEs as values and replicas as keys (URIs). :param site : The name of the site """ result = [] try: rses = parse_expression("site=%s" % site) except InvalidRSEExpression: return result except Exception: return result rses = [i['rse'] for i in rses] for replica in replicas: if replicas[replica] in rses: result.append(replica) return result
def get_sources(dest_rse, scheme, req): allowed_rses = [] if req['request_type'] == RequestType.STAGEIN: rses = rse_core.list_rses(filters={'staging_buffer': dest_rse['rse']}) allowed_rses = [x['rse'] for x in rses] allowed_source_rses = [] if req['attributes']: if type(req['attributes']) is dict: req_attributes = json.loads(json.dumps(req['attributes'])) else: req_attributes = json.loads(str(req['attributes'])) source_replica_expression = req_attributes["source_replica_expression"] if source_replica_expression: try: parsed_rses = parse_expression(source_replica_expression, session=None) except InvalidRSEExpression, e: logging.error("Invalid RSE exception %s for request %s: %s" % (source_replica_expression, req['request_id'], e)) allowed_source_rses = [] else: allowed_source_rses = [x['rse'] for x in parsed_rses]
def test_tag_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """ assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression(self.tag1)]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_complement(self): """ RSE_EXPRESSION_PARSER (CORE) Test complement operator """ assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s\\%s" % (self.tag1, self.rse3))]), sorted([self.rse1_id, self.rse2_id]))
def test_intersect(self): """ RSE_EXPRESSION_PARSER (CORE) Test intersect operator """ assert_equal([rse['id'] for rse in rse_expression_parser.parse_expression("%s&%s=uk" % (self.tag2, self.attribute))], [self.rse4_id])
def test_complicated_expression_1(self): """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1""" assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_complicated_expression_2(self): """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 2""" assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))]), sorted([self.rse1_id, self.rse2_id, self.rse5_id]))
def test_invalid_expression_unconnected_operator(self): """ RSE_EXPRESSION_PARSER (CORE) Test invalid rse expression: unconnected operator""" rse_expression_parser.parse_expression("TEST_RSE1|")
def test_invalid_expression_wrong_parantheses(self): """ RSE_EXPRESSION_PARSER (CORE) Test invalid rse expression: wrong parantheses """ rse_expression_parser.parse_expression("TEST_RSE1)")
def test_unknown_RSE(self): """ RSE_EXPRESSION_PARSER (CORE) Test unknown RSE """ rse_expression_parser.parse_expression("TEST_RSE999")
def test_simple_rse_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE reference """ assert_equal([rse['id'] for rse in rse_expression_parser.parse_expression(self.rse1)], [self.rse1_id])
def test_attribute_reference(self): """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """ assert_equal([rse['id'] for rse in rse_expression_parser.parse_expression("%s=uk" % self.attribute)], [self.rse4_id])
def check_rule(rule, session): """ Checks a rule Returns a report object """ report = {'rule_id': rule.id, 'num_files': 0, 'did_type': None, 'num_datasets': None, # In case of a container 'missing_locks': 0, 'locks_placed_wrong': 0, 'missing_replicas': 0, 'failed_parse': 0} try: rse_set = parse_expression(expression=rule.rse_expression, session=session) except: report['failed_parse'] = 1 return report did_type = session.query(models.DataIdentifier.did_type).filter_by(scope=rule.scope, name=rule.name).one()[0] report['did_type'] = str(did_type) if did_type == DIDType.CONTAINER: report['num_datasets'] = 0 child_dids = list_child_dids(scope=rule.scope, name=rule.name, session=session) for did in child_dids: if did['type'] == DIDType.DATASET: report['num_datasets'] += 1 files = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).filter_by(scope=did['scope'], name=did['name']).all() report['num_files'] += len(files) report = check_locks(report=report, rule_id=rule.id, copies=rule.copies, rse_set=rse_set, grouping=rule.grouping, did=(did['scope'], did['name']), did_type=DIDType.DATASET, files=files, session=session) report = check_replicas(report=report, copies=rule.copies, rse_set=rse_set, did=(did['scope'], did['name']), did_type=DIDType.DATASET, files=files, session=session) elif did_type == DIDType.DATASET: files = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).filter_by(scope=rule.scope, name=rule.name).all() report['num_files'] = len(files) report = check_locks(report=report, rule_id=rule.id, copies=rule.copies, rse_set=rse_set, grouping=rule.grouping, did=(rule.scope, rule.name), did_type=DIDType.DATASET, files=files, session=session) report = check_replicas(report=report, copies=rule.copies, rse_set=rse_set, did=(rule.scope, rule.name), did_type=DIDType.DATASET, files=files, session=session) else: report['num_files'] = 1 report = check_locks(report=report, rule_id=rule.id, copies=rule.copies, rse_set=rse_set, grouping=rule.grouping, did=(rule.scope, rule.name), did_type=DIDType.FILE, files=[(rule.scope, rule.name)], session=session) report = check_replicas(report=report, copies=rule.copies, rse_set=rse_set, did=(rule.scope, rule.name), did_type=DIDType.DATASET, files=[(rule.scope, rule.name)], session=session) return report
if include_rses: try: parsed_rses = parse_expression(include_rses, session=None) except InvalidRSEExpression, e: logging.error("Invalid RSE exception %s to include RSEs" % (include_rses)) else: for rse in parsed_rses: if rse not in working_rses: working_rses.append(rse) if not (rses or include_rses): working_rses = rses_list if exclude_rses: try: parsed_rses = parse_expression(exclude_rses, session=None) except InvalidRSEExpression, e: logging.error("Invalid RSE exception %s to exclude RSEs: %s" % (exclude_rses, e)) else: working_rses = [rse for rse in working_rses if rse not in parsed_rses] working_rses = [rsemgr.get_rse_info(rse['rse']) for rse in working_rses] return working_rses def get_requests(rse_id=None, process=0, total_processes=1, thread=0, total_threads=1, mock=False, bulk=100, activity=None): ts = time.time() reqs = request.get_next(request_type=[RequestType.TRANSFER, RequestType.STAGEIN, RequestType.STAGEOUT], state=RequestState.QUEUED,
def test_parantheses(self): """ RSE_EXPRESSION_PARSER (CORE) Test parantheses """ assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("(%s)" % self.tag1)]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_order_of_operations(self): """ RSE_EXPRESSION_PARSER (CORE) Test order of operations """ assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s\\%s|%s=fr" % (self.tag1, self.rse3, self.attribute))]), sorted([self.rse1_id, self.rse2_id, self.rse3_id])) assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s\\(%s|%s=fr)" % (self.tag1, self.rse3, self.attribute))]), sorted([self.rse1_id, self.rse2_id]))
def test_union(self): """ RSE_EXPRESSION_PARSER (CORE) Test union operator """ assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s|%s" % (self.tag1, self.tag2))]), sorted([self.rse1_id, self.rse2_id, self.rse3_id, self.rse4_id, self.rse5_id]))