def get_rse_by_country(self, institute=None, institute_country=None): """ This function returns a site based on the user's institute and institute country. If institute = '' the user no longer belong to CMS Experiment. This part is now handled by the mapping algorithm, with assigns to these kind of users a default policy, called TestPolicy. It can be removed. """ if not institute: return None # TODO: Should have multiple countries in the institute policy and have a default per country like FNAL if institute_country == 'US': with open('config_institute_policy.json') as institutes_per_rse: rses_by_country = json.load(institutes_per_rse) institutes_by_country = rses_by_country[institute_country] # Can uncomment to go back to using Test RSEs. Better to use a different JSON file rucio_rses = [r['rse'] for r in self.client.list_rses()] for rse_key, institutes_val in institutes_by_country.items(): rse = rse_key # Not needed anymore + '_Test' if institute in institutes_val: if rse not in rucio_rses: raise RSENotFound('RSE %s not found' % rse) return rse return u'T3_US_FNALLPC' # This last return should not be hardcoded! Needs to be obtained from the JSON file else: # for other policies pass
def get_rses_to_process(rses, include_rses, exclude_rses, vos): """ Return the list of RSEs to process based on rses, include_rses and exclude_rses :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :param vos: VOs on which to look for RSEs. Only used in multi-VO mode. If None, we either use all VOs if run from "def", :returns: A list of RSEs to process """ multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: if vos: logging.log(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.') vos = ['def'] else: if vos: invalid = set(vos) - set([v['vo'] for v in list_vos()]) if invalid: msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid])) raise VONotFound(msg) else: vos = [v['vo'] for v in list_vos()] logging.log(logging.INFO, 'Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos]))) cache_key = 'rses_to_process' if multi_vo: cache_key += '@%s' % '-'.join(vo for vo in vos) result = REGION.get(cache_key) if result is not NO_VALUE: return result all_rses = [] for vo in vos: all_rses.extend(list_rses(filters={'vo': vo})) if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] REGION.set(cache_key, rses) logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses])) return rses
def get_rses_to_process(rses, include_rses, exclude_rses): """ Return the list of RSEs to process based on rses, include_rses and exclude_rses :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :returns: A list of RSEs to process """ result = REGION.get('rses_to_process') if result is not NO_VALUE: return result all_rses = list_rses() if rses: if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): logging.warning( 'Ignoring argument rses, this is only available in a single-VO setup. Please try an RSE Expression with include_rses if it is required.' ) rses = all_rses else: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format( 's' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] REGION.set('rses_to_process', rses) logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses])) return rses
def get_rses_to_process(rses, include_rses, exclude_rses): """ Return the list of RSEs to process based on rses, include_rses and exclude_rses :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :returns: A list of RSEs to process """ result = REGION.get('rses_to_process') if result is not NO_VALUE: return result all_rses = list_rses() if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format( 's' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] REGION.set('rses_to_process', rses) logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses])) return rses
def run(total_workers=1, chunk_size=100, threads_per_worker=None, once=False, greedy=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, vos=None, delay_seconds=0): """ Starts up the reaper threads. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param threads_per_worker: Total number of threads created by each worker. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :param vos: VOs on which to look for RSEs. Only used in multi-VO mode. If None, we either use all VOs if run from "def", or the current VO otherwise. """ if rucio.db.sqla.util.is_old_db(): raise DatabaseException('Database was not updated, daemon won\'t start') logging.info('Reaper1 daemon will be deprecated and replaced by reaper2 with Rucio release 1.25 (~March 2021)!') logging.info('main: starting processes') multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: if vos: logging.warning('Ignoring argument vos, this is only applicable in a multi-VO setup.') vos = ['def'] else: if vos: invalid = set(vos) - set([v['vo'] for v in list_vos()]) if invalid: msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid])) raise VONotFound(msg) else: vos = [v['vo'] for v in list_vos()] logging.info('Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos]))) all_rses = [] for vo in vos: all_rses.extend(rse_core.list_rses(filters={'vo': vo})) if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if not rses: logging.error('Reaper: No RSEs found. Exiting.') return logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses])) threads = [] nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1 rses = random.sample(rses, len(rses)) for worker in range(total_workers): for child in range(threads_per_worker or 1): rses_list = rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker] if not rses_list: logging.warning('Reaper: Empty RSEs list for worker %(worker)s' % locals()) continue kwargs = {'worker_number': worker, 'child_number': child, 'total_children': threads_per_worker or 1, 'once': once, 'chunk_size': chunk_size, 'greedy': greedy, 'rses': rses_list, 'delay_seconds': delay_seconds, 'scheme': scheme} threads.append(threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, child: %s' % (worker, child))) [t.start() for t in threads] while threads[0].is_alive(): [t.join(timeout=3.14) for t in threads]
def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, vos=None, delay_seconds=0): """ Starts up the reaper threads. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param once: If True, only runs one iteration of the main loop. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. (Single-VO only) :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :param vos: VOs on which to look for RSEs. Only used in multi-VO mode. If None, we either use all VOs if run from "def", or the current VO otherwise. """ setup_logging() if rucio.db.sqla.util.is_old_db(): raise DatabaseException('Database was not updated, daemon won\'t start') logging.info('main: starting processes') multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: if vos: logging.warning('Ignoring argument vos, this is only applicable in a multi-VO setup.') vos = ['def'] else: if vos: invalid = set(vos) - set([v['vo'] for v in list_vos()]) if invalid: msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid])) raise VONotFound(msg) else: vos = [v['vo'] for v in list_vos()] logging.info('Light Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos]))) all_rses = [] for vo in vos: all_rses.extend(rse_core.list_rses(filters={'vo': vo})) if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if not rses: logging.error('Light Reaper: No RSEs found. Exiting.') return threads = [] for worker in range(total_workers): kwargs = {'worker_number': worker, 'total_workers': total_workers, 'rses': rses, 'once': once, 'chunk_size': chunk_size, 'scheme': scheme} threads.append(threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, Total_Workers: %s' % (worker, total_workers))) [t.start() for t in threads] while threads[0].is_alive(): [t.join(timeout=3.14) for t in threads]
def run(threads=1, chunk_size=100, once=False, greedy=False, rses=None, scheme=None, exclude_rses=None, include_rses=None, delay_seconds=0, sleep_time=60): """ Starts up the reaper threads. :param threads: The total number of workers. :param chunk_size: The size of chunk for deletion. :param threads_per_worker: Total number of threads created by each worker. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. :param delay_seconds: The delay to query replicas in BEING_DELETED state. :param sleep_time: Time between two cycles. """ logging.info('main: starting processes') all_rses = list_rses() if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format( 's' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] if not rses: logging.error('Reaper: No RSEs found. Exiting.') return logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses])) logging.info('starting reaper threads') threads_list = [ threading.Thread(target=reaper, kwargs={ 'once': once, 'rses': rses, 'chunk_size': chunk_size, 'greedy': greedy, 'sleep_time': sleep_time, 'delay_seconds': delay_seconds, 'scheme': scheme }) for _ in range(0, threads) ] for thread in threads_list: thread.start() logging.info('waiting for interrupts') # To populate the cache get_rses_to_hostname_mapping() # Interruptible joins require a timeout. while threads_list: threads_list = [ thread.join(timeout=3.14) for thread in threads_list if thread and thread.isAlive() ]
def run(total_workers=1, chunk_size=100, threads_per_worker=None, once=False, greedy=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, delay_seconds=0): """ Starts up the reaper threads. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param threads_per_worker: Total number of threads created by each worker. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param include_rses: RSE expression to include RSEs. """ logging.info('main: starting processes') all_rses = rse_core.list_rses() if rses: invalid = set(rses) - set([rse['rse'] for rse in all_rses]) if invalid: msg = 'RSE{} {} cannot be found'.format( 's' if len(invalid) > 1 else '', ', '.join([repr(rse) for rse in invalid])) raise RSENotFound(msg) rses = [rse for rse in all_rses if rse['rse'] in rses] else: rses = all_rses if exclude_rses: excluded_rses = parse_expression(exclude_rses) rses = [rse for rse in rses if rse not in excluded_rses] if include_rses: included_rses = parse_expression(include_rses) rses = [rse for rse in rses if rse in included_rses] if not rses: logging.error('Reaper: No RSEs found. Exiting.') return logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses])) threads = [] nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1 rses = random.sample(rses, len(rses)) for worker in range(total_workers): for child in range(threads_per_worker or 1): rses_list = rses[worker * nb_rses_per_worker:worker * nb_rses_per_worker + nb_rses_per_worker] if not rses_list: logging.warning( 'Reaper: Empty RSEs list for worker %(worker)s' % locals()) continue kwargs = { 'worker_number': worker, 'child_number': child + 1, 'total_children': threads_per_worker or 1, 'once': once, 'chunk_size': chunk_size, 'greedy': greedy, 'rses': rses_list, 'delay_seconds': delay_seconds, 'scheme': scheme } threads.append( threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, child: %s' % (worker, child + 1))) [t.start() for t in threads] while threads[0].is_alive(): [t.join(timeout=3.14) for t in threads]