def rule_rebalancer( rse_expression: str, move_subscriptions: bool = False, use_dump: bool = False, sleep_time: int = 300, once: bool = True, dry_run: bool = False, ) -> None: """ Create a rule_rebalancer worker :param rse_expression: The RSE expression where the rule rebalancing is applied. :param move_subscription: To allow rebalancing of subscription rules. Not implemented yet. :param use_dump: To use dump instead of DB query. :param sleep_time: Time between two cycles. :param once: Run only once. :param dry_run: To run in dry run mode (i.e. rules are not created). """ run_daemon( once=once, graceful_stop=graceful_stop, executable="rucio-bb8", logger_prefix="rucio-bb8", partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, rse_expression=rse_expression, move_subscriptions=move_subscriptions, use_dump=use_dump, dry_run=dry_run, ), )
def reaper(rses, include_rses, exclude_rses, vos=None, chunk_size=100, once=False, greedy=False, scheme=None, delay_seconds=0, sleep_time=60, auto_exclude_threshold=100, auto_exclude_timeout=600): """ Main loop to select and delete files. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param include_rses: RSE expression to include RSEs. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. :param vos: VOs on which to look for RSEs. Only used in multi-VO mode. If None, we either use all VOs if run from "def", or the current VO otherwise. :param chunk_size: The size of chunk for deletion. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param scheme: Force the reaper to use a particular protocol, e.g., mock. :param delay_seconds: The delay to query replicas in BEING_DELETED state. :param sleep_time: Time between two cycles. :param auto_exclude_threshold: Number of service unavailable exceptions after which the RSE gets temporarily excluded. :param auto_exclude_timeout: Timeout for temporarily excluded RSEs. """ executable = 'reaper' oidc_account = config_get_bool('reaper', 'oidc_account', False, '') oidc_scope = config_get('reaper', 'oidc_scope', False, 'delete') oidc_audience = config_get('reaper', 'oidc_audience', False, 'rse') run_daemon(once=once, graceful_stop=GRACEFUL_STOP, executable=executable, logger_prefix=executable, partition_wait_time=0 if once else 10, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, rses=rses, include_rses=include_rses, exclude_rses=exclude_rses, vos=vos, chunk_size=chunk_size, greedy=greedy, scheme=scheme, delay_seconds=delay_seconds, auto_exclude_threshold=auto_exclude_threshold, auto_exclude_timeout=auto_exclude_timeout, oidc_account=oidc_account, oidc_scope=oidc_scope, oidc_audience=oidc_audience, ))
def account_update(once=False, sleep_time=10): """ Main loop to check and update the Account Counters. """ run_daemon( once=once, graceful_stop=graceful_stop, executable='abacus-account', logger_prefix='account_update', partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=run_once, )
def preparer(once, sleep_time, bulk, partition_wait_time=10): # Make an initial heartbeat so that all instanced daemons have the correct worker number on the next try logger_prefix = executable = 'conveyor-preparer' run_daemon( once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=logger_prefix, partition_wait_time=partition_wait_time, sleep_time=sleep_time, run_once_fnc=functools.partial(run_once, bulk=bulk), activities=None, )
def finisher(once=False, sleep_time=60, activities=None, bulk=100, db_bulk=1000, partition_wait_time=10): """ Main loop to update the replicas and rules based on finished requests. """ try: conveyor_config = {item[0]: item[1] for item in items('conveyor')} except ConfigNotFound: logging.log(logging.INFO, 'No configuration found for conveyor') conveyor_config = {} # Get suspicious patterns suspicious_patterns = conveyor_config.get('suspicious_pattern', []) if suspicious_patterns: pattern = str(suspicious_patterns) patterns = pattern.split(",") suspicious_patterns = [re.compile(pat.strip()) for pat in patterns] logging.log( logging.DEBUG, "Suspicious patterns: %s" % [pat.pattern for pat in suspicious_patterns]) retry_protocol_mismatches = conveyor_config.get( 'retry_protocol_mismatches', False) logger_prefix = executable = 'conveyor-finisher' if activities: activities.sort() executable += '--activities ' + str(activities) run_daemon( once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=logger_prefix, partition_wait_time=partition_wait_time, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, bulk=bulk, db_bulk=db_bulk, suspicious_patterns=suspicious_patterns, retry_protocol_mismatches=retry_protocol_mismatches, ), activities=activities, )
def collection_replica_update(once=False, limit=1000, sleep_time=10): """ Main loop to check and update the collection replicas. """ run_daemon( once=once, graceful_stop=graceful_stop, executable='abacus-collection-replica', logger_prefix='collection_replica_update', partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, limit=limit, ), )
def rule_injector(once=False, sleep_time=60): """ Main loop to check for asynchronous creation of replication rules """ executable = 'judge-injector' paused_rules = {} # {rule_id: datetime} run_daemon(once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=executable, partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, paused_rules=paused_rules, ))
def poller(once=False, activities=None, sleep_time=60, fts_bulk=100, db_bulk=1000, older_than=60, activity_shares=None, partition_wait_time=10): """ Main loop to check the status of a transfer primitive with a transfertool. """ try: timeout = config_get('conveyor', 'poll_timeout') timeout = float(timeout) except NoOptionError: timeout = None multi_vo = config_get_bool('common', 'multi_vo', False, None) logger_prefix = executable = 'conveyor-poller' if activities: activities.sort() executable += '--activities ' + str(activities) if activity_shares: activities.sort() executable += '--activity_shares' + str(activity_shares) if FILTER_TRANSFERTOOL: executable += ' --filter-transfertool ' + FILTER_TRANSFERTOOL run_daemon( once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=logger_prefix, partition_wait_time=partition_wait_time, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, fts_bulk=fts_bulk, db_bulk=db_bulk, older_than=older_than, activity_shares=activity_shares, multi_vo=multi_vo, timeout=timeout, ), activities=activities, )
def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10): """ Main loop to select and delete dids. """ executable = 'undertaker' paused_dids = {} # {(scope, name): datetime} run_daemon(once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=executable, partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, paused_dids=paused_dids, chunk_size=chunk_size, ))
def re_evaluator(once=False, sleep_time=30, did_limit=100): """ Main loop to check the re-evaluation of dids. """ paused_dids = {} # {(scope, name): datetime} run_daemon( once=once, graceful_stop=graceful_stop, executable='judge-evaluator', logger_prefix='re_evaluator', partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, did_limit=did_limit, paused_dids=paused_dids, ) )
def throttler(once=False, sleep_time=600, partition_wait_time=10): """ Main loop to check rse transfer limits. """ logging.info('Throttler starting') logger_prefix = executable = 'conveyor-throttler' run_daemon( once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=logger_prefix, partition_wait_time=partition_wait_time, sleep_time=sleep_time, run_once_fnc=run_once, activities=None, )
def transmogrifier(bulk: int = 5, once: bool = False, sleep_time: int = 60) -> None: """ Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash, identifies the subscriptions matching the DIDs and submit a replication rule for each DID matching a subscription. :param bulk: The number of requests to process. :param once: Run only once. :param sleep_time: Time between two cycles. """ run_daemon( once=once, graceful_stop=graceful_stop, executable="transmogrifier", logger_prefix="transmogrifier", partition_wait_time=1, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, bulk=bulk, ), )
def submitter(once=False, rses=None, partition_wait_time=10, bulk=100, group_bulk=1, group_policy='rule', source_strategy=None, activities=None, sleep_time=600, max_sources=4, archive_timeout_override=None, filter_transfertool=FILTER_TRANSFERTOOL, transfertool=TRANSFER_TOOL, transfertype=TRANSFER_TYPE, ignore_availability=False): """ Main loop to submit a new transfer primitive to a transfertool. """ try: partition_hash_var = config_get('conveyor', 'partition_hash_var') except NoOptionError: partition_hash_var = None try: scheme = config_get('conveyor', 'scheme') except NoOptionError: scheme = None try: failover_scheme = config_get('conveyor', 'failover_scheme') except NoOptionError: failover_scheme = None try: timeout = config_get('conveyor', 'submit_timeout') timeout = float(timeout) except NoOptionError: timeout = None try: bring_online = config_get_int('conveyor', 'bring_online') except NoOptionError: bring_online = 43200 try: max_time_in_queue = {} timelife_conf = config_get('conveyor', 'max_time_in_queue') timelife_confs = timelife_conf.split(",") for conf in timelife_confs: act, timelife = conf.split(":") max_time_in_queue[act.strip()] = int(timelife.strip()) except NoOptionError: max_time_in_queue = {} if 'default' not in max_time_in_queue: max_time_in_queue['default'] = 168 logging.debug("Maximum time in queue for different activities: %s", max_time_in_queue) logger_prefix = executable = "conveyor-submitter" if activities: activities.sort() executable += '--activities ' + str(activities) if filter_transfertool: executable += ' --filter-transfertool ' + filter_transfertool if rses: rse_ids = [rse['id'] for rse in rses] else: rse_ids = None transfertools = transfertool.split(',') transfertool_kwargs = { FTS3Transfertool: { 'group_policy': group_policy, 'group_bulk': group_bulk, 'source_strategy': source_strategy, 'max_time_in_queue': max_time_in_queue, 'bring_online': bring_online, 'default_lifetime': 172800, 'archive_timeout_override': archive_timeout_override, }, GlobusTransferTool: { 'group_policy': transfertype, 'group_bulk': group_bulk, }, } run_daemon( once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=logger_prefix, partition_wait_time=partition_wait_time, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, bulk=bulk, group_bulk=group_bulk, filter_transfertool=filter_transfertool, transfertools=transfertools, ignore_availability=ignore_availability, scheme=scheme, failover_scheme=failover_scheme, partition_hash_var=partition_hash_var, rse_ids=rse_ids, timeout=timeout, transfertool_kwargs=transfertool_kwargs, ), activities=activities, )
def stager(once=False, rses=None, bulk=100, group_bulk=1, group_policy='rule', source_strategy=None, activities=None, sleep_time=600): """ Main loop to submit a new transfer primitive to a transfertool. """ try: scheme = config_get('conveyor', 'scheme') except NoOptionError: scheme = None try: failover_scheme = config_get('conveyor', 'failover_scheme') except NoOptionError: failover_scheme = None try: bring_online = config_get_int('conveyor', 'bring_online') except NoOptionError: bring_online = 43200 try: max_time_in_queue = {} timelife_conf = config_get('conveyor', 'max_time_in_queue') timelife_confs = timelife_conf.split(",") for conf in timelife_confs: act, timelife = conf.split(":") max_time_in_queue[act.strip()] = int(timelife.strip()) except NoOptionError: max_time_in_queue = {} if 'default' not in max_time_in_queue: max_time_in_queue['default'] = 168 logging.debug("Maximum time in queue for different activities: %s" % max_time_in_queue) logger_prefix = executable = 'conveyor-stager' if activities: activities.sort() executable += '--activities ' + str(activities) if rses: rse_ids = [rse['id'] for rse in rses] else: rse_ids = None transfertool_kwargs = { FTS3Transfertool: { 'group_policy': group_policy, 'group_bulk': group_bulk, 'source_strategy': source_strategy, 'max_time_in_queue': max_time_in_queue, 'bring_online': bring_online, 'default_lifetime': -1, } } run_daemon( once=once, graceful_stop=graceful_stop, executable=executable, logger_prefix=logger_prefix, partition_wait_time=None, sleep_time=sleep_time, run_once_fnc=functools.partial( run_once, bulk=bulk, group_bulk=group_bulk, scheme=scheme, failover_scheme=failover_scheme, rse_ids=rse_ids, transfertool_kwargs=transfertool_kwargs, ), activities=activities, )