def test_3(self): # enough RSEs and global quota, also after after change -> 2 RSE set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=20) set_global_account_limit(account=self.account, rse_expression=self.rse_2_name, bytes=20) set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20) set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=20) copies = 2 rses = [self.rse_1, self.rse_2] rse_selector = RSESelector(self.account, rses, None, copies) assert_equal(len(rse_selector.rses), 2) rse_selector.select_rse(10, [self.mock1_id], copies=1) rse_selector.select_rse(10, [self.mock2_id], copies=1) rses = rse_selector.select_rse(5, [], copies=2) assert_equal(len(rses), 2)
def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None): """ Select a new target RSE for a rebalanced rule. :param parent_rule rule that is rebalanced. :param current_rse_id: RSE of the source. :param rse_expression: RSE Expression of the source rule. :param subscription_id: Subscription ID of the source rule. :param rse_attributes: The attributes of the source rse. :param other_rses: Other RSEs with existing dataset replicas. :param exclude_expression: Exclude this rse_expression from being target_rses. :param force_expression: Force a specific rse_expression as target. :param session: The DB Session :returns: New RSE expression """ if rse_attributes['type'] != 'DATADISK' and force_expression is None: print('WARNING: dest RSE(s) has to be provided with --force-expression for rebalancing of non-datadisk RSES.') raise InsufficientTargetRSEs current_rse = get_rse_name(rse_id=current_rse_id) current_rse_expr = current_rse # if parent rule has a vo, enforce it vo = parent_rule['scope'].vo if exclude_expression: target_rse = '(%s)\\%s' % (exclude_expression, current_rse_expr) else: target_rse = current_rse_expr rses = parse_expression(expression=rse_expression, filter={'vo': vo}, session=session) # TODO: dest rse selection should be configurable, there might be cases when tier is not defined, or concept of DATADISKS is not present. # if subscription_id: # pass # # get_subscription_by_id(subscription_id, session) if force_expression is not None: if parent_rule['grouping'] != RuleGrouping.NONE: rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter={'vo': vo, 'availability_write': True}, session=session) else: # in order to avoid replication of the part of distributed dataset not present at rabalanced rse -> rses in force_expression # this will be extended with development of delayed rule rses = parse_expression(expression='((%s)|(%s))\\%s' % (force_expression, rse_expression, target_rse), filter={'vo': vo, 'availability_write': True}, session=session) elif len(rses) > 1: # Just define the RSE Expression without the current_rse return '(%s)\\%s' % (rse_expression, target_rse) else: if rse_attributes['tier'] is True or int(rse_attributes['tier']) == 1: # Tier 1 should go to another Tier 1 expression = '(tier=1&type=DATADISK)\\{}'.format(target_rse) elif int(rse_attributes['tier']) == 2: # Tier 2 should go to another Tier 2 expression = '(tier=2&type=DATADISK)\\{}'.format(target_rse) elif int(rse_attributes['tier']) == 3: # Tier 3 will go to Tier 2, since we don't have enough t3s expression = '((tier=2&type=DATADISK)\\datapolicynucleus=1)\\{}'.format(target_rse) rses = parse_expression(expression=expression, filter={'vo': vo, 'availability_write': True}, session=session) rseselector = RSESelector(account=InternalAccount('ddmadmin', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session) return get_rse_name([rse_id for rse_id, _, _ in rseselector.select_rse(size=0, preferred_rse_ids=[], blocklist=other_rses)][0], session=session)
def test_2(self): # enough RSEs and global quota, but not enough global quota after change -> 1 RSE set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=10) set_global_account_limit(account=self.account, rse_expression=self.rse_2_name, bytes=10) set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20) set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=20) copies = 2 rses = [self.rse_1, self.rse_2] rse_selector = RSESelector(self.account, rses, None, copies) assert len(rse_selector.rses) == 2 rse_selector.select_rse(10, [self.mock1_id], copies=1) rses = rse_selector.select_rse(5, [], copies=1) assert len(rses) == 1 assert rses[0][0] == self.mock2_id
def select_target_rse(current_rse, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None): """ Select a new target RSE for a rebalanced rule. :param current_rse: RSE of the source. :param rse_expression: RSE Expression of the source rule. :param subscription_id: Subscription ID of the source rule. :param rse_attributes: The attributes of the source rse. :param other_rses: Other RSEs with existing dataset replicas. :param exclude_expression: Exclude this rse_expression from being target_rses. :param force_expression: Force a specific rse_expression as target. :param session: The DB Session :returns: New RSE expression """ if exclude_expression: target_rse = '(%s)\\%s' % (exclude_expression, current_rse) else: target_rse = current_rse rses = parse_expression(expression=rse_expression, session=session) # if subscription_id: # pass # # get_subscription_by_id(subscription_id, session) if force_expression is not None: rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter={'availability_write': True}, session=session) elif len(rses) > 1: # Just define the RSE Expression without the current_rse return '(%s)\\%s' % (rse_expression, target_rse) elif rse_attributes['tier'] is True or rse_attributes['tier'] == '1': # Tier 1 should go to another Tier 1 rses = parse_expression(expression='(tier=1&type=DATADISK)\\%s' % target_rse, filter={'availability_write': True}, session=session) elif rse_attributes['tier'] == 2 or rse_attributes['tier'] == '2': # Tier 2 should go to another Tier 2 rses = parse_expression(expression='(tier=2&type=DATADISK)\\%s' % target_rse, filter={'availability_write': True}, session=session) rseselector = RSESelector(account='ddmadmin', rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session) return get_rse_name([ rse_id for rse_id, _, _ in rseselector.select_rse( size=0, preferred_rse_ids=[], blacklist=other_rses) ][0], session=session)
def transmogrifier(bulk=5, once=False, sleep_time=60): """ Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash, identifies the subscriptions matching the DIDs and submit a replication rule for each DID matching a subscription. :param thread: Thread number at startup. :param bulk: The number of requests to process. :param once: Run only once. :param sleep_time: Time between two cycles. """ executable = 'transmogrifier' hostname = socket.getfqdn() pid = os.getpid() hb_thread = threading.current_thread() heartbeat.sanity_check(executable=executable, hostname=hostname) while not graceful_stop.is_set(): heart_beat = heartbeat.live(executable, hostname, pid, hb_thread) dids, subscriptions = [], [] tottime = 0 prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads']) try: # Get the new DIDs based on the is_new flag for did in list_new_dids(thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], chunk_size=bulk, did_type=None): dids.append({ 'scope': did['scope'], 'did_type': str(did['did_type']), 'name': did['name'] }) sub_dict = {3: []} # Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest # The priority is defined as 'policyid' for sub in list_subscriptions(None, None): if sub['state'] != SubscriptionState.INACTIVE and sub[ 'lifetime'] and (datetime.now() > sub['lifetime']): update_subscription( name=sub['name'], account=sub['account'], metadata={'state': SubscriptionState.INACTIVE}) elif sub['state'] in [ SubscriptionState.ACTIVE, SubscriptionState.UPDATED ]: priority = 3 if 'policyid' in sub: if int(sub['policyid']) not in sub_dict: sub_dict[int(sub['policyid'])] = [] priority = int(sub['policyid']) sub_dict[priority].append(sub) priorities = list(sub_dict.keys()) priorities.sort() # Order the subscriptions according to their priority for priority in priorities: subscriptions.extend(sub_dict[priority]) except SubscriptionNotFound as error: logging.warning(prepend_str + 'No subscriptions defined: %s' % (str(error))) time.sleep(10) continue except Exception as error: logging.error( prepend_str + 'Failed to get list of new DIDs or subscriptions: %s' % (str(error))) try: results = {} start_time = time.time() blacklisted_rse_id = [ rse['id'] for rse in list_rses({'availability_write': False}) ] logging.debug(prepend_str + 'In transmogrifier worker') identifiers = [] # Loop over all the new dids for did in dids: did_success = True if did['did_type'] == str( DIDType.DATASET) or did['did_type'] == str( DIDType.CONTAINER): did_tag = '%s:%s' % (did['scope'].internal, did['name']) results[did_tag] = [] try: metadata = get_metadata(did['scope'], did['name']) # Loop over all the subscriptions for subscription in subscriptions: # Check if the DID match the subscription if is_matching_subscription( subscription, did, metadata) is True: filter_string = loads(subscription['filter']) split_rule = filter_string.get( 'split_rule', False) stime = time.time() results[did_tag].append(subscription['id']) logging.info(prepend_str + '%s:%s matches subscription %s' % (did['scope'], did['name'], subscription['name'])) rules = loads( subscription['replication_rules']) created_rules = {} cnt = 0 for rule_dict in rules: cnt += 1 created_rules[cnt] = [] # Get all the rule and subscription parameters grouping = rule_dict.get( 'grouping', 'DATASET') lifetime = rule_dict.get('lifetime', None) ignore_availability = rule_dict.get( 'ignore_availability', None) weight = rule_dict.get('weight', None) source_replica_expression = rule_dict.get( 'source_replica_expression', None) locked = rule_dict.get('locked', None) if locked == 'True': locked = True else: locked = False purge_replicas = rule_dict.get( 'purge_replicas', False) if purge_replicas == 'True': purge_replicas = True else: purge_replicas = False rse_expression = str( rule_dict['rse_expression']) comment = str(subscription['comments']) subscription_id = str(subscription['id']) account = subscription['account'] copies = int(rule_dict['copies']) activity = rule_dict.get( 'activity', 'User Subscriptions') try: validate_schema(name='activity', obj=activity) except InputValidationError as error: logging.error( prepend_str + 'Error validating the activity %s' % (str(error))) activity = 'User Subscriptions' if lifetime: lifetime = int(lifetime) str_activity = "".join(activity.split()) success = False nattempt = 5 attemptnr = 0 skip_rule_creation = False selected_rses = [] chained_idx = rule_dict.get( 'chained_idx', None) if chained_idx: params = {} if rule_dict.get( 'associated_site_idx', None): params[ 'associated_site_idx'] = rule_dict.get( 'associated_site_idx', None) logging.debug( '%s Chained subscription identified. Will use %s', prepend_str, str(created_rules[chained_idx])) algorithm = rule_dict.get( 'algorithm', None) selected_rses = select_algorithm( algorithm, created_rules[chained_idx], params) else: # In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm if split_rule: vo = account.vo rses = parse_expression( rse_expression, filter={'vo': vo}) list_of_rses = [ rse['id'] for rse in rses ] # Check that some rule doesn't already exist for this DID and subscription preferred_rse_ids = [] for rule in list_rules( filters={ 'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name'] }): already_existing_rses = [ (rse['rse'], rse['id']) for rse in parse_expression( rule['rse_expression'], filter={'vo': vo}) ] for rse, rse_id in already_existing_rses: if (rse_id in list_of_rses ) and ( rse_id not in preferred_rse_ids): preferred_rse_ids.append( rse_id) if len(preferred_rse_ids ) >= copies: skip_rule_creation = True rse_id_dict = {} for rse in rses: rse_id_dict[ rse['id']] = rse['rse'] try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist= blacklisted_rse_id) ] except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error: logging.warning( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites' % (subscription['name'], account, str(error))) # Now including the blacklisted sites try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist=[]) ] ignore_availability = True except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error: logging.error( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.' % (subscription['name'], account, str(error))) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str(error.__class__. __name__)), delta=1) # The DID won't be reevaluated at the next cycle did_success = did_success and True continue for attempt in range(0, nattempt): attemptnr = attempt nb_rule = 0 # Try to create the rule try: if split_rule: if not skip_rule_creation: for rse in selected_rses: if isinstance( selected_rses, dict): source_replica_expression = selected_rses[ rse].get( 'source_replica_expression', None) weight = selected_rses[ rse].get( 'weight', None) logging.info( prepend_str + 'Will insert one rule for %s:%s on %s' % (did['scope'], did['name'], rse)) rule_ids = add_rule( dids=[{ 'scope': did['scope'], 'name': did['name'] }], account=account, copies=1, rse_expression=rse, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription_id, source_replica_expression =source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment) created_rules[ cnt].append( rule_ids[0]) nb_rule += 1 if nb_rule == copies: success = True break else: rule_ids = add_rule( dids=[{ 'scope': did['scope'], 'name': did['name'] }], account=account, copies=copies, rse_expression= rse_expression, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription['id'], source_replica_expression= source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment) created_rules[cnt].append( rule_ids[0]) nb_rule += 1 monitor.record_counter( counters= 'transmogrifier.addnewrule.done', delta=nb_rule) monitor.record_counter( counters= 'transmogrifier.addnewrule.activity.%s' % str_activity, delta=nb_rule) success = True break except (InvalidReplicationRule, InvalidRuleWeight, InvalidRSEExpression, StagingAreaRuleRequiresLifetime, DuplicateRule) as error: # Errors that won't be retried success = True logging.error(prepend_str + '%s' % (str(error))) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str( error.__class__.__name__)), delta=1) break except (ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, InsufficientAccountLimit, DatabaseException, RSEBlacklisted, RSEWriteBlocked) as error: # Errors to be retried logging.error( prepend_str + '%s Will perform an other attempt %i/%i' % (str(error), attempt + 1, nattempt)) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str( error.__class__.__name__)), delta=1) except Exception: # Unexpected errors monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.unknown', delta=1) exc_type, exc_value, exc_traceback = exc_info( ) logging.critical( prepend_str + ''.join( format_exception( exc_type, exc_value, exc_traceback)).strip( )) did_success = (did_success and success) if (attemptnr + 1) == nattempt and not success: logging.error( prepend_str + 'Rule for %s:%s on %s cannot be inserted' % (did['scope'], did['name'], rse_expression)) else: logging.info( prepend_str + '%s rule(s) inserted in %f seconds' % (str(nb_rule), time.time() - stime)) except DataIdentifierNotFound as error: logging.warning(prepend_str + error) if did_success: if did['did_type'] == str(DIDType.FILE): monitor.record_counter( counters='transmogrifier.did.file.processed', delta=1) elif did['did_type'] == str(DIDType.DATASET): monitor.record_counter( counters='transmogrifier.did.dataset.processed', delta=1) elif did['did_type'] == str(DIDType.CONTAINER): monitor.record_counter( counters='transmogrifier.did.container.processed', delta=1) monitor.record_counter( counters='transmogrifier.did.processed', delta=1) identifiers.append({ 'scope': did['scope'], 'name': did['name'], 'did_type': DIDType.from_sym(did['did_type']) }) time1 = time.time() # Mark the DIDs as processed for identifier in chunks(identifiers, 100): _retrial(set_new_dids, identifier, None) logging.info(prepend_str + 'Time to set the new flag : %f' % (time.time() - time1)) tottime = time.time() - start_time for sub in subscriptions: update_subscription( name=sub['name'], account=sub['account'], metadata={'last_processed': datetime.now()}) logging.info(prepend_str + 'It took %f seconds to process %i DIDs' % (tottime, len(dids))) logging.debug(prepend_str + 'DIDs processed : %s' % (str(dids))) monitor.record_counter(counters='transmogrifier.job.done', delta=1) monitor.record_timer(stat='transmogrifier.job.duration', time=1000 * tottime) except Exception: exc_type, exc_value, exc_traceback = exc_info() logging.critical(prepend_str + ''.join( format_exception(exc_type, exc_value, exc_traceback)).strip()) monitor.record_counter(counters='transmogrifier.job.error', delta=1) monitor.record_counter(counters='transmogrifier.addnewrule.error', delta=1) if once is True: break if tottime < sleep_time: logging.info(prepend_str + 'Will sleep for %s seconds' % (sleep_time - tottime)) time.sleep(sleep_time - tottime) heartbeat.die(executable, hostname, pid, hb_thread) logging.info(prepend_str + 'Graceful stop requested') logging.info(prepend_str + 'Graceful stop done')
def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None): """ Select a new target RSE for a rebalanced rule. :param parent_rule rule that is rebalanced. :param current_rse_id: RSE of the source. :param rse_expression: RSE Expression of the source rule. :param subscription_id: Subscription ID of the source rule. :param rse_attributes: The attributes of the source rse. :param other_rses: Other RSEs with existing dataset replicas. :param exclude_expression: Exclude this rse_expression from being target_rses. :param force_expression: Force a specific rse_expression as target. :param session: The DB Session. :returns: New RSE expression. """ current_rse = get_rse_name(rse_id=current_rse_id) current_rse_expr = current_rse # if parent rule has a vo, enforce it vo = parent_rule['scope'].vo if exclude_expression: target_rse = '((%s)|(%s))' % (exclude_expression, current_rse_expr) else: target_rse = current_rse_expr list_target_rses = [ rse['rse'] for rse in parse_expression( expression=target_rse, filter_={'vo': vo}, session=session) ] list_target_rses.sort() rses = parse_expression(expression=rse_expression, filter_={'vo': vo}, session=session) # TODO: Implement subscription rebalancing if force_expression is not None: if parent_rule['grouping'] != RuleGrouping.NONE: rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter_={ 'vo': vo, 'availability_write': True }, session=session) else: # in order to avoid replication of the part of distributed dataset not present at rebalanced rse -> rses in force_expression # this will be extended with development of delayed rule rses = parse_expression( expression='((%s)|(%s))\\%s' % (force_expression, rse_expression, target_rse), filter_={ 'vo': vo, 'availability_write': True }, session=session) else: # force_expression is not set, RSEs will be selected as rse_expression\rse list_rses = [rse['rse'] for rse in rses] list_rses.sort() if list_rses == list_target_rses: raise InsufficientTargetRSEs( 'Not enough RSEs to rebalance rule %s' % parent_rule['id']) else: rses = parse_expression(expression='(%s)\\%s' % (rse_expression, target_rse), filter_={ 'vo': vo, 'availability_write': True }, session=session) rseselector = RSESelector(account=InternalAccount('root', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session) return get_rse_name([ rse_id for rse_id, _, _ in rseselector.select_rse( size=0, preferred_rse_ids=[], blocklist=other_rses) ][0], session=session)
rse_id_dict = {} for rse in rses: rse_id_dict[rse['id']] = rse['rse'] try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist=blacklisted_rse_id ) ] except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight) as error: logging.warning( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites' % (subscription['name'], account, str(error))) # Now including the blacklisted sites try: rseselector = RSESelector(