def list_replication_rules(filters={}, vo='def', session=None): """ Lists replication rules based on a filter. :param filters: dictionary of attributes by which the results should be filtered. :param vo: The VO to act on. :param session: The database session in use. """ # If filters is empty, create a new dict to avoid overwriting the function's default if not filters: filters = {} if 'scope' in filters: scope = filters['scope'] else: scope = '*' filters['scope'] = InternalScope(scope=scope, vo=vo) if 'account' in filters: account = filters['account'] else: account = '*' filters['account'] = InternalAccount(account=account, vo=vo) rules = rule.list_rules(filters, session=session) for r in rules: yield api_update_return_dict(r, session=session)
def list_replication_rules(filters={}): """ Lists replication rules based on a filter. :param filters: dictionary of attributes by which the results should be filtered. """ return rule.list_rules(filters)
def list_replication_rules(filters={}): """ Lists replication rules based on a filter. :param filters: dictionary of attributes by which the results should be filtered. """ return rule.list_rules(filters)
def test_add_rule_with_r2d2_container_treating_and_duplicate_rule(self): """ JUDGE INJECTOR (CORE): Add a replication rule with an r2d2 container treatment and duplicate rule""" scope = 'mock' container = 'asdf.r2d2_request.2016-04-01-15-00-00.ads.' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe') datasets = [] for i in range(3): files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) datasets.append(dataset) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') attach_dids(scope, container, [{ 'scope': scope, 'name': dataset }], 'jdoe') add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=False) rule_id = add_rule(dids=[{ 'scope': scope, 'name': container }], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=True)[0] approve_rule(rule_id, approver='root') assert (get_rule(rule_id)['state'] == RuleState.INJECT) rule_injector(once=True) # Check if there is a rule for each file with assert_raises(RuleNotFound): get_rule(rule_id) for dataset in datasets: assert (len( [r for r in list_rules({ 'scope': scope, 'name': dataset })]) > 0)
def list_replication_rules(filters={}): """ Lists replication rules based on a filter. :param filters: dictionary of attributes by which the results should be filtered. """ if 'scope' in filters: filters['scope'] = InternalScope(filters['scope']) if 'account' in filters: filters['account'] = InternalAccount(filters['account']) rules = rule.list_rules(filters) for r in rules: yield api_update_return_dict(r)
def list_replication_rules(filters={}, vo='def'): """ Lists replication rules based on a filter. :param filters: dictionary of attributes by which the results should be filtered. :param vo: The VO to act on. """ if 'scope' in filters: scope = filters['scope'] else: scope = '*' filters['scope'] = InternalScope(scope=scope, vo=vo) if 'account' in filters: account = filters['account'] else: account = '*' filters['account'] = InternalAccount(account=account, vo=vo) rules = rule.list_rules(filters) for r in rules: yield api_update_return_dict(r)
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ tmp_scope = 'mock' nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() add_rse(rse) set_account_limit('jdoe', get_rse_id(rse), -1) dsns2 = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': 'jdoe', 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for i in range(nbdatasets)] add_dids(dids=dsns2, account='root') replicas = list() for dsn in dsns2: files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root') replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert(get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None) for dsn in dsns2: assert(get_did(scope='archive', name=dsn['name'])['name'] == dsn['name']) assert(len([x for x in list_rules(filters={'scope': 'archive', 'name': dsn['name']})]) == 1)
def transmogrifier(bulk=5, once=False, sleep_time=60): """ Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash, identifies the subscriptions matching the DIDs and submit a replication rule for each DID matching a subscription. :param thread: Thread number at startup. :param bulk: The number of requests to process. :param once: Run only once. :param sleep_time: Time between two cycles. """ executable = 'transmogrifier' hostname = socket.getfqdn() pid = os.getpid() hb_thread = threading.current_thread() heartbeat.sanity_check(executable=executable, hostname=hostname) while not graceful_stop.is_set(): heart_beat = heartbeat.live(executable, hostname, pid, hb_thread) dids, subscriptions = [], [] tottime = 0 prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads']) try: # Get the new DIDs based on the is_new flag for did in list_new_dids(thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], chunk_size=bulk, did_type=None): dids.append({ 'scope': did['scope'], 'did_type': str(did['did_type']), 'name': did['name'] }) sub_dict = {3: []} # Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest # The priority is defined as 'policyid' for sub in list_subscriptions(None, None): if sub['state'] != SubscriptionState.INACTIVE and sub[ 'lifetime'] and (datetime.now() > sub['lifetime']): update_subscription( name=sub['name'], account=sub['account'], metadata={'state': SubscriptionState.INACTIVE}) elif sub['state'] in [ SubscriptionState.ACTIVE, SubscriptionState.UPDATED ]: priority = 3 if 'policyid' in sub: if int(sub['policyid']) not in sub_dict: sub_dict[int(sub['policyid'])] = [] priority = int(sub['policyid']) sub_dict[priority].append(sub) priorities = list(sub_dict.keys()) priorities.sort() # Order the subscriptions according to their priority for priority in priorities: subscriptions.extend(sub_dict[priority]) except SubscriptionNotFound as error: logging.warning(prepend_str + 'No subscriptions defined: %s' % (str(error))) time.sleep(10) continue except Exception as error: logging.error( prepend_str + 'Failed to get list of new DIDs or subscriptions: %s' % (str(error))) try: results = {} start_time = time.time() blacklisted_rse_id = [ rse['id'] for rse in list_rses({'availability_write': False}) ] logging.debug(prepend_str + 'In transmogrifier worker') identifiers = [] # Loop over all the new dids for did in dids: did_success = True if did['did_type'] == str( DIDType.DATASET) or did['did_type'] == str( DIDType.CONTAINER): did_tag = '%s:%s' % (did['scope'].internal, did['name']) results[did_tag] = [] try: metadata = get_metadata(did['scope'], did['name']) # Loop over all the subscriptions for subscription in subscriptions: # Check if the DID match the subscription if is_matching_subscription( subscription, did, metadata) is True: filter_string = loads(subscription['filter']) split_rule = filter_string.get( 'split_rule', False) stime = time.time() results[did_tag].append(subscription['id']) logging.info(prepend_str + '%s:%s matches subscription %s' % (did['scope'], did['name'], subscription['name'])) rules = loads( subscription['replication_rules']) created_rules = {} cnt = 0 for rule_dict in rules: cnt += 1 created_rules[cnt] = [] # Get all the rule and subscription parameters grouping = rule_dict.get( 'grouping', 'DATASET') lifetime = rule_dict.get('lifetime', None) ignore_availability = rule_dict.get( 'ignore_availability', None) weight = rule_dict.get('weight', None) source_replica_expression = rule_dict.get( 'source_replica_expression', None) locked = rule_dict.get('locked', None) if locked == 'True': locked = True else: locked = False purge_replicas = rule_dict.get( 'purge_replicas', False) if purge_replicas == 'True': purge_replicas = True else: purge_replicas = False rse_expression = str( rule_dict['rse_expression']) comment = str(subscription['comments']) subscription_id = str(subscription['id']) account = subscription['account'] copies = int(rule_dict['copies']) activity = rule_dict.get( 'activity', 'User Subscriptions') try: validate_schema(name='activity', obj=activity) except InputValidationError as error: logging.error( prepend_str + 'Error validating the activity %s' % (str(error))) activity = 'User Subscriptions' if lifetime: lifetime = int(lifetime) str_activity = "".join(activity.split()) success = False nattempt = 5 attemptnr = 0 skip_rule_creation = False selected_rses = [] chained_idx = rule_dict.get( 'chained_idx', None) if chained_idx: params = {} if rule_dict.get( 'associated_site_idx', None): params[ 'associated_site_idx'] = rule_dict.get( 'associated_site_idx', None) logging.debug( '%s Chained subscription identified. Will use %s', prepend_str, str(created_rules[chained_idx])) algorithm = rule_dict.get( 'algorithm', None) selected_rses = select_algorithm( algorithm, created_rules[chained_idx], params) else: # In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm if split_rule: vo = account.vo rses = parse_expression( rse_expression, filter={'vo': vo}) list_of_rses = [ rse['id'] for rse in rses ] # Check that some rule doesn't already exist for this DID and subscription preferred_rse_ids = [] for rule in list_rules( filters={ 'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name'] }): already_existing_rses = [ (rse['rse'], rse['id']) for rse in parse_expression( rule['rse_expression'], filter={'vo': vo}) ] for rse, rse_id in already_existing_rses: if (rse_id in list_of_rses ) and ( rse_id not in preferred_rse_ids): preferred_rse_ids.append( rse_id) if len(preferred_rse_ids ) >= copies: skip_rule_creation = True rse_id_dict = {} for rse in rses: rse_id_dict[ rse['id']] = rse['rse'] try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist= blacklisted_rse_id) ] except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error: logging.warning( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites' % (subscription['name'], account, str(error))) # Now including the blacklisted sites try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist=[]) ] ignore_availability = True except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error: logging.error( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.' % (subscription['name'], account, str(error))) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str(error.__class__. __name__)), delta=1) # The DID won't be reevaluated at the next cycle did_success = did_success and True continue for attempt in range(0, nattempt): attemptnr = attempt nb_rule = 0 # Try to create the rule try: if split_rule: if not skip_rule_creation: for rse in selected_rses: if isinstance( selected_rses, dict): source_replica_expression = selected_rses[ rse].get( 'source_replica_expression', None) weight = selected_rses[ rse].get( 'weight', None) logging.info( prepend_str + 'Will insert one rule for %s:%s on %s' % (did['scope'], did['name'], rse)) rule_ids = add_rule( dids=[{ 'scope': did['scope'], 'name': did['name'] }], account=account, copies=1, rse_expression=rse, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription_id, source_replica_expression =source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment) created_rules[ cnt].append( rule_ids[0]) nb_rule += 1 if nb_rule == copies: success = True break else: rule_ids = add_rule( dids=[{ 'scope': did['scope'], 'name': did['name'] }], account=account, copies=copies, rse_expression= rse_expression, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription['id'], source_replica_expression= source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment) created_rules[cnt].append( rule_ids[0]) nb_rule += 1 monitor.record_counter( counters= 'transmogrifier.addnewrule.done', delta=nb_rule) monitor.record_counter( counters= 'transmogrifier.addnewrule.activity.%s' % str_activity, delta=nb_rule) success = True break except (InvalidReplicationRule, InvalidRuleWeight, InvalidRSEExpression, StagingAreaRuleRequiresLifetime, DuplicateRule) as error: # Errors that won't be retried success = True logging.error(prepend_str + '%s' % (str(error))) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str( error.__class__.__name__)), delta=1) break except (ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, InsufficientAccountLimit, DatabaseException, RSEBlacklisted, RSEWriteBlocked) as error: # Errors to be retried logging.error( prepend_str + '%s Will perform an other attempt %i/%i' % (str(error), attempt + 1, nattempt)) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str( error.__class__.__name__)), delta=1) except Exception: # Unexpected errors monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.unknown', delta=1) exc_type, exc_value, exc_traceback = exc_info( ) logging.critical( prepend_str + ''.join( format_exception( exc_type, exc_value, exc_traceback)).strip( )) did_success = (did_success and success) if (attemptnr + 1) == nattempt and not success: logging.error( prepend_str + 'Rule for %s:%s on %s cannot be inserted' % (did['scope'], did['name'], rse_expression)) else: logging.info( prepend_str + '%s rule(s) inserted in %f seconds' % (str(nb_rule), time.time() - stime)) except DataIdentifierNotFound as error: logging.warning(prepend_str + error) if did_success: if did['did_type'] == str(DIDType.FILE): monitor.record_counter( counters='transmogrifier.did.file.processed', delta=1) elif did['did_type'] == str(DIDType.DATASET): monitor.record_counter( counters='transmogrifier.did.dataset.processed', delta=1) elif did['did_type'] == str(DIDType.CONTAINER): monitor.record_counter( counters='transmogrifier.did.container.processed', delta=1) monitor.record_counter( counters='transmogrifier.did.processed', delta=1) identifiers.append({ 'scope': did['scope'], 'name': did['name'], 'did_type': DIDType.from_sym(did['did_type']) }) time1 = time.time() # Mark the DIDs as processed for identifier in chunks(identifiers, 100): _retrial(set_new_dids, identifier, None) logging.info(prepend_str + 'Time to set the new flag : %f' % (time.time() - time1)) tottime = time.time() - start_time for sub in subscriptions: update_subscription( name=sub['name'], account=sub['account'], metadata={'last_processed': datetime.now()}) logging.info(prepend_str + 'It took %f seconds to process %i DIDs' % (tottime, len(dids))) logging.debug(prepend_str + 'DIDs processed : %s' % (str(dids))) monitor.record_counter(counters='transmogrifier.job.done', delta=1) monitor.record_timer(stat='transmogrifier.job.duration', time=1000 * tottime) except Exception: exc_type, exc_value, exc_traceback = exc_info() logging.critical(prepend_str + ''.join( format_exception(exc_type, exc_value, exc_traceback)).strip()) monitor.record_counter(counters='transmogrifier.job.error', delta=1) monitor.record_counter(counters='transmogrifier.addnewrule.error', delta=1) if once is True: break if tottime < sleep_time: logging.info(prepend_str + 'Will sleep for %s seconds' % (sleep_time - tottime)) time.sleep(sleep_time - tottime) heartbeat.die(executable, hostname, pid, hb_thread) logging.info(prepend_str + 'Graceful stop requested') logging.info(prepend_str + 'Graceful stop done')
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ if get_policy() != 'atlas': LOG.info("Skipping atlas-specific test") return tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for _ in range(nbdatasets)] add_dids(dids=dsns2, account=root) replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for _ in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert (get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is None) for dsn in dsns2: assert (get_did(scope=InternalScope('archive', **self.vo), name=dsn['name'])['name'] == dsn['name']) assert (len([ x for x in list_rules( filters={ 'scope': InternalScope('archive', **self.vo), 'name': dsn['name'] }) ]) == 1)
def add_files(lfns, account, ignore_availability, vo='def', session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :param vo: The VO to act on :param session: The session used """ rule_extension_list = [] attachments = [] # The list of scopes is necessary for the extract_scope filter_ = {'scope': InternalScope(scope='*', vo=vo)} scopes = list_scopes(filter_=filter_, session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] try: lifetime_dict = config_get(section='dirac', option='lifetime', session=session) lifetime_dict = loads(lifetime_dict) except ConfigNotFound: lifetime_dict = {} except JSONDecodeError as err: raise InvalidType('Problem parsing lifetime option in dirac section : %s' % str(err)) except Exception as err: raise RucioException(str(err)) for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope, vo=vo) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope, vo=vo) # Compute lifetime lifetime = None if dsn_scope in lifetime_dict: lifetime = lifetime_dict[dsn_scope] else: for pattern in lifetime_dict: if re.match(pattern, dsn_scope): lifetime = lifetime_dict[pattern] break exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) # to maintain a compatibility between master and LTS-1.26 branches remove keywords for first 3 arguments add_did(dsn_scope, dsn_name, DIDType.DATASET, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': None, 'grouping': 'NONE'}], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': dsn_scope, 'name': dsn_name}]}) rule_extension_list.append((dsn_scope, dsn_name)) if lifetime and (dsn_scope, dsn_name) not in rule_extension_list: # Reset the lifetime of the rule to the configured value rule = [rul for rul in list_rules({'scope': dsn_scope, 'name': dsn_name, 'account': InternalAccount(account, vo=vo)}, session=session) if rul['rse_expression'] == 'ANY=true'] if rule: update_rule(rule[0]['id'], options={'lifetime': lifetime}, session=session) rule_extension_list.append((dsn_scope, dsn_name)) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes_ = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32} if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account, vo=vo), ignore_availability=ignore_availability, session=session) add_rule(dids=[{'scope': lfn_scope, 'name': filename}], account=InternalAccount(account, vo=vo), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]}) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope, vo=vo) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(child_scope, lpn, DIDType.CONTAINER, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': child_scope, 'name': lpn}]}) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account, vo=vo), ignore_duplicate=True, session=session)
'bytes': 1L, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in xrange(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse=rse, dids=files, account='root') replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert (get_replica(scope=replica['scope'], name=replica['name'], rse=rse)['tombstone'] is None) for dsn in dsns2: assert (get_did(scope='archive', name=dsn['name'])['name'] == dsn['name']) assert (len([ x for x in list_rules(filters={ 'scope': 'archive', 'name': dsn['name'] }) ]) == 1)
success = False nattempt = 5 attemptnr = 0 skip_rule_creation = False if split_rule: rses = parse_expression(rse_expression) list_of_rses = [ rse['rse'] for rse in rses ] # Check that some rule doesn't already exist for this DID and subscription preferred_rse_ids = [] for rule in list_rules( filters={ 'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name'] }): already_existing_rses = [ (rse['rse'], rse['id']) for rse in parse_expression( rule['rse_expression']) ] for rse, rse_id in already_existing_rses: if (rse in list_of_rses) and ( rse_id not in preferred_rse_ids): preferred_rse_ids.append( rse_id) if len(preferred_rse_ids) >= copies:
def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool: worker_number, total_workers, logger = heartbeat_handler.live() dids, subscriptions = [], [] tottime = 0 try: # Get the new DIDs based on the is_new flag logger(logging.DEBUG, "Listing new dids") for did in list_new_dids( thread=worker_number, total_threads=total_workers, chunk_size=bulk, did_type=None, ): dids.append({ "scope": did["scope"], "did_type": str(did["did_type"]), "name": did["name"], }) logger(logging.INFO, "%i new dids to process", len(dids)) sub_dict = {3: []} # Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest # The priority is defined as 'policyid' logger(logging.DEBUG, "Listing active subscriptions") for sub in list_subscriptions(None, None): if (sub["state"] != SubscriptionState.INACTIVE and sub["lifetime"] and (datetime.now() > sub["lifetime"])): update_subscription( name=sub["name"], account=sub["account"], metadata={"state": SubscriptionState.INACTIVE}, ) elif sub["state"] in [ SubscriptionState.ACTIVE, SubscriptionState.UPDATED ]: priority = 3 if "policyid" in sub: if int(sub["policyid"]) not in sub_dict: sub_dict[int(sub["policyid"])] = [] priority = int(sub["policyid"]) sub_dict[priority].append(sub) priorities = list(sub_dict.keys()) priorities.sort() # Order the subscriptions according to their priority for priority in priorities: subscriptions.extend(sub_dict[priority]) logger(logging.INFO, "%i active subscriptions", len(subscriptions)) except SubscriptionNotFound as error: logger(logging.WARNING, "No subscriptions defined: %s" % (str(error))) must_sleep = True return must_sleep except Exception as error: logger( logging.ERROR, "Failed to get list of new DIDs or subscriptions: %s" % (str(error)), ) must_sleep = False return must_sleep results = {} start_time = time.time() blocklisted_rse_id = [ rse["id"] for rse in list_rses({"availability_write": False}) ] identifiers = [] # Loop over all the new dids for did in dids: _, _, logger = heartbeat_handler.live() did_success = True if did["did_type"] == str(DIDType.DATASET) or did["did_type"] == str( DIDType.CONTAINER): did_tag = "%s:%s" % (did["scope"].internal, did["name"]) results[did_tag] = [] try: metadata = get_metadata(did["scope"], did["name"]) # Loop over all the subscriptions for subscription in subscriptions: # Check if the DID match the subscription if is_matching_subscription(subscription, did, metadata) is True: filter_string = loads(subscription["filter"]) split_rule = filter_string.get("split_rule", False) stime = time.time() results[did_tag].append(subscription["id"]) logger( logging.INFO, "%s:%s matches subscription %s" % (did["scope"], did["name"], subscription["name"]), ) rules = loads(subscription["replication_rules"]) created_rules = {} cnt = 0 for rule_dict in rules: cnt += 1 created_rules[cnt] = [] # Get all the rule and subscription parameters grouping = rule_dict.get("grouping", "DATASET") lifetime = rule_dict.get("lifetime", None) ignore_availability = rule_dict.get( "ignore_availability", None) weight = rule_dict.get("weight", None) source_replica_expression = rule_dict.get( "source_replica_expression", None) locked = rule_dict.get("locked", None) if locked == "True": locked = True else: locked = False purge_replicas = rule_dict.get( "purge_replicas", False) if purge_replicas == "True": purge_replicas = True else: purge_replicas = False rse_expression = str(rule_dict["rse_expression"]) comment = str(subscription["comments"] )[:RULES_COMMENT_LENGTH] if "comments" in rule_dict: comment = str(rule_dict["comments"]) subscription_id = str(subscription["id"]) account = subscription["account"] copies = int(rule_dict["copies"]) activity = rule_dict.get("activity", "User Subscriptions") try: validate_schema(name="activity", obj=activity, vo=account.vo) except InputValidationError as error: logger( logging.ERROR, "Error validating the activity %s" % (str(error)), ) activity = "User Subscriptions" if lifetime: lifetime = int(lifetime) str_activity = "".join(activity.split()) success = False nattempt = 5 attemptnr = 0 skip_rule_creation = False selected_rses = [] chained_idx = rule_dict.get("chained_idx", None) if chained_idx: params = {} if rule_dict.get("associated_site_idx", None): params[ "associated_site_idx"] = rule_dict.get( "associated_site_idx", None) logger( logging.DEBUG, "Chained subscription identified. Will use %s", str(created_rules[chained_idx]), ) algorithm = rule_dict.get("algorithm", None) selected_rses = select_algorithm( algorithm, created_rules[chained_idx], params) else: # In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm if split_rule: preferred_rses = set() for rule in list_rules( filters={ "subscription_id": subscription_id, "scope": did["scope"], "name": did["name"], }): for rse_dict in parse_expression( rule["rse_expression"], filter_={"vo": account.vo}, ): preferred_rses.add(rse_dict["rse"]) preferred_rses = list(preferred_rses) try: ( selected_rses, preferred_unmatched, ) = resolve_rse_expression( rse_expression, account, weight=weight, copies=copies, size=0, preferred_rses=preferred_rses, blocklist=blocklisted_rse_id, ) except ( InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota, ) as error: logger( logging.WARNING, 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blocklisted sites' % ( subscription["name"], account, str(error), ), ) # Now including the blocklisted sites try: ( selected_rses, preferred_unmatched, ) = resolve_rse_expression( rse_expression, account, weight=weight, copies=copies, size=0, preferred_rses=preferred_rses, ) ignore_availability = True except ( InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota, ) as error: logger( logging.ERROR, 'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.' % ( subscription["name"], account, str(error), ), ) monitor.record_counter( name= "transmogrifier.addnewrule.errortype.{exception}", labels={ "exception": str(error.__class__. __name__) }, ) # The DID won't be reevaluated at the next cycle did_success = did_success and True continue if (len(preferred_rses) - len(preferred_unmatched) >= copies): skip_rule_creation = True for attempt in range(0, nattempt): attemptnr = attempt nb_rule = 0 # Try to create the rule try: if split_rule: if not skip_rule_creation: for rse in selected_rses: if isinstance( selected_rses, dict): source_replica_expression = ( selected_rses[rse].get( "source_replica_expression", None, )) weight = selected_rses[ rse].get( "weight", None) logger( logging.INFO, "Will insert one rule for %s:%s on %s" % (did["scope"], did["name"], rse), ) rule_ids = add_rule( dids=[{ "scope": did["scope"], "name": did["name"], }], account=account, copies=1, rse_expression=rse, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription_id, source_replica_expression= source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment, ) created_rules[cnt].append( rule_ids[0]) nb_rule += 1 if nb_rule == copies: success = True break else: rule_ids = add_rule( dids=[{ "scope": did["scope"], "name": did["name"], }], account=account, copies=copies, rse_expression=rse_expression, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id=subscription["id"], source_replica_expression= source_replica_expression, activity=activity, purge_replicas=purge_replicas, ignore_availability= ignore_availability, comment=comment, ) created_rules[cnt].append(rule_ids[0]) nb_rule += 1 monitor.record_counter( name="transmogrifier.addnewrule.done", delta=nb_rule, ) monitor.record_counter( name= "transmogrifier.addnewrule.activity.{activity}", delta=nb_rule, labels={"activity": str_activity}, ) success = True break except ( InvalidReplicationRule, InvalidRuleWeight, InvalidRSEExpression, StagingAreaRuleRequiresLifetime, DuplicateRule, ) as error: # Errors that won't be retried success = True logger(logging.ERROR, str(error)) monitor.record_counter( name= "transmogrifier.addnewrule.errortype.{exception}", labels={ "exception": str(error.__class__.__name__) }, ) break except ( ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, InsufficientAccountLimit, DatabaseException, RSEWriteBlocked, ) as error: # Errors to be retried logger( logging.ERROR, "%s Will perform an other attempt %i/%i" % (str(error), attempt + 1, nattempt), ) monitor.record_counter( name= "transmogrifier.addnewrule.errortype.{exception}", labels={ "exception": str(error.__class__.__name__) }, ) except Exception: # Unexpected errors monitor.record_counter( name= "transmogrifier.addnewrule.errortype.{exception}", labels={"exception": "unknown"}, ) logger(logging.ERROR, "Unexpected error", exc_info=True) did_success = did_success and success if (attemptnr + 1) == nattempt and not success: logger( logging.ERROR, "Rule for %s:%s on %s cannot be inserted" % (did["scope"], did["name"], rse_expression), ) else: logger( logging.INFO, "%s rule(s) inserted in %f seconds" % (str(nb_rule), time.time() - stime), ) except DataIdentifierNotFound as error: logger(logging.WARNING, str(error)) if did_success: if did["did_type"] == str(DIDType.FILE): monitor.record_counter( name="transmogrifier.did.file.processed") elif did["did_type"] == str(DIDType.DATASET): monitor.record_counter( name="transmogrifier.did.dataset.processed") elif did["did_type"] == str(DIDType.CONTAINER): monitor.record_counter( name="transmogrifier.did.container.processed", delta=1) monitor.record_counter(name="transmogrifier.did.processed", delta=1) identifiers.append({ "scope": did["scope"], "name": did["name"], "did_type": did["did_type"], }) time1 = time.time() # Mark the DIDs as processed for identifier in chunks(identifiers, 100): _retrial(set_new_dids, identifier, None) logger(logging.DEBUG, "Time to set the new flag : %f" % (time.time() - time1)) tottime = time.time() - start_time for sub in subscriptions: update_subscription( name=sub["name"], account=sub["account"], metadata={"last_processed": datetime.now()}, ) logger(logging.INFO, "It took %f seconds to process %i DIDs" % (tottime, len(dids))) logger(logging.DEBUG, "DIDs processed : %s" % (str(dids))) monitor.record_counter(name="transmogrifier.job.done", delta=1) monitor.record_timer(name="transmogrifier.job.duration", time=1000 * tottime) must_sleep = True return must_sleep