def test_update_nonexisting_subscription(self): """ SUBSCRIPTION (API): Test the update of a non-existing subscription """ subscription_name = uuid() update_subscription(name=subscription_name, account='root', metadata={'filter': { 'project': [ 'toto', ] }}, issuer='root')
def PUT(self, account, name): """ Update an existing subscription. HTTP Success: 201 Created HTTP Error: 400 Bad Request 401 Unauthorized 404 Not Found 500 Internal Error """ json_data = data() try: params = loads(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: filter = params['filter'] except KeyError: filter = None try: replication_rules = params['replication_rules'] except KeyError: replication_rules = None try: comments = params['comments'] except KeyError: comments = None try: lifetime = params['lifetime'] except KeyError: lifetime = None try: retroactive = params['retroactive'] except KeyError: retroactive = None try: dry_run = params['dry_run'] except KeyError: dry_run = None try: priority = params['priority'] except KeyError: priority = None try: update_subscription(name=name, account=account, filter=filter, replication_rules=replication_rules, comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority, issuer=ctx.env.get('issuer')) except InvalidObject, error: raise generate_http_error(400, 'InvalidObject', error[0][0])
def put(self, account, name): """ Update an existing subscription. .. :quickref: Subscription; Update a subscription. :param account: The account name. :param name: The subscription name. :status 201: Created. :status 400: Cannot decode json parameter list. :status 401: Invalid Auth Token. :status 404: Subscription Not Found. :status 500: Internal Error. """ json_data = request.data.decode() try: params = loads(json_data) params = params['options'] except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') metadata = {} metadata['filter'] = params.get('filter', None) metadata['replication_rules'] = params.get('replication_rules', None) metadata['comments'] = params.get('comments', None) metadata['lifetime'] = params.get('lifetime', None) metadata['retroactive'] = params.get('retroactive', None) metadata['priority'] = params.get('priority', None) try: update_subscription(name=name, account=account, metadata=metadata, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) except (InvalidObject, TypeError) as error: return generate_http_error_flask(400, 'InvalidObject', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except SubscriptionNotFound as error: return generate_http_error_flask(404, 'SubscriptionNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: logging.exception("Internal Error") return str(error), 500 return 'Created', 201
def PUT(self, account, name): """ Update an existing subscription. HTTP Success: 201 Created HTTP Error: 400 Bad Request 401 Unauthorized 404 Not Found 500 Internal Error """ json_data = data() try: params = loads(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: filter = params['filter'] except KeyError: filter = None try: replication_rules = params['replication_rules'] except KeyError: replication_rules = None try: comments = params['comments'] except KeyError: comments = None try: lifetime = params['lifetime'] except KeyError: lifetime = None try: retroactive = params['retroactive'] except KeyError: retroactive = None try: dry_run = params['dry_run'] except KeyError: dry_run = None try: update_subscription(name=name, account=account, filter=filter, replication_rules=replication_rules, comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run) except SubscriptionNotFound, e: raise generate_http_error(404, 'SubscriptionNotFound', e[0][0])
def PUT(self, account, name): """ Update an existing subscription. HTTP Success: 201 Created HTTP Error: 400 Bad Request 401 Unauthorized 404 Not Found 500 Internal Error """ json_data = data() try: params = loads(json_data.decode()) params = params['options'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') metadata = {} metadata['filter'] = params.get('filter', None) metadata['replication_rules'] = params.get('replication_rules', None) metadata['comments'] = params.get('comments', None) metadata['lifetime'] = params.get('lifetime', None) metadata['retroactive'] = params.get('retroactive', None) metadata['priority'] = params.get('priority', None) try: update_subscription(name=name, account=account, metadata=metadata, issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo')) except (InvalidObject, TypeError) as error: raise generate_http_error(400, 'InvalidObject', error.args[0]) except AccessDenied as error: raise generate_http_error(401, 'AccessDenied', error.args[0]) except SubscriptionNotFound as error: raise generate_http_error(404, 'SubscriptionNotFound', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: raise InternalError(error) raise Created()
def test_create_and_update_and_list_subscription(self): """ SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """ subscription_name = uuid() result = add_subscription(name=subscription_name, account='root', filter={'project': ['data12_900GeV', 'data12_8TeV', 'data13_900GeV', 'data13_8TeV'], 'datatype': ['AOD', ], 'excluded_pattern': '(_tid|physics_(Muons|JetTauEtmiss|Egamma)\..*\.ESD|express_express(?!.*NTUP|.*\.ESD|.*RAW)|(physics|express)(?!.*NTUP).* \ \.x|physics_WarmStart|calibration(?!_PixelBeam.merge.(NTUP_IDVTXLUMI|AOD))|merge.HIST|NTUP_MUONCALIB|NTUP_TRIG)', 'account': 'tier0'}, replication_rules=[(2, 'T1_DATATAPE', True, True), (1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment') with assert_raises(TypeError): result = update_subscription(name=subscription_name, account='root', filter='toto') with assert_raises(InvalidObject): result = update_subscription(name=subscription_name, account='root', filter={'project': 'toto'}) result = update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]}) assert_equal(result, None) result = list_subscriptions(name=subscription_name, account='root') sub = [] for r in result: sub.append(r) assert_equal(len(sub), 1) assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
def put(self, account, name): """ Update an existing subscription. .. :quickref: Subscription; Update a subscription. :param account: The account name. :param name: The subscription name. :status 201: Created. :status 400: Cannot decode json parameter list. :status 401: Invalid Auth Token. :status 404: Subscription Not Found. """ parameters = json_parameters() options = param_get(parameters, 'options') metadata = { 'filter': None, 'replication_rules': None, 'comments': None, 'lifetime': None, 'retroactive': None, 'priority': None, } for keyword in metadata: metadata[keyword] = param_get(options, keyword, default=metadata[keyword]) try: update_subscription(name=name, account=account, metadata=metadata, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) except (InvalidObject, TypeError) as error: return generate_http_error_flask(400, InvalidObject.__name__, error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, error) except SubscriptionNotFound as error: return generate_http_error_flask(404, error) return 'Created', 201
def test_create_and_update_and_list_subscription(self): """ SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """ subscription_name = uuid() with assert_raises(InvalidObject): result = add_subscription(name=subscription_name, account='root', filter={'project': self.projects, 'datatype': ['AOD', ], 'excluded_pattern': self.pattern1, 'account': ['tier0', ]}, replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'noactivity'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root') result = add_subscription(name=subscription_name, account='root', filter={'project': self.projects, 'datatype': ['AOD', ], 'excluded_pattern': self.pattern1, 'account': ['tier0', ]}, replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root') with assert_raises(TypeError): result = update_subscription(name=subscription_name, account='root', filter='toto', issuer='root') with assert_raises(InvalidObject): result = update_subscription(name=subscription_name, account='root', filter={'project': 'toto'}, issuer='root') result = update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]}, issuer='root') assert_equal(result, None) result = list_subscriptions(name=subscription_name, account='root') sub = [] for res in result: sub.append(res) assert_equal(len(sub), 1) assert_equal(loads(sub[0]['filter'])['project'][0], 'toto')
def test_update_nonexisting_subscription(self): """ SUBSCRIPTION (API): Test the update of a non-existing subscription """ subscription_name = uuid() update_subscription(name=subscription_name, account='root', filter={'project': ['toto', ]})
def transmogrifier(bulk=5, once=False, sleep_time=60): """ Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash, identifies the subscriptions matching the DIDs and submit a replication rule for each DID matching a subscription. :param thread: Thread number at startup. :param bulk: The number of requests to process. :param once: Run only once. :param sleep_time: Time between two cycles. """ executable = ' '.join(argv) hostname = socket.getfqdn() pid = os.getpid() hb_thread = threading.current_thread() heartbeat.sanity_check(executable=executable, hostname=hostname) while not graceful_stop.is_set(): heart_beat = heartbeat.live(executable, hostname, pid, hb_thread) dids, subscriptions = [], [] tottime = 0 prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads']) try: # Get the new DIDs based on the is_new flag for did in list_new_dids(thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], chunk_size=bulk): dids.append({ 'scope': did['scope'], 'did_type': str(did['did_type']), 'name': did['name'] }) sub_dict = {3: []} # Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest # The priority is defined as 'policyid' for sub in list_subscriptions(None, None): if sub['state'] != SubscriptionState.INACTIVE and sub[ 'lifetime'] and (datetime.now() > sub['lifetime']): update_subscription( name=sub['name'], account=sub['account'], metadata={'state': SubscriptionState.INACTIVE}, issuer='root') elif sub['state'] in [ SubscriptionState.ACTIVE, SubscriptionState.UPDATED ]: priority = 3 if 'policyid' in sub: if int(sub['policyid']) not in sub_dict: sub_dict[int(sub['policyid'])] = [] priority = int(sub['policyid']) sub_dict[priority].append(sub) priorities = list(sub_dict.keys()) priorities.sort() # Order the subscriptions according to their priority for priority in priorities: subscriptions.extend(sub_dict[priority]) except SubscriptionNotFound as error: logging.warning(prepend_str + 'No subscriptions defined: %s' % (str(error))) time.sleep(10) continue except Exception as error: logging.error( prepend_str + 'Failed to get list of new DIDs or subscriptions: %s' % (str(error))) try: results = {} start_time = time.time() blacklisted_rse_id = [ rse['id'] for rse in list_rses({'availability_write': False}) ] logging.debug(prepend_str + 'In transmogrifier worker') identifiers = [] # Loop over all the new dids for did in dids: did_success = True if did['did_type'] == str( DIDType.DATASET) or did['did_type'] == str( DIDType.CONTAINER): results['%s:%s' % (did['scope'], did['name'])] = [] try: metadata = get_metadata(did['scope'], did['name']) # Loop over all the subscriptions for subscription in subscriptions: # Check if the DID match the subscription if is_matching_subscription( subscription, did, metadata) is True: filter_string = loads(subscription['filter']) split_rule = filter_string.get( 'split_rule', False) if split_rule == 'true': split_rule = True elif split_rule == 'false': split_rule = False stime = time.time() results['%s:%s' % (did['scope'], did['name'])].append( subscription['id']) logging.info(prepend_str + '%s:%s matches subscription %s' % (did['scope'], did['name'], subscription['name'])) for rule_string in loads( subscription['replication_rules']): # Get all the rule and subscription parameters grouping = rule_string.get( 'grouping', 'DATASET') lifetime = rule_string.get( 'lifetime', None) ignore_availability = rule_string.get( 'ignore_availability', None) weight = rule_string.get('weight', None) source_replica_expression = rule_string.get( 'source_replica_expression', None) locked = rule_string.get('locked', None) if locked == 'True': locked = True else: locked = False purge_replicas = rule_string.get( 'purge_replicas', False) if purge_replicas == 'True': purge_replicas = True else: purge_replicas = False rse_expression = str( rule_string['rse_expression']) comment = str(subscription['comments']) subscription_id = str(subscription['id']) account = subscription['account'] copies = int(rule_string['copies']) activity = rule_string.get( 'activity', 'User Subscriptions') try: validate_schema(name='activity', obj=activity) except InputValidationError as error: logging.error( prepend_str + 'Error validating the activity %s' % (str(error))) activity = 'User Subscriptions' if lifetime: lifetime = int(lifetime) str_activity = "".join(activity.split()) success = False nattempt = 5 attemptnr = 0 skip_rule_creation = False if split_rule: rses = parse_expression(rse_expression) list_of_rses = [ rse['rse'] for rse in rses ] # Check that some rule doesn't already exist for this DID and subscription preferred_rse_ids = [] for rule in list_rules( filters={ 'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name'] }): already_existing_rses = [ (rse['rse'], rse['id']) for rse in parse_expression( rule['rse_expression']) ] for rse, rse_id in already_existing_rses: if (rse in list_of_rses) and ( rse_id not in preferred_rse_ids): preferred_rse_ids.append( rse_id) if len(preferred_rse_ids) >= copies: skip_rule_creation = True rse_id_dict = {} for rse in rses: rse_id_dict[rse['id']] = rse['rse'] try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist=blacklisted_rse_id ) ] except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error: logging.warning( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites' % (subscription['name'], account, str(error))) # Now including the blacklisted sites try: rseselector = RSESelector( account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids)) selected_rses = [ rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse( 0, preferred_rse_ids= preferred_rse_ids, copies=copies, blacklist=[]) ] ignore_availability = True except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error: logging.error( prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.' % (subscription['name'], account, str(error))) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str(error.__class__. __name__)), delta=1) # The DID won't be reevaluated at the next cycle did_success = did_success and True continue for attempt in range(0, nattempt): attemptnr = attempt nb_rule = 0 # Try to create the rule try: if split_rule: if not skip_rule_creation: for rse in selected_rses: logging.info( prepend_str + 'Will insert one rule for %s:%s on %s' % (did['scope'], did['name'], rse)) add_rule( dids=[{ 'scope': did['scope'], 'name': did['name'] }], account=account, copies=1, rse_expression=rse, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription_id, source_replica_expression =source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment) nb_rule += 1 if nb_rule == copies: success = True break else: add_rule( dids=[{ 'scope': did['scope'], 'name': did['name'] }], account=account, copies=copies, rse_expression= rse_expression, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked, subscription_id= subscription['id'], source_replica_expression= source_replica_expression, activity=activity, purge_replicas= purge_replicas, ignore_availability= ignore_availability, comment=comment) nb_rule += 1 monitor.record_counter( counters= 'transmogrifier.addnewrule.done', delta=nb_rule) monitor.record_counter( counters= 'transmogrifier.addnewrule.activity.%s' % str_activity, delta=nb_rule) success = True break except (InvalidReplicationRule, InvalidRuleWeight, InvalidRSEExpression, StagingAreaRuleRequiresLifetime, DuplicateRule) as error: # Errors that won't be retried success = True logging.error(prepend_str + '%s' % (str(error))) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str( error.__class__.__name__)), delta=1) break except (ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, InsufficientAccountLimit, DatabaseException, RSEBlacklisted) as error: # Errors to be retried logging.error( prepend_str + '%s Will perform an other attempt %i/%i' % (str(error), attempt + 1, nattempt)) monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.%s' % (str( error.__class__.__name__)), delta=1) except Exception as error: # Unexpected errors monitor.record_counter( counters= 'transmogrifier.addnewrule.errortype.unknown', delta=1) exc_type, exc_value, exc_traceback = exc_info( ) logging.critical( prepend_str + ''.join( format_exception( exc_type, exc_value, exc_traceback)).strip( )) did_success = (did_success and success) if (attemptnr + 1) == nattempt and not success: logging.error( prepend_str + 'Rule for %s:%s on %s cannot be inserted' % (did['scope'], did['name'], rse_expression)) else: logging.info( prepend_str + '%s rule(s) inserted in %f seconds' % (str(nb_rule), time.time() - stime)) except DataIdentifierNotFound as error: logging.warning(prepend_str + error) if did_success: if did['did_type'] == str(DIDType.FILE): monitor.record_counter( counters='transmogrifier.did.file.processed', delta=1) elif did['did_type'] == str(DIDType.DATASET): monitor.record_counter( counters='transmogrifier.did.dataset.processed', delta=1) elif did['did_type'] == str(DIDType.CONTAINER): monitor.record_counter( counters='transmogrifier.did.container.processed', delta=1) monitor.record_counter( counters='transmogrifier.did.processed', delta=1) identifiers.append({ 'scope': did['scope'], 'name': did['name'], 'did_type': DIDType.from_sym(did['did_type']) }) time1 = time.time() # Mark the DIDs as processed for identifier in chunks(identifiers, 100): _retrial(set_new_dids, identifier, None) logging.info(prepend_str + 'Time to set the new flag : %f' % (time.time() - time1)) tottime = time.time() - start_time for sub in subscriptions: update_subscription( name=sub['name'], account=sub['account'], metadata={'last_processed': datetime.now()}, issuer='root') logging.info(prepend_str + 'It took %f seconds to process %i DIDs' % (tottime, len(dids))) logging.debug(prepend_str + 'DIDs processed : %s' % (str(dids))) monitor.record_counter(counters='transmogrifier.job.done', delta=1) monitor.record_timer(stat='transmogrifier.job.duration', time=1000 * tottime) except Exception: exc_type, exc_value, exc_traceback = exc_info() logging.critical(prepend_str + ''.join( format_exception(exc_type, exc_value, exc_traceback)).strip()) monitor.record_counter(counters='transmogrifier.job.error', delta=1) monitor.record_counter(counters='transmogrifier.addnewrule.error', delta=1) if once is True: break if tottime < sleep_time: logging.info(prepend_str + 'Will sleep for %s seconds' % (sleep_time - tottime)) time.sleep(sleep_time - tottime) heartbeat.die(executable, hostname, pid, hb_thread) logging.info(prepend_str + 'Graceful stop requested') logging.info(prepend_str + 'Graceful stop done')
def transmogrifier(bulk=5, once=False): """ Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash, identifies the subscriptions matching the DIDs and submit a replication rule for each DID matching a subscription. :param thread: Thread number at startup. :param bulk: The number of requests to process. :param once: Run only once. """ executable = ' '.join(argv) hostname = socket.getfqdn() pid = os.getpid() hb_thread = threading.current_thread() heartbeat.sanity_check(executable=executable, hostname=hostname) while not graceful_stop.is_set(): heart_beat = heartbeat.live(executable, hostname, pid, hb_thread) dids, subscriptions = [], [] tottime = 0 prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads']) try: for did in list_new_dids(thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], chunk_size=bulk): dids.append({ 'scope': did['scope'], 'did_type': str(did['did_type']), 'name': did['name'] }) sub_dict = {3: []} for sub in list_subscriptions(None, None): if sub['state'] != SubscriptionState.INACTIVE and sub[ 'lifetime'] and (datetime.now() > sub['lifetime']): update_subscription(name=sub['name'], account=sub['account'], state=SubscriptionState.INACTIVE) elif sub['state'] in [ SubscriptionState.ACTIVE, SubscriptionState.UPDATED ]: priority = 3 if 'policyid' in sub: if int(sub['policyid']) not in sub_dict: sub_dict[int(sub['policyid'])] = [] priority = int(sub['policyid']) sub_dict[priority].append(sub) priorities = sub_dict.keys() priorities.sort() for priority in priorities: subscriptions.extend(sub_dict[priority]) except SubscriptionNotFound, error: logging.warning(prepend_str + 'No subscriptions defined: %s' % (str(error))) time.sleep(10) continue except Exception, error: logging.error( prepend_str + 'Failed to get list of new DIDs or subscriptions: %s' % (str(error))) if once: break else: continue
def test_create_and_update_and_list_subscription(self): """ SUBSCRIPTION (API): Test the creation of a new subscription, update it, list it """ rse1, _ = self.rse_factory.make_mock_rse() rse2, _ = self.rse_factory.make_mock_rse() rse_expression = '%s|%s' % (rse1, rse2) subscription_name = uuid() with pytest.raises(InvalidObject): result = add_subscription(name=subscription_name, account='root', filter_={ 'project': self.projects, 'datatype': [ 'AOD', ], 'excluded_pattern': self.pattern1, 'account': [ 'tier0', ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': rse_expression, 'copies': 2, 'activity': 'noactivity' }], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root', **self.vo) result = add_subscription(name=subscription_name, account='root', filter_={ 'project': self.projects, 'datatype': [ 'AOD', ], 'excluded_pattern': self.pattern1, 'account': [ 'tier0', ] }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': rse_expression, 'copies': 2, 'activity': 'Data Brokering' }], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root', **self.vo) with pytest.raises(TypeError): result = update_subscription(name=subscription_name, account='root', metadata={'filter': 'toto'}, issuer='root', **self.vo) with pytest.raises(InvalidObject): result = update_subscription( name=subscription_name, account='root', metadata={'filter': { 'project': 'toto' }}, issuer='root', **self.vo) result = update_subscription( name=subscription_name, account='root', metadata={'filter': { 'project': [ 'toto', ] }}, issuer='root', **self.vo) assert result is None result = list_subscriptions(name=subscription_name, account='root', **self.vo) sub = [] for res in result: sub.append(res) assert len(sub) == 1 assert loads(sub[0]['filter'])['project'][0] == 'toto'