def test_to_repair_a_rule_with_DATASET_grouping_whose_transfer_failed( self): """ JUDGE REPAIRER: Test to repair a rule with 1 failed transfer (lock)""" rule_repairer(once=True) # Clean out the repairer scope = InternalScope('mock') files = create_files(4, scope, self.rse4_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=1, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, activity='DebugJudge')[0] successful_transfer( scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) successful_transfer( scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) failed_transfer( scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) failed_transfer( scope=scope, name=files[3]['name'], rse_id=get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id) assert (rule_id == get_rule(rule_id)['id'].replace('-', '').lower()) assert (RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) assert (RuleState.REPLICATING == get_rule(rule_id)['state']) assert (get_replica_locks( scope=files[2]['scope'], name=files[2]['name'])[0].rse_id == get_replica_locks( scope=files[3]['scope'], name=files[3]['name'])[0].rse_id) assert (get_replica_locks( scope=files[1]['scope'], name=files[1]['name'])[0].rse_id == get_replica_locks( scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
def _create_tmp_DID(self, type='DATASET'): did_name = 'fe_test_did_%s' % generate_uuid() add_did(scope=self.tmp_scope, name=did_name, did_type='DATASET', account=self.root) return did_name
def test_judge_evaluate_detach_datasetlock(self): """ JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container""" re_evaluator(once=True) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), self.jdoe) attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe) # Add a rule to the Container add_rule(dids=[{'scope': scope, 'name': container}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) # Check if the datasetlock is there locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) > 0) detach_dids(scope, container, [{'scope': scope, 'name': dataset}]) # Fake judge re_evaluator(once=True) locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) == 0)
def test_delete_dids(self): """ DATA IDENTIFIERS (CORE): Delete dids """ tmp_scope = 'mock' dsns = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'did_type': DIDType.DATASET} for i in xrange(5)] for dsn in dsns: add_did(scope=tmp_scope, name=dsn['name'], type='DATASET', account='root') delete_dids(dids=dsns, account='root')
def test_judge_evaluate_detach_datasetlock(self): """ JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container""" re_evaluator(once=True) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe') attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe') # Add a rule to the Container add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) # Check if the datasetlock is there locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) > 0) detach_dids(scope, container, [{'scope': scope, 'name': dataset}]) # Fake judge re_evaluator(once=True) locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) == 0)
def test_account_counter_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS""" re_evaluator(once=True) account_update(once=True) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) account_update(once=True) account_counter_before = get_counter(self.rse1_id, 'jdoe') detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) account_update(once=True) account_counter_after = get_counter(self.rse1_id, 'jdoe') assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes']) assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_list_rules_states(self): """ SUBSCRIPTION (API): Test listing of rule states for subscription """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() add_rse(site_a) add_rse(site_b) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subscription_name = uuid() id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'}, replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment') subscriptions = list_subscriptions(name=subscription_name, account='root') # workaround until add_subscription returns the id id = None for s in subscriptions: id = s['id'] # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) for r in list_subscription_rule_states(account='root', name=subscription_name): assert_equal(r[3], 2)
def test_judge_expire_rule_with_child_rule(self): """ JUDGE CLEANER: Test the judge when deleting expired rules with child rules""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] child_rule = add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=1, rse_expression=self.rse3, grouping='NONE', weight='fakeweight', lifetime=-3, locked=False, subscription_id=None)[0] update_rule(rule_id, {'child_rule_id': child_rule}) rule_cleaner(once=True)
def test_bb8_rebalance_rule(self): """ BB8: Test the rebalance rule method""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] rule = {} try: rule = get_rule(rule_id) except: assert_raises(RuleNotFound, get_rule, rule_id) child_rule = rebalance_rule(rule, 'Rebalance', self.rse3, priority=3) rule_cleaner(once=True) assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow()) assert(get_rule(rule_id)['child_rule_id'] == child_rule) rule_cleaner(once=True) assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow()) successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse3_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse3_id, nowait=False) with assert_raises(UnsupportedOperation): delete_rule(rule_id) successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse3_id, nowait=False) rule_cleaner(once=True) assert(get_rule(child_rule)['state'] == RuleState.OK)
def test_judge_add_files_to_dataset(self): """ JUDGE EVALUATOR: Test the judge when adding files to dataset""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) # Add a first rule to the DS add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) attach_dids(scope, dataset, files, self.jdoe) re_evaluator(once=True) files = create_files(3, scope, self.rse1_id) attach_dids(scope, dataset, files, self.jdoe) # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert (len( get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
def test_add_rule_container_dataset_with_weights(self): """ REPLICATION RULE (CORE): Add a replication rule on a container, DATASET Grouping, WEIGHTS""" scope = 'mock' container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe') all_files = [] dataset_files = [] for i in xrange(3): files = create_files(3, scope, self.rse1) all_files.extend(files) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe') dataset_files.append({'scope': scope, 'name': dataset, 'files': files}) add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=False, subscription_id=None) t1 = set([self.rse1_id, self.rse3_id, self.rse5_id]) for dataset in dataset_files: first_locks = None for file in dataset['files']: if first_locks is None: first_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]) rse_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]) assert(len(t1.intersection(rse_locks)) == 2) assert(len(first_locks.intersection(rse_locks)) == 2) assert_in(self.rse1_id, rse_locks)
def test_judge_deny_rule(self): """ JUDGE INJECTOR: Test the judge when asking approval for a rule and denying it""" scope = 'mock' files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS rule_id = add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='jdoe', copies=1, rse_expression=self.rse4, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, ask_approval=True)[0] assert (get_rule(rule_id)['state'] == RuleState.WAITING_APPROVAL) deny_rule(rule_id=rule_id, approver='root') assert_raises(RuleNotFound, get_rule, rule_id)
def test_judge_inject_rule(self): """ JUDGE INJECTOR: Test the judge when injecting a rule""" scope = 'mock' files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS rule_id = add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, asynchronous=True)[0] assert (get_rule(rule_id)['state'] == RuleState.INJECT) rule_injector(once=True) # Check if the Locks are created properly for file in files: assert (len( get_replica_locks(scope=file['scope'], name=file['name'])) == 2) assert (get_rule(rule_id)['state'] == RuleState.REPLICATING)
def test_list_rules_states(self): """ SUBSCRIPTION (API): Test listing of rule states for subscription """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() add_rse(site_a) add_rse(site_b) # Add quota set_account_limit('root', get_rse_id(site_a), -1) set_account_limit('root', get_rse_id(site_b), -1) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subscription_name = uuid() subid = add_subscription(name=subscription_name, account='root', filter={'account': ['root', ], 'scope': [tmp_scope, ]}, replication_rules=[{'lifetime': 86400, 'rse_expression': 'MOCK|MOCK2', 'copies': 2, 'activity': 'Data Brokering'}], lifetime=100000, retroactive=0, dry_run=0, comments='This is a comment', issuer='root') # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) for rule in list_subscription_rule_states(account='root', name=subscription_name): assert_equal(rule[3], 2)
def test_account_counter_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS""" re_evaluator(once=True) account_update(once=True) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) account_update(once=True) account_counter_before = get_usage(self.rse1_id, self.jdoe) detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) account_update(once=True) account_counter_after = get_usage(self.rse1_id, self.jdoe) assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes']) assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_run_transmogrifier(self): """ SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """ rse1, _ = self.rse_factory.make_mock_rse() rse2, _ = self.rse_factory.make_mock_rse() rse3, _ = self.rse_factory.make_mock_rse() rse_expression = '%s|%s|%s' % (rse1, rse2, rse3) tmp_scope = InternalScope('mock_' + uuid()[:8], **self.vo) root = InternalAccount('root', **self.vo) add_scope(tmp_scope, root) subscription_name = uuid() dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root) subid = self.sub_client.add_subscription(name=subscription_name, account='root', filter_={ 'scope': [ tmp_scope.external, ], 'pattern': 'dataset-.*', 'split_rule': True }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': rse_expression, 'copies': 2, 'activity': 'Data Brokering' }], lifetime=None, retroactive=0, dry_run=0, comments='Ni ! Ni!', priority=1) run(threads=1, bulk=1000000, once=True) rules = [ rule for rule in self.did_client.list_did_rules( scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid) ] assert len(rules) == 2 set_new_dids([ { 'scope': tmp_scope, 'name': dsn }, ], 1) run(threads=1, bulk=1000000, once=True) rules = [ rule for rule in self.did_client.list_did_rules( scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid) ] assert len(rules) == 2
def test_judge_inject_delayed_rule(self): """ JUDGE INJECTOR: Test the judge when injecting a delayed rule""" scope = InternalScope('mock', **self.vo) files = create_files(1, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) [file] = files # Add a delayed rule rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, delay_injection=3600)[0] rule = get_rule(rule_id) assert rule['state'] == RuleState.INJECT assert rule['updated_at'] < rule['created_at'] assert datetime.utcnow() + timedelta(seconds=3550) < rule['created_at'] < datetime.utcnow() + timedelta(seconds=3650) # The time to create the rule has not yet arrived. The injector must skip this rule, no locks must be created rule_injector(once=True) assert get_rule(rule_id)['state'] == RuleState.INJECT assert not get_replica_locks(scope=file['scope'], name=file['name']) # simulate that time to inject the rule has arrived @transactional_session def __update_created_at(session=None): session.query(ReplicationRule).filter_by(id=rule_id).one().created_at = datetime.utcnow() __update_created_at() # The injector must create the locks now rule_injector(once=True) assert get_rule(rule_id)['state'] == RuleState.REPLICATING assert len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2
def test_touch_dids(self): """ DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp""" tmp_scope = 'mock' tmp_dsn1 = 'dsn_%s' % generate_uuid() tmp_dsn2 = 'dsn_%s' % generate_uuid() add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root') add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root') now = datetime.utcnow() now -= timedelta(microseconds=now.microsecond) assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1)) assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2)) touch_dids(dids=[{ 'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now }]) assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1)) assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_to_repair_a_rule_with_only_1_rse_whose_transfers_failed(self): """ JUDGE REPAIRER: Test to repair a rule with only 1 rse whose transfers failed (lock)""" rule_repairer(once=True) # Clean out the repairer scope = 'mock' files = create_files(4, scope, self.rse4, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] successful_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) failed_transfer(scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) failed_transfer(scope=scope, name=files[3]['name'], rse_id=get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id) cancel_request_did(scope=scope, name=files[2]['name'], dest_rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) cancel_request_did(scope=scope, name=files[3]['name'], dest_rse_id=get_replica_locks(scope=files[3]['scope'], name=files[2]['name'])[0].rse_id) assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower()) assert(RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) # Stil assert STUCK because of delays: assert(RuleState.STUCK == get_rule(rule_id)['state']) assert(get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id == get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
def test_repair_a_rule_with_source_replica_expression(self): """ JUDGE EVALUATOR: Test the judge when a with two rules with source_replica_expression""" scope = 'mock' files = create_files(3, scope, self.rse4) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS rule_id1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rule_id2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, source_replica_expression=self.rse1)[0] assert(RuleState.REPLICATING == get_rule(rule_id1)['state']) assert(RuleState.STUCK == get_rule(rule_id2)['state']) successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse1_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse1_id, nowait=False) successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse1_id, nowait=False) # Also make replicas AVAILABLE session = get_session() replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[0]['name'], rse_id=self.rse1_id).one() replica.state = ReplicaState.AVAILABLE replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[1]['name'], rse_id=self.rse1_id).one() replica.state = ReplicaState.AVAILABLE replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[2]['name'], rse_id=self.rse1_id).one() replica.state = ReplicaState.AVAILABLE session.commit() rule_repairer(once=True) assert(RuleState.OK == get_rule(rule_id1)['state']) assert(RuleState.REPLICATING == get_rule(rule_id2)['state'])
def test_to_repair_a_rule_with_NONE_grouping_whose_transfer_failed(self): """ JUDGE REPAIRER: Test to repair a rule with 1 failed transfer (lock)""" rule_repairer(once=True) # Clean out the repairer scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse4_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0] failed_rse_id = get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['state'] == ReplicaState.COPYING) assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['lock_cnt'] == 1) successful_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) failed_transfer(scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower()) assert(RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) assert(RuleState.REPLICATING == get_rule(rule_id)['state']) assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['state'] == ReplicaState.UNAVAILABLE) assert(get_replica(scope=files[2]['scope'], name=files[2]['name'], rse_id=failed_rse_id)['lock_cnt'] == 0)
def test_repair_a_rule_with_source_replica_expression(self): """ JUDGE EVALUATOR: Test the judge when a with two rules with source_replica_expression""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse4_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) # Add a first rule to the DS rule_id1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rule_id2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, source_replica_expression=self.rse1)[0] assert(RuleState.REPLICATING == get_rule(rule_id1)['state']) assert(RuleState.STUCK == get_rule(rule_id2)['state']) successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse1_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse1_id, nowait=False) successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse1_id, nowait=False) # Also make replicas AVAILABLE session = get_session() replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[0]['name'], rse_id=self.rse1_id).one() replica.state = ReplicaState.AVAILABLE replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[1]['name'], rse_id=self.rse1_id).one() replica.state = ReplicaState.AVAILABLE replica = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=files[2]['name'], rse_id=self.rse1_id).one() replica.state = ReplicaState.AVAILABLE session.commit() rule_repairer(once=True) assert(RuleState.OK == get_rule(rule_id1)['state']) assert(RuleState.REPLICATING == get_rule(rule_id2)['state'])
def test_to_repair_a_rule_with_only_1_rse_whose_transfers_failed(self): """ JUDGE REPAIRER: Test to repair a rule with only 1 rse whose transfers failed (lock)""" rule_repairer(once=True) # Clean out the repairer scope = InternalScope('mock', **self.vo) files = create_files(4, scope, self.rse4_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] successful_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=get_replica_locks(scope=files[1]['scope'], name=files[2]['name'])[0].rse_id, nowait=False) failed_transfer(scope=scope, name=files[2]['name'], rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) failed_transfer(scope=scope, name=files[3]['name'], rse_id=get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id) cancel_request_did(scope=scope, name=files[2]['name'], dest_rse_id=get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id) cancel_request_did(scope=scope, name=files[3]['name'], dest_rse_id=get_replica_locks(scope=files[3]['scope'], name=files[2]['name'])[0].rse_id) assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower()) assert(RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) # Stil assert STUCK because of delays: assert(RuleState.STUCK == get_rule(rule_id)['state']) assert(get_replica_locks(scope=files[2]['scope'], name=files[2]['name'])[0].rse_id == get_replica_locks(scope=files[3]['scope'], name=files[3]['name'])[0].rse_id)
def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blocklisted(self): """ JUDGE REPAIRER: Test to repair a rule with only 1 rse whose site is blocklisted""" rse = rse_name_generator() rse_id = add_rse(rse, **self.vo) update_rse(rse_id, {'availability_write': False}) set_local_account_limit(self.jdoe, rse_id, -1) rule_repairer(once=True) # Clean out the repairer scope = InternalScope('mock', **self.vo) files = create_files(4, scope, self.rse4_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True, activity='DebugJudge')[0] assert(RuleState.STUCK == get_rule(rule_id)['state']) rule_repairer(once=True) # Stil assert STUCK because of ignore_availability: assert(RuleState.STUCK == get_rule(rule_id)['state']) region = make_region().configure('dogpile.cache.memcached', expiration_time=3600, arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True}) region.delete(sha256(rse.encode()).hexdigest()) update_rse(rse_id, {'availability_write': True}) rule_repairer(once=True) assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
def testdid(vo): did_name = 'testdid_%s' % generate_uuid() mock_scope = InternalScope('mock', vo=vo) didtype = 'DATASET' account = InternalAccount('root', vo=vo) add_did(scope=mock_scope, name=did_name, type=didtype, account=account) yield {'name': did_name, 'scope': mock_scope} delete_dids(dids=[{'name': did_name, 'scope': mock_scope, 'did_type': didtype, 'purge_replicas': True}], account=account)
def test_add_rule(self): """ REPLICATION RULE (CLIENT): Add a replication rule """ scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE') assert_is_instance(ret, list)
def test_get_metadata(self): """ DID Meta (JSON): Get did meta """ skip_without_json() did_name = 'mock_did_%s' % generate_uuid() meta_key = 'my_key_%s' % generate_uuid() meta_value = 'my_value_%s' % generate_uuid() add_did(scope=self.tmp_scope, name=did_name, type='DATASET', account=self.root) set_metadata(scope=self.tmp_scope, name=did_name, key=meta_key, value=meta_value) assert get_metadata(scope=self.tmp_scope, name=did_name, plugin='JSON')[meta_key] == meta_value
def test_delete_dids(self): """ DATA IDENTIFIERS (CORE): Delete dids """ tmp_scope = 'mock' dsns = [{'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'purge_replicas': False, 'did_type': DIDType.DATASET} for i in range(5)] for dsn in dsns: add_did(scope=tmp_scope, name=dsn['name'], type='DATASET', account='root') delete_dids(dids=dsns, account='root')
def test_update_dids(self): """ DATA IDENTIFIERS (CORE): Update file size and checksum""" tmp_scope = 'mock' dsn = 'dsn_%s' % generate_uuid() lfn = 'lfn.%s' % str(generate_uuid()) add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') files = [{'scope': tmp_scope, 'name': lfn, 'bytes': 724963570L, 'adler32': '0cc737eb', 'meta': {'guid': str(generate_uuid()), 'events': 100}}]
def test_run_transmogrifier(self): """ SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') subscription_name = uuid() dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subid = self.sub_client.add_subscription(name=subscription_name, account='root', filter={ 'scope': [ tmp_scope, ], 'pattern': 'dataset-.*', 'split_rule': True }, replication_rules=[{ 'lifetime': 86400, 'rse_expression': 'MOCK-POSIX|MOCK2|MOCK3', 'copies': 2, 'activity': 'Data Brokering' }], lifetime=None, retroactive=0, dry_run=0, comments='Ni ! Ni!', priority=1) run(threads=1, bulk=1000000, once=True) rules = [ rule for rule in self.did_client.list_did_rules(scope=tmp_scope, name=dsn) if str(rule['subscription_id']) == str(subid) ] assert_equal(len(rules), 2) set_new_dids([ { 'scope': tmp_scope, 'name': dsn }, ], 1) run(threads=1, bulk=1000000, once=True) rules = [ rule for rule in self.did_client.list_did_rules(scope=tmp_scope, name=dsn) if str(rule['subscription_id']) == str(subid) ] assert_equal(len(rules), 2)
def test_get_did_meta(self): """ DID Meta (Hardcoded): Get did meta """ did_name = 'mock_did_%s' % generate_uuid() dataset_meta = {'project': 'data12_8TeV'} add_did(scope=self.tmp_scope, name=did_name, type='DATASET', meta=dataset_meta, account=self.root) assert get_metadata(scope=self.tmp_scope, name=did_name)['project'] == 'data12_8TeV'
def test_get_rule(self): """ REPLICATION RULE (CORE): Test to get a previously created rule""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] assert(rule_id == get_rule(rule_id)['id'].replace('-', '').lower()) assert_raises(RuleNotFound, get_rule, uuid())
def test_add_rule_with_r2d2_container_treating_and_duplicate_rule(self): """ JUDGE INJECTOR (CORE): Add a replication rule with an r2d2 container treatment and duplicate rule""" scope = 'mock' container = 'asdf.r2d2_request.2016-04-01-15-00-00.ads.' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe') datasets = [] for i in range(3): files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) datasets.append(dataset) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') attach_dids(scope, container, [{ 'scope': scope, 'name': dataset }], 'jdoe') add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=False) rule_id = add_rule(dids=[{ 'scope': scope, 'name': container }], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=900, locked=False, subscription_id=None, ask_approval=True)[0] approve_rule(rule_id, approver='root') assert (get_rule(rule_id)['state'] == RuleState.INJECT) rule_injector(once=True) # Check if there is a rule for each file with assert_raises(RuleNotFound): get_rule(rule_id) for dataset in datasets: assert (len( [r for r in list_rules({ 'scope': scope, 'name': dataset })]) > 0)
def test_dataset_lock(self): """ DATASETLOCK (CLIENT): Get a datasetlock for a specific dataset""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=True, subscription_id=None)[0] rule_ids = [lock['rule_id'] for lock in self.lock_client.get_dataset_locks(scope=scope, name=dataset)] assert_in(rule_id_1, rule_ids)
def test_abacus_collection_replica_cleanup(self): """ ABACUS (COLLECTION REPLICA): Test if the cleanup procedure works correctly. """ collection_replica.run(once=True) db_session = session.get_session() rse1 = rse_name_generator() rse_id1 = add_rse(rse1, **self.vo) rse2 = rse_name_generator() rse_id2 = add_rse(rse2, **self.vo) scope = InternalScope('mock', **self.vo) dataset = 'dataset_%s' % generate_uuid() jdoe = InternalAccount('jdoe', **self.vo) add_did(scope, dataset, DIDType.DATASET, jdoe) models.CollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, state=ReplicaState.AVAILABLE, bytes=1).save(session=db_session, flush=False) models.CollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, state=ReplicaState.AVAILABLE, bytes=1).save(session=db_session, flush=False) models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, did_type=DIDType.DATASET).save( session=db_session, flush=False) models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, did_type=DIDType.DATASET).save( session=db_session, flush=False) models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, did_type=DIDType.DATASET).save( session=db_session, flush=False) models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, did_type=DIDType.DATASET).save( session=db_session, flush=False) models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=None, did_type=DIDType.DATASET).save( session=db_session, flush=False) db_session.commit() assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3 self.did_client.set_metadata(scope.external, dataset, 'lifetime', -1)
def test_delete_rule(self): """ REPLICATION RULE (CLIENT): Delete a replication rule """ scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] ret = self.rule_client.delete_replication_rule(rule_id=rule_id) assert(ret is True) assert_raises(RuleNotFound, self.rule_client.delete_replication_rule, rule_id)
def test_get_rule_by_account(self): """ ACCOUNT (CLIENT): Get Replication Rule by account """ scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') ret = self.rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE') get = self.account_client.list_account_rules('jdoe') rules = [rule['id'] for rule in get] assert_in(ret[0], rules)
def test_locked_rule(self): """ REPLICATION RULE (CLIENT): Delete a locked replication rule""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=True, subscription_id=None)[0] assert_raises(AccessDenied, delete_rule, rule_id_1) self.rule_client.update_replication_rule(rule_id=rule_id_1, options={'locked': False}) delete_rule(rule_id=rule_id_1)
def test_add_rule_duplicate(self): """ REPLICATION RULE (CORE): Add a replication rule duplicate""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) # Add a second rule and check if the right locks are created assert_raises(DuplicateRule, add_rule, dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)
def test_add_did_meta(self): """ DID Meta (Hardcoded): Add did meta """ did_name = 'mock_did_%s' % generate_uuid() add_did(scope=self.tmp_scope, name=did_name, type='DATASET', account=self.root) set_metadata(scope=self.tmp_scope, name=did_name, key='project', value='data12_8TeV') assert get_metadata(scope=self.tmp_scope, name=did_name)['project'] == 'data12_8TeV'
def test_judge_expire_rule(self): """ JUDGE CLEANER: Test the judge when deleting expired rules""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) attach_dids(scope, dataset, files, self.jdoe) add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=-3, locked=False, subscription_id=None)[0] add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=3, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] rule_cleaner(once=True) for file in files: rse_locks = get_replica_locks(scope=file['scope'], name=file['name']) assert (len(rse_locks) == 5)
def test_add_did_to_followed(self): """ DATA IDENTIFIERS (CORE): Mark a did as followed """ tmp_scope = InternalScope('mock') dsn = 'dsn_%s' % generate_uuid() root = InternalAccount('root') add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root) add_did_to_followed(scope=tmp_scope, name=dsn, account=root) users = get_users_following_did(scope=tmp_scope, name=dsn) rows = 0 for user in users: rows += 1 assert_equal(rows, 1)
def test_delete_rule(self): """ REPLICATION RULE (CORE): Test to delete a previously created rule""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] delete_rule(rule_id) for file in files: rse_locks = get_replica_locks(scope=file['scope'], name=file['name']) assert(len(rse_locks) == 0) assert_raises(RuleNotFound, delete_rule, uuid())
def test_rule_add_fails_account_limit(self): """ REPLICATION RULE (CORE): Test if a rule fails correctly when account limit conflict""" scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') set_account_limit(account='jdoe', rse_id=self.rse1_id, bytes=5) assert_raises(InsufficientAccountLimit, add_rule, dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) delete_account_limit(account='jdoe', rse_id=self.rse1_id)
def test_get_did_with_dynamic(self): """ DATA IDENTIFIERS (CORE): Get did with dynamic resolve of size""" tmp_scope = 'mock' tmp_dsn1 = 'dsn_%s' % generate_uuid() tmp_dsn2 = 'dsn_%s' % generate_uuid() tmp_dsn3 = 'dsn_%s' % generate_uuid() tmp_dsn4 = 'dsn_%s' % generate_uuid() add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root') add_replica(rse='MOCK', scope=tmp_scope, name=tmp_dsn2, bytes=10, account='root') add_replica(rse='MOCK', scope=tmp_scope, name=tmp_dsn3, bytes=10, account='root') attach_dids(scope=tmp_scope, name=tmp_dsn1, dids=[{ 'scope': tmp_scope, 'name': tmp_dsn2 }, { 'scope': tmp_scope, 'name': tmp_dsn3 }], account='root') add_did(scope=tmp_scope, name=tmp_dsn4, type=DIDType.CONTAINER, account='root') attach_dids(scope=tmp_scope, name=tmp_dsn4, dids=[{ 'scope': tmp_scope, 'name': tmp_dsn1 }], account='root') assert_equal( get_did(scope=tmp_scope, name=tmp_dsn1, dynamic=True)['bytes'], 20) assert_equal( get_did(scope=tmp_scope, name=tmp_dsn4, dynamic=True)['bytes'], 20)
def test_get_users_following_did(self): """ DATA IDENTIFIERS (CORE): Get the list of users following a did """ tmp_scope = InternalScope('mock', **self.vo) dsn = 'dsn_%s' % generate_uuid() root = InternalAccount('root', **self.vo) add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account=root) add_did_to_followed(scope=tmp_scope, name=dsn, account=root) users = get_users_following_did(scope=tmp_scope, name=dsn) rows = 0 for user in users: rows += 1 assert rows == 1
def test_add_rule_with_ignore_availability(self): """ REPLICATION RULE (CORE): Add a replication rule with ignore_availability setting""" rse = rse_name_generator() add_rse(rse) update_rse(rse, {'availability_write': False}) scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') with assert_raises(InvalidRSEExpression): add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0] add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True)[0]
def test_add_rule_with_purge(self): """ REPLICATION RULE (CORE): Add a replication rule with purge setting""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, purge_replicas=True)[0] delete_rule(rule_id) # Check if the Locks are created properly for file in files: replica = get_replica(rse=self.rse4, scope=file['scope'], name=file['name']) assert(replica['tombstone'] == OBSOLETE)
def test_add_rule_dataset_none_with_weights(self): """ REPLICATION RULE (CORE): Add a replication rule on a dataset, NONE Grouping, WEIGHTS""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight="fakeweight", lifetime=None, locked=False, subscription_id=None) # Check if the Locks are created properly t1 = set([self.rse1_id, self.rse3_id, self.rse5_id]) for file in files: rse_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]) assert(len(t1.intersection(rse_locks)) == 2) assert_in(self.rse1_id, rse_locks)
def test_change_rule_lifetime(self): """ REPLICATION RULE (CLIENT): Change rule lifetime""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight='fakeweight', lifetime=150, locked=True, subscription_id=None)[0] get = self.rule_client.get_replication_rule(rule_id_1) self.rule_client.update_replication_rule(rule_id_1, options={'lifetime': 10000}) get2 = self.rule_client.get_replication_rule(rule_id_1) assert(get['expires_at'] != get2['expires_at'])
def test_touch_dids(self): """ DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp""" tmp_scope = 'mock' tmp_dsn1 = 'dsn_%s' % generate_uuid() tmp_dsn2 = 'dsn_%s' % generate_uuid() add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root') add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root') now = datetime.utcnow() now -= timedelta(microseconds=now.microsecond) assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn1)) assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2)) touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET, 'accessed_at': now}]) assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1)) assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
def test_list_rules_states(self): """ SUBSCRIPTION (REST): Test listing of rule states for subscription """ tmp_scope = 'mock_' + uuid()[:8] add_scope(tmp_scope, 'root') mw = [] site_a = 'RSE%s' % uuid().upper() site_b = 'RSE%s' % uuid().upper() add_rse(site_a) add_rse(site_b) # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, type=DIDType.DATASET, account='root') subscription_name = uuid() id = add_subscription(name=subscription_name, account='root', filter={'account': 'root'}, replication_rules=[(1, 'T1_DATADISK', False, True)], lifetime=100000, retroactive=0, dry_run=0, comments='We want a shrubbery') subscriptions = list_subscriptions(name=subscription_name, account='root') # workaround until add_subscription returns the id id = None for s in subscriptions: id = s['id'] # Add two rules add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_a, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account='root', copies=1, rse_expression=site_b, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=id) headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) assert_equal(r1.status, 200) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Auth-Token': str(token)} r2 = TestApp(subs_app.wsgifunc(*mw)).get('/%s/%s/Rules/States' % ('root', subscription_name), headers=headers2, expect_errors=True) for line in r2.body.split('\n'): print line rs = loads(line) if rs[1] == subscription_name: break assert_equal(rs[3], 2)
def test_list_rules_by_did(self): """ DID (CLIENT): List Replication Rules per DID """ scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] rule_id_2 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] ret = self.did_client.list_did_rules(scope=scope, name=dataset) ids = [rule['id'] for rule in ret] assert_in(rule_id_1, ids) assert_in(rule_id_2, ids)
def test_dataset_callback_no(self): """ REPLICATION RULE (CORE): Test dataset callback should not be sent""" scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') set_status(scope=scope, name=dataset, open=False) rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, notify='C')[0] successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse3_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse3_id, nowait=False) # Check if rule exists assert(False == check_dataset_ok_callback(scope, dataset, self.rse3, rule_id))
def test_add_rule_container_none(self): """ REPLICATION RULE (CORE): Add a replication rule on a container, NONE Grouping""" scope = 'mock' container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe') all_files = [] for i in xrange(3): files = create_files(3, scope, self.rse1) all_files.extend(files) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe') add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.T2, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) for file in all_files: rse_locks = set([lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])]) assert_in(self.rse4_id, rse_locks) assert_not_in(self.rse5_id, rse_locks)
def test_rse_counter_unavailable_replicas(self): """ REPLICATION RULE (CORE): Test if creating UNAVAILABLE replicas updates the RSE Counter correctly""" rse_update(once=True) rse_counter_before = get_rse_counter(self.rse3_id) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Check if the rse has been updated correctly rse_update(once=True) rse_counter_after = get_rse_counter(self.rse3_id) assert(rse_counter_before['bytes'] + 3*100 == rse_counter_after['bytes']) assert(rse_counter_before['files'] + 3 == rse_counter_after['files'])
def test_delete_rule_and_cancel_transfers(self): """ REPLICATION RULE (CORE): Test to delete a previously created rule and do not cancel overlapping transfers""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') rule_id_1 = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=3, rse_expression=self.T1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0] delete_rule(rule_id_1) for file in files: rse_locks = get_replica_locks(scope=file['scope'], name=file['name']) assert(len(rse_locks) == 5) # TODO Need to check transfer queue here, this is actually not the check of this test case assert_raises(RuleNotFound, delete_rule, uuid())
def test_account_counter_rule_create(self): """ REPLICATION RULE (CORE): Test if the account counter is updated correctly when new rule is created""" account_update(once=True) account_counter_before = get_account_counter(self.rse1_id, 'jdoe') scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) # Check if the counter has been updated correctly account_update(once=True) account_counter_after = get_account_counter(self.rse1_id, 'jdoe') assert(account_counter_before['bytes'] + 3*100 == account_counter_after['bytes']) assert(account_counter_before['files'] + 3 == account_counter_after['files'])
def test_add_rules_datasets_none(self): """ REPLICATION RULE (CORE): Add replication rules to multiple datasets, NONE Grouping""" scope = 'mock' files1 = create_files(3, scope, self.rse4) dataset1 = 'dataset_' + str(uuid()) add_did(scope, dataset1, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset1, files1, 'jdoe') files2 = create_files(3, scope, self.rse4) dataset2 = 'dataset_' + str(uuid()) add_did(scope, dataset2, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset2, files2, 'jdoe') # Add the rules to both DS add_rules(dids=[{'scope': scope, 'name': dataset1}, {'scope': scope, 'name': dataset2}], rules=[{'account': 'jdoe', 'copies': 1, 'rse_expression': self.T1, 'grouping': 'NONE', 'weight': None, 'lifetime': None, 'locked': False, 'subscription_id': None}, {'account': 'root', 'copies': 1, 'rse_expression': self.T1, 'grouping': 'NONE', 'weight': 'fakeweight', 'lifetime': None, 'locked': False, 'subscription_id': None}]) # Check if the Locks are created properly for file in files1: rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])] assert(rse_locks[0] == rse_locks[1]) for file in files2: rse_locks = [lock['rse_id'] for lock in get_replica_locks(scope=file['scope'], name=file['name'])] assert(rse_locks[0] == rse_locks[1])