def test_account_counter_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS""" re_evaluator(once=True) account_update(once=True) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) account_update(once=True) account_counter_before = get_counter(self.rse1_id, 'jdoe') detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) account_update(once=True) account_counter_after = get_counter(self.rse1_id, 'jdoe') assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes']) assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_judge_add_files_to_dataset(self): """ JUDGE EVALUATOR: Test the judge when adding files to dataset""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) # Add a first rule to the DS add_rule(dids=[{ 'scope': scope, 'name': dataset }], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) attach_dids(scope, dataset, files, self.jdoe) re_evaluator(once=True) files = create_files(3, scope, self.rse1_id) attach_dids(scope, dataset, files, self.jdoe) # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert (len( get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
def test_judge_evaluate_detach_datasetlock(self): """ JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container""" re_evaluator(once=True) scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), 'jdoe') attach_dids(scope, container, [{'scope': scope, 'name': dataset}], 'jdoe') # Add a rule to the Container add_rule(dids=[{'scope': scope, 'name': container}], account='jdoe', copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) # Check if the datasetlock is there locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) > 0) detach_dids(scope, container, [{'scope': scope, 'name': dataset}]) # Fake judge re_evaluator(once=True) locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) == 0)
def test_judge_add_files_to_dataset_rule_on_container(self): """ JUDGE EVALUATOR: Test the judge when attaching file to dataset with rule on two levels of containers""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) parent_container = 'dataset_' + str(uuid()) add_did(scope, parent_container, DIDType.from_sym('CONTAINER'), self.jdoe) attach_dids(scope, parent_container, [{'scope': scope, 'name': dataset}], self.jdoe) parent_parent_container = 'dataset_' + str(uuid()) add_did(scope, parent_parent_container, DIDType.from_sym('CONTAINER'), self.jdoe) attach_dids(scope, parent_parent_container, [{'scope': scope, 'name': parent_container}], self.jdoe) # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': parent_parent_container}], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) # create more files and attach them more_files = create_files(3, scope, self.rse1_id) attach_dids(scope, dataset, more_files, self.jdoe) re_evaluator(once=True) # Check if the Locks are created properly for file in more_files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
def test_judge_evaluate_detach_datasetlock(self): """ JUDGE EVALUATOR: Test if the a datasetlock is detached correctly when removing a dataset from a container""" re_evaluator(once=True) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), self.jdoe) attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe) # Add a rule to the Container add_rule(dids=[{'scope': scope, 'name': container}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) # Check if the datasetlock is there locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) > 0) detach_dids(scope, container, [{'scope': scope, 'name': dataset}]) # Fake judge re_evaluator(once=True) locks = [ds_lock for ds_lock in get_dataset_locks(scope=scope, name=dataset)] assert(len(locks) == 0)
def test_account_counter_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the account counter is updated correctly when a file is removed from a DS""" re_evaluator(once=True) account_update(once=True) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) account_update(once=True) account_counter_before = get_usage(self.rse1_id, self.jdoe) detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) account_update(once=True) account_counter_after = get_usage(self.rse1_id, self.jdoe) assert(account_counter_before['bytes'] - 100 == account_counter_after['bytes']) assert(account_counter_before['files'] - 1 == account_counter_after['files'])
def test_judge_add_files_to_dataset_with_2_rules(self): """ JUDGE EVALUATOR: Test the judge when adding files to dataset with 2 rules""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') # Add a first rule to the DS add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='jdoe', copies=1, rse_expression=self.rse5, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) add_rule(dids=[{ 'scope': scope, 'name': dataset }], account='root', copies=1, rse_expression=self.rse5, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) attach_dids(scope, dataset, files, 'jdoe') re_evaluator(once=True) files = create_files(3, scope, self.rse1) attach_dids(scope, dataset, files, 'jdoe') # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert (len( get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
def test_judge_add_dataset_to_container(self): """ JUDGE EVALUATOR: Test the judge when adding dataset to container""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') attach_dids(scope, dataset, files, 'jdoe') parent_container = 'dataset_' + str(uuid()) add_did(scope, parent_container, DIDType.from_sym('CONTAINER'), 'jdoe') # Add a first rule to the DS add_rule(dids=[{ 'scope': scope, 'name': parent_container }], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) attach_dids(scope, parent_container, [{ 'scope': scope, 'name': dataset }], 'jdoe') # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert (len( get_replica_locks(scope=file['scope'], name=file['name'])) == 2) # Check if the DatasetLocks are created properly dataset_locks = [ lock for lock in get_dataset_locks(scope=scope, name=dataset) ] assert (len(dataset_locks) == 2)
def test_judge_add_files_to_dataset(self): """ JUDGE EVALUATOR: Test the judge when adding files to dataset""" scope = 'mock' files = create_files(3, scope, self.rse1) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') # Add a first rule to the DS add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None) attach_dids(scope, dataset, files, 'jdoe') re_evaluator(once=True) files = create_files(3, scope, self.rse1) attach_dids(scope, dataset, files, 'jdoe') # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
def test_dataset_callback_with_evaluator(self): """ REPLICATION RULE (CORE): Test dataset callback with judge evaluator""" scope = 'mock' files = create_files(3, scope, self.rse1, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, notify='C')[0] assert(False == check_dataset_ok_callback(scope, dataset, self.rse3, rule_id)) attach_dids(scope, dataset, files, 'jdoe') set_status(scope=scope, name=dataset, open=False) assert(False == check_dataset_ok_callback(scope, dataset, self.rse3, rule_id)) re_evaluator(once=True) successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse3_id, nowait=False) successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse3_id, nowait=False) successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse3_id, nowait=False) assert(True == check_dataset_ok_callback(scope, dataset, self.rse3, rule_id))
def test_repair_a_rule_with_missing_locks(self): """ JUDGE EVALUATOR: Test the judge when a rule gets STUCK from re_evaluating and there are missing locks""" scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse4_id) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.DATASET, self.jdoe) # Add a first rule to the DS rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] attach_dids(scope, dataset, files, self.jdoe) # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) # Add more files to the DID files2 = create_files(3, scope, self.rse4_id) attach_dids(scope, dataset, files2, self.jdoe) # Mark the rule STUCK to fake that the re-evaluation failed session = get_session() rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one() rule.state = RuleState.STUCK session.commit() rule_repairer(once=True) for file in files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) for file in files2: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) assert(len(set([lock.rse_id for lock in get_replica_locks(scope=files[0]['scope'], name=files[0]['name'])]).intersection(set([lock.rse_id for lock in get_replica_locks(scope=file['scope'], name=file['name'])]))) == 2) assert(12 == get_rule(rule_id)['locks_replicating_cnt'])
def test_repair_a_rule_with_missing_locks(self): """ JUDGE EVALUATOR: Test the judge when a rule gets STUCK from re_evaluating and there are missing locks""" scope = 'mock' files = create_files(3, scope, self.rse4) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe') # Add a first rule to the DS rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] attach_dids(scope, dataset, files, 'jdoe') # Fake judge re_evaluator(once=True) # Check if the Locks are created properly for file in files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) # Add more files to the DID files2 = create_files(3, scope, self.rse4) attach_dids(scope, dataset, files2, 'jdoe') # Mark the rule STUCK to fake that the re-evaluation failed session = get_session() rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one() rule.state = RuleState.STUCK session.commit() rule_repairer(once=True) for file in files: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) for file in files2: assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2) assert(len(set([lock.rse_id for lock in get_replica_locks(scope=files[0]['scope'], name=files[0]['name'])]).intersection(set([lock.rse_id for lock in get_replica_locks(scope=file['scope'], name=file['name'])]))) == 2) assert(12 == get_rule(rule_id)['locks_replicating_cnt'])
def test_judge_evaluate_detach(self): """ JUDGE EVALUATOR: Test if the detach is done correctly""" re_evaluator(once=True) scope = InternalScope('mock', **self.vo) container = 'container_' + str(uuid()) add_did(scope, container, DIDType.from_sym('CONTAINER'), self.jdoe) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe) scope = InternalScope('mock', **self.vo) files = create_files(3, scope, self.rse1_id, bytes=100) dataset = 'dataset_' + str(uuid()) add_did(scope, dataset, DIDType.from_sym('DATASET'), self.jdoe) attach_dids(scope, dataset, files, self.jdoe) attach_dids(scope, container, [{'scope': scope, 'name': dataset}], self.jdoe) # Add a first rule to the Container rule_id = add_rule(dids=[{'scope': scope, 'name': container}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)[0] # Fake judge re_evaluator(once=True) assert(9 == get_rule(rule_id)['locks_ok_cnt']) detach_dids(scope, dataset, [files[0]]) # Fake judge re_evaluator(once=True) assert(8 == get_rule(rule_id)['locks_ok_cnt'])
def test_tpc(containerized_rses, root_account, test_scope, did_factory, rse_client, rule_client, artifact): if len(containerized_rses) < 2: pytest.skip( "TPC tests need at least 2 containerized rse's for execution}") rse1_name, rse1_id = containerized_rses[0] rse2_name, rse2_id = containerized_rses[1] base_file_name = generate_uuid() test_file = did_factory.upload_test_file(rse1_name, name=base_file_name + '.000', return_full_item=True) test_file_did_str = '%s:%s' % (test_file['did_scope'], test_file['did_name']) test_file_did = {'scope': test_scope, 'name': test_file['did_name']} test_file_name_hash = hashlib.md5( test_file_did_str.encode('utf-8')).hexdigest() test_file_expected_pfn = '%s/%s/%s/%s' % ( test_file_did['scope'], test_file_name_hash[0:2], test_file_name_hash[2:4], test_file_did['name']) rse1_hostname = rse_client.get_protocols(rse1_name)[0]['hostname'] rse2_hostname = rse_client.get_protocols(rse2_name)[0]['hostname'] rule_id = add_rule(dids=[test_file_did], account=root_account, copies=1, rse_expression=rse2_name, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) rule = rule_client.get_replication_rule(rule_id[0]) re_evaluator(once=True) assert rule['locks_ok_cnt'] == 0 assert rule['locks_replicating_cnt'] == 1 [[_, [transfer_path]] ] = next_transfers_to_submit(rses=[rse1_id, rse2_id]).items() assert transfer_path[0].rws.rule_id == rule_id[0] src_url = transfer_path[0].legacy_sources[0][1] dest_url = transfer_path[0].dest_url check_url(src_url, rse1_hostname, test_file_expected_pfn) check_url(dest_url, rse2_hostname, test_file_expected_pfn) # Run Submitter submitter.submitter(once=True) # Get FTS transfer job id request = get_request_by_did(rse_id=rse2_id, **test_file_did) fts_transfer_id = request['external_id'] # Check FTS transfer job assert fts_transfer_id is not None # Wait for the FTS transfer to finish fts_transfer_status = None for _ in range(MAX_POLL_WAIT_SECONDS): fts_transfer_status = poll_fts_transfer_status(fts_transfer_id) if fts_transfer_status not in ['SUBMITTED', 'ACTIVE']: break time.sleep(1) assert fts_transfer_status == 'FINISHED' poller.run(once=True, older_than=0) finisher.run(once=True) rule = rule_client.get_replication_rule(rule_id[0]) assert rule['locks_ok_cnt'] == 1 assert rule['locks_replicating_cnt'] == 0 if artifact is not None: date = datetime.date.today().strftime("%Y-%m-%d") with open(artifact, 'w') as artifact_file: artifact_file.write( f"/var/log/fts3/{date}/{rse1_name.lower()}__{rse2_name.lower()}/*__{fts_transfer_id}" )
def test_bb8_full_workflow(vo, root_account, jdoe_account, rse_factory, mock_scope, did_factory): """BB8: Test the rebalance rule method""" config_core.set(section='bb8', option='allowed_accounts', value='jdoe') tot_rses = 4 rses = [rse_factory.make_posix_rse() for _ in range(tot_rses)] rse1, rse1_id = rses[0] rse2, rse2_id = rses[1] rse3, rse3_id = rses[2] rse4, rse4_id = rses[3] # Add Tags # RSE 1 and 2 nmatch expression T1=true # RSE 3 and 4 nmatch expression T2=true T1 = tag_generator() T2 = tag_generator() add_rse_attribute(rse1_id, T1, True) add_rse_attribute(rse2_id, T1, True) add_rse_attribute(rse3_id, T2, True) add_rse_attribute(rse4_id, T2, True) # Add fake weights add_rse_attribute(rse1_id, "fakeweight", 10) add_rse_attribute(rse2_id, "fakeweight", 0) add_rse_attribute(rse3_id, "fakeweight", 0) add_rse_attribute(rse4_id, "fakeweight", 0) add_rse_attribute(rse1_id, "freespace", 1) add_rse_attribute(rse2_id, "freespace", 1) add_rse_attribute(rse3_id, "freespace", 1) add_rse_attribute(rse4_id, "freespace", 1) # Add quota set_local_account_limit(jdoe_account, rse1_id, -1) set_local_account_limit(jdoe_account, rse2_id, -1) set_local_account_limit(jdoe_account, rse3_id, -1) set_local_account_limit(jdoe_account, rse4_id, -1) set_local_account_limit(root_account, rse1_id, -1) set_local_account_limit(root_account, rse2_id, -1) set_local_account_limit(root_account, rse3_id, -1) set_local_account_limit(root_account, rse4_id, -1) # Invalid the cache because the result of parse_expression is cached REGION.invalidate() tot_datasets = 4 # Create a list of datasets datasets = [did_factory.make_dataset() for _ in range(tot_datasets)] dsn = [dataset['name'] for dataset in datasets] rules = list() base_unit = 100000000000 nb_files1 = 7 nb_files2 = 5 nb_files3 = 3 nb_files4 = 2 file_size = 1 * base_unit rule_to_rebalance = None # Add one secondary file files = create_files(1, mock_scope, rse1_id, bytes_=1) add_rule(dids=[{ 'scope': mock_scope, 'name': files[0]['name'] }], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0] for cnt in range(3, tot_rses): add_replicas(rses[cnt][1], files, jdoe_account) add_rule(dids=[{ 'scope': mock_scope, 'name': files[0]['name'] }], account=jdoe_account, copies=1, rse_expression=rses[cnt][0], grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0] rule_cleaner(once=True) # Create dataset 1 of 800 GB and create a rule on RSE 1 and RSE 3 files = create_files(nb_files1, mock_scope, rse1_id, bytes_=file_size) attach_dids(mock_scope, dsn[0], files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[0] }], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rules.append(rule_id) add_replicas(rse3_id, files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[0] }], account=jdoe_account, copies=1, rse_expression=rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rules.append(rule_id) # Create dataset 2 of 500 GB and create a rule on RSE 1 and RSE 2 files = create_files(nb_files2, mock_scope, rse1_id, bytes_=file_size) attach_dids(mock_scope, dsn[1], files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[1] }], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rules.append(rule_id) add_replicas(rse2_id, files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[1] }], account=jdoe_account, copies=1, rse_expression=rse2, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rules.append(rule_id) # Create dataset 3 of 300 GB and create a rule on RSE 1. The copy on RSE 3 is secondary files = create_files(nb_files3, mock_scope, rse1_id, bytes_=file_size) attach_dids(mock_scope, dsn[2], files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[2] }], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rule_to_rebalance = rule_id rules.append(rule_id) add_replicas(rse3_id, files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[2] }], account=jdoe_account, copies=1, rse_expression=rse3, grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0] rule_cleaner(once=True) try: rule = get_rule(rule_id) except: pytest.raises(RuleNotFound, get_rule, rule_id) # Create dataset 4 of 200 GB and create a rule on RSE 3. The copy on RSE 2 is secondary files = create_files(nb_files4, mock_scope, rse3_id, bytes_=file_size) attach_dids(mock_scope, dsn[3], files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[3] }], account=jdoe_account, copies=1, rse_expression=rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0] rules.append(rule_id) add_replicas(rse2_id, files, jdoe_account) rule_id = add_rule(dids=[{ 'scope': mock_scope, 'name': dsn[3] }], account=jdoe_account, copies=1, rse_expression=rse2, grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0] rule_cleaner(once=True) try: rule = get_rule(rule_id) except: pytest.raises(RuleNotFound, get_rule, rule_id) for dataset in dsn: set_status(mock_scope, dataset, open=False) for rse in rses: fill_rse_expired(rse[1]) set_rse_usage(rse_id=rse[1], source='min_free_space', used=2 * base_unit, free=2 * base_unit, session=None) set_rse_usage(rse_id=rse[1], source='storage', used=15 * base_unit, free=2 * base_unit, session=None) set_rse_usage(rse_id=rse2_id, source='min_free_space', used=1 * base_unit, free=1 * base_unit, session=None) set_rse_usage(rse_id=rse2_id, source='storage', used=6 * base_unit, free=5 * base_unit, session=None) run_abacus(once=True, threads=1, fill_history_table=False, sleep_time=10) # Summary : # RSE 1 : 1500 GB primary + 1 B secondary tot_space = [ src for src in get_rse_usage(rse1_id) if src['source'] == 'rucio' ][0] expired = [ src for src in get_rse_usage(rse1_id) if src['source'] == 'expired' ][0] assert tot_space['used'] == (nb_files1 + nb_files2 + nb_files3) * file_size + 1 assert expired['used'] == 1 # RSE 2 : 500 GB primary + 100 GB secondary tot_space = [ src for src in get_rse_usage(rse2_id) if src['source'] == 'rucio' ][0] expired = [ src for src in get_rse_usage(rse2_id) if src['source'] == 'expired' ][0] assert tot_space['used'] == (nb_files2 + nb_files4) * file_size assert expired['used'] == nb_files4 * file_size # Total primary on T1=true : 2000 GB # Total secondary on T1=true : 200 GB # Ratio secondary / primary = 10 % # Ratio on RSE 1 : 0 % # Ratio on RSE 2 : 40 % # Now run BB8 re_evaluator(once=True, sleep_time=30, did_limit=100) bb8_run(once=True, rse_expression='%s=true' % str(T1), move_subscriptions=False, use_dump=False, sleep_time=300, threads=1, dry_run=False) for rule_id in rules: rule = get_rule(rule_id) if rule_id != rule_to_rebalance: assert (rule['child_rule_id'] is None) else: assert (rule['child_rule_id'] is not None) assert ( rule['expires_at'] <= datetime.utcnow() + timedelta(seconds=1) ) # timedelta needed to prevent failure due to rounding effects child_rule_id = rule['child_rule_id'] child_rule = get_rule(child_rule_id) assert (child_rule['rse_expression'] == rse2) # For teardown, delete child rule update_rule(child_rule_id, {'lifetime': -86400}) rule_cleaner(once=True) for dataset in dsn: set_metadata(mock_scope, dataset, 'lifetime', -86400) undertaker.run(once=True)
def test_tpc(containerized_rses, root_account, test_scope, did_factory, rse_client, rule_client, artifact): if len(containerized_rses) < 2: pytest.skip( "TPC tests need at least 2 containerized rse's for execution}") rse1_name, rse1_id = containerized_rses[0] rse2_name, rse2_id = containerized_rses[1] base_file_name = generate_uuid() test_file = did_factory.upload_test_file(rse1_name, name=base_file_name + '.000', return_full_item=True) test_file_did_str = '%s:%s' % (test_file['did_scope'], test_file['did_name']) test_file_did = {'scope': test_scope, 'name': test_file['did_name']} test_file_name_hash = hashlib.md5( test_file_did_str.encode('utf-8')).hexdigest() test_file_expected_pfn = '%s/%s/%s/%s' % ( test_file_did['scope'], test_file_name_hash[0:2], test_file_name_hash[2:4], test_file_did['name']) rse1_hostname = rse_client.get_protocols(rse1_name)[0]['hostname'] rse2_hostname = rse_client.get_protocols(rse2_name)[0]['hostname'] rule_id = add_rule(dids=[test_file_did], account=root_account, copies=1, rse_expression=rse2_name, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None) rule = rule_client.get_replication_rule(rule_id[0]) re_evaluator(once=True) assert rule['locks_ok_cnt'] == 0 assert rule['locks_replicating_cnt'] == 1 transfer_requestss = get_transfer_requests_and_source_replicas( rses=[rse1_id, rse2_id]) for transfer_requests in transfer_requestss: for transfer_request in transfer_requests: if transfer_requests[transfer_request]['rule_id'] == rule_id[0]: src_url = transfer_requests[transfer_request]['sources'][0][1] dest_url = transfer_requests[transfer_request]['dest_urls'][0] check_url(src_url, rse1_hostname, test_file_expected_pfn) check_url(dest_url, rse2_hostname, test_file_expected_pfn) # Run Submitter submitter.run(once=True) # Get FTS transfer job info fts_transfer_id, fts_transfer_status = list_fts_transfer() # Check FTS transfer job assert fts_transfer_id is not None assert fts_transfer_status in ['SUBMITTED', 'ACTIVE'] fts_transfer_status = poll_fts_transfer_status(fts_transfer_id) assert fts_transfer_status == 'FINISHED' poller.run(once=True, older_than=0) finisher.run(once=True) rule = rule_client.get_replication_rule(rule_id[0]) assert rule['locks_ok_cnt'] == 1 assert rule['locks_replicating_cnt'] == 0 if artifact is not None: date = datetime.date.today().strftime("%Y-%m-%d") with open(artifact, 'w') as artifact_file: artifact_file.write( f"/var/log/fts3/{date}/{rse1_name.lower()}__{rse2_name.lower()}/*__{fts_transfer_id}" )