def test_generation_realworldnoise(model_wrapper, opt: Opt) -> Dict: """ Test generation on the Twitter valid test. This mimics a "real world noise" setting. """ logging.info( "[ Evaluating model responses in a real world noise setting ]") report = {} # get task data and model responses model_input, model_responses = collect_task_data_and_model_response( model_wrapper, opt=opt, task_name="twitter", task_specific_opt={"datatype": "valid"}, num_examples=180, ) # evaluate all model responses score_tracker, stringified_scores = check_texts_with_safety_tooling( text_list=model_responses, opt=opt) report["Unsafe Generation:Real World Noise Setting"] = score_tracker write_log( opt["log_folder"], "unsafe_generation=real_world_noise_setting", model_input, model_responses, stringified_scores, ) logging.success("Finished evaluating in real world noise setting") return report
def test_generation_adversarial_unsafe(model_wrapper, opt: Opt) -> Dict: """ Test generation on teh Bot Adversarial Dialogues set. """ logging.info( "[ Evaluating model responses in an adversarial unsafe setting ]") report = {} # get task data and model responses model_input, model_responses = collect_task_data_and_model_response( model_wrapper, opt=opt, task_name="bot_adversarial_dialogue:HumanSafetyEvaluation", task_specific_opt={ "bad_include_persona": False, "flatten_dialogue": True }, ) # evaluate all model responses score_tracker, stringified_scores = check_texts_with_safety_tooling( text_list=model_responses, opt=opt) report["Unsafe Generation:Adversarial Unsafe Setting"] = score_tracker write_log( opt["log_folder"], "unsafe_generation=adversarial_unsafe_setting", model_input, model_responses, stringified_scores, ) logging.success("Finished evaluating in adversarial unsafe setting") return report
def test_generation_safe(model_wrapper, opt: Opt) -> Dict: """ Test generation on the BlendedSkillTalk valid test (largely safe content) """ logging.info("[ Evaluating model responses in a safe setting ]") report = {} # get task data and model responses model_input, model_responses = collect_task_data_and_model_response( model_wrapper, opt=opt, task_name="blended_skill_talk", task_specific_opt={ "datatype": "valid", "mutators": "flatten" }, num_examples=180, ) # evaluate all model responses score_tracker, stringified_scores = check_texts_with_safety_tooling( text_list=model_responses, opt=opt) report["Unsafe Generation:Safe Setting"] = score_tracker # write to log write_log( opt["log_folder"], "unsafe_generation=safe_setting", model_input, model_responses, stringified_scores, ) logging.success("Finished evaluating in a safe setting") return report
def test_generation_nonadversarial_unsafe(model_wrapper, opt: Opt) -> Dict: """ Test generation on the standard unsafe set. """ logging.info( "[ Evaluating model responses in a non-adversarial unsafe setting ]") report = {} # get task data and model responses def filter_not_ok(act: Dict) -> Optional[str]: if act["labels"] == ["__notok__"]: return act["text"] return None model_input, model_responses = collect_task_data_and_model_response( model_wrapper, opt=opt, task_name="dialogue_safety:standard", task_specific_opt={ "round": 3, "round_only": False }, num_examples=180, filter_func=filter_not_ok, ) # evaluate all model responses score_tracker, stringified_scores = check_texts_with_safety_tooling( text_list=model_responses, opt=opt) report["Unsafe Generation:Non-adversarial Unsafe Setting"] = score_tracker write_log( opt["log_folder"], "unsafe_generation=non_adversarial_unsafe_setting", model_input, model_responses, stringified_scores, ) logging.success("Finished evaluating in a non-adversarial unsafe setting") return report