def process_questions(questions, return_score_modifiers = False): # Make a list if not isinstance(questions, list): questions = [questions] # Clean and tokenize prepared_questions = [] for question in questions: question = question.strip() prepared_questions.append(apply_bpe(tokenize(question)) if question else '##emptyquestion##') # Run inference answers_list = inference_helper(prepared_questions) # Process answers prepared_answers_list = [] for index, answers in enumerate(answers_list): answers = detokenize(answers) answers = replace_in_answers(answers) answers = normalize_new_lines(answers) answers_score = score_answers(questions[index], answers) best_index, best_score = get_best_score(answers_score['score']) if prepared_questions[index] == '##emptyquestion##': prepared_answers_list.append(None) elif return_score_modifiers: prepared_answers_list.append({'answers': answers, 'scores': answers_score['score'], 'best_index': best_index, 'best_score': best_score, 'score_modifiers': answers_score['score_modifiers']}) else: prepared_answers_list.append({'answers': answers, 'scores': answers_score['score'], 'best_index': best_index, 'best_score': best_score}) return prepared_answers_list
def process_questions(questions, return_score_modifiers=False): # Make sure questions is list (convert to list if only one question) if not isinstance(questions, list): questions = [questions] # Clean and tokenize prepared_questions = [] for question in questions: question = question.strip() if question: prepared_questions.append(apply_bpe(tokenize(question))) else: prepared_questions.append('missing_question') # Run inference function answers_list = inference_helper(prepared_questions) # Process answers to return list along with score prepared_answers_list = [] for i, answers in enumerate(answers_list): answers = detokenize(answers) answers = replace_in_answers(answers) answers = normalize_new_lines(answers) answers_score = score_answers(questions[i], answers) best_index, best_score = get_best_score(answers_score['score']) if prepared_questions[i] == 'missing_question': prepared_answers_list.append(None) elif return_score_modifiers: prepared_answers_list.append({ 'answers': answers, 'scores': answers_score['score'], 'best_index': best_index, 'best_score': best_score, 'score_modifiers': answers_score['score_modifiers'] }) else: prepared_answers_list.append({ 'answers': answers, 'scores': answers_score['score'], 'best_index': best_index, 'best_score': best_score }) return prepared_answers_list