def init(self, config):
assert (isinstance(config, YamlConfigFileHandler))
self.config = config
self.test_type = config.get_config('type', self.test_type)
self.server = config.get_config("server", self.server)
options = config.get_config(
"options", {
'suspend_on_error': self.suspend_on_error,
'question_interval': self.question_interval
})
self.suspend_on_error = options.get('suspend_on_error',
self.suspend_on_error)
self.question_interval = options.get('question_interval',
self.question_interval)
robots = []
if self.test_type == 'knowledge':
robots = config.get_config("usernames", None)
if not robots:
robots = config.get_config("username", None)
elif self.test_type == 'api':
robots = config.get_config('name', 'Unknown')
if isinstance(robots, basestring):
self.robots = [robots]
else:
self.robots = robots
self.questions = config.get_config("questions", self.questions)
self.answers = config.get_config("answers", self.answers)
configuration = config.get_config(
"output", {
'print_info': self.print_info,
'print_conversation': self.print_conversation,
'print_details': self.print_details,
'print_correct_answer': self.print_correct_answer,
'print_csv': self.print_csv
})
self.print_info = configuration.get('print_info', self.print_info)
self.print_conversation = configuration.get('print_conversation',
self.print_conversation)
self.print_details = configuration.get('print_details',
self.print_details)
self.print_correct_answer = configuration.get(
'print_correct_answer', self.print_correct_answer)
self.print_csv = configuration.get('print_csv', self.print_csv)
if self.questions and len(self.questions) > 0:
report_logger.info("grader ready, there are {0} questions".format(
len(self.questions)))
def main():
report_logger.info("QiwuGrader ver {0}".format(GRADER_VERSION))
# use dns cache
_set_dns_cache()
# test configuration file name
test_config_file_name = 'test.yml'
# number of sessions per hour
test_session = 1
# test session length (seconds)
test_length = 5
# Parse parameters from the command line
test_config_file_name_list = []
argv = sys.argv[1:]
for yml in sys.argv[1:]:
if yml.endswith('.yml'):
test_config_file_name_list.append(yml)
argv.remove(yml)
if len(argv) >= 1:
test_session = int(argv[0])
if len(argv) == 2:
test_length = int(argv[1])
# Test all configs
if not test_config_file_name_list:
test_config_file_name_list.append('test.yml')
# if test_session == 1: # single session grade
roll_log_file = init_log_file()
for test_config_file_name in test_config_file_name_list:
run(test_config_file_name,
test_session=test_session,
test_length=test_length)
if test_config_file_name != test_config_file_name_list[-1]:
report_logger.info('=====' * 20)
if test_session == 1:
roll_log_file()
def run(test_config_file_name, test_session, test_length):
test_config = YamlConfigFileHandler(test_config_file_name)
test_config_id = test_config.get_config('name',
test_config.get_config('id', ''))
if test_config_file_name.find(test_config_id) == -1:
report_logger.error(
"file name: {} is different from config id/name: {}".format(
test_config_file_name, test_config_id))
return
if test_session == -1: # forever stable test
while True:
grader = Grader()
grader.init(test_config)
grader.test()
elif test_session == 1: # single session grade
# init_log_file()
grader = Grader()
grader.init(test_config)
grader.test()
elif test_length == -1: # single session several round grade
for i in range(1, test_session):
grader = Grader()
grader.init(test_config)
grader.test()
elif test_session > 1: # multi session grade
# calculate thread spawn interval
spawn_interval = test_length / (test_session * 1.0)
# determine grader class
use_process = False
handler_count = test_session
session_per_handler = 1
Handler_Class = GraderThread
# use process to speed up grade
if test_session > 512 or spawn_interval < 0.5:
use_process = True
handler_count = multiprocessing.cpu_count()
session_per_handler = test_session // handler_count
Handler_Class = GraderProcess
# thread safe success counter
success_count = SharedCounter()
success_time_count = SharedCounter(val_type='d')
# process time counter
process_time = time.time()
# thread group
threads = []
# if not use_process and test_session <= 100:
# init_log_file()
report_logger.info(
"Testing {0} sessions in {1} seconds, interval: {2}, using class {3}"
.format(test_session, test_length, spawn_interval,
Handler_Class.__name__))
report_logger.info("Warming up ...")
warm_up_time = time.time()
# Spawn threads
sessions_to_distribute = test_session
while sessions_to_distribute > 0:
session_per_process = sessions_to_distribute > session_per_handler and session_per_handler or sessions_to_distribute
grader_handler = Handler_Class(success_count,
test_config,
success_time_count,
loop=session_per_process,
spawn_interval=spawn_interval *
handler_count)
grader_handler.init()
threads.append(grader_handler)
sessions_to_distribute -= session_per_process
report_logger.info(
"Warm up process finished in {0} seconds".format(time.time() -
warm_up_time))
launch_time = time.time()
# Start threads
for grader_handler in threads:
grader_handler.start()
# Wait for spawn interval
sleep(spawn_interval)
report_logger.info("{0} sessions started in {1}".format(
int(test_session),
time.time() - launch_time))
# Wait for all threads to finish
for grader_handler in threads:
grader_handler.join()
questions_count = success_count.value(
) * grader_handler.get_question_number()
report_logger.info(
"Result: {0} / {1} passed. Total time: {2}\nSuccess time: {3} Passed: {4} Success avg: {5}"
.format(success_count.value(), int(test_session),
time.time() - process_time, success_time_count.value(),
questions_count, success_time_count.value()))
def test_robot(self, name, questions, answers, sessions=None):
if sessions is None:
sessions = {}
if self.test_type == 'api':
test_service = SingleDialogueHandler(self.config)
else:
test_service = pMsgHandler(self.config, test_logger)
uid = self.uid_prefix + id_generator(20)
last_session = 1
test_service.robot_name = name
# items_op = getattr(questions, "iteritems", None)
# question_items = callable(items_op) and questions.iteritems() or questions.items()
grade = 0
total_time = 0
for i, question in questions.items():
if sessions.get(i, -1) != last_session:
uid = self.uid_prefix + id_generator(20)
last_session = sessions.get(i, -1)
process_time = time.time()
# turn question into a string and handle chat
if type(question) == int:
question_str = str(question)
else:
question_str = question
response = test_service.handle_chat(uid,
question_str,
login_wait=0)
data = None
response = to_str(response)
if response.find(AnswerTokenType.QUERY_ATTACH.value) != -1 \
or response.find(StringExtractor.DATA_ATTACH_SEPARATOR):
try:
data, response = self.handle_string(response)
except Exception:
test_logger.info("Failed to retrive data")
chat_key = None
if hasattr(test_service, 'tokens') and uid in test_service.tokens:
chat_key = test_service.tokens[uid]
process_time = time.time() - process_time
correct = True
if response.find(self.TIMEOUT_REPLY) == 0:
correct = False
if response == self.REQUEST_TIMEOUT:
correct = False
if response == self.REQUEST_ERROR_REPLY:
correct = False
answer_str = 'No answer found for question {0}'.format(i)
if correct and answers and i in answers and answers[i]:
response = to_str(response)
# if data:
# temp_res = response
# response = data
# data = temp_res
answer = answers[i]
correct = False
if isinstance(answer, dict):
if 'multi' in answer:
answer_str = json.dumps(
answer, ensure_ascii=False).encode('utf-8')
for j, item in enumerate(answer['multi']):
if response.find(item.encode('utf-8')) != -1:
correct = True
break
if 'regex' in answer:
answer_str = answer['regex'].encode('utf-8')
match_obj = re.match(answer_str, response)
if match_obj:
correct = True
else:
answer_str = answer.encode('utf-8')
if response.find(to_str(answer_str)) != -1:
correct = True
if correct:
grade += 1
total_time += process_time
if self.print_csv:
csv_string = str(last_session)
csv_string += ",Question {0}".format(i)
csv_string += "," + uid
csv_string += "," + to_str(question.encode('utf-8')).replace(
",", "，")
csv_string += "," + to_str(response).replace(",", "，")
csv_string += "," + to_str(answer_str).replace(",", "，")
csv_string += "," + (correct and "Passed" or "Wrong")
csv_string += "," + "{:.5f}".format(process_time)
if data:
csv_string += "," + str(len(data))
csv_string += "," + data
csv_logger.info(csv_string)
if self.print_info:
if self.print_conversation:
if self.print_details:
question_str = to_str(question_str.encode('utf-8'))
test_logger.info(
"Session {0} Question {1}: {2}".format(
last_session, i, question_str))
response = to_str(response)
if chat_key:
test_logger.info("Chatkey: {0})".format(chat_key))
test_logger.info("Response :" + response)
if self.print_correct_answer or not correct:
answer_str = to_str(answer_str)
test_logger.info("Answer: " + answer_str)
test_logger.info((correct and "Passed" or "Wrong") +
" for " + uid)
test_logger.info(
"Processed in {:.5f} seconds".format(process_time))
test_logger.info("=============")
else:
test_logger.info(
"Question {0}: {1} in {2} seconds".format(
i, correct and "Passed" or "Wrong",
process_time))
if not correct:
test_logger.info("Answer: " + to_str(response))
elif not correct:
test_logger.warning("Q {0} WA vc: {1}".format(
i, to_str(response)))
# print ("Q {0} Wrong answer: {1}".format(i, response))
if not correct and self.suspend_on_error:
break
if not correct and self.pause_on_error:
test_logger.info(
"Pause thread for 10 seconds due to incorrect response")
time.sleep(10)
if self.question_interval > 0.1:
time.sleep(self.question_interval)
# end of question loop
if self.print_csv:
csv_logger.info("{0},{1} / {2},{3},{4}".format(
name, grade, len(questions), total_time,
total_time / len(questions)))
report_logger.info("{0} grade: {1} / {2}\ntime: {3} avg: {4}".format(
name, grade, len(questions), total_time,
total_time / len(questions)))
return grade == len(questions) and True or False, total_time
def init(self, config: YamlConfigFileHandler):
assert (isinstance(config, YamlConfigFileHandler))
self.config = config
self.test_type = config.get_config('type', self.test_type)
self.server = config.get_config("server", self.server)
options = config.get_config(
"options", {
'shuffle_session': self.shuffle_session,
'pause_on_error': self.pause_on_error,
'suspend_on_error': self.suspend_on_error,
'question_interval': self.question_interval
})
self.shuffle_session = options.get('shuffle_session',
self.shuffle_session)
self.pause_on_error = options.get('pause_on_error',
self.pause_on_error)
self.suspend_on_error = options.get('suspend_on_error',
self.suspend_on_error)
self.question_interval = options.get('question_interval',
self.question_interval)
robots = []
if self.test_type == 'knowledge':
robots = config.get_config("usernames", None)
if not robots:
robots = config.get_config("username", None)
elif self.test_type == 'api':
robots = config.get_config('name', 'Unknown')
if isinstance(robots, basestring):
self.robots = [robots]
else:
self.robots = robots
questions_xlsx = config.filename.replace('.yml', '.xlsx')
questions_json = config.filename.replace('.yml', '.json')
questions_txt = config.filename.replace('.yml', '.txt')
if os.path.exists(questions_xlsx):
self.init_xlsx(questions_xlsx)
elif os.path.exists(questions_json):
self.init_json(questions_json)
elif os.path.exists(questions_txt):
self.init_txt(questions_txt)
else:
self.questions = config.get_config("questions", self.questions)
self.answers = config.get_config("answers", self.answers)
if self.shuffle_session:
self.shuffle_sessions()
configuration = config.get_config(
"output", {
'print_info': self.print_info,
'print_conversation': self.print_conversation,
'print_details': self.print_details,
'print_correct_answer': self.print_correct_answer,
'print_csv': self.print_csv
})
self.print_info = configuration.get('print_info', self.print_info)
self.print_conversation = configuration.get('print_conversation',
self.print_conversation)
self.print_details = configuration.get('print_details',
self.print_details)
self.print_correct_answer = configuration.get(
'print_correct_answer', self.print_correct_answer)
self.print_csv = configuration.get('print_csv', self.print_csv)
self.uid_prefix = config.get_config('uid_prefix', self.uid_prefix)
if self.questions and len(self.questions) > 0:
report_logger.info("grader ready, there are {0} questions".format(
len(self.questions)))
def test_robot(self, name, questions, answers):
if self.test_type == 'api':
test_service = SingleDialogueHandler(self.config)
else:
test_service = pMsgHandler(self.server, test_logger)
uid = "grader_" + id_generator(10)
test_service.robot_name = name
if self.print_csv:
write_utf_bom()
# items_op = getattr(questions, "iteritems", None)
# question_items = callable(items_op) and questions.iteritems() or questions.items()
grade = 0
total_time = 0
for i, question in questions.items():
process_time = time.time()
# turn question into a string and handle chat
if type(question) == int:
question_str = str(question)
else:
question_str = question
response = test_service.handle_chat(
uid, question_str, login_wait=self.question_interval)
chat_key = None
if hasattr(test_service, 'tokens'):
chat_key = test_service.tokens[uid]
process_time = time.time() - process_time
correct = True
answer_str = 'No answer found for question {0}'.format(i)
if answers and i in answers:
answer = answers[i]
correct = False
if isinstance(answer, dict):
if 'multi' in answer:
answer_str = json.dumps(
answer, ensure_ascii=False).encode('utf-8')
for j, item in enumerate(answer['multi']):
if response.find(item.encode('utf-8')) != -1:
correct = True
break
if 'regex' in answer:
answer_str = answer['regex'].encode('utf-8')
match_obj = re.match(answer_str, response)
if match_obj:
correct = True
else:
answer_str = answer.encode('utf-8')
if response.find(answer_str) != -1:
correct = True
else:
if response == self.ERROR_REPLY:
correct = False
if correct:
grade += 1
total_time += process_time
if self.print_csv:
csv_string = "Question {0},{1}".format(
i,
to_str(question.encode('utf-8')).replace(",", "，"))
csv_string += "," + to_str(response).replace(",", "，")
csv_string += "," + to_str(answer_str).replace(",", "，")
csv_string += "," + (correct and "Passed" or "Wrong")
csv_string += "," + "Processed in {:.5f} seconds".format(
process_time)
csv_logger.info(csv_string)
if self.print_info:
if self.print_conversation:
if self.print_details:
question_str = to_str(question_str.encode('utf-8'))
test_logger.info("Question {0}: {1}".format(
i, question_str))
response = to_str(response)
if chat_key:
test_logger.info(
"Chatkey: {1} Response: {0}".format(
response, chat_key))
else:
test_logger.info("Response :" + response)
if self.print_correct_answer or not correct:
answer_str = to_str(answer_str)
test_logger.info("Answer: " + answer_str)
test_logger.info(correct and "Passed" or "Wrong")
test_logger.info(
"Processed in {:.5f} seconds".format(process_time))
test_logger.info("=============")
else:
test_logger.info(
"Question {0}: {1} in {2} seconds".format(
i, correct and "Passed" or "Wrong",
process_time))
if not correct:
test_logger.info("Answer: " + response)
elif not correct:
test_logger.warning("Q {0} Wrong answer: {1}".format(
i, to_str(response)))
# print ("Q {0} Wrong answer: {1}".format(i, response))
if not correct and self.suspend_on_error:
break
if self.question_interval > 0.1:
time.sleep(self.question_interval)
# end of question loop
if self.print_csv:
csv_logger.info("{0},{1} / {2},{3},{4}".format(
name, grade, len(questions), total_time,
total_time / len(questions)))
report_logger.info("{0} grade: {1} / {2}\ntime: {3} avg: {4}".format(
name, grade, len(questions), total_time,
total_time / len(questions)))
return grade == len(questions) and True or False, total_time