def top_meet(self, rail, source, url, open_type):
screen_cut(rail)
text_line = get_text_line()
print(text_line)
if not text_line:
print("未发现题目")
return
questions = get_questions(text_line)
questions = deal_questions(questions, source=source)
get_answer(questions, url=url)
open_webboswer(questions[0], open_type=open_type)
def billon_super_man(self, rail, source, url, open_type):
screen_cut(cs_mac_rail_top)
text_line = get_text_line()
print(text_line)
if not text_line:
print("未发现题目")
return
questions = get_questions(text_line)
questions = deal_questions(questions, source=source)
get_answer(questions, url=url)
open_webboswer(questions[0], open_type=open_type)
def start_survey(self):
try:
q_index = 0
while True:
q = self.survey_step()
if q:
print()
print(f"Question {q_index}.")
print(f'-' * 80)
print(q)
answer = get_answer()
if answer == 'n':
continue
else:
q.answer = answer
self.questions.append(q)
q_index += 1
except KeyboardInterrupt:
if len(self.questions):
result = [q.is_answer_correct() for q in self.questions]
acc = round(sum(result) * 100 / len(result), 2)
print()
print(f"Intruision detection rate: {acc}")
print()
def predict(self,passage :str,question :str):
example = input_to_squad_example(passage,question)
features = squad_examples_to_features(example,self.tokenizer,self.max_seq_length,self.doc_stride,self.max_query_length)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_example_index)
eval_sampler = SequentialSampler(dataset)
eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=1)
all_results = []
for batch in eval_dataloader:
batch = tuple(t.to(self.device) for t in batch)
with torch.no_grad():
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2]
}
example_indices = batch[3]
outputs = self.model(**inputs)
for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item()]
unique_id = int(eval_feature.unique_id)
result = RawResult(unique_id = unique_id,
start_logits = to_list(outputs[0][i]),
end_logits = to_list(outputs[1][i]))
all_results.append(result)
answer = get_answer(example,features,all_results,self.n_best_size,self.max_answer_length,self.do_lower_case)
return answer
def reward(document, sampled_starts, sampled_ends, greedy_start, greedy_end,
gold_start, gold_end):
rewards = []
gold_answer, baseline_answer = get_answer(document, gold_start, gold_end,
int(greedy_start),
int(greedy_end))
baseline = f1_score(baseline_answer, gold_answer)
em = exact_match_score(baseline_answer, gold_answer)
for i in range(4):
gold_answer, sample_answer = get_answer(document, gold_start, gold_end,
int(sampled_starts[i]),
int(sampled_ends[i]))
f1 = f1_score(sample_answer, gold_answer)
normalized_reward = f1 - baseline
rewards.append(normalized_reward)
return rewards, baseline, em
def final_write(file_buffer):
danglings = catch_danglings(join(prefix, "texts"), file_buffer)
print("These files will be deleted:")
for d in danglings:
print(' ' + d)
print('continue? (y/n)')
ans = get_answer(['y', 'n'])
if ans == 'n':
print('Cancelled!')
return
print('Writing...')
with Pool() as p:
delete_result = p.map_async(remove, danglings)
write_result = p.starmap_async(write_file, list(file_buffer.items()))
p.close()
p.join()
def final_write(file_buffer):
danglings = catch_danglings(join(prefix, "texts"), file_buffer)
print("These files will be deleted:")
for d in danglings:
print(' ' + d)
print('continue? (y/n)')
ans = get_answer(['y', 'n'])
if ans == 'n':
print('Cancelled!')
return
print('Writing...')
with Pool() as p:
delete_result = p.map_async(remove, danglings)
write_result = p.starmap_async(write_file, list(file_buffer.items()))
p.close()
p.join()
def run_benchmark(templates):
"""
Compare performance of algorithm against a sample dataset
"""
answered = 0
with open(f'./datasets/sample-questions.json') as file:
training_data = json.load(file)
total = len(training_data)
for index, question in enumerate(training_data):
log.info(f'Testing {index +1} of {total}: {question}')
answer = get_answer(question, templates)
if answer != None and len(answer) > 1:
log.info(answer)
answered += 1
else:
log.info('INCORRECT\n')
log.info(f'Answered: {answered}\nTotal: {total}')
def post_answer(headers, cookies):
question_str, answer_d = get_question(headers, cookies)
answer_str = get_answer(question_str, answer_d.keys())
session = requests.Session()
data = {
"formhash": "74889ea9",
"answer": answer_d[answer_str],
"submit": "true"
}
resp = session.post(url=URLConfig.ANSWER_URL,
headers=headers,
cookies=cookies,
data=data)
soup = Soup(resp.text, "html.parser")
res = soup.find("div", {"id": "messagetext"}).p.text.split("setTimeout")[0]
print(res)
correct = False if res == WRONG else True
mark_answer(question_str, answer_str, correct)
def server(listen):
sockets = {listen.fileno(): listen}
receive = {}
send = {}
poll = select.poll()
poll.register(listen, select.POLLIN)
for fd, event in all_listen_event(poll):
sock = sockets[fd]
if event and (select.POLLHUP or select.POLLNVAL or select.POLLERR):
"""
开始接受/发送数据处理。
"""
poll.unregister(sock)
del sockets[fd]
elif sock is listen:
client_sock, address = sock.accept()
client_sock.setblocking(False)
sockets[client_sock.fileno] = client_sock
poll.register(client_sock, client_sock.POOLIN)
elif event and event.POOLIN:
more_data = sock.recv(4096)
if not more_data:
sock.close()
continue
data = receive.pop(sock, b'') + more_data
if more_data.endwith(b'?'):
send[sock] = utils.get_answer(data)
poll.modify(sock, sock.POLLOUT)
else:
receive[sock] = data
elif event and event.POOLOUT:
data = send.pop(sock, b'')
n = sock.send(data)
if n < len(data):
send[sock] = data[n + 1:]
else:
poll.modify(sock, sock.POLLIN)
def check_answer(message):
try:
answer, dictOut = utils.get_answer(message.chat.id,
message.text) # this is LIST
if not answer:
bot.send_message(message.chat.id,
'Чтобы что-то случилось, нажмите /show')
else:
keyboard_hider = types.ReplyKeyboardHide()
if message.text in answer:
print(message.text)
print(dictOut)
m = str(message.text)
rrr = dictOut[m]
bot.send_message(message.chat.id,
rrr,
reply_markup=keyboard_hider)
else:
bot.send_message(message.chat.id,
'неверный ввод',
reply_markup=keyboard_hider)
except TypeError:
bot.send_message(message.chat.id, 'ERROR что-то случилось, нажмите')
def main():
parser = cli.init_parser()
args = parser.parse_args()
log.setLevel(logging.DEBUG if args.debug else logging.INFO)
config.DEBUG = args.debug
config.UPDATE = args.properties or args.templates
config.BENCHMARK = args.benchmark
config.SIMILARITY_METRIC = args.metric
config.THRESHOLD = args.similarity
config.FIGURES = args.figures
log.debug(f'Started in DEBUG mode')
log.info('Hit CTRL+D to exit')
properties = None
filtered_properties = None
templates = []
# update properties if asked or if we're trying to update templates without a properties.json file
if args.properties or (args.templates and not has_properties_cache()):
log.info('Updating properties')
timer.tic()
properties = update()
log.info(f'Cached properties in: {timer.tocvalue()}')
if args.templates or not has_templates_cache():
log.info('Regenerating templates from cached properties')
log.info(
f'Using {config.MIN_PROPERTY_REFERENCE_COUNT} as minimum reference requirement'
)
timer.tic()
if properties == None:
properties = load_properties_from_cache()
total_properties = 0
for property_type in properties:
total_properties += len(properties[property_type])
log.info(
f'{len(properties.keys())} property types with {total_properties} total properties found'
)
filtered_properties = get_filtered_properties(properties)
templates = generate_templates_from_properties(filtered_properties)
log.info(f'Generated {len(templates)} question templates')
save_to_file(templates, filename=config.TEMPLATES_FILENAME)
log.info(f'Templates created in: {timer.tocvalue()}')
else:
log.debug('Loading templates from cache')
templates = load_templates_from_cache()
if args.benchmark and args.question:
log.error('Cannot ask question and run benchmarks at the same time')
sys.exit(-1)
if args.log or args.benchmark:
now = datetime.now()
current_time = now.strftime("%Y:%m:%d-%H:%M:%S")
log_file = logging.FileHandler(
f'results/{current_time}-benchmarks.log')
log.addHandler(log_file)
log.info(f'Loaded {len(templates)} templates')
log.info(
f"Using '{config.SIMILARITY_METRIC}' as similarity metric, with threshold of {config.THRESHOLD}"
)
if args.benchmark:
log.info('Running benchmarks...')
run_benchmark(templates)
elif args.question:
answer = get_answer(args.question, templates)
log.info(answer)
elif args.ask:
while True:
try:
question = input("Ask a question:\n")
if question == '':
continue
answer = get_answer(question, templates)
log.info(answer)
log.info('')
except KeyboardInterrupt as interrupt:
log.info('')
continue
except EOFError:
log.info('\nExiting\n')
sys.exit(0)
else:
parser.print_help()
def predict(self, passages: list, question: str):
examples = input_to_squad_example(passages, question)
features = squad_examples_to_features(examples,
self.tokenizer,
self.max_seq_length,
self.doc_stride,
self.max_query_length,
vsl=self.vsl)
if not self.use_tf and not self.use_onnx_runtime:
torch_input_ids = torch.tensor([f.input_ids for f in features],
dtype=torch.long).to(self.device)
torch_input_mask = torch.tensor([f.input_mask for f in features],
dtype=torch.long).to(self.device)
torch_segment_ids = torch.tensor([f.segment_ids for f in features],
dtype=torch.long).to(self.device)
torch_example_index = torch.arange(torch_input_ids.size(0),
dtype=torch.long).to(
self.device)
all_results = []
if self.use_tf:
if self.tf_onnx:
# this is the TF graph converted from ONNX
inputs = {
'input_ids:0': [f.input_ids for f in features],
'attention_mask:0': [f.input_mask for f in features],
'token_type_ids:0': [f.segment_ids for f in features]
}
start_logits, end_logits = self.model.run(
['Squeeze_49:0', 'Squeeze_50:0'], feed_dict=inputs)
else:
# this is the original TF graph
inputs = {
'input_ids:0': [f.input_ids for f in features],
'input_mask:0': [f.input_mask for f in features],
'segment_ids:0': [f.segment_ids for f in features]
}
start_logits, end_logits = self.model.run(
['start_logits:0', 'end_logits:0'], feed_dict=inputs)
example_indices = np.arange(len(features))
outputs = [start_logits, end_logits]
elif self.use_onnx_runtime:
inputs = {
self.model.get_inputs()[0].name:
np.array([f.input_ids for f in features]),
self.model.get_inputs()[1].name:
np.array([f.input_mask for f in features]),
self.model.get_inputs()[2].name:
np.array([f.segment_ids for f in features])
}
output_names = [
self.model.get_outputs()[0].name,
self.model.get_outputs()[1].name
]
example_indices = np.arange(len(features))
outputs = self.model.run(output_names, inputs)
else:
example_indices = torch_example_index
if self.use_jit:
outputs = self.model(torch_input_ids, torch_input_mask,
torch_segment_ids)
else:
with torch.no_grad():
inputs = {
'input_ids': torch_input_ids,
'attention_mask': torch_input_mask,
'token_type_ids': torch_segment_ids
}
outputs = self.model(**inputs)
for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item()]
unique_id = int(eval_feature.unique_id)
result = RawResult(unique_id=unique_id,
start_logits=to_list(outputs[0][i]),
end_logits=to_list(outputs[1][i]))
all_results.append(result)
answers = get_answer(examples, features, all_results, self.n_best_size,
self.max_answer_length, self.do_lower_case)
return answers
def replace(target_file, field, newdata, original):
## Tries to merge translation to framework
## if translation exists and conflicts with new one
## asks for user to manual merge
## target - file in framework related to file should be translated
## field - path of field in game assets json file which should be translated
## newdata - translated string
## original - path to file in game assets should be translated
target = join(root_dir, target_file)
data, index = get_data(field, target, original)
if not (type(newdata) is str):
return
if data is None:
print("Cannot get data: " + newdata)
print("Target file: " + target)
print("Assets file: " + original)
return
changed = False
olddata = ""
if "DeniedAlternatives" not in data[index]:
data[index]["DeniedAlternatives"] = list()
if "Rus" in data[index]["Texts"]:
olddata = data[index]["Texts"]["Rus"]
if olddata == newdata or len(newdata) == 0:
return
elif newdata in data[index]["DeniedAlternatives"]:
return
elif len(olddata) == 0:
changed = True
data[index]["Texts"]["Rus"] = newdata
else:
print("Target: " + target)
print("Origin: " + original)
print("Used in:")
i = 0
for f, fields in data[index]["Files"].items():
if i > 5:
print("...and in " + str(len(data[index]["Files"])-i) + " more files")
break
print(" " + f)
for p in fields:
print(" at " + p)
i += 1
print("Denied variants:")
for d in data[index]["DeniedAlternatives"]:
print(' ' + d)
print("Field: " + field)
print("English text:")
print(' "' + data[index]["Texts"]["Eng"] + '"')
print("Old Russian text:")
print(' "' + data[index]["Texts"]["Rus"] + '"')
print("New Russian text:")
print(" \"" + newdata + '"')
print("What text should be used?")
print(" n - new text")
print(" o - old text")
print(" e - enter manually")
answer = get_answer(["n", "o", "e", "i"])
if answer == "n":
print("Setting to the new data...")
if olddata not in data[index]["DeniedAlternatives"]:
insort_left(data[index]["DeniedAlternatives"], olddata)
if newdata in data[index]["DeniedAlternatives"]:
data[index]["DeniedAlternatives"].remove(newdata)
data[index]["Texts"]["Rus"] = newdata
changed = True
elif answer == "e":
print("Enter new data:")
answer = get_answer(3)
data[index]["Texts"]["Rus"] = answer
changed = True
if newdata not in data[index]["DeniedAlternatives"] and newdata != answer:
insort_left(data[index]["DeniedAlternatives"], newdata)
changed = True
if olddata not in data[index]["DeniedAlternatives"] and olddata != answer:
insort_left(data[index]["DeniedAlternatives"], olddata)
changed = True
if answer in data[index]["DeniedAlternatives"]:
data[index]["DeniedAlternatives"].remove(answer)
print("Written: " + answer)
elif answer == "i":
import code
code.InteractiveConsole(locals=globals()).interact()
else:
print("Keeping old data...")
if newdata not in data[index]["DeniedAlternatives"]:
insort_left(data[index]["DeniedAlternatives"],newdata)
changed = True
if olddata in data[index]["DeniedAlternatives"]:
data[index]["DeniedAlternatives"].remove(olddata)
changed = True
if changed:
pass
with copen(target, "w", 'utf-8') as f:
json.dump(data, f, ensure_ascii = False, indent = 2, sort_keys=True)
