def get_response():
print(request.json)
score = request.json['score']
history = request.json['history']
# if score < 0.5:
# bot_message = generate_response(model_reddit, tokenizer_reddit, ' {} '.format(tokenizer_reddit.eos_token).join(history), config)
# else:
out_ids = sample_sequence(custom_personality, list(map(tokenizer.encode, history)), tokenizer, model, args)
bot_message = tokenizer.decode(out_ids, skip_special_tokens=True)
score += CLOSENESS
return jsonify(response = bot_message, score = score)
async def toto(item: Item, text: str):
logging.info(f"text: {text}")
logging.info(pformat(item))
personality = [tokenizer.encode(x) for x in item.persona]
if text is not None and item.context[-1] != text:
item.context.append(text)
history = [tokenizer.encode(o) for o in item.context]
with torch.no_grad():
out_ids = sample_sequence(personality, history, tokenizer, model, args)
res = tokenizer.decode(out_ids, skip_special_tokens=True)
logging.info(f"context={item.context}, res='{res}'")
return {"text": res}
def act(self):
reply = {}
if self.args.eval_type == "hits@1" and len(self.candidates) > 0:
instances = defaultdict(list)
for candidate, _ in self.candidates:
instance, _ = build_input_from_segments(
self.persona, self.history, candidate, self.tokenizer)
for input_name, input_array in instance.items():
instances[input_name].append(input_array)
inputs = pad_dataset(instances,
padding=self.special_tokens_ids[-1])
tensor_inputs = {}
for input_name in ["input_ids", "mc_token_ids", "token_type_ids"]:
tensor = torch.tensor(inputs[input_name],
device=self.args.device)
tensor = tensor.view((-1, len(self.candidates)) +
tensor.shape[1:])
tensor_inputs[input_name] = tensor
with torch.no_grad():
_, mc_logits = self.model_checkpoint(**tensor_inputs)
val, ind = torch.sort(mc_logits[0], descending=True)
ypred = self.candidates[ind[0].item()][1] # match
tc = []
for j in range(len(self.candidates)):
tc.append(self.candidates[ind[j].item()][1])
reply = {'text': ypred, 'text_candidates': tc}
else:
# We are in interactive of f1 evaluation mode => just sample
with torch.no_grad():
out_ids = sample_sequence(self.persona, self.history,
self.tokenizer,
self.model_checkpoint,
self.args) # YW: TODO: out_ids, _?
out_text = self.tokenizer.decode(
out_ids,
skip_special_tokens=True,
clean_up_tokenization_spaces=(self.args.eval_type != 'f1'))
# print('out_text:', out_text)
reply = {'text': out_text}
return reply
def ask():
try:
start = time.time()
logging.info('prediction requested')
params = get_params()
logger.debug(json.dumps(params, indent=2))
all_utterances = params.get('utterances', '')
if all_utterances == '':
all_utterances = []
elif isinstance(all_utterances, str):
hist_str = html.unescape(all_utterances)
all_utterances = json.loads(hist_str)
user_input = params.get('user_input', None)
if user_input is not None:
all_utterances.append(user_input)
utterances = all_utterances[-(2 * args.max_history + 1):]
#if 'user_input' in params:
# raise DeprecationWarning('Parameter "user_input" is deprecated. Append the user_input to the utterances '
# 'instead.')
# create required format of context: dict with entry_id -> list of sentences (strings)
#if isinstance(params.get('background', None), str):
# params['background'] = {'user': params['background']}
background = params.get('background', None) if params.get(
'keep_background', False) else None
if isinstance(background, str):
if background == '':
background = None
else:
backgr_str = html.unescape(background)
background = json.loads(backgr_str)
if background is not None:
for k in background:
background[k]['external'] = True
if not params.get('dont_fetch', False):
assert context_fetcher is not None, 'No context/background fetcher initialized. Please provide a background with every request.'
try:
# use only the considered utterances to query background
background = context_fetcher(' '.join(utterances),
previous_context=background)
except InvalidUsage as e:
response = e.to_dict()
response['status_code'] = e.status_code
logger.warning(f'context_fetcher exception: {response}')
except AssertionError as e:
logger.warning(e)
background_encoded = None
if background is not None and len(background) > 0:
background_keys, background_encoded = zip(
*[(k, tokenizer.encode(b['text']))
for k, b in background.items()])
params['background'] = background
else:
background_keys = []
if 'background' in params:
# delete for html template
del params['background']
personality_encoded = None
if 'personality' in params:
personality_encoded = tokenizer.encode(params['personality'])
utterances_encoded = [
tokenizer.encode(utterance) for utterance in utterances
]
all_utterance_types = params.get('utterance_types', None)
if all_utterance_types is not None:
assert len(all_utterances) == len(all_utterance_types), f'number of utterance elements [{len(utterances)}] does not match ' \
f'number of utterance_types [{len(all_utterance_types)}]'
utterance_types_encoded = []
allowed_hist_types = ', '.join(tokenizer.all_special_tokens)
for hist_type in all_utterance_types[-(2 * args.max_history + 1):]:
assert hist_type in tokenizer.all_special_tokens, f'Unknown type for utterances element: {hist_type}. ' \
f'Use only these types: {allowed_hist_types}'
utterance_types_encoded.append(
tokenizer.convert_tokens_to_ids(hist_type))
else:
# if no utterance types are available, assume the last utterance was from the user and then the type
# alternates, i.e. starting from last utterance in reverse order: <user>, <bot>, <user>, <bot>, ...
utterance_types_encoded = [
tokenizer.convert_tokens_to_ids(TYPE_USER) if
(len(utterances) - i) %
2 else tokenizer.convert_tokens_to_ids(TYPE_BOT)
for i, h in enumerate(utterances)
]
# predict only if any utterances / user_input (was added to utterances) is available
if len(utterances) > 0:
# if explanations are requested:
if params.get('explain', False):
out_ids, eos, last_ids, explanations = sample_sequence(
background=background_encoded,
personality=personality_encoded,
utterances=utterances_encoded,
utterance_types=utterance_types_encoded,
tokenizer=tokenizer,
model=model,
args=args,
explain=True,
replace_unknown=True)
explanations_merged = merge_sample_explanations(explanations)
all_tokens = [tokenizer.decode([tok]) for tok in last_ids[0]]
special_and_texts, explanation_annotations = create_explanation_annotations(
tokens=all_tokens,
expl=sum(explanations_merged.values()),
split_tokens=tokenizer.all_special_tokens,
return_tuples=True,
return_spans=False)
special_tokens, explanation_texts = zip(*special_and_texts)
explanation_texts = list(explanation_texts)
explanation_texts_w_annotations = create_annotated_spans(
explanation_texts, explanation_annotations)
# add prediction
explanation_texts[-1] = tokenizer.decode(out_ids)
explanation_texts_w_annotations[
-1] = f'<span class="prediction">{html.escape(tokenizer.decode(out_ids))}</span>'
# just for compatibility
#params['explanation'] = {'utterances': [], 'background': {}}
params['explanation_annotations'] = {
'utterances': [],
'background': {}
}
# just for sanity check
params['explanation_texts'] = {
'utterances': [],
'background': {}
}
n_background = 0
for i, _ in enumerate(explanation_texts):
if special_tokens[i] in ['<user>', '<bot>']:
# compatibility
#params['explanation']['utterances'].append(explanation_texts_w_annotations[i])
params['explanation_annotations']['utterances'].append(
explanation_annotations[i])
# sanity check (should equal params['utterances'])
params['explanation_texts']['utterances'].append(
explanation_texts[i])
elif special_tokens[i] == '<background>':
# compatibility
#params['explanation']['background'][background_keys[n_background]] = explanation_texts_w_annotations[i]
params['explanation_annotations']['background'][
background_keys[
n_background]] = explanation_annotations[i]
# sanity check
params['explanation_texts']['background'][
background_keys[n_background]] = explanation_texts[
i]
n_background += 1
else:
with torch.no_grad():
out_ids, eos = sample_sequence(
background=background_encoded,
personality=personality_encoded,
utterances=utterances_encoded,
utterance_types=utterance_types_encoded,
tokenizer=tokenizer,
model=model,
args=args,
explain=False,
replace_unknown=True)
utterances_encoded.append(out_ids)
params['prediction'] = tokenizer.decode(out_ids,
skip_special_tokens=True)
params['utterances'] = [
tokenizer.decode(utterance) for utterance in utterances_encoded
]
# sanity check
if 'explanation_texts' in params:
for i, u in enumerate(params['utterances']):
assert u == params['explanation_texts']['utterances'][i], \
f'utterance [{u}] does not equal text returned by visualize_explanation ' \
f'[{params["explanation_text"]["utterances"][i]}]'
del params['explanation_texts']
utterance_types_encoded.append(
tokenizer.convert_tokens_to_ids(TYPE_BOT))
params['utterance_types'] = tokenizer.convert_ids_to_tokens(
utterance_types_encoded)
if eos is not None:
params['eos'] = tokenizer.convert_ids_to_tokens([eos])[0]
else:
params['eos'] = None
logger.debug('predicted:\n%s' % params['prediction'])
# add annotations only when not explaining
if params.get('background', None) is not None:
pos_start = 0
# deprecated, just for compatibility (use entity_annotations instead)
params['utterances_annotated'] = []
params['entity_annotations'] = []
for h in params['utterances']:
params['utterances_annotated'].append(
insert_annotations(s=h,
annotations=params['background'],
offset=pos_start))
for k, entity in params['background'].items():
start = entity['offsetStart'] - pos_start
if 0 <= start < len(h):
l = entity['offsetEnd'] - entity['offsetStart']
params['entity_annotations'].append((start, l, k))
# increase (1 for space)
pos_start += 1 + len(h)
params['model'] = checkpoint_fn
http_accept = params.get('HTTP_ACCEPT', False) or 'text/html'
if 'application/json' in http_accept:
json_data = json.dumps(params)
response = Response(json_data, mimetype=http_accept)
response.headers['Access-Control-Allow-Origin'] = '*'
else:
response = Response(render_template('chat.html',
root=args.root,
**params),
mimetype='text/html')
logger.info("Time spent handling the request: %f" %
(time.time() - start))
except Exception as e:
tb = traceback.format_exc()
logger.error(tb)
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
raise InvalidUsage('%s: %s @line %s in %s' %
(type(e).__name__, str(e), exc_tb.tb_lineno, fname))
return response
def act(self):
reply = {}
if self.args.eval_type == "hits@1" and len(self.candidates) > 0:
instances = defaultdict(list)
for candidate, _ in self.candidates:
instance, _ = build_input_from_segments(self.persona, self.history, candidate, self.tokenizer)
for input_name, input_array in instance.items():
instances[input_name].append(input_array)
inputs = pad_dataset(instances, padding=self.special_tokens_ids[-1])
tensor_inputs = {}
for input_name in ["input_ids", "mc_token_ids", "token_type_ids"]:
tensor = torch.tensor(inputs[input_name], device=self.args.device)
tensor = tensor.view((-1, len(self.candidates)) + tensor.shape[1:])
tensor_inputs[input_name] = tensor
with torch.no_grad():
_, mc_logits = self.model_checkpoint(**tensor_inputs)
val, ind = torch.sort(mc_logits[0], descending=True)
ypred = self.candidates[ind[0].item()][1] # match
tc = []
for j in range(len(self.candidates)):
tc.append(self.candidates[ind[j].item()][1])
reply = {'text': ypred, 'text_candidates': tc}
else:
# We are in interactive of f1 evaluation mode => just sample
with torch.no_grad():
out_ids = sample_sequence(self.persona, self.history, self.tokenizer, self.model_checkpoint, self.args) # YW: TODO: out_ids, _?
# Get a generated response
out_text = self.tokenizer.decode(out_ids, skip_special_tokens=True,
clean_up_tokenization_spaces=(self.args.eval_type != 'f1'))
out_text_org = out_text
out_text = out_text.replace(' \' ', '\'') # TODO: tbd
out_text = out_text.replace(' \'', '\'')
# persona NLI
profiles = []
for profile in self.persona:
profile_text = self.tokenizer.decode(profile, skip_special_tokens=True, clean_up_tokenization_spaces=False)
profile_text = profile_text.replace(' \' ', '\'') # TODO: tbd
profile_text = profile_text.replace(' \'', '\'')
profiles.append(profile_text)
nli_score, reward_score, c_score, current_con_en = nli_engine(out_text, profiles, self.nli_tokenizer, self.nli_model, eval=True)
self.nli_scores += nli_score # persona NLI
self.reward_scores += reward_score # reward function
self.c_scores += c_score # C score
self.sample_num += 1
self.con_en += current_con_en # if this persona contains a contradicted/entail profile or not (not applied)
# internal repetition
response_tok = out_text_org.split()
intrep_1gram = intrep_frac(response_tok)
# if 2-gram or 3-gram are going to be used:
''''
# intrep_2gram
response_tok_2gram = get_ngrams(out_text, 2)
intrep_2gram = intrep_frac(response_tok_2gram)
# intrep_3gram
response_tok_3gram = get_ngrams(out_text, 3)
intrep_3gram = intrep_frac(response_tok_3gram)
'''
intern_rep_reward = intrep_1gram
self.intrep_scores += intern_rep_reward
# bleu
label_text = self.tokenizer.decode(self.labels, skip_special_tokens=True, clean_up_tokenization_spaces=False)
current_bleu = bleu_rewarder(out_text_org, label_text)
self.bleu_scores += current_bleu
# fine-tuned GPT-based language model
lm_tokenize_input = self.lm_tokenizer.tokenize(out_text)
# lm_tensor_input = torch.tensor([lm_tokenizer.convert_tokens_to_ids(lm_tokenize_input)]).to(args.device)
lm_tensor_input = torch.tensor([[self.special_tokens_ids[0]] + self.lm_tokenizer.convert_tokens_to_ids(lm_tokenize_input) + [self.special_tokens_ids[-1]]]).to(self.args.device)
lm_loss = self.lm_model(lm_tensor_input, lm_labels=lm_tensor_input)
lm_ppl = math.exp(lm_loss.item())
self.lm_ppl_scores += lm_ppl
print('out_text:', out_text)
print('current nli:', self.nli_scores)
print('current score:', self.reward_scores / self.sample_num)
print('current c_score_macro:', self.c_scores / self.sample_num)
current_c_score_micro = (self.nli_scores[1] - self.nli_scores[0]) / sum(self.nli_scores)
cn_res = nli_score[1] - nli_score[0] # cn: C_new (persona level)
# C_new calculation
if cn_res > 0:
current_cn = 1
elif cn_res < 0:
current_cn = -1
else:
current_cn = 0
self.cnm += current_cn
print('current c_new:', self.cnm / self.sample_num)
print('current c_score_micro:', current_c_score_micro)
print('current con_en:', self.con_en)
print('current intrep score:', self.intrep_scores / self.sample_num)
print('current BLEU:', self.bleu_scores / self.sample_num)
print('current PPL:', self.lm_ppl_scores / self.sample_num)
reply = {'text': out_text}
return reply
def pred_attacks(df,
model_path,
args,
output_clm,
context='title',
attacks_clm='premise_counter_premise_pairs',
post_clm='post',
rand_premise_idx=-1,
rand_premises_clm='',
baseline=False,
random=False):
model, tokenizer = interact.load_model(model_path)
predictions = []
for row_idx, row in df.iterrows():
post_attacks = []
for premise_counter in row[attacks_clm]:
if context == 'title+full_post':
argument = [tokenizer.encode(row['title'])] + [
tokenizer.encode(sent) for sent in row[post_clm]
]
else:
argument = [tokenizer.encode(
row['title'])] if context == 'title' else [
tokenizer.encode(sent) for sent in row[post_clm]
]
#print([row['title']]+row[post_clm])
#print(premise_counter[0])
if baseline:
argument = trim_argument(argument, 510, args)
pred_counter = interact.sample_sequence(argument, [],
tokenizer,
model,
args,
baseline=True)
elif random:
if rand_premise_idx != None:
weak_premise_encoded = [
tokenizer.encode(
row[rand_premises_clm][rand_premise_idx])
]
else:
weak_premise_encoded = [
tokenizer.encode(row[rand_premises_clm])
]
argument = trim_argument(argument, 510, args)
pred_counter = interact.sample_sequence(
argument, weak_premise_encoded, tokenizer, model, args)
else:
weak_premise_encoded = [
tokenizer.encode(premise) for premise in premise_counter[0]
]
argument = trim_argument(argument, 510, args)
pred_counter = interact.sample_sequence(
argument, weak_premise_encoded, tokenizer, model, args)
#remove quoted text
#pred_counter = tokenizer.decode(pred_counter)
#for p in premise_counter[0]:
# pred_counter = pred_counter.replace(p.lower(), '')
post_attacks.append([premise_counter[0], pred_counter])
predictions.append(post_attacks)
df[output_clm] = predictions
return df
def perform_attacks_hua_df(df,
model_path,
args,
output_clm,
context,
weak_premise_clm,
weak_premise_idx=None,
baseline=False):
model, tokenizer = interact.load_model(model_path)
pred_attacks = []
for idx, row in df.iterrows():
if baseline:
argument = [row['claim']] + row['post']
argument = [tokenizer.encode(sentence) for sentence in argument]
argument = trim_argument(argument, 510, args)
pred_counter = interact.sample_sequence(argument, [],
tokenizer,
model,
args,
baseline=True)
pred_counter = pred_counter.replace(row['claim'].lower(), '')
for sent in row['post']:
pred_counter = pred_counter.replace(sent.lower(), '')
else:
if context == 'title+full_post':
argument = [tokenizer.encode(row['claim'])] + [
tokenizer.encode(sent) for sent in row['post']
]
else:
argument = [tokenizer.encode(row['claim'])
] if context == 'title' else [
tokenizer.encode(sent) for sent in row['post']
]
if weak_premise_idx != None:
weak_premise_encoded = [
tokenizer.encode(premise)
for premise in row[weak_premise_clm][weak_premise_idx]
]
else:
weak_premise_encoded = [
tokenizer.encode(premise)
for premise in row[weak_premise_clm]
]
if args.premise_extra:
argument = trim_argument(
argument, 510 -
len([token for p in weak_premise_encoded for token in p]),
args)
else:
argument = trim_argument(argument, 510, args)
pred_counter = interact.sample_sequence(argument,
weak_premise_encoded,
tokenizer, model, args)
#remove quoted text
#pred_counter = tokenizer.decode(pred_counter)
for p in row[weak_premise_clm]:
pred_counter = pred_counter.replace(p.lower(), '')
pred_attacks.append(pred_counter)
df[output_clm] = pred_attacks
return df
