def get_response(): print(request.json) score = request.json['score'] history = request.json['history'] # if score < 0.5: # bot_message = generate_response(model_reddit, tokenizer_reddit, ' {} '.format(tokenizer_reddit.eos_token).join(history), config) # else: out_ids = sample_sequence(custom_personality, list(map(tokenizer.encode, history)), tokenizer, model, args) bot_message = tokenizer.decode(out_ids, skip_special_tokens=True) score += CLOSENESS return jsonify(response = bot_message, score = score)
async def toto(item: Item, text: str): logging.info(f"text: {text}") logging.info(pformat(item)) personality = [tokenizer.encode(x) for x in item.persona] if text is not None and item.context[-1] != text: item.context.append(text) history = [tokenizer.encode(o) for o in item.context] with torch.no_grad(): out_ids = sample_sequence(personality, history, tokenizer, model, args) res = tokenizer.decode(out_ids, skip_special_tokens=True) logging.info(f"context={item.context}, res='{res}'") return {"text": res}
def act(self): reply = {} if self.args.eval_type == "hits@1" and len(self.candidates) > 0: instances = defaultdict(list) for candidate, _ in self.candidates: instance, _ = build_input_from_segments( self.persona, self.history, candidate, self.tokenizer) for input_name, input_array in instance.items(): instances[input_name].append(input_array) inputs = pad_dataset(instances, padding=self.special_tokens_ids[-1]) tensor_inputs = {} for input_name in ["input_ids", "mc_token_ids", "token_type_ids"]: tensor = torch.tensor(inputs[input_name], device=self.args.device) tensor = tensor.view((-1, len(self.candidates)) + tensor.shape[1:]) tensor_inputs[input_name] = tensor with torch.no_grad(): _, mc_logits = self.model_checkpoint(**tensor_inputs) val, ind = torch.sort(mc_logits[0], descending=True) ypred = self.candidates[ind[0].item()][1] # match tc = [] for j in range(len(self.candidates)): tc.append(self.candidates[ind[j].item()][1]) reply = {'text': ypred, 'text_candidates': tc} else: # We are in interactive of f1 evaluation mode => just sample with torch.no_grad(): out_ids = sample_sequence(self.persona, self.history, self.tokenizer, self.model_checkpoint, self.args) # YW: TODO: out_ids, _? out_text = self.tokenizer.decode( out_ids, skip_special_tokens=True, clean_up_tokenization_spaces=(self.args.eval_type != 'f1')) # print('out_text:', out_text) reply = {'text': out_text} return reply
def ask(): try: start = time.time() logging.info('prediction requested') params = get_params() logger.debug(json.dumps(params, indent=2)) all_utterances = params.get('utterances', '') if all_utterances == '': all_utterances = [] elif isinstance(all_utterances, str): hist_str = html.unescape(all_utterances) all_utterances = json.loads(hist_str) user_input = params.get('user_input', None) if user_input is not None: all_utterances.append(user_input) utterances = all_utterances[-(2 * args.max_history + 1):] #if 'user_input' in params: # raise DeprecationWarning('Parameter "user_input" is deprecated. Append the user_input to the utterances ' # 'instead.') # create required format of context: dict with entry_id -> list of sentences (strings) #if isinstance(params.get('background', None), str): # params['background'] = {'user': params['background']} background = params.get('background', None) if params.get( 'keep_background', False) else None if isinstance(background, str): if background == '': background = None else: backgr_str = html.unescape(background) background = json.loads(backgr_str) if background is not None: for k in background: background[k]['external'] = True if not params.get('dont_fetch', False): assert context_fetcher is not None, 'No context/background fetcher initialized. Please provide a background with every request.' try: # use only the considered utterances to query background background = context_fetcher(' '.join(utterances), previous_context=background) except InvalidUsage as e: response = e.to_dict() response['status_code'] = e.status_code logger.warning(f'context_fetcher exception: {response}') except AssertionError as e: logger.warning(e) background_encoded = None if background is not None and len(background) > 0: background_keys, background_encoded = zip( *[(k, tokenizer.encode(b['text'])) for k, b in background.items()]) params['background'] = background else: background_keys = [] if 'background' in params: # delete for html template del params['background'] personality_encoded = None if 'personality' in params: personality_encoded = tokenizer.encode(params['personality']) utterances_encoded = [ tokenizer.encode(utterance) for utterance in utterances ] all_utterance_types = params.get('utterance_types', None) if all_utterance_types is not None: assert len(all_utterances) == len(all_utterance_types), f'number of utterance elements [{len(utterances)}] does not match ' \ f'number of utterance_types [{len(all_utterance_types)}]' utterance_types_encoded = [] allowed_hist_types = ', '.join(tokenizer.all_special_tokens) for hist_type in all_utterance_types[-(2 * args.max_history + 1):]: assert hist_type in tokenizer.all_special_tokens, f'Unknown type for utterances element: {hist_type}. ' \ f'Use only these types: {allowed_hist_types}' utterance_types_encoded.append( tokenizer.convert_tokens_to_ids(hist_type)) else: # if no utterance types are available, assume the last utterance was from the user and then the type # alternates, i.e. starting from last utterance in reverse order: <user>, <bot>, <user>, <bot>, ... utterance_types_encoded = [ tokenizer.convert_tokens_to_ids(TYPE_USER) if (len(utterances) - i) % 2 else tokenizer.convert_tokens_to_ids(TYPE_BOT) for i, h in enumerate(utterances) ] # predict only if any utterances / user_input (was added to utterances) is available if len(utterances) > 0: # if explanations are requested: if params.get('explain', False): out_ids, eos, last_ids, explanations = sample_sequence( background=background_encoded, personality=personality_encoded, utterances=utterances_encoded, utterance_types=utterance_types_encoded, tokenizer=tokenizer, model=model, args=args, explain=True, replace_unknown=True) explanations_merged = merge_sample_explanations(explanations) all_tokens = [tokenizer.decode([tok]) for tok in last_ids[0]] special_and_texts, explanation_annotations = create_explanation_annotations( tokens=all_tokens, expl=sum(explanations_merged.values()), split_tokens=tokenizer.all_special_tokens, return_tuples=True, return_spans=False) special_tokens, explanation_texts = zip(*special_and_texts) explanation_texts = list(explanation_texts) explanation_texts_w_annotations = create_annotated_spans( explanation_texts, explanation_annotations) # add prediction explanation_texts[-1] = tokenizer.decode(out_ids) explanation_texts_w_annotations[ -1] = f'<span class="prediction">{html.escape(tokenizer.decode(out_ids))}</span>' # just for compatibility #params['explanation'] = {'utterances': [], 'background': {}} params['explanation_annotations'] = { 'utterances': [], 'background': {} } # just for sanity check params['explanation_texts'] = { 'utterances': [], 'background': {} } n_background = 0 for i, _ in enumerate(explanation_texts): if special_tokens[i] in ['<user>', '<bot>']: # compatibility #params['explanation']['utterances'].append(explanation_texts_w_annotations[i]) params['explanation_annotations']['utterances'].append( explanation_annotations[i]) # sanity check (should equal params['utterances']) params['explanation_texts']['utterances'].append( explanation_texts[i]) elif special_tokens[i] == '<background>': # compatibility #params['explanation']['background'][background_keys[n_background]] = explanation_texts_w_annotations[i] params['explanation_annotations']['background'][ background_keys[ n_background]] = explanation_annotations[i] # sanity check params['explanation_texts']['background'][ background_keys[n_background]] = explanation_texts[ i] n_background += 1 else: with torch.no_grad(): out_ids, eos = sample_sequence( background=background_encoded, personality=personality_encoded, utterances=utterances_encoded, utterance_types=utterance_types_encoded, tokenizer=tokenizer, model=model, args=args, explain=False, replace_unknown=True) utterances_encoded.append(out_ids) params['prediction'] = tokenizer.decode(out_ids, skip_special_tokens=True) params['utterances'] = [ tokenizer.decode(utterance) for utterance in utterances_encoded ] # sanity check if 'explanation_texts' in params: for i, u in enumerate(params['utterances']): assert u == params['explanation_texts']['utterances'][i], \ f'utterance [{u}] does not equal text returned by visualize_explanation ' \ f'[{params["explanation_text"]["utterances"][i]}]' del params['explanation_texts'] utterance_types_encoded.append( tokenizer.convert_tokens_to_ids(TYPE_BOT)) params['utterance_types'] = tokenizer.convert_ids_to_tokens( utterance_types_encoded) if eos is not None: params['eos'] = tokenizer.convert_ids_to_tokens([eos])[0] else: params['eos'] = None logger.debug('predicted:\n%s' % params['prediction']) # add annotations only when not explaining if params.get('background', None) is not None: pos_start = 0 # deprecated, just for compatibility (use entity_annotations instead) params['utterances_annotated'] = [] params['entity_annotations'] = [] for h in params['utterances']: params['utterances_annotated'].append( insert_annotations(s=h, annotations=params['background'], offset=pos_start)) for k, entity in params['background'].items(): start = entity['offsetStart'] - pos_start if 0 <= start < len(h): l = entity['offsetEnd'] - entity['offsetStart'] params['entity_annotations'].append((start, l, k)) # increase (1 for space) pos_start += 1 + len(h) params['model'] = checkpoint_fn http_accept = params.get('HTTP_ACCEPT', False) or 'text/html' if 'application/json' in http_accept: json_data = json.dumps(params) response = Response(json_data, mimetype=http_accept) response.headers['Access-Control-Allow-Origin'] = '*' else: response = Response(render_template('chat.html', root=args.root, **params), mimetype='text/html') logger.info("Time spent handling the request: %f" % (time.time() - start)) except Exception as e: tb = traceback.format_exc() logger.error(tb) exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] raise InvalidUsage('%s: %s @line %s in %s' % (type(e).__name__, str(e), exc_tb.tb_lineno, fname)) return response
def act(self): reply = {} if self.args.eval_type == "hits@1" and len(self.candidates) > 0: instances = defaultdict(list) for candidate, _ in self.candidates: instance, _ = build_input_from_segments(self.persona, self.history, candidate, self.tokenizer) for input_name, input_array in instance.items(): instances[input_name].append(input_array) inputs = pad_dataset(instances, padding=self.special_tokens_ids[-1]) tensor_inputs = {} for input_name in ["input_ids", "mc_token_ids", "token_type_ids"]: tensor = torch.tensor(inputs[input_name], device=self.args.device) tensor = tensor.view((-1, len(self.candidates)) + tensor.shape[1:]) tensor_inputs[input_name] = tensor with torch.no_grad(): _, mc_logits = self.model_checkpoint(**tensor_inputs) val, ind = torch.sort(mc_logits[0], descending=True) ypred = self.candidates[ind[0].item()][1] # match tc = [] for j in range(len(self.candidates)): tc.append(self.candidates[ind[j].item()][1]) reply = {'text': ypred, 'text_candidates': tc} else: # We are in interactive of f1 evaluation mode => just sample with torch.no_grad(): out_ids = sample_sequence(self.persona, self.history, self.tokenizer, self.model_checkpoint, self.args) # YW: TODO: out_ids, _? # Get a generated response out_text = self.tokenizer.decode(out_ids, skip_special_tokens=True, clean_up_tokenization_spaces=(self.args.eval_type != 'f1')) out_text_org = out_text out_text = out_text.replace(' \' ', '\'') # TODO: tbd out_text = out_text.replace(' \'', '\'') # persona NLI profiles = [] for profile in self.persona: profile_text = self.tokenizer.decode(profile, skip_special_tokens=True, clean_up_tokenization_spaces=False) profile_text = profile_text.replace(' \' ', '\'') # TODO: tbd profile_text = profile_text.replace(' \'', '\'') profiles.append(profile_text) nli_score, reward_score, c_score, current_con_en = nli_engine(out_text, profiles, self.nli_tokenizer, self.nli_model, eval=True) self.nli_scores += nli_score # persona NLI self.reward_scores += reward_score # reward function self.c_scores += c_score # C score self.sample_num += 1 self.con_en += current_con_en # if this persona contains a contradicted/entail profile or not (not applied) # internal repetition response_tok = out_text_org.split() intrep_1gram = intrep_frac(response_tok) # if 2-gram or 3-gram are going to be used: '''' # intrep_2gram response_tok_2gram = get_ngrams(out_text, 2) intrep_2gram = intrep_frac(response_tok_2gram) # intrep_3gram response_tok_3gram = get_ngrams(out_text, 3) intrep_3gram = intrep_frac(response_tok_3gram) ''' intern_rep_reward = intrep_1gram self.intrep_scores += intern_rep_reward # bleu label_text = self.tokenizer.decode(self.labels, skip_special_tokens=True, clean_up_tokenization_spaces=False) current_bleu = bleu_rewarder(out_text_org, label_text) self.bleu_scores += current_bleu # fine-tuned GPT-based language model lm_tokenize_input = self.lm_tokenizer.tokenize(out_text) # lm_tensor_input = torch.tensor([lm_tokenizer.convert_tokens_to_ids(lm_tokenize_input)]).to(args.device) lm_tensor_input = torch.tensor([[self.special_tokens_ids[0]] + self.lm_tokenizer.convert_tokens_to_ids(lm_tokenize_input) + [self.special_tokens_ids[-1]]]).to(self.args.device) lm_loss = self.lm_model(lm_tensor_input, lm_labels=lm_tensor_input) lm_ppl = math.exp(lm_loss.item()) self.lm_ppl_scores += lm_ppl print('out_text:', out_text) print('current nli:', self.nli_scores) print('current score:', self.reward_scores / self.sample_num) print('current c_score_macro:', self.c_scores / self.sample_num) current_c_score_micro = (self.nli_scores[1] - self.nli_scores[0]) / sum(self.nli_scores) cn_res = nli_score[1] - nli_score[0] # cn: C_new (persona level) # C_new calculation if cn_res > 0: current_cn = 1 elif cn_res < 0: current_cn = -1 else: current_cn = 0 self.cnm += current_cn print('current c_new:', self.cnm / self.sample_num) print('current c_score_micro:', current_c_score_micro) print('current con_en:', self.con_en) print('current intrep score:', self.intrep_scores / self.sample_num) print('current BLEU:', self.bleu_scores / self.sample_num) print('current PPL:', self.lm_ppl_scores / self.sample_num) reply = {'text': out_text} return reply
def pred_attacks(df, model_path, args, output_clm, context='title', attacks_clm='premise_counter_premise_pairs', post_clm='post', rand_premise_idx=-1, rand_premises_clm='', baseline=False, random=False): model, tokenizer = interact.load_model(model_path) predictions = [] for row_idx, row in df.iterrows(): post_attacks = [] for premise_counter in row[attacks_clm]: if context == 'title+full_post': argument = [tokenizer.encode(row['title'])] + [ tokenizer.encode(sent) for sent in row[post_clm] ] else: argument = [tokenizer.encode( row['title'])] if context == 'title' else [ tokenizer.encode(sent) for sent in row[post_clm] ] #print([row['title']]+row[post_clm]) #print(premise_counter[0]) if baseline: argument = trim_argument(argument, 510, args) pred_counter = interact.sample_sequence(argument, [], tokenizer, model, args, baseline=True) elif random: if rand_premise_idx != None: weak_premise_encoded = [ tokenizer.encode( row[rand_premises_clm][rand_premise_idx]) ] else: weak_premise_encoded = [ tokenizer.encode(row[rand_premises_clm]) ] argument = trim_argument(argument, 510, args) pred_counter = interact.sample_sequence( argument, weak_premise_encoded, tokenizer, model, args) else: weak_premise_encoded = [ tokenizer.encode(premise) for premise in premise_counter[0] ] argument = trim_argument(argument, 510, args) pred_counter = interact.sample_sequence( argument, weak_premise_encoded, tokenizer, model, args) #remove quoted text #pred_counter = tokenizer.decode(pred_counter) #for p in premise_counter[0]: # pred_counter = pred_counter.replace(p.lower(), '') post_attacks.append([premise_counter[0], pred_counter]) predictions.append(post_attacks) df[output_clm] = predictions return df
def perform_attacks_hua_df(df, model_path, args, output_clm, context, weak_premise_clm, weak_premise_idx=None, baseline=False): model, tokenizer = interact.load_model(model_path) pred_attacks = [] for idx, row in df.iterrows(): if baseline: argument = [row['claim']] + row['post'] argument = [tokenizer.encode(sentence) for sentence in argument] argument = trim_argument(argument, 510, args) pred_counter = interact.sample_sequence(argument, [], tokenizer, model, args, baseline=True) pred_counter = pred_counter.replace(row['claim'].lower(), '') for sent in row['post']: pred_counter = pred_counter.replace(sent.lower(), '') else: if context == 'title+full_post': argument = [tokenizer.encode(row['claim'])] + [ tokenizer.encode(sent) for sent in row['post'] ] else: argument = [tokenizer.encode(row['claim']) ] if context == 'title' else [ tokenizer.encode(sent) for sent in row['post'] ] if weak_premise_idx != None: weak_premise_encoded = [ tokenizer.encode(premise) for premise in row[weak_premise_clm][weak_premise_idx] ] else: weak_premise_encoded = [ tokenizer.encode(premise) for premise in row[weak_premise_clm] ] if args.premise_extra: argument = trim_argument( argument, 510 - len([token for p in weak_premise_encoded for token in p]), args) else: argument = trim_argument(argument, 510, args) pred_counter = interact.sample_sequence(argument, weak_premise_encoded, tokenizer, model, args) #remove quoted text #pred_counter = tokenizer.decode(pred_counter) for p in row[weak_premise_clm]: pred_counter = pred_counter.replace(p.lower(), '') pred_attacks.append(pred_counter) df[output_clm] = pred_attacks return df