def get_persona_faiss_selected(args, tokenizer):
""" Prepare the dataset for training and evaluation """
personachat = get_dataset_with_no_tokenizer(tokenizer, args.dataset_path,
args.dataset_cache)
logger.info("Build inputs and labels")
datasets = {"train": defaultdict(list), "valid": defaultdict(list)}
persona_faiss_selected = []
history_faiss_selected = []
persona_faiss_index = []
history_faiss_index = []
persona_complete = parse_data('./Dataset/train_self_original.txt')
persona_complete = persona_complete[:20]
for dataset_name, dataset in personachat.items():
num_candidates = len(dataset[0]["utterances"][0]["candidates"])
if args.num_candidates > 0 and dataset_name == 'train':
num_candidates = min(args.num_candidates, num_candidates)
for dialog in dataset:
#persona = dialog["personality"].copy()
persona = dialog["persona_info"]
#persona2 = dialog["persona_info2"].copy()
#persona_selected = faiss(replyanddialog)
#index: all persona1 sentences or all personalities
#model1 = SentenceTransformer('bert-large-nli-mean-tokens')
#model = SentenceTransformer('sentence-transformers/distiluse-base-multilingual-cased')
model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')
embeddings_persona = model.encode(persona_complete,
show_progress_bar=True)
#data_train list of set of list of all personalities (not duplicated)
# Step 1: Change data type
#embeddings_persona = np.array([embedding for embedding in embeddings_persona]).astype("float32")
# Step 2: Instantiate the index
index = faiss.IndexFlatL2(embeddings_persona.shape[1])
# Step 3: Pass the index to IndexIDMap
index = faiss.IndexIDMap(index)
# Step 4: Add vectors and their IDs
index.add_with_ids(
embeddings_persona,
np.array(list(range(0, embeddings_persona.shape[0]))))
for _ in range(args.personality_permutations):
for utterance in dialog["utterances"]:
history = utterance["history"][-(2 * args.max_history +
1):]
for j, candidate in enumerate(
utterance["candidates"][-num_candidates:]):
history_encoded = model.encode(history,
show_progress_bar=True)
D, I = index.search(np.array(history_encoded), k=5)
history_faiss_selected.append(history)
persona_faiss_selected.append(
persona_complete[I[0][1]])
#persona = [persona[-1]] + persona[:-1] # permuted personalities
return persona_faiss_selected
def get_data_loaders(args, tokenizer):
""" Prepare the dataset for training and evaluation """
#Dataset is charged in variable. It is already tokenidez
personachat = get_dataset(tokenizer, args.dataset_path, args.dataset_cache)
personachat_raw = get_dataset_with_no_tokenizer(tokenizer,
args.dataset_path,
args.dataset_cache)
with open('data_faiss_fase1_opcion2_chatbot_personalidad_reducida.pkl',
'rb') as f:
persona_selected_list = pickle.load(f)
count_persona = 0
#personachat_personalities = get_dataset_personalities(tokenizer,args.dataset_path,args.dataset_cache)
logger.info("Build inputs and labels")
#Dictionary inside dictionary
datasets = {"train": defaultdict(list), "valid": defaultdict(list)}
persona_info_raw = {"train": [], "valid": []}
personas = {
"persona_tokens": personachat,
"persona_history": personachat_raw
}
for dataset_name, dataset in personachat.items():
for persona_index, personaset in personachat_raw.items():
for dialog in personaset:
persona_info_raw[persona_index].append(
dialog["persona_info"].copy())
num_candidates = len(dataset[0]["utterances"][0]["candidates"])
if args.num_candidates > 0 and dataset_name == 'train':
num_candidates = min(args.num_candidates, num_candidates)
count = 0
for dialog in dataset:
#persona1_raw = persona_info_raw[dataset_name][count].copy()
#persona1 = dialog["persona_info"].copy()
#persona_selected = get_persona_faiss_selected(args,tokenizer)
#persona2 = dialog["persona_info2"].copy()
#persona_selected = faiss(replyanddialog)
#index: all persona1 sentences or all personalities
#for _ in range(args.personality_permutations):
for utterance in dialog["utterances"]:
history = utterance["history"][-(2 * args.max_history + 1):]
for j, candidate in enumerate(
utterance["candidates"][-num_candidates:]):
lm_labels = bool(j == num_candidates - 1)
#D, I = index.search(np.array([history]), k=10)
#print(f'L2 distance: {D.flatten().tolist()}\n\nMAG paper IDs: {I.flatten().tolist()}')
persona_selected = persona_selected_list[count_persona]
persona_selected_tokenized = tokenize(
tokenizer, persona_selected)
instance = build_input_from_segments(
persona_selected_tokenized, history, candidate,
tokenizer, lm_labels)
for input_name, input_array in instance.items():
datasets[dataset_name][input_name].append(input_array)
count_persona = count_persona + 1
datasets[dataset_name]["mc_labels"].append(num_candidates - 1)
datasets[dataset_name]["n_candidates"] = num_candidates
#count_persona = count_persona + 1
#persona1 = [persona1[-1]] + persona1[:-1] # permuted personalities
#persona2 = [persona2[-1]] + persona2[:-1] # permuted personalities
count = count + 1
logger.info("Pad inputs and convert to Tensor")
tensor_datasets = {"train": [], "valid": []}
for dataset_name, dataset in datasets.items():
dataset = pad_dataset(dataset,
padding=tokenizer.convert_tokens_to_ids(
SPECIAL_TOKENS_2[-1]))
for input_name in MODEL_INPUTS:
tensor = torch.tensor(dataset[input_name])
if input_name != "mc_labels":
tensor = tensor.view((-1,
datasets[dataset_name]["n_candidates"]) +
tensor.shape[1:])
tensor_datasets[dataset_name].append(tensor)
logger.info("Build train and validation dataloaders")
train_dataset, valid_dataset = TensorDataset(
*tensor_datasets["train"]), TensorDataset(*tensor_datasets["valid"])
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset) if args.distributed else None
valid_sampler = torch.utils.data.distributed.DistributedSampler(
valid_dataset) if args.distributed else None
train_loader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size,
shuffle=(not args.distributed))
valid_loader = DataLoader(valid_dataset,
sampler=valid_sampler,
batch_size=args.valid_batch_size,
shuffle=False)
logger.info("Train dataset (Batch, Candidates, Seq length): {}".format(
train_dataset.tensors[0].shape))
logger.info("Valid dataset (Batch, Candidates, Seq length): {}".format(
valid_dataset.tensors[0].shape))
return train_loader, valid_loader, train_sampler, valid_sampler
def get_persona_faiss_selected(args):
""" Prepare the dataset for training and evaluation """
tokenizer = ""
personachat = get_dataset_with_no_tokenizer(tokenizer, args.dataset_path,
args.dataset_cache)
logger.info("Build inputs and labels")
datasets = {"train": defaultdict(list), "valid": defaultdict(list)}
persona_faiss_selected = []
history_faiss_selected = []
persona_faiss_index = []
history_faiss_index = []
persona_complete = parse_data('./Dataset/train_self_original.txt')
model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')
count = 0
#embeddings_persona = model.encode(persona_complete, show_progress_bar=False)
# Step 1: Change data type
#embeddings_persona = np.array([embedding for embedding in embeddings_persona]).astype("float32")
# Step 2: Instantiate the index
#index = faiss.IndexFlatL2(embeddings_persona.shape[1])
# Step 3: Pass the index to IndexIDMap
#index = faiss.IndexIDMap(index)
# Step 4: Add vectors and their IDs
#index.add_with_ids(embeddings_persona, np.array(list(range(0,embeddings_persona.shape[0]))))
for dataset_name, dataset in personachat.items():
num_candidates = len(dataset[0]["utterances"][0]["candidates"])
if args.num_candidates > 0 and dataset_name == 'train':
num_candidates = min(args.num_candidates, num_candidates)
for dialog in tqdm(dataset):
#persona = dialog["personality"].copy()
persona = dialog["persona_info"]
#persona2 = dialog["persona_info2"].copy()
#persona_selected = faiss(replyanddialog)
#index: all persona1 sentences or all personalities
#model1 = SentenceTransformer('bert-large-nli-mean-tokens')
#model = SentenceTransformer('sentence-transformers/distiluse-base-multilingual-cased')
embeddings_persona = model.encode(persona, show_progress_bar=False)
# Step 1: Change data type
embeddings_persona = np.array([
embedding for embedding in embeddings_persona
]).astype("float32")
# Step 2: Instantiate the index
index = faiss.IndexFlatL2(embeddings_persona.shape[1])
# Step 3: Pass the index to IndexIDMap
index = faiss.IndexIDMap(index)
# Step 4: Add vectors and their IDs
index.add_with_ids(
embeddings_persona,
np.array(list(range(0, embeddings_persona.shape[0]))))
#if count==4:
# break
count = count + 1
#data_train list of set of list of all personalities (not duplicated)
for _ in range(args.personality_permutations):
for utterance in dialog["utterances"]:
history = utterance["history"][-(2 * args.max_history +
1):]
for j, candidate in enumerate(
utterance["candidates"][-num_candidates:]):
#historysplitted = " ".join(history)
history_encoded_user = model.encode(
[history[-1]], show_progress_bar=False)
D, I = index.search(np.array(history_encoded_user),
k=len(persona))
history_faiss_selected.append(history)
index_to_be_removed = I[0][0]
persona2 = persona[:index_to_be_removed] + persona[
index_to_be_removed + 1:]
embeddings_persona2 = model.encode(
persona2, show_progress_bar=False)
# Step 1: Change data type
embeddings_persona2 = np.array([
embedding for embedding in embeddings_persona2
]).astype("float32")
# Step 2: Instantiate the index
index2 = faiss.IndexFlatL2(
embeddings_persona2.shape[1])
# Step 3: Pass the index to IndexIDMap
index2 = faiss.IndexIDMap(index2)
# Step 4: Add vectors and their IDs
index2.add_with_ids(
embeddings_persona2,
np.array(
list(range(0, embeddings_persona2.shape[0]))))
persona_faiss_index.append([I[0][1:-1].tolist()])
persona_list = []
for i in I[0][1:-1]:
persona_list.append(persona[i])
if len(history) > 1:
history_encoded_chatbot = model.encode(
[history[-2]], show_progress_bar=False)
else:
history_encoded_chatbot = model.encode(
[history[-1]], show_progress_bar=False)
T, J = index2.search(np.array(history_encoded_user),
k=len(persona2))
persona_faiss_selected.append(persona2[J[0][0]])
#persona = [persona[-1]] + persona[:-1] # permuted personalities
#break
return persona_faiss_selected, persona_faiss_index