def main(args):
# Dataset functions
vocab = Vocabulary('./data/vocabulary.json', padding=args.padding)
kb_vocab = Vocabulary('./data/vocabulary.json', padding=4)
print('Loading datasets.')
training = Data(args.training_data, vocab, kb_vocab)
validation = Data(args.validation_data, vocab, kb_vocab)
training.load()
validation.load()
training.transform()
training.kb_out()
validation.transform()
validation.kb_out()
print('Datasets Loaded.')
print('Compiling Model.')
model = KVMMModel(pad_length=args.padding,
embedding_size=args.embedding,
vocab_size=vocab.size(),
batch_size=batch_size,
n_chars=vocab.size(),
n_labels=vocab.size(),
encoder_units=200,
decoder_units=200).to(device)
print(model)
model_optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss()
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
print_every = 100
start = time.time()
n_iters = 500000
iter = 0
while iter < n_iters:
training_data = training.generator(batch_size)
input_tensors = training_data[0][0]
target_tensors = training_data[1]
kbs = training_data[0][1]
iter += 1
loss = train(input_tensors, target_tensors, kbs, model,
model_optimizer, criterion, vocab, kb_vocab)
print_loss_total += loss
plot_loss_total += loss
if iter % print_every == 0:
validation_data = validation.generator(batch_size)
validation_inputs = validation_data[0][0]
validation_kbs = validation_data[0][1]
validation_targets = validation_data[1]
accuracy = evaluate(model, validation_inputs, validation_targets,
validation_kbs)
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('%s (%d %d%%) %.4f - val_accuracy %f' %
(timeSince(start, iter / n_iters), iter,
iter / n_iters * 100, print_loss_avg, accuracy))
torch.save(model.state_dict(), 'model_weights.pytorch')
def main(args):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
# Dataset functions
vocab = Vocabulary('./data/vocabulary.json', padding=args.padding)
vocab = Vocabulary('./data/vocabulary.json', padding=args.padding)
kb_vocab = Vocabulary('./data/vocabulary.json', padding=4)
print('Loading datasets.')
training = Data(args.training_data, vocab, kb_vocab)
validation = Data(args.validation_data, vocab, kb_vocab)
training.load()
validation.load()
training.transform()
training.kb_out()
validation.transform()
validation.kb_out()
print('Datasets Loaded.')
print('Compiling Model.')
model = memnn(pad_length=args.padding,
embedding_size=args.embedding,
vocab_size=vocab.size(),
batch_size=args.batch_size,
n_chars=vocab.size(),
n_labels=vocab.size(),
embedding_learnable=True,
encoder_units=200,
decoder_units=200,
trainable=True)
model.summary()
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
])
print('Model Compiled.')
print('Training. Ctrl+C to end early.')
try:
model.fit_generator(generator=training.generator(args.batch_size),
steps_per_epoch=300,
validation_data=validation.generator(
args.batch_size),
validation_steps=10,
workers=1,
verbose=1,
epochs=args.epochs)
except KeyboardInterrupt as e:
print('Model training stopped early.')
model.save_weights("model_weights_nkbb.hdf5")
print('Model training complete.')
predicted.append(run_example(model, kbs, vocabulary, example))
outdf['input'].append(example)
outdf['output'].append(predicted[-1])
return predicted
if __name__ == "__main__":
no_unks = False
dialog_type = "schedule"
file_name = "-" + dialog_type + "-epoch-80" # "-2409"
pad_length = 20
df = pd.read_csv("../data/test_data - " + dialog_type + ".csv",
delimiter=";")
inputs = list(df["input"])
outputs = list(df["output"])
vocab = Vocabulary('../data/vocabulary-full.json', padding=pad_length)
kb_vocabulary = Vocabulary('../data/vocabulary-full.json', padding=4)
model = memnn(pad_length=20,
embedding_size=200,
batch_size=1,
vocab_size=vocab.size(),
n_chars=vocab.size(),
n_labels=vocab.size(),
embedding_learnable=True,
encoder_units=200,
decoder_units=200)
weights_file = "../weights/model_weights_nkbb" + file_name + ".hdf5"
model.load_weights(weights_file, by_name=True)
def main_examples(dialog_type,
underscore,
kb,
iteration=500000,
delimiter=",",
final_folder="",
test_mode=""):
if final_folder == "":
final_folder = dialog_type[3:]
try:
df = pd.read_csv("data/test_data" + underscore + test_mode +
dialog_type + kb + ".csv",
encoding="ISO-8859-1",
delimiter=delimiter)
except FileNotFoundError:
df = pd.read_csv("../data/test_data" + underscore + test_mode +
dialog_type + kb + ".csv",
encoding="ISO-8859-1",
delimiter=delimiter)
inputs = list(df["input"])
outputs = list(df["output"])
actual_clusters = list(df["actual_cluster"])
try:
vocab = Vocabulary('data/vocabulary-train' + dialog_type +
' - perfect decomposition - preprocessedFinal.json',
padding=pad_length)
kb_vocabulary = Vocabulary(
'data/vocabulary-train' + dialog_type +
' - perfect decomposition - preprocessedFinal.json',
padding=4)
except FileNotFoundError:
vocab = Vocabulary('../data/vocabulary-train' + dialog_type +
' - perfect decomposition - preprocessedFinal.json',
padding=pad_length)
kb_vocabulary = Vocabulary(
'../data/vocabulary-train' + dialog_type +
' - perfect decomposition - preprocessedFinal.json',
padding=4)
model = KVMMModel(pad_length=20,
embedding_size=200,
batch_size=1,
vocab_size=vocab.size(),
n_chars=vocab.size(),
n_labels=vocab.size(),
encoder_units=200,
decoder_units=200).to(device)
weights_file = "final-" + final_folder + "/model_weights_" + dialog_type[
3:] + "_iter_" + str(iteration) + ".pytorch"
try:
try:
model.load_state_dict(torch.load(weights_file))
except RuntimeError:
model.load_state_dict(torch.load(weights_file, map_location='cpu'))
except FileNotFoundError:
try:
model.load_state_dict(torch.load("../" + weights_file))
except RuntimeError:
model.load_state_dict(
torch.load("../" + weights_file, map_location='cpu'))
kbfile = "data/normalised_kbtuples.csv"
try:
df = pd.read_csv(kbfile)
except FileNotFoundError:
df = pd.read_csv("../" + kbfile)
kbs = list(df["subject"] + " " + df["relation"])
# print(kbs[:3])
kbs = np.array(list(map(kb_vocabulary.string_to_int, kbs)))
kbs = np.repeat(kbs[np.newaxis, :, :], 1, axis=0)
data = run_examples(model, kbs, vocab, inputs, outputs)
df = pd.DataFrame(
columns=["inputs", "outputs", "prediction", "actual_cluster"])
d = {'outputs': [], 'inputs': [], 'predictions': [], "actual_cluster": []}
for index, (i, o, p) in enumerate(zip(inputs, outputs, data)):
d["outputs"].append(str(o))
d["inputs"].append(str(i))
d["predictions"].append(str(p))
d["actual_cluster"].append(actual_clusters[index])
df = pd.DataFrame(d)
return df
"""
All Accuracy
https://github.com/rasmusbergpalm/normalization/blob/master/train.py#L10
"""
return K.mean(
K.all(K.equal(K.max(y_true, axis=-1),
K.cast(K.argmax(y_pred, axis=-1), K.floatx())),
axis=1))
# Configuration
training_data = './training.csv'
validation_data = './validation.csv'
# Dataset functions
input_vocab = Vocabulary('./human_vocab.json', padding=config.padding)
output_vocab = Vocabulary('./machine_vocab.json', padding=config.padding)
print('Loading datasets...')
training = Data(training_data, input_vocab, output_vocab)
validation = Data(validation_data, input_vocab, output_vocab)
training.load()
validation.load()
training.transform()
validation.transform()
print('Datasets Loaded.')
def build_models(pad_length=config.padding,
def main(args):
# Dataset functions
vocab = Vocabulary(args.vocabulary_data, padding=args.padding)
kb_vocab = Vocabulary(args.vocabulary_data, padding=4) # 7
print('Loading datasets.')
# Callback.__init__(self)
if args.training_data.find("schedule") != -1:
train_file_name = "schedule"
elif args.training_data.find("navigate") != -1:
train_file_name = "navigate"
elif args.training_data.find("weather") != -1:
train_file_name = "weather"
elif args.training_data.find("ubuntu") != -1:
train_file_name = "ubuntu"
elif args.training_data.find("original") != -1:
train_file_name = "original"
else:
train_file_name = "unknown"
if args.save_path == "default":
args.save_path = "weights/model_weights_" + train_file_name
training = Data(args.training_data, vocab, kb_vocab,
args.generated_training_data)
validation = Data(args.validation_data, vocab, kb_vocab)
training.load()
validation.load()
training.transform()
training.kb_out()
validation.transform()
validation.kb_out()
print('Datasets Loaded.')
print('Compiling Model.')
model = KVMMModel(pad_length=args.padding,
embedding_size=args.embedding,
vocab_size=vocab.size(),
batch_size=batch_size,
n_chars=vocab.size(),
n_labels=vocab.size(),
encoder_units=200,
decoder_units=200).to(device)
print(model)
# Training using Adam Optimizer
model_optimizer = optim.Adam(model.parameters(), lr=0.001)
# Training using cross-entropy loss
criterion = nn.CrossEntropyLoss()
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
print_every = 100
save_every = 10000
start = time.time()
n_iters = 500000
iter = 0
while iter < n_iters:
training_data = training.generator(batch_size)
input_tensors = training_data[0][0]
target_tensors = training_data[1]
kbs = training_data[0][1]
iter += 1
loss = train(input_tensors, target_tensors, kbs, model,
model_optimizer, criterion, vocab, kb_vocab)
print_loss_total += loss
plot_loss_total += loss
if iter % print_every == 0:
validation_data = validation.generator(batch_size)
validation_inputs = validation_data[0][0]
validation_kbs = validation_data[0][1]
validation_targets = validation_data[1]
accuracy = evaluate(model, validation_inputs, validation_targets,
validation_kbs)
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('%s (%d %d%%) %.4f - val_accuracy %f' %
(timeSince(start, iter / n_iters), iter,
iter / n_iters * 100, print_loss_avg, accuracy))
if iter % save_every == 0:
torch.save(model.state_dict(),
args.save_path + "_iter_" + str(iter) + ".pytorch")
if len(sort)>k:
ndf["u"+str(k+1)].append(sort[k][0])
else:
ndf["u" + str(k + 1)].append("None")
ndf = pd.DataFrame(ndf)
ndf.to_csv("cl_reranked_with_lstmfk_kb_t1.csv")
print("Saved to file")
#tmp.append(res[0])
#reranked.append(tmp)
if __name__ == "__main__":
pad_length = 20
df = pd.read_csv("../data/ranked_responsesfk_kb.csv",encoding="latin1")
vocab = Vocabulary('../fkdata/vocabl_fk.json', padding=pad_length)
model = dualenc(pad_length=20,
embedding_size=100,
batch_size=1,
vocab_size=vocab.size(),
n_chars=vocab.size(),
n_labels=vocab.size(),
encoder_units=256,
decoder_units=256)
weights_file = "../cl_modellstm_weightsfk_kb.hdf5"
model.load_weights(weights_file, by_name=True)
run_examples(model,vocab,df)
#print(data[:3])
'''ndf = {"input": [], "response": [], "u1": [], "u2": [], "u3": [], "u4": [], "u5": [], "u6": [], "u7": [], "u8": [],"u9": [], "u10": []}
ndf = {"input": [], "response": [], "ranked": []}
for i,(inp,out) in enumerate(zip(df["input"],df["response"])):
required=False, default=20, type=int)
named_args.add_argument('-t', '--training-data', metavar='|',
help="""Location of training data""",
required=False, default='./data/train_data.csv')
named_args.add_argument('-v', '--validation-data', metavar='|',
help="""Location of validation data""",
required=False, default='./data/val_data.csv')
named_args.add_argument('-b', '--batch-size', metavar='|',
help="""Location of validation data""",
required=False, default=100, type=int)
args = parser.parse_args()
print(args)
vocab = Vocabulary('./data/vocabulary.json',
padding=args.padding)
model = memnn(pad_length=args.padding,
embedding_size=args.embedding,
vocab_size=vocab.size(),
batch_size=args.batch_size,
n_chars=vocab.size(),
n_labels=vocab.size(),
embedding_learnable=True,
encoder_units=200,
decoder_units=200,trainable=True)
model.load_weights("model_weights_nkbb.hdf5")
test_data = load_test_data()
model.predict(test_data) # ("How is the weather in San Francisco today?")