(datasetName, dataColumns),
]
# :: Prepares the dataset to be used with the LSTM-network. Creates and stores cPickle files in the pkl/ folder ::
pickleFile = perpareDataset(embeddingsPath, datasetFiles)
######################################################
#
# The training of the network starts here
#
######################################################
#Load the embeddings and the dataset
embeddings, word2Idx, datasets = loadDatasetPickle(pickleFile)
data = datasets[datasetName]
print("Dataset:", datasetName)
print(data['mappings'].keys())
print("Label key: ", labelKey)
print("Train Sentences:", len(data['trainMatrix']))
print("Dev Sentences:", len(data['devMatrix']))
print("Test Sentences:", len(data['testMatrix']))
model = BiLSTM(params)
model.setMappings(embeddings, data['mappings'])
model.setTrainDataset(data, labelKey)
model.verboseBuild = True
model.modelSavePath = "models/%s/%s/[DevScore]_[TestScore]_[Epoch].h5" % (
datasetName, labelKey) #Enable this line to save the model to the disk
model.evaluate(50)
f1_file.write('\n____________________________\n')
f1_file.write('function: ' + f)
f1_file.write('\n____________________________\n')
f1_file.write('Units,1,2,3,4,5,max,min,average,standard deviation'
) # average results after
f1_file.write('\n' + str(size[0]) + ',')
params['LSTM-Size'] = size
params['scores'] = []
params["dev_scores"] = []
for i in range(num_repeats):
model = BiLSTM(params)
model.setMappings(embeddings, data['mappings'])
model.setTrainDataset(data, labelKey)
model.verboseBuild = True
model.evaluate(epochs)
max = np.max(params['scores'])
min = np.min(params['scores'])
average = np.average(params['scores'])
stddev = np.std(params['scores']) * 100
print("length of scores ", len(params['scores']))
params['scores'] += [max, min, average, stddev]
for s in params['scores']:
f1_file.write(str(s) + ',')
f1_file.write("\t" + ",".join([str(x) for x in params["dev_scores"]]))
f1_file.flush()
os.fsync(f1_file)
