def createDataSet(ngrams, testDataProportion): ngramLength = len(ngrams[0]) a = SupervisedDataSet(ngramLength,1) for g in ngrams: a.addSample(g[0:ngramLength],g[ngramLength-1]) return a.splitWithProportion(testDataProportion)
'''
#step2： construct data set
# define that the input of data set is 784 demensions, output is 10 demension
# the first feature extraction
#DS = SupervisedDataSet(784,10)
# the first feature extraction
DS = SupervisedDataSet(28, 10)
#add sample data set
for i in range(len(traininglist)):
DS.addSample(traininglist[i], traininglabels[i])
X = DS['input']
Y = DS['target']
dataTrain, dataTest = DS.splitWithProportion(0.8)
xTrain, yTrain = dataTrain['input'], dataTrain['target']
xTest, yTest = dataTest['input'], dataTest['target']
#step3
# trainner use BP algorithm
verbose = True
trainer = BackpropTrainer(fnn,
dataTrain,
verbose=True,
learningrate=0.5,
lrdecay=0.5,
momentum=0)
trainer.trainUntilConvergence(DS, maxEpochs=10)
NetworkWriter.writeToFile(fnn, 'networkClassifier.txt')
# main execution
if __name__ == "__main__":
# initialize the used network and dataset
net = buildNetwork(num_chars * 2, hidden_layer_size, 1, bias=True)
ds = SupervisedDataSet(num_chars * 2, 1)
# readin names from files
result = readin('data/female.txt', 1) + readin('data/male.txt', 0)
shuffle(result)
#add entries to the dataset
for entry in result:
ds.addSample(entry[0], (entry[1]))
leftDs, rightDs = ds.splitWithProportion(training_set_percentage)
if (use_stored_weights == False):
print("successs rate over all names before training:")
print(validate_dataset(result, net))
print()
#train the network
print("starting to train the network")
trainer = BackpropTrainer(net, leftDs)
trainer.trainUntilConvergence(maxEpochs=max_epochs)
#store learned parameters to file
print("validate against the validation dataset")
print(validate_dataset(rightDs, net))
print()
