def plotLSTMresult(experiment, window, xaxis=None, label=None):
(iteration, truth, predictions, train, params) = loadExperiment(experiment)
(square_deviation, x) = computeAccuracy(predictions, truth, iteration)
if xaxis is not None:
x = xaxis
plotAccuracy((square_deviation, x), truth, train=train, window=window,
label=label, params=params)
def load_plot_return_errors(forgettingFactor,
algorithmname,
nHidden,
dataset='nyc_taxi',
skipTrain=500,
window=100):
forgettingFactor = str(forgettingFactor)
nHidden = str(nHidden)
filepath = './prediction/' + dataset + '_FF' + forgettingFactor + algorithmname + nHidden + '_pred.csv'
if os.path.isfile(filepath):
(elmTruth, elmPrediction) = loadExperimentResult(filepath)
squareDeviation = computeSquareDeviation(elmPrediction, elmTruth)
squareDeviation[:skipTrain] = None
nrse_temp = plotAccuracy((squareDeviation, xaxisDate),
elmTruth,
window=window,
errorType='square_deviation',
label=algorithmname + '_#HN=' + nHidden)
nrse_temp = np.sqrt(np.nanmean(nrse_temp)) / np.nanstd(elmTruth)
nrse_temp = min(1000, nrse_temp)
mape_temp = computeAltMAPE(elmTruth, elmPrediction, skipTrain)
mape_temp = min(1000, mape_temp)
else:
nrse_temp = 0
mape_temp = 0
return [nrse_temp, mape_temp]
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window1001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-1000')
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window3001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-3000')
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window5001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-5000')
# load TM prediction
filePath = './data/' + 'nyc_taxi' + '.csv'
data = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['datetime', 'value', 'timeofday', 'dayofweek'])
dataSet = 'nyc_taxi_perturb'
filePath = './prediction/' + dataSet + '_TM_pred.csv'
predData_TM = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5'])
truth = predData_TM['value']
predData_TM_five_step = np.roll(predData_TM['prediction5'], 5)
iteration = predData_TM.index
(square_deviation, x) = computeAccuracy(predData_TM_five_step, truth, iteration)
square_deviation[:5000] = None
x = pd.to_datetime(data['datetime'])
plotAccuracy((square_deviation, x),
truth,
window=window,
label='TM')
plt.legend()
plt.savefig(figPath + 'continuous_perturb.pdf')
truth = np.roll(actual_data, -5)
from nupic.encoders.scalar import ScalarEncoder as NupicScalarEncoder
encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True)
from plot import computeLikelihood, plotAccuracy
bucketIndex2 = []
negLL = []
minProb = 0.0001
for i in xrange(len(truth)):
bucketIndex2.append(np.where(encoder.encode(truth[i]))[0])
outOfBucketProb = 1 - sum(predictions[i,:])
prob = predictions[i, bucketIndex2[i]]
if prob == 0:
prob = outOfBucketProb
if prob < minProb:
prob = minProb
negLL.append( -np.log(prob))
negLL = computeLikelihood(predictions, truth, encoder)
negLL[:5000] = np.nan
x = range(len(negLL))
plt.figure()
plotAccuracy((negLL, x), truth, window=480, errorType='negLL')
np.save('./result/'+dataSet+classifierType+'TMprediction.npy', predictions)
np.save('./result/'+dataSet+classifierType+'TMtruth.npy', truth)
os.path.join("tm/results", "high-order-noise", "inject_noise_after0.0",
"0.log")
]
for experiment in experiments:
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
plotAccuracy((accuracy, x),
data['trains'],
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
pyplot.xlim([10500, 14500])
pyplot.xlabel('Number of elements seen')
pyplot.ylabel(' Accuracy ')
pyplot.legend(['LSTM', 'HTM'])
pyplot.savefig('./result/temporal_noise_train_with_noise.pdf')
experiments = [
os.path.join("lstm/results", "high-order-noise",
"inject_noise_after12000.0", "0.log"),
os.path.join("tm/results", "high-order-noise",
"inject_noise_after12000.0", "0.log")
nrmseLSTM1000Perturb = expResultPerturb1000.error
nrmseLSTM3000Perturb = expResultPerturb3000.error
nrmseLSTM6000Perturb = expResultPerturb6000.error
nrmseLSTMOnlinePerturb = expResultPerturbOnline.error
mapeLSTM1000Perturb = np.abs(expResultPerturb1000.truth - expResultPerturb1000.predictions)
mapeLSTM3000Perturb = np.abs(expResultPerturb3000.truth - expResultPerturb3000.predictions)
mapeLSTM6000Perturb = np.abs(expResultPerturb6000.truth - expResultPerturb6000.predictions)
mapeLSTMOnlinePerturb = np.abs(expResultPerturbOnline.truth - expResultPerturbOnline.predictions)
plt.figure()
window = 400
plotAccuracy(
(mapeLSTM1000Perturb, xaxisDatetime),
truthLSTM3000Perturb,
window=window,
errorType="mape",
label="LSTM1000",
train=expResultPerturb1000.train,
)
plotAccuracy(
(mapeLSTM3000Perturb, xaxisDatetime), truthLSTM3000Perturb, window=window, errorType="mape", label="LSTM3000"
)
plotAccuracy(
(mapeLSTM6000Perturb, xaxisDatetime), truthLSTM6000Perturb, window=window, errorType="mape", label="LSTM6000"
)
plotAccuracy(
(mapeLSTMOnlinePerturb, xaxisDatetime),
truth_LSTMonline_perturb,
experiments.append(os.path.join("elm/results",
"high-order-distributed-random-perturbed",
"seed0.0",
"0.log"))
for experiment in experiments:
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
# perturbAt = data['sequenceCounter'][10000]
plotAccuracy((accuracy, x),
data['trains'],
window=200,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
# plt.xlim([1200, 1750])
plt.xlabel('# of sequences seen')
plt.axvline(x=10000, color='k')
plt.legend(['HTM', 'LSTM-1000', 'LSTM-9000', 'Online-ELM'], loc=4)
plt.savefig('./result/model_performance_high_order_prediction.pdf')
plt.figure(1)
for length in lengths:
expResult = expResults[length]
accuracy = expResult["accuracy"]
numIteration = expResult["numIteration"]
numSequences = expResult["numSequences"]
movingData = movingAverage(accuracy, min(len(accuracy), 100))
numSequenceRequired.append(
numSequences[np.where(np.array(movingData) >= 0.999)[0][1]])
numIterationRequired.append(
numIteration[np.where(np.array(movingData) >= 0.999)[0][1]])
plt.figure(1)
plotAccuracy((accuracy, numSequences),
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
plt.xlabel('# of sequences seen')
plt.figure(2)
plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
window=1000,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
plt.xlabel('# of sequences seen')
for fig in [1, 2]:
plt.figure(fig)
(NRMSE_LSTM6000, expResult_LSTM6000) = \
plotLSTMresult('results/nyc_taxi_experiment_continuous/learning_window6001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-6000')
dataSet = 'nyc_taxi'
filePath = './prediction/' + dataSet + '_TM_pred.csv'
predData_TM = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5'])
tm_truth = np.roll(predData_TM['value'], -5)
predData_TM_five_step = np.array(predData_TM['prediction5'])
square_deviation = computeSquareDeviation(predData_TM_five_step, tm_truth)
square_deviation[:6000] = None
NRMSE_TM = plotAccuracy((square_deviation, xaxis_datetime),
tm_truth,
window=window,
errorType='square_deviation',
label='TM')
filePath = './prediction/' + dataSet + '_ESN_pred.csv'
predData_ESN = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5'])
esn_truth = np.roll(predData_ESN['value'], -5)
predData_ESN_five_step = np.array(predData_ESN['prediction5'])
square_deviation = computeSquareDeviation(predData_ESN_five_step, esn_truth)
square_deviation[:6000] = None
NRMSE_ESN = plotAccuracy((square_deviation, xaxis_datetime),
tm_truth,
window=window,
experiments = [
os.path.join("lstm/results", "high-order-noise", "inject_noise_after0.0", "0.log"),
os.path.join("tm/results", "high-order-noise", "inject_noise_after0.0", "0.log"),
]
for experiment in experiments:
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(
data["predictions"], data["truths"], data["iterations"], resets=data["resets"], randoms=data["randoms"]
)
plotAccuracy(
(accuracy, x),
data["trains"],
window=100,
type=type,
label="NoiseExperiment",
hideTraining=True,
lineSize=1.0,
)
pyplot.xlim([10500, 14500])
pyplot.xlabel("Number of elements seen")
pyplot.ylabel(" Accuracy ")
pyplot.legend(["LSTM", "HTM"])
pyplot.savefig("./result/temporal_noise_train_with_noise.pdf")
experiments = [
os.path.join("lstm/results", "high-order-noise", "inject_noise_after12000.0", "0.log"),
os.path.join("tm/results", "high-order-noise", "inject_noise_after12000.0", "0.log"),
]
expResultPerturbOnline = ExperimentResult(
'results/nyc_taxi_experiment_continuous_perturb_online/learning_window'+str(200.0)+'/')
nrmseLSTM1000Perturb = expResultPerturb1000.error
nrmseLSTM3000Perturb = expResultPerturb3000.error
nrmseLSTM6000Perturb = expResultPerturb6000.error
nrmseLSTMOnlinePerturb = expResultPerturbOnline.error
mapeLSTM1000Perturb = np.abs(expResultPerturb1000.truth - expResultPerturb1000.predictions)
mapeLSTM3000Perturb = np.abs(expResultPerturb3000.truth - expResultPerturb3000.predictions)
mapeLSTM6000Perturb = np.abs(expResultPerturb6000.truth - expResultPerturb6000.predictions)
mapeLSTMOnlinePerturb = np.abs(expResultPerturbOnline.truth - expResultPerturbOnline.predictions)
plt.figure()
window = 400
plotAccuracy((mapeLSTM1000Perturb, xaxisDatetime), truthLSTM3000Perturb,
window=window, errorType='mape', label='LSTM1000', train=expResultPerturb1000.train)
plotAccuracy((mapeLSTM3000Perturb, xaxisDatetime), truthLSTM3000Perturb,
window=window, errorType='mape', label='LSTM3000')
plotAccuracy((mapeLSTM6000Perturb, xaxisDatetime), truthLSTM6000Perturb,
window=window, errorType='mape', label='LSTM6000')
plotAccuracy((mapeLSTMOnlinePerturb, xaxisDatetime), truth_LSTMonline_perturb,
window=window, errorType='mape', label='LSTM-online')
plotAccuracy((mapeTMPerturb, xaxisDatetime), tmTruthPerturb,
window=window, errorType='mape', label='TM')
plt.axvline(xaxisDatetime[13152], color='black', linestyle='--')
plt.xlim([xaxisDatetime[13000], xaxisDatetime[15000]])
predData_TM = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5'])
predData_TM_five_step = np.array(predData_TM['prediction5'])
iteration = predData_TM.index
tm_pred_perturb_truth = np.roll(predData_TM['value'], -5)
tm_pred_perturb = np.array(predData_TM['prediction5'])
negLL_tm_perturb = computeLikelihood(tm_prediction_perturb, tm_truth_perturb, encoder)
negLL_tm_perturb[:6000] = None
nrmse_tm_perturb = computeSquareDeviation(tm_pred_perturb, tm_pred_perturb_truth)
mape_tm_perturb = np.abs(tm_pred_perturb - tm_pred_perturb_truth)
plt.figure()
window = 480
plotAccuracy((negLL_LSTM3000_perturb, xaxis_datetime), truth_LSTM3000_perturb,
window=window, errorType='negLL', label='LSTM3000', train=expResult_perturb.train)
# plotAccuracy((negLL_LSTM3000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM')
plotAccuracy((negLL_LSTM6000_perturb, xaxis_datetime), truth_LSTM6000_perturb, window=window, errorType='negLL', label='LSTM6000')
# plotAccuracy((negLL_LSTM6000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM')
plotAccuracy((negLL_tm_perturb, xaxis_datetime), tm_truth_perturb, window=window, errorType='negLL', label='TM')
plt.axvline(xaxis_datetime[13152], color='black', linestyle='--')
plt.xlim([xaxis_datetime[13000], xaxis_datetime[15000]])
plt.legend()
plt.ylim([1.2, 2.3])
plt.ylabel('Negative Log-Likelihood')
plt.savefig(figPath + 'example_perturbation.pdf')
expResult_perturb_1000 = ExperimentResult(
# nrmseTM = plotAccuracy((squareDeviation, xaxisDate),
# tmTruth,
# window=window,
# errorType='square_deviation',
# label='TM', skipRecordNum=skipTrain)
(esnTruth,
esnPrediction) = loadExperimentResult('./prediction/' + dataSet +
'_esn_online_pred.csv')
squareDeviation = computeSquareDeviation(esnPrediction, esnTruth)
squareDeviation[:skipTrain] = None
nrmseESN = plotAccuracy((squareDeviation, xaxisDate),
tmTruth,
window=window,
errorType='square_deviation',
label='ESN')
#
# (knnTruth, knnPrediction) = loadExperimentResult('./prediction/' + dataSet + '_plainKNN_pred.csv')
#
# squareDeviation = computeSquareDeviation(knnPrediction, knnTruth)
# squareDeviation[:skipTrain] = None
#
# nrmseKNN = plotAccuracy((squareDeviation, xaxisDate),
# tmTruth,
# window=window,
# errorType='square_deviation',
# label='KNN')
#
nrmseLSTM6000Perturb = expResultPerturb6000.error
nrmseLSTMOnlinePerturb = expResultPerturbOnline.error
mapeLSTM1000Perturb = np.abs(expResultPerturb1000.truth -
expResultPerturb1000.predictions)
mapeLSTM3000Perturb = np.abs(expResultPerturb3000.truth -
expResultPerturb3000.predictions)
mapeLSTM6000Perturb = np.abs(expResultPerturb6000.truth -
expResultPerturb6000.predictions)
mapeLSTMOnlinePerturb = np.abs(expResultPerturbOnline.truth -
expResultPerturbOnline.predictions)
plt.figure()
window = 400
plotAccuracy((mapeLSTM1000Perturb, xaxisDatetime),
truthLSTM3000Perturb,
window=window,
errorType='mape',
label='LSTM1000',
train=expResultPerturb1000.train)
plotAccuracy((mapeLSTM3000Perturb, xaxisDatetime),
truthLSTM3000Perturb,
window=window,
errorType='mape',
label='LSTM3000')
plotAccuracy((mapeLSTM6000Perturb, xaxisDatetime),
truthLSTM6000Perturb,
window=window,
errorType='mape',
label='LSTM6000')
plt.figure(1)
for length in lengths:
expResult = expResults[length]
accuracy = expResult["accuracy"]
numIteration = expResult["numIteration"]
numSequences = expResult["numSequences"]
movingData = movingAverage(accuracy, min(len(accuracy), 100))
numSequenceRequired.append(
numSequences[np.where(np.array(movingData) >= 0.999)[0][1]])
numIterationRequired.append(
numIteration[np.where(np.array(movingData) >= 0.999)[0][1]])
plt.figure(1)
plotAccuracy((accuracy, numSequences),
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
plt.xlabel('# of sequences seen')
plt.figure(2)
plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
window=1000,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
plt.xlabel('# of sequences seen')
for fig in [1, 2]:
plt.figure(fig)
truth = np.roll(actualData, -5)
predictions = np.transpose(likelihoodsVecAll)
negLLCLA = computeLikelihood(predictions, truth, encoder)
negLLCLA[:5904] = np.nan
predictions = np.transpose(likelihoodsVecAllNN)
negLLNN = computeLikelihood(predictions, truth, encoder)
negLLNN[:5904] = np.nan
# save predicted distribution for likelihood calculation
np.save('./result/' + datasetName + 'TMprediction.npy', predictions)
np.save('./result/' + datasetName + 'TMtruth.npy', truth)
plt.figure()
plotAccuracy((negLLCLA, range(len(negLLCLA))), truth, window=480, errorType='negLL')
plotAccuracy((negLLNN, range(len(negLLNN))), truth, window=480, errorType='negLL')
# Compare NN classifier and CLA classifier
plt.figure()
shiftedActualData = np.roll(np.array(actualData), -_options.stepsAhead)
plt.plot(shiftedActualData)
plt.plot(predictDataNN)
plt.plot(predictDataCLA)
plt.legend(['True', 'NN', 'CLA'])
plt.xlim([16600, 17000])
fig, ax = plt.subplots(nrows=1, ncols=3)
inds = np.arange(2)
ax1 = ax[0]
width = 0.5
names=['step', 'value', 'prediction5'])
esnPredPerturbTruth = np.roll(predDataESN['value'], -5)
esnPredPerturb = np.array(predDataESN['prediction5'])
negLL_tm_perturb = computeLikelihood(tm_prediction_perturb, tm_truth_perturb,
encoder)
negLL_tm_perturb[:6000] = None
nrmse_tm_perturb = computeSquareDeviation(tm_pred_perturb,
tm_pred_perturb_truth)
mape_tm_perturb = np.abs(tm_pred_perturb - tm_pred_perturb_truth)
mape_esn_perturb = np.abs(esnPredPerturb - esnPredPerturbTruth)
plt.figure()
plotAccuracy((negLL_LSTM3000_perturb, xaxis_datetime),
truth_LSTM3000_perturb,
window=window,
errorType='negLL',
label='LSTM3000',
train=expResult_perturb.train)
# plotAccuracy((negLL_LSTM3000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM')
plotAccuracy((negLL_LSTM6000_perturb, xaxis_datetime),
truth_LSTM6000_perturb,
window=window,
errorType='negLL',
label='LSTM6000')
# plotAccuracy((negLL_LSTM6000_perturb_baseline, xaxis_datetime), truth_LSTM3000_perturb, window=window, errorType='negLL', label='TM')
plotAccuracy((negLL_tm_perturb, xaxis_datetime),
tm_truth_perturb,
window=window,
errorType='negLL',
(nrmseLSTM6000, expResultLSTM6000) = \
plotLSTMresult('results/nyc_taxi_experiment_continuous/learning_window6001.0/',
window, xaxis=xaxisDate, label='continuous LSTM-6000')
dataSet = 'nyc_taxi'
filePath = './prediction/' + dataSet + '_TM_pred.csv'
(tmTruth, tmPrediction) = loadExperimentResult('./prediction/' + dataSet +
'_TM_pred.csv')
squareDeviation = computeSquareDeviation(tmPrediction, tmTruth)
squareDeviation[:skipTrain] = None
nrmseTM = plotAccuracy((squareDeviation, xaxisDate),
tmTruth,
window=window,
errorType='square_deviation',
label='TM')
(esnTruth,
esnPrediction) = loadExperimentResult('./prediction/' + dataSet +
'_esn_online_pred.csv')
squareDeviation = computeSquareDeviation(esnPrediction, esnTruth)
squareDeviation[:skipTrain] = None
nrmseESN = plotAccuracy((squareDeviation, xaxisDate),
tmTruth,
window=window,
errorType='square_deviation',
label='ESN')
(nrmseLSTMonline, expResultLSTMonline) = plotLSTMresult(
'results/nyc_taxi_experiment_continuous_online/learning_window100.0/',
window, xaxis=xaxisDate, label='continuous LSTM-online')
dataSet = 'nyc_taxi'
filePath = './prediction/' + dataSet + '_TM_pred.csv'
(tmTruth, tmPrediction) = loadExperimentResult('./prediction/' + dataSet + '_TM_pred.csv')
squareDeviation = computeSquareDeviation(tmPrediction, tmTruth)
squareDeviation[:skipTrain] = None
nrmseTM = plotAccuracy((squareDeviation, xaxisDate),
tmTruth,
window=window,
errorType='square_deviation',
label='TM')
(esnTruth, esnPrediction) = loadExperimentResult('./prediction/' + dataSet + '_esn_online_pred.csv')
squareDeviation = computeSquareDeviation(esnPrediction, esnTruth)
squareDeviation[:skipTrain] = None
nrmseESN = plotAccuracy((squareDeviation, xaxisDate),
tmTruth,
window=window,
errorType='square_deviation',
label='ESN')
minProb = 0.0001
for i in xrange(len(truth)):
bucketIndex2.append(np.where(encoder.encode(truth[i]))[0])
outOfBucketProb = 1 - sum(predictions[i, :])
prob = predictions[i, bucketIndex2[i]]
if prob == 0:
prob = outOfBucketProb
if prob < minProb:
prob = minProb
negLL.append(-np.log(prob))
negLL = computeLikelihood(predictions, truth, encoder)
negLL[:5000] = np.nan
x = range(len(negLL))
plt.figure()
plotAccuracy((negLL, x), truth, window=480, errorType='negLL')
np.save('./result/' + dataSet + classifierType + 'TMprediction.npy',
predictions)
np.save('./result/' + dataSet + classifierType + 'TMtruth.npy', truth)
plt.figure()
activeCellNumAvg = movingAverage(activeCellNum, 100)
plt.plot(np.array(activeCellNumAvg) / tm.numberOfCells())
plt.xlabel('data records')
plt.ylabel('sparsity')
plt.xlim([0, 5000])
plt.savefig('result/sparsity_over_training.pdf')
plt.figure()
dataSet = 'nyc_taxi'
filePath = './prediction/' + dataSet + '_TM_pred.csv'
predData_TM = pd.read_csv(filePath,
header=0,
skiprows=[1, 2],
names=['step', 'value', 'prediction5'])
tm_truth = np.roll(predData_TM['value'], -5)
predData_TM_five_step = np.array(predData_TM['prediction5'])
square_deviation = computeSquareDeviation(predData_TM_five_step, tm_truth)
square_deviation[:6000] = None
NRMSE_TM = plotAccuracy((square_deviation, xaxis_datetime),
tm_truth,
window=window,
errorType='square_deviation',
label='TM')
filePath = './prediction/' + dataSet + '_ESN_pred.csv'
predData_ESN = pd.read_csv(filePath,
header=0,
skiprows=[1, 2],
names=['step', 'value', 'prediction5'])
esn_truth = np.roll(predData_ESN['value'], -5)
predData_ESN_five_step = np.array(predData_ESN['prediction5'])
square_deviation = computeSquareDeviation(predData_ESN_five_step, esn_truth)
square_deviation[:6000] = None
NRMSE_ESN = plotAccuracy((square_deviation, xaxis_datetime),
