def loadRepeatedExperiments(expDir, killCellPercentRange, seedRange,
killCellAt):
meanAccuracy = []
stdAccuracy = []
for killCellPercent in killCellPercentRange:
accuracyList = []
for seed in range(10):
if killCellPercent == 0:
experiment = os.path.join(
expDir, "kill_cell_percent{:1.1f}seed{:1.1f}/0.log".format(
killCellPercent, seed))
else:
experiment = os.path.join(
expDir, "kill_cell_percent{:1.2f}seed{:1.1f}/0.log".format(
killCellPercent, seed))
expResults = readExperiment(experiment)
print "Load Experiment: ", experiment
(accuracy, x) = computeAccuracy(expResults['predictions'],
expResults['truths'],
expResults['iterations'],
resets=expResults['resets'],
randoms=expResults['randoms'])
idx = numpy.array([i for i in range(len(x)) if x[i] > killCellAt])
accuracy = numpy.array(accuracy)
accuracyList.append(
float(numpy.sum(accuracy[idx])) / len(accuracy[idx]))
meanAccuracy.append(numpy.mean(accuracyList))
stdAccuracy.append(numpy.std(accuracyList))
return meanAccuracy, stdAccuracy
def plotLSTMresult(experiment, window, xaxis=None, label=None):
(iteration, truth, predictions, train, params) = loadExperiment(experiment)
(square_deviation, x) = computeAccuracy(predictions, truth, iteration)
if xaxis is not None:
x = xaxis
plotAccuracy((square_deviation, x), truth, train=train, window=window,
label=label, params=params)
def loadRepeatedExperiments(expDir, killCellPercentRange, seedRange, killCellAt):
meanAccuracy = []
stdAccuracy = []
for killCellPercent in killCellPercentRange:
accuracyList = []
for seed in range(10):
if killCellPercent == 0:
experiment = os.path.join(
expDir,"kill_cell_percent{:1.1f}seed{:1.1f}/0.log".format(
killCellPercent, seed))
else:
experiment = os.path.join(
expDir,"kill_cell_percent{:1.2f}seed{:1.1f}/0.log".format(
killCellPercent, seed))
expResults = readExperiment(experiment)
print "Load Experiment: ", experiment
(accuracy, x) = computeAccuracy(expResults['predictions'],
expResults['truths'],
expResults['iterations'],
resets=expResults['resets'],
randoms=expResults['randoms'])
idx = numpy.array([i for i in range(len(x)) if x[i] > killCellAt])
accuracy = numpy.array(accuracy)
accuracyList.append(float(numpy.sum(accuracy[idx])) / len(accuracy[idx]))
meanAccuracy.append(numpy.mean(accuracyList))
stdAccuracy.append(numpy.std(accuracyList))
return meanAccuracy, stdAccuracy
def loadExperiment(experiment):
print "Loading experiment ", experiment
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
return (accuracy, x)
def loadExperiment(experiment):
print "Loading experiment ", experiment
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
return (accuracy, x)
pyplot.close('all')
if __name__ == '__main__':
experiments = [
os.path.join("lstm/results", "high-order-noise",
"inject_noise_after0.0", "0.log"),
os.path.join("tm/results", "high-order-noise", "inject_noise_after0.0",
"0.log")
]
for experiment in experiments:
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
plotAccuracy((accuracy, x),
data['trains'],
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
pyplot.xlim([10500, 14500])
pyplot.xlabel('Number of elements seen')
pyplot.ylabel(' Accuracy ')
pyplot.legend(['LSTM', 'HTM'])
outdir = 'tm/result/'
KILLCELL_PERCENT = list(numpy.arange(7) / 10.0)
accuracyListTM = []
accuracyListLSTM = []
accuracyListELM = []
for killCellPercent in KILLCELL_PERCENT:
# HTM experiments
experiment = os.path.join(outdir, "kill_cell_percent{:1.1f}".format(
killCellPercent)) + '/0.log'
expResults = readExperiment(experiment)
killCellAt = 10000
(accuracy, x) = computeAccuracy(expResults['predictions'][killCellAt:],
expResults['truths'][killCellAt:],
expResults['iterations'][killCellAt:],
resets=expResults['resets'][killCellAt:],
randoms=expResults['randoms'][killCellAt:])
accuracyListTM.append(float(numpy.sum(accuracy)) / len(accuracy))
# LSTM experiments
experiment = 'lstm/results/high-order-distributed-random-kill-cell/' \
'kill_cell_percent' + "{:1.2f}".format(killCellPercent) + '/0.log'
expResults = readExperiment(experiment)
killCellAt = 10000
(accuracy, x) = computeAccuracy(expResults['predictions'][killCellAt:],
expResults['truths'][killCellAt:],
expResults['iterations'][killCellAt:],
resets=expResults['resets'][killCellAt:],
mpl.rcParams['pdf.fonttype'] = 42
pyplot.ion()
if __name__ == '__main__':
experiments = [
os.path.join("lstm/results", "high-order-noise", "0.log"),
os.path.join("tm/results", "high-order-noise", "0.log")
]
for experiment in experiments:
data = readExperiment(experiment)
(accuracy,
x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'],
sequenceCounter=data['sequenceCounter'])
injectNoiseAt = data['sequenceCounter'][12000]
plotAccuracy((accuracy, x),
data['trains'],
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
pyplot.xlim([1200, 1750])
pyplot.savefig('./model_performance_after_temporal_noise.pdf')
pyplot.legend(['LSTM', 'HTM'])
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window1001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-1000')
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window3001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-3000')
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window5001.0/',
window, xaxis=xaxis_datetime, label='continuous LSTM-5000')
# load TM prediction
filePath = './data/' + 'nyc_taxi' + '.csv'
data = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['datetime', 'value', 'timeofday', 'dayofweek'])
dataSet = 'nyc_taxi_perturb'
filePath = './prediction/' + dataSet + '_TM_pred.csv'
predData_TM = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5'])
truth = predData_TM['value']
predData_TM_five_step = np.roll(predData_TM['prediction5'], 5)
iteration = predData_TM.index
(square_deviation, x) = computeAccuracy(predData_TM_five_step, truth, iteration)
square_deviation[:5000] = None
x = pd.to_datetime(data['datetime'])
plotAccuracy((square_deviation, x),
truth,
window=window,
label='TM')
plt.legend()
plt.savefig(figPath + 'continuous_perturb.pdf')
outdir = 'tm/result/'
KILLCELL_PERCENT = list(numpy.arange(7) / 10.0)
accuracyListTM = []
accuracyListLSTM = []
for killCellPercent in KILLCELL_PERCENT:
experiment = os.path.join(
outdir,
"kill_cell_percent{:1.1f}".format(killCellPercent)) + '/0.log'
(predictions, truths, iterations, resets, randoms, trains,
killCell) = readExperiment(experiment)
killCellAt = 10000
(accuracy, x) = computeAccuracy(predictions[killCellAt:],
truths[killCellAt:],
iterations[killCellAt:],
resets=resets[killCellAt:],
randoms=randoms[killCellAt:])
accuracyListTM.append(float(numpy.sum(accuracy)) / len(accuracy))
experiment = 'lstm/results/high-order-distributed-random-kill-cell/' \
'kill_cell_percent' + "{:1.2f}".format(killCellPercent) + '/0.log'
(predictions, truths, iterations, resets, randoms,
killCell) = readExperiment(experiment)
killCellAt = 10000
(accuracy, x) = computeAccuracy(predictions[killCellAt:],
truths[killCellAt:],
iterations[killCellAt:],
experiments.append(os.path.join("lstm/results",
"high-order-distributed-random-perturbed",
"seed0.0learning_window{:6.1f}".format(window),
"0.log"))
experiments.append(os.path.join("elm/results",
"high-order-distributed-random-perturbed",
"seed0.0",
"0.log"))
for experiment in experiments:
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
# perturbAt = data['sequenceCounter'][10000]
plotAccuracy((accuracy, x),
data['trains'],
window=200,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
# plt.xlim([1200, 1750])
plt.xlabel('# of sequences seen')
mpl.rcParams["pdf.fonttype"] = 42
pyplot.ion()
pyplot.close("all")
if __name__ == "__main__":
experiments = [
os.path.join("lstm/results", "high-order-noise", "inject_noise_after0.0", "0.log"),
os.path.join("tm/results", "high-order-noise", "inject_noise_after0.0", "0.log"),
]
for experiment in experiments:
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(
data["predictions"], data["truths"], data["iterations"], resets=data["resets"], randoms=data["randoms"]
)
plotAccuracy(
(accuracy, x),
data["trains"],
window=100,
type=type,
label="NoiseExperiment",
hideTraining=True,
lineSize=1.0,
)
pyplot.xlim([10500, 14500])
pyplot.xlabel("Number of elements seen")
pyplot.ylabel(" Accuracy ")
try:
# Load raw experiment results
# You have to run the experiments in ./tm
# python tm_suite.py --experiment="high-order-variable-length" -d
expResults = {}
for length in lengths:
experiment = os.path.join(tmResults,
"sequence_length"+"{:.1f}".format(length),
"0.log")
data = readExperiment(experiment)
(accuracy, numIteration, numSequences) = computeAccuracy(
data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'],
sequenceCounter=data['sequenceCounter'])
expResult = {"length": length,
"accuracy": accuracy,
"numIteration": numIteration,
"numSequences": numSequences}
expResults[length] = expResult
output = open(os.path.join(tmResults,
'SequenceLengthExperiment.pkl'), 'wb')
pickle.dump(expResults, output, -1)
output.close()
except:
print "Cannot find raw experiment results"
