def loadRepeatedExperiments(expDir, killCellPercentRange, seedRange, killCellAt): meanAccuracy = [] stdAccuracy = [] for killCellPercent in killCellPercentRange: accuracyList = [] for seed in range(10): if killCellPercent == 0: experiment = os.path.join( expDir, "kill_cell_percent{:1.1f}seed{:1.1f}/0.log".format( killCellPercent, seed)) else: experiment = os.path.join( expDir, "kill_cell_percent{:1.2f}seed{:1.1f}/0.log".format( killCellPercent, seed)) expResults = readExperiment(experiment) print "Load Experiment: ", experiment (accuracy, x) = computeAccuracy(expResults['predictions'], expResults['truths'], expResults['iterations'], resets=expResults['resets'], randoms=expResults['randoms']) idx = numpy.array([i for i in range(len(x)) if x[i] > killCellAt]) accuracy = numpy.array(accuracy) accuracyList.append( float(numpy.sum(accuracy[idx])) / len(accuracy[idx])) meanAccuracy.append(numpy.mean(accuracyList)) stdAccuracy.append(numpy.std(accuracyList)) return meanAccuracy, stdAccuracy
def plotLSTMresult(experiment, window, xaxis=None, label=None): (iteration, truth, predictions, train, params) = loadExperiment(experiment) (square_deviation, x) = computeAccuracy(predictions, truth, iteration) if xaxis is not None: x = xaxis plotAccuracy((square_deviation, x), truth, train=train, window=window, label=label, params=params)
def loadRepeatedExperiments(expDir, killCellPercentRange, seedRange, killCellAt): meanAccuracy = [] stdAccuracy = [] for killCellPercent in killCellPercentRange: accuracyList = [] for seed in range(10): if killCellPercent == 0: experiment = os.path.join( expDir,"kill_cell_percent{:1.1f}seed{:1.1f}/0.log".format( killCellPercent, seed)) else: experiment = os.path.join( expDir,"kill_cell_percent{:1.2f}seed{:1.1f}/0.log".format( killCellPercent, seed)) expResults = readExperiment(experiment) print "Load Experiment: ", experiment (accuracy, x) = computeAccuracy(expResults['predictions'], expResults['truths'], expResults['iterations'], resets=expResults['resets'], randoms=expResults['randoms']) idx = numpy.array([i for i in range(len(x)) if x[i] > killCellAt]) accuracy = numpy.array(accuracy) accuracyList.append(float(numpy.sum(accuracy[idx])) / len(accuracy[idx])) meanAccuracy.append(numpy.mean(accuracyList)) stdAccuracy.append(numpy.std(accuracyList)) return meanAccuracy, stdAccuracy
def loadExperiment(experiment): print "Loading experiment ", experiment data = readExperiment(experiment) (accuracy, x) = computeAccuracy(data['predictions'], data['truths'], data['iterations'], resets=data['resets'], randoms=data['randoms']) return (accuracy, x)
pyplot.close('all') if __name__ == '__main__': experiments = [ os.path.join("lstm/results", "high-order-noise", "inject_noise_after0.0", "0.log"), os.path.join("tm/results", "high-order-noise", "inject_noise_after0.0", "0.log") ] for experiment in experiments: data = readExperiment(experiment) (accuracy, x) = computeAccuracy(data['predictions'], data['truths'], data['iterations'], resets=data['resets'], randoms=data['randoms']) plotAccuracy((accuracy, x), data['trains'], window=100, type=type, label='NoiseExperiment', hideTraining=True, lineSize=1.0) pyplot.xlim([10500, 14500]) pyplot.xlabel('Number of elements seen') pyplot.ylabel(' Accuracy ') pyplot.legend(['LSTM', 'HTM'])
outdir = 'tm/result/' KILLCELL_PERCENT = list(numpy.arange(7) / 10.0) accuracyListTM = [] accuracyListLSTM = [] accuracyListELM = [] for killCellPercent in KILLCELL_PERCENT: # HTM experiments experiment = os.path.join(outdir, "kill_cell_percent{:1.1f}".format( killCellPercent)) + '/0.log' expResults = readExperiment(experiment) killCellAt = 10000 (accuracy, x) = computeAccuracy(expResults['predictions'][killCellAt:], expResults['truths'][killCellAt:], expResults['iterations'][killCellAt:], resets=expResults['resets'][killCellAt:], randoms=expResults['randoms'][killCellAt:]) accuracyListTM.append(float(numpy.sum(accuracy)) / len(accuracy)) # LSTM experiments experiment = 'lstm/results/high-order-distributed-random-kill-cell/' \ 'kill_cell_percent' + "{:1.2f}".format(killCellPercent) + '/0.log' expResults = readExperiment(experiment) killCellAt = 10000 (accuracy, x) = computeAccuracy(expResults['predictions'][killCellAt:], expResults['truths'][killCellAt:], expResults['iterations'][killCellAt:], resets=expResults['resets'][killCellAt:],
mpl.rcParams['pdf.fonttype'] = 42 pyplot.ion() if __name__ == '__main__': experiments = [ os.path.join("lstm/results", "high-order-noise", "0.log"), os.path.join("tm/results", "high-order-noise", "0.log") ] for experiment in experiments: data = readExperiment(experiment) (accuracy, x) = computeAccuracy(data['predictions'], data['truths'], data['iterations'], resets=data['resets'], randoms=data['randoms'], sequenceCounter=data['sequenceCounter']) injectNoiseAt = data['sequenceCounter'][12000] plotAccuracy((accuracy, x), data['trains'], window=100, type=type, label='NoiseExperiment', hideTraining=True, lineSize=1.0) pyplot.xlim([1200, 1750]) pyplot.savefig('./model_performance_after_temporal_noise.pdf') pyplot.legend(['LSTM', 'HTM'])
plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window1001.0/', window, xaxis=xaxis_datetime, label='continuous LSTM-1000') plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window3001.0/', window, xaxis=xaxis_datetime, label='continuous LSTM-3000') plotLSTMresult('results/nyc_taxi_experiment_perturb/learning_window5001.0/', window, xaxis=xaxis_datetime, label='continuous LSTM-5000') # load TM prediction filePath = './data/' + 'nyc_taxi' + '.csv' data = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['datetime', 'value', 'timeofday', 'dayofweek']) dataSet = 'nyc_taxi_perturb' filePath = './prediction/' + dataSet + '_TM_pred.csv' predData_TM = pd.read_csv(filePath, header=0, skiprows=[1, 2], names=['step', 'value', 'prediction5']) truth = predData_TM['value'] predData_TM_five_step = np.roll(predData_TM['prediction5'], 5) iteration = predData_TM.index (square_deviation, x) = computeAccuracy(predData_TM_five_step, truth, iteration) square_deviation[:5000] = None x = pd.to_datetime(data['datetime']) plotAccuracy((square_deviation, x), truth, window=window, label='TM') plt.legend() plt.savefig(figPath + 'continuous_perturb.pdf')
outdir = 'tm/result/' KILLCELL_PERCENT = list(numpy.arange(7) / 10.0) accuracyListTM = [] accuracyListLSTM = [] for killCellPercent in KILLCELL_PERCENT: experiment = os.path.join( outdir, "kill_cell_percent{:1.1f}".format(killCellPercent)) + '/0.log' (predictions, truths, iterations, resets, randoms, trains, killCell) = readExperiment(experiment) killCellAt = 10000 (accuracy, x) = computeAccuracy(predictions[killCellAt:], truths[killCellAt:], iterations[killCellAt:], resets=resets[killCellAt:], randoms=randoms[killCellAt:]) accuracyListTM.append(float(numpy.sum(accuracy)) / len(accuracy)) experiment = 'lstm/results/high-order-distributed-random-kill-cell/' \ 'kill_cell_percent' + "{:1.2f}".format(killCellPercent) + '/0.log' (predictions, truths, iterations, resets, randoms, killCell) = readExperiment(experiment) killCellAt = 10000 (accuracy, x) = computeAccuracy(predictions[killCellAt:], truths[killCellAt:], iterations[killCellAt:],
experiments.append(os.path.join("lstm/results", "high-order-distributed-random-perturbed", "seed0.0learning_window{:6.1f}".format(window), "0.log")) experiments.append(os.path.join("elm/results", "high-order-distributed-random-perturbed", "seed0.0", "0.log")) for experiment in experiments: data = readExperiment(experiment) (accuracy, x) = computeAccuracy(data['predictions'], data['truths'], data['iterations'], resets=data['resets'], randoms=data['randoms']) # perturbAt = data['sequenceCounter'][10000] plotAccuracy((accuracy, x), data['trains'], window=200, type=type, label='NoiseExperiment', hideTraining=True, lineSize=1.0) # plt.xlim([1200, 1750]) plt.xlabel('# of sequences seen')
mpl.rcParams["pdf.fonttype"] = 42 pyplot.ion() pyplot.close("all") if __name__ == "__main__": experiments = [ os.path.join("lstm/results", "high-order-noise", "inject_noise_after0.0", "0.log"), os.path.join("tm/results", "high-order-noise", "inject_noise_after0.0", "0.log"), ] for experiment in experiments: data = readExperiment(experiment) (accuracy, x) = computeAccuracy( data["predictions"], data["truths"], data["iterations"], resets=data["resets"], randoms=data["randoms"] ) plotAccuracy( (accuracy, x), data["trains"], window=100, type=type, label="NoiseExperiment", hideTraining=True, lineSize=1.0, ) pyplot.xlim([10500, 14500]) pyplot.xlabel("Number of elements seen") pyplot.ylabel(" Accuracy ")
try: # Load raw experiment results # You have to run the experiments in ./tm # python tm_suite.py --experiment="high-order-variable-length" -d expResults = {} for length in lengths: experiment = os.path.join(tmResults, "sequence_length"+"{:.1f}".format(length), "0.log") data = readExperiment(experiment) (accuracy, numIteration, numSequences) = computeAccuracy( data['predictions'], data['truths'], data['iterations'], resets=data['resets'], randoms=data['randoms'], sequenceCounter=data['sequenceCounter']) expResult = {"length": length, "accuracy": accuracy, "numIteration": numIteration, "numSequences": numSequences} expResults[length] = expResult output = open(os.path.join(tmResults, 'SequenceLengthExperiment.pkl'), 'wb') pickle.dump(expResults, output, -1) output.close() except: print "Cannot find raw experiment results"