def analyzeResult(x, accuracy, perturbAt=10000, movingAvg=True, smooth=True): if movingAvg: accuracy = movingAverage(accuracy, min(len(accuracy), 100)) x = np.array(x) accuracy = np.array(accuracy) if smooth: # perform smoothing convolution mask = np.ones(shape=(100,)) mask = mask/np.sum(mask) # extend accuracy vector to eliminate boundary effect of convolution accuracy = np.concatenate((accuracy, np.ones((200, ))*accuracy[-1])) accuracy = np.convolve(accuracy, mask, 'same') accuracy = accuracy[:len(x)] perturbAtX = np.where(x > perturbAt)[0][0] finalAccuracy = accuracy[perturbAtX-len(mask)/2] learnTime = min(np.where(np.logical_and(accuracy > finalAccuracy * 0.99, x < x[perturbAtX - len(mask)/2-1]))[0]) learnTime = x[learnTime] finalAccuracyAfterPerturbation = accuracy[-1] learnTimeAfterPerturbation = min(np.where( np.logical_and(accuracy > finalAccuracyAfterPerturbation * 0.99, x > x[perturbAtX + len(mask)]))[0]) learnTimeAfterPerturbation = x[learnTimeAfterPerturbation] - perturbAt result = {"finalAccuracy": finalAccuracy, "learnTime": learnTime, "finalAccuracyAfterPerturbation": finalAccuracyAfterPerturbation, "learnTimeAfterPerturbation": learnTimeAfterPerturbation} return result
def loadExperiment(experiment): print "Loading experiment ", experiment data = readExperiment(experiment) (accuracy, x) = computeAccuracy(data['predictions'], data['truths'], data['iterations'], resets=data['resets'], randoms=data['randoms']) accuracy = movingAverage(accuracy, min(len(accuracy), 100)) return (accuracy, x)
def analyzeResult(x, accuracy, perturbAt=10000, movingAvg=True, smooth=True): if movingAvg: accuracy = movingAverage(accuracy, min(len(accuracy), 100)) x = np.array(x) accuracy = np.array(accuracy) if smooth: # perform smoothing convolution mask = np.ones(shape=(100, )) mask = mask / np.sum(mask) # extend accuracy vector to eliminate boundary effect of convolution accuracy = np.concatenate((accuracy, np.ones((200, )) * accuracy[-1])) accuracy = np.convolve(accuracy, mask, 'same') accuracy = accuracy[:len(x)] perturbAtX = np.where(x > perturbAt)[0][0] finalAccuracy = accuracy[perturbAtX - len(mask) / 2] learnTime = min( np.where( np.logical_and(accuracy > finalAccuracy * 0.99, x < x[perturbAtX - len(mask) / 2 - 1]))[0]) learnTime = x[learnTime] finalAccuracyAfterPerturbation = accuracy[-1] learnTimeAfterPerturbation = min( np.where( np.logical_and(accuracy > finalAccuracyAfterPerturbation * 0.99, x > x[perturbAtX + len(mask)]))[0]) learnTimeAfterPerturbation = x[learnTimeAfterPerturbation] - perturbAt result = { "finalAccuracy": finalAccuracy, "learnTime": learnTime, "finalAccuracyAfterPerturbation": finalAccuracyAfterPerturbation, "learnTimeAfterPerturbation": learnTimeAfterPerturbation } return result
# python suite.py --experiment="high-order-distributed-random-perturbed" -d expResults = {} expResultsAnaly = {} # HTM tmResults = os.path.join("tm/results", "high-order-distributed-random-perturbed") accuracyAll = [] exptLabel = 'HTM' expResultsAnaly[exptLabel] = [] for seed in range(20): experiment = os.path.join(tmResults, "seed" + "{:.1f}".format(seed), "0.log") (accuracy, x) = loadExperiment(experiment) expResultsAnaly[exptLabel].append(analyzeResult(x, accuracy)) accuracy = movingAverage(accuracy, min(len(accuracy), 100)) accuracyAll.append(np.array(accuracy)) (meanAccuracy, stdAccuracy) = calculateMeanStd(accuracyAll) expResults[exptLabel] = { 'x': x, 'meanAccuracy': meanAccuracy, 'stdAccuracy': stdAccuracy} # ELM elmResults = os.path.join("elm/results", "high-order-distributed-random-perturbed") accuracyAll = [] exptLabel = 'ELM' expResultsAnaly[exptLabel] = [] for seed in range(10): experiment = os.path.join(elmResults,
expResults = pickle.load(input) # load processed experiment results and plot them numSequenceRequired = [] numIterationRequired = [] lengths = np.sort(expResults.keys()) plt.close('all') plt.figure(1) for length in lengths: expResult = expResults[length] accuracy = expResult["accuracy"] numIteration = expResult["numIteration"] numSequences = expResult["numSequences"] movingData = movingAverage(accuracy, min(len(accuracy), 100)) numSequenceRequired.append( numSequences[np.where(np.array(movingData) >= 0.999)[0][1]]) numIterationRequired.append( numIteration[np.where(np.array(movingData) >= 0.999)[0][1]]) plt.figure(1) plotAccuracy((accuracy, numSequences), window=100, type=type, label='NoiseExperiment', hideTraining=True, lineSize=1.0) plt.xlabel('# of sequences seen') plt.figure(2) plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
# python suite.py --experiment="high-order-distributed-random-perturbed" -d expResults = {} expResultsAnaly = {} # HTM tmResults = os.path.join("tm/results", "high-order-distributed-random-perturbed") accuracyAll = [] exptLabel = 'HTM' expResultsAnaly[exptLabel] = [] for seed in range(20): experiment = os.path.join(tmResults, "seed" + "{:.1f}".format(seed), "0.log") (accuracy, x) = loadExperiment(experiment) expResultsAnaly[exptLabel].append(analyzeResult(x, accuracy)) accuracy = movingAverage(accuracy, min(len(accuracy), 100)) accuracyAll.append(np.array(accuracy)) (meanAccuracy, stdAccuracy) = calculateMeanStd(accuracyAll) expResults[exptLabel] = { 'x': x, 'meanAccuracy': meanAccuracy, 'stdAccuracy': stdAccuracy } # ELM elmResults = os.path.join("elm/results", "high-order-distributed-random-perturbed") accuracyAll = [] exptLabel = 'ELM'