def analyzeResult(x, accuracy, perturbAt=10000, movingAvg=True, smooth=True):
if movingAvg:
accuracy = movingAverage(accuracy, min(len(accuracy), 100))
x = np.array(x)
accuracy = np.array(accuracy)
if smooth:
# perform smoothing convolution
mask = np.ones(shape=(100,))
mask = mask/np.sum(mask)
# extend accuracy vector to eliminate boundary effect of convolution
accuracy = np.concatenate((accuracy, np.ones((200, ))*accuracy[-1]))
accuracy = np.convolve(accuracy, mask, 'same')
accuracy = accuracy[:len(x)]
perturbAtX = np.where(x > perturbAt)[0][0]
finalAccuracy = accuracy[perturbAtX-len(mask)/2]
learnTime = min(np.where(np.logical_and(accuracy > finalAccuracy * 0.99,
x < x[perturbAtX - len(mask)/2-1]))[0])
learnTime = x[learnTime]
finalAccuracyAfterPerturbation = accuracy[-1]
learnTimeAfterPerturbation = min(np.where(
np.logical_and(accuracy > finalAccuracyAfterPerturbation * 0.99,
x > x[perturbAtX + len(mask)]))[0])
learnTimeAfterPerturbation = x[learnTimeAfterPerturbation] - perturbAt
result = {"finalAccuracy": finalAccuracy,
"learnTime": learnTime,
"finalAccuracyAfterPerturbation": finalAccuracyAfterPerturbation,
"learnTimeAfterPerturbation": learnTimeAfterPerturbation}
return result
def loadExperiment(experiment):
print "Loading experiment ", experiment
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
accuracy = movingAverage(accuracy, min(len(accuracy), 100))
return (accuracy, x)
def loadExperiment(experiment):
print "Loading experiment ", experiment
data = readExperiment(experiment)
(accuracy, x) = computeAccuracy(data['predictions'],
data['truths'],
data['iterations'],
resets=data['resets'],
randoms=data['randoms'])
accuracy = movingAverage(accuracy, min(len(accuracy), 100))
return (accuracy, x)
def analyzeResult(x, accuracy, perturbAt=10000, movingAvg=True, smooth=True):
if movingAvg:
accuracy = movingAverage(accuracy, min(len(accuracy), 100))
x = np.array(x)
accuracy = np.array(accuracy)
if smooth:
# perform smoothing convolution
mask = np.ones(shape=(100, ))
mask = mask / np.sum(mask)
# extend accuracy vector to eliminate boundary effect of convolution
accuracy = np.concatenate((accuracy, np.ones((200, )) * accuracy[-1]))
accuracy = np.convolve(accuracy, mask, 'same')
accuracy = accuracy[:len(x)]
perturbAtX = np.where(x > perturbAt)[0][0]
finalAccuracy = accuracy[perturbAtX - len(mask) / 2]
learnTime = min(
np.where(
np.logical_and(accuracy > finalAccuracy * 0.99,
x < x[perturbAtX - len(mask) / 2 - 1]))[0])
learnTime = x[learnTime]
finalAccuracyAfterPerturbation = accuracy[-1]
learnTimeAfterPerturbation = min(
np.where(
np.logical_and(accuracy > finalAccuracyAfterPerturbation * 0.99,
x > x[perturbAtX + len(mask)]))[0])
learnTimeAfterPerturbation = x[learnTimeAfterPerturbation] - perturbAt
result = {
"finalAccuracy": finalAccuracy,
"learnTime": learnTime,
"finalAccuracyAfterPerturbation": finalAccuracyAfterPerturbation,
"learnTimeAfterPerturbation": learnTimeAfterPerturbation
}
return result
# python suite.py --experiment="high-order-distributed-random-perturbed" -d
expResults = {}
expResultsAnaly = {}
# HTM
tmResults = os.path.join("tm/results",
"high-order-distributed-random-perturbed")
accuracyAll = []
exptLabel = 'HTM'
expResultsAnaly[exptLabel] = []
for seed in range(20):
experiment = os.path.join(tmResults,
"seed" + "{:.1f}".format(seed), "0.log")
(accuracy, x) = loadExperiment(experiment)
expResultsAnaly[exptLabel].append(analyzeResult(x, accuracy))
accuracy = movingAverage(accuracy, min(len(accuracy), 100))
accuracyAll.append(np.array(accuracy))
(meanAccuracy, stdAccuracy) = calculateMeanStd(accuracyAll)
expResults[exptLabel] = {
'x': x, 'meanAccuracy': meanAccuracy, 'stdAccuracy': stdAccuracy}
# ELM
elmResults = os.path.join("elm/results",
"high-order-distributed-random-perturbed")
accuracyAll = []
exptLabel = 'ELM'
expResultsAnaly[exptLabel] = []
for seed in range(10):
experiment = os.path.join(elmResults,
expResults = pickle.load(input)
# load processed experiment results and plot them
numSequenceRequired = []
numIterationRequired = []
lengths = np.sort(expResults.keys())
plt.close('all')
plt.figure(1)
for length in lengths:
expResult = expResults[length]
accuracy = expResult["accuracy"]
numIteration = expResult["numIteration"]
numSequences = expResult["numSequences"]
movingData = movingAverage(accuracy, min(len(accuracy), 100))
numSequenceRequired.append(
numSequences[np.where(np.array(movingData) >= 0.999)[0][1]])
numIterationRequired.append(
numIteration[np.where(np.array(movingData) >= 0.999)[0][1]])
plt.figure(1)
plotAccuracy((accuracy, numSequences),
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
plt.xlabel('# of sequences seen')
plt.figure(2)
plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
expResults = pickle.load(input)
# load processed experiment results and plot them
numSequenceRequired = []
numIterationRequired = []
lengths = np.sort(expResults.keys())
plt.close('all')
plt.figure(1)
for length in lengths:
expResult = expResults[length]
accuracy = expResult["accuracy"]
numIteration = expResult["numIteration"]
numSequences = expResult["numSequences"]
movingData = movingAverage(accuracy, min(len(accuracy), 100))
numSequenceRequired.append(
numSequences[np.where(np.array(movingData) >= 0.999)[0][1]])
numIterationRequired.append(
numIteration[np.where(np.array(movingData) >= 0.999)[0][1]])
plt.figure(1)
plotAccuracy((accuracy, numSequences),
window=100,
type=type,
label='NoiseExperiment',
hideTraining=True,
lineSize=1.0)
plt.xlabel('# of sequences seen')
plt.figure(2)
plotAccuracy((expResult["accuracyAll"], expResult["numSequencesAll"]),
# python suite.py --experiment="high-order-distributed-random-perturbed" -d
expResults = {}
expResultsAnaly = {}
# HTM
tmResults = os.path.join("tm/results",
"high-order-distributed-random-perturbed")
accuracyAll = []
exptLabel = 'HTM'
expResultsAnaly[exptLabel] = []
for seed in range(20):
experiment = os.path.join(tmResults,
"seed" + "{:.1f}".format(seed), "0.log")
(accuracy, x) = loadExperiment(experiment)
expResultsAnaly[exptLabel].append(analyzeResult(x, accuracy))
accuracy = movingAverage(accuracy, min(len(accuracy), 100))
accuracyAll.append(np.array(accuracy))
(meanAccuracy, stdAccuracy) = calculateMeanStd(accuracyAll)
expResults[exptLabel] = {
'x': x,
'meanAccuracy': meanAccuracy,
'stdAccuracy': stdAccuracy
}
# ELM
elmResults = os.path.join("elm/results",
"high-order-distributed-random-perturbed")
accuracyAll = []
exptLabel = 'ELM'
