def runMultiColumnExperiment():
"""
Mean number of observations needed to unambiguously recognize an object with
multi-column networks as the set of columns increases. We train each network
with 100 objects and plot the average number of sensations required to
unambiguously recognize an object.
"""
# We run 10 trials for each column number and then analyze results
numTrials = 10
columnRange = [1, 2, 3, 4, 5, 6, 7, 8]
featureRange = [5, 10, 20, 30]
objectRange = [100]
# Comment this out if you are re-running analysis on already saved results.
# Very useful for debugging the plots
runExperimentPool(
numObjects=objectRange,
numLocations=[10],
numFeatures=featureRange,
numColumns=columnRange,
numPoints=10,
numWorkers=cpu_count() - 1,
nTrials=numTrials,
resultsName="object_convergence_multi_column_results.pkl")
# Analyze results
with open("object_convergence_multi_column_results.pkl", "rb") as f:
results = cPickle.load(f)
plotConvergenceByColumn(results, columnRange, featureRange, numTrials)
def runSingleColumnExperiment():
"""
Mean number of sensations needed to unambiguously recognize an object with a
single column network as the set of learned objects increases. We train models
on varying numbers of objects, from 1 to 100 and plot the average number of
sensations required to unambiguously recognize a single object.
"""
# We run 10 trials for each column number and then analyze results
numTrials = 10
columnRange = [1]
featureRange = [5, 10, 20, 30]
objectRange = [2, 10, 20, 30, 40, 50, 60, 80, 100]
# Comment this out if you are re-running analysis on already saved results.
# Very useful for debugging the plots
runExperimentPool(numObjects=objectRange,
numLocations=[10],
numFeatures=featureRange,
numColumns=columnRange,
numPoints=10,
nTrials=numTrials,
numWorkers=cpu_count() - 1,
resultsName="object_convergence_results.pkl")
# Analyze results
with open("object_convergence_results.pkl", "rb") as f:
results = cPickle.load(f)
plotConvergenceByObject(results, objectRange, featureRange, numTrials)
def runSensationsNoiseExperiment():
"""
Calculcate Recognition accuracy as a function of the number of sensations.
Colored lines correspond to noise levels in the location input
"""
noise = [0.0, 0.2, 0.4, 0.5, 0.6, 0.7]
objects = [100]
locations = [5000]
features = [5000]
columns = [1]
points = 10
settlingTime = 3
numTrials = 10
results = runExperimentPool(numObjects=objects,
numLocations=locations,
numFeatures=features,
numColumns=columns,
locationNoiseRange=noise,
numWorkers=cpu_count() - 1,
numPoints=points,
settlingTime=settlingTime,
nTrials=numTrials,
enableFeedback=[False],
resultsName="sensation_noise_results.pkl")
# with open("sensation_noise_results.pkl", "rb") as f:
# results = cPickle.load(f)
# Group results by noise
noise_key = lambda x: x['locationNoise']
sorted_results = sorted(results, key=noise_key)
grouped_results = [
(k, numpy.mean([row['classificationPerSensation'] for row in v],
axis=0))
for k, v in groupby(sorted_results, key=noise_key)
]
plotSensationsNoise(grouped_results)
def single_column_accuracy_comparison():
"""
Plot accuracy of the ideal observer (with and without locations) as the number
of sensations increases.
"""
pointRange = 1
numTrials = 10
args = []
resultsDir = os.path.dirname(os.path.realpath(__file__))
for t in range(numTrials):
for useLocation in [0, 1]:
args.append(
{"numObjects": 100,
"numLocations": 10,
"numFeatures": 10,
"numColumns": 1,
"trialNum": t,
"pointRange": pointRange,
"numPoints": 10,
"useLocation": useLocation
}
)
print "{} experiments to run, {} workers".format(len(args), cpu_count())
idealResultsFile = os.path.join(resultsDir, "ideal_model_result.pkl")
# Run all experiments and pickle results for later use
pool = Pool(processes=cpu_count())
resultsIdeal = pool.map(run_ideal_classifier, args)
with open(idealResultsFile, "wb") as f:
cPickle.dump(resultsIdeal, f)
# run HTM network
columnRange = [1]
objectRange = [100]
numAmbiguousLocationsRange = [0]
htmResultsFile = os.path.join(resultsDir, "single_column_convergence_results.pkl")
runExperimentPool(
numObjects=objectRange,
numLocations=[10],
numFeatures=[10],
numColumns=columnRange,
numPoints=10,
settlingTime=1,
nTrials=numTrials,
numWorkers=cpu_count(),
ambiguousLocationsRange=numAmbiguousLocationsRange,
resultsName=htmResultsFile)
# Read results from pickle files
with open(idealResultsFile, "rb") as f:
resultsIdeal = cPickle.load(f)
with open(htmResultsFile, "rb") as f:
resultsModel = cPickle.load(f)
# plot accuracy across sensations
accuracyIdeal = 0
accuracyBOF = 0
for r in resultsIdeal:
if r["useLocation"]:
accuracyIdeal += np.array(r['accuracy'])
else:
accuracyBOF += np.array(r['accuracy'])
accuracyIdeal /= len(resultsIdeal) / 2
accuracyBOF /= len(resultsIdeal) / 2
numTouches = len(accuracyIdeal)
accuracyModel = 0
for r in resultsModel:
accuracyModel += np.array(r['classificationPerSensation'])
accuracyModel /= len(resultsModel)
plt.figure()
plt.plot(np.arange(numTouches)+1, accuracyIdeal, '-o', label='Ideal observer (with location')
plt.plot(np.arange(numTouches) + 1, accuracyBOF, '-s', label='Ideal observer (no location)')
plt.plot(np.arange(numTouches)+1, accuracyModel, '-^', label='Sensorimotor network')
plt.xlabel("Number of sensations")
plt.ylabel("Accuracy")
plt.legend()
plt.savefig('plots/ideal_observer_comparison_single_column.pdf')
def run_multiple_column_experiment():
"""
Compare the ideal observer against a multi-column sensorimotor network.
"""
# Create the objects
featureRange = [5, 10, 20, 30]
pointRange = 1
objectRange = [100]
numLocations = [10]
numPoints = 10
numTrials = 10
columnRange = [1, 2, 3, 4, 5, 6, 7, 8]
useLocation = 1
resultsDir = os.path.dirname(os.path.realpath(__file__))
args = []
for c in reversed(columnRange):
for o in reversed(objectRange):
for l in numLocations:
for f in featureRange:
for t in range(numTrials):
args.append(
{"numObjects": o,
"numLocations": l,
"numFeatures": f,
"numColumns": c,
"trialNum": t,
"pointRange": pointRange,
"numPoints": numPoints,
"useLocation": useLocation
}
)
print "Number of experiments:",len(args)
idealResultsFile = os.path.join(resultsDir,
"ideal_multi_column_useLocation_{}.pkl".format(useLocation))
pool = Pool(processes=cpu_count())
result = pool.map(run_ideal_classifier, args)
# Pickle results for later use
with open(idealResultsFile, "wb") as f:
cPickle.dump(result, f)
htmResultsFile = os.path.join(resultsDir, "column_convergence_results.pkl")
runExperimentPool(
numObjects=objectRange,
numLocations=[10],
numFeatures=featureRange,
numColumns=columnRange,
numPoints=10,
nTrials=numTrials,
numWorkers=cpu_count(),
resultsName=htmResultsFile)
with open(htmResultsFile, "rb") as f:
results = cPickle.load(f)
with open(idealResultsFile, "rb") as f:
resultsIdeal = cPickle.load(f)
plt.figure()
plotConvergenceByColumn(results, columnRange, featureRange, numTrials)
plotConvergenceByColumn(resultsIdeal, columnRange, featureRange, numTrials,
"--")
plt.savefig('plots/ideal_observer_multiple_column.pdf')
# Here we want to see how the number of objects affects convergence for a
# single column.
# This experiment is run using a process pool
if False:
# We run 10 trials for each column number and then analyze results
numTrials = 10
columnRange = [1]
featureRange = [5, 10, 20, 30]
objectRange = [2, 10, 20, 30, 40, 50, 60, 80, 100]
# Comment this out if you are re-running analysis on already saved results.
# Very useful for debugging the plots
runExperimentPool(numObjects=objectRange,
numLocations=[10],
numFeatures=featureRange,
numColumns=columnRange,
numPoints=10,
nTrials=numTrials,
numWorkers=cpu_count() - 1,
resultsName="object_convergence_results.pkl")
# Analyze results
with open("object_convergence_results.pkl", "rb") as f:
results = cPickle.load(f)
plotConvergenceByObject(results, objectRange, featureRange)
# Here we want to see how the number of objects affects convergence for
# multiple columns.
if False:
# We run 10 trials for each column number and then analyze results
numTrials = 10
# single column.
# This experiment is run using a process pool
if False:
# We run 10 trials for each column number and then analyze results
numTrials = 10
columnRange = [1]
featureRange = [5,10,20,30]
objectRange = [2,10,20,30,40,50,60,80,100]
# Comment this out if you are re-running analysis on already saved results.
# Very useful for debugging the plots
runExperimentPool(
numObjects=objectRange,
numLocations=[10],
numFeatures=featureRange,
numColumns=columnRange,
numPoints=10,
nTrials=numTrials,
numWorkers=cpu_count() - 1,
resultsName="object_convergence_results.pkl")
# Analyze results
with open("object_convergence_results.pkl","rb") as f:
results = cPickle.load(f)
plotConvergenceByObject(results, objectRange, featureRange)
# Here we want to see how the number of objects affects convergence for
# multiple columns.
if False:
def runFeatureLocationNoiseExperiment():
"""
Evaluated robustness of a single column network to noise. After the network
learned a set of objects, we added varying amounts of random noise to the
sensory and location inputs. The noise affected the active bits in the
input without changing its overall sparsity (see Materials and Methods).
Recognition accuracy after 30 touches is plotted as a function of noise
"""
noise = numpy.arange(0, 0.8, .1)
objects = [50]
locations = [10]
features = [10]
columns = [1]
points = 10
settlingTime = 3
numTrials = 10
feature_noise_results = runExperimentPool(
numObjects=objects,
numLocations=locations,
numFeatures=features,
numColumns=columns,
featureNoiseRange=noise,
numWorkers=cpu_count() - 1,
numPoints=points,
nTrials=numTrials,
enableFeedback=[False],
settlingTime=settlingTime,
resultsName="feature_noise_results.pkl")
# with open("feature_noise_results.pkl", "rb") as f:
# feature_noise_results = cPickle.load(f)
# Group Feature results by noise
feature_noise_key = lambda x: x['featureNoise']
sorted_results = sorted(feature_noise_results, key=feature_noise_key)
feature_noise_results = [
(k, numpy.mean(next(v)['classificationAccuracy']))
for k, v in groupby(sorted_results, key=feature_noise_key)
]
location_noise_results = runExperimentPool(
numObjects=objects,
numLocations=locations,
numFeatures=features,
numColumns=columns,
locationNoiseRange=noise,
numWorkers=cpu_count() - 1,
numPoints=points,
settlingTime=settlingTime,
nTrials=numTrials,
enableFeedback=[False],
resultsName="location_noise_results.pkl")
# with open("location_noise_results.pkl", "rb") as f:
# location_noise_results = cPickle.load(f)
# Group Location results by noise
location_noise_key = lambda x: x['locationNoise']
sorted_results = sorted(location_noise_results, key=location_noise_key)
location_noise_results = [
(k, numpy.mean(next(v)['classificationAccuracy']))
for k, v in groupby(sorted_results, key=location_noise_key)
]
plotFeatureLocationNoise(feature_results=feature_noise_results,
location_results=location_noise_results)
def single_column_accuracy_comparison():
"""
Plot accuracy of the ideal observer (with and without locations) as the number
of sensations increases.
"""
pointRange = 1
numTrials = 10
args = []
resultsDir = os.path.dirname(os.path.realpath(__file__))
for t in range(numTrials):
for useLocation in [0, 1]:
args.append({
"numObjects": 100,
"numLocations": 10,
"numFeatures": 10,
"numColumns": 1,
"trialNum": t,
"pointRange": pointRange,
"numPoints": 10,
"useLocation": useLocation
})
print "{} experiments to run, {} workers".format(len(args), cpu_count())
idealResultsFile = os.path.join(resultsDir, "ideal_model_result.pkl")
# Run all experiments and pickle results for later use
pool = Pool(processes=cpu_count())
resultsIdeal = pool.map(run_ideal_classifier, args)
with open(idealResultsFile, "wb") as f:
cPickle.dump(resultsIdeal, f)
# run HTM network
columnRange = [1]
objectRange = [100]
numAmbiguousLocationsRange = [0]
htmResultsFile = os.path.join(resultsDir,
"single_column_convergence_results.pkl")
runExperimentPool(numObjects=objectRange,
numLocations=[10],
numFeatures=[10],
numColumns=columnRange,
numPoints=10,
settlingTime=1,
nTrials=numTrials,
numWorkers=cpu_count(),
ambiguousLocationsRange=numAmbiguousLocationsRange,
resultsName=htmResultsFile)
# Read results from pickle files
with open(idealResultsFile, "rb") as f:
resultsIdeal = cPickle.load(f)
with open(htmResultsFile, "rb") as f:
resultsModel = cPickle.load(f)
# plot accuracy across sensations
accuracyIdeal = 0
accuracyBOF = 0
for r in resultsIdeal:
if r["useLocation"]:
accuracyIdeal += np.array(r['accuracy'])
else:
accuracyBOF += np.array(r['accuracy'])
accuracyIdeal /= len(resultsIdeal) / 2
accuracyBOF /= len(resultsIdeal) / 2
numTouches = len(accuracyIdeal)
accuracyModel = 0
for r in resultsModel:
accuracyModel += np.array(r['classificationPerSensation'])
accuracyModel /= len(resultsModel)
plt.figure()
plt.plot(np.arange(numTouches) + 1,
accuracyIdeal,
'-o',
label='Ideal observer (with location')
plt.plot(np.arange(numTouches) + 1,
accuracyBOF,
'-s',
label='Ideal observer (no location)')
plt.plot(np.arange(numTouches) + 1,
accuracyModel,
'-^',
label='Sensorimotor network')
plt.xlabel("Number of sensations")
plt.ylabel("Accuracy")
plt.legend()
plt.savefig('plots/ideal_observer_comparison_single_column.pdf')
def run_multiple_column_experiment():
"""
Compare the ideal observer against a multi-column sensorimotor network.
"""
# Create the objects
featureRange = [5, 10, 20, 30]
pointRange = 1
objectRange = [100]
numLocations = [10]
numPoints = 10
numTrials = 10
columnRange = [1, 2, 3, 4, 5, 6, 7, 8]
useLocation = 1
resultsDir = os.path.dirname(os.path.realpath(__file__))
args = []
for c in reversed(columnRange):
for o in reversed(objectRange):
for l in numLocations:
for f in featureRange:
for t in range(numTrials):
args.append({
"numObjects": o,
"numLocations": l,
"numFeatures": f,
"numColumns": c,
"trialNum": t,
"pointRange": pointRange,
"numPoints": numPoints,
"useLocation": useLocation
})
print "Number of experiments:", len(args)
idealResultsFile = os.path.join(
resultsDir,
"ideal_multi_column_useLocation_{}.pkl".format(useLocation))
pool = Pool(processes=cpu_count())
result = pool.map(run_ideal_classifier, args)
# Pickle results for later use
with open(idealResultsFile, "wb") as f:
cPickle.dump(result, f)
htmResultsFile = os.path.join(resultsDir, "column_convergence_results.pkl")
runExperimentPool(numObjects=objectRange,
numLocations=[10],
numFeatures=featureRange,
numColumns=columnRange,
numPoints=10,
nTrials=numTrials,
numWorkers=cpu_count(),
resultsName=htmResultsFile)
with open(htmResultsFile, "rb") as f:
results = cPickle.load(f)
with open(idealResultsFile, "rb") as f:
resultsIdeal = cPickle.load(f)
plt.figure()
plotConvergenceByColumn(results, columnRange, featureRange, numTrials)
plotConvergenceByColumn(resultsIdeal, columnRange, featureRange, numTrials,
"--")
plt.savefig('plots/ideal_observer_multiple_column.pdf')
