def evaluate(self):
data = Dataset('datasets/textures/')
max_dist = -32
timer = genetics.speed_fitness(threshold=60, maximum=2 * 60)
sensor = htm.EyeSensor(self.eye)
sp = htm.SpatialPooler(self.sp,
input_sdr=sensor.optic_sdr,
column_sdr=SDR(self.columns),
radii=self.radii,
multisegment_experiment=None)
classifier = htm.SDR_Classifier(self.sdrc,
sp.columns, (len(data.names), ),
output_type='pdf')
baseline = htm.RandomOutputClassifier((len(data.names), ))
time_per_image = int(round(self.time_per_image))
num_images = int(round(self.image_cycles))
if self.debug:
sampler = htm.EyeSensorSampler(sensor, num_images, 30)
memory_perf = 0 # Debug takes extra memory, this is disabled during debug.
sp_score = 0
baseline_score = 0
# Outer loop through images.
for img_num in range(num_images):
# Setup for a new image
data.random_image()
sensor.new_image(data.current_image)
# Determine where the eye will look on the image.
positions = data.points_near_label(max_dist=max_dist,
number=time_per_image)
# Inner loop through samples from each image.
for sample_point in positions:
# Classify the image.
sensor.randomize_view() # Get a new orientation and scale.
sensor.position = sample_point
sp.compute(input_sdr=sensor.view())
sp_prediction = classifier.predict(sp.columns)
baseline_prediction = baseline.predict()
# Compare results to labels.
label_sample_points = sensor.input_space_sample_points(20)
labels = data.sample_labels(label_sample_points)
sp_score += data.compare_label_samples(sp_prediction, labels)
baseline_score += data.compare_label_samples(
baseline_prediction, labels)
# Learn.
sp.learn()
classifier.train(sp.columns.index, labels)
baseline.train(labels)
# Sample memory usage.
if img_num == min(10, num_images - 1) and not self.debug:
# I want each process to take no more than 20% of my RAM or 2.5
# GB. This should let me run 4 processes + linux + firefox.
memory_perf = genetics.memory_fitness(2e9, 4e9)
sp_score /= sp.age
baseline_score /= sp.age
time_perf = timer.done(
) # Stop the timer before debug opens windows and returns control to user.
if self.debug:
print(sp.statistics())
sampler.view_samples()
return {
'baseline': baseline_score,
'score': sp_score,
'time': time_perf,
'memory': memory_perf,
}
def evaluate(self, debug):
# SETUP
inputs, objects = object_dataset()
enc = EnumEncoder(self.enc_size, self.enc_sparsity, diag=False)
enc.output_sdr = SDR(enc.output_sdr, average_overlap_alpha=self.alpha)
sp = SpatialPooler(
SpatialPoolerParameters(
permanence_inc=self.inc,
permanence_dec=self.dec,
permanence_thresh=self.thresh,
potential_pool=self.pp,
sparsity=self.col_sparsity,
boosting_alpha=self.alpha,
),
input_sdr=SDR(enc.output_sdr,
activation_frequency_alpha=self.alpha),
column_sdr=SDR((self.cols, )),
multisegment_experiment=self.prox_segs,
init_dist=self.init_dist,
)
sdrc = SDR_Classifier(SDRC_Parameters(alpha=0.001),
input_sdr=sp.columns,
output_shape=(num_objects, ),
output_type='index')
def measure():
# Compute every sensation for every object.
objects_columns = []
for obj in objects:
objects_columns.append([])
for sensation in obj:
enc.encode(sensation)
sp.compute(input_sdr=enc.output_sdr)
objects_columns[-1].append(SDR(sp.columns))
# Measure classification accuracy.
score = 0
max_score = 0
for object_id, obj_cols in enumerate(objects_columns):
for sp_cols in obj_cols:
prediction = np.argmax(sdrc.predict(sp_cols))
if prediction == object_id:
score += 1
max_score += 1
score = score / max_score
# Measure column overlap within objects.
overlap_stability = 0
comparisions = 0
for obj_cols in objects_columns:
for c1, c2 in itertools.combinations(obj_cols, 2):
overlap_stability += c1.overlap(c2)
comparisions += 1
stability = overlap_stability / comparisions
# Measure column overlap between objects.
overlap_between_objects = 0
comparisions = 0
skip_compare = itertools.cycle([False] + [True] * 24)
for obj1_cols, obj2_cols in itertools.combinations(
objects_columns, 2):
for c1 in obj1_cols:
for c2 in obj2_cols:
if next(skip_compare):
continue
overlap_between_objects += c1.overlap(c2)
comparisions += 1
distinctiveness = overlap_between_objects / comparisions
return stability, distinctiveness, score
if debug:
untrained_stability, untrained_distinctiveness, untrained_score = measure(
)
print('NUM-OBJ ', num_objects)
print("OBJ-SIZE", object_size)
print(sp.statistics())
print('UNTRAINED INTER OBJECT OVERLAP', untrained_distinctiveness)
print('UNTRAINED INTRA OBJECT OVERLAP', untrained_stability)
print('UNTRAINED SCORE', untrained_score)
print('Training ...')
# RUN ONLINE
prev_cols = SDR(sp.columns)
prev_cols.zero()
for step in range(num_objects * (object_size**2) * 10):
if step % 2 * object_size == 0:
object_id = random.choice(range(len(objects)))
sensation = random.choice(objects[object_id])
enc.encode(sensation)
sp.compute(input_sdr=enc.output_sdr)
sp.stabilize(prev_cols, self.min_stab)
prev_cols = SDR(sp.columns)
sp.learn()
sdrc.train(sp.columns, (object_id, ))
trained_stability, trained_distinctiveness, trained_score = measure()
if debug:
print('TRAINED FOR CYCLES', step + 1)
print('TRAINED INTER OBJECT OVERLAP', trained_distinctiveness)
print('TRAINED INTRA OBJECT OVERLAP', trained_stability)
print('TRAINED SCORE', trained_score)
print()
print('Encoder Output', enc.output_sdr.statistics())
print(sp.statistics())
return {
'score': trained_score,
'ovlp': trained_stability / trained_distinctiveness,
'memory': genetics.memory_fitness(4e9, 5e9),
}
def evaluate(self):
cell_death_experiment = None
reward_adjustment_experiment = False
adversarial_dataset_experiment = False
if adversarial_dataset_experiment:
datastream = AdversarialDataset()
else:
datastream = Dataset(num_sequences=self.num_seq)
# SETUP AI.
start_time = time.time()
enc = htm.EnumEncoder(self.enc_bits, self.enc_spar, diag=False)
sp = htm.SpatialPooler(
self.sp,
input_sdr=enc.output_sdr,
column_sdr=SDR((self.cols, )),
)
sp_sdrc = htm.SDR_Classifier(htm.SDRC_Parameters(alpha=0.001),
sp.columns, datastream.class_shape,
'index')
tm = htm.TemporalMemory(self.tm, sp.columns)
tm_sdrc = htm.SDR_Classifier(htm.SDRC_Parameters(alpha=0.001),
tm.active, datastream.class_shape,
'index')
bg = basal_ganglia.BasalGanglia(self.bg,
tm.active,
future_discount=.95)
d1_sdrc = htm.SDR_Classifier(htm.SDRC_Parameters(alpha=0.001),
bg.d1.active, datastream.class_shape,
'index')
d2_sdrc = htm.SDR_Classifier(htm.SDRC_Parameters(alpha=0.001),
bg.d2.active, datastream.class_shape,
'index')
memory_score = genetics.memory_fitness(2e9, 3e9)
# SETUP ACCUMULATORS.
anom_rand = 0
anom_rand_total = 0
anom_pred = 0
anom_pred_total = 0
sp_seq_score = 0
tm_seq_score = 0
d1_seq_score = 0
d2_seq_score = 0
sequence_total = 0
td_error = 0
td_error_total = 0 # RMS of TD-Error
baseline = 0 # RMS(reward). If EV === 0 then this is also the RMS TD-Error.
event_step = None # A vertical line is drawn on the TD-Error graph at this step.
if self.debug:
input_history = []
reward_history = []
anomalous_input_history = []
ev_history = []
td_error_history = []
anomaly_history = []
def plot_striatum_performance_vs_reward():
# Measure classification accuracy of each sequence.
d1_seq_scores = []
d2_seq_scores = []
for seq_idx, seq in enumerate(datastream.sequences):
reward = datastream.rewards[seq_idx]
d1_seq_scores.append(0)
d2_seq_scores.append(0)
seqence_classification_samples = 0
for measurement in range(3):
# Add random inputs at the start of the seqence.
reset_steps = random.randrange(
min(datastream.filler_length),
max(datastream.filler_length) + 1)
reset_noise = [
random.choice(datastream.inputs)
for step in range(reset_steps)
]
seq = seq + reset_noise
for step, inp in enumerate(seq):
enc.encode(inp)
sp.compute()
tm.compute()
bg.compute(tm.active, reward=None) # No learning.
# Filter out the random noise at the start of the sequence.
if step not in range(reset_steps):
d1_seq_cls = d1_sdrc.predict(bg.d1.active)
d2_seq_cls = d2_sdrc.predict(bg.d2.active)
d1_seq_scores[seq_idx] += d1_seq_cls[
seq_idx] / np.sum(d1_seq_cls)
d2_seq_scores[seq_idx] += d2_seq_cls[
seq_idx] / np.sum(d2_seq_cls)
seqence_classification_samples += 1
d1_seq_scores[seq_idx] /= seqence_classification_samples
d2_seq_scores[seq_idx] /= seqence_classification_samples
# Plot the relationship between sequence value and which striatum
# populations learned to recognise the sequence.
plt.figure('Reward versus Striatum')
plt.subplot(1, 2, 1)
plt.title('D1')
plt.plot(datastream.rewards, d1_seq_scores, 'ro')
plt.xlabel('Sequence Reward')
plt.ylabel('Classification Accuracy')
plt.subplot(1, 2, 2)
plt.title('D2')
plt.plot(datastream.rewards, d2_seq_scores, 'bo')
plt.xlabel('Sequence Reward')
plt.ylabel('Classification Accuracy')
# RUN ONLINE.
print("Num Cycles", self.cycles)
for step in range(self.cycles):
inp, reward = next(datastream)
enc.encode(inp)
sp.compute()
tm.compute()
bg.compute(tm.active, reward)
sp.learn()
tm.learn()
if cell_death_experiment is not None and step == self.cycles // 2:
tm.active.kill_cells(cell_death_experiment)
bg.d1.active.kill_cells(cell_death_experiment)
bg.d2.active.kill_cells(cell_death_experiment)
bg.gpe.active.kill_cells(cell_death_experiment)
bg.gpi.active.kill_cells(cell_death_experiment)
print("KILLED %g %% OF CELLS" % (cell_death_experiment * 100))
event_step = step
# Measure performance.
if bg.td_error is not None:
baseline += reward**2
td_error += bg.td_error**2
td_error_total += 1
if datastream.anomallous:
anom_rand += tm.anomaly
anom_rand_total += 1
else:
anom_pred += tm.anomaly
anom_pred_total += 1
# Train and test sequence classifiers for every part of the system.
if datastream.state[0] == 'sequence':
sp_seq_cls = sp_sdrc.predict(sp.columns)
tm_seq_cls = tm_sdrc.predict(tm.active)
d1_seq_cls = d1_sdrc.predict(bg.d1.active)
d2_seq_cls = d2_sdrc.predict(bg.d2.active)
sequence_total += 1
seq_idx = datastream.state[1]
# SDR Classifier outputs a PDF. At creation, PDF may be beneath
# the minumum representable floating point value.
sp_seq_score += np.nan_to_num(sp_seq_cls[seq_idx] /
np.sum(sp_seq_cls))
tm_seq_score += np.nan_to_num(tm_seq_cls[seq_idx] /
np.sum(tm_seq_cls))
d1_seq_score += np.nan_to_num(d1_seq_cls[seq_idx] /
np.sum(d1_seq_cls))
d2_seq_score += np.nan_to_num(d2_seq_cls[seq_idx] /
np.sum(d2_seq_cls))
sp_sdrc.train(sp.columns, (seq_idx, ))
tm_sdrc.train(tm.active, (seq_idx, ))
d1_sdrc.train(bg.d1.active, (seq_idx, ))
d2_sdrc.train(bg.d2.active, (seq_idx, ))
if self.debug:
# Record everything for debugging.
input_history.append(inp)
reward_history.append(reward)
anomalous_input_history.append(datastream.anomallous)
ev_history.append(bg.expected_value)
td_error_history.append(
bg.td_error if bg.td_error is not None else 0)
anomaly_history.append(tm.anomaly)
if reward_adjustment_experiment and step == self.cycles // 2:
plot_striatum_performance_vs_reward()
plt.show()
print('ADJUSTING ALL REWARDS!')
datastream.adjust_rewards()
event_step = step
# REPORT.
sp_seq_score = sp_seq_score / sequence_total
tm_seq_score = tm_seq_score / sequence_total
d1_seq_score = d1_seq_score / sequence_total
d2_seq_score = d2_seq_score / sequence_total
anom_pred = anom_pred / anom_pred_total
anom_rand = anom_rand / anom_rand_total
baseline = (baseline / td_error_total)**.5
td_error = (td_error / td_error_total)**.5
fitness = {
'anom_pred': anom_pred,
'anom_rand': anom_rand,
'seq_sp': sp_seq_score,
'seq_tm': tm_seq_score,
'seq_d1': d1_seq_score,
'seq_d2': d2_seq_score,
'td_error': td_error / baseline,
'time': (time.time() - start_time) / 60,
'memory': memory_score,
}
if self.debug:
print(sp.statistics())
print(tm.statistics())
print(bg.statistics())
print("TD-Error Baseline", baseline, "Measured", td_error)
print(fitness)
plot_striatum_performance_vs_reward()
# Plot the Reward, Expected Value, and TD-Error.
steps = np.arange(len(input_history))
plt.figure('Reinforcement Learning Graph')
plt.title(
'Reward is Red, Expected Value is Green, TD-Error is Blue.')
plt.xlabel('Step Number')
plt.plot(steps, reward_history, 'r', steps, ev_history, 'g', steps,
td_error_history, 'b')
# Plot the Anomaly.
plt.figure('Anomaly Graph')
plt.title('Anomaly is Green, Unpredictable input is Red.')
plt.plot(steps, anomaly_history, 'g', steps,
anomalous_input_history, 'r')
plt.xlabel('Step Number')
# Smooth and plot the TD error alone.
alpha = .005
td_err_cleaned = []
avg = 0
for td_err in td_error_history:
avg = avg * (1 - alpha) + abs(td_err) * alpha
td_err_cleaned.append(avg)
plt.figure('TD Error')
plt.title('Exponential Rolling average of |TD Error|, alpha = %g' %
alpha)
plt.plot(steps, td_err_cleaned, 'b')
plt.xlabel('Step Number')
if event_step is not None:
plt.axvline(event_step)
plt.show()
return fitness
def evaluate(self, debug):
# Setup test and train datasets and perform lexical analysis. First get
# full text of training dataset into a string.
if self.dataset == 'gutenberg':
text_stream = read_corpus(debug=debug)
elif self.dataset == 'states':
text_stream = state_name_reader(debug=debug)
train_dataset = []
for i in range(self.train_time):
char = next(text_stream)
train_dataset.append(char)
train_dataset = ''.join(train_dataset)
# Search for words in the dataset. Store the words as keys in a
# histogram of word occurances.
word_regex = r"\w(\w|')*"
word_iter = re.finditer(word_regex, train_dataset)
word_hist = {}
# train_word_spans stores where each word is located, list of pairs of
# (start, end) index into train_dataset.
train_word_spans = []
for match in word_iter:
span = match.span() # Returns pair of (start-index, end-index)
word = train_dataset[span[0]:span[1]]
if word not in word_hist:
word_hist[word] = 0
word_hist[word] += 1
train_word_spans.append(span)
# Sort words by the number of times the occur in the train_dataset.
# Break ties randomly.
word_list = list(word_hist.keys())
word_freq = [
-(word_hist[word] + random.random()) for word in word_list
]
word_rank = np.take(word_list, np.argsort(word_freq))
# Get word_list and word_freq out of memory, from here on use word_rank & word_hist.
word_list = None
word_freq = None
# Select some common words to test vocabulary with.
test_words = word_rank[:self.test_words].tolist()
# Assign each vocabulary word an integer identifier. A test words
# identifier doubles as its index into the test_words list.
if False:
test_words.sort() # Make the index easier for humans to use.
# The first entry is special B/C when the SDRC can't identify the word
# at all, it outputs all zeros. Then np.argmax() outputs as index 0 as
# the best prediction.
test_words.insert(0, "WORD_UNKNOWN")
word_hist[test_words[0]] = 0
test_word_id_lookup = {
word: index
for index, word in enumerate(test_words)
}
# Search for examples of the vocabulary words used in sentances. First
# read a large amount of sample text. Only relevant sections of the
# test_dataset are used, the ranges of text are stored in variable
# test_sentance_spans.
test_dataset = []
for i in range(int(self.test_time)):
char = next(text_stream)
test_dataset.append(char)
test_dataset = ''.join(test_dataset)
word_iter = re.finditer(word_regex, test_dataset)
# The following two lists hold pairs of (start, end) slice indexes into
# test_dataset. They are NOT the same length because overlapping
# test_sentance_spans are merged into a single example containing
# several of the vocabulary words.
test_word_spans = [] # Spans of just the test vocabulary words.
test_sentance_spans = [
] # Spans of test words with preceding context included.
test_hist = {word: 0 for word in test_words}
for match in word_iter:
span = match.span() # Returns pair of (start-index, end-index)
start, end = span
word = test_dataset[start:end]
if word not in test_word_id_lookup.keys():
continue
# Ignore test vocabulary words after they've been seen many times.
if test_hist[word] >= self.test_sample:
continue
test_hist[word] += 1
test_word_spans.append(span)
context_start = max(0, start - self.min_context)
if test_sentance_spans and test_sentance_spans[-1][
1] >= context_start:
# Extend the last test sentance and test this additional word using it.
context_start = test_sentance_spans[-1][0]
test_sentance_spans[-1] = (context_start, end)
else:
# Add a new test sentance.
test_sentance_spans.append((context_start, end))
len_test_dataset = sum(e - s for s, e in test_sentance_spans)
if debug:
print('Training dataset size:', self.train_time, 'characters,',
len(train_word_spans), 'words,', len(word_hist),
'unique words.')
print('Test vocabulary size:', len(test_words), 'words.')
min_freq = min(word_hist[word] for word in test_words[1:])
max_freq = max(word_hist[word] for word in test_words[1:])
print('Test vocabulary samples:',
', '.join(random.sample(test_words[1:], 6)) + '.')
print(
'Test vocabulary min & max occurances in training dataset: %d - %d.'
% (min_freq, max_freq))
test_hist_values = list(test_hist[word] for word in test_words[1:])
min_freq = min(test_hist_values)
avg_freq = np.mean(test_hist_values)
max_freq = max(test_hist_values)
print(
'Test vocabulary min/mean/max occurances in testing dataset: %d / %.1f / %d.'
% (min_freq, avg_freq, max_freq))
print('Test dataset size:', len_test_dataset, 'characters,',
len(test_word_spans), 'vocabulary words.')
print('Test sentance average length: %.1f characters.' %
(len_test_dataset / len(test_sentance_spans)))
if self.list_test_words:
print('Index) Word, Train samples, Test samples.')
if False:
# Sort by number of samples in dataset.
# TODO: This would be more useful if it sorted the actual test_words list.
test_freq = [-test_hist[word] for word in test_words]
test_rank = np.take(test_words, np.argsort(test_freq))
ordered_words = test_rank
else:
# Sort by index.
ordered_words = test_words
# Print index of test words.
for word in ordered_words:
index = test_word_id_lookup[word]
train_samples = word_hist[word]
test_samples = test_hist[word]
fmt_str = '%3d) %-15s\t%2d, %2d'
print(fmt_str % (index, word, train_samples, test_samples))
if True:
# Look at some test sentances
sample_spans = random.sample(
test_sentance_spans, min(10, len(test_sentance_spans)))
sample_sentances = [
test_dataset[s[0]:s[1]] for s in sample_spans
]
print('Sample test sentances:\n\t',
'\n\n\t'.join(sample_sentances))
print()
# After seeing all of the words in either dataset, wait forever for the
# next word, or until the dataset is finished playing. This case is
# needed when the dataset ends with white space.
train_word_spans.append((float('inf'), float('inf')))
test_word_spans.append((float('inf'), float('inf')))
# if len(test_words) != self.test_words + 1:
# raise ValueError('Could not find %d test words'%self.test_words)
# Setup AI.
timer = genetics.speed_fitness(self.time_limit / 3, self.time_limit)
enc = encoders.EnumEncoder(self.enc_bits,
self.enc_sparsity,
diag=False)
# Make the context SDR which both L4 and L23 use to predict the future.
context_size = self.l4.cells + self.l23.cells
context = SDR((context_size, ))
l4 = unified.Unified(
self.l4,
input_sdr=enc.output_sdr,
context_sdr=context,
macro_columns=(1, ),
radii=self.l4_radii,
)
l23 = unified.Unified(
self.l23,
input_sdr=l4.active,
context_sdr=context,
macro_columns=(1, ),
radii=self.l23_radii,
)
l4_sdrc = classifiers.SDR_Classifier(self.sdrc, l4.active,
(len(test_words), ), 'index')
l23_sdrc = classifiers.SDR_Classifier(self.sdrc, l23.active,
(len(test_words), ), 'index')
def reset():
l4.reset()
l23.reset()
def compute(learn=True):
context.assign_flat_concatenate([l4.active, l23.active])
if self.l4_only:
# Test L4 in isolation, Disable feedback from L2/3 to L4.
zeros_like_l23 = SDR(l23.active)
zeros_like_l23.zero()
context.assign_flat_concatenate([l4.active, zeros_like_l23])
l4.compute()
l23.compute()
if learn:
l4.learn()
l23.learn()
if debug:
print('SDR DEBUG:', sdr.debug)
if self.l4_only:
print("L4 Isolated, Disabled L2/3 -> L4 Feedback.")
if False:
print('L4', l4.statistics())
print('L23', l23.statistics())
# Train by reading books.
if self.train_no_stability:
self.l23.min_stability = 0
assert (debug)
print('L23 min stability set to', self.l23.min_stability)
if debug:
print('Training ...')
word = None # Current word or None, AI trains to predict this variable.
word_index = None # Index of current word in test_data, or None if its not a test word.
word_span_index = 0 # Index of current word in train_dataset
reset()
for step in range(self.train_time):
# Determine the current word.
start, end = train_word_spans[word_span_index]
if step == start:
word = train_dataset[start:end]
try:
word_index = (test_word_id_lookup[word], )
except KeyError: # Word is not in vocabulary test words, SDRC should ignore it.
word_index = None
if step == end:
word = None
word_index = None
word_span_index += 1
# Process the next letter of the book.
char = train_dataset[step]
enc.encode(char)
compute(learn=True)
if word_index is not None and step == end - 1:
l4_sdrc.train(input_sdr=None, out=word_index)
l23_sdrc.train(input_sdr=None, out=word_index)
# Test. Measure:
# 1) Stability,
# 2) Anomaly,
# 3) Word recognition accuracy and cross-catagory confusion.
real_min_stab = l23.args.min_stability
if self.test_no_stability:
l23.args.min_stability = 0
if debug:
print('Testing ...')
if l23.args.min_stability != real_min_stab:
print('L23 min stability changed to', l23.args.min_stability)
else:
print('L23 min stability remains at', l23.args.min_stability)
l23_stability = 0. # Accumulates the L2/3 stability.
l4_anomaly = 0. # Accumulates the L4 anomaly.
l23_anomaly = 0. # Accumulates the L2/3 anomaly.
l4_accuracy = 0. # Accumulates the L4 word classificatioon accuracy.
l23_accuracy = 0. # Accumulates the L2/3 word classificatioon accuracy.
max_accuracy = 0. # Number of samples accumulated in variable 'l23_accuracy'.
l4_end_accuracy = 0. # Like 'l4_accuracy' but only measured on the final letter of the word.
l23_end_accuracy = 0. # Like 'l23_accuracy' but only measured on the final letter of the word.
max_end_accuracy = 0. # Number of samples accumulated in variable 'l23_end_accuracy'.
l23_confusion = np.zeros((len(test_words), len(test_words)))
l4_confusion = np.zeros((len(test_words), len(test_words)))
next_span_index = 0 # Index of current word in test_word_spans (or next word if not currently on a word).
for sentance_start, sentance_end in test_sentance_spans:
reset()
word_index = None # Index of current word, or None.
for index in range(sentance_start, sentance_end):
# Determine the current word. Allow words to immediately follow
# each other, they in case they're seperated by a reset and zero
# characters of context.
word_start, word_end = test_word_spans[next_span_index]
if index >= word_end:
word_index = None
next_span_index += 1
word_start, word_end = test_word_spans[next_span_index]
if index >= word_start:
word = test_dataset[word_start:word_end]
word_index = test_word_id_lookup[word]
# Process the current character.
char = test_dataset[index]
enc.encode(char)
compute(learn=False)
# Measure.
if real_min_stab > 0:
l23_stability += min(l23.stability,
real_min_stab) / real_min_stab
else:
l23_stability += 1
l4_anomaly += l4.anomaly
l23_anomaly += l23.anomaly
if word_index is not None:
l4_prediction = l4_sdrc.predict()
l23_prediction = l23_sdrc.predict()
l4_best_guess = np.argmax(l4_prediction)
l23_best_guess = np.argmax(l23_prediction)
if l23_best_guess == word_index:
l23_accuracy += 1
if index == word_end - 1:
l23_end_accuracy += 1
if l4_best_guess == word_index:
l4_accuracy += 1
if index == word_end - 1:
l4_end_accuracy += 1
max_accuracy += 1
if index == word_end - 1:
max_end_accuracy += 1
# Update confusion matixes. Prediction is a PDF, sum must equal 1.
if True:
l23_confusion[word_index, l23_best_guess] += 1
if index == word_end - 1:
l4_confusion[word_index, l4_best_guess] += 1
else:
l23_prediction_sum = np.sum(l23_prediction)
if l23_prediction_sum != 0.:
l23_prediction /= l23_prediction_sum
l23_confusion[word_index, :] += l23_prediction
l4_prediction_sum = np.sum(l4_prediction)
if l4_prediction_sum != 0.:
l4_prediction /= l4_prediction_sum
l4_confusion[word_index, :] += l4_prediction
# Divide all accumulators by the number of samples added to them.
l23_stability /= len_test_dataset
l23_accuracy /= max_accuracy
l23_end_accuracy /= max_end_accuracy
l23_anomaly /= len_test_dataset
l4_accuracy /= max_accuracy
l4_end_accuracy /= max_end_accuracy
l4_anomaly /= len_test_dataset
for label_idx, label in enumerate(test_words):
# Divide by the number of PDF's which have accumulated at each
# label, each PDF has sum of 1.
l23_num_samples = np.sum(l23_confusion[label_idx, :])
if l23_num_samples != 0:
l23_confusion[label_idx, :] /= l23_num_samples
l4_num_samples = np.sum(l4_confusion[label_idx, :])
if l4_num_samples != 0:
l4_confusion[label_idx, :] /= l4_num_samples
def plot_sentance_stability(string):
plt.figure('Stability')
plt.ylim(-0.01, 1.01)
plt.xlim(-0.5, len(string) - 0.5)
plt.xlabel('Time')
plt.ylabel('L2/3 Overlap')
plt.axhline(real_min_stab)
stability = []
confidence = []
anomaly = []
reset()
for step, char in enumerate(string):
enc.encode(char)
compute(learn=False)
stability.append(l23.stability)
anomaly.append(l23.anomaly)
prediction = l23_sdrc.predict()
best_guess = test_words[np.argmax(prediction)]
confidence.append(np.max(prediction) / np.sum(prediction))
#
plt.axvline(step + .5, color='grey', alpha=0.25)
plt.text(step - 0.25, .98, char)
if char.isalpha():
plt.text(step - 0.25,
0.95,
best_guess,
rotation='vertical')
# TODO: Determine which steps it learns on by looking at the dataset.
elif step - 1 >= 0 and string[step - 1].isalpha():
plt.axvspan(step - 1.5,
step - .5,
color='yellow',
alpha=0.5)
plt.axvspan(step - .5, step + .5, color='yellow', alpha=0.5)
plt.plot(
np.arange(len(string)),
stability,
'r-',
)
# np.arange(len(string)), confidence, 'b-',)
# np.arange(len(string)), anomaly, 'b-',)
# plt.title('L2/3 Overlap is Red, Confidence is Blue')
# plt.title('L2/3 Overlap is Red, Anomaly is Blue')
plt.title((
'Top: Input Letter, Middle: Best Guess,\n' +
'Bottom Graph: Red Line L2/3 Stability, Blue Line: Target Stability, Learning Enabled on Yellow Steps.'
))
# Report.
fitness = {
'L23_stability': l23_stability,
'L23_accuracy': l23_accuracy,
'L23_end_accuracy': l23_end_accuracy,
'L23_anomaly': l23_anomaly,
'L4_accuracy': l4_accuracy,
'L4_end_accuracy': l4_end_accuracy,
'L4_anomaly': l4_anomaly,
'speed': timer.done(),
'memory': genetics.memory_fitness(2e9, 3e9),
}
if debug:
print()
print('L4', l4.statistics())
print('L23', l23.statistics())
span = random.choice(test_sentance_spans)
sentance = test_dataset[span[0]:span[1]]
sentance = sentance[
-100:] # Don't show too much text in one figure.
if self.show_typos:
sentance = ' '.join(
[mutate_word(w) for w in sentance.split(' ')])
plot_sentance_stability(sentance)
plt.figure('L23 Confusion Matrix')
plt.imshow(l23_confusion, interpolation='nearest')
plt.xlabel('Prediction')
plt.ylabel('Label')
plt.figure('L4 Confusion Matrix')
plt.imshow(l4_confusion, interpolation='nearest')
plt.xlabel('Prediction')
plt.ylabel('Label')
plt.show()
return fitness
def evaluate_recognition(parameters, debug=False):
data = Dataset('datasets/textures/')
train_data, test_data = data.split_dataset(75, 25, verbosity=0)
max_dist = -32
timer = genetics.speed_fitness(threshold = 30, maximum = 2*60)
eye_params = parameters.eye
sp_params = parameters.sp
sdrc_params = parameters.sdrc
sensor = htm.EyeSensor(eye_params)
sp = htm.SpatialPooler(sp_params, sensor.view_shape)
classifier = htm.SDR_Classifier(sdrc_params, sp.column_dimensions, (len(data.names),), output_type='pdf')
baseline_classifier = htm.RandomOutputClassifier((len(data.names),))
time_per_image = 10
training_time = int(round(time_per_image * parameters.image_cycles))
if debug:
training_time = time_per_image * 2
for i in range(training_time):
# Setup for a new image
if i % time_per_image == 0:
train_data.random_image()
sensor.new_image(train_data.current_image)
eye_positions = train_data.points_near_label(
max_dist = max_dist,
number=time_per_image)
# Setup and compute for this cycle
sensor.randomize_view() # Random scale and orientation
sensor.position = eye_positions.pop()
labels_pdf = train_data.sample_labels(sensor.input_space_sample_points())
view = sensor.view()
columns = sp.compute(view)
classifier.train(columns, labels_pdf)
baseline_classifier.train(labels_pdf)
# Sample the memory usage.
if i == time_per_image-1:
memory_performance = genetics.memory_fitness([sensor, sp, classifier], 0.5e9, 2e9)
testing_time = min(1000, time_per_image * 100)
if debug:
testing_time = time_per_image * 2
score = 0 # Regular accuracy, summed results of dataset.compare_label_samples
baseline_score = 0
for i in range(testing_time):
# Setup for a new image
if i % time_per_image == 0:
test_data.random_image()
sensor.new_image(test_data.current_image)
eye_positions = test_data.points_near_label(
max_dist = max_dist,
number = time_per_image)
# Setup and compute for this cycle
sensor.randomize_view()
sensor.position = eye_positions.pop()
view = sensor.view()
columns = sp.compute(view, learn=False)
prediction = classifier.predict(columns)
baseline = baseline_classifier.predict()
# Compare results to labels
labels = test_data.sample_labels(sensor.input_space_sample_points())
score += test_data.compare_label_samples(prediction, labels)
baseline_score += test_data.compare_label_samples(baseline, labels)
score /= testing_time
baseline_score /= testing_time
return {
'recognition': score,
'baseline': baseline_score,
'time': timer.done(),
'memory': memory_performance,
}