def testZeroActiveColumns(self):
tm = TemporalMemory(
columnDimensions=[32],
cellsPerColumn=4,
activationThreshold=3,
initialPermanence=.21,
connectedPermanence=.5,
minThreshold=2,
maxNewSynapseCount=3,
permanenceIncrement=.10,
permanenceDecrement=.10,
predictedSegmentDecrement=0.0,
seed=42)
previousActiveColumns = [0]
previousActiveCells = [0, 1, 2, 3]
expectedActiveCells = [4]
segment = tm.createSegment(expectedActiveCells[0])
tm.connections.createSynapse(segment, previousActiveCells[0], .5)
tm.connections.createSynapse(segment, previousActiveCells[1], .5)
tm.connections.createSynapse(segment, previousActiveCells[2], .5)
tm.connections.createSynapse(segment, previousActiveCells[3], .5)
tm.compute(previousActiveColumns, True)
self.assertFalse(len(tm.getActiveCells()) == 0)
self.assertFalse(len(tm.getWinnerCells()) == 0)
self.assertFalse(len(tm.getPredictiveCells()) == 0)
zeroColumns = []
tm.compute(zeroColumns, True)
self.assertTrue(len(tm.getActiveCells()) == 0)
self.assertTrue(len(tm.getWinnerCells()) == 0)
self.assertTrue(len(tm.getPredictiveCells()) == 0)
def testZeroActiveColumns(self):
tm = TemporalMemory(
columnDimensions=[32],
cellsPerColumn=4,
activationThreshold=3,
initialPermanence=.21,
connectedPermanence=.5,
minThreshold=2,
maxNewSynapseCount=3,
permanenceIncrement=.10,
permanenceDecrement=.10,
predictedSegmentDecrement=0.0,
seed=42)
previousActiveColumns = [0]
previousActiveCells = [0, 1, 2, 3]
expectedActiveCells = [4]
segment = tm.createSegment(expectedActiveCells[0])
tm.connections.createSynapse(segment, previousActiveCells[0], .5)
tm.connections.createSynapse(segment, previousActiveCells[1], .5)
tm.connections.createSynapse(segment, previousActiveCells[2], .5)
tm.connections.createSynapse(segment, previousActiveCells[3], .5)
tm.compute(previousActiveColumns, True)
self.assertFalse(len(tm.getActiveCells()) == 0)
self.assertFalse(len(tm.getWinnerCells()) == 0)
self.assertFalse(len(tm.getPredictiveCells()) == 0)
zeroColumns = []
tm.compute(zeroColumns, True)
self.assertTrue(len(tm.getActiveCells()) == 0)
self.assertTrue(len(tm.getWinnerCells()) == 0)
self.assertTrue(len(tm.getPredictiveCells()) == 0)
def testActivateCorrectlyPredictiveCells(self):
tm = TemporalMemory(
columnDimensions=[32],
cellsPerColumn=4,
activationThreshold=3,
initialPermanence=.21,
connectedPermanence=.5,
minThreshold=2,
maxNewSynapseCount=3,
permanenceIncrement=.10,
permanenceDecrement=.10,
predictedSegmentDecrement=0.0,
seed=42)
previousActiveColumns = [0]
activeColumns = [1]
previousActiveCells = [0,1,2,3]
expectedActiveCells = [4]
activeSegment = tm.createSegment(expectedActiveCells[0])
tm.connections.createSynapse(activeSegment, previousActiveCells[0], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[1], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[2], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[3], .5)
tm.compute(previousActiveColumns, True)
self.assertEqual(expectedActiveCells, tm.getPredictiveCells())
tm.compute(activeColumns, True)
self.assertEqual(expectedActiveCells, tm.getActiveCells())
def testActivateCorrectlyPredictiveCells(self):
tm = TemporalMemory(
columnDimensions=[32],
cellsPerColumn=4,
activationThreshold=3,
initialPermanence=.21,
connectedPermanence=.5,
minThreshold=2,
maxNewSynapseCount=3,
permanenceIncrement=.10,
permanenceDecrement=.10,
predictedSegmentDecrement=0.0,
seed=42)
previousActiveColumns = [0]
activeColumns = [1]
previousActiveCells = [0,1,2,3]
expectedActiveCells = [4]
activeSegment = tm.createSegment(expectedActiveCells[0])
tm.connections.createSynapse(activeSegment, previousActiveCells[0], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[1], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[2], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[3], .5)
tm.compute(previousActiveColumns, True)
self.assertEqual(expectedActiveCells, tm.getPredictiveCells())
tm.compute(activeColumns, True)
self.assertEqual(expectedActiveCells, tm.getActiveCells())
def testNoChangeToMatchingSegmentsInPredictedActiveColumn(self):
tm = TemporalMemory(columnDimensions=[32],
cellsPerColumn=4,
activationThreshold=3,
initialPermanence=.21,
connectedPermanence=.50,
minThreshold=2,
maxNewSynapseCount=3,
permanenceIncrement=.10,
permanenceDecrement=.10,
predictedSegmentDecrement=0.0,
seed=42)
previousActiveColumns = [0]
activeColumns = [1]
previousActiveCells = [0, 1, 2, 3]
expectedActiveCells = [4]
otherburstingCells = [5, 6, 7]
activeSegment = tm.connections.createSegment(expectedActiveCells[0])
tm.connections.createSynapse(activeSegment, previousActiveCells[0], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[1], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[2], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[3], .5)
matchingSegmentOnSameCell = tm.connections.createSegment(
expectedActiveCells[0])
s1 = tm.connections.createSynapse(matchingSegmentOnSameCell,
previousActiveCells[0], .3)
s2 = tm.connections.createSynapse(matchingSegmentOnSameCell,
previousActiveCells[1], .3)
matchingSegmentOnOtherCell = tm.connections.createSegment(
otherburstingCells[0])
s3 = tm.connections.createSynapse(matchingSegmentOnOtherCell,
previousActiveCells[0], .3)
s4 = tm.connections.createSynapse(matchingSegmentOnOtherCell,
previousActiveCells[1], .3)
tm.compute(previousActiveColumns, True)
self.assertEqual(expectedActiveCells, tm.getPredictiveCells())
tm.compute(activeColumns, True)
self.assertAlmostEqual(.3,
tm.connections.dataForSynapse(s1).permanence)
self.assertAlmostEqual(.3,
tm.connections.dataForSynapse(s2).permanence)
self.assertAlmostEqual(.3,
tm.connections.dataForSynapse(s3).permanence)
self.assertAlmostEqual(.3,
tm.connections.dataForSynapse(s4).permanence)
def testNoChangeToMatchingSegmentsInPredictedActiveColumn(self):
tm = TemporalMemory(
columnDimensions=[32],
cellsPerColumn=4,
activationThreshold=3,
initialPermanence=.21,
connectedPermanence=.50,
minThreshold=2,
maxNewSynapseCount=3,
permanenceIncrement=.10,
permanenceDecrement=.10,
predictedSegmentDecrement=0.0,
seed=42)
previousActiveColumns = [0]
activeColumns = [1]
previousActiveCells = [0,1,2,3]
expectedActiveCells = [4]
otherburstingCells = [5,6,7]
activeSegment = tm.createSegment(expectedActiveCells[0])
tm.connections.createSynapse(activeSegment, previousActiveCells[0], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[1], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[2], .5)
tm.connections.createSynapse(activeSegment, previousActiveCells[3], .5)
matchingSegmentOnSameCell = tm.createSegment(
expectedActiveCells[0])
s1 = tm.connections.createSynapse(matchingSegmentOnSameCell,
previousActiveCells[0], .3)
s2 = tm.connections.createSynapse(matchingSegmentOnSameCell,
previousActiveCells[1], .3)
matchingSegmentOnOtherCell = tm.createSegment(
otherburstingCells[0])
s3 = tm.connections.createSynapse(matchingSegmentOnOtherCell,
previousActiveCells[0], .3)
s4 = tm.connections.createSynapse(matchingSegmentOnOtherCell,
previousActiveCells[1], .3)
tm.compute(previousActiveColumns, True)
self.assertEqual(expectedActiveCells, tm.getPredictiveCells())
tm.compute(activeColumns, True)
self.assertAlmostEqual(.3, tm.connections.dataForSynapse(s1).permanence)
self.assertAlmostEqual(.3, tm.connections.dataForSynapse(s2).permanence)
self.assertAlmostEqual(.3, tm.connections.dataForSynapse(s3).permanence)
self.assertAlmostEqual(.3, tm.connections.dataForSynapse(s4).permanence)
tm = TemporalMemory(columnDimensions=columnDimensions)
c = SDRClassifier(steps=[1], alpha=0.1, actValueAlpha=0.1, verbosity=0)
x_true = x[1:]
x_predict = np.zeros(len(x) - 1)
for i, xi in tqdm(enumerate(x[:-1])):
encoded = encoder.encode(xi)
bucketIdx = np.where(encoded > 0)[0][0]
spd = np.zeros(columnDimensions[0])
sp.compute(encoded, True, spd)
active_indices = np.where(spd > 0)[0]
tm.compute(active_indices)
active_cell_indices = tm.getActiveCells()
predictive_cell_indices = tm.getPredictiveCells()
patternNZ = np.asarray(active_cell_indices)
patternNZ = np.append(patternNZ, predictive_cell_indices)
patternNZ = patternNZ.astype(np.int)
patternNZ = list(set(patternNZ))
result = c.compute(recordNum=i,
patternNZ=patternNZ,
classification={
"bucketIdx": bucketIdx,
"actValue": xi
},
learn=True,
infer=True)
topPredictions = sorted(zip(result[1], result["actualValues"]),
reverse=True)[0]
# We repeat the sequence 10 times
for i in range(10):
# Send each letter in the sequence in order
for j in range(5):
activeColumns = set([i for i, j in zip(count(), x[j]) if j == 1])
# The compute method performs one step of learning and/or inference. Note:
# here we just perform learning but you can perform prediction/inference and
# learning in the same step if you want (online learning).
tm.compute(activeColumns, learn = True)
# The following print statements can be ignored.
# Useful for tracing internal states
print("active cells " + str(tm.getActiveCells()))
print("predictive cells " + str(tm.getPredictiveCells()))
print("winner cells " + str(tm.getWinnerCells()))
print("# of active segments " + str(tm.connections.numSegments()))
# The reset command tells the TM that a sequence just ended and essentially
# zeros out all the states. It is not strictly necessary but it's a bit
# messier without resets, and the TM learns quicker with resets.
tm.reset()
#######################################################################
#
# Step 3: send the same sequence of vectors and look at predictions made by
# temporal memory
for j in range(5):
print "\n\n--------","ABCDE"[j],"-----------"
# We repeat the sequence 10 times
for i in range(10):
# Send each letter in the sequence in order
for j in range(5):
activeColumns = set([i for i, j in zip(count(), x[j]) if j == 1])
# The compute method performs one step of learning and/or inference. Note:
# here we just perform learning but you can perform prediction/inference and
# learning in the same step if you want (online learning).
tm.compute(activeColumns, learn=True)
# The following print statements can be ignored.
# Useful for tracing internal states
print("active cells " + str(tm.getActiveCells()))
print("predictive cells " + str(tm.getPredictiveCells()))
print("winner cells " + str(tm.getWinnerCells()))
print("# of active segments " + str(tm.connections.numSegments()))
# The reset command tells the TM that a sequence just ended and essentially
# zeros out all the states. It is not strictly necessary but it's a bit
# messier without resets, and the TM learns quicker with resets.
tm.reset()
#######################################################################
#
# Step 3: send the same sequence of vectors and look at predictions made by
# temporal memory
for j in range(5):
print "\n\n--------", "ABCDE"[j], "-----------"
print "Raw input vector : " + formatRow(x[j])
# We repeat the sequence 10 times
for i in range(10):
# Send each letter in the sequence in order
for j in range(5):
activeColumns = set([i for i, j in zip(count(), x[j]) if j == 1])
# The compute method performs one step of learning and/or inference. Note:
# here we just perform learning but you can perform prediction/inference and
# learning in the same step if you want (online learning).
tm.compute(activeColumns, learn = True)
# The following print statements can be ignored.
# Useful for tracing internal states
print("active cells " + str(tm.getActiveCells()))
print("predictive cells " + str(tm.getPredictiveCells()))
print("winner cells " + str(tm.getWinnerCells()))
print("# of active segments " + str(tm.connections.numSegments()))
# The reset command tells the TM that a sequence just ended and essentially
# zeros out all the states. It is not strictly necessary but it's a bit
# messier without resets, and the TM learns quicker with resets.
tm.reset()
#######################################################################
#
# Step 3: send the same sequence of vectors and look at predictions made by
# temporal memory
for j in range(5):
print "\n\n--------","ABCDE"[j],"-----------"
for x in range(len(Data)):
encoder = multiencode(var_encoders, Data, x)
# e_val = RDSE.encode(Data['value'][x])
# e_tod = TODE.encode(Data.index[x])
# e_wend = WENDE.encode(Data.index[x])
# encoder = np.concatenate([e_val])
SP.compute(encoder, False, active_columns)
col_index = active_columns.nonzero()[0]
TM.compute(col_index, learn=True)
if x > 0:
inter = set(col_index).intersection(Prev_pred_col)
inter_l = len(inter)
active_l = len(col_index)
A_score[x] = 1 - (inter_l / active_l)
Data.iat[x, -2] = A_score[x]
Prev_pred_col = list(set(x // cell_col for x in TM.getPredictiveCells()))
end = time.time()
print(end - start)
W = 72
W_prim = 5
eps = 1e-6
AL_score = np.zeros(len(Data))
for x in range(len(Data)):
if x > 0:
W_vec = A_score[max(0, x - W):x]
W_prim_vec = A_score[max(0, x - W_prim):x]
AL_score[x] = 1 - 2 * norm.sf(
abs(np.mean(W_vec) - np.mean(W_prim_vec)) /
def HTM_AD(
Data='Test',
vars={'value': ['num']},
prec_param=5,
pooler_out=2024, # Number of columns of the pooler output
cell_col=5, # HTM cells per column
W=72, # Window parameter
W_prim=5, # Local window for anomaly detection likelihood
eps=1e-6, # to Avoid by zero divisions
athreshold=0.95):
"""
This function performs HTM based anomaly detection on a time series provided
:param Data:
:param vars: Possible values: num, tod, weekend
:param prec_param: A parameter that defines how much precision the number encoder has
The encoder precision depends on the variability of the data,
The real precision is computed taking into account both the precision parameter and data std
A high precision might mean a high error at predicting the variable value in noisy variables
:param pooler_out: Number of columns of the pooler output
:param cell_col: HTM cells per column
:param W: Window parameter
:param W_prim: Local window for anomaly detection likelihood
:param eps: to Avoid by zero divisions
:param athreshold: To classify based on anomaly likelihood whether there is an anomaly or not
:return: The Data + 3 columns
Anomaly: indicates the error of within the value predicted by the HTM network
Anomaly_likelihood: indicates the likelihood of the data into being anomalous
Anomaly_flag: classifies the data in anomalous vs non anomalous
"""
if Data == 'Test': # If there is not data available, simply loads the temperature benchmark dataset
# Import data
Data = pd.read_csv('anomaly_API/Data/sample.csv',
parse_dates=True,
index_col='timestamp')
Data = Data.resample('H').bfill().interpolate()
TODE = DateEncoder(timeOfDay=(21, 1))
WENDE = DateEncoder(weekend=21)
var_encoders = set()
# Spatial Pooler Parameters
for x in vars:
for y in vars[x]:
if y == 'num':
exec(
"RDSE_" + x +
" = RandomDistributedScalarEncoder(resolution=Data['" + x +
"'].std()/prec_param)", locals(), globals())
var_encoders.add(Encoder(x, ["RDSE_" + x]))
elif y == 'weekend':
var_encoders.add(Encoder(x, ["WENDE"]))
elif y == 'tod':
var_encoders.add(Encoder(x, ["TODE"]))
else:
return {"error": "Variable encoder type is not recognized "}
encoder_width = 0 # Computes encoder width
for x in var_encoders:
for y in x.encoders:
exec("s = " + y + ".getWidth()", locals(), globals())
encoder_width += s
SP = SpatialPooler(
inputDimensions=encoder_width,
columnDimensions=pooler_out,
potentialPct=0.8,
globalInhibition=True,
numActiveColumnsPerInhArea=pooler_out //
50, # Gets 2% of the total area
boostStrength=1.0,
wrapAround=False)
TM = TemporalMemory(columnDimensions=(pooler_out, ),
cellsPerColumn=cell_col)
Data['Anomaly'] = 0.0
Data['Anomaly_Likelihood'] = 0.0
# Train Spatial Pooler
print("Spatial pooler learning")
start = time.time()
active_columns = np.zeros(pooler_out)
for x in range(len(Data)):
encoder = multiencode(var_encoders, Data, x)
SP.compute(encoder, True, active_columns)
end = time.time()
print(end - start)
# Temporal pooler
print("Temporal pooler learning")
start = time.time()
A_score = np.zeros(len(Data))
for x in range(len(Data)):
encoder = multiencode(var_encoders, Data, x)
SP.compute(encoder, False, active_columns)
col_index = active_columns.nonzero()[0]
TM.compute(col_index, learn=True)
if x > 0:
inter = set(col_index).intersection(Prev_pred_col)
inter_l = len(inter)
active_l = len(col_index)
A_score[x] = 1 - (inter_l / active_l)
Data.iat[x, -2] = A_score[x]
Prev_pred_col = list(
set(x // cell_col for x in TM.getPredictiveCells()))
end = time.time()
print(end - start)
AL_score = np.zeros(len(Data))
# Computes the likelihood of the anomaly
for x in range(len(Data)):
if x > 0:
W_vec = A_score[max(0, x - W):x]
W_prim_vec = A_score[max(0, x - W_prim):x]
AL_score[x] = 1 - 2 * norm.sf(
abs(np.mean(W_vec) - np.mean(W_prim_vec)) /
max(np.std(W_vec), eps))
Data.iat[x, -1] = AL_score[x]
Data['Anomaly_flag'] = athreshold < Data['Anomaly_Likelihood']
return Data
start = time.time()
A_score = np.zeros(len(Data))
for x in range(len(Data)):
encoder = multiencode(var_encoders, Data, x)
SP.compute(encoder, False, active_columns)
col_index = active_columns.nonzero()[0]
TM.compute(col_index, learn=True)
if x > 0:
inter = set(col_index).intersection(Prev_pred_col)
inter_l = len(inter)
active_l = len(col_index)
A_score[x] = 1 - (inter_l / active_l)
Data.iat[x, -2] = A_score[x]
Prev_pred_col = list(
set(x // cell_col for x in TM.getPredictiveCells()))
end = time.time()
print(end - start)
AL_score = np.zeros(len(Data))
for x in range(len(Data)):
if x > 0:
W_vec = A_score[max(0, x - W):max(x - W_prim, 0)]
W_prim_vec = A_score[max(0, x - W_prim):x]
AL_score[x] = 1 - 2 * norm.sf(
abs(np.mean(W_vec) - np.mean(W_prim_vec)) /
max(np.std(W_vec), eps))
Data.iat[x, -1] = AL_score[x]
dataplot = Data.copy()
encoder_output = np.concatenate((bits_time, bits_scalar))
####################################################
sdr_output = np.zeros(N_COLUMNS)
sp.compute(encoder_output, True, sdr_output)
active_columns = np.nonzero(sdr_output)[0]
####################################################
tm.compute(active_columns, learn=True)
####################################################
anom_score[i] = anomaly_score.compute(tm.getActiveCells(),
tm.getPredictiveCells())
anom_logscore[i] = anomaly_likelihood.computeLogLikelihood(anom_score[i])
if i % 100 == 0:
print(i)
anom_score_futuro = np.zeros((5000 + 1, )) ## excluir isso aqui
anom_logscore_futuro = np.zeros((5000 + 1, )) ## excluir isso aqui
for i, linha in enumerate(
sign[7220000:7225000, :]
): ##esse for é o do arquivo teste pra ver se é possível encontrar as anomalias
#excluir esse for depois
scalar_encoder.encodeIntoArray(linha[1], bits_scalar)
