def testVeryFewScores(self): """ This calls estimateAnomalyLikelihoods and updateAnomalyLikelihoods with one or no scores. """ # Generate an estimate using two data points data1 = _generateSampleData(mean=42.0, variance=1e-10) _, _, estimatorParams = (an.estimateAnomalyLikelihoods(data1[0:2])) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) # Check that the estimated mean is that value dParams = estimatorParams["distribution"] self.assertWithinEpsilon(dParams["mean"], data1[0][2]) # Can't generate an estimate using no data points data1 = numpy.zeros(0) with self.assertRaises(ValueError): an.estimateAnomalyLikelihoods(data1) # Can't update with no scores with self.assertRaises(ValueError): an.updateAnomalyLikelihoods(data1, estimatorParams)
def testVeryFewScores(self): """ This calls estimateAnomalyLikelihoods and updateAnomalyLikelihoods with one or no scores. """ # Generate an estimate using two data points data1 = _generateSampleData(mean=42.0, variance=1e-10) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data1[0:2]) ) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) # Check that the estimated mean is that value dParams = estimatorParams["distribution"] self.assertWithinEpsilon(dParams["mean"], data1[0][2]) # Can't generate an estimate using no data points data1 = numpy.zeros(0) with self.assertRaises(ValueError): an.estimateAnomalyLikelihoods(data1) # Can't update with no scores with self.assertRaises(ValueError): an.updateAnomalyLikelihoods(data1, estimatorParams)
def testUpdateAnomalyLikelihoods(self): """ A slight more complex test. This calls estimateAnomalyLikelihoods to estimate the distribution on fake data, followed by several calls to updateAnomalyLikelihoods. """ # ------------------------------------------ # Step 1. Generate an initial estimate using fake distribution of anomaly # scores. data1 = _generateSampleData(mean=0.2)[0:1000] _, _, estimatorParams = an.estimateAnomalyLikelihoods(data1, averagingWindow=5) # ------------------------------------------ # Step 2. Generate some new data with a higher average anomaly # score. Using the estimator from step 1, to compute likelihoods. Now we # should see a lot more anomalies. data2 = _generateSampleData(mean=0.6)[0:300] likelihoods2, avgRecordList2, estimatorParams2 = an.updateAnomalyLikelihoods(data2, estimatorParams) self.assertEqual(len(likelihoods2), len(data2)) self.assertEqual(len(avgRecordList2), len(data2)) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) # The new running total should be different self.assertNotEqual(estimatorParams2["movingAverage"]["total"], estimatorParams["movingAverage"]["total"]) # We should have many more samples where likelihood is < 0.01, but not all self.assertGreaterEqual(numpy.sum(likelihoods2 < 0.01), 25) self.assertLessEqual(numpy.sum(likelihoods2 < 0.01), 250) # ------------------------------------------ # Step 3. Generate some new data with the expected average anomaly score. We # should see fewer anomalies than in Step 2. data3 = _generateSampleData(mean=0.2)[0:1000] likelihoods3, avgRecordList3, estimatorParams3 = an.updateAnomalyLikelihoods(data3, estimatorParams2) self.assertEqual(len(likelihoods3), len(data3)) self.assertEqual(len(avgRecordList3), len(data3)) self.assertTrue(an.isValidEstimatorParams(estimatorParams3)) # The new running total should be different self.assertNotEqual(estimatorParams3["movingAverage"]["total"], estimatorParams["movingAverage"]["total"]) self.assertNotEqual(estimatorParams3["movingAverage"]["total"], estimatorParams2["movingAverage"]["total"]) # We should have a small number samples where likelihood is < 0.02, but at # least one self.assertGreaterEqual(numpy.sum(likelihoods3 < 0.01), 1) self.assertLessEqual(numpy.sum(likelihoods3 < 0.01), 100) # ------------------------------------------ # Step 4. Validate that sending data incrementally is the same as sending # in one batch allData = data1 allData.extend(data2) allData.extend(data3) # Compute moving average of all the data and check it's the same _, historicalValuesAll, totalAll = an._anomalyScoreMovingAverage(allData, windowSize=5) self.assertEqual(sum(historicalValuesAll), sum(estimatorParams3["movingAverage"]["historicalValues"])) self.assertEqual(totalAll, estimatorParams3["movingAverage"]["total"])
def testCaseUnusuallyHighSpikeFrequency(self): """ Test B: one anomaly spike every 20 records. Then we suddenly get a bunch in a row. The likelihood of those spikes should be low. """ data = self._addSampleData(spikePeriod=20, numSamples=1019) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # If we continue to see the same distribution, we should get reasonable # likelihoods data = self._addSampleData(numSamples=119, spikePeriod=20) likelihoods1, _, estimatorParams1 = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The minimum likelihood should be reasonably high self.assertTrue((likelihoods1.min() > 0.1 )) data = self._addSampleData(numSamples=20, spikePeriod=2) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams1) ) # The likelihood once you get past the initial averaging should be very low. self.assertTrue((likelihoods2[5:].sum() / 15.0) < 0.001)
def testCaseUnusuallyHighSpikeFrequency(self): """ Test B: one anomaly spike every 20 records. Then we suddenly get a bunch in a row. The likelihood of those spikes should be low. """ data = self._addSampleData(spikePeriod=20, numSamples=1019) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # If we continue to see the same distribution, we should get reasonable # likelihoods data = self._addSampleData(numSamples=119, spikePeriod=20) likelihoods1, _, estimatorParams1 = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The minimum likelihood should be reasonably high self.assertTrue((likelihoods1.min() > 0.1 )) data = self._addSampleData(numSamples=20, spikePeriod=2) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams1) ) # The likelihood once you get past the initial averaging should be very low. self.assertTrue((likelihoods2[5:].sum() / 15.0) < 0.001)
def testFlatAnomalyScores(self): """ This calls estimateAnomalyLikelihoods with flat distributions and ensures things don't crash. """ # Generate an estimate using fake distribution of anomaly scores. data1 = _generateSampleData(mean=42.0, variance=1e-10) likelihoods, avgRecordList, estimatorParams = ( an.estimateAnomalyLikelihoods(data1[0:1000]) ) self.assertEqual(len(likelihoods), 1000) self.assertEqual(len(avgRecordList), 1000) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) ## Check that the estimated mean is correct dParams = estimatorParams["distribution"] self.assertWithinEpsilon(dParams["mean"], data1[0][2]) # If you deviate from the mean, you should get probability 0 # Test this by sending in just slightly different values. data2 = _generateSampleData(mean=42.5, variance=1e-10) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data2[0:10], estimatorParams) ) # The likelihoods should go to zero very quickly self.assertLessEqual(likelihoods2.sum(), 0.01) # Test edge case where anomaly scores are very close to 0 # In this case we don't let likelihood to get too low. An average # anomaly score of 0.1 should be essentially zero, but an average # of 0.04 should be higher data3 = _generateSampleData(mean=0.01, variance=1e-6) _, _, estimatorParams3 = ( an.estimateAnomalyLikelihoods(data3[0:1000]) ) data4 = _generateSampleData(mean=0.1, variance=1e-6) likelihoods4, _, estimatorParams4 = ( an.updateAnomalyLikelihoods(data4[0:20], estimatorParams3) ) # Average of 0.1 should go to zero self.assertLessEqual(likelihoods4[10:].mean(), 0.002) data5 = _generateSampleData(mean=0.05, variance=1e-6) likelihoods5, _, _ = ( an.updateAnomalyLikelihoods(data5[0:20], estimatorParams4) ) # The likelihoods should be low but not near zero self.assertLessEqual(likelihoods5[10:].mean(), 0.28) self.assertGreater(likelihoods5[10:].mean(), 0.015)
def testFlatAnomalyScores(self): """ This calls estimateAnomalyLikelihoods with flat distributions and ensures things don't crash. """ # Generate an estimate using fake distribution of anomaly scores. data1 = _generateSampleData(mean=42.0, variance=1e-10) likelihoods, avgRecordList, estimatorParams = ( an.estimateAnomalyLikelihoods(data1[0:1000]) ) self.assertEqual(len(likelihoods), 1000) self.assertEqual(len(avgRecordList), 1000) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) ## Check that the estimated mean is correct dParams = estimatorParams["distribution"] self.assertWithinEpsilon(dParams["mean"], data1[0][2]) # If you deviate from the mean, you should get probability 0 # Test this by sending in just slightly different values. data2 = _generateSampleData(mean=42.5, variance=1e-10) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data2[0:10], estimatorParams) ) # The likelihoods should go to zero very quickly self.assertLessEqual(likelihoods2.sum(), 0.01) # Test edge case where anomaly scores are very close to 0 # In this case we don't let likelihood to get too low. An average # anomaly score of 0.1 should be essentially zero, but an average # of 0.04 should be higher data3 = _generateSampleData(mean=0.01, variance=1e-6) _, _, estimatorParams3 = ( an.estimateAnomalyLikelihoods(data3[0:1000]) ) data4 = _generateSampleData(mean=0.1, variance=1e-6) likelihoods4, _, estimatorParams4 = ( an.updateAnomalyLikelihoods(data4[0:20], estimatorParams3) ) # Average of 0.1 should go to zero self.assertLessEqual(likelihoods4[10:].mean(), 0.002) data5 = _generateSampleData(mean=0.05, variance=1e-6) likelihoods5, _, _ = ( an.updateAnomalyLikelihoods(data5[0:20], estimatorParams4) ) # The likelihoods should be low but not near zero self.assertLessEqual(likelihoods5[10:].mean(), 0.28) self.assertGreater(likelihoods5[10:].mean(), 0.015)
def testCaseContinuousBunchesOfSpikes(self): """ Test D: bunches of anomalies every 20 records that continue. This should not be anomalous. """ # Generate initial data data = [] for _ in range(30): data = self._addSampleData(data, spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=3, numSamples=10) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # Now feed in the same distribution data = self._addSampleData(spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=3, numSamples=10) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The likelihood should be reasonable high everywhere self.assertTrue(likelihoods2.min() > 0.01)
def testCaseIncreasedAnomalyScore(self): """ Test F: small anomaly score every 20 records, but then a large one when you would expect a small one. This should be anomalous. """ # Generate initial data data = [] data = self._addSampleData(data, spikePeriod=20, spikeValue=0.4, numSamples=1000) _, _, estimatorParams = (an.estimateAnomalyLikelihoods(data)) # Now feed in a more frequent distribution data = self._addSampleData(spikePeriod=20, spikeValue=1.0, numSamples=100) likelihoods2, _, _ = (an.updateAnomalyLikelihoods( data, estimatorParams)) # We should detect highly unusual behavior self.assertTrue(likelihoods2.min() < 0.0003) # We should detect it pretty often self.assertTrue((likelihoods2 < 0.0003).sum() > 40)
def likelihood(self, value, anomalyScore, dttm): """ Given the current metric value, plus the current anomaly score, output the anomalyLikelihood for this record. """ dataPoint = (dttm, value, anomalyScore) # We ignore the first probationaryPeriod data points if len(self._historicalScores) < self._probationaryPeriod: likelihood = 0.5 else: # On a rolling basis we re-estimate the distribution every 100 iterations if self._distribution is None or (self._iteration % 100 == 0): _, _, self._distribution = ( anomaly_likelihood.estimateAnomalyLikelihoods( self._historicalScores, skipRecords = self._numentaLearningPeriod) ) likelihoods, _, self._distribution = ( anomaly_likelihood.updateAnomalyLikelihoods([dataPoint], self._distribution) ) likelihood = 1.0 - likelihoods[0] # Before we exit update historical scores and iteration self._historicalScores.append(dataPoint) self._iteration += 1 return likelihood
def likelihood(self, value, anomalyScore, dttm): """ Given the current metric value, plus the current anomaly score, output the anomalyLikelihood for this record. """ dataPoint = (dttm, value, anomalyScore) # We ignore the first probationaryPeriod data points if len(self._historicalScores) < self._probationaryPeriod: likelihood = 0.5 else: # On a rolling basis we re-estimate the distribution every 100 iterations if self._distribution is None or (self._iteration % 100 == 0): _, _, self._distribution = ( anomaly_likelihood.estimateAnomalyLikelihoods( self._historicalScores, skipRecords = self._numentaLearningPeriod) ) likelihoods, _, self._distribution = ( anomaly_likelihood.updateAnomalyLikelihoods([dataPoint], self._distribution) ) likelihood = 1.0 - likelihoods[0] # Before we exit update historical scores and iteration self._historicalScores.append(dataPoint) self._iteration += 1 return likelihood
def testCaseIncreasedAnomalyScore(self): """ Test F: small anomaly score every 20 records, but then a large one when you would expect a small one. This should be anomalous. """ # Generate initial data data = [] data = self._addSampleData(data, spikePeriod=20, spikeValue=0.4, numSamples=1000) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data) ) # Now feed in a more frequent distribution data = self._addSampleData(spikePeriod=20, spikeValue=1.0, numSamples=100) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # We should detect highly unusual behavior self.assertTrue(likelihoods2.min() < 0.0003) # We should detect it pretty often self.assertTrue((likelihoods2 < 0.0003).sum() > 40)
def testCaseIncreasedSpikeFrequency(self): """ Test E: bunches of anomalies every 20 records that become even more frequent. This should be anomalous. """ # Generate initial data data = [] for _ in range(30): data = self._addSampleData(data, spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=3, numSamples=10) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # Now feed in a more frequent distribution data = self._addSampleData(spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=1, numSamples=10) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The likelihood should become anomalous but only near the end self.assertTrue(likelihoods2[0:30].min() > 0.01) self.assertTrue(likelihoods2[-5:].min() < 0.002)
def testCaseContinuousBunchesOfSpikes(self): """ Test D: bunches of anomalies every 20 records that continue. This should not be anomalous. """ # Generate initial data data = [] for _ in range(30): data = self._addSampleData(data, spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=3, numSamples=10) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # Now feed in the same distribution data = self._addSampleData(spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=3, numSamples=10) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The likelihood should be reasonable high everywhere self.assertTrue(likelihoods2.min() > 0.01)
def testCaseIncreasedSpikeFrequency(self): """ Test E: bunches of anomalies every 20 records that become even more frequent. This should be anomalous. """ # Generate initial data data = [] for _ in range(30): data = self._addSampleData(data, spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=3, numSamples=10) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # Now feed in a more frequent distribution data = self._addSampleData(spikePeriod=0, numSamples=30) data = self._addSampleData(data, spikePeriod=1, numSamples=10) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The likelihood should become anomalous but only near the end self.assertTrue(likelihoods2[0:30].min() > 0.01) self.assertTrue(likelihoods2[-5:].min() < 0.002)
def testBadParams(self): """ Calls updateAnomalyLikelihoods with bad params. """ # Generate an estimate using one data point data1 = _generateSampleData(mean=42.0, variance=1e-10) _, _, estimatorParams = (an.estimateAnomalyLikelihoods(data1[0:1])) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) # Can't pass in a bad params structure with self.assertRaises(ValueError): an.updateAnomalyLikelihoods(data1, {"haha": "heehee"}) # Can't pass in something not a dict with self.assertRaises(ValueError): an.updateAnomalyLikelihoods(data1, 42.0)
def testBadParams(self): """ Calls updateAnomalyLikelihoods with bad params. """ # Generate an estimate using one data point data1 = _generateSampleData(mean=42.0, variance=1e-10) _, _, estimatorParams = an.estimateAnomalyLikelihoods(data1[0:1]) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) # Can't pass in a bad params structure with self.assertRaises(ValueError): an.updateAnomalyLikelihoods(data1, {"haha": "heehee"}) # Can't pass in something not a dict with self.assertRaises(ValueError): an.updateAnomalyLikelihoods(data1, 42.0)
def testCaseSingleSpike(self): """ No anomalies, and then you see a single spike. The likelihood of that spike should be 0 """ data = self._addSampleData(spikePeriod=0, numSamples=1000) _, _, estimatorParams = (an.estimateAnomalyLikelihoods(data[0:1000])) data = self._addSampleData(numSamples=1, spikePeriod=1) likelihoods1, _, _ = (an.updateAnomalyLikelihoods( data, estimatorParams)) self.assertWithinEpsilon(likelihoods1[0], 0.0)
def testCaseSingleSpike(self): """ No anomalies, and then you see a single spike. The likelihood of that spike should be 0 """ data = self._addSampleData(spikePeriod=0, numSamples=1000) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) data = self._addSampleData(numSamples=1, spikePeriod=1) likelihoods1, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) self.assertWithinEpsilon(likelihoods1[0], 0.0)
def testCaseMissingSpike(self): """ Test C: one anomaly every 20 records, but then see none. The likelihood at the end should be very low. """ # Initial data data = self._addSampleData(spikePeriod=20, numSamples=1019) _, _, estimatorParams = (an.estimateAnomalyLikelihoods(data[0:1000])) # Now feed in none data = self._addSampleData(numSamples=100, spikePeriod=0) likelihoods2, _, _ = (an.updateAnomalyLikelihoods( data, estimatorParams)) # The likelihood once you get past the initial averaging should be very low. self.assertTrue((likelihoods2[5:].sum() / 15.0) < 0.0001)
def testCaseMissingSpike(self): """ Test C: one anomaly every 20 records, but then see none. The likelihood at the end should be very low. """ # Initial data data = self._addSampleData(spikePeriod=20, numSamples=1019) _, _, estimatorParams = ( an.estimateAnomalyLikelihoods(data[0:1000]) ) # Now feed in none data = self._addSampleData(numSamples=100, spikePeriod=0) likelihoods2, _, _ = ( an.updateAnomalyLikelihoods(data, estimatorParams) ) # The likelihood once you get past the initial averaging should be very low. self.assertTrue((likelihoods2[5:].sum() / 15.0) < 0.0001)
def testNABAnomalyLikelihood(self): """ Tests the specific calls to nupic/algorithms/anomaly_likelihood as they"re made in "NAB/detectors/numenta/numenta_detector.py". Note "NAB/.../numenta_detector.py" has its own class AnomalyLikelihood, different from nupic/algorithms/anomaly_likelihood.AnomalyLikelihood, but which calls the functions estimateAnomalyLikelihoods() and updateAnomalyLikelihoods() from "nupic/algorithms/anomaly_likelihood.py". """ # AnomalyLikelihood object initial values iteration = 0 probationaryPeriod = 4 historicalScores = [] likelihoodList = [] for dataPoint in self.data: # Ignore the first probationaryPeriod data points if len(historicalScores) < probationaryPeriod: likelihood = 0.5 else: if iteration % 4 == 0: _, _, distribution = an.estimateAnomalyLikelihoods( historicalScores, skipRecords = probationaryPeriod) likelihoods, _, distribution = an.updateAnomalyLikelihoods( [dataPoint], distribution) likelihood = 1.0 - likelihoods[0] historicalScores.append(dataPoint) iteration += 1 likelihoodList.append(likelihood) truthLikelihoodList = [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.044565462999999972, 0.044565462999999972, 0.044565462999999972, 0.044565462999999972, 0.90319951499999995, 0.90319951499999995, 0.90319951499999995, 0.90319951499999995, 0.78814460099999994, 0.78814460099999994, 0.78814460099999994, 0.78814460099999994] for i in xrange(len(likelihoodList)): self.assertAlmostEqual(likelihoodList[i], truthLikelihoodList[i], msg="unequal values are at index %i" % i)
def testNABAnomalyLikelihood(self): """ Tests the specific calls to nupic/algorithms/anomaly_likelihood as they"re made in "NAB/detectors/numenta/numenta_detector.py". Note "NAB/.../numenta_detector.py" has its own class AnomalyLikelihood, different from nupic/algorithms/anomaly_likelihood.AnomalyLikelihood, but which calls the functions estimateAnomalyLikelihoods() and updateAnomalyLikelihoods() from "nupic/algorithms/anomaly_likelihood.py". """ # AnomalyLikelihood object initial values iteration = 0 probationaryPeriod = 4 historicalScores = [] likelihoodList = [] for dataPoint in self.data: # Ignore the first probationaryPeriod data points if len(historicalScores) < probationaryPeriod: likelihood = 0.5 else: if iteration % 4 == 0: _, _, distribution = an.estimateAnomalyLikelihoods( historicalScores, skipRecords=probationaryPeriod) likelihoods, _, distribution = an.updateAnomalyLikelihoods( [dataPoint], distribution) likelihood = 1.0 - likelihoods[0] historicalScores.append(dataPoint) iteration += 1 likelihoodList.append(likelihood) truthLikelihoodList = [ 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.044565462999999972, 0.044565462999999972, 0.044565462999999972, 0.044565462999999972, 0.90319951499999995, 0.90319951499999995, 0.90319951499999995, 0.90319951499999995, 0.78814460099999994, 0.78814460099999994, 0.78814460099999994, 0.78814460099999994 ] for i in xrange(len(likelihoodList)): self.assertAlmostEqual(likelihoodList[i], truthLikelihoodList[i], msg="unequal values are at index %i" % i)
def testUpdateAnomalyLikelihoods(self): """ A slight more complex test. This calls estimateAnomalyLikelihoods to estimate the distribution on fake data, followed by several calls to updateAnomalyLikelihoods. """ #------------------------------------------ # Step 1. Generate an initial estimate using fake distribution of anomaly # scores. data1 = _generateSampleData(mean=0.2)[0:1000] _, _, estimatorParams = (an.estimateAnomalyLikelihoods( data1, averagingWindow=5)) #------------------------------------------ # Step 2. Generate some new data with a higher average anomaly # score. Using the estimator from step 1, to compute likelihoods. Now we # should see a lot more anomalies. data2 = _generateSampleData(mean=0.6)[0:300] likelihoods2, avgRecordList2, estimatorParams2 = ( an.updateAnomalyLikelihoods(data2, estimatorParams)) self.assertEqual(len(likelihoods2), len(data2)) self.assertEqual(len(avgRecordList2), len(data2)) self.assertTrue(an.isValidEstimatorParams(estimatorParams)) # The new running total should be different self.assertNotEqual(estimatorParams2["movingAverage"]["total"], estimatorParams["movingAverage"]["total"]) # We should have many more samples where likelihood is < 0.01, but not all self.assertGreaterEqual(numpy.sum(likelihoods2 < 0.01), 25) self.assertLessEqual(numpy.sum(likelihoods2 < 0.01), 250) #------------------------------------------ # Step 3. Generate some new data with the expected average anomaly score. We # should see fewer anomalies than in Step 2. data3 = _generateSampleData(mean=0.2)[0:1000] likelihoods3, avgRecordList3, estimatorParams3 = ( an.updateAnomalyLikelihoods(data3, estimatorParams2)) self.assertEqual(len(likelihoods3), len(data3)) self.assertEqual(len(avgRecordList3), len(data3)) self.assertTrue(an.isValidEstimatorParams(estimatorParams3)) # The new running total should be different self.assertNotEqual(estimatorParams3["movingAverage"]["total"], estimatorParams["movingAverage"]["total"]) self.assertNotEqual(estimatorParams3["movingAverage"]["total"], estimatorParams2["movingAverage"]["total"]) # We should have a small number samples where likelihood is < 0.02, but at # least one self.assertGreaterEqual(numpy.sum(likelihoods3 < 0.01), 1) self.assertLessEqual(numpy.sum(likelihoods3 < 0.01), 100) #------------------------------------------ # Step 4. Validate that sending data incrementally is the same as sending # in one batch allData = data1 allData.extend(data2) allData.extend(data3) # Compute moving average of all the data and check it's the same _, historicalValuesAll, totalAll = (an._anomalyScoreMovingAverage( allData, windowSize=5)) self.assertEqual( sum(historicalValuesAll), sum(estimatorParams3["movingAverage"]["historicalValues"])) self.assertEqual(totalAll, estimatorParams3["movingAverage"]["total"])
def updateModelAnomalyScores(self, engine, metricObj, metricDataRows): """ Calculate the anomaly scores based on the anomaly likelihoods. Update anomaly scores in the given metricDataRows MetricData instances, and calculate new anomaly likelihood params for the model. :param engine: SQLAlchemy engine object :type engine: sqlalchemy.engine.Engine :param metricObj: the model's Metric instance :param metricDataRows: a sequence of MetricData instances in the processed order (ascending by timestamp) with updated raw_anomaly_score and zeroed out anomaly_score corresponding to the new model inference results, but not yet updated in the database. Will update their anomaly_score properties, as needed. :returns: new anomaly likelihood params for the model *NOTE:* the processing must be idempotent due to the "at least once" delivery semantics of the message bus *NOTE:* the performance goal is to minimize costly database access and avoid falling behind while processing model results, especially during the model's initial "catch-up" phase when large inference result batches are prevalent. """ # When populated, a cached list of MetricData instances for updating # anomaly likelyhood params statsSampleCache = None # Index into metricDataRows where processing is to resume startRowIndex = 0 statisticsRefreshInterval = self._getStatisticsRefreshInterval( batchSize=len(metricDataRows)) if metricObj.status != MetricStatus.ACTIVE: raise MetricNotActiveError( "getAnomalyLikelihoodParams failed because metric=%s is not ACTIVE; " "status=%s; resource=%s" % (metricObj.uid, metricObj.status, metricObj.server,)) modelParams = jsonDecode(metricObj.model_params) anomalyParams = modelParams.get("anomalyLikelihoodParams", None) if not anomalyParams: # We don't have a likelihood model yet. Create one if we have sufficient # records with raw anomaly scores (anomalyParams, statsSampleCache, startRowIndex) = ( self._initAnomalyLikelihoodModel(engine=engine, metricObj=metricObj, metricDataRows=metricDataRows)) # Do anomaly likelihood processing on the rest of the new samples # NOTE: this loop will be skipped if there are still not enough samples for # creating the anomaly likelihood params while startRowIndex < len(metricDataRows): # Determine where to stop processing rows prior to next statistics refresh if (statsSampleCache is None or len(statsSampleCache) >= self._statisticsMinSampleSize): # We're here if: # a. We haven't tried updating anomaly likelihood stats yet # OR # b. We already updated anomaly likelihood stats (we had sufficient # samples for it) # TODO: unit-test endRowID = (anomalyParams["last_rowid_for_stats"] + statisticsRefreshInterval) if endRowID < metricDataRows[startRowIndex].rowid: # We're here if: # a. Statistics refresh interval is smaller than during last stats # update; this is the typical/normal case when backlog catch-up # is tapering off, and refresh interval is reduced for smaller # batches. OR # b. There is a gap of anomaly scores preceding the start of the # current chunk. OR # c. Statistics config changed. # TODO: unit-test self._log.warning( "Anomaly run cutoff precedes samples (smaller stats " "refreshInterval or gap in anomaly scores or statistics config " "changed) : model=%s; rows=[%s..%s]", metricObj.uid, metricDataRows[startRowIndex].rowid, endRowID) if statsSampleCache is not None: # We already attempted to update anomaly likelihood params, so fix # up endRowID to make sure we make progress and don't get stuck in # an infinite loop endRowID = metricDataRows[startRowIndex].rowid self._log.warning( "Advanced anomaly run cutoff to make progress: " "model=%s; rows=[%s..%s]", metricObj.uid, metricDataRows[startRowIndex].rowid, endRowID) else: # During prior iteration, there were not enough samples in cache for # updating anomaly params # We extend the end row so that there will be enough samples # to avoid getting stuck in this rut in the current and following # iterations # TODO: unit-test this endRowID = metricDataRows[startRowIndex].rowid + ( self._statisticsMinSampleSize - len(statsSampleCache) - 1) # Translate endRowID into metricDataRows limitIndex for current run if endRowID < metricDataRows[startRowIndex].rowid: # Cut-off precedes the remaining samples # Normally shouldn't be here (unless statistics config changed or there # is a gap in anomaly scores in metric_data table) # TODO: unit-test this # Set limit to bypass processing of samples for immediate refresh of # anomaly likelihood params limitIndex = startRowIndex self._log.warning( "Anomaly run cutoff precedes samples, so forcing refresh of anomaly " "likelihood params: modelInfo=<%s>; rows=[%s..%s]", getMetricLogPrefix(metricObj), metricDataRows[startRowIndex].rowid, endRowID) else: # Cutoff is either inside or after the remaining samples # TODO: unit-test this limitIndex = startRowIndex + min( len(metricDataRows) - startRowIndex, endRowID + 1 - metricDataRows[startRowIndex].rowid) # Process the next new sample run self._log.debug( "Starting anomaly run: model=%s; " "startRowIndex=%s; limitIndex=%s; rows=[%s..%s]; " "last_rowid_for_stats=%s; refreshInterval=%s; batchSize=%s", metricObj.uid, startRowIndex, limitIndex, metricDataRows[startRowIndex].rowid, endRowID, anomalyParams["last_rowid_for_stats"], statisticsRefreshInterval, len(metricDataRows)) consumedSamples = [] for md in itertools.islice(metricDataRows, startRowIndex, limitIndex): consumedSamples.append(md) (likelihood,), _, anomalyParams["params"] = ( algorithms.updateAnomalyLikelihoods( ((md.timestamp, md.metric_value, md.raw_anomaly_score),), anomalyParams["params"])) # TODO: the float "cast" here seems redundant md.anomaly_score = float(1.0 - likelihood) # If anomaly score > 0.99 then we greedily update the statistics. 0.99 # should not repeat too often, but to be safe we wait a few more # records before updating again, in order to avoid overloading the DB. # # TODO: the magic 0.99 and the magic 3 value below should either # be constants or config settings. Where should they be defined? if (md.anomaly_score > 0.99 and (anomalyParams["last_rowid_for_stats"] + 3) < md.rowid): if statsSampleCache is None or ( len(statsSampleCache) + len(consumedSamples) >= self._statisticsMinSampleSize): # TODO: unit-test this self._log.info("Forcing refresh of anomaly params for model=%s due " "to exceeded anomaly_score threshold in sample=%r", metricObj.uid, md) break if startRowIndex + len(consumedSamples) < len(metricDataRows) or ( consumedSamples[-1].rowid >= endRowID): # We stopped before the end of new samples, including a bypass-run, # or stopped after processing the last item and need one final refresh # of anomaly params anomalyParams, statsSampleCache = self._refreshAnomalyParams( engine=engine, metricID=metricObj.uid, statsSampleCache=statsSampleCache, consumedSamples=consumedSamples, defaultAnomalyParams=anomalyParams) startRowIndex += len(consumedSamples) # <--- while return anomalyParams
def updateModelAnomalyScores(self, engine, metricObj, metricDataRows): """ Calculate the anomaly scores based on the anomaly likelihoods. Update anomaly scores in the given metricDataRows MetricData instances, and calculate new anomaly likelihood params for the model. :param engine: SQLAlchemy engine object :type engine: sqlalchemy.engine.Engine :param metricObj: the model's Metric instance :param metricDataRows: a sequence of MetricData instances in the processed order (ascending by timestamp) with updated raw_anomaly_score and zeroed out anomaly_score corresponding to the new model inference results, but not yet updated in the database. Will update their anomaly_score properties, as needed. :returns: new anomaly likelihood params for the model *NOTE:* the processing must be idempotent due to the "at least once" delivery semantics of the message bus *NOTE:* the performance goal is to minimize costly database access and avoid falling behind while processing model results, especially during the model's initial "catch-up" phase when large inference result batches are prevalent. """ # When populated, a cached list of MetricData instances for updating # anomaly likelyhood params statsSampleCache = None # Index into metricDataRows where processing is to resume startRowIndex = 0 statisticsRefreshInterval = self._getStatisticsRefreshInterval( batchSize=len(metricDataRows)) if metricObj.status != MetricStatus.ACTIVE: raise MetricNotActiveError( "getAnomalyLikelihoodParams failed because metric=%s is not ACTIVE; " "status=%s; resource=%s" % ( metricObj.uid, metricObj.status, metricObj.server, )) modelParams = jsonDecode(metricObj.model_params) anomalyParams = modelParams.get("anomalyLikelihoodParams", None) if not anomalyParams: # We don't have a likelihood model yet. Create one if we have sufficient # records with raw anomaly scores (anomalyParams, statsSampleCache, startRowIndex) = (self._initAnomalyLikelihoodModel( engine=engine, metricObj=metricObj, metricDataRows=metricDataRows)) # Do anomaly likelihood processing on the rest of the new samples # NOTE: this loop will be skipped if there are still not enough samples for # creating the anomaly likelihood params while startRowIndex < len(metricDataRows): # Determine where to stop processing rows prior to next statistics refresh if (statsSampleCache is None or len(statsSampleCache) >= self._statisticsMinSampleSize): # We're here if: # a. We haven't tried updating anomaly likelihood stats yet # OR # b. We already updated anomaly likelyhood stats (we had sufficient # samples for it) # TODO: unit-test endRowID = (anomalyParams["last_rowid_for_stats"] + statisticsRefreshInterval) if endRowID < metricDataRows[startRowIndex].rowid: # We're here if: # a. Statistics refresh interval is smaller than during last stats # update; this is the typical/normal case when backlog catch-up # is tapering off, and refresh interval is reduced for smaller # batches. OR # b. There is a gap of anomaly scores preceeding the start of the # current chunk. OR # c. Statistics config changed. # TODO: unit-test self._log.warning( "Anomaly run cutoff precedes samples (smaller stats " "refreshInterval or gap in anomaly scores or statistics config " "changed) : model=%s; rows=[%s..%s]", metricObj.uid, metricDataRows[startRowIndex].rowid, endRowID) if statsSampleCache is not None: # We already attempted to update anomaly likelihood params, so fix # up endRowID to make sure we make progress and don't get stuck in # an infinite loop endRowID = metricDataRows[startRowIndex].rowid self._log.warning( "Advanced anomaly run cutoff to make progress: " "model=%s; rows=[%s..%s]", metricObj.uid, metricDataRows[startRowIndex].rowid, endRowID) else: # During prior iteration, there were not enough samples in cache for # updating anomaly params # We extend the end row so that there will be enough samples # to avoid getting stuck in this rut in the current and following # iterations # TODO: unit-test this endRowID = metricDataRows[startRowIndex].rowid + ( self._statisticsMinSampleSize - len(statsSampleCache) - 1) # Translate endRowID into metricDataRows limitIndex for current run if endRowID < metricDataRows[startRowIndex].rowid: # Cut-off precedes the remaining samples # Normally shouldn't be here (unless statistics config changed or there # is a gap in anomaly scores in metric_data table) # TODO: unit-test this # Set limit to bypass processing of samples for immediate refresh of # anomaly likelihood params limitIndex = startRowIndex self._log.warning( "Anomaly run cutoff precedes samples, so forcing refresh of anomaly " "likelihood params: modelInfo=<%s>; rows=[%s..%s]", getMetricLogPrefix(metricObj), metricDataRows[startRowIndex].rowid, endRowID) else: # Cutoff is either inside or after the remaining samples # TODO: unit-test this limitIndex = startRowIndex + min( len(metricDataRows) - startRowIndex, endRowID + 1 - metricDataRows[startRowIndex].rowid) # Process the next new sample run self._log.debug( "Starting anomaly run: model=%s; " "startRowIndex=%s; limitIndex=%s; rows=[%s..%s]; " "last_rowid_for_stats=%s; refreshInterval=%s; batchSize=%s", metricObj.uid, startRowIndex, limitIndex, metricDataRows[startRowIndex].rowid, endRowID, anomalyParams["last_rowid_for_stats"], statisticsRefreshInterval, len(metricDataRows)) consumedSamples = [] for md in itertools.islice(metricDataRows, startRowIndex, limitIndex): consumedSamples.append(md) (likelihood, ), _, anomalyParams["params"] = ( algorithms.updateAnomalyLikelihoods( ((md.timestamp, md.metric_value, md.raw_anomaly_score), ), anomalyParams["params"])) # TODO: the float "cast" here seems redundant md.anomaly_score = float(1.0 - likelihood) # If anomaly score > 0.99 then we greedily update the statistics. 0.99 # should not repeat too often, but to be safe we wait a few more # records before updating again, in order to avoid overloading the DB. # # TODO: the magic 0.99 and the magic 3 value below should either # be constants or config settings. Where should they be defined? if (md.anomaly_score > 0.99 and (anomalyParams["last_rowid_for_stats"] + 3) < md.rowid): if statsSampleCache is None or ( len(statsSampleCache) + len(consumedSamples) >= self._statisticsMinSampleSize): # TODO: unit-test this self._log.info( "Forcing refresh of anomaly params for model=%s due " "to exceeded anomaly_score threshold in sample=%r", metricObj.uid, md) break if startRowIndex + len(consumedSamples) < len(metricDataRows) or ( consumedSamples[-1].rowid >= endRowID): # We stopped before the end of new samples, including a bypass-run, # or stopped after processing the last item and need one final refresh # of anomaly params anomalyParams, statsSampleCache = self._refreshAnomalyParams( engine=engine, metricID=metricObj.uid, statsSampleCache=statsSampleCache, consumedSamples=consumedSamples, defaultAnomalyParams=anomalyParams) startRowIndex += len(consumedSamples) # <--- while return anomalyParams