def testMovingAverage(self):
"""
Test that the (internal) moving average maintains the averages correctly,
even for null initial condition and when the number of values goes over
windowSize. Pass in integers and floats.
"""
historicalValues = []
total = 0
windowSize = 3
newAverage, historicalValues, total = (MovingAverage.compute(
historicalValues, total, 3, windowSize))
self.assertEqual(newAverage, 3.0)
self.assertEqual(historicalValues, [3.0])
self.assertEqual(total, 3.0)
newAverage, historicalValues, total = (MovingAverage.compute(
historicalValues, total, 4, windowSize))
self.assertEqual(newAverage, 3.5)
self.assertListEqual(historicalValues, [3.0, 4.0])
self.assertEqual(total, 7.0)
newAverage, historicalValues, total = (MovingAverage.compute(
historicalValues, total, 5.0, windowSize))
self.assertEqual(newAverage, 4.0)
self.assertListEqual(historicalValues, [3.0, 4.0, 5.0])
self.assertEqual(total, 12.0)
# Ensure the first value gets popped
newAverage, historicalValues, total = (MovingAverage.compute(
historicalValues, total, 6.0, windowSize))
self.assertEqual(newAverage, 5.0)
self.assertListEqual(historicalValues, [4.0, 5.0, 6.0])
self.assertEqual(total, 15.0)
def testMovingAverage(self):
"""
Test that the (internal) moving average maintains the averages correctly,
even for null initial condition and when the number of values goes over
windowSize. Pass in integers and floats.
"""
historicalValues = []
total = 0
windowSize = 3
newAverage, historicalValues, total = MovingAverage.compute(historicalValues, total, 3, windowSize)
self.assertEqual(newAverage, 3.0)
self.assertEqual(historicalValues, [3.0])
self.assertEqual(total, 3.0)
newAverage, historicalValues, total = MovingAverage.compute(historicalValues, total, 4, windowSize)
self.assertEqual(newAverage, 3.5)
self.assertListEqual(historicalValues, [3.0, 4.0])
self.assertEqual(total, 7.0)
newAverage, historicalValues, total = MovingAverage.compute(historicalValues, total, 5.0, windowSize)
self.assertEqual(newAverage, 4.0)
self.assertListEqual(historicalValues, [3.0, 4.0, 5.0])
self.assertEqual(total, 12.0)
# Ensure the first value gets popped
newAverage, historicalValues, total = MovingAverage.compute(historicalValues, total, 6.0, windowSize)
self.assertEqual(newAverage, 5.0)
self.assertListEqual(historicalValues, [4.0, 5.0, 6.0])
self.assertEqual(total, 15.0)
def _anomalyScoreMovingAverage(anomalyScores, windowSize=10, verbosity=0):
"""
Given a list of anomaly scores return a list of averaged records.
anomalyScores is assumed to be a list of records of the form:
[datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]
Each record in the returned list list contains:
[datetime, value, averagedScore]
*Note:* we only average the anomaly score.
"""
historicalValues = []
total = 0.0
averagedRecordList = [] # Aggregated records
for record in anomalyScores:
# Skip (but log) records without correct number of entries
if not isinstance(record, (list, tuple)) or len(record) != 3:
if verbosity >= 1:
print("Malformed record:", record)
continue
avg, historicalValues, total = MovingAverage.compute(historicalValues, total, record[2], windowSize)
averagedRecordList.append([record[0], record[1], avg])
if verbosity > 2:
print("Aggregating input record:", record)
print("Result:", [record[0], record[1], avg])
return averagedRecordList, historicalValues, total
def _anomalyScoreMovingAverage(anomalyScores,
windowSize=10,
verbosity=0,
):
"""
Given a list of anomaly scores return a list of averaged records.
anomalyScores is assumed to be a list of records of the form:
[datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]
Each record in the returned list list contains:
[datetime, value, averagedScore]
*Note:* we only average the anomaly score.
"""
historicalValues = []
total = 0.0
averagedRecordList = [] # Aggregated records
for record in anomalyScores:
# Skip (but log) records without correct number of entries
if not isinstance(record, (list, tuple)) or len(record) != 3:
if verbosity >= 1:
print "Malformed record:", record
continue
avg, historicalValues, total = (
MovingAverage.compute(historicalValues, total, record[2], windowSize)
)
averagedRecordList.append( [record[0], record[1], avg] )
if verbosity > 2:
print "Aggregating input record:", record
print "Result:", [record[0], record[1], avg]
return averagedRecordList, historicalValues, total
def updateAnomalyLikelihoods(anomalyScores,
params,
verbosity=0): # pylint: disable=W0613
"""
Compute updated probabilities for anomalyScores using the given params.
:param anomalyScores: a list of records. Each record is a list with the
following three elements: [timestamp, value, score]
Example::
[datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]
:param params: the JSON dict returned by estimateAnomalyLikelihoods
:param verbosity: integer controlling extent of printouts for debugging
:type verbosity: int
:returns: 3-tuple consisting of:
- likelihoods
numpy array of likelihoods, one for each aggregated point
- avgRecordList
list of averaged input records
- params
an updated JSON object containing the state of this metric.
"""
if verbosity > 3:
print "In updateAnomalyLikelihoods."
print "Number of anomaly scores:", len(anomalyScores)
print "First 20:", anomalyScores[0:min(20, len(anomalyScores))]
print "Params:", params
if len(anomalyScores) == 0:
raise ValueError("Must have at least one anomalyScore")
if not isValidEstimatorParams(params):
raise ValueError("'params' is not a valid params structure")
# For backward compatibility.
if not params.has_key("historicalLikelihoods"):
params["historicalLikelihoods"] = [1.0]
# Compute moving averages of these new scores using the previous values
# as well as likelihood for these scores using the old estimator
historicalValues = params["movingAverage"]["historicalValues"]
total = params["movingAverage"]["total"]
windowSize = params["movingAverage"]["windowSize"]
aggRecordList = numpy.zeros(len(anomalyScores), dtype=float)
likelihoods = numpy.zeros(len(anomalyScores), dtype=float)
for i, v in enumerate(anomalyScores):
newAverage, historicalValues, total = (
MovingAverage.compute(historicalValues, total, v[2], windowSize)
)
aggRecordList[i] = newAverage
likelihoods[i] = normalProbability(newAverage, params["distribution"])
# Filter the likelihood values. First we prepend the historical likelihoods
# to the current set. Then we filter the values. We peel off the likelihoods
# to return and the last windowSize values to store for later.
likelihoods2 = params["historicalLikelihoods"] + list(likelihoods)
filteredLikelihoods = _filterLikelihoods(likelihoods2)
likelihoods[:] = filteredLikelihoods[-len(likelihoods):]
historicalLikelihoods = likelihoods2[-min(windowSize, len(likelihoods2)):]
# Update the estimator
newParams = {
"distribution": params["distribution"],
"movingAverage": {
"historicalValues": historicalValues,
"total": total,
"windowSize": windowSize,
},
"historicalLikelihoods": historicalLikelihoods,
}
assert len(newParams["historicalLikelihoods"]) <= windowSize
if verbosity > 3:
print "Number of likelihoods:", len(likelihoods)
print "First 20 likelihoods:", likelihoods[0:min(20, len(likelihoods))]
print "Leaving updateAnomalyLikelihoods."
return (likelihoods, aggRecordList, newParams)
def updateAnomalyLikelihoods(anomalyScores,
params,
verbosity=0):
"""
Compute updated probabilities for anomalyScores using the given params.
:param anomalyScores: a list of records. Each record is a list with the
following three elements: [timestamp, value, score]
Example::
[datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]
:param params: the JSON dict returned by estimateAnomalyLikelihoods
:param verbosity: integer controlling extent of printouts for debugging
:type verbosity: int
:returns: 3-tuple consisting of:
- likelihoods
numpy array of likelihoods, one for each aggregated point
- avgRecordList
list of averaged input records
- params
an updated JSON object containing the state of this metric.
"""
if verbosity > 3:
print "In updateAnomalyLikelihoods."
print "Number of anomaly scores:", len(anomalyScores)
print "First 20:", anomalyScores[0:min(20, len(anomalyScores))]
print "Params:", params
if len(anomalyScores) == 0:
raise ValueError("Must have at least one anomalyScore")
if not isValidEstimatorParams(params):
raise ValueError("'params' is not a valid params structure")
# For backward compatibility.
if not params.has_key("historicalLikelihoods"):
params["historicalLikelihoods"] = [1.0]
# Compute moving averages of these new scores using the previous values
# as well as likelihood for these scores using the old estimator
historicalValues = params["movingAverage"]["historicalValues"]
total = params["movingAverage"]["total"]
windowSize = params["movingAverage"]["windowSize"]
aggRecordList = numpy.zeros(len(anomalyScores), dtype=float)
likelihoods = numpy.zeros(len(anomalyScores), dtype=float)
for i, v in enumerate(anomalyScores):
newAverage, historicalValues, total = (
MovingAverage.compute(historicalValues, total, v[2], windowSize)
)
aggRecordList[i] = newAverage
likelihoods[i] = normalProbability(newAverage, params["distribution"])
# Filter the likelihood values. First we prepend the historical likelihoods
# to the current set. Then we filter the values. We peel off the likelihoods
# to return and the last windowSize values to store for later.
likelihoods2 = params["historicalLikelihoods"] + list(likelihoods)
filteredLikelihoods = _filterLikelihoods(likelihoods2)
likelihoods[:] = filteredLikelihoods[-len(likelihoods):]
historicalLikelihoods = likelihoods2[-min(windowSize, len(likelihoods2)):]
# Update the estimator
newParams = {
"distribution": params["distribution"],
"movingAverage": {
"historicalValues": historicalValues,
"total": total,
"windowSize": windowSize,
},
"historicalLikelihoods": historicalLikelihoods,
}
assert len(newParams["historicalLikelihoods"]) <= windowSize
if verbosity > 3:
print "Number of likelihoods:", len(likelihoods)
print "First 20 likelihoods:", likelihoods[0:min(20, len(likelihoods))]
print "Leaving updateAnomalyLikelihoods."
return (likelihoods, aggRecordList, newParams)
