def difference_in_means(
in_stream, out_stream,
long_window_size, short_window_size,
threshold):
map_window(f, in_stream, out_stream,
long_window_size, step_size=1)
def compute(in_streams):
def subtract_mean(window):
return window[-1] - sum(window) / float(len(window))
def magnitude_of_vector(coordinates):
return math.sqrt(sum([v * v for v in coordinates]))
def simple_anomaly(value, threshold):
if value > threshold: return 1.0
else: return 0.0
zero_mean_streams = [
Stream('zero mean e'),
Stream('zero mean n'),
Stream('zero mean z')
]
magnitude_stream = Stream('magnitude')
anomaly_stream = Stream('anomalies')
filenames = ['zero_mean_e.txt', 'zero_mean_n.txt', 'zero_mean_z.txt']
for i in range(3):
map_window(func=subtract_mean,
in_stream=in_streams[i],
out_stream=zero_mean_streams[i],
window_size=8000,
step_size=1)
zip_map(func=magnitude_of_vector,
in_streams=zero_mean_streams,
out_stream=magnitude_stream)
map_element(func=simple_anomaly,
in_stream=magnitude_stream,
out_stream=anomaly_stream,
threshold=0.1)
stream_to_file(in_stream=magnitude_stream, filename='magnitude.txt')
stream_to_file(in_stream=anomaly_stream, filename='anomaly.txt')
def window_dot_product(in_stream,
out_stream,
multiplicand_vector,
step_size=1):
"""
Parameters
----------
in_stream: Stream
input stream of agent
out_stream: Stream
output stream of agent
multiplicand_vector: list or NumPy array
length of multiplicand_vector must be strictly positive
The dot product is applied between each sliding window
and the multiplicand_vector
step_size: int
Must be positive
The amount by which the sliding window moves on each step.
Operation
---------
Creates an agent which carries out the dot product of the
multiplicand_vector and each sliding window.
"""
def f(window, multiplicand_vector):
return np.dot(window, multiplicand_vector)
window_size = len(multiplicand_vector)
map_window(f,
in_stream,
out_stream,
window_size,
multiplicand_vector=multiplicand_vector)
def map_window_examples_simple():
#----------------------------------------------------------------
# Example with no state and no additional parameters
#----------------------------------------------------------------
# Create streams
x = Stream('x')
s = Stream()
# Create agents
map_window(func=sum,
in_stream=x,
out_stream=s,
window_size=3,
step_size=10)
# Explanation of agent
# The function sum() returns the sum of the window.
# out_stream[j] = \
# func(in_stream[j*step_size : j*step_size + window_size])
# for all j.
# So if window_size is 3 and step_size is 10:
# s[j] = x[10*j] + x[10*j+1] + x[10*j+2], for all j
# For window_size of 3 and step_size of 10, the output
# stream will contain:
# [0 + 1 + 2, 10 + 11 + 12, 20 + 21 + 22, 30 + 31 + 32]
# Put data into input streams and run.
DATA = list(range(40))
x.extend(DATA)
run()
# Inspect output
assert recent_values(s) == [3, 33, 63, 93]
def anomaly_stream(in_stream, out_stream, long_term_duration,
short_term_duration, threshold, cutoff):
"""
Parameters
----------
in_stream: Stream
input stream of function
out_stream: Stream
output stream of function
long_term_duration: int
short_term_duration: int
cutoff: int or float
threshold: int or float
Note
----
out_stream is a stream of anomalies determined by the function
detect_anomaly().
An anomaly occurs when average of the clipped window is
significantly lower (determined by threshold) than the average of
the most recent short_term_duration elements in the window.
"""
map_window(func=short_term_long_term_anomaly,
in_stream=in_stream,
out_stream=out_stream,
window_size=long_term_duration,
step_size=1,
short_term_duration=short_term_duration,
threshold=threshold,
cutoff=cutoff)
def kmeans_sliding_windows(in_stream, out_stream, window_size, step_size,
num_clusters):
# The initial state is set to 0.
# (Note that setting state to None implies no state, and so that
# won't work.)
kmeans_object = KMeansForSlidingWindows(num_clusters)
map_window(kmeans_object.func, in_stream, out_stream, window_size,
step_size)
def add(in_stream_array, out_stream_array, window_size, addend):
def f(an_array):
return _multivalue(an_array + addend)
map_window(f,
in_stream_array,
out_stream_array,
window_size=window_size,
step_size=window_size)
def multiply(in_stream_array, out_stream_array, window_size, multiplicand):
def f(an_array):
return _multivalue(an_array * multiplicand)
map_window(f,
in_stream_array,
out_stream_array,
window_size=window_size,
step_size=window_size)
def map_window_example_simple_with_state_and_parameter():
#----------------------------------------------------------------
# Example with state and additional parameters
#----------------------------------------------------------------
# Declare functions
# This function returns an element of the output stream and then
# next state.
# The parameter is multiplicand.
def maxes(in_stream_window, state, multiplicand):
next_output_element = sum(in_stream_window) > multiplicand * state
next_state = max(state, sum(in_stream_window))
return (next_output_element, next_state)
# Create streams
x = Stream('x')
m = Stream('m')
# Create agents
map_window(func=maxes,
in_stream=x,
out_stream=m,
state=0,
window_size=2,
step_size=3,
multiplicand=1.5)
# Explanation of agent
# in_stream_window on step j is:
# x[step_size*j : step_size*j + window_size]
# For j > 0: state[j] = \
# max(state[j-1], sum(x[step_size*j : step_size*j + window_size]))
# m[j] = sum(x[step_size*j : step_size*j + window_size]) > \
# multiplicand * state[j]
# With a window_size of 2 and step_size of 3 and multiplicand of 1.5
# state[j] is max(state[j-1], x[3*j, 3*j+1])
# m[j] is (x[3*j, 3*j+1]) > 1.5*state[j]
# Put data into input streams and run.
DATA = [1, 1, 0, -1, 0, 3, 1, 5, 11, 1]
# With window_size=3, step_size=2, and this DATA the output steps are:
# Initially state = 0, because of the specification in max_window
# m[0] is (DATA[0]+DATA[1]) > 1.5*0, which is (1 + 1) > 0 or True.
# state[1] is max(0, DATA[0]+DATA[1]) which is 2.
# m[1] is (DATA[3]+DATA[4]) > 1.5*2 which is (-1 + 0) > 2 or False.
# state[2] is max(2, DATA[3]+DATA[4]) which is 2.
# m[2] is (DATA[6]+DATA[7]) > 1.5*2 which is (1 + 5) > 2 or True.
# state[3] is max(2, DATA[6]+DATA[7]) which is 6.
# So the output stream is:
# [True, False, True, ....]
x.extend(DATA)
run()
# Inspect output
assert recent_values(m) == [True, False, True]
def compute_func(in_streams, out_streams):
check_list = [1, 5, 9, 13, 17]
t = Stream()
map_window(func=sum,
in_stream=in_streams[0],
out_stream=t,
window_size=2,
step_size=2)
check_correctness_of_output(in_stream=t, check_list=check_list)
stream_to_file(
in_stream=t,
filename='single_process_single_source_map_window_example_1.dat')
def compute_func(in_streams, out_streams):
merged_stream = Stream('merge of two ntp server offsets')
averaged_stream = Stream('sliding window average of offsets')
blend(func=lambda x: x,
in_streams=in_streams,
out_stream=merged_stream)
map_window(func=average_of_list,
in_stream=merged_stream,
out_stream=averaged_stream,
window_size=2,
step_size=1)
stream_to_file(in_stream=averaged_stream, filename='average.dat')
def compute(in_stream):
def subtract_mean(window):
return window[-1] - sum(window) / float(len(window))
zero_mean_stream = Stream('zero mean')
input_stream = Stream('input')
map_window(func=subtract_mean,
in_stream=in_stream,
out_stream=zero_mean_stream,
window_size=50,
step_size=1)
map_window(func=lambda window: window[-1],
in_stream=in_stream,
out_stream=input_stream,
window_size=50,
step_size=1)
stream_to_file(in_stream=zero_mean_stream, filename='zero_mean_z.txt')
def map_window_example_simple_with_parameter():
#----------------------------------------------------------------
# Example with no state and additional parameters
#----------------------------------------------------------------
# The additional parameter is threshold
# Since this agent has no state, this function returns a single
# value which is appended to the output stream.
# Declare functions
def thresh(in_stream_window, threshold):
return sum(in_stream_window) > threshold
# Create streams
x = Stream('x')
t = Stream('t')
# Create agents
map_window(func=thresh,
in_stream=x,
out_stream=t,
window_size=3,
step_size=2,
threshold=0)
# Explanation of agent
# For all j, : t[j] is True if and only if
# sum(x[window_size*j : step_size*j+window_size]) > threshold.
# With window_size of 3 and step_size of 2, and threshold of 5
# the output is
# [sum(x[0:3]) > 0, sum(x[2:5] > 0, sum(x[4:7] > 0, ...]
#
# Note: You can use any names as arguments in the
# definition of func, as in:
# def thresh(v, s): return sum(v) > w
# Put data into input streams and run.
DATA = [1, 1, 0, -1, 0, 3, -1, -20, 11, 1]
# With window_size=3, step_size=2, and this DATA the output is:
# [(1 + 1 + 0 > 0), (0 + (-1) + 0 > 0), (0 + 3 + (-1) > 0), ..] which is
# [True, False, True]
x.extend(DATA)
run()
# Inspect output
assert recent_values(t) == [True, False, True, False]
def map_window_example_simple_with_state():
#----------------------------------------------------------------
# Example with state and no additional parameters
#----------------------------------------------------------------
# Declare functions
def comp(this_list, state):
# this_list is a list
# state is a number
# return next element of output stream, next state
next_state = sum(this_list)
next_output_element = sum(this_list) > state
return next_output_element, next_state
# Create streams
x = Stream('x')
c = Stream('c')
# Create agents
map_window(func=comp,
in_stream=x,
out_stream=c,
state=0,
window_size=2,
step_size=4)
# Explanation of agent
# The initial state is 0
# The initial value of this_list is x[:window_size]
# The zeroth element of the output stream is:
# func(x[:window_size], state)
# c[0] = sum(x[:window_size]) > 0 because 0 is the initial state
# c[j] = sum(x[step_size*j:step_size*j+window_size]) >
# sum(x[step_size*(j-1):step_size*(j-1)+window_size])
# Put data into input streams and run.
DATA = [1, 10, 0, -1, 2, 3, -10, -20, 11, 1]
# With window_size=2, step_size=4, and this DATA the output is:
# [(1 + 10 > 0), (2 + 3 > 1 + 10), (11 + 1 > 2 + 3)] which is
# [True, False, True]
x.extend(DATA)
run()
# Inspect output
assert recent_values(c) == [True, False, True]
def heavy_hitters_stream(
in_stream, out_stream, window_size,
heavy_hitters_object):
"""
Parameters
----------
in_stream: Stream
The input stream of the agent.
out_stream: Stream
The output stream of the agent.
Each element of the output stream is a dict which
represents the heavy hitters.
window_size: int, positive
An element is appended to the output stream when
ever the input stream is a multiple of window_size.
heavy_hitters_object: HeavyHitters
An instance of HeavyHitters.
"""
def f(window):
for element in window:
heavy_hitters_object.add(element)
return copy.copy(heavy_hitters_object.heavy_hitters)
map_window(f, in_stream, out_stream, window_size, step_size=window_size)
def pick_orientation(scaled, timestamps, orientation):
""" Sends picks on a single orientation, either 'n', 'e', or 'z'. """
# ---------------------------------------------------------------
# CREATE AGENTS AND STREAMS
# ---------------------------------------------------------------
# 1. DECIMATE SCALED DATA.
# Window of size DECIMATION is decimated to its average.
decimated = Stream('decimated')
map_window(lambda v: sum(v) / float(len(v)), scaled, decimated,
window_size=self.decimation, step_size=self.decimation)
# 2. DECIMATE TIMESTAMPS.
# Window of size DECIMATION is decimated to its last value.
decimated_timestamps = Stream('decimated_timestamps')
map_window(lambda window: window[-1],
timestamps, decimated_timestamps,
window_size=self.decimation, step_size=self.decimation)
# 3. DEMEAN (subtract mean from) DECIMATED STREAM.
# Subtract mean of window from the window's last value.
# Move sliding window forward by 1 step.
demeaned = Stream('demeaned', initial_value=[0.0] * (LTA_count - 1))
map_window(lambda window: window[-1] - sum(window) / float(len(window)),
decimated, demeaned,
window_size=LTA_count, step_size=1)
# 4. MERGE TIMESTAMPS WITH DEMEANED ACCELERATIONS.
# Merges decimated_timestamps and demeaned to get timestamped_data.
timestamped_data = Stream('timestamped_data')
zip_streams(in_streams=[decimated_timestamps, demeaned], out_stream=timestamped_data)
# 5. DETECT PICKS.
# Output a pick if the value part of the time_value (t_v) exceeds threshold.
picks = Stream('picks')
filter_element(lambda t_v: abs(t_v[1]) > self.pick_threshold, timestamped_data, picks)
# 6. QUENCH PICKS.
# An element is a (timestamp, value).
# Start a new quench when timestamp > QUENCH_PERIOD + last_quench.
# Update the last quench when a new quench is initiated.
# Initially the last_quench (i.e. state) is 0.
quenched_picks = Stream('quenched_picks')
# f is the filtering function
def f(timestamped_value, last_quench, QUENCH_PERIOD):
timestamp, value = timestamped_value
new_quench = timestamp > QUENCH_PERIOD + last_quench
last_quench = timestamp if new_quench else last_quench
# return filter condition (new_quench) and next state (last_quench)
return new_quench, last_quench
filter_element(f, picks, quenched_picks, state=0, QUENCH_PERIOD=2)
# 7. SEND QUENCHED PICKS.
self.send_event(quenched_picks)
def g(in_stream, out_stream, window_size, step_size, **kwargs):
return map_window(func, in_stream, out_stream,
window_size, step_size, **kwargs)
def f(in_streams, out_streams):
"""
Compute Function for Sensor Reader
Parameters
----------
in_streams: list of input Streams - acceleration reordered to conform SAF standard of [N, E, Z]
in_streams[0] - acceleration N
in_streams[1] - acceleration E
in_streams[2] - acceleration Z
in_streams[3] - timestamp
out_streams: list of Streams
out_streams[0] - acceleration N (averaged and picked)
out_streams[1] - acceleration E (averaged and picked)
out_streams[2] - acceleration Z (averaged and picked)
out_streams[3] - timestamp
"""
n_acc = len(in_streams) - 1
# DECLARE STREAMS
scaled_acc = [Stream('scaled_acc_' + str(i)) for i in range(n_acc)]
inverted_acc = Stream('inverted_acc') # stream for inverted acceleration E
averaged_acc = [Stream('averaged_acc_' + str(i)) for i in range(n_acc)]
acc_timestamp = Stream(
'acc_timestamp') # timestamp corresponding to the averaged_acc
acc_merged = Stream(
'acc_merged') # acceleration stream merged with timestamp stream
acc_picked = Stream(
'acc_picked') # stream of acc data picked according to timestamp
# CREATE AGENTS
# 1. SCALE ACCELERATION
# our special order CSN Phidgets are scaled to +/- 2g instead of +/- 6g
def scale_g(v):
return PHIDGETS_ACCELERATION_TO_G * v
for i in range(n_acc):
map_element(func=scale_g,
in_stream=in_streams[i],
out_stream=scaled_acc[i])
# TODO: CHECK AND REPORT MISSING SAMPLES
# if self.last_phidgets_timestamp:
# sample_increment = int(
# round((phidgets_timestamp - self.last_phidgets_timestamp) / PHIDGETS_NOMINAL_DATA_INTERVAL))
# if sample_increment > 4 * self.decimation:
# logging.warn('Missing >3 samples: last sample %s current sample %s missing samples %s', \
# self.last_phidgets_timestamp, phidgets_timestamp, sample_increment)
# elif sample_increment == 0:
# logging.warn('Excess samples: last sample %s current sample %s equiv samples %s', \
# self.last_phidgets_timestamp, phidgets_timestamp, sample_increment)
# 2. INVERT ACCELERATION E
# invert channel 1 (E-W) - this results in +1g being reported when the sensor is resting on its E side
def invert_channel(v):
return -1 * v
map_element(func=invert_channel,
in_stream=scaled_acc[1],
out_stream=inverted_acc)
# 3. AVERAGE WINDOW
def average_samples(window):
return sum(window) / float(len(window))
# average for inverted channel
map_window(func=average_samples,
in_stream=inverted_acc,
out_stream=averaged_acc[1],
window_size=PHIDGETS_DECIMATION,
step_size=PHIDGETS_DECIMATION)
for i in [0, 2]:
map_window(func=average_samples,
in_stream=scaled_acc[i],
out_stream=averaged_acc[i],
window_size=PHIDGETS_DECIMATION,
step_size=PHIDGETS_DECIMATION)
# 4. OBTAIN CORRESPONDING TIMESTAMP
def get_timestamp(window):
return window[-1]
map_window(func=get_timestamp,
in_stream=in_streams[3],
out_stream=acc_timestamp,
window_size=PHIDGETS_DECIMATION,
step_size=PHIDGETS_DECIMATION)
# 5. ZIP ACCELERATION AND TIMESTAMP STREAMS
zip_stream(in_streams=averaged_acc + [acc_timestamp],
out_stream=acc_merged)
# 6. QUENCH SENSOR READING
def timestamp_picker(v, state):
if v[3] - state > PICKER_INTERVAL: # generate output
return False, v[3]
else:
return True, state
filter_element(func=timestamp_picker,
in_stream=acc_merged,
out_stream=acc_picked,
state=0)
# 7. UNZIP STREAM - to pass streams to other processes
unzip(in_stream=acc_picked, out_streams=out_streams)
def g(in_streams, out_streams):
"""
Compute Function for Picker
Parameters
----------
in_streams: list of input Streams passed from sensor reader process (f)
in_streams[0] - acceleration N
in_streams[1] - acceleration E
in_streams[2] - acceleration Z
in_streams[3] - timestamp
"""
# DECLARE STREAMS
adjusted_acc = [
Stream('adjusted_acc_{}'.format(i))
for i in range(len(in_streams) - 1)
]
adjusted_timestamp = Stream('adjusted_timestamp')
merged_acc = Stream('acc_merged')
filtered_acc = Stream('filtered_acc')
quenched_acc = Stream('quenched_acc')
# DEFINE AGENTS
# 1. ADJUST LTA - subtract long-term-average from sample data
def adjust_lta(window):
return abs(window[-1] - sum(window) / len(window))
for i in range(len(in_streams) - 1):
map_window(func=adjust_lta,
in_stream=in_streams[i],
out_stream=adjusted_acc[i],
window_size=LTA_COUNT,
step_size=1)
# 2. ADJUST TIMESTAMP - obtain timestamp corresponding to each window
def adjust_timestamp(window):
return window[-1]
map_window(func=adjust_timestamp,
in_stream=in_streams[-1],
out_stream=adjusted_timestamp,
window_size=LTA_COUNT,
step_size=1)
# 3. ZIP STREAM - zip acceleration and timestamp streams
zip_stream(in_streams=adjusted_acc + [adjusted_timestamp],
out_stream=merged_acc)
# 4. DETECT ANOMALY - filter out small magnitude to report only large acceleration
def detect_anomaly(v):
return any(map(lambda x: x > PICKER_THRESHOLD, v[:-1]))
filter_element(func=detect_anomaly,
in_stream=merged_acc,
out_stream=filtered_acc)
# 5. QUENCH PICKER
def quench_picker(v, state):
timestamp = v[3]
if timestamp - state < MINIMUM_REPICK_INTERVAL_SECONDS:
return _no_value, state
else:
state = timestamp
return v, state
map_element(func=quench_picker,
in_stream=filtered_acc,
out_stream=quenched_acc,
state=0)
# 6. STREAM RESULTS TO FILE - for test purposes
stream_to_file(quenched_acc, './phidget_data.txt')
from stream import Stream, StreamArray
from stream import _no_value, _multivalue
from check_agent_parameter_types import *
from recent_values import recent_values
from op import map_window
scheduler = Stream.scheduler
# In the following, x is a stream that must be declared
# before the functions are called.
x = Stream('x')
#----------------------------------------------------------------
# Example with no state and no additional parameters
#----------------------------------------------------------------
s = Stream()
map_window(func=sum, in_stream=x, out_stream=s, window_size=2, step_size=10)
#y[j] = f(x[j*step_size : j*step_size + window_size]) for all j.
#----------------------------------------------------------------
# Example with state and no additional parameters
#----------------------------------------------------------------
this_list = range(10)
def comp(this_list, sum_of_previous_list):
# Current state is sum_of_previous_list
# Next state will be sum(this_list)
return sum(this_list) > sum_of_previous_list, sum(this_list)
c = Stream()
def test_pick_orientation_with_verbose_output():
PHIDGETS_ACCELERATION_TO_G = 1.0 / 3.0
DECIMATION = 2
LTA_count = 2
PICK_THRESHOLD = 0.5
# ---------------------------------------------------------------
# Input streams
# raw is the stream of raw acceleration data along one axis.
# timestamps is the stream of timestamps
scaled = Stream('scaled')
timestamps = Stream('timestamps')
# Decimate acceleration.
# Window of size DECIMATION is decimated to its average.
# Input = scaled
# Output = decimated.
decimated = Stream('decimated')
map_window(lambda v: sum(v) / float(len(v)),
scaled,
decimated,
window_size=DECIMATION,
step_size=DECIMATION)
# Decimate timestamps.
# Window of size DECIMATION is decimated to its last value.
# Input = timestamps
# Output = decimated_timestamps.
decimated_timestamps = Stream('decimated_timestamps')
map_window(lambda window: window[-1],
timestamps,
decimated_timestamps,
window_size=DECIMATION,
step_size=DECIMATION)
# Demean (subtract mean) from decimated stream.
# Subtract mean of window from the window's last value.
# Move sliding window forward by 1 step.
# Input = decimated
# Output = demeaned
demeaned = Stream('demeaned', initial_value=[0.0] * (LTA_count - 1))
map_window(lambda window: window[-1] - sum(window) / float(len(window)),
decimated,
demeaned,
window_size=LTA_count,
step_size=1)
# Add timestamps to demeaned accelerations.
# Merges decimated_timestamps and demeaned to get timestamped_data.
# Inputs = decimated_timestamps, demeaned
# Outputs = timestamped_data
timestamped_data = Stream('timestamped_data')
zip_streams(in_streams=[decimated_timestamps, demeaned],
out_stream=timestamped_data)
# Detect picks.
# Output a pick if the value part of the time_value (t_v) exceeds threshold.
# Input = timestamped_data
# Output = picks
picks = Stream('picks')
filter_element(lambda t_v: abs(t_v[1]) > PICK_THRESHOLD, timestamped_data,
picks)
# Quench picks.
# An element is a (timestamp, value).
# Start a new quench when timestamp > QUENCH_PERIOD + last_quench.
# Update the last quench when a new quench is initiated.
# Initially the last_quench (i.e. state) is 0.
# Input = picks
# Output = quenched_picks
quenched_picks = Stream('quenched_picks')
# f is the filtering function
def f(timestamped_value, last_quench, QUENCH_PERIOD):
timestamp, value = timestamped_value
new_quench = timestamp > QUENCH_PERIOD + last_quench
last_quench = timestamp if new_quench else last_quench
# return filter condition (new_quench) and next state (last_quench)
return new_quench, last_quench
filter_element(f, picks, quenched_picks, state=0, QUENCH_PERIOD=2)
# Send quenched picks.
send_event(quenched_picks, orientation='n')
# ---------------------------------------------------------------
# ---------------------------------------------------------------
# Drive test
print_stream(timestamps, 'timestamps')
print_stream(scaled, 'scaled')
print_stream(decimated, 'decimated')
print_stream(decimated_timestamps, 'decimated_timestamps')
print_stream(demeaned, 'demeaned')
print_stream(timestamped_data, 'timestamped_data')
print_stream(picks, 'picks')
scaled.extend([1.0, 1.0, 2.0, 4.0, 4.0, 20.0, 8.0, 8.0, 4.0, 6.0])
timestamps.extend(list(range(12)))
run()
from stream import Stream, StreamArray
from stream import _no_value, _multivalue
from check_agent_parameter_types import *
from recent_values import recent_values
from op import map_window
# In the following examples, x is a stream that is an input stream of
# the agents.
x = Stream('x')
#----------------------------------------------------------------
# Example with no state and no additional parameters
#----------------------------------------------------------------
s = Stream()
map_window(func=sum, in_stream=x, out_stream=s,
window_size=2, step_size=10)
# out_stream[j] = \
# func(in_stream[j*step_size : j*step_size + window_size])
# for all j.
# So:
# s[j] = x[10*j] + x[10*j+1], for all j
#----------------------------------------------------------------
# Example with state and no additional parameters
#----------------------------------------------------------------
this_list = range(10)
def comp(this_list, state):
# Current state is sum_of_previous_list
# Next state will be sum(this_list)
# return next element of output stream, next state
next_state = sum(this_list)
def detect_anomaly(window, short_window_size, threshold, cutoff):
current_average = statistics.mean(window[-short_window_size:])
clipped_window = [v for v in window if v < cutoff]
long_term_average = statistics.mean(clipped_window)
print current_average, long_term_average
anomaly = current_average > long_term_average * threshold
return anomaly
if __name__ == '__main__':
import random
input_sequence = [random.random() for _ in range(100)]
input_sequence.extend([10 + random.random() for _ in range(5)])
input_sequence.extend([random.random() for _ in range(100)])
# Create streams
s = Stream('s')
t = Stream('t')
map_window(
func=detect_anomaly, in_stream=s, out_stream=t,
window_size=50, step_size=1, short_window_size=5,
threshold=3, cutoff=0.9)
print_stream(t)
s.extend(input_sequence)
# Execute a step of the scheduler
Stream.scheduler.step()
def f(in_streams, out_streams):
"""
Parameters
----------
in_streams: list of Stream
in_streams is usually a list of 3 streams indicating
measurements in x, y and z directions or equivalently
in e, n, and z (for east, north, vertical) directions.
Each triaxial sensor generates x, y and z streams.
out_streams: list of Stream
out_streams has only one element, which is a
Stream of int. An element of this stream is either
1.0 or 0.0. An element is 1.0 to indicate that an
anomaly was detected and is 0.0 otherwise.
"""
# DECLARE STREAMS
# 1. zero_means
# Array of stream with one stream for each stream in in_streams
# zero_means[0, 1, 2] usually represent streams in the x, y, z direction generated by a single triaxial sensor.
zero_means = [
Stream('zero_means_' + str(i)) for i in range(len(in_streams))
]
# magnitudes is a stream of magnitudes of a vector from its x, y, z values
magnitudes = Stream('magnitudes')
# CREATE AGENTS
# 1. subtract_mean agent
# Define the terminating function
def subtract_mean(window):
return window[-1] - sum(window) / float(len(window))
# Wrap the terminating function to create an agent
for i in range(len(in_streams)):
map_window(func=subtract_mean,
in_stream=in_streams[i],
out_stream=zero_means[i],
window_size=500,
step_size=1,
initial_value=0.0)
# 2. magnitude agent
# Define the terminating function
def magnitude_of_vector(coordinates):
return math.sqrt(sum([v * v for v in coordinates]))
# Wrap the terminating function to create an agent
zip_map(func=magnitude_of_vector,
in_streams=zero_means,
out_stream=magnitudes)
# 3. local anomaly agent
# Define the terminating function
def simple_anomaly(value):
if value > ANOMALY_THRESHOLD:
return 1.0
else:
return 0.0
# Wrap the terminating function to create an agent
map_element(func=simple_anomaly,
in_stream=magnitudes,
out_stream=out_streams[0])
