class SessionProcessor(PDU):
""" Module which receives notifications that a new measurement is added
to a trajectory. It searches for a nearby trajectory in the processed
sessions DB, and if none is found, creates a new one.
Search is done via spatial & temporal criteria.
"""
QUEUE = 'session_processor'
def __init__(self, **kwargs):
super(SessionProcessor, self).__init__(**kwargs)
self.processed_session_store = ProcessedSessionStore()
def process_message(self, message):
# TODO(andrei): implement source_session_id -> dest_session_id
# affinity table. Right now it's numerically unstable because of lack
# of that.
dst_session_id = self.processed_session_store.session_ending_at(
message['time'], message['info'])
# If no closely matching session is found, create a new one
if dst_session_id is None:
dst_session_id = self.processed_session_store.new_session_id()
# Write the measurement to the "closest" matching destination session
self.processed_session_store.set(sid=dst_session_id,
timestamp=message['time'],
info=message['info'])
def setUp(self):
super(ProcessedSessionStoreTest, self).setUp()
settings.REDIS_SERVER = 'localhost'
self.redis = redis.StrictRedis(host=settings.REDIS_SERVER,
port=settings.REDIS_PORT,
db=settings.REDIS_SESSION_DB)
self.redis.flushdb()
self.store = ProcessedSessionStore()
class ProcessedSessionStoreTest(TestCase):
@classmethod
def setUpClass(cls):
super(ProcessedSessionStoreTest, cls).setUpClass()
# TODO(andrei): move this into a custom test runner
log.setup_logging(level=logging.DEBUG)
def setUp(self):
super(ProcessedSessionStoreTest, self).setUp()
settings.REDIS_SERVER = 'localhost'
self.redis = redis.StrictRedis(host=settings.REDIS_SERVER,
port=settings.REDIS_PORT,
db=settings.REDIS_SESSION_DB)
self.redis.flushdb()
self.store = ProcessedSessionStore()
def test_new_session_id_returns_different_every_time(self):
""" Check that new_session_id() always returns different results. """
N = random.randint(5, 10)
session_ids = [self.store.new_session_id() for _ in xrange(N)]
eq_(len(session_ids), len(set(session_ids)),
"Generated session ids should all be different!")
def _create_new_session(self, N=None,
min_t=None, max_t=None,
min_x=None, max_x=None,
min_y=None, max_y=None,
min_z=None, max_z=None):
N = N or random.randint(50, 100)
min_t = min_t or int(time.time()) - 10
max_t = max_t or int(time.time())
sid = self.store.new_session_id()
for _ in xrange(N):
t = random.randint(min_t, max_t)
info = {}
if min_x is not None and max_x is not None:
info['X'] = random.uniform(min_x, max_x)
if min_y is not None and max_y is not None:
info['Y'] = random.uniform(min_y, max_y)
if min_z is not None and max_z is not None:
info['Z'] = random.uniform(min_z, max_z)
self.store.set(sid, t, info)
return sid
def test_no_session_returned_if_all_are_stale(self):
""" Given a new measurement, and a set of stale sessions,
check that no session is returned for matching with our without
position.
NOTE: for positional matching we deliberately set the session
coordinates equal to the ones of the measurement so that we
ensure the check is done temporally and not spatially.
"""
num_sessions = random.randint(10, 20)
t = int(time.time() * 1000)
threshold = ProcessedSessionStore.TIME_MATCHING_THRESHOLD_MS
old_time_min = t - threshold - 20
old_time_max = t - threshold - 10
for _ in xrange(num_sessions):
self._create_new_session(min_t=old_time_min, max_t=old_time_max)
eq_(self.store.session_ending_at(t, {}), None,
"No stale session should be returned")
# Add some sessions with positional matching
X = random.uniform(100, 500)
Y = random.uniform(100, 500)
Z = random.uniform(100, 500)
for _ in xrange(num_sessions):
self._create_new_session(min_t=old_time_min, max_t=old_time_max,
min_x=X, max_x=X,
min_y=Y, max_y=Y,
min_z=Z, max_z=Z)
eq_(self.store.session_ending_at(t, {'X': X, 'Y': Y, 'Z': Z}), None,
"No stale session should be returned")
def test_new_session_is_returned_with_temporal_match(self):
num_sessions = random.randint(10, 20)
t = int(time.time() * 1000)
threshold = ProcessedSessionStore.TIME_MATCHING_THRESHOLD_MS
old_time_min = t - threshold - 20
old_time_max = t - threshold - 10
for _ in xrange(num_sessions - 1):
self._create_new_session(min_t=old_time_min, max_t=old_time_max)
good_sid = self._create_new_session(min_t=t - threshold + 1,
max_t=t)
eq_(self.store.session_ending_at(t, {}), good_sid,
"The good session (having fresh updated_at) should be returned")
X = random.uniform(100, 500)
Y = random.uniform(100, 500)
Z = random.uniform(100, 500)
eq_(self.store.session_ending_at(t, {'X': X, 'Y': Y, 'Z': Z}), None,
"No session should be returned even if they are fresh because "
"they have no positional data")
def test_new_session_with_old_positional_data_is_not_returned(self):
num_sessions = random.randint(10, 20)
t = int(time.time() * 1000)
threshold = ProcessedSessionStore.TIME_MATCHING_THRESHOLD_MS
positional_threshold = ProcessedSessionStore.POSITION_MATCHING_THRESHOLD_MS
old_time_min = t - threshold - 20
old_time_max = t - threshold - 10
for _ in xrange(num_sessions - 1):
self._create_new_session(min_t=old_time_min, max_t=old_time_max)
X = random.uniform(100, 500)
Y = random.uniform(100, 500)
Z = random.uniform(100, 500)
# Create a "good" session. So far, it has no positional data.
# We will add some "old" positional data, and see that it doesn't
# get returned.
good_sid = self._create_new_session(min_t=t - threshold + 1,
max_t=t)
N = random.randint(50, 100)
for _ in xrange(N):
old_timestamp = random.randint(t - positional_threshold - 20,
t - positional_threshold - 10)
self.store.set(good_sid, old_timestamp, {'X': X, 'Y': Y, 'Z': Z})
eq_(self.store.session_ending_at(t, {}), good_sid,
"The good session (having fresh updated_at) should be returned")
eq_(self.store.session_ending_at(t, {'X': X, 'Y': Y, 'Z': Z}), None,
"Good session should not be returned because it has old "
"positional data")
def test_new_session_with_fresh_but_far_positional_data_is_not_returned(self):
num_sessions = random.randint(10, 20)
t = int(time.time() * 1000)
threshold = ProcessedSessionStore.TIME_MATCHING_THRESHOLD_MS
positional_threshold = ProcessedSessionStore.POSITION_MATCHING_THRESHOLD_MS
distance_threshold = ProcessedSessionStore.POSITION_MATCHING_THRESHOLD_MM
old_time_min = t - threshold - 20
old_time_max = t - threshold - 10
for _ in xrange(num_sessions - 1):
self._create_new_session(min_t=old_time_min, max_t=old_time_max)
X = random.uniform(100, 500)
Y = random.uniform(100, 500)
Z = random.uniform(100, 500)
# Create a "good" session. So far, it has no positional data.
# We will add some "old" positional data, and see that it doesn't
# get returned.
good_sid = self._create_new_session(min_t=t - threshold + 1,
max_t=t)
N = random.randint(50, 100)
for _ in xrange(N):
new_timestamp = random.randint(t - positional_threshold + 1, t)
info = {'X': X, 'Y': Y, 'Z': Z}
coord = random.choice(info.keys())
sign = random.choice([-1, 1])
info[coord] += sign * (distance_threshold + random.uniform(0.5, 1))
self.store.set(good_sid, new_timestamp, info)
eq_(self.store.session_ending_at(t, {}), good_sid,
"The good session (having fresh updated_at) should be returned")
eq_(self.store.session_ending_at(t, {'X': X, 'Y': Y, 'Z': Z}), None,
"Good session should not be returned because it has new but "
"far away positional data")
def test_new_session_with_fresh_and_near_positional_data_is_returned(self):
num_sessions = random.randint(10, 20)
t = int(time.time() * 1000)
threshold = ProcessedSessionStore.TIME_MATCHING_THRESHOLD_MS
positional_threshold = ProcessedSessionStore.POSITION_MATCHING_THRESHOLD_MS
distance_threshold = ProcessedSessionStore.POSITION_MATCHING_THRESHOLD_MM
old_time_min = t - threshold - 20
old_time_max = t - threshold - 10
for _ in xrange(num_sessions - 1):
self._create_new_session(min_t=old_time_min, max_t=old_time_max)
X = random.uniform(100, 500)
Y = random.uniform(100, 500)
Z = random.uniform(100, 500)
# Create a "good" session. So far, it has no positional data.
# We will add some "old" positional data, and see that it doesn't
# get returned.
good_sid = self._create_new_session(min_t=t - threshold + 1,
max_t=t)
N = random.randint(50, 100)
for _ in xrange(N):
new_timestamp = random.randint(t - positional_threshold + 1, t)
info = {'X': X, 'Y': Y, 'Z': Z}
coord = random.choice(info.keys())
sign = random.choice([-1, 1])
info[coord] += sign * (distance_threshold - random.uniform(0.5, 1))
self.store.set(good_sid, new_timestamp, info)
eq_(self.store.session_ending_at(t, {}), good_sid,
"The good session (having fresh updated_at) should be returned")
eq_(self.store.session_ending_at(t, {'X': X, 'Y': Y, 'Z': Z}), good_sid,
"Good session should be returned because it has both new "
"and near positional data")
def __init__(self, **kwargs):
super(SessionProcessor, self).__init__(**kwargs)
self.processed_session_store = ProcessedSessionStore()
