def testConvertMotionActivity_ios(self):
entry = json.load(open("emission/tests/data/netTests/ios.activity.txt"))
formatted_entry = enuf.convert_to_common_format(ad.AttrDict(entry))
self.assertEquals(formatted_entry.data.confidence, 100)
self.assertEquals(formatted_entry.data.type, ema.MotionTypes.STILL.value)
self.assertEquals(formatted_entry.data.ts, 1446513827.479381)
self.assertTrue(formatted_entry.data.fmt_time.startswith("2015-11-02T17:23:47"))
def testConvertTransition_ios(self):
entry = json.load(open("emission/tests/data/netTests/ios.transition.txt"))
formatted_entry = enuf.convert_to_common_format(ad.AttrDict(entry))
self.assertEquals(formatted_entry.data.curr_state, et.State.WAITING_FOR_TRIP_START.value)
self.assertEquals(formatted_entry.data.transition, et.TransitionType.STOPPED_MOVING.value)
self.assertEquals(formatted_entry.metadata.write_ts, 1446577206.122407)
self.assertTrue(formatted_entry.data.fmt_time.startswith("2015-11-03T11:00:06.122"))
def testConvertTransition(self):
entry = json.load(open("emission/tests/data/netTests/android.transition.txt"))
formatted_entry = enuf.convert_to_common_format(ad.AttrDict(entry))
self.assertEquals(formatted_entry.data.curr_state, et.State.WAITING_FOR_TRIP_START.value)
self.assertEquals(formatted_entry.data.transition, et.TransitionType.INITIALIZE.value)
self.assertEquals(formatted_entry.metadata.write_ts, 1436821510.445)
self.assertTrue(formatted_entry.data.fmt_time.startswith("2015-07-13T14:05:10.445"))
def testConvertMotionActivity(self):
entry = json.load(open("emission/tests/data/netTests/android.activity.txt"))
logging.debug("entry.keys() = %s" % entry.keys())
formatted_entry = enuf.convert_to_common_format(ad.AttrDict(entry))
self.assertEquals(formatted_entry.data.confidence, 100)
self.assertEquals(formatted_entry.data.type, ema.MotionTypes.TILTING.value)
self.assertEquals(formatted_entry.data.ts, 1436826360.493)
self.assertTrue(formatted_entry.data.fmt_time.startswith("2015-07-13T15:26:00.493"))
def moveToLongTerm(self):
"""
In order to move to the long term, we need to do the following:
a) determine the time range to be processed. We do this by checking the
pipeline state. this does not leak information since the process
will run whether there is data for it to work on or not. So the
pipeline state is stored outside of the user cache.
b) process the time range. pass in a function that works on every entry
to convert it to the appropriate format.
c) delete the time range once it is processed (in usercache or here?)
d) update the pipeline state to reflect the new range (here)
"""
# Error handling: if any of the entries has an error in processing, we
# move it to a separate "error_usercache" and process the rest. The
# stage is still marked successful. This means that the stage can never
# be unsuccessful. We could try to keep it, but then the delete query
# below will get significantly more complicated.
uc = enua.UserCache.getUserCache(self.user_id)
messages = uc.getMessage()
# Here, we assume that the user only has data from a single platform.
# Since this is a temporary hack, this is fine
if len(messages) == 0:
logging.debug("No messages to process")
# Since we didn't get the current time range, there is no current
# state, so we don't need to mark it as done
# esp.mark_usercache_done(None)
return
time_query = esp.get_time_range_for_usercache(self.user_id)
ts = etsa.TimeSeries.get_time_series(self.user_id)
curr_entry_it = uc.getMessage(None, time_query)
last_ts_processed = None
for entry_doc in curr_entry_it:
unified_entry = None
try:
# We don't want to use our wrapper classes yet because they are based on the
# standard long-term formats, and we don't yet know whether the
# incoming entries are consistent with them. That's why we have the
# convert_to_common_format step. So let's just wrap this in a
# generic attrdict for now.
entry = ad.AttrDict(entry_doc)
unified_entry = enuf.convert_to_common_format(entry)
ts.insert(unified_entry)
last_ts_processed = ecwe.Entry(unified_entry).metadata.write_ts
time_query.endTs = last_ts_processed
except pymongo.errors.DuplicateKeyError as e:
logging.info("document already present in timeseries, skipping since read-only")
except Exception as e:
logging.exception("Backtrace time")
logging.warn("Got error %s while saving entry %s -> %s"% (e, entry, unified_entry))
ts.insert_error(entry_doc)
logging.debug("Deleting all entries for query %s" % time_query)
uc.clearProcessedMessages(time_query)
esp.mark_usercache_done(self.user_id, last_ts_processed)
def testConvertLocation_ios(self):
entry = json.load(open("emission/tests/data/netTests/ios.location.txt"))
formatted_entry = enuf.convert_to_common_format(ad.AttrDict(entry))
self.assertEquals(formatted_entry.data.accuracy, 65)
self.assertEquals(formatted_entry.data.latitude, 37.39974810579324)
self.assertEquals(formatted_entry.data.longitude, -122.0808742899394)
self.assertEquals(formatted_entry.data.loc, geojson.Point((-122.0808742899394, 37.39974810579324)))
self.assertEquals(formatted_entry.data.ts, 1446503965.190834)
self.assertTrue(formatted_entry.data.fmt_time.startswith("2015-11-02T14:39:25.190"))
self.assertEquals(formatted_entry.metadata.write_ts, 1446503965.760821)
self.assertTrue(formatted_entry.metadata.write_fmt_time.startswith("2015-11-02T14:39:25.760"))
def testConvertLocation(self):
entry = json.load(open("emission/tests/data/netTests/android.location.raw.txt"))
formatted_entry = enuf.convert_to_common_format(ad.AttrDict(entry))
self.assertEquals(formatted_entry.data.accuracy, 52.5)
self.assertEquals(formatted_entry.data.latitude, 37.3885529)
self.assertEquals(formatted_entry.data.longitude, -122.0879696)
self.assertEquals(formatted_entry.data.loc, geojson.Point((-122.0879696, 37.3885529)))
self.assertEquals(formatted_entry.data.ts, 1436826356.852)
self.assertTrue(formatted_entry.data.fmt_time.startswith("2015-07-13T15:25:56.852"))
self.assertEquals(formatted_entry.metadata.write_ts, 1436826357.115)
self.assertTrue(formatted_entry.metadata.write_fmt_time.startswith("2015-07-13T15:25:57.115"))
