def testJul22SplitAroundReboot(self): dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data)
def testZeroDurationPlaceInterpolationSingleSync(self): # Test for 545114feb5ac15caac4110d39935612525954b71 dataFile_1 = "emission/tests/data/real_examples/shankari_2016-01-12" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-01-13" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 12}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 13}) cacheKey_1 = "diary/trips-2016-01-12" cacheKey_2 = "diary/trips-2016-01-13" ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile_1) self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) # Although we process the day's data in two batches, we should get the same result self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) # Although we process the day's data in two batches, we should get the same result self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data)
def testZeroDurationPlaceInterpolationMultiSync(self): # Test for 545114feb5ac15caac4110d39935612525954b71 dataFile_1 = "emission/tests/data/real_examples/shankari_2016-01-12" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-01-13" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 12}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 13}) cacheKey_1 = "diary/trips-2016-01-12" cacheKey_2 = "diary/trips-2016-01-13" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data)
def testJul22SplitAroundReboot(self): dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) # Although we process the day's data in two batches, we should get the same result self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) # Although we process the day's data in two batches, we should get the same result self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data)
def testAug10MultiSyncEndNotDetected(self): # Re-run, but with multiple calls to sync data # This tests the effect of online versus offline analysis and segmentation with potentially partial data dataFile = "emission/tests/data/real_examples/shankari_2016-08-10" start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 9}) end_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10}) cacheKey = "diary/trips-2016-08-10" with open( "emission/tests/data/real_examples/shankari_2016-08-910.ground_truth" ) as gtf: ground_truth = json.load(gtf, object_hook=bju.object_hook) logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().estimated_document_count()) with open(dataFile) as df: all_entries = json.load(df, object_hook=bju.object_hook) ts_1030 = arrow.get("2016-08-10T10:30:00-07:00").timestamp logging.debug("ts_1030 = %s, converted back = %s" % (ts_1030, arrow.get(ts_1030).to("America/Los_Angeles"))) before_1030_entries = [ e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1030 ] after_1030_entries = [ e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1030 ] # First load all data from the 9th. Otherwise, the missed trip is the first trip, # and we don't set the last_ts_processed # See the code around "logging.debug("len(segmentation_points) == 0, early return")" etc.setupRealExample( self, "emission/tests/data/real_examples/shankari_2016-08-09") # Sync at 10:30 to capture all the points on the trip *to* the optometrist # Skip the last few points to ensure that the trip end is skipped self.entries = before_1030_entries[0:-2] etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Then sync after 10:30 self.entries = after_1030_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) self.persistGroundTruthIfNeeded(api_result, dataFile, start_ld, cacheKey) # Although we process the day's data in two batches, we should get the same result self.compare_approx_result(ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testOct07MultiSyncSpuriousEndDetected(self): # Re-run, but with multiple calls to sync data # This tests the effect of online versus offline analysis and segmentation with potentially partial data dataFile = "emission/tests/data/real_examples/issue_436_assertion_error" start_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 0o7}) end_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 0o7}) cacheKey = "diary/trips-2016-10-07" with open(dataFile + ".ground_truth") as gtf: ground_truth = json.load(gtf, object_hook=bju.object_hook) logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().estimated_document_count()) with open(dataFile) as df: all_entries = json.load(df, object_hook=bju.object_hook) # 18:01 because the transition was at 2016-02-22T18:00:09.623404-08:00, so right after # 18:00 ts_1800 = arrow.get("2016-10-07T18:33:11-07:00").timestamp logging.debug("ts_1800 = %s, converted back = %s" % (ts_1800, arrow.get(ts_1800).to("America/Los_Angeles"))) before_1800_entries = [ e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1800 ] after_1800_entries = [ e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1800 ] # Sync at 18:00 to capture all the points on the trip *to* the optometrist # Skip the last few points to ensure that the trip end is skipped etc.createAndFillUUID(self) self.entries = before_1800_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Then sync after 18:00 self.entries = after_1800_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) self.persistGroundTruthIfNeeded(api_result, dataFile, start_ld, cacheKey) # Although we process the day's data in two batches, we should get the same result self.compare_approx_result(ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testResetToFuture(self): """ - Load data for both days - Run pipelines - Reset to a date after the two - Verify that all is well - Re-run pipelines and ensure that there are no errors """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Reset to a date well after the two days reset_ts = arrow.get("2017-07-24").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Data should be untouched because of early return api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data) # Re-running the pipeline again should not affect anything etc.runIntakePipeline(self.testUUID)
def testFeb22MultiSyncEndNotDetected(self): # Re-run, but with multiple calls to sync data # This tests the effect of online versus offline analysis and segmentation with potentially partial data dataFile = "emission/tests/data/real_examples/iphone_2016-02-22" start_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22}) end_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22}) cacheKey = "diary/trips-2016-02-22" ground_truth = json.load(open(dataFile + ".ground_truth"), object_hook=bju.object_hook) logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().count()) all_entries = json.load(open(dataFile), object_hook=bju.object_hook) # 18:01 because the transition was at 2016-02-22T18:00:09.623404-08:00, so right after # 18:00 ts_1800 = arrow.get("2016-02-22T18:00:30-08:00").timestamp logging.debug("ts_1800 = %s, converted back = %s" % (ts_1800, arrow.get(ts_1800).to("America/Los_Angeles"))) before_1800_entries = [ e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1800 ] after_1800_entries = [ e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1800 ] # Sync at 18:00 to capture all the points on the trip *to* the optometrist # Skip the last few points to ensure that the trip end is skipped import uuid self.testUUID = uuid.uuid4() self.entries = before_1800_entries[0:-2] etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Then sync after 18:00 self.entries = after_1800_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Although we process the day's data in two batches, we should get the same result self.compare_approx_result(ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testAug11(self): # This is a more complex day. Tests: # PR #352 (should not split trip to Oakland) # PR #348 (trip from station to OAK DOT) # PR #357 (trip to Radio Shack is complete and not truncated) # PR #345 (no cleaned trips are skipped) dataFile = "emission/tests/data/real_examples/shankari_2016-08-11" ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 11}) cacheKey = "diary/trips-2016-08-11" ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testAug11(self): # This is a more complex day. Tests: # PR #352 (should not split trip to Oakland) # PR #348 (trip from station to OAK DOT) # PR #357 (trip to Radio Shack is complete and not truncated) # PR #345 (no cleaned trips are skipped) dataFile = "emission/tests/data/real_examples/shankari_2016-08-11" ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 11}) cacheKey = "diary/trips-2016-08-11" with open(dataFile + ".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data)
def testJun21(self): # This is a more complex day. Tests: # PR #357 (spurious trip at 14:00 should be segmented and skipped) # PR #358 (trip back from bella's house at 16:00) dataFile = "emission/tests/data/real_examples/shankari_2016-06-21" ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 21}) cacheKey = "diary/trips-2016-06-21" with open(dataFile + ".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data)
def testJun20(self): # This is a fairly straightforward day. Tests mainly: # - ordering of trips # - handling repeated location entries with different write timestamps # We have two identical location points with ts = 1466436483.395 and write_ts = 1466436496.4, 1466436497.047 dataFile = "emission/tests/data/real_examples/shankari_2016-06-20" ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 20}) cacheKey = "diary/trips-2016-06-20" with open(dataFile + ".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data)
def testAug10(self): # This is a more complex day. Tests: # PR #302 (trip to optometrist) # PR #352 (split optometrist trip) dataFile = "emission/tests/data/real_examples/shankari_2016-08-10" ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10}) cacheKey = "diary/trips-2016-08-10" with open(dataFile+".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testJun20(self): # This is a fairly straightforward day. Tests mainly: # - ordering of trips # - handling repeated location entries with different write timestamps # We have two identical location points with ts = 1466436483.395 and write_ts = 1466436496.4, 1466436497.047 dataFile = "emission/tests/data/real_examples/shankari_2016-06-20" ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 20}) cacheKey = "diary/trips-2016-06-20" with open(dataFile+".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testAug10(self): # This is a more complex day. Tests: # PR #302 (trip to optometrist) # PR #352 (split optometrist trip) dataFile = "emission/tests/data/real_examples/shankari_2016-08-10" ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10}) cacheKey = "diary/trips-2016-08-10" ground_truth = json.load(open(dataFile + ".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data)
def get_trips_for_day(user_uuid, day, force_refresh): """ The day argument here is a string such as 2015-10-01 or 2016-01-01. We will parse this to a datetime, which we will use to query the data in the timeseries. We could also cache the timeline views in a separate collection and just look up from there. The challenge is to then decide when to recompute a view - we can't use the standard technique that we use for the other stages because we will have to recompute the timeline for the current day multiple times, for example. """ # I was going to read from the user cache if it existed there, and recreate # from scratch if it didn't. But that would involve adding a getDocument # field to the usercache, which I had intentionally not added before this. # The problem with adding a getDocument method is that then the usercache # is no longer a cache - it is "storage" that is used internally. If we # want to do that, we should really store it as a materialized view and not # only in the usercache, which should be a cache of values stored elsewhere. parsed_dt = dup.parse(day) start_dt = ecwl.LocalDate({ 'year': parsed_dt.year, 'month': parsed_dt.month, 'day': parsed_dt.day }) end_dt = start_dt return gfc.get_geojson_for_dt(user_uuid, start_dt, end_dt)
def testSunilShortTrips(self): dataFile = "emission/tests/data/real_examples/sunil_2016-07-27" start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 27}) cacheKey = "diary/trips-2016-07-27" etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.assertEqual(api_result, [])
def testAirTripHawaiiEnd(self): dataFile = "emission/tests/data/real_examples/shankari_2016-08-04" start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 4}) cacheKey = "diary/trips-2016-07-27" ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testSunilShortTrips(self): dataFile = "emission/tests/data/real_examples/sunil_2016-07-27" start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 27}) cacheKey = "diary/trips-2016-07-27" ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.assertEqual(api_result, [])
def testAug10MultiSyncEndDetected(self): # Re-run, but with multiple calls to sync data # This tests the effect of online versus offline analysis and segmentation with potentially partial data dataFile = "emission/tests/data/real_examples/shankari_2016-08-10" start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 9}) end_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10}) cacheKey = "diary/trips-2016-08-10" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2016-08-910.ground_truth"), object_hook=bju.object_hook) logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().count()) all_entries = None with open(dataFile) as secondfp: all_entries = json.load(secondfp, object_hook = bju.object_hook) ts_1030 = arrow.get("2016-08-10T10:30:00-07:00").timestamp logging.debug("ts_1030 = %s, converted back = %s" % (ts_1030, arrow.get(ts_1030).to("America/Los_Angeles"))) before_1030_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1030] after_1030_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1030] # First load all data from the 9th. Otherwise, the missed trip is the first trip, # and we don't set the last_ts_processed # See the code around "logging.debug("len(segmentation_points) == 0, early return")" etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2016-08-09") # Sync at 10:30 to capture all the points on the trip *to* the optometrist self.entries = before_1030_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Then sync after 10:30 self.entries = after_1030_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Although we process the day's data in two batches, we should get the same result self.compare_approx_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testResetToFuture(self): """ - Load data for both days - Run pipelines - Reset to a date after the two - Verify that all is well - Re-run pipelines and ensure that there are no errors """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Reset to a date well after the two days reset_ts = arrow.get("2017-07-24").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Data should be untouched because of early return api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data) # Re-running the pipeline again should not affect anything etc.runIntakePipeline(self.testUUID)
def testOverriddenModeHack(self): # Test for https://github.com/e-mission/e-mission-server/issues/457 dataFile = "emission/tests/data/real_examples/test_overriden_mode_hack.jul-31" start_ld = ecwl.LocalDate({'year': 2017, 'month': 7, 'day': 31}) cacheKey = "diary/trips-2017-07-31" ground_truth = json.load(open("emission/tests/data/real_examples/test_overriden_mode_hack.jul-31.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testJan16SpeedAssert(self): # Test for https://github.com/e-mission/e-mission-server/issues/457 dataFile = "emission/tests/data/real_examples/another_speed_assertion_failure.jan-16" start_ld = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 16}) cacheKey = "diary/trips-2016-01-16" ground_truth = json.load(open("emission/tests/data/real_examples/another_speed_assertion_failure.jan-16.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testJumpSmoothingSectionStart(self): dataFile = "emission/tests/data/real_examples/shankari_2016-independence_day_jump_bus_start" start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 15}) cacheKey = "diary/trips-2016-08-15" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2016-independence_day.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testTsMismatch(self): # Test for https://github.com/e-mission/e-mission-server/issues/457 dataFile = "emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12" start_ld = ecwl.LocalDate({'year': 2016, 'month': 12, 'day': 12}) cacheKey = "diary/trips-2016-12-12" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testTsMismatch(self): # Test for https://github.com/e-mission/e-mission-server/issues/457 dataFile = "emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12" start_ld = ecwl.LocalDate({'year': 2016, 'month': 12, 'day': 12}) cacheKey = "diary/trips-2016-12-12" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testJumpSmoothingSectionStart(self): dataFile = "emission/tests/data/real_examples/shankari_2016-independence_day_jump_bus_start" start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 15}) cacheKey = "diary/trips-2016-08-15" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2016-independence_day.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testOverriddenModeHack(self): # Test for https://github.com/e-mission/e-mission-server/issues/457 dataFile = "emission/tests/data/real_examples/test_overriden_mode_hack.jul-31" start_ld = ecwl.LocalDate({'year': 2017, 'month': 7, 'day': 31}) cacheKey = "diary/trips-2017-07-31" ground_truth = json.load(open("emission/tests/data/real_examples/test_overriden_mode_hack.jul-31.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testJan16SpeedAssert(self): # Test for https://github.com/e-mission/e-mission-server/issues/457 dataFile = "emission/tests/data/real_examples/another_speed_assertion_failure.jan-16" start_ld = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 16}) cacheKey = "diary/trips-2016-01-16" ground_truth = json.load(open("emission/tests/data/real_examples/another_speed_assertion_failure.jan-16.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testResetToStart(self): """ - Load data for both days - Run pipelines - Verify that all is well - Reset to start - Verify that there is no analysis data - Re-run pipelines - Verify that all is well """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Check results: so far, so good api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data) # Reset pipeline to start epr.reset_user_to_start(self.testUUID, is_dry_run=False) # Now there are no results api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.assertEqual(api_result, []) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.assertEqual(api_result, []) # Re-run the pipeline again etc.runIntakePipeline(self.testUUID) # Should be back to ground truth api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data)
def testResetToPast(self): """ - Load data for both days - Run pipelines - Verify that all is well - Reset to a date before both - Verify that analysis data for the both days is removed - Re-run pipelines - Verify that all is well """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Verify that all is well api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data) # Reset to a date well before the two days reset_ts = arrow.get("2015-07-24").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Data should be completely deleted api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.assertEqual(api_result, []) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.assertEqual(api_result, []) # Re-running the pipeline again etc.runIntakePipeline(self.testUUID) # Should reconstruct everything api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data)
def testFeb22ShortTripsDistance(self): dataFile = "emission/tests/data/real_examples/iphone_3_2016-02-22" start_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22}) end_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22}) cacheKey = "diary/trips-2016-02-22" ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testIosJumpsAndUntrackedSquishing(self): # Test for a2c0ee4e3ceafa822425ceef299dcdb01c9b32c9 dataFile = "emission/tests/data/real_examples/sunil_2016-07-20" start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 20}) cacheKey = "diary/trips-2016-07-20" ground_truth = json.load(open("emission/tests/data/real_examples/sunil_2016-07-20.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testIndexLengthChange(self): # Test for 94f67b4848611fa01c4327a0fa0cab97c2247744 dataFile = "emission/tests/data/real_examples/shankari_2015-08-23" start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 23}) cacheKey = "diary/trips-2015-08-23" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2015-08-23.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testIosJumpsAndUntrackedSquishing(self): # Test for a2c0ee4e3ceafa822425ceef299dcdb01c9b32c9 dataFile = "emission/tests/data/real_examples/sunil_2016-07-20" start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 20}) cacheKey = "diary/trips-2016-07-20" ground_truth = json.load(open("emission/tests/data/real_examples/sunil_2016-07-20.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testIndexLengthChange(self): # Test for 94f67b4848611fa01c4327a0fa0cab97c2247744 dataFile = "emission/tests/data/real_examples/shankari_2015-08-23" start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 23}) cacheKey = "diary/trips-2015-08-23" ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2015-08-23.ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testOct07MultiSyncSpuriousEndDetected(self): # Re-run, but with multiple calls to sync data # This tests the effect of online versus offline analysis and segmentation with potentially partial data dataFile = "emission/tests/data/real_examples/issue_436_assertion_error" start_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 07}) end_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 07}) cacheKey = "diary/trips-2016-10-07" ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook) logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().count()) all_entries = json.load(open(dataFile), object_hook = bju.object_hook) # 18:01 because the transition was at 2016-02-22T18:00:09.623404-08:00, so right after # 18:00 ts_1800 = arrow.get("2016-10-07T18:33:11-07:00").timestamp logging.debug("ts_1800 = %s, converted back = %s" % (ts_1800, arrow.get(ts_1800).to("America/Los_Angeles"))) before_1800_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1800] after_1800_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1800] # Sync at 18:00 to capture all the points on the trip *to* the optometrist # Skip the last few points to ensure that the trip end is skipped import uuid self.testUUID = uuid.uuid4() self.entries = before_1800_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Then sync after 18:00 self.entries = after_1800_entries etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Although we process the day's data in two batches, we should get the same result self.compare_approx_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testAug27TooMuchExtrapolation(self): dataFile = "emission/tests/data/real_examples/shankari_2015-aug-27" start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27}) end_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27}) cacheKey = "diary/trips-2015-08-27" with open(dataFile+".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def standardMatchDataGroundTruth(self, dataFile, ld, cacheKey): with open(dataFile+".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testAug27TooMuchExtrapolation(self): dataFile = "emission/tests/data/real_examples/shankari_2015-aug-27" start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27}) end_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27}) cacheKey = "diary/trips-2015-08-27" with open(dataFile + ".ground_truth") as gfp: ground_truth = json.load(gfp, object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld) # Although we process the day's data in two batches, we should get the same result self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth).data)
def get_trips_for_day(user_uuid, day, force_refresh): """ The day argument here is a string such as 2015-10-01 or 2016-01-01. We will parse this to a datetime, which we will use to query the data in the timeseries. We could also cache the timeline views in a separate collection and just look up from there. The challenge is to then decide when to recompute a view - we can't use the standard technique that we use for the other stages because we will have to recompute the timeline for the current day multiple times, for example. """ # I was going to read from the user cache if it existed there, and recreate # from scratch if it didn't. But that would involve adding a getDocument # field to the usercache, which I had intentionally not added before this. # The problem with adding a getDocument method is that then the usercache # is no longer a cache - it is "storage" that is used internally. If we # want to do that, we should really store it as a materialized view and not # only in the usercache, which should be a cache of values stored elsewhere. start_dt = dup.parse(day) end_dt = start_dt + pydt.timedelta(days=1) return gfc.get_geojson_for_dt(user_uuid, start_dt, end_dt)
def testJun21(self): # This is a more complex day. Tests: # PR #357 (spurious trip at 14:00 should be segmented and skipped) # PR #358 (trip back from bella's house at 16:00) dataFile = "emission/tests/data/real_examples/shankari_2016-06-21" ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 21}) cacheKey = "diary/trips-2016-06-21" ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook) etc.setupRealExample(self, dataFile) etc.runIntakePipeline(self.testUUID) # runIntakePipeline does not run the common trips, habitica or store views to cache # So let's manually store to the cache # tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld) # enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query) # cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID, # "metadata.key": cacheKey}) api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld) # self.compare_result(cached_result, ground_truth) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth).data)
def testResetToTsInMiddleOfPlace(self): """ - Load data for both days - Run pipelines - Verify that all is well - Reset to a date between the two - Verify that analysis data for the first day is unchanged - Verify that analysis data for the second day does not exist - Re-run pipelines - Verify that all is well """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Check results: so far, so good api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data) # Reset pipeline to july 23. # Note that this is actually 22nd 16:00 PDT, so this is partway # through the 22nd reset_ts = arrow.get("2016-07-23").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # First day is unchanged, except that the last place doesn't have # exit data. # TODO: Modify ground truth to capture this change # Until then, we know that this will fail # api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) # self.compare_result(ad.AttrDict({'result': api_result}).result, # ad.AttrDict(ground_truth_1).data) # Second day does not exist api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) logging.debug(json.dumps(api_result, indent=4, default=bju.default)) self.assertEqual(api_result, []) # Re-run the pipeline again etc.runIntakePipeline(self.testUUID) # Should be back to ground truth api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data)
def testResetToTsInMiddleOfTrip(self): """ - Load data for both days - Run pipelines - Verify that all is well - Reset to a date between the two - Verify that analysis data for the first day is unchanged - Verify that analysis data for the second day does not exist - Re-run pipelines - Verify that all is well """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Check results: so far, so good api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data) # Reset pipeline to july 24. # Note that this is actually 23nd 16:00 PDT # This will reset in the middle of the untracked time, which is # technically a trip, and will allow us to test the trip resetting # code reset_ts = arrow.get("2016-07-24").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Second day does not exist api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) logging.debug(json.dumps(api_result, indent=4, default=bju.default)) self.assertEqual(api_result, []) # Re-run the pipeline again etc.runIntakePipeline(self.testUUID) # Should be back to ground truth api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result(ad.AttrDict({'result': api_result}).result, ad.AttrDict(ground_truth_2).data)
import dateutil.parser as dup import json import datetime as pydt import time # Our imports import emission.core.get_database as edb import emission.net.usercache.abstract_usercache as enua import emission.analysis.plotting.geojson.geojson_feature_converter as gfc def get_trips_for_day(user_uuid, day) """ The day argument here is a string such as 2015-10-01 or 2016-01-01. We will parse this to a datetime, which we will use to query the data in the timeseries. We could also cache the timeline views in a separate collection and just look up from there. The challenge is to then decide when to recompute a view - we can't use the standard technique that we use for the other stages because we will have to recompute the timeline for the current day multiple times, for example. """ # I was going to read from the user cache if it existed there, and recreate # from scratch if it didn't. But that would involve adding a getDocument # field to the usercache, which I had intentionally not added before this. # The problem with adding a getDocument method is that then the usercache # is no longer a cache - it is "storage" that is used internally. If we # want to do that, we should really store it as a materialized view and not # only in the usercache, which should be a cache of values stored elsewhere. start_dt = dup.parse(day) end_dt = start_dt + pydt.timedelta(days=1) return gfc.get_geojson_for_dt(user_uuid, start_dt, end_dt)
def testResetToTsInMiddleOfPlace(self): """ - Load data for both days - Run pipelines - Verify that all is well - Reset to a date between the two - Verify that analysis data for the first day is unchanged - Verify that analysis data for the second day does not exist - Re-run pipelines - Verify that all is well """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Check results: so far, so good api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data) # Reset pipeline to july 23. # Note that this is actually 22nd 16:00 PDT, so this is partway # through the 22nd reset_ts = arrow.get("2016-07-23").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # First day is unchanged, except that the last place doesn't have # exit data. # TODO: Modify ground truth to capture this change # Until then, we know that this will fail # api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) # self.compare_result(ad.AttrDict({'result': api_result}).result, # ad.AttrDict(ground_truth_1).data) # Second day does not exist api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) logging.debug(json.dumps(api_result, indent=4, default=bju.default)) self.assertEqual(api_result, []) # Re-run the pipeline again etc.runIntakePipeline(self.testUUID) # Should be back to ground truth api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data)
def testResetToTsInMiddleOfTrip(self): """ - Load data for both days - Run pipelines - Verify that all is well - Reset to a date between the two - Verify that analysis data for the first day is unchanged - Verify that analysis data for the second day does not exist - Re-run pipelines - Verify that all is well """ # Load all data dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22" dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25" start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22}) start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25}) cacheKey_1 = "diary/trips-2016-07-22" cacheKey_2 = "diary/trips-2016-07-25" ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"), object_hook=bju.object_hook) ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"), object_hook=bju.object_hook) # Run both pipelines etc.setupRealExample(self, dataFile_1) etc.runIntakePipeline(self.testUUID) self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook) etc.setupRealExampleWithEntries(self) etc.runIntakePipeline(self.testUUID) # Check results: so far, so good api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data) # Reset pipeline to july 24. # Note that this is actually 23nd 16:00 PDT # This will reset in the middle of the untracked time, which is # technically a trip, and will allow us to test the trip resetting # code reset_ts = arrow.get("2016-07-24").timestamp epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Second day does not exist api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) logging.debug(json.dumps(api_result, indent=4, default=bju.default)) self.assertEqual(api_result, []) # Re-run the pipeline again etc.runIntakePipeline(self.testUUID) # Should be back to ground truth api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_1).data) api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2) self.compare_result( ad.AttrDict({ 'result': api_result }).result, ad.AttrDict(ground_truth_2).data)