def testJul22SplitAroundReboot(self):
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
def testZeroDurationPlaceInterpolationSingleSync(self):
# Test for 545114feb5ac15caac4110d39935612525954b71
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-01-12"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-01-13"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 12})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 13})
cacheKey_1 = "diary/trips-2016-01-12"
cacheKey_2 = "diary/trips-2016-01-13"
ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"),
object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"),
object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile_1)
self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
def testZeroDurationPlaceInterpolationMultiSync(self):
# Test for 545114feb5ac15caac4110d39935612525954b71
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-01-12"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-01-13"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 12})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 13})
cacheKey_1 = "diary/trips-2016-01-12"
cacheKey_2 = "diary/trips-2016-01-13"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
def testJul22SplitAroundReboot(self):
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"),
object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"),
object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
def testAug10MultiSyncEndNotDetected(self):
# Re-run, but with multiple calls to sync data
# This tests the effect of online versus offline analysis and segmentation with potentially partial data
dataFile = "emission/tests/data/real_examples/shankari_2016-08-10"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 9})
end_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10})
cacheKey = "diary/trips-2016-08-10"
with open(
"emission/tests/data/real_examples/shankari_2016-08-910.ground_truth"
) as gtf:
ground_truth = json.load(gtf, object_hook=bju.object_hook)
logging.info("Before loading, timeseries db size = %s" %
edb.get_timeseries_db().estimated_document_count())
with open(dataFile) as df:
all_entries = json.load(df, object_hook=bju.object_hook)
ts_1030 = arrow.get("2016-08-10T10:30:00-07:00").timestamp
logging.debug("ts_1030 = %s, converted back = %s" %
(ts_1030, arrow.get(ts_1030).to("America/Los_Angeles")))
before_1030_entries = [
e for e in all_entries
if ad.AttrDict(e).metadata.write_ts <= ts_1030
]
after_1030_entries = [
e for e in all_entries
if ad.AttrDict(e).metadata.write_ts > ts_1030
]
# First load all data from the 9th. Otherwise, the missed trip is the first trip,
# and we don't set the last_ts_processed
# See the code around "logging.debug("len(segmentation_points) == 0, early return")"
etc.setupRealExample(
self, "emission/tests/data/real_examples/shankari_2016-08-09")
# Sync at 10:30 to capture all the points on the trip *to* the optometrist
# Skip the last few points to ensure that the trip end is skipped
self.entries = before_1030_entries[0:-2]
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Then sync after 10:30
self.entries = after_1030_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
self.persistGroundTruthIfNeeded(api_result, dataFile, start_ld,
cacheKey)
# Although we process the day's data in two batches, we should get the same result
self.compare_approx_result(ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data,
time_fuzz=60,
distance_fuzz=100)
def testOct07MultiSyncSpuriousEndDetected(self):
# Re-run, but with multiple calls to sync data
# This tests the effect of online versus offline analysis and segmentation with potentially partial data
dataFile = "emission/tests/data/real_examples/issue_436_assertion_error"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 0o7})
end_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 0o7})
cacheKey = "diary/trips-2016-10-07"
with open(dataFile + ".ground_truth") as gtf:
ground_truth = json.load(gtf, object_hook=bju.object_hook)
logging.info("Before loading, timeseries db size = %s" %
edb.get_timeseries_db().estimated_document_count())
with open(dataFile) as df:
all_entries = json.load(df, object_hook=bju.object_hook)
# 18:01 because the transition was at 2016-02-22T18:00:09.623404-08:00, so right after
# 18:00
ts_1800 = arrow.get("2016-10-07T18:33:11-07:00").timestamp
logging.debug("ts_1800 = %s, converted back = %s" %
(ts_1800, arrow.get(ts_1800).to("America/Los_Angeles")))
before_1800_entries = [
e for e in all_entries
if ad.AttrDict(e).metadata.write_ts <= ts_1800
]
after_1800_entries = [
e for e in all_entries
if ad.AttrDict(e).metadata.write_ts > ts_1800
]
# Sync at 18:00 to capture all the points on the trip *to* the optometrist
# Skip the last few points to ensure that the trip end is skipped
etc.createAndFillUUID(self)
self.entries = before_1800_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Then sync after 18:00
self.entries = after_1800_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
self.persistGroundTruthIfNeeded(api_result, dataFile, start_ld,
cacheKey)
# Although we process the day's data in two batches, we should get the same result
self.compare_approx_result(ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data,
time_fuzz=60,
distance_fuzz=100)
def testResetToFuture(self):
"""
- Load data for both days
- Run pipelines
- Reset to a date after the two
- Verify that all is well
- Re-run pipelines and ensure that there are no errors
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"),
object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"),
object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Reset to a date well after the two days
reset_ts = arrow.get("2017-07-24").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# Data should be untouched because of early return
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
# Re-running the pipeline again should not affect anything
etc.runIntakePipeline(self.testUUID)
def testFeb22MultiSyncEndNotDetected(self):
# Re-run, but with multiple calls to sync data
# This tests the effect of online versus offline analysis and segmentation with potentially partial data
dataFile = "emission/tests/data/real_examples/iphone_2016-02-22"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22})
end_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22})
cacheKey = "diary/trips-2016-02-22"
ground_truth = json.load(open(dataFile + ".ground_truth"),
object_hook=bju.object_hook)
logging.info("Before loading, timeseries db size = %s" %
edb.get_timeseries_db().count())
all_entries = json.load(open(dataFile), object_hook=bju.object_hook)
# 18:01 because the transition was at 2016-02-22T18:00:09.623404-08:00, so right after
# 18:00
ts_1800 = arrow.get("2016-02-22T18:00:30-08:00").timestamp
logging.debug("ts_1800 = %s, converted back = %s" %
(ts_1800, arrow.get(ts_1800).to("America/Los_Angeles")))
before_1800_entries = [
e for e in all_entries
if ad.AttrDict(e).metadata.write_ts <= ts_1800
]
after_1800_entries = [
e for e in all_entries
if ad.AttrDict(e).metadata.write_ts > ts_1800
]
# Sync at 18:00 to capture all the points on the trip *to* the optometrist
# Skip the last few points to ensure that the trip end is skipped
import uuid
self.testUUID = uuid.uuid4()
self.entries = before_1800_entries[0:-2]
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Then sync after 18:00
self.entries = after_1800_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_approx_result(ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data,
time_fuzz=60,
distance_fuzz=100)
def testAug11(self):
# This is a more complex day. Tests:
# PR #352 (should not split trip to Oakland)
# PR #348 (trip from station to OAK DOT)
# PR #357 (trip to Radio Shack is complete and not truncated)
# PR #345 (no cleaned trips are skipped)
dataFile = "emission/tests/data/real_examples/shankari_2016-08-11"
ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 11})
cacheKey = "diary/trips-2016-08-11"
ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testAug11(self):
# This is a more complex day. Tests:
# PR #352 (should not split trip to Oakland)
# PR #348 (trip from station to OAK DOT)
# PR #357 (trip to Radio Shack is complete and not truncated)
# PR #345 (no cleaned trips are skipped)
dataFile = "emission/tests/data/real_examples/shankari_2016-08-11"
ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 11})
cacheKey = "diary/trips-2016-08-11"
with open(dataFile + ".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data)
def testJun21(self):
# This is a more complex day. Tests:
# PR #357 (spurious trip at 14:00 should be segmented and skipped)
# PR #358 (trip back from bella's house at 16:00)
dataFile = "emission/tests/data/real_examples/shankari_2016-06-21"
ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 21})
cacheKey = "diary/trips-2016-06-21"
with open(dataFile + ".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data)
def testJun20(self):
# This is a fairly straightforward day. Tests mainly:
# - ordering of trips
# - handling repeated location entries with different write timestamps
# We have two identical location points with ts = 1466436483.395 and write_ts = 1466436496.4, 1466436497.047
dataFile = "emission/tests/data/real_examples/shankari_2016-06-20"
ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 20})
cacheKey = "diary/trips-2016-06-20"
with open(dataFile + ".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data)
def testAug10(self):
# This is a more complex day. Tests:
# PR #302 (trip to optometrist)
# PR #352 (split optometrist trip)
dataFile = "emission/tests/data/real_examples/shankari_2016-08-10"
ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10})
cacheKey = "diary/trips-2016-08-10"
with open(dataFile+".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testJun20(self):
# This is a fairly straightforward day. Tests mainly:
# - ordering of trips
# - handling repeated location entries with different write timestamps
# We have two identical location points with ts = 1466436483.395 and write_ts = 1466436496.4, 1466436497.047
dataFile = "emission/tests/data/real_examples/shankari_2016-06-20"
ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 20})
cacheKey = "diary/trips-2016-06-20"
with open(dataFile+".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testAug10(self):
# This is a more complex day. Tests:
# PR #302 (trip to optometrist)
# PR #352 (split optometrist trip)
dataFile = "emission/tests/data/real_examples/shankari_2016-08-10"
ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10})
cacheKey = "diary/trips-2016-08-10"
ground_truth = json.load(open(dataFile + ".ground_truth"),
object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data)
def get_trips_for_day(user_uuid, day, force_refresh):
"""
The day argument here is a string such as 2015-10-01 or 2016-01-01. We will
parse this to a datetime, which we will use to query the data in the
timeseries. We could also cache the timeline views in a separate collection
and just look up from there. The challenge is to then decide when to
recompute a view - we can't use the standard technique that we use for the
other stages because we will have to recompute the timeline for the current
day multiple times, for example.
"""
# I was going to read from the user cache if it existed there, and recreate
# from scratch if it didn't. But that would involve adding a getDocument
# field to the usercache, which I had intentionally not added before this.
# The problem with adding a getDocument method is that then the usercache
# is no longer a cache - it is "storage" that is used internally. If we
# want to do that, we should really store it as a materialized view and not
# only in the usercache, which should be a cache of values stored elsewhere.
parsed_dt = dup.parse(day)
start_dt = ecwl.LocalDate({
'year': parsed_dt.year,
'month': parsed_dt.month,
'day': parsed_dt.day
})
end_dt = start_dt
return gfc.get_geojson_for_dt(user_uuid, start_dt, end_dt)
def testSunilShortTrips(self):
dataFile = "emission/tests/data/real_examples/sunil_2016-07-27"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 27})
cacheKey = "diary/trips-2016-07-27"
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.assertEqual(api_result, [])
def testAirTripHawaiiEnd(self):
dataFile = "emission/tests/data/real_examples/shankari_2016-08-04"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 4})
cacheKey = "diary/trips-2016-07-27"
ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testSunilShortTrips(self):
dataFile = "emission/tests/data/real_examples/sunil_2016-07-27"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 27})
cacheKey = "diary/trips-2016-07-27"
ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.assertEqual(api_result, [])
def testAug10MultiSyncEndDetected(self):
# Re-run, but with multiple calls to sync data
# This tests the effect of online versus offline analysis and segmentation with potentially partial data
dataFile = "emission/tests/data/real_examples/shankari_2016-08-10"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 9})
end_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 10})
cacheKey = "diary/trips-2016-08-10"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2016-08-910.ground_truth"),
object_hook=bju.object_hook)
logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().count())
all_entries = None
with open(dataFile) as secondfp:
all_entries = json.load(secondfp, object_hook = bju.object_hook)
ts_1030 = arrow.get("2016-08-10T10:30:00-07:00").timestamp
logging.debug("ts_1030 = %s, converted back = %s" % (ts_1030, arrow.get(ts_1030).to("America/Los_Angeles")))
before_1030_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1030]
after_1030_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1030]
# First load all data from the 9th. Otherwise, the missed trip is the first trip,
# and we don't set the last_ts_processed
# See the code around "logging.debug("len(segmentation_points) == 0, early return")"
etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2016-08-09")
# Sync at 10:30 to capture all the points on the trip *to* the optometrist
self.entries = before_1030_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Then sync after 10:30
self.entries = after_1030_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_approx_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testResetToFuture(self):
"""
- Load data for both days
- Run pipelines
- Reset to a date after the two
- Verify that all is well
- Re-run pipelines and ensure that there are no errors
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Reset to a date well after the two days
reset_ts = arrow.get("2017-07-24").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# Data should be untouched because of early return
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
# Re-running the pipeline again should not affect anything
etc.runIntakePipeline(self.testUUID)
def testOverriddenModeHack(self):
# Test for https://github.com/e-mission/e-mission-server/issues/457
dataFile = "emission/tests/data/real_examples/test_overriden_mode_hack.jul-31"
start_ld = ecwl.LocalDate({'year': 2017, 'month': 7, 'day': 31})
cacheKey = "diary/trips-2017-07-31"
ground_truth = json.load(open("emission/tests/data/real_examples/test_overriden_mode_hack.jul-31.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testJan16SpeedAssert(self):
# Test for https://github.com/e-mission/e-mission-server/issues/457
dataFile = "emission/tests/data/real_examples/another_speed_assertion_failure.jan-16"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 16})
cacheKey = "diary/trips-2016-01-16"
ground_truth = json.load(open("emission/tests/data/real_examples/another_speed_assertion_failure.jan-16.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testJumpSmoothingSectionStart(self):
dataFile = "emission/tests/data/real_examples/shankari_2016-independence_day_jump_bus_start"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 15})
cacheKey = "diary/trips-2016-08-15"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2016-independence_day.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testTsMismatch(self):
# Test for https://github.com/e-mission/e-mission-server/issues/457
dataFile = "emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 12, 'day': 12})
cacheKey = "diary/trips-2016-12-12"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testTsMismatch(self):
# Test for https://github.com/e-mission/e-mission-server/issues/457
dataFile = "emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 12, 'day': 12})
cacheKey = "diary/trips-2016-12-12"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_single_positional_indexer.dec-12.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testJumpSmoothingSectionStart(self):
dataFile = "emission/tests/data/real_examples/shankari_2016-independence_day_jump_bus_start"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 8, 'day': 15})
cacheKey = "diary/trips-2016-08-15"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2016-independence_day.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testOverriddenModeHack(self):
# Test for https://github.com/e-mission/e-mission-server/issues/457
dataFile = "emission/tests/data/real_examples/test_overriden_mode_hack.jul-31"
start_ld = ecwl.LocalDate({'year': 2017, 'month': 7, 'day': 31})
cacheKey = "diary/trips-2017-07-31"
ground_truth = json.load(open("emission/tests/data/real_examples/test_overriden_mode_hack.jul-31.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testJan16SpeedAssert(self):
# Test for https://github.com/e-mission/e-mission-server/issues/457
dataFile = "emission/tests/data/real_examples/another_speed_assertion_failure.jan-16"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 1, 'day': 16})
cacheKey = "diary/trips-2016-01-16"
ground_truth = json.load(open("emission/tests/data/real_examples/another_speed_assertion_failure.jan-16.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testResetToStart(self):
"""
- Load data for both days
- Run pipelines
- Verify that all is well
- Reset to start
- Verify that there is no analysis data
- Re-run pipelines
- Verify that all is well
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Check results: so far, so good
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
# Reset pipeline to start
epr.reset_user_to_start(self.testUUID, is_dry_run=False)
# Now there are no results
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.assertEqual(api_result, [])
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.assertEqual(api_result, [])
# Re-run the pipeline again
etc.runIntakePipeline(self.testUUID)
# Should be back to ground truth
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
def testResetToPast(self):
"""
- Load data for both days
- Run pipelines
- Verify that all is well
- Reset to a date before both
- Verify that analysis data for the both days is removed
- Re-run pipelines
- Verify that all is well
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Verify that all is well
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
# Reset to a date well before the two days
reset_ts = arrow.get("2015-07-24").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# Data should be completely deleted
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.assertEqual(api_result, [])
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.assertEqual(api_result, [])
# Re-running the pipeline again
etc.runIntakePipeline(self.testUUID)
# Should reconstruct everything
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
def testFeb22ShortTripsDistance(self):
dataFile = "emission/tests/data/real_examples/iphone_3_2016-02-22"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22})
end_ld = ecwl.LocalDate({'year': 2016, 'month': 2, 'day': 22})
cacheKey = "diary/trips-2016-02-22"
ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testIosJumpsAndUntrackedSquishing(self):
# Test for a2c0ee4e3ceafa822425ceef299dcdb01c9b32c9
dataFile = "emission/tests/data/real_examples/sunil_2016-07-20"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 20})
cacheKey = "diary/trips-2016-07-20"
ground_truth = json.load(open("emission/tests/data/real_examples/sunil_2016-07-20.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testIndexLengthChange(self):
# Test for 94f67b4848611fa01c4327a0fa0cab97c2247744
dataFile = "emission/tests/data/real_examples/shankari_2015-08-23"
start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 23})
cacheKey = "diary/trips-2015-08-23"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2015-08-23.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testIosJumpsAndUntrackedSquishing(self):
# Test for a2c0ee4e3ceafa822425ceef299dcdb01c9b32c9
dataFile = "emission/tests/data/real_examples/sunil_2016-07-20"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 20})
cacheKey = "diary/trips-2016-07-20"
ground_truth = json.load(open("emission/tests/data/real_examples/sunil_2016-07-20.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testIndexLengthChange(self):
# Test for 94f67b4848611fa01c4327a0fa0cab97c2247744
dataFile = "emission/tests/data/real_examples/shankari_2015-08-23"
start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 23})
cacheKey = "diary/trips-2015-08-23"
ground_truth = json.load(open("emission/tests/data/real_examples/shankari_2015-08-23.ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, start_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testOct07MultiSyncSpuriousEndDetected(self):
# Re-run, but with multiple calls to sync data
# This tests the effect of online versus offline analysis and segmentation with potentially partial data
dataFile = "emission/tests/data/real_examples/issue_436_assertion_error"
start_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 07})
end_ld = ecwl.LocalDate({'year': 2016, 'month': 10, 'day': 07})
cacheKey = "diary/trips-2016-10-07"
ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook)
logging.info("Before loading, timeseries db size = %s" % edb.get_timeseries_db().count())
all_entries = json.load(open(dataFile), object_hook = bju.object_hook)
# 18:01 because the transition was at 2016-02-22T18:00:09.623404-08:00, so right after
# 18:00
ts_1800 = arrow.get("2016-10-07T18:33:11-07:00").timestamp
logging.debug("ts_1800 = %s, converted back = %s" % (ts_1800, arrow.get(ts_1800).to("America/Los_Angeles")))
before_1800_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts <= ts_1800]
after_1800_entries = [e for e in all_entries if ad.AttrDict(e).metadata.write_ts > ts_1800]
# Sync at 18:00 to capture all the points on the trip *to* the optometrist
# Skip the last few points to ensure that the trip end is skipped
import uuid
self.testUUID = uuid.uuid4()
self.entries = before_1800_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Then sync after 18:00
self.entries = after_1800_entries
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_approx_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data, time_fuzz=60, distance_fuzz=100)
def testAug27TooMuchExtrapolation(self):
dataFile = "emission/tests/data/real_examples/shankari_2015-aug-27"
start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27})
end_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27})
cacheKey = "diary/trips-2015-08-27"
with open(dataFile+".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def standardMatchDataGroundTruth(self, dataFile, ld, cacheKey):
with open(dataFile+".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testAug27TooMuchExtrapolation(self):
dataFile = "emission/tests/data/real_examples/shankari_2015-aug-27"
start_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27})
end_ld = ecwl.LocalDate({'year': 2015, 'month': 8, 'day': 27})
cacheKey = "diary/trips-2015-08-27"
with open(dataFile + ".ground_truth") as gfp:
ground_truth = json.load(gfp, object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld, end_ld)
# Although we process the day's data in two batches, we should get the same result
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth).data)
def get_trips_for_day(user_uuid, day, force_refresh):
"""
The day argument here is a string such as 2015-10-01 or 2016-01-01. We will
parse this to a datetime, which we will use to query the data in the
timeseries. We could also cache the timeline views in a separate collection
and just look up from there. The challenge is to then decide when to
recompute a view - we can't use the standard technique that we use for the
other stages because we will have to recompute the timeline for the current
day multiple times, for example.
"""
# I was going to read from the user cache if it existed there, and recreate
# from scratch if it didn't. But that would involve adding a getDocument
# field to the usercache, which I had intentionally not added before this.
# The problem with adding a getDocument method is that then the usercache
# is no longer a cache - it is "storage" that is used internally. If we
# want to do that, we should really store it as a materialized view and not
# only in the usercache, which should be a cache of values stored elsewhere.
start_dt = dup.parse(day)
end_dt = start_dt + pydt.timedelta(days=1)
return gfc.get_geojson_for_dt(user_uuid, start_dt, end_dt)
def testJun21(self):
# This is a more complex day. Tests:
# PR #357 (spurious trip at 14:00 should be segmented and skipped)
# PR #358 (trip back from bella's house at 16:00)
dataFile = "emission/tests/data/real_examples/shankari_2016-06-21"
ld = ecwl.LocalDate({'year': 2016, 'month': 6, 'day': 21})
cacheKey = "diary/trips-2016-06-21"
ground_truth = json.load(open(dataFile+".ground_truth"), object_hook=bju.object_hook)
etc.setupRealExample(self, dataFile)
etc.runIntakePipeline(self.testUUID)
# runIntakePipeline does not run the common trips, habitica or store views to cache
# So let's manually store to the cache
# tc_query = estt.TimeComponentQuery("data.star_local_dt", ld, ld)
# enuah.UserCacheHandler.getUserCacheHandler(self.testUUID).storeTimelineToCache(tc_query)
# cached_result = edb.get_usercache_db().find_one({'user_id': self.testUUID,
# "metadata.key": cacheKey})
api_result = gfc.get_geojson_for_dt(self.testUUID, ld, ld)
# self.compare_result(cached_result, ground_truth)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth).data)
def testResetToTsInMiddleOfPlace(self):
"""
- Load data for both days
- Run pipelines
- Verify that all is well
- Reset to a date between the two
- Verify that analysis data for the first day is unchanged
- Verify that analysis data for the second day does not exist
- Re-run pipelines
- Verify that all is well
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Check results: so far, so good
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
# Reset pipeline to july 23.
# Note that this is actually 22nd 16:00 PDT, so this is partway
# through the 22nd
reset_ts = arrow.get("2016-07-23").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# First day is unchanged, except that the last place doesn't have
# exit data.
# TODO: Modify ground truth to capture this change
# Until then, we know that this will fail
# api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
# self.compare_result(ad.AttrDict({'result': api_result}).result,
# ad.AttrDict(ground_truth_1).data)
# Second day does not exist
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
logging.debug(json.dumps(api_result, indent=4, default=bju.default))
self.assertEqual(api_result, [])
# Re-run the pipeline again
etc.runIntakePipeline(self.testUUID)
# Should be back to ground truth
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
def testResetToTsInMiddleOfTrip(self):
"""
- Load data for both days
- Run pipelines
- Verify that all is well
- Reset to a date between the two
- Verify that analysis data for the first day is unchanged
- Verify that analysis data for the second day does not exist
- Re-run pipelines
- Verify that all is well
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1+".ground_truth"), object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2+".ground_truth"), object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook = bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Check results: so far, so good
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
# Reset pipeline to july 24.
# Note that this is actually 23nd 16:00 PDT
# This will reset in the middle of the untracked time, which is
# technically a trip, and will allow us to test the trip resetting
# code
reset_ts = arrow.get("2016-07-24").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# Second day does not exist
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
logging.debug(json.dumps(api_result, indent=4, default=bju.default))
self.assertEqual(api_result, [])
# Re-run the pipeline again
etc.runIntakePipeline(self.testUUID)
# Should be back to ground truth
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2, start_ld_2)
self.compare_result(ad.AttrDict({'result': api_result}).result,
ad.AttrDict(ground_truth_2).data)
import dateutil.parser as dup
import json
import datetime as pydt
import time
# Our imports
import emission.core.get_database as edb
import emission.net.usercache.abstract_usercache as enua
import emission.analysis.plotting.geojson.geojson_feature_converter as gfc
def get_trips_for_day(user_uuid, day)
"""
The day argument here is a string such as 2015-10-01 or 2016-01-01. We will
parse this to a datetime, which we will use to query the data in the
timeseries. We could also cache the timeline views in a separate collection
and just look up from there. The challenge is to then decide when to
recompute a view - we can't use the standard technique that we use for the
other stages because we will have to recompute the timeline for the current
day multiple times, for example.
"""
# I was going to read from the user cache if it existed there, and recreate
# from scratch if it didn't. But that would involve adding a getDocument
# field to the usercache, which I had intentionally not added before this.
# The problem with adding a getDocument method is that then the usercache
# is no longer a cache - it is "storage" that is used internally. If we
# want to do that, we should really store it as a materialized view and not
# only in the usercache, which should be a cache of values stored elsewhere.
start_dt = dup.parse(day)
end_dt = start_dt + pydt.timedelta(days=1)
return gfc.get_geojson_for_dt(user_uuid, start_dt, end_dt)
def testResetToTsInMiddleOfPlace(self):
"""
- Load data for both days
- Run pipelines
- Verify that all is well
- Reset to a date between the two
- Verify that analysis data for the first day is unchanged
- Verify that analysis data for the second day does not exist
- Re-run pipelines
- Verify that all is well
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"),
object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"),
object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Check results: so far, so good
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
# Reset pipeline to july 23.
# Note that this is actually 22nd 16:00 PDT, so this is partway
# through the 22nd
reset_ts = arrow.get("2016-07-23").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# First day is unchanged, except that the last place doesn't have
# exit data.
# TODO: Modify ground truth to capture this change
# Until then, we know that this will fail
# api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1, start_ld_1)
# self.compare_result(ad.AttrDict({'result': api_result}).result,
# ad.AttrDict(ground_truth_1).data)
# Second day does not exist
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
logging.debug(json.dumps(api_result, indent=4, default=bju.default))
self.assertEqual(api_result, [])
# Re-run the pipeline again
etc.runIntakePipeline(self.testUUID)
# Should be back to ground truth
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
def testResetToTsInMiddleOfTrip(self):
"""
- Load data for both days
- Run pipelines
- Verify that all is well
- Reset to a date between the two
- Verify that analysis data for the first day is unchanged
- Verify that analysis data for the second day does not exist
- Re-run pipelines
- Verify that all is well
"""
# Load all data
dataFile_1 = "emission/tests/data/real_examples/shankari_2016-07-22"
dataFile_2 = "emission/tests/data/real_examples/shankari_2016-07-25"
start_ld_1 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 22})
start_ld_2 = ecwl.LocalDate({'year': 2016, 'month': 7, 'day': 25})
cacheKey_1 = "diary/trips-2016-07-22"
cacheKey_2 = "diary/trips-2016-07-25"
ground_truth_1 = json.load(open(dataFile_1 + ".ground_truth"),
object_hook=bju.object_hook)
ground_truth_2 = json.load(open(dataFile_2 + ".ground_truth"),
object_hook=bju.object_hook)
# Run both pipelines
etc.setupRealExample(self, dataFile_1)
etc.runIntakePipeline(self.testUUID)
self.entries = json.load(open(dataFile_2), object_hook=bju.object_hook)
etc.setupRealExampleWithEntries(self)
etc.runIntakePipeline(self.testUUID)
# Check results: so far, so good
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
# Reset pipeline to july 24.
# Note that this is actually 23nd 16:00 PDT
# This will reset in the middle of the untracked time, which is
# technically a trip, and will allow us to test the trip resetting
# code
reset_ts = arrow.get("2016-07-24").timestamp
epr.reset_user_to_ts(self.testUUID, reset_ts, is_dry_run=False)
# Second day does not exist
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
logging.debug(json.dumps(api_result, indent=4, default=bju.default))
self.assertEqual(api_result, [])
# Re-run the pipeline again
etc.runIntakePipeline(self.testUUID)
# Should be back to ground truth
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_1,
start_ld_1)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_1).data)
api_result = gfc.get_geojson_for_dt(self.testUUID, start_ld_2,
start_ld_2)
self.compare_result(
ad.AttrDict({
'result': api_result
}).result,
ad.AttrDict(ground_truth_2).data)
