def testTripGeojson(self):
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
eaicl.filter_current_sections(self.testUUID)
tl = esdtl.get_raw_timeline(self.testUUID, 1440658800, 1440745200)
self.assertEquals(len(tl.trips), 9)
eaicr.clean_and_resample(self.testUUID)
eacimp.predict_mode(self.testUUID)
tl = esdtl.get_cleaned_timeline(self.testUUID, 1440658800, 1440745200)
tl.fill_start_end_places()
created_trips = tl.trips
self.assertEquals(len(created_trips), 9)
trip_geojson = gjfc.trip_to_geojson(created_trips[0], tl)
logging.debug("first trip_geojson = %s" % bju.dumps(trip_geojson, indent=4))
self.assertEquals(trip_geojson.type, "FeatureCollection")
self.assertEquals(trip_geojson.properties["feature_type"], "trip")
self.assertEquals(len(trip_geojson.features), 5)
day_geojson = gjfc.get_geojson_for_timeline(self.testUUID, tl)
self.assertEquals(len(day_geojson), 8)
self.assertEquals(day_geojson[-1].type, "FeatureCollection")
self.assertEquals(day_geojson[-1].properties["feature_type"], "trip")
self.assertEquals(len(day_geojson[-1].features), 5)
def testTripGeojson(self):
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
eaicl.filter_current_sections(self.testUUID)
tl = esdtl.get_raw_timeline(self.testUUID, 1440658800, 1440745200)
self.assertEqual(len(tl.trips), 9)
eaicr.clean_and_resample(self.testUUID)
eacimp.predict_mode(self.testUUID)
tl = esdtl.get_cleaned_timeline(self.testUUID, 1440658800, 1440745200)
tl.fill_start_end_places()
created_trips = tl.trips
self.assertEqual(len(created_trips), 9)
trip_geojson = gjfc.trip_to_geojson(created_trips[0], tl)
logging.debug("first trip_geojson = %s" %
bju.dumps(trip_geojson, indent=4))
self.assertEqual(trip_geojson.type, "FeatureCollection")
self.assertEqual(trip_geojson.properties["feature_type"], "trip")
self.assertEqual(len(trip_geojson.features), 5)
day_geojson = gjfc.get_geojson_for_timeline(self.testUUID, tl)
self.assertEqual(len(day_geojson), 8)
self.assertEqual(day_geojson[-1].type, "FeatureCollection")
self.assertEqual(day_geojson[-1].properties["feature_type"], "trip")
self.assertEqual(len(day_geojson[-1].features), 5)
def runIntakePipeline(uuid):
# Move these imports here so that we don't inadvertently load the modules,
# and any related config modules, before we want to
import emission.analysis.intake.cleaning.filter_accuracy as eaicf
import emission.storage.timeseries.format_hacks.move_filter_field as estfm
import emission.analysis.intake.segmentation.trip_segmentation as eaist
import emission.analysis.intake.segmentation.section_segmentation as eaiss
import emission.analysis.intake.cleaning.location_smoothing as eaicl
import emission.analysis.intake.cleaning.clean_and_resample as eaicr
import emission.analysis.classification.inference.mode.pipeline as eacimp
eaicf.filter_accuracy(uuid)
eaist.segment_current_trips(uuid)
eaiss.segment_current_sections(uuid)
eaicl.filter_current_sections(uuid)
eaicr.clean_and_resample(uuid)
eacimp.predict_mode(uuid)
def runIntakePipeline(uuid):
# Move these imports here so that we don't inadvertently load the modules,
# and any related config modules, before we want to
import emission.analysis.intake.cleaning.filter_accuracy as eaicf
import emission.storage.timeseries.format_hacks.move_filter_field as estfm
import emission.analysis.intake.segmentation.trip_segmentation as eaist
import emission.analysis.intake.segmentation.section_segmentation as eaiss
import emission.analysis.intake.cleaning.location_smoothing as eaicl
import emission.analysis.intake.cleaning.clean_and_resample as eaicr
import emission.analysis.classification.inference.mode.pipeline as eacimp
eaicf.filter_accuracy(uuid)
eaist.segment_current_trips(uuid)
eaiss.segment_current_sections(uuid)
eaicl.filter_current_sections(uuid)
eaicr.clean_and_resample(uuid)
eacimp.predict_mode(uuid)
def run_pipeline():
edb.pm_address = request.json['pm_address']
print(edb.pm_address)
# uuid is a filler and just needs to be consistent for each user.
# These can be removed but require refactoring all code locations
# that use the uuid.
uuid = request.json['uuid']
uh = euah.UserCacheHandler.getUserCacheHandler(uuid)
with ect.Timer() as uct:
logging.info("*" * 10 + "moving to long term" + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "moving to long term" + "*" * 10)
uh.moveToLongTerm()
esds.store_pipeline_time(uuid, ecwp.PipelineStages.USERCACHE.name,
time.time(), uct.elapsed)
# Hack until we delete these spurious entries
# https://github.com/e-mission/e-mission-server/issues/407#issuecomment-2484868
# Hack no longer works after the stats are in the timeseries because
# every user, even really old ones, have the pipeline run for them,
# which inserts pipeline_time stats.
# Let's strip out users who only have pipeline_time entries in the timeseries
# I wonder if this (distinct versus count) is the reason that the pipeline has
# become so much slower recently. Let's try to actually delete the
# spurious entries or at least mark them as obsolete and see if that helps.
print(edb.get_timeseries_db().find({"user_id": uuid}).distinct("metadata.key"))
if edb.get_timeseries_db().find({"user_id": uuid}).distinct("metadata.key") == ["stats/pipeline_time"]:
logging.debug("Found no entries for %s, skipping" % uuid)
return
with ect.Timer() as aft:
logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10)
eaicf.filter_accuracy(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.ACCURACY_FILTERING.name,
time.time(), aft.elapsed)
with ect.Timer() as tst:
logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10)
eaist.segment_current_trips(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.TRIP_SEGMENTATION.name,
time.time(), tst.elapsed)
with ect.Timer() as sst:
logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10)
eaiss.segment_current_sections(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.SECTION_SEGMENTATION.name,
time.time(), sst.elapsed)
with ect.Timer() as jst:
logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10)
eaicl.filter_current_sections(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.JUMP_SMOOTHING.name,
time.time(), jst.elapsed)
with ect.Timer() as crt:
logging.info("*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10)
eaicr.clean_and_resample(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.CLEAN_RESAMPLING.name,
time.time(), crt.elapsed)
with ect.Timer() as crt:
logging.info("*" * 10 + "UUID %s: inferring transportation mode" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: inferring transportation mode" % uuid + "*" * 10)
eacimp.predict_mode(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.MODE_INFERENCE.name,
time.time(), crt.elapsed)
with ect.Timer() as ogt:
logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10)
# use store data
uh.storeViewsToCache()
esds.store_pipeline_time(uuid, ecwp.PipelineStages.OUTPUT_GEN.name,
time.time(), ogt.elapsed)
def run_intake_pipeline_for_user(uuid):
uh = euah.UserCacheHandler.getUserCacheHandler(uuid)
with ect.Timer() as uct:
logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10)
uh.moveToLongTerm()
esds.store_pipeline_time(uuid, ecwp.PipelineStages.USERCACHE.name,
time.time(), uct.elapsed)
# Hack until we delete these spurious entries
# https://github.com/e-mission/e-mission-server/issues/407#issuecomment-2484868
# Hack no longer works after the stats are in the timeseries because
# every user, even really old ones, have the pipeline run for them,
# which inserts pipeline_time stats.
# Let's strip out users who only have pipeline_time entries in the timeseries
# I wonder if this (distinct versus count) is the reason that the pipeline has
# become so much slower recently. Let's try to actually delete the
# spurious entries or at least mark them as obsolete and see if that helps.
if edb.get_timeseries_db().find({"user_id": uuid}).distinct("metadata.key") == ["stats/pipeline_time"]:
logging.debug("Found no entries for %s, skipping" % uuid)
return
with ect.Timer() as aft:
logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10)
eaicf.filter_accuracy(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.ACCURACY_FILTERING.name,
time.time(), aft.elapsed)
with ect.Timer() as tst:
logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10)
eaist.segment_current_trips(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.TRIP_SEGMENTATION.name,
time.time(), tst.elapsed)
with ect.Timer() as sst:
logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10)
eaiss.segment_current_sections(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.SECTION_SEGMENTATION.name,
time.time(), sst.elapsed)
with ect.Timer() as jst:
logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10)
eaicl.filter_current_sections(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.JUMP_SMOOTHING.name,
time.time(), jst.elapsed)
with ect.Timer() as crt:
logging.info("*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10)
eaicr.clean_and_resample(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.CLEAN_RESAMPLING.name,
time.time(), crt.elapsed)
with ect.Timer() as crt:
logging.info("*" * 10 + "UUID %s: inferring transportation mode" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: inferring transportation mode" % uuid + "*" * 10)
eacimp.predict_mode(uuid)
esds.store_pipeline_time(uuid, ecwp.PipelineStages.MODE_INFERENCE.name,
time.time(), crt.elapsed)
with ect.Timer() as act:
logging.info("*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10)
autocheck.give_points_for_all_tasks(uuid)
esds.store_pipeline_time(uuid, "AUTOCHECK_POINTS",
time.time(), act.elapsed)
with ect.Timer() as ogt:
logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10)
print(str(arrow.now()) + "*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10)
uh.storeViewsToCache()
esds.store_pipeline_time(uuid, ecwp.PipelineStages.OUTPUT_GEN.name,
time.time(), ogt.elapsed)