def testE2E(self):
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
esdtmq.make_tour_model_from_raw_user_data(self.testUUID)
tm = esdtmq.get_tour_model(self.testUUID)
self.assertTrue(len(tm["common_trips"]) > 0)
self.assertTrue(len(tm["common_places"]) > 0)
def testNoData(self):
fake_user_id = "new_fake"
esdtmq.make_tour_model_from_raw_user_data(fake_user_id)
tm = esdtmq.get_tour_model(fake_user_id)
logging.debug("in testNoData, tour model = %s" % tm)
self.assertTrue(len(tm["common_places"]) == 0)
self.assertTrue(len(tm["common_trips"]) == 0)
def testE2E(self):
etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-27")
eaicf.filter_accuracy(self.testUUID)
estfm.move_all_filters_to_data()
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
esdtmq.make_tour_model_from_raw_user_data(self.testUUID)
tm = esdtmq.get_tour_model(self.testUUID)
self.assertTrue(len(tm["common_trips"]) > 0)
self.assertTrue(len(tm["common_places"]) > 0)
def testE2E(self):
etc.runIntakePipeline(self.testUUID)
esdtmq.make_tour_model_from_raw_user_data(self.testUUID)
tm = esdtmq.get_tour_model(self.testUUID)
self.assertTrue(len(tm["common_trips"]) > 0)
self.assertTrue(len(tm["common_places"]) > 0)
def setup(self):
def run_model_pipeline(process_number, uuid_list):
"""
Run the modeling stage of the pipeline.
"""
for uuid in uuid_list:
if uuid is None:
continue
try:
run_model_pipeline_for_user(uuid)
except Exception as e:
print "dang flabbit failed on error %s" % e
def run_mode_inference_pipeline_for_user(uuid):
MIP = ModeInferencePipeline() #I don't think its this simple
allConfirmedTripsQuery = ModeInferencePipeline.getSectionQueryWithGroundTruth({'$ne': ''})
#These are all the trips that have a confirmed mode. We will be training off of this.
#(MIP.modeList, MIP.confirmedSections) = MIP.loadTrainingDataStep(allConfirmedTripsQuery)
MIP.runPipeline()
def update_model_for_user(uuid, algorithm_id=1):
MIP = pipeline.ModelInferencePipeline()
MIP.runModelBuildingStage(uuid = uuid, algorithm_id= algorithm_id)
#user.model = MIP.getModel
def predict_for_user(uuid, timerange, model=None):
if model is None:
pass
#model = user.model
MIP = pipeline.ModelInferencePipeline()
MIP.runModeInferenceStage(uuid, timerange, model=model)
def run_mode_pipeline(process_number, uuid_list):
MIP = pipeline.ModelInferencePipeline()
for user_id in uuid_list:
try:
run_mode_pipeline_for_user(MIP, user_id)
except e:
logging.debug("Skipping user %s because of error" % user_id)
logging.debug("error %s" % e)
mark_mode_failed_for_user(user_id)
def run_mode_pipeline_for_user(MIP, uuid):
timerange = get_time_range_for_mode_inference(uuid)
MIP.runPipelineModelStage(uuid, timerange)
mark_mode_inference_done_for_user(uuid, MIP.getLastTimestamp})
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.DEBUG)
long_term_uuid_list = esta.TimeSeries.get_uuid_list()
logging.info("*" * 10 + "long term UUID list = %s" % long_term_uuid_list)
for uuid in long_term_uuid_list:
if uuid is None:
continue
logging.info("*" * 10 + "UUID %s: finding common trips" % uuid + "*" * 10)
esdtmq.make_tour_model_from_raw_user_data(uuid)
if __name__ == '__main__':
cache_uuid_list = enua.UserCache.get_uuid_list()
logging.info("cache UUID list = %s" % cache_uuid_list)
for uuid in cache_uuid_list:
logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10)
uh = euah.UserCacheHandler.getUserCacheHandler(uuid)
uh.moveToLongTerm()
long_term_uuid_list = esta.TimeSeries.get_uuid_list()
logging.info("*" * 10 + "long term UUID list = %s" % long_term_uuid_list)
for uuid in long_term_uuid_list:
logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10)
eaicf.filter_accuracy(uuid)
logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10)
eaist.segment_current_trips(uuid)
logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10)
eaiss.segment_current_sections(uuid)
logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10)
eaicl.filter_current_sections(uuid)
logging.info("*" * 10 + "UUID %s: finding common trips" % uuid + "*" * 10)
esdtmq.make_tour_model_from_raw_user_data(uuid)
logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10)
uh = euah.UserCacheHandler.getUserCacheHandler(uuid)
uh.storeViewsToCache()
from future import standard_library
standard_library.install_aliases()
from builtins import *
import sys
import logging
import emission.net.usercache.abstract_usercache_handler as euah
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.abstract_timeseries as esta
import emission.storage.decorations.tour_model_queries as esdtmq
import emission.analysis.intake.cleaning.filter_accuracy as eaicf
import emission.analysis.intake.segmentation.trip_segmentation as eaist
import emission.analysis.intake.segmentation.section_segmentation as eaiss
import emission.analysis.intake.cleaning.location_smoothing as eaicl
import emission.analysis.intake.cleaning.clean_and_resample as eaicr
import emission.net.ext_service.habitica.executor as autocheck
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.DEBUG)
long_term_uuid_list = esta.TimeSeries.get_uuid_list()
logging.info("*" * 10 + "long term UUID list = %s" % long_term_uuid_list)
for uuid in long_term_uuid_list:
if uuid is None:
continue
logging.info("*" * 10 + "UUID %s: finding common trips" % uuid +
"*" * 10)
esdtmq.make_tour_model_from_raw_user_data(uuid)
def testE2E(self):
etc.runIntakePipeline(self.testUUID)
esdtmq.make_tour_model_from_raw_user_data(self.testUUID)
tm = esdtmq.get_tour_model(self.testUUID)
self.assertTrue(len(tm["common_trips"]) > 0)
self.assertTrue(len(tm["common_places"]) > 0)
def testNoData(self):
fake_user_id = "new_fake"
esdtmq.make_tour_model_from_raw_user_data(fake_user_id)
tm = esdtmq.get_tour_model(fake_user_id)
self.assertTrue(len(tm["common_places"]) == 0)
self.assertTrue(len(tm["common_trips"]) == 0)
