def testE2E(self): eaist.segment_current_trips(self.testUUID) eaiss.segment_current_sections(self.testUUID) esdtmq.make_tour_model_from_raw_user_data(self.testUUID) tm = esdtmq.get_tour_model(self.testUUID) self.assertTrue(len(tm["common_trips"]) > 0) self.assertTrue(len(tm["common_places"]) > 0)
def testNoData(self): fake_user_id = "new_fake" esdtmq.make_tour_model_from_raw_user_data(fake_user_id) tm = esdtmq.get_tour_model(fake_user_id) logging.debug("in testNoData, tour model = %s" % tm) self.assertTrue(len(tm["common_places"]) == 0) self.assertTrue(len(tm["common_trips"]) == 0)
def testE2E(self): etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-27") eaicf.filter_accuracy(self.testUUID) estfm.move_all_filters_to_data() eaist.segment_current_trips(self.testUUID) eaiss.segment_current_sections(self.testUUID) esdtmq.make_tour_model_from_raw_user_data(self.testUUID) tm = esdtmq.get_tour_model(self.testUUID) self.assertTrue(len(tm["common_trips"]) > 0) self.assertTrue(len(tm["common_places"]) > 0)
def testE2E(self): etc.runIntakePipeline(self.testUUID) esdtmq.make_tour_model_from_raw_user_data(self.testUUID) tm = esdtmq.get_tour_model(self.testUUID) self.assertTrue(len(tm["common_trips"]) > 0) self.assertTrue(len(tm["common_places"]) > 0)
def setup(self): def run_model_pipeline(process_number, uuid_list): """ Run the modeling stage of the pipeline. """ for uuid in uuid_list: if uuid is None: continue try: run_model_pipeline_for_user(uuid) except Exception as e: print "dang flabbit failed on error %s" % e def run_mode_inference_pipeline_for_user(uuid): MIP = ModeInferencePipeline() #I don't think its this simple allConfirmedTripsQuery = ModeInferencePipeline.getSectionQueryWithGroundTruth({'$ne': ''}) #These are all the trips that have a confirmed mode. We will be training off of this. #(MIP.modeList, MIP.confirmedSections) = MIP.loadTrainingDataStep(allConfirmedTripsQuery) MIP.runPipeline() def update_model_for_user(uuid, algorithm_id=1): MIP = pipeline.ModelInferencePipeline() MIP.runModelBuildingStage(uuid = uuid, algorithm_id= algorithm_id) #user.model = MIP.getModel def predict_for_user(uuid, timerange, model=None): if model is None: pass #model = user.model MIP = pipeline.ModelInferencePipeline() MIP.runModeInferenceStage(uuid, timerange, model=model) def run_mode_pipeline(process_number, uuid_list): MIP = pipeline.ModelInferencePipeline() for user_id in uuid_list: try: run_mode_pipeline_for_user(MIP, user_id) except e: logging.debug("Skipping user %s because of error" % user_id) logging.debug("error %s" % e) mark_mode_failed_for_user(user_id) def run_mode_pipeline_for_user(MIP, uuid): timerange = get_time_range_for_mode_inference(uuid) MIP.runPipelineModelStage(uuid, timerange) mark_mode_inference_done_for_user(uuid, MIP.getLastTimestamp}) if __name__ == '__main__': logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG) long_term_uuid_list = esta.TimeSeries.get_uuid_list() logging.info("*" * 10 + "long term UUID list = %s" % long_term_uuid_list) for uuid in long_term_uuid_list: if uuid is None: continue logging.info("*" * 10 + "UUID %s: finding common trips" % uuid + "*" * 10) esdtmq.make_tour_model_from_raw_user_data(uuid)
if __name__ == '__main__': cache_uuid_list = enua.UserCache.get_uuid_list() logging.info("cache UUID list = %s" % cache_uuid_list) for uuid in cache_uuid_list: logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) uh = euah.UserCacheHandler.getUserCacheHandler(uuid) uh.moveToLongTerm() long_term_uuid_list = esta.TimeSeries.get_uuid_list() logging.info("*" * 10 + "long term UUID list = %s" % long_term_uuid_list) for uuid in long_term_uuid_list: logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) eaicf.filter_accuracy(uuid) logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) eaist.segment_current_trips(uuid) logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) eaiss.segment_current_sections(uuid) logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) eaicl.filter_current_sections(uuid) logging.info("*" * 10 + "UUID %s: finding common trips" % uuid + "*" * 10) esdtmq.make_tour_model_from_raw_user_data(uuid) logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) uh = euah.UserCacheHandler.getUserCacheHandler(uuid) uh.storeViewsToCache()
from future import standard_library standard_library.install_aliases() from builtins import * import sys import logging import emission.net.usercache.abstract_usercache_handler as euah import emission.net.usercache.abstract_usercache as enua import emission.storage.timeseries.abstract_timeseries as esta import emission.storage.decorations.tour_model_queries as esdtmq import emission.analysis.intake.cleaning.filter_accuracy as eaicf import emission.analysis.intake.segmentation.trip_segmentation as eaist import emission.analysis.intake.segmentation.section_segmentation as eaiss import emission.analysis.intake.cleaning.location_smoothing as eaicl import emission.analysis.intake.cleaning.clean_and_resample as eaicr import emission.net.ext_service.habitica.executor as autocheck if __name__ == '__main__': logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG) long_term_uuid_list = esta.TimeSeries.get_uuid_list() logging.info("*" * 10 + "long term UUID list = %s" % long_term_uuid_list) for uuid in long_term_uuid_list: if uuid is None: continue logging.info("*" * 10 + "UUID %s: finding common trips" % uuid + "*" * 10) esdtmq.make_tour_model_from_raw_user_data(uuid)
def testNoData(self): fake_user_id = "new_fake" esdtmq.make_tour_model_from_raw_user_data(fake_user_id) tm = esdtmq.get_tour_model(fake_user_id) self.assertTrue(len(tm["common_places"]) == 0) self.assertTrue(len(tm["common_trips"]) == 0)