def testResetActiveTransportation(self): self.testAutomaticRewardActiveTransportation() #Get user data before resetting user_before = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_before['bike_count']), 1500) habits_before = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_before = [habit['history'] for habit in habits_before['data'] if habit['text'] == self.new_task_text] #Reset reset_ts = arrow.Arrow(2016,5,3,9).timestamp autocheck.reset_all_tasks_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Check timestamp user_after = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_after['last_timestamp']), reset_ts) # Re-score points # This should give points for the second and third sections # So I expect to see an additional distance of 2.5 + 3.5 km = 6km autocheck.give_points_for_all_tasks(self.testUUID) #Get user data after scoring and check results # We already had bike_count = 1500, and this is a round number, so it # should continue to be 1500 user_after = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_after['bike_count']), 0) # and we should have 6 points more? habits_after = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_after = [habit['history'] for habit in habits_after['data'] if habit['text'] == self.new_task_text] logging.debug("bike_pts_after = %s" % (len(bike_pts_after[0]) - len(bike_pts_before[0]))) self.assertTrue(len(bike_pts_after[0]) - len(bike_pts_before[0]) == 3)
def testResetActiveTransportation(self): self.testAutomaticRewardActiveTransportation() #Get user data before resetting user_before = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_before['bike_count']), 1500) habits_before = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_before = [ habit['history'] for habit in habits_before['data'] if habit['text'] == self.new_task_text ] #Reset reset_ts = arrow.Arrow(2016, 5, 3, 9).timestamp autocheck.reset_all_tasks_to_ts(self.testUUID, reset_ts, is_dry_run=False) # Check timestamp user_after = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_after['last_timestamp']), reset_ts) # Re-score points # This should give points for the second and third sections # So I expect to see an additional distance of 2.5 + 3.5 km = 6km autocheck.give_points_for_all_tasks(self.testUUID) #Get user data after scoring and check results # We already had bike_count = 1500, and this is a round number, so it # should continue to be 1500 user_after = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_after['bike_count']), 0) # and we should have 6 points more? habits_after = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_after = [ habit['history'] for habit in habits_after['data'] if habit['text'] == self.new_task_text ] logging.debug("bike_pts_after = %s" % (len(bike_pts_after[0]) - len(bike_pts_before[0]))) self.assertTrue(len(bike_pts_after[0]) - len(bike_pts_before[0]) == 3)
def testAutomaticRewardActiveTransportation(self): # Create a task that we can retrieve later self.new_task_text = randomGen() new_habit = { 'type': "habit", 'text': self.new_task_text, 'notes': 'AUTOCHECK: {"mapper": "active_distance",' '"args": {"walk_scale": 1000, "bike_scale": 3000}}' } habit_id = proxy.create_habit(self.testUUID, new_habit) self.dummy_task = enehat.Task() self.dummy_task.task_id = habit_id logging.debug("in testAutomaticRewardActiveTransportation," "the new habit id is = %s and task is %s" % (habit_id, self.dummy_task)) #Create test data -- code copied from TestTimeGrouping key = (2016, 5, 3) test_section_list = [] # # Since PST is UTC-7, all of these will be in the same UTC day # 13:00, 17:00, 21:00 # so we expect the local date and UTC bins to be the same test_section_list.append( self._createTestSection( arrow.Arrow(2016, 5, 3, 6, tzinfo=tz.gettz(PST)), PST)) test_section_list.append( self._createTestSection( arrow.Arrow(2016, 5, 3, 10, tzinfo=tz.gettz(PST)), PST)) test_section_list.append( self._createTestSection( arrow.Arrow(2016, 5, 3, 14, tzinfo=tz.gettz(PST)), PST)) self._fillModeDistanceDuration(test_section_list) #logging.debug("durations = %s" % [s.data.duration for s in test_section_list]) summary_ts = earmt.group_by_timestamp( self.testUUID, arrow.Arrow(2016, 5, 1).timestamp, arrow.Arrow(2016, 6, 1).timestamp, None, [earmts.get_distance]) logging.debug( "in testAutomaticRewardActiveTransportation, result = %s" % summary_ts) #Get user data before scoring user_before = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertIsNone(user_before) # Needed to work, otherwise sections from may won't show up in the query! modification = { "last_timestamp": arrow.Arrow(2016, 5, 1).timestamp, "bike_count": 0, "walk_count": 0 } autocheck.save_task_state(self.testUUID, self.dummy_task, modification) user_before = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_before['bike_count']), 0) habits_before = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_before = [ habit['history'] for habit in habits_before['data'] if habit['text'] == self.new_task_text ] #Score points autocheck.give_points_for_all_tasks(self.testUUID) #Get user data after scoring and check results user_after = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_after['bike_count']), 1500) habits_after = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_after = [ habit['history'] for habit in habits_after['data'] if habit['text'] == self.new_task_text ] self.assertTrue(len(bike_pts_after[0]) - len(bike_pts_before[0]) == 2)
def testAutomaticRewardActiveTransportation(self): # Create a task that we can retrieve later self.new_task_text = randomGen() new_habit = {'type': "habit", 'text': self.new_task_text, 'notes': 'AUTOCHECK: {"mapper": "active_distance",' '"args": {"walk_scale": 1000, "bike_scale": 3000}}'} habit_id = proxy.create_habit(self.testUUID, new_habit) self.dummy_task = enehat.Task() self.dummy_task.task_id = habit_id logging.debug("in testAutomaticRewardActiveTransportation," "the new habit id is = %s and task is %s" % (habit_id, self.dummy_task)) #Create test data -- code copied from TestTimeGrouping key = (2016, 5, 3) test_section_list = [] # # Since PST is UTC-7, all of these will be in the same UTC day # 13:00, 17:00, 21:00 # so we expect the local date and UTC bins to be the same test_section_list.append( self._createTestSection(arrow.Arrow(2016,5,3,6, tzinfo=tz.gettz(PST)), PST)) test_section_list.append( self._createTestSection(arrow.Arrow(2016,5,3,10, tzinfo=tz.gettz(PST)), PST)) test_section_list.append( self._createTestSection(arrow.Arrow(2016,5,3,14, tzinfo=tz.gettz(PST)), PST)) self._fillModeDistanceDuration(test_section_list) #logging.debug("durations = %s" % [s.data.duration for s in test_section_list]) summary_ts = earmt.group_by_timestamp(self.testUUID, arrow.Arrow(2016,5,1).timestamp, arrow.Arrow(2016,6,1).timestamp, None, [earmts.get_distance]) logging.debug("in testAutomaticRewardActiveTransportation, result = %s" % summary_ts) #Get user data before scoring user_before = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertIsNone(user_before) # Needed to work, otherwise sections from may won't show up in the query! modification = {"last_timestamp": arrow.Arrow(2016,5,1).timestamp, "bike_count": 0, "walk_count":0} autocheck.save_task_state(self.testUUID, self.dummy_task, modification) user_before = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_before['bike_count']), 0) habits_before = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_before = [habit['history'] for habit in habits_before['data'] if habit['text'] == self.new_task_text] #Score points autocheck.give_points_for_all_tasks(self.testUUID) #Get user data after scoring and check results user_after = autocheck.get_task_state(self.testUUID, self.dummy_task) self.assertEqual(int(user_after['bike_count']),1500) habits_after = proxy.habiticaProxy(self.testUUID, 'GET', "/api/v3/tasks/user?type=habits", None).json() bike_pts_after = [habit['history'] for habit in habits_after['data'] if habit['text'] == self.new_task_text] self.assertTrue(len(bike_pts_after[0]) - len(bike_pts_before[0]) == 2)
def run_intake_pipeline_for_user(uuid): uh = euah.UserCacheHandler.getUserCacheHandler(uuid) with ect.Timer() as uct: logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) uh.moveToLongTerm() esds.store_pipeline_time(uuid, ecwp.PipelineStages.USERCACHE.name, time.time(), uct.elapsed) # Hack until we delete these spurious entries # https://github.com/e-mission/e-mission-server/issues/407#issuecomment-2484868 if edb.get_timeseries_db().find({"user_id": uuid}).count() == 0: logging.debug("Found no entries for %s, skipping" % uuid) return with ect.Timer() as aft: logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) eaicf.filter_accuracy(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.ACCURACY_FILTERING.name, time.time(), aft.elapsed) with ect.Timer() as tst: logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) eaist.segment_current_trips(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.TRIP_SEGMENTATION.name, time.time(), tst.elapsed) with ect.Timer() as sst: logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) eaiss.segment_current_sections(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.SECTION_SEGMENTATION.name, time.time(), sst.elapsed) with ect.Timer() as jst: logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) eaicl.filter_current_sections(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.JUMP_SMOOTHING.name, time.time(), jst.elapsed) with ect.Timer() as crt: logging.info("*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) eaicr.clean_and_resample(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.CLEAN_RESAMPLING.name, time.time(), crt.elapsed) with ect.Timer() as act: logging.info( "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) autocheck.give_points_for_all_tasks(uuid) esds.store_pipeline_time(uuid, "AUTOCHECK_POINTS", time.time(), act.elapsed) with ect.Timer() as ogt: logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) uh.storeViewsToCache() esds.store_pipeline_time(uuid, ecwp.PipelineStages.OUTPUT_GEN.name, time.time(), ogt.elapsed)
def run_intake_pipeline_for_user(uuid): uh = euah.UserCacheHandler.getUserCacheHandler(uuid) with ect.Timer() as uct: logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) uh.moveToLongTerm() esds.store_pipeline_time(uuid, ecwp.PipelineStages.USERCACHE.name, time.time(), uct.elapsed) # Hack until we delete these spurious entries # https://github.com/e-mission/e-mission-server/issues/407#issuecomment-2484868 # Hack no longer works after the stats are in the timeseries because # every user, even really old ones, have the pipeline run for them, # which inserts pipeline_time stats. # Let's strip out users who only have pipeline_time entries in the timeseries # I wonder if this (distinct versus count) is the reason that the pipeline has # become so much slower recently. Let's try to actually delete the # spurious entries or at least mark them as obsolete and see if that helps. if edb.get_timeseries_db().find({ "user_id": uuid }).distinct("metadata.key") == ["stats/pipeline_time"]: logging.debug("Found no entries for %s, skipping" % uuid) return with ect.Timer() as aft: logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) eaicf.filter_accuracy(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.ACCURACY_FILTERING.name, time.time(), aft.elapsed) with ect.Timer() as tst: logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) eaist.segment_current_trips(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.TRIP_SEGMENTATION.name, time.time(), tst.elapsed) with ect.Timer() as sst: logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) eaiss.segment_current_sections(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.SECTION_SEGMENTATION.name, time.time(), sst.elapsed) with ect.Timer() as jst: logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) eaicl.filter_current_sections(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.JUMP_SMOOTHING.name, time.time(), jst.elapsed) with ect.Timer() as crt: logging.info("*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) eaicr.clean_and_resample(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.CLEAN_RESAMPLING.name, time.time(), crt.elapsed) with ect.Timer() as crt: logging.info("*" * 10 + "UUID %s: inferring transportation mode" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: inferring transportation mode" % uuid + "*" * 10) eacimp.predict_mode(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.MODE_INFERENCE.name, time.time(), crt.elapsed) with ect.Timer() as act: logging.info( "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) autocheck.give_points_for_all_tasks(uuid) esds.store_pipeline_time(uuid, "AUTOCHECK_POINTS", time.time(), act.elapsed) with ect.Timer() as ogt: logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) uh.storeViewsToCache() esds.store_pipeline_time(uuid, ecwp.PipelineStages.OUTPUT_GEN.name, time.time(), ogt.elapsed)
def run_intake_pipeline_for_user(uuid): uh = euah.UserCacheHandler.getUserCacheHandler(uuid) with ect.Timer() as uct: logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) uh.moveToLongTerm() esds.store_pipeline_time(uuid, ecwp.PipelineStages.USERCACHE.name, time.time(), uct.elapsed) # Hack until we delete these spurious entries # https://github.com/e-mission/e-mission-server/issues/407#issuecomment-2484868 # Hack no longer works after the stats are in the timeseries because # every user, even really old ones, have the pipeline run for them, # which inserts pipeline_time stats. # Let's strip out users who only have pipeline_time entries in the timeseries # I wonder if this (distinct versus count) is the reason that the pipeline has # become so much slower recently. Let's try to actually delete the # spurious entries or at least mark them as obsolete and see if that helps. if edb.get_timeseries_db().find({"user_id": uuid}).distinct("metadata.key") == ["stats/pipeline_time"]: logging.debug("Found no entries for %s, skipping" % uuid) return with ect.Timer() as aft: logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) eaicf.filter_accuracy(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.ACCURACY_FILTERING.name, time.time(), aft.elapsed) with ect.Timer() as tst: logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) eaist.segment_current_trips(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.TRIP_SEGMENTATION.name, time.time(), tst.elapsed) with ect.Timer() as sst: logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) eaiss.segment_current_sections(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.SECTION_SEGMENTATION.name, time.time(), sst.elapsed) with ect.Timer() as jst: logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) eaicl.filter_current_sections(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.JUMP_SMOOTHING.name, time.time(), jst.elapsed) with ect.Timer() as crt: logging.info("*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) eaicr.clean_and_resample(uuid) esds.store_pipeline_time(uuid, ecwp.PipelineStages.CLEAN_RESAMPLING.name, time.time(), crt.elapsed) with ect.Timer() as act: logging.info("*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) autocheck.give_points_for_all_tasks(uuid) esds.store_pipeline_time(uuid, "AUTOCHECK_POINTS", time.time(), act.elapsed) with ect.Timer() as ogt: logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) print(str(arrow.now()) + "*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) uh.storeViewsToCache() esds.store_pipeline_time(uuid, ecwp.PipelineStages.OUTPUT_GEN.name, time.time(), ogt.elapsed)
def run_intake_pipeline_for_user(uuid): uh = euah.UserCacheHandler.getUserCacheHandler(uuid) logging.info("*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: moving to long term" % uuid + "*" * 10) uh.moveToLongTerm() # Hack until we delete these spurious entries # https://github.com/e-mission/e-mission-server/issues/407#issuecomment-2484868 if edb.get_timeseries_db().find({"user_id": uuid}).count() == 0: logging.debug("Found no entries for %s, skipping" % uuid) return logging.info("*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: filter accuracy if needed" % uuid + "*" * 10) eaicf.filter_accuracy(uuid) logging.info("*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into trips" % uuid + "*" * 10) eaist.segment_current_trips(uuid) logging.info("*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: segmenting into sections" % uuid + "*" * 10) eaiss.segment_current_sections(uuid) logging.info("*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: smoothing sections" % uuid + "*" * 10) eaicl.filter_current_sections(uuid) logging.info("*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: cleaning and resampling timeline" % uuid + "*" * 10) eaicr.clean_and_resample(uuid) logging.info("*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: checking active mode trips to autocheck habits" % uuid + "*" * 10) autocheck.give_points_for_all_tasks(uuid) logging.info("*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) print( str(arrow.now()) + "*" * 10 + "UUID %s: storing views to cache" % uuid + "*" * 10) uh.storeViewsToCache()