def testLocalDateReadWrite(self): ts = esta.TimeSeries.get_time_series(self.testUUID) start_ts = arrow.now().timestamp ma_ts = 1460586729 local_dt = ecwl.LocalDate.get_local_date(ma_ts, "America/Los_Angeles") fmt_time = arrow.get(ma_ts).to("America/Los_Angeles").isoformat() ma = ecwm.Motionactivity({ "ts": 1460586729, "local_dt": local_dt, "fmt_time": fmt_time }) ma_entry = ecwe.Entry.create_entry(self.testUUID, "background/motion_activity", ma) ts.insert(ma_entry) ret_entry = ecwe.Entry( ts.get_entry_at_ts("background/motion_activity", "data.ts", 1460586729)) self.assertGreaterEqual(ret_entry.metadata.write_ts, start_ts) metadata_dt = arrow.get(ret_entry.metadata.write_ts).to( ret_entry.metadata.time_zone).datetime self.assertEqual(metadata_dt.hour, ret_entry.metadata.write_local_dt.hour) self.assertEqual(metadata_dt.minute, ret_entry.metadata.write_local_dt.minute) self.assertEqual(metadata_dt.weekday(), ret_entry.metadata.write_local_dt.weekday) self.assertEqual(ret_entry.data.local_dt.hour, 15) self.assertEqual(ret_entry.data.local_dt.month, 4) self.assertEqual(ret_entry.data.local_dt.weekday, 2) self.assertEqual(ret_entry.data.fmt_time, "2016-04-13T15:32:09-07:00")
def create_motion_entry_from_leg(leg, user_id): #TODO: Update with all possible/supported OTP modes. Also check for leg == None #Also, make sure this timestamp is correct timestamp = float(otp_time_to_ours(leg['startTime']).timestamp) print("*** Leg Start Time: %s" % arrow.get(timestamp).format()) opt_mode_to_motion_type = { 'BICYCLE': ecwm.MotionTypes.BICYCLING.value, 'CAR': ecwm.MotionTypes.IN_VEHICLE.value, 'RAIL': ecwm.MotionTypes.IN_VEHICLE.value, 'WALK': ecwm.MotionTypes.WALKING.value } new_motion_activity = ecwm.Motionactivity( ts=timestamp, type=opt_mode_to_motion_type[leg['mode']], #The following two lines were added to satisfy the formatters/android/motion_activity.py script zzaKM=opt_mode_to_motion_type[leg['mode']], zzaKN=100.0, fmt_time=arrow.get(timestamp).to('UTC').format(), local_dt=ecwld.LocalDate.get_local_date(timestamp, 'UTC'), confidence=100.0) entry = ecwe.Entry.create_entry(user_id, "background/motion_activity", new_motion_activity, create_id=True) #This field ('type') is required by the server when we push the entry to the user cache # so we add it here. entry['metadata']['type'] = 'sensor-data' #For some reason the android formater overwrites ts with metadata.write_ts. #so we need to set write_ts to ts to make sure they become the same. entry['metadata']['write_ts'] = timestamp entry['metadata']['platform'] = 'android' return entry
def is_filtered(self, curr_activity_doc): curr_activity = ecwm.Motionactivity(curr_activity_doc) logging.debug("curr activity = %s" % curr_activity) if (curr_activity.confidence > self.confidence_threshold and curr_activity.type not in self.ignore_modes_list): return True else: return False
def extend_activity_to_location(self, motion_change, location_point): new_mc = ecwm.Motionactivity({ 'type': motion_change.type, 'confidence': motion_change.confidence, 'ts': location_point.data.ts, 'local_dt': location_point.data.local_dt, 'fmt_time': location_point.data.fmt_time }) return new_mc
def segment_into_motion_changes(self, timeseries, time_query): """ Use the motion changes detected on the phone to detect sections (consecutive chains of points) that have a consistent motion. :param timeseries: the time series for this user :param time_query: the range to consider for segmentation :return: a list of tuples [(start_motion, end_motion)] that represent the ranges with a consistent motion. The gap between end_motion[n] and start_motion[n+1] represents the transition between the activities. We don't actually know the motion/activity in that range with any level of confidence. We need a policy on how to deal with them (combine with first, combine with second, split in the middle). This policy can be enforced when we map the activity changes to locations. """ motion_df = timeseries.get_data_df("background/motion_activity", time_query) filter_mask = motion_df.apply(self.is_filtered, axis=1) # Calling np.nonzero on the filter_mask even if it was related trips with zero sections # has not been a problem before this - the subsequent check on the # length of the filtered dataframe was sufficient. But now both Tom and # I have hit it (on 18th and 21st of Sept) so let's handle it proactively here. if filter_mask.shape == (0, 0): logging.info("Found filter_mask with shape (0,0), returning blank") return [] logging.debug("filtered points %s" % np.nonzero(filter_mask.to_numpy())) logging.debug("motion_df = %s" % motion_df.head()) filtered_df = motion_df[filter_mask] if len(filtered_df) == 0: # If there were no entries in the filtered_df, then there are no sections, # and we need to return an empty list. This check enforces that... return [] motion_change_list = [] prev_motion = None curr_start_motion = ecwm.Motionactivity(filtered_df.iloc[0]) # curr_section = ad.AttrDict({"user_id": trip.user_id, "loc_filter": trip.loc_filter, # "start_ts": trip.start_ts, "start_time": trip.start_time, # "activity": no_tilting_points_df.iloc[0].activity}) for idx, row in filtered_df.iterrows(): curr_motion = ecwm.Motionactivity(row) if curr_motion.type != curr_start_motion.type: # Because the curr_start_motion is initialized with the first # motion. So when idx == 0, the activities will be equal and # this is guaranteed to not be invoked assert (idx > 0) logging.debug( "At %s, found new activity %s compared to current %s - creating new section with start_time %s" % (curr_motion.fmt_time, curr_motion.type, curr_start_motion.type, curr_motion.fmt_time)) # complete this section motion_change_list.append((curr_start_motion, curr_motion)) curr_start_motion = curr_motion else: logging.debug( "At %s, retained existing activity %s because of no change" % (curr_motion.fmt_time, curr_motion.type)) prev_motion = curr_motion logging.info("Detected trip end! Ending section at %s" % curr_motion.fmt_time) motion_change_list.append((curr_start_motion, curr_motion)) # Go from activities to # Merge short sections # Sometimes, the sections flip-flop around return motion_change_list
def segment_into_motion_changes(self, timeseries, time_query): """ Use the motion changes detected on the phone to detect sections (consecutive chains of points) that have a consistent motion. :param timeseries: the time series for this user :param time_query: the range to consider for segmentation :return: a list of tuples [(start_motion, end_motion)] that represent the ranges with a consistent motion. The gap between end_motion[n] and start_motion[n+1] represents the transition between the activities. We don't actually know the motion/activity in that range with any level of confidence. We need a policy on how to deal with them (combine with first, combine with second, split in the middle). This policy can be enforced when we map the activity changes to locations. """ motion_df = timeseries.get_data_df("background/motion_activity", time_query) filter_mask = motion_df.apply(self.is_filtered, axis=1) # Calling np.nonzero on the filter_mask even if it was related trips with zero sections # has not been a problem before this - the subsequent check on the # length of the filtered dataframe was sufficient. But now both Tom and # I have hit it (on 18th and 21st of Sept) so let's handle it proactively here. if filter_mask.shape == (0, 0): logging.info("Found filter_mask with shape (0,0), returning blank") return [] # Replace RUNNING with WALKING so that we don't get extra, incorrect sections # https://github.com/e-mission/e-mission-server/issues/577#issuecomment-379496118 motion_df["type"].replace([8], 7, inplace=True) logging.debug("filtered points %s" % np.nonzero(filter_mask)) logging.debug("motion_df = %s" % motion_df.head()) filtered_df = motion_df[filter_mask] filtered_df.reset_index(inplace=True) if len(filtered_df) == 0: # If there were no entries in the filtered_df, then there are no sections, # and we need to return an empty list. This check enforces that... return [] motion_change_list = [] prev_motion = None curr_start_motion = ecwm.Motionactivity(filtered_df.iloc[0]) # curr_section = ad.AttrDict({"user_id": trip.user_id, "loc_filter": trip.loc_filter, # "start_ts": trip.start_ts, "start_time": trip.start_time, # "activity": no_tilting_points_df.iloc[0].activity}) for idx, row in filtered_df.iterrows(): curr_motion = ecwm.Motionactivity(row) curr_motion.update({"idx": idx}) # Since the start motion is set upstream makes sure to set an idx # for it too if curr_motion.ts == curr_start_motion.ts: curr_start_motion.update({"idx": idx}) if curr_motion.type != curr_start_motion.type: # Because the curr_start_motion is initialized with the first # motion. So when idx == 0, the activities will be equal and # this is guaranteed to not be invoked assert (idx > 0) logging.debug( "At idx %d, time %s, found new activity %s compared to current %s" % (idx, curr_motion.fmt_time, curr_motion.type, curr_start_motion.type)) curr_end_motion = get_curr_end_motion(prev_motion, curr_motion) logging.debug( "creating new section for %s at %s -> %s with start_time %s -> %s" % (curr_start_motion.type, curr_start_motion["idx"], curr_end_motion["idx"], curr_start_motion.fmt_time, curr_end_motion.fmt_time)) # complete this section motion_change_list.append((curr_start_motion, curr_end_motion)) curr_start_motion = curr_end_motion else: logging.debug( "At %s, retained existing activity %s because of no change" % (curr_motion.fmt_time, curr_motion.type)) prev_motion = curr_motion logging.info("Detected trip end! Ending section at %s" % curr_motion.fmt_time) motion_change_list.append((curr_start_motion, curr_motion)) smoothed_motion_list = ffd.FlipFlopDetection( motion_change_list, self).merge_flip_flop_sections() return smoothed_motion_list