def testLocalDateReadWrite(self):
ts = esta.TimeSeries.get_time_series(self.testUUID)
start_ts = arrow.now().timestamp
ma_ts = 1460586729
local_dt = ecwl.LocalDate.get_local_date(ma_ts, "America/Los_Angeles")
fmt_time = arrow.get(ma_ts).to("America/Los_Angeles").isoformat()
ma = ecwm.Motionactivity({
"ts": 1460586729,
"local_dt": local_dt,
"fmt_time": fmt_time
})
ma_entry = ecwe.Entry.create_entry(self.testUUID,
"background/motion_activity", ma)
ts.insert(ma_entry)
ret_entry = ecwe.Entry(
ts.get_entry_at_ts("background/motion_activity", "data.ts",
1460586729))
self.assertGreaterEqual(ret_entry.metadata.write_ts, start_ts)
metadata_dt = arrow.get(ret_entry.metadata.write_ts).to(
ret_entry.metadata.time_zone).datetime
self.assertEqual(metadata_dt.hour,
ret_entry.metadata.write_local_dt.hour)
self.assertEqual(metadata_dt.minute,
ret_entry.metadata.write_local_dt.minute)
self.assertEqual(metadata_dt.weekday(),
ret_entry.metadata.write_local_dt.weekday)
self.assertEqual(ret_entry.data.local_dt.hour, 15)
self.assertEqual(ret_entry.data.local_dt.month, 4)
self.assertEqual(ret_entry.data.local_dt.weekday, 2)
self.assertEqual(ret_entry.data.fmt_time, "2016-04-13T15:32:09-07:00")
def create_motion_entry_from_leg(leg, user_id):
#TODO: Update with all possible/supported OTP modes. Also check for leg == None
#Also, make sure this timestamp is correct
timestamp = float(otp_time_to_ours(leg['startTime']).timestamp)
print("*** Leg Start Time: %s" % arrow.get(timestamp).format())
opt_mode_to_motion_type = {
'BICYCLE': ecwm.MotionTypes.BICYCLING.value,
'CAR': ecwm.MotionTypes.IN_VEHICLE.value,
'RAIL': ecwm.MotionTypes.IN_VEHICLE.value,
'WALK': ecwm.MotionTypes.WALKING.value
}
new_motion_activity = ecwm.Motionactivity(
ts=timestamp,
type=opt_mode_to_motion_type[leg['mode']],
#The following two lines were added to satisfy the formatters/android/motion_activity.py script
zzaKM=opt_mode_to_motion_type[leg['mode']],
zzaKN=100.0,
fmt_time=arrow.get(timestamp).to('UTC').format(),
local_dt=ecwld.LocalDate.get_local_date(timestamp, 'UTC'),
confidence=100.0)
entry = ecwe.Entry.create_entry(user_id,
"background/motion_activity",
new_motion_activity,
create_id=True)
#This field ('type') is required by the server when we push the entry to the user cache
# so we add it here.
entry['metadata']['type'] = 'sensor-data'
#For some reason the android formater overwrites ts with metadata.write_ts.
#so we need to set write_ts to ts to make sure they become the same.
entry['metadata']['write_ts'] = timestamp
entry['metadata']['platform'] = 'android'
return entry
def is_filtered(self, curr_activity_doc):
curr_activity = ecwm.Motionactivity(curr_activity_doc)
logging.debug("curr activity = %s" % curr_activity)
if (curr_activity.confidence > self.confidence_threshold
and curr_activity.type not in self.ignore_modes_list):
return True
else:
return False
def extend_activity_to_location(self, motion_change, location_point):
new_mc = ecwm.Motionactivity({
'type': motion_change.type,
'confidence': motion_change.confidence,
'ts': location_point.data.ts,
'local_dt': location_point.data.local_dt,
'fmt_time': location_point.data.fmt_time
})
return new_mc
def segment_into_motion_changes(self, timeseries, time_query):
"""
Use the motion changes detected on the phone to detect sections (consecutive chains of points)
that have a consistent motion.
:param timeseries: the time series for this user
:param time_query: the range to consider for segmentation
:return: a list of tuples [(start_motion, end_motion)] that represent the ranges with a consistent motion.
The gap between end_motion[n] and start_motion[n+1] represents the transition between the activities.
We don't actually know the motion/activity in that range with any level of confidence. We need a policy on
how to deal with them (combine with first, combine with second, split in the middle). This policy can be
enforced when we map the activity changes to locations.
"""
motion_df = timeseries.get_data_df("background/motion_activity",
time_query)
filter_mask = motion_df.apply(self.is_filtered, axis=1)
# Calling np.nonzero on the filter_mask even if it was related trips with zero sections
# has not been a problem before this - the subsequent check on the
# length of the filtered dataframe was sufficient. But now both Tom and
# I have hit it (on 18th and 21st of Sept) so let's handle it proactively here.
if filter_mask.shape == (0, 0):
logging.info("Found filter_mask with shape (0,0), returning blank")
return []
logging.debug("filtered points %s" %
np.nonzero(filter_mask.to_numpy()))
logging.debug("motion_df = %s" % motion_df.head())
filtered_df = motion_df[filter_mask]
if len(filtered_df) == 0:
# If there were no entries in the filtered_df, then there are no sections,
# and we need to return an empty list. This check enforces that...
return []
motion_change_list = []
prev_motion = None
curr_start_motion = ecwm.Motionactivity(filtered_df.iloc[0])
# curr_section = ad.AttrDict({"user_id": trip.user_id, "loc_filter": trip.loc_filter,
# "start_ts": trip.start_ts, "start_time": trip.start_time,
# "activity": no_tilting_points_df.iloc[0].activity})
for idx, row in filtered_df.iterrows():
curr_motion = ecwm.Motionactivity(row)
if curr_motion.type != curr_start_motion.type:
# Because the curr_start_motion is initialized with the first
# motion. So when idx == 0, the activities will be equal and
# this is guaranteed to not be invoked
assert (idx > 0)
logging.debug(
"At %s, found new activity %s compared to current %s - creating new section with start_time %s"
% (curr_motion.fmt_time, curr_motion.type,
curr_start_motion.type, curr_motion.fmt_time))
# complete this section
motion_change_list.append((curr_start_motion, curr_motion))
curr_start_motion = curr_motion
else:
logging.debug(
"At %s, retained existing activity %s because of no change"
% (curr_motion.fmt_time, curr_motion.type))
prev_motion = curr_motion
logging.info("Detected trip end! Ending section at %s" %
curr_motion.fmt_time)
motion_change_list.append((curr_start_motion, curr_motion))
# Go from activities to
# Merge short sections
# Sometimes, the sections flip-flop around
return motion_change_list
def segment_into_motion_changes(self, timeseries, time_query):
"""
Use the motion changes detected on the phone to detect sections (consecutive chains of points)
that have a consistent motion.
:param timeseries: the time series for this user
:param time_query: the range to consider for segmentation
:return: a list of tuples [(start_motion, end_motion)] that represent the ranges with a consistent motion.
The gap between end_motion[n] and start_motion[n+1] represents the transition between the activities.
We don't actually know the motion/activity in that range with any level of confidence. We need a policy on
how to deal with them (combine with first, combine with second, split in the middle). This policy can be
enforced when we map the activity changes to locations.
"""
motion_df = timeseries.get_data_df("background/motion_activity",
time_query)
filter_mask = motion_df.apply(self.is_filtered, axis=1)
# Calling np.nonzero on the filter_mask even if it was related trips with zero sections
# has not been a problem before this - the subsequent check on the
# length of the filtered dataframe was sufficient. But now both Tom and
# I have hit it (on 18th and 21st of Sept) so let's handle it proactively here.
if filter_mask.shape == (0, 0):
logging.info("Found filter_mask with shape (0,0), returning blank")
return []
# Replace RUNNING with WALKING so that we don't get extra, incorrect sections
# https://github.com/e-mission/e-mission-server/issues/577#issuecomment-379496118
motion_df["type"].replace([8], 7, inplace=True)
logging.debug("filtered points %s" % np.nonzero(filter_mask))
logging.debug("motion_df = %s" % motion_df.head())
filtered_df = motion_df[filter_mask]
filtered_df.reset_index(inplace=True)
if len(filtered_df) == 0:
# If there were no entries in the filtered_df, then there are no sections,
# and we need to return an empty list. This check enforces that...
return []
motion_change_list = []
prev_motion = None
curr_start_motion = ecwm.Motionactivity(filtered_df.iloc[0])
# curr_section = ad.AttrDict({"user_id": trip.user_id, "loc_filter": trip.loc_filter,
# "start_ts": trip.start_ts, "start_time": trip.start_time,
# "activity": no_tilting_points_df.iloc[0].activity})
for idx, row in filtered_df.iterrows():
curr_motion = ecwm.Motionactivity(row)
curr_motion.update({"idx": idx})
# Since the start motion is set upstream makes sure to set an idx
# for it too
if curr_motion.ts == curr_start_motion.ts:
curr_start_motion.update({"idx": idx})
if curr_motion.type != curr_start_motion.type:
# Because the curr_start_motion is initialized with the first
# motion. So when idx == 0, the activities will be equal and
# this is guaranteed to not be invoked
assert (idx > 0)
logging.debug(
"At idx %d, time %s, found new activity %s compared to current %s"
% (idx, curr_motion.fmt_time, curr_motion.type,
curr_start_motion.type))
curr_end_motion = get_curr_end_motion(prev_motion, curr_motion)
logging.debug(
"creating new section for %s at %s -> %s with start_time %s -> %s"
% (curr_start_motion.type, curr_start_motion["idx"],
curr_end_motion["idx"], curr_start_motion.fmt_time,
curr_end_motion.fmt_time))
# complete this section
motion_change_list.append((curr_start_motion, curr_end_motion))
curr_start_motion = curr_end_motion
else:
logging.debug(
"At %s, retained existing activity %s because of no change"
% (curr_motion.fmt_time, curr_motion.type))
prev_motion = curr_motion
logging.info("Detected trip end! Ending section at %s" %
curr_motion.fmt_time)
motion_change_list.append((curr_start_motion, curr_motion))
smoothed_motion_list = ffd.FlipFlopDetection(
motion_change_list, self).merge_flip_flop_sections()
return smoothed_motion_list
