def turn_into_new_trip(self, user_id):
print "new trip"
ts = esta.TimeSeries.get_time_series(user_id)
trip = ecwrt.Rawtrip()
sections = []
our_json = self.get_json()
mode_list = set ( )
if "plan" not in our_json:
print("While querying alternatives from %s to %s" % (self.start_point, self.end_point))
print("query URL is %s" % self.make_url())
print("Response %s does not have a plan " % our_json)
raise PathNotFoundException(our_json['debugOutput'])
trip.start_loc = gj.Point( (float(our_json["plan"]["from"]["lat"]), float(our_json["plan"]["from"]["lon"])) )
trip.end_loc = gj.Point( (float(our_json["plan"]["to"]["lat"]), float(our_json["plan"]["to"]["lon"])) )
trip.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
our_json['plan']['itineraries'][0]["startTime"]).timestamp, "UTC")
trip.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
our_json['plan']['itineraries'][0]["endTime"]).timestamp, "UTC")
trip_id = ts.insert(ecwe.Entry.create_entry(user_id, "segmentation/raw_trip", trip))
for leg in our_json["plan"]["itineraries"][0]['legs']:
section = ecws.Section()
section.trip_id = trip_id
section.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
leg["startTime"]).timestamp, "UTC")
section.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
leg["endTime"]).timestamp, "UTC")
section.distance = float(leg["distance"])
section.start_loc = gj.Point( (float(leg["from"]["lat"]), float(leg["from"]["lon"])) )
section.end_loc = gj.Point( (float(leg["to"]["lat"]), float(leg["to"]["lon"])) )
ts.insert_data(user_id, "segmentation/raw_section", section)
def turn_into_new_trip(self, user_id):
print("new trip")
ts = esta.TimeSeries.get_time_series(user_id)
trip = ecwrt.Rawtrip()
sections = []
our_json = self.get_json()
mode_list = set ( )
if "plan" not in our_json:
print("While querying alternatives from %s to %s" % (self.start_point, self.end_point))
print("query URL is %s" % self.make_url())
print("Response %s does not have a plan " % our_json)
raise PathNotFoundException(our_json['debugOutput'])
trip.start_loc = gj.Point( (float(our_json["plan"]["from"]["lat"]), float(our_json["plan"]["from"]["lon"])) )
trip.end_loc = gj.Point( (float(our_json["plan"]["to"]["lat"]), float(our_json["plan"]["to"]["lon"])) )
trip.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
our_json['plan']['itineraries'][0]["startTime"]).timestamp, "UTC")
trip.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
our_json['plan']['itineraries'][0]["endTime"]).timestamp, "UTC")
trip_id = ts.insert(ecwe.Entry.create_entry(user_id, "segmentation/raw_trip", trip))
for leg in our_json["plan"]["itineraries"][0]['legs']:
section = ecws.Section()
section.trip_id = trip_id
section.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
leg["startTime"]).timestamp, "UTC")
section.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
leg["endTime"]).timestamp, "UTC")
section.distance = float(leg["distance"])
section.start_loc = gj.Point( (float(leg["from"]["lat"]), float(leg["from"]["lon"])) )
section.end_loc = gj.Point( (float(leg["to"]["lat"]), float(leg["to"]["lon"])) )
ts.insert_data(user_id, "segmentation/raw_section", section)
def setUp(self):
etc.setupRealExample(
self, "emission/tests/data/real_examples/shankari_2015-aug-27")
# eaicf.filter_accuracy(self.testUUID)
etc.runIntakePipeline(self.testUUID)
# estfm.move_all_filters_to_data()
logging.info("After loading, timeseries db size = %s" %
edb.get_timeseries_db().count())
self.day_start_ts = 1440658800
self.day_end_ts = 1440745200
self.day_start_dt = esdldq.get_local_date(self.day_start_ts,
"America/Los_Angeles")
self.day_end_dt = esdldq.get_local_date(self.day_end_ts,
"America/Los_Angeles")
# If we don't delete the time components, we end up with the upper and
# lower bounds = 0, which basically matches nothing.
del self.day_start_dt['hour']
del self.day_end_dt['hour']
del self.day_start_dt['minute']
del self.day_end_dt['minute']
del self.day_start_dt['second']
del self.day_end_dt['second']
def range_mode_heatmap(mode, start_ts, end_ts):
start_dt = esdl.get_local_date(start_ts, "UTC")
end_dt = esdl.get_local_date(end_ts, "UTC")
time_query = esttc.TimeComponentQuery("data.ts", start_dt, end_dt)
loc_entry_list = esda.get_entries(esda.CLEANED_LOCATION_KEY, user_id=None,
time_query=time_query, geo_query=None,
extra_query_list=[esdlq.get_mode_query(mode)])
return {"latlng": [e.data.loc.coordinates for e in loc_entry_list]}
def setUp(self):
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-27")
eaicf.filter_accuracy(self.testUUID)
estfm.move_all_filters_to_data()
logging.info(
"After loading, timeseries db size = %s" % edb.get_timeseries_db().count())
self.day_start_ts = 1440658800
self.day_end_ts = 1440745200
self.day_start_dt = esdldq.get_local_date(self.day_start_ts, "America/Los_Angeles")
self.day_end_dt = esdldq.get_local_date(self.day_end_ts, "America/Los_Angeles")
def setUp(self):
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-27")
eaicf.filter_accuracy(self.testUUID)
estfm.move_all_filters_to_data()
logging.info(
"After loading, timeseries db size = %s" % edb.get_timeseries_db().count())
self.day_start_ts = 1440658800
self.day_end_ts = 1440745200
self.day_start_dt = esdldq.get_local_date(self.day_start_ts, "America/Los_Angeles")
self.day_end_dt = esdldq.get_local_date(self.day_end_ts, "America/Los_Angeles")
def range_mode_heatmap(mode, start_ts, end_ts):
start_dt = esdl.get_local_date(start_ts, "UTC")
end_dt = esdl.get_local_date(end_ts, "UTC")
time_query = esttc.TimeComponentQuery("data.ts", start_dt, end_dt)
loc_entry_list = esda.get_entries(
esda.CLEANED_LOCATION_KEY,
user_id=None,
time_query=time_query,
geo_query=None,
extra_query_list=[esdlq.get_mode_query(mode)])
return {"latlng": [e.data.loc.coordinates for e in loc_entry_list]}
def setUp(self):
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-21")
self.testUUID1 = self.testUUID
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-27")
etc.runIntakePipeline(self.testUUID1)
etc.runIntakePipeline(self.testUUID)
logging.info(
"After loading, timeseries db size = %s" % edb.get_timeseries_db().count())
self.aug_start_ts = 1438387200
self.aug_end_ts = 1441065600
self.day_start_dt = esdldq.get_local_date(self.aug_start_ts, "America/Los_Angeles")
self.day_end_dt = esdldq.get_local_date(self.aug_end_ts, "America/Los_Angeles")
def setUp(self):
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-21")
self.testUUID1 = self.testUUID
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-27")
etc.runIntakePipeline(self.testUUID1)
etc.runIntakePipeline(self.testUUID)
logging.info(
"After loading, timeseries db size = %s" % edb.get_timeseries_db().count())
self.aug_start_ts = 1438387200
self.aug_end_ts = 1441065600
self.day_start_dt = esdldq.get_local_date(self.aug_start_ts, "America/Los_Angeles")
self.day_end_dt = esdldq.get_local_date(self.aug_end_ts, "America/Los_Angeles")
def testGetLocalDay(self):
adt = arrow.get(pydt.datetime(2016, 1, 1, 9, 46, 0, 0))
test_dt = ecsdlq.get_local_date(adt.timestamp, "America/Los_Angeles")
test_trip = ecwt.Trip({'start_local_dt': test_dt, 'start_fmt_time': adt.isoformat()})
test_handler = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
self.assertEqual(test_handler.get_local_day_from_fmt_time(test_trip), "2016-01-01")
self.assertEqual(test_handler.get_local_day_from_local_dt(test_trip), "2016-01-01")
def _fillDates(self, object, prefix, ardt, timezone):
object["%sts" % prefix] = ardt.timestamp
object["%slocal_dt" % prefix] = esdl.get_local_date(ardt.timestamp,
timezone)
object["%sfmt_time" % prefix] = ardt.to(timezone).isoformat()
logging.debug("After filling entries, keys are %s" % object.keys())
return object
def create_measurement(coordinate, timestamp, velocity, altitude, user_id):
#TODO: Rename to create_location_measurement
"""
Creates location entry.
"""
new_loc = ecwl.Location(
ts = timestamp,
latitude = coordinate[0],
longitude = coordinate[1],
sensed_speed = velocity,
accuracy = 0,
bearing = 0,
filter = 'distance',
fmt_time = arrow.get(timestamp).to('UTC').format(),
#This should not be neseceary. TODO: Figure out how we can avoind this.
loc = gj.Point( (coordinate[1], coordinate[0]) ),
local_dt = ecsdlq.get_local_date(timestamp, 'UTC'),
altitude = altitude
)
entry = ecwe.Entry.create_entry(user_id,"background/filtered_location", new_loc, create_id=True)
#This field ('type') is required by the server when we push the entry to the user cache
# so we add it here. Also we just chose an abritrary formater. In the future we might want to
# create a fromater group called fake user.
entry['metadata']['type'] = 'sensor-data'
entry['metadata']['platform'] = 'android'
#entry['data']['bearing'] = 0
return entry
def create_motion_entry_from_leg(leg, user_id):
#TODO: Update with all possible/supported OTP modes. Also check for leg == None
#Also, make sure this timestamp is correct
timestamp = float(otp_time_to_ours(leg['startTime']).timestamp)
print("*** Leg Start Time: %s" % arrow.get(timestamp).format())
opt_mode_to_motion_type = {
'BICYCLE': ecwm.MotionTypes.BICYCLING.value,
'CAR': ecwm.MotionTypes.IN_VEHICLE.value,
'RAIL': ecwm.MotionTypes.IN_VEHICLE.value,
'WALK': ecwm.MotionTypes.WALKING.value
}
new_motion_activity = ecwm.Motionactivity(
ts = timestamp,
type = opt_mode_to_motion_type[leg['mode']],
#The following two lines were added to satisfy the formatters/android/motion_activity.py script
zzaKM = opt_mode_to_motion_type[leg['mode']],
zzaKN = 100.0,
fmt_time = arrow.get(timestamp).to('UTC').format(),
local_dt = ecsdlq.get_local_date(timestamp, 'UTC'),
confidence = 100.0
)
entry = ecwe.Entry.create_entry(user_id, "background/motion_activity", new_motion_activity, create_id=True)
#This field ('type') is required by the server when we push the entry to the user cache
# so we add it here.
entry['metadata']['type'] = 'sensor-data'
#For some reason the android formater overwrites ts with metadata.write_ts.
#so we need to set write_ts to ts to make sure they become the same.
entry['metadata']['write_ts'] = timestamp
entry['metadata']['platform'] = 'android'
return entry
def _fillDates(self, object, prefix, ardt, timezone):
object["%sts" % prefix] = ardt.timestamp
object["%slocal_dt" % prefix] = esdl.get_local_date(ardt.timestamp,
timezone)
object["%sfmt_time" % prefix] = ardt.to(timezone).isoformat()
logging.debug("After filling entries, keys are %s" % object.keys())
return object
def testLocalDateReadWrite(self):
ts = esta.TimeSeries.get_time_series(self.testUUID)
start_ts = arrow.now().timestamp
ma_ts = 1460586729
local_dt = esdl.get_local_date(ma_ts, "America/Los_Angeles")
fmt_time = arrow.get(ma_ts).to("America/Los_Angeles").isoformat()
ma = ecwm.Motionactivity({
"ts": 1460586729,
"local_dt": local_dt,
"fmt_time": fmt_time
})
ma_entry = ecwe.Entry.create_entry(self.testUUID, "background/motion_activity",
ma)
ts.insert(ma_entry)
ret_entry = ecwe.Entry(ts.get_entry_at_ts("background/motion_activity", "data.ts", 1460586729))
self.assertGreaterEqual(ret_entry.metadata.write_ts, start_ts)
metadata_dt = arrow.get(ret_entry.metadata.write_ts).to(ret_entry.metadata.time_zone).datetime
self.assertEqual(metadata_dt.hour, ret_entry.metadata.write_local_dt.hour)
self.assertEqual(metadata_dt.minute, ret_entry.metadata.write_local_dt.minute)
self.assertEqual(metadata_dt.weekday(), ret_entry.metadata.write_local_dt.weekday)
self.assertEqual(ret_entry.data.local_dt.hour, 15)
self.assertEqual(ret_entry.data.local_dt.month, 4)
self.assertEqual(ret_entry.data.local_dt.weekday, 2)
self.assertEqual(ret_entry.data.fmt_time, "2016-04-13T15:32:09-07:00")
def create_motion_entry_from_leg(leg, user_id):
#TODO: Update with all possible/supported OTP modes. Also check for leg == None
#Also, make sure this timestamp is correct
timestamp = float(otp_time_to_ours(leg['startTime']).timestamp)
print("*** Leg Start Time: %s" % arrow.get(timestamp).format())
opt_mode_to_motion_type = {
'BICYCLE': ecwm.MotionTypes.BICYCLING.value,
'CAR': ecwm.MotionTypes.IN_VEHICLE.value,
'RAIL': ecwm.MotionTypes.IN_VEHICLE.value,
'WALK': ecwm.MotionTypes.WALKING.value
}
new_motion_activity = ecwm.Motionactivity(
ts=timestamp,
type=opt_mode_to_motion_type[leg['mode']],
#The following two lines were added to satisfy the formatters/android/motion_activity.py script
zzaKM=opt_mode_to_motion_type[leg['mode']],
zzaKN=100.0,
fmt_time=arrow.get(timestamp).to('UTC').format(),
local_dt=ecsdlq.get_local_date(timestamp, 'UTC'),
confidence=100.0)
entry = ecwe.Entry.create_entry(user_id,
"background/motion_activity",
new_motion_activity,
create_id=True)
#This field ('type') is required by the server when we push the entry to the user cache
# so we add it here.
entry['metadata']['type'] = 'sensor-data'
#For some reason the android formater overwrites ts with metadata.write_ts.
#so we need to set write_ts to ts to make sure they become the same.
entry['metadata']['write_ts'] = timestamp
entry['metadata']['platform'] = 'android'
return entry
def create_measurement(coordinate, timestamp, velocity, altitude, user_id):
#TODO: Rename to create_location_measurement
"""
Creates location entry.
"""
new_loc = ecwl.Location(
ts=timestamp,
latitude=coordinate[0],
longitude=coordinate[1],
sensed_speed=velocity,
accuracy=0,
bearing=0,
filter='distance',
fmt_time=arrow.get(timestamp).to('UTC').format(),
#This should not be neseceary. TODO: Figure out how we can avoind this.
loc=gj.Point((coordinate[1], coordinate[0])),
local_dt=ecsdlq.get_local_date(timestamp, 'UTC'),
altitude=altitude)
entry = ecwe.Entry.create_entry(user_id,
"background/filtered_location",
new_loc,
create_id=True)
#This field ('type') is required by the server when we push the entry to the user cache
# so we add it here. Also we just chose an abritrary formater. In the future we might want to
# create a fromater group called fake user.
entry['metadata']['type'] = 'sensor-data'
entry['metadata']['platform'] = 'android'
#entry['data']['bearing'] = 0
return entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts) / 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = float(
entry.data.mTime) / 1000 # convert the ms from the phone to secs
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts)/ 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = float(entry.data.mTime) / 1000 # convert the ms from the phone to secs
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def testGetLocalDay(self):
adt = arrow.get(pydt.datetime(2016, 1, 1, 9, 46, 0, 0))
test_dt = ecsdlq.get_local_date(adt.timestamp, "America/Los_Angeles")
test_trip = ecwt.Trip({'start_local_dt': test_dt, 'start_fmt_time': adt.isoformat()})
test_handler = enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1)
self.assertEqual(test_handler.get_local_day_from_fmt_time(test_trip), "2016-01-01")
self.assertEqual(test_handler.get_local_day_from_local_dt(test_trip), "2016-01-01")
def resample(filtered_loc_list, interval):
"""
TODO: There is a problem with working on this on a section by section basis.
Namely, how do we deal with the ends. In particular, note that our sections
are generated by looking at motion activity while our points are generated
by looking at locations, and the two streams are not synchronized. So we have
"transition areas" between sections that have no points. Resampling will
help with this because we can ensure that arbitrary time point - for example,
the transition between sections, has an associated location point. But that
fix really needs to be made in the segmentation code - by this point, the
original timestamps have been replaced by the timestamps of the start and
end location points. But we can also have a gap if the duration of the section
is not a multiple of the interval, as it is likely to be. In this case, we will
manually add the last entry to ensure that we adequately capture the end point.
:param filtered_loc_list:
:param interval:
:return:
"""
loc_df = pd.DataFrame(filtered_loc_list)
# See https://github.com/e-mission/e-mission-server/issues/268 for log traces
#
# basically, on iOS, due to insufficient smoothing, it is possible for us to
# have very small segments. Some of these contain zero points, and we skip them
# in the segmentation stage. Some of them contain one point, and we don't.
# Ideally, we would strip these sections too and merge the stops on the two sides
# But that is going to take more time and effort than I have here.
#
# So let's just return the one point without resampling in that case, and move on for now
if len(loc_df) == 0 or len(loc_df) == 1:
return loc_df
logging.debug("Resampling entry list %s of size %s" % (loc_df.head(), len(filtered_loc_list)))
start_ts = loc_df.ts.iloc[0]
end_ts = loc_df.ts.iloc[-1]
tz_ranges_df = _get_tz_ranges(loc_df)
logging.debug("tz_ranges_df = %s" % tz_ranges_df)
lat_fn = spi.interp1d(x=loc_df.ts, y=loc_df.latitude, bounds_error=False,
fill_value='extrapolate')
lng_fn = spi.interp1d(x=loc_df.ts, y=loc_df.longitude, bounds_error=False,
fill_value='extrapolate')
altitude_fn = spi.interp1d(x=loc_df.ts, y=loc_df.altitude, bounds_error=False,
fill_value='extrapolate')
ts_new = np.append(np.arange(start_ts, end_ts, 30), [end_ts])
logging.debug("After resampling, using %d points" % ts_new.size)
lat_new = lat_fn(ts_new)
lng_new = lng_fn(ts_new)
alt_new = altitude_fn(ts_new)
tz_new = [_get_timezone(ts, tz_ranges_df) for ts in ts_new]
ld_new = [esdl.get_local_date(ts, tz) for (ts, tz) in zip(ts_new, tz_new)]
loc_new = [gj.Point((lng, lat)) for (lng, lat) in zip(lng_new, lat_new)]
fmt_time_new = [arrow.get(ts).to(tz).isoformat() for
(ts, tz) in zip(ts_new, tz_new)]
loc_df_new = pd.DataFrame({"latitude": lat_new, "longitude": lng_new,
"loc": loc_new, "ts": ts_new, "local_dt": ld_new,
"fmt_time": fmt_time_new, "altitude": alt_new})
return loc_df_new
def resample(filtered_loc_list, interval):
"""
TODO: There is a problem with working on this on a section by section basis.
Namely, how do we deal with the ends. In particular, note that our sections
are generated by looking at motion activity while our points are generated
by looking at locations, and the two streams are not synchronized. So we have
"transition areas" between sections that have no points. Resampling will
help with this because we can ensure that arbitrary time point - for example,
the transition between sections, has an associated location point. But that
fix really needs to be made in the segmentation code - by this point, the
original timestamps have been replaced by the timestamps of the start and
end location points. But we can also have a gap if the duration of the section
is not a multiple of the interval, as it is likely to be. In this case, we will
manually add the last entry to ensure that we adequately capture the end point.
:param filtered_loc_list:
:param interval:
:return:
"""
loc_df = pd.DataFrame(filtered_loc_list)
start_ts = loc_df.ts.iloc[0]
end_ts = loc_df.ts.iloc[-1]
tz_ranges_df = _get_tz_ranges(loc_df)
logging.debug("tz_ranges_df = %s" % tz_ranges_df)
lat_fn = spi.interp1d(x=loc_df.ts,
y=loc_df.latitude,
bounds_error=False,
fill_value='extrapolate')
lng_fn = spi.interp1d(x=loc_df.ts,
y=loc_df.longitude,
bounds_error=False,
fill_value='extrapolate')
altitude_fn = spi.interp1d(x=loc_df.ts,
y=loc_df.altitude,
bounds_error=False,
fill_value='extrapolate')
ts_new = np.append(np.arange(start_ts, end_ts, 30), [end_ts])
lat_new = lat_fn(ts_new)
lng_new = lng_fn(ts_new)
alt_new = altitude_fn(ts_new)
tz_new = [_get_timezone(ts, tz_ranges_df) for ts in ts_new]
ld_new = [esdl.get_local_date(ts, tz) for (ts, tz) in zip(ts_new, tz_new)]
loc_new = [gj.Point((lng, lat)) for (lng, lat) in zip(lng_new, lat_new)]
fmt_time_new = [
arrow.get(ts).to(tz).isoformat() for (ts, tz) in zip(ts_new, tz_new)
]
loc_df_new = pd.DataFrame({
"latitude": lat_new,
"longitude": lng_new,
"loc": loc_new,
"ts": ts_new,
"local_dt": ld_new,
"fmt_time": fmt_time_new,
"altitude": alt_new
})
return loc_df_new
def Berkeley_pop_route(start_ts, end_ts):
berkeley_json = {"geometry": {
"type": "Polygon",
"coordinates": [[
[-122.267443, 37.864693], [-122.267443, 37.880687], [-122.250985, 37.880687], [-122.250985, 37.864693], [-122.267443, 37.864693]
]]
}
}
# box = [ [-122.267443, 37.864693], [-122.250985, 37.880687] ]
start_dt = esdl.get_local_date(start_ts, "UTC")
end_dt = esdl.get_local_date(end_ts, "UTC")
time_query = esttc.TimeComponentQuery("data.ts", start_dt, end_dt)
geo_query = estg.GeoQuery(["data.loc"], berkeley_json)
loc_entry_list = esda.get_entries(esda.CLEANED_LOCATION_KEY, user_id=None,
time_query=time_query,
geo_query=geo_query)
return {"latlng": [e.data.loc.coordinates for e in loc_entry_list]}
def get_local_date(fmt_time, timezone):
"""
When we parse the fmt time, we get a timezone offset, but not the timezone string.
Timezone string seems more portable, so we want to use that instead.
So we need to get it from somewhere else and pass it in here
"""
adt = arrow.get(fmt_time)
logging.debug("after parsing, adt = %s" % adt)
return ecsdlq.get_local_date(adt.timestamp, timezone)
def get_local_date(fmt_time, timezone):
"""
When we parse the fmt time, we get a timezone offset, but not the timezone string.
Timezone string seems more portable, so we want to use that instead.
So we need to get it from somewhere else and pass it in here
"""
adt = arrow.get(fmt_time)
logging.debug("after parsing, adt = %s" % adt)
return ecsdlq.get_local_date(adt.timestamp, timezone)
def expand_metadata_times(m):
logging.debug("write_ts = %s" % m.write_ts)
# Write DT will also be in local time to allow us to search for all trips on a particular
# day or all movement in a particular location between a particular time range without doing
# string parsing. If we didn't do this, there is no way to know when a
# particular event occurred in local time without string parsing.
# The timestamp is in UTC and doesn't know the local time.
# The fmt_time is in local time, but is a string, so query ranges are hard
# to search for
m.write_local_dt = ecsdlq.get_local_date(m.write_ts, m.time_zone)
m.write_fmt_time = arrow.get(m.write_ts).to(m.time_zone).isoformat()
def expand_metadata_times(m):
logging.debug("write_ts = %s" % m.write_ts)
# Write DT will also be in local time to allow us to search for all trips on a particular
# day or all movement in a particular location between a particular time range without doing
# string parsing. If we didn't do this, there is no way to know when a
# particular event occurred in local time without string parsing.
# The timestamp is in UTC and doesn't know the local time.
# The fmt_time is in local time, but is a string, so query ranges are hard
# to search for
m.write_local_dt = ecsdlq.get_local_date(m.write_ts, m.time_zone)
m.write_fmt_time = arrow.get(m.write_ts).to(m.time_zone).isoformat()
def create_start_location_from_trip_plan(plan):
#TODO: Old function. Should be removed
converted_time = otp_time_to_ours(plan['itineraries'][0]["startTime"])
time_stamp = converted_time.timestamp
local_dt = ecsdlq.get_local_date(time_stamp, 'UTC')
fmt_time = converted_time.to("UTC").format()
loc = gj.Point((float(plan["from"]["lon"]), float(plan["from"]["lat"])))
start_loc = ecwl.Location(ts=time_stamp,
local_dt=local_dt,
fmt_time=fmt_time,
loc=loc)
return start_loc
def create_metadata_for_result(key):
import emission.storage.decorations.local_date_queries as esdl
import arrow
m = Metadata()
m.key = key
m.platform = "server"
m.write_ts = time.time()
m.time_zone = "America/Los_Angeles"
m.write_local_dt = esdl.get_local_date(m.write_ts, m.time_zone)
m.write_fmt_time = arrow.get(m.write_ts).to(m.time_zone).isoformat()
return m
def create_end_location_from_leg(leg):
#TODO: Old function. Should be removed
converted_time = otp_time_to_ours(leg['endTime'])
time_stamp = converted_time.timestamp
local_dt = ecsdlq.get_local_date(time_stamp, 'UTC')
fmt_time = converted_time.to("UTC").format()
loc = gj.Point((float(leg["to"]["lon"]), float(leg["to"]["lat"])))
end_loc = ecwl.Location(ts=time_stamp,
local_dt=local_dt,
fmt_time=fmt_time,
loc=loc)
return end_loc
def create_metadata_for_result(key):
import emission.storage.decorations.local_date_queries as esdl
import arrow
m = Metadata()
m.key = key
m.platform = "server"
m.write_ts = time.time()
m.time_zone = "America/Los_Angeles"
m.write_local_dt = esdl.get_local_date(m.write_ts, m.time_zone)
m.write_fmt_time = arrow.get(m.write_ts).to(m.time_zone).isoformat()
return m
def Berkeley_pop_route(start_ts, end_ts):
berkeley_json = {
"geometry": {
"type":
"Polygon",
"coordinates":
[[[-122.267443, 37.864693], [-122.267443, 37.880687],
[-122.250985, 37.880687], [-122.250985, 37.864693],
[-122.267443, 37.864693]]]
}
}
# box = [ [-122.267443, 37.864693], [-122.250985, 37.880687] ]
start_dt = esdl.get_local_date(start_ts, "UTC")
end_dt = esdl.get_local_date(end_ts, "UTC")
time_query = esttc.TimeComponentQuery("data.ts", start_dt, end_dt)
geo_query = estg.GeoQuery(["data.loc"], berkeley_json)
loc_entry_list = esda.get_entries(esda.CLEANED_LOCATION_KEY,
user_id=None,
time_query=time_query,
geo_query=geo_query)
return {"lnglat": [e.data.loc.coordinates for e in loc_entry_list]}
def create_end_location_from_leg(leg):
#TODO: Old function. Should be removed
converted_time = otp_time_to_ours(leg['endTime'])
time_stamp = converted_time.timestamp
local_dt = ecsdlq.get_local_date(time_stamp, 'UTC')
fmt_time = converted_time.to("UTC").format()
loc = gj.Point( (float(leg["to"]["lon"]), float(leg["to"]["lat"])) )
end_loc = ecwl.Location(
ts =time_stamp,
local_dt =local_dt,
fmt_time= fmt_time,
loc = loc
)
return end_loc
def setUp(self):
etc.setupRealExample(self,
"emission/tests/data/real_examples/shankari_2015-aug-27")
# eaicf.filter_accuracy(self.testUUID)
etc.runIntakePipeline(self.testUUID)
# estfm.move_all_filters_to_data()
logging.info(
"After loading, timeseries db size = %s" % edb.get_timeseries_db().count())
self.day_start_ts = 1440658800
self.day_end_ts = 1440745200
self.day_start_dt = esdldq.get_local_date(self.day_start_ts, "America/Los_Angeles")
self.day_end_dt = esdldq.get_local_date(self.day_end_ts, "America/Los_Angeles")
# If we don't delete the time components, we end up with the upper and
# lower bounds = 0, which basically matches nothing.
del self.day_start_dt['hour']
del self.day_end_dt['hour']
del self.day_start_dt['minute']
del self.day_end_dt['minute']
del self.day_start_dt['second']
del self.day_end_dt['second']
def create_start_location_from_trip_plan(plan):
#TODO: Old function. Should be removed
converted_time = otp_time_to_ours(plan['itineraries'][0]["startTime"])
time_stamp = converted_time.timestamp
local_dt = ecsdlq.get_local_date(time_stamp, 'UTC')
fmt_time = converted_time.to("UTC").format()
loc = gj.Point( (float(plan["from"]["lon"]), float(plan["from"]["lat"])) )
start_loc = ecwl.Location(
ts =time_stamp,
local_dt =local_dt,
fmt_time= fmt_time,
loc = loc
)
return start_loc
def resample(filtered_loc_list, interval):
"""
TODO: There is a problem with working on this on a section by section basis.
Namely, how do we deal with the ends. In particular, note that our sections
are generated by looking at motion activity while our points are generated
by looking at locations, and the two streams are not synchronized. So we have
"transition areas" between sections that have no points. Resampling will
help with this because we can ensure that arbitrary time point - for example,
the transition between sections, has an associated location point. But that
fix really needs to be made in the segmentation code - by this point, the
original timestamps have been replaced by the timestamps of the start and
end location points. But we can also have a gap if the duration of the section
is not a multiple of the interval, as it is likely to be. In this case, we will
manually add the last entry to ensure that we adequately capture the end point.
:param filtered_loc_list:
:param interval:
:return:
"""
loc_df = pd.DataFrame(filtered_loc_list)
start_ts = loc_df.ts.iloc[0]
end_ts = loc_df.ts.iloc[-1]
tz_ranges_df = _get_tz_ranges(loc_df)
logging.debug("tz_ranges_df = %s" % tz_ranges_df)
lat_fn = spi.interp1d(x=loc_df.ts, y=loc_df.latitude, bounds_error=False,
fill_value='extrapolate')
lng_fn = spi.interp1d(x=loc_df.ts, y=loc_df.longitude, bounds_error=False,
fill_value='extrapolate')
altitude_fn = spi.interp1d(x=loc_df.ts, y=loc_df.altitude, bounds_error=False,
fill_value='extrapolate')
ts_new = np.append(np.arange(start_ts, end_ts, 30), [end_ts])
lat_new = lat_fn(ts_new)
lng_new = lng_fn(ts_new)
alt_new = altitude_fn(ts_new)
tz_new = [_get_timezone(ts, tz_ranges_df) for ts in ts_new]
ld_new = [esdl.get_local_date(ts, tz) for (ts, tz) in zip(ts_new, tz_new)]
loc_new = [gj.Point((lng, lat)) for (lng, lat) in zip(lng_new, lat_new)]
fmt_time_new = [arrow.get(ts).to(tz).isoformat() for
(ts, tz) in zip(ts_new, tz_new)]
loc_df_new = pd.DataFrame({"latitude": lat_new, "longitude": lng_new,
"loc": loc_new, "ts": ts_new, "local_dt": ld_new,
"fmt_time": fmt_time_new, "altitude": alt_new})
return loc_df_new
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" % (entry.metadata.write_ts, metadata.write_ts))
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" %
(entry.metadata.write_ts, metadata.write_ts))
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
formatted_entry.data = data
return formatted_entry
def timestamp_fill_times(key, ignored, metric_summary):
dt = arrow.get(key)
metric_summary.ts = dt.timestamp
metric_summary.local_dt = esdl.get_local_date(dt.timestamp, 'UTC')
metric_summary.fmt_time = dt.isoformat()
def timestamp_fill_times(key, ignored, metric_summary):
dt = arrow.get(key)
metric_summary.ts = dt.timestamp
metric_summary.local_dt = esdl.get_local_date(dt.timestamp, 'UTC')
metric_summary.fmt_time = dt.isoformat()
def test_create_start_location_form_leg(self):
legs = self.opt_trip_1.get_json()["plan"]["itineraries"][0]['legs']
first_leg = legs[0]
start_loc = otp.create_start_location_from_leg(first_leg)
self.assertEqual(start_loc.ts,otp.otp_time_to_ours(first_leg['startTime']).timestamp )
self.assertEqual(start_loc.local_dt, ecsdlq.get_local_date(start_loc.ts, 'UTC'))
def expand_data_times(d,m):
d.local_dt = ecsdlq.get_local_date(d.ts, m.time_zone)
d.fmt_time = arrow.get(d.ts).to(m.time_zone).isoformat()
def expand_start_end_data_times(d,m):
d.start_local_dt = ecsdlq.get_local_date(d.start_ts, m.time_zone)
d.start_fmt_time = arrow.get(d.start_ts).to(m.time_zone).isoformat()
d.end_local_dt = ecsdlq.get_local_date(d.end_ts, m.time_zone)
d.end_fmt_time = arrow.get(d.end_ts).to(m.time_zone).isoformat()
def resample(filtered_loc_list, interval):
"""
TODO: There is a problem with working on this on a section by section basis.
Namely, how do we deal with the ends. In particular, note that our sections
are generated by looking at motion activity while our points are generated
by looking at locations, and the two streams are not synchronized. So we have
"transition areas" between sections that have no points. Resampling will
help with this because we can ensure that arbitrary time point - for example,
the transition between sections, has an associated location point. But that
fix really needs to be made in the segmentation code - by this point, the
original timestamps have been replaced by the timestamps of the start and
end location points. But we can also have a gap if the duration of the section
is not a multiple of the interval, as it is likely to be. In this case, we will
manually add the last entry to ensure that we adequately capture the end point.
:param filtered_loc_list:
:param interval:
:return:
"""
loc_df = pd.DataFrame(filtered_loc_list)
# See https://github.com/e-mission/e-mission-server/issues/268 for log traces
#
# basically, on iOS, due to insufficient smoothing, it is possible for us to
# have very small segments. Some of these contain zero points, and we skip them
# in the segmentation stage. Some of them contain one point, and we don't.
# Ideally, we would strip these sections too and merge the stops on the two sides
# But that is going to take more time and effort than I have here.
#
# So let's just return the one point without resampling in that case, and move on for now
if len(loc_df) == 1:
return loc_df
logging.debug("Resampling entry list %s of size %s" %
(loc_df.head(), len(filtered_loc_list)))
start_ts = loc_df.ts.iloc[0]
end_ts = loc_df.ts.iloc[-1]
tz_ranges_df = _get_tz_ranges(loc_df)
logging.debug("tz_ranges_df = %s" % tz_ranges_df)
lat_fn = spi.interp1d(x=loc_df.ts,
y=loc_df.latitude,
bounds_error=False,
fill_value='extrapolate')
lng_fn = spi.interp1d(x=loc_df.ts,
y=loc_df.longitude,
bounds_error=False,
fill_value='extrapolate')
altitude_fn = spi.interp1d(x=loc_df.ts,
y=loc_df.altitude,
bounds_error=False,
fill_value='extrapolate')
ts_new = np.append(np.arange(start_ts, end_ts, 30), [end_ts])
lat_new = lat_fn(ts_new)
lng_new = lng_fn(ts_new)
alt_new = altitude_fn(ts_new)
tz_new = [_get_timezone(ts, tz_ranges_df) for ts in ts_new]
ld_new = [esdl.get_local_date(ts, tz) for (ts, tz) in zip(ts_new, tz_new)]
loc_new = [gj.Point((lng, lat)) for (lng, lat) in zip(lng_new, lat_new)]
fmt_time_new = [
arrow.get(ts).to(tz).isoformat() for (ts, tz) in zip(ts_new, tz_new)
]
loc_df_new = pd.DataFrame({
"latitude": lat_new,
"longitude": lng_new,
"loc": loc_new,
"ts": ts_new,
"local_dt": ld_new,
"fmt_time": fmt_time_new,
"altitude": alt_new
})
return loc_df_new
def expand_data_times(d, m):
d.local_dt = ecsdlq.get_local_date(d.ts, m.time_zone)
d.fmt_time = arrow.get(d.ts).to(m.time_zone).isoformat()
