def testFeatureGenWithOnePoint(self):
# ensure that the start and end datetimes are the same, since the average calculation uses
# the total distance and the total duration
ts = esta.TimeSeries.get_time_series(self.testUUID)
trackpoint1 = ecwlo.Location({u'coordinates': [0, 0], 'type': 'Point'})
ts.insert_data(self.testUUID, "analysis/recreated_location",
trackpoint1)
testSeg = ecws.Section({
"start_loc": trackpoint1,
"end_loc": trackpoint1,
"distance": 500,
"sensed_mode": 1,
"duration": 150,
"start_ts": arrow.now().timestamp,
"end_ts": arrow.now().timestamp,
"_id": 2,
"speeds": [],
"distances": [],
})
testSegEntry = ecwe.Entry.create_entry(self.testUUID,
"analysis/cleaned_section",
testSeg)
d = testSegEntry.data
m = testSegEntry.metadata
enufc.expand_start_end_data_times(d, m)
testSegEntry["data"] = d
testSegEntry["metadata"] = m
inserted_id = ts.insert(testSegEntry)
featureMatrix = np.zeros([1, len(self.pipeline.featureLabels)])
resultVector = np.zeros(1)
self.pipeline.updateFeatureMatrixRowWithSection(
featureMatrix, 0, testSegEntry)
logging.debug("featureMatrix = %s" % featureMatrix)
self.assertEqual(np.count_nonzero(featureMatrix[0][5:16]), 0)
self.assertEqual(np.count_nonzero(featureMatrix[0][19:21]), 0)
def turn_into_new_trip(self, user_id):
print("new trip")
ts = esta.TimeSeries.get_time_series(user_id)
trip = ecwrt.Rawtrip()
sections = []
our_json = self.get_json()
mode_list = set ( )
if "plan" not in our_json:
print("While querying alternatives from %s to %s" % (self.start_point, self.end_point))
print("query URL is %s" % self.make_url())
print("Response %s does not have a plan " % our_json)
raise PathNotFoundException(our_json['debugOutput'])
trip.start_loc = gj.Point( (float(our_json["plan"]["from"]["lat"]), float(our_json["plan"]["from"]["lon"])) )
trip.end_loc = gj.Point( (float(our_json["plan"]["to"]["lat"]), float(our_json["plan"]["to"]["lon"])) )
trip.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
our_json['plan']['itineraries'][0]["startTime"]).timestamp, "UTC")
trip.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
our_json['plan']['itineraries'][0]["endTime"]).timestamp, "UTC")
trip_id = ts.insert(ecwe.Entry.create_entry(user_id, "segmentation/raw_trip", trip))
for leg in our_json["plan"]["itineraries"][0]['legs']:
section = ecws.Section()
section.trip_id = trip_id
section.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
leg["startTime"]).timestamp, "UTC")
section.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours(
leg["endTime"]).timestamp, "UTC")
section.distance = float(leg["distance"])
section.start_loc = gj.Point( (float(leg["from"]["lat"]), float(leg["from"]["lon"])) )
section.end_loc = gj.Point( (float(leg["to"]["lat"]), float(leg["to"]["lon"])) )
ts.insert_data(user_id, "segmentation/raw_section", section)
def setUp(self):
# We need to access the database directly sometimes in order to
# forcibly insert entries for the tests to pass. But we put the import
# in here to reduce the temptation to use the database directly elsewhere.
import emission.core.get_database as edb
import uuid
self.testUUID = uuid.uuid4()
self.trips = json.load(
open("emission/tests/data/smoothing_data/trip_list.txt"),
object_hook=bju.object_hook)
for trip in self.trips:
trip["user_id"] = self.testUUID
edb.get_trip_new_db().save(trip)
self.trips = [ecwt.Trip(t) for t in self.trips]
self.sections = json.load(
open("emission/tests/data/smoothing_data/section_list.txt"),
object_hook=bju.object_hook)
for section in self.sections:
section["user_id"] = self.testUUID
edb.get_section_new_db().save(section)
self.sections = [ecws.Section(s) for s in self.sections]
self.ts = esta.TimeSeries.get_time_series(self.testUUID)
def get_sections_for_trip(user_id, trip_id):
"""
Get the set of sections that are children of this trip.
"""
section_doc_cursor = edb.get_section_new_db().find({
"user_id": user_id,
"trip_id": trip_id
}).sort("start_ts", pymongo.ASCENDING)
return [ecws.Section(doc) for doc in section_doc_cursor]
def testQuerySectionsForTrip(self):
new_trip = self.create_fake_trip()
new_section = ecwc.Section()
new_section.trip_id = new_trip.get_id()
new_section.start_ts = 5
new_section.end_ts = 6
ts = esta.TimeSeries.get_time_series(self.testUserId)
ts.insert_data(self.testUserId, esda.RAW_SECTION_KEY, new_section)
ret_entries = esdt.get_raw_sections_for_trip(self.testUserId, new_trip.get_id())
self.assertEqual([entry.data for entry in ret_entries], [new_section])
def _createTestSection(self, start_ardt, start_timezone):
section = ecws.Section()
self._fillDates(section, "start_", start_ardt, start_timezone)
# Hackily fill in the end with the same values as the start
# so that the field exists
# in cases where the end is important (mainly for range timezone
# calculation with local times), it can be overridden using _fillDates
# from the test case
self._fillDates(section, "end_", start_ardt, start_timezone)
#logging.debug("created section %s" % (section.start_fmt_time))
entry = ecwe.Entry.create_entry(self.testUUID, esda.CLEANED_SECTION_KEY,
section, create_id=True)
self.ts.insert(entry)
return entry
def testQuerySections(self):
new_section = ecws.Section()
new_section.start_ts = 5
new_section.end_ts = 6
new_section.trip_id = self.test_trip_id
esta.TimeSeries.get_time_series(self.testUserId).insert_data(self.testUserId,
esda.RAW_SECTION_KEY,
new_section)
ret_arr_one = esds.get_sections_for_trip(self.testUserId, self.test_trip_id)
self.assertEqual(len(ret_arr_one), 1)
self.assertEqual([entry.data for entry in ret_arr_one], [new_section])
ret_arr_list = esds.get_sections_for_trip_list(self.testUserId, [self.test_trip_id])
self.assertEqual(ret_arr_one, ret_arr_list)
ret_arr_time = esda.get_objects(esda.RAW_SECTION_KEY, self.testUserId,
estt.TimeQuery("data.start_ts", 4, 6))
self.assertEqual([entry.data for entry in ret_arr_list], ret_arr_time)
def segment_trip_into_sections(user_id, trip_entry, trip_source):
ts = esta.TimeSeries.get_time_series(user_id)
time_query = esda.get_time_query_for_trip_like(esda.RAW_TRIP_KEY,
trip_entry.get_id())
distance_from_place = _get_distance_from_start_place_to_end(trip_entry)
if (trip_source == "DwellSegmentationTimeFilter"):
import emission.analysis.intake.segmentation.section_segmentation_methods.smoothed_high_confidence_motion as shcm
shcmsm = shcm.SmoothedHighConfidenceMotion(60, 100, [
ecwm.MotionTypes.TILTING, ecwm.MotionTypes.UNKNOWN,
ecwm.MotionTypes.STILL
])
else:
assert (trip_source == "DwellSegmentationDistFilter")
import emission.analysis.intake.segmentation.section_segmentation_methods.smoothed_high_confidence_with_visit_transitions as shcmvt
shcmsm = shcmvt.SmoothedHighConfidenceMotionWithVisitTransitions(
49,
50,
[
ecwm.MotionTypes.TILTING,
ecwm.MotionTypes.UNKNOWN,
ecwm.MotionTypes.STILL,
ecwm.MotionTypes.NONE, # iOS only
ecwm.MotionTypes.STOPPED_WHILE_IN_VEHICLE
]) # iOS only
segmentation_points = shcmsm.segment_into_sections(ts, distance_from_place,
time_query)
# Since we are segmenting an existing trip into sections, we do not need to worry about linking with
# a prior place, since it will be linked through the trip object.
# So this is much simpler than the trip case.
# Again, since this is segmenting a trip, we can just start with a section
prev_section_entry = None
# TODO: Should we link the locations to the trips this way, or by using a foreign key?
# If we want to use a foreign key, then we need to include the object id in the data df as well so that we can
# set it properly.
ts = esta.TimeSeries.get_time_series(user_id)
get_loc_for_ts = lambda time: ecwl.Location(
ts.get_entry_at_ts("background/filtered_location", "data.ts", time)[
"data"])
trip_start_loc = get_loc_for_ts(trip_entry.data.start_ts)
trip_end_loc = get_loc_for_ts(trip_entry.data.end_ts)
logging.debug("trip_start_loc = %s, trip_end_loc = %s" %
(trip_start_loc, trip_end_loc))
for (i, (start_loc_doc, end_loc_doc,
sensed_mode)) in enumerate(segmentation_points):
logging.debug("start_loc_doc = %s, end_loc_doc = %s" %
(start_loc_doc, end_loc_doc))
get_loc_for_row = lambda row: ts.df_row_to_entry(
"background/filtered_location", row).data
start_loc = get_loc_for_row(start_loc_doc)
end_loc = get_loc_for_row(end_loc_doc)
logging.debug("start_loc = %s, end_loc = %s" % (start_loc, end_loc))
section = ecwc.Section()
section.trip_id = trip_entry.get_id()
if prev_section_entry is None:
# This is the first point, so we want to start from the start of the trip, not the start of this segment
start_loc = trip_start_loc
if i == len(segmentation_points) - 1:
# This is the last point, so we want to end at the end of the trip, not at the end of this segment
# Particularly in this case, if we don't do this, then the trip end may overshoot the section end
end_loc = trip_end_loc
fill_section(section, start_loc, end_loc, sensed_mode)
# We create the entry after filling in the section so that we know
# that the data is included properly
section_entry = ecwe.Entry.create_entry(user_id,
esda.RAW_SECTION_KEY,
section,
create_id=True)
if prev_section_entry is not None:
# If this is not the first section, create a stop to link the two sections together
# The expectation is prev_section -> stop -> curr_section
stop = ecws.Stop()
stop.trip_id = trip_entry.get_id()
stop_entry = ecwe.Entry.create_entry(user_id,
esda.RAW_STOP_KEY,
stop,
create_id=True)
logging.debug("stop = %s, stop_entry = %s" % (stop, stop_entry))
stitch_together(prev_section_entry, stop_entry, section_entry)
ts.insert(stop_entry)
ts.update(prev_section_entry)
# After we go through the loop, we will be left with the last section,
# which does not have an ending stop. We insert that too.
ts.insert(section_entry)
prev_section_entry = section_entry
def create_new_section(user_id, trip_id):
_id = edb.get_section_new_db().save({
"user_id": user_id,
"trip_id": trip_id
})
return ecws.Section({"_id": _id, "user_id": user_id, "trip_id": trip_id})
def _get_sections_for_query(section_query, sort_field):
logging.debug("Returning sections for query %s" % section_query)
section_doc_cursor = edb.get_section_new_db().find(section_query).sort(
sort_field, pymongo.ASCENDING)
# TODO: Fix "TripIterator" and return it instead of this list
return [ecws.Section(doc) for doc in section_doc_cursor]
def get_section(section_id):
return ecws.Section(edb.get_section_new_db().find_one({"_id": section_id}))
