def clearRelatedDb(self):
edb.get_timeseries_db().remove()
edb.get_place_db().remove()
edb.get_stop_db().remove()
edb.get_trip_new_db().remove()
edb.get_section_new_db().remove()
def clearRelatedDb(self):
edb.get_timeseries_db().remove()
edb.get_place_db().remove()
edb.get_stop_db().remove()
edb.get_trip_new_db().remove()
edb.get_section_new_db().remove()
def del_objects(args):
del_query = {}
if args.user_id != "all":
del_query['user_id'] = uuid.UUID(args.user_id)
if args.date is None:
trip_query = del_query
place_query = del_query
else:
day_dt = pydt.datetime.strptime(args.date, "%Y-%m-%d")
logging.debug("day_dt is %s" % day_dt)
day_ts = time.mktime(day_dt.timetuple())
logging.debug("day_ts is %s" % day_ts)
trip_query = copy.copy(del_query)
trip_query.update({"start_ts": {"$gt": day_ts}})
place_query = copy.copy(del_query)
place_query.update({"exit_ts": {"$gt": day_ts}})
print "trip_query = %s" % trip_query
print "place_query = %s" % place_query
# Since sections have the same basic structure as trips and stops have the
# same basic structure as places, we can reuse the queries
print "Deleting trips for %s after %s" % (args.user_id, args.date)
print edb.get_trip_new_db().remove(trip_query)
print "Deleting sections for %s after %s" % (args.user_id, args.date)
print edb.get_section_new_db().remove(trip_query)
print "Deleting places for %s after %s" % (args.user_id, args.date)
print edb.get_place_db().remove(place_query)
print "Deleting stops for %s after %s" % (args.user_id, args.date)
print edb.get_stop_db().remove(place_query)
def clearRelatedDb(self):
edb.get_timeseries_db().remove({'user_id': self.testUUID})
edb.get_place_db().remove({'user_id': self.testUUID})
edb.get_stop_db().remove({'user_id': self.testUUID})
edb.get_trip_new_db().remove({'user_id': self.testUUID})
edb.get_section_new_db().remove({'user_id': self.testUUID})
def testSaveStop(self):
new_stop = esds.create_new_stop(self.testUserId, self.test_trip_id)
new_stop.enter_ts = 5
new_stop.exit_ts = 6
esds.save_stop(new_stop)
self.assertEqual(edb.get_stop_db().find({"exit_ts": 6}).count(), 1)
self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["_id"], new_stop.get_id())
self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["user_id"], self.testUserId)
self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["trip_id"], self.test_trip_id)
def testSaveStop(self):
new_stop = esds.create_new_stop(self.testUserId, self.test_trip_id)
new_stop.enter_ts = 5
new_stop.exit_ts = 6
esds.save_stop(new_stop)
self.assertEqual(edb.get_stop_db().find({"exit_ts": 6}).count(), 1)
self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["_id"],
new_stop.get_id())
self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["user_id"],
self.testUserId)
self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["trip_id"],
self.test_trip_id)
def get_stops_for_trip(user_id, trip_id):
"""
Get the set of sections that are children of this trip.
"""
stop_doc_cursor = edb.get_stop_db().find({"user_id": user_id, "trip_id": trip_id}).sort("enter_ts", pymongo.ASCENDING)
logging.debug("About to execute query %s" % {"user_id": user_id, "trip_id": trip_id})
return [ecwst.Stop(doc) for doc in stop_doc_cursor]
def get_stops_for_trip(user_id, trip_id):
"""
Get the set of sections that are children of this trip.
"""
stop_doc_cursor = edb.get_stop_db().find({
"user_id": user_id,
"trip_id": trip_id
}).sort("enter_ts", pymongo.ASCENDING)
logging.debug("About to execute query %s" % {
"user_id": user_id,
"trip_id": trip_id
})
return [ecwst.Stop(doc) for doc in stop_doc_cursor]
def create_new_stop(user_id, trip_id):
_id = edb.get_stop_db().save({'user_id': user_id, "trip_id": trip_id})
logging.debug("Created new stop %s for user %s" % (_id, user_id))
return ecws.Stop({"_id": _id, 'user_id': user_id, "trip_id": trip_id})
def setUp(self):
self.testUserId = uuid.uuid4()
edb.get_stop_db().remove()
self.test_trip_id = "test_trip_id"
else:
logging.warning("No exit timestamp found, skipping")
collection.save(entry)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
"key", help="the key representing the stream that we want to fix")
parser.add_argument(
"-f",
"--filename",
help=
"a saved timeline whose local_dt needs to be fixed. If this is specified, key is ignored"
)
args = parser.parse_args()
if args.filename is not None:
fix_file(args.filename)
elif args.key == "trips":
fix_trips_or_sections(edb.get_trip_new_db())
elif args.key == "sections":
fix_trips_or_sections(edb.get_section_new_db())
elif args.key == "places":
fix_stops_or_places(edb.get_place_db())
elif args.key == "stops":
fix_stops_or_places(edb.get_stop_db())
else:
fix_timeseries(args.key)
logging.info("About to convert %s entries" % result_cursor.count())
for i, wrapper in enumerate(result_cursor):
entry = convert_wrapper_to_entry(key, wrapper)
if entry.get_id() != wrapper["_id"]:
logging.warn("entry.id = %s, wrapper.id = %s" % (entry.get_id(), wrapper["_id"]))
if i % 10000 == 0:
print "converted %s -> %s" % (wrapper, entry)
edb.get_timeseries_db().insert(entry)
collection.remove(wrapper)
def move_ts_entries(key):
tdb = edb.get_timeseries_db()
atdb = edb.get_analysis_timeseries_db()
result_cursor = tdb.find({'metadata.key': key})
logging.info("About to convert %s entries" % result_cursor.count())
for i, entry_doc in enumerate(result_cursor):
if i % 10000 == 0:
print "moved %s from one ts to the other" % (entry_doc)
atdb.insert(entry_doc)
tdb.remove(entry_doc)
if __name__ == '__main__':
# No arguments - muahahahaha. Just going to copy known fields over.
convert_collection(edb.get_trip_new_db(), "segmentation/raw_trip")
convert_collection(edb.get_place_db(), "segmentation/raw_place")
convert_collection(edb.get_section_new_db(), "segmentation/raw_section")
convert_collection(edb.get_stop_db(), "segmentation/raw_stop")
move_ts_entries("analysis/smoothing")
def get_stop(stop_id):
return ecws.Stop(edb.get_stop_db().find_one({"_id": stop_id}))
def _get_stops_for_query(stop_query, sort_key):
logging.debug("Returning stops for query %s" % stop_query)
stop_doc_cursor = edb.get_stop_db().find(stop_query).sort(
sort_key, pymongo.ASCENDING)
# TODO: Fix "TripIterator" and return it instead of this list
return [ecws.Stop(doc) for doc in stop_doc_cursor]
def save_stop(stop):
edb.get_stop_db().save(stop)
def create_new_stop(user_id, trip_id):
_id = edb.get_stop_db().save({'user_id': user_id, "trip_id": trip_id})
logging.debug("Created new stop %s for user %s" % (_id, user_id))
return ecws.Stop({"_id": _id, 'user_id': user_id, "trip_id": trip_id})
def get_stop(stop_id):
return ecws.Stop(edb.get_stop_db().find_one({"_id": stop_id}))
else:
exit_tz = "America/Los_Angeles"
logging.debug("exit metadata timezone = %s" % exit_tz)
entry['exit_local_dt'] = get_local_date(entry['exit_fmt_time'], exit_tz)
else:
logging.warning("No exit timestamp found, skipping")
collection.save(entry)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("key",
help="the key representing the stream that we want to fix")
parser.add_argument("-f", "--filename",
help="a saved timeline whose local_dt needs to be fixed. If this is specified, key is ignored")
args = parser.parse_args()
if args.filename is not None:
fix_file(args.filename)
elif args.key == "trips":
fix_trips_or_sections(edb.get_trip_new_db())
elif args.key == "sections":
fix_trips_or_sections(edb.get_section_new_db())
elif args.key == "places":
fix_stops_or_places(edb.get_place_db())
elif args.key == "stops":
fix_stops_or_places(edb.get_stop_db())
else:
fix_timeseries(args.key)
def save_stop(stop):
edb.get_stop_db().save(stop)
def _get_stops_for_query(stop_query, sort_key):
logging.debug("Returning stops for query %s" % stop_query)
stop_doc_cursor = edb.get_stop_db().find(stop_query).sort(sort_key, pymongo.ASCENDING)
# TODO: Fix "TripIterator" and return it instead of this list
return [ecws.Stop(doc) for doc in stop_doc_cursor]
def setUp(self):
self.testUserId = uuid.uuid4()
edb.get_stop_db().remove()
self.test_trip_id = "test_trip_id"