def clearRelatedDb(self): edb.get_timeseries_db().remove() edb.get_place_db().remove() edb.get_stop_db().remove() edb.get_trip_new_db().remove() edb.get_section_new_db().remove()
def del_objects(args): del_query = {} if args.user_id != "all": del_query['user_id'] = uuid.UUID(args.user_id) if args.date is None: trip_query = del_query place_query = del_query else: day_dt = pydt.datetime.strptime(args.date, "%Y-%m-%d") logging.debug("day_dt is %s" % day_dt) day_ts = time.mktime(day_dt.timetuple()) logging.debug("day_ts is %s" % day_ts) trip_query = copy.copy(del_query) trip_query.update({"start_ts": {"$gt": day_ts}}) place_query = copy.copy(del_query) place_query.update({"exit_ts": {"$gt": day_ts}}) print "trip_query = %s" % trip_query print "place_query = %s" % place_query # Since sections have the same basic structure as trips and stops have the # same basic structure as places, we can reuse the queries print "Deleting trips for %s after %s" % (args.user_id, args.date) print edb.get_trip_new_db().remove(trip_query) print "Deleting sections for %s after %s" % (args.user_id, args.date) print edb.get_section_new_db().remove(trip_query) print "Deleting places for %s after %s" % (args.user_id, args.date) print edb.get_place_db().remove(place_query) print "Deleting stops for %s after %s" % (args.user_id, args.date) print edb.get_stop_db().remove(place_query)
def clearRelatedDb(self): edb.get_timeseries_db().remove({'user_id': self.testUUID}) edb.get_place_db().remove({'user_id': self.testUUID}) edb.get_stop_db().remove({'user_id': self.testUUID}) edb.get_trip_new_db().remove({'user_id': self.testUUID}) edb.get_section_new_db().remove({'user_id': self.testUUID})
def testSaveStop(self): new_stop = esds.create_new_stop(self.testUserId, self.test_trip_id) new_stop.enter_ts = 5 new_stop.exit_ts = 6 esds.save_stop(new_stop) self.assertEqual(edb.get_stop_db().find({"exit_ts": 6}).count(), 1) self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["_id"], new_stop.get_id()) self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["user_id"], self.testUserId) self.assertEqual(edb.get_stop_db().find_one({"exit_ts": 6})["trip_id"], self.test_trip_id)
def get_stops_for_trip(user_id, trip_id): """ Get the set of sections that are children of this trip. """ stop_doc_cursor = edb.get_stop_db().find({"user_id": user_id, "trip_id": trip_id}).sort("enter_ts", pymongo.ASCENDING) logging.debug("About to execute query %s" % {"user_id": user_id, "trip_id": trip_id}) return [ecwst.Stop(doc) for doc in stop_doc_cursor]
def get_stops_for_trip(user_id, trip_id): """ Get the set of sections that are children of this trip. """ stop_doc_cursor = edb.get_stop_db().find({ "user_id": user_id, "trip_id": trip_id }).sort("enter_ts", pymongo.ASCENDING) logging.debug("About to execute query %s" % { "user_id": user_id, "trip_id": trip_id }) return [ecwst.Stop(doc) for doc in stop_doc_cursor]
def create_new_stop(user_id, trip_id): _id = edb.get_stop_db().save({'user_id': user_id, "trip_id": trip_id}) logging.debug("Created new stop %s for user %s" % (_id, user_id)) return ecws.Stop({"_id": _id, 'user_id': user_id, "trip_id": trip_id})
def setUp(self): self.testUserId = uuid.uuid4() edb.get_stop_db().remove() self.test_trip_id = "test_trip_id"
else: logging.warning("No exit timestamp found, skipping") collection.save(entry) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( "key", help="the key representing the stream that we want to fix") parser.add_argument( "-f", "--filename", help= "a saved timeline whose local_dt needs to be fixed. If this is specified, key is ignored" ) args = parser.parse_args() if args.filename is not None: fix_file(args.filename) elif args.key == "trips": fix_trips_or_sections(edb.get_trip_new_db()) elif args.key == "sections": fix_trips_or_sections(edb.get_section_new_db()) elif args.key == "places": fix_stops_or_places(edb.get_place_db()) elif args.key == "stops": fix_stops_or_places(edb.get_stop_db()) else: fix_timeseries(args.key)
logging.info("About to convert %s entries" % result_cursor.count()) for i, wrapper in enumerate(result_cursor): entry = convert_wrapper_to_entry(key, wrapper) if entry.get_id() != wrapper["_id"]: logging.warn("entry.id = %s, wrapper.id = %s" % (entry.get_id(), wrapper["_id"])) if i % 10000 == 0: print "converted %s -> %s" % (wrapper, entry) edb.get_timeseries_db().insert(entry) collection.remove(wrapper) def move_ts_entries(key): tdb = edb.get_timeseries_db() atdb = edb.get_analysis_timeseries_db() result_cursor = tdb.find({'metadata.key': key}) logging.info("About to convert %s entries" % result_cursor.count()) for i, entry_doc in enumerate(result_cursor): if i % 10000 == 0: print "moved %s from one ts to the other" % (entry_doc) atdb.insert(entry_doc) tdb.remove(entry_doc) if __name__ == '__main__': # No arguments - muahahahaha. Just going to copy known fields over. convert_collection(edb.get_trip_new_db(), "segmentation/raw_trip") convert_collection(edb.get_place_db(), "segmentation/raw_place") convert_collection(edb.get_section_new_db(), "segmentation/raw_section") convert_collection(edb.get_stop_db(), "segmentation/raw_stop") move_ts_entries("analysis/smoothing")
def get_stop(stop_id): return ecws.Stop(edb.get_stop_db().find_one({"_id": stop_id}))
def _get_stops_for_query(stop_query, sort_key): logging.debug("Returning stops for query %s" % stop_query) stop_doc_cursor = edb.get_stop_db().find(stop_query).sort( sort_key, pymongo.ASCENDING) # TODO: Fix "TripIterator" and return it instead of this list return [ecws.Stop(doc) for doc in stop_doc_cursor]
def save_stop(stop): edb.get_stop_db().save(stop)
else: exit_tz = "America/Los_Angeles" logging.debug("exit metadata timezone = %s" % exit_tz) entry['exit_local_dt'] = get_local_date(entry['exit_fmt_time'], exit_tz) else: logging.warning("No exit timestamp found, skipping") collection.save(entry) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("key", help="the key representing the stream that we want to fix") parser.add_argument("-f", "--filename", help="a saved timeline whose local_dt needs to be fixed. If this is specified, key is ignored") args = parser.parse_args() if args.filename is not None: fix_file(args.filename) elif args.key == "trips": fix_trips_or_sections(edb.get_trip_new_db()) elif args.key == "sections": fix_trips_or_sections(edb.get_section_new_db()) elif args.key == "places": fix_stops_or_places(edb.get_place_db()) elif args.key == "stops": fix_stops_or_places(edb.get_stop_db()) else: fix_timeseries(args.key)
def _get_stops_for_query(stop_query, sort_key): logging.debug("Returning stops for query %s" % stop_query) stop_doc_cursor = edb.get_stop_db().find(stop_query).sort(sort_key, pymongo.ASCENDING) # TODO: Fix "TripIterator" and return it instead of this list return [ecws.Stop(doc) for doc in stop_doc_cursor]