def copy_to_usercache(): # Step 1: Copy data back to user cache error_it = edb.get_timeseries_error_db().find() logging.info("Found %d errors in this round" % error_it.count()) for error in error_it: logging.debug("Copying entry %s" % error["metadata"]) save_result = edb.get_usercache_db().save(error) remove_result = edb.get_timeseries_error_db().remove(error["_id"]) logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result)) logging.info("step copy_to_usercache DONE")
def copy_to_usercache(): # Step 1: Copy data back to user cache error_it = edb.get_timeseries_error_db().find() uc = edb.get_usercache_db() te = edb.get_timeseries_error_db() logging.info("Found %d errors in this round" % error_it.count()) for error in error_it: logging.debug("Copying entry %s" % error["metadata"]) save_result = uc.save(error) remove_result = te.remove(error["_id"]) logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result)) logging.info("step copy_to_usercache DONE")
def insert_error(self, entry): """ """ logging.debug("insert_error called") if "user_id" not in entry: entry["user_id"] = self.user_id elif entry["user_id"] != self.user_id: raise AttributeError("Saving entry for %s in timeseries for %s" % (entry["user_id"], self.user_id)) else: logging.debug("entry was fine, no need to fix it") logging.debug("Inserting entry %s into error timeseries" % entry) edb.get_timeseries_error_db().insert(entry)
def testMoveToLongTerm(self): # 5 mins of data, every 30 secs = 10 entries per entry type. There are # 3 entry types, so 30 entries # First all the entries are in the usercache self.assertEqual(len(self.uc1.getMessage()), 30) self.assertEqual(len(list(self.ts1.find_entries())), 0) self.assertEqual(len(self.uc2.getMessage()), 30) self.assertEqual(len(list(self.ts2.find_entries())), 0) self.assertEqual(len(self.ucios.getMessage()), 30) self.assertEqual(len(list(self.tsios.find_entries())), 0) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() # So we end up with all user1 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30) # Then, we move entries for the ios user into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm() self.assertEqual(len(self.ucios.getMessage()), 0) self.assertEqual(len(list(self.tsios.find_entries())), 30) # 30 entries from android + 30 entries from ios = 60 self.assertEqual(edb.get_timeseries_db().find().count(), 60) self.assertEqual(edb.get_timeseries_error_db().find().count(), 0) # But all existing entries still in usercache for the second user self.assertEqual(len(self.uc2.getMessage()), 30) self.assertEqual(len(list(self.ts2.find_entries())), 0)
def testMoveToLongTerm(self): # 5 mins of data, every 30 secs = 10 entries per entry type. There are # 3 entry types, so 30 entries # First all the entries are in the usercache self.assertEqual(len(self.uc1.getMessage()), 30) self.assertEqual(len(list(self.ts1.find_entries())), 0) self.assertEqual(len(self.uc2.getMessage()), 30) self.assertEqual(len(list(self.ts2.find_entries())), 0) self.assertEqual(len(self.ucios.getMessage()), 30) self.assertEqual(len(list(self.tsios.find_entries())), 0) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() # So we end up with all user1 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30) # Then, we move entries for the ios user into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm() self.assertEqual(len(self.ucios.getMessage()), 0) self.assertEqual(len(list(self.tsios.find_entries())), 30) # 30 entries from android + 30 entries from ios = 60 self.assertEqual(edb.get_timeseries_db().estimated_document_count(), 60) self.assertEqual(edb.get_timeseries_error_db().estimated_document_count(), 0) # But all existing entries still in usercache for the second user self.assertEqual(len(self.uc2.getMessage()), 30) self.assertEqual(len(list(self.ts2.find_entries())), 0)
def testMoveWhenEmpty(self): # 5 mins of data, every 30 secs = 10 entries per entry type. There are # 3 entry types, so 30 entries # First all the entries are in the usercache self.assertEqual(len(self.uc1.getMessage()), 30) self.assertEqual(len(list(self.ts1.find_entries())), 0) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler( self.testUserUUID1).moveToLongTerm() # So we end up with all user1 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30) # Add an invalid type edb.get_usercache_db().insert_one({ 'user_id': self.testUserUUID1, '_id': boi.ObjectId('572d3621d282b8f30def7e85'), 'data': { u'transition': None, 'currState': u'STATE_ONGOING_TRIP' }, 'metadata': { 'plugin': 'none', 'write_ts': self.curr_ts - 25, 'time_zone': u'America/Los_Angeles', 'platform': u'ios', 'key': u'statemachine/transition', 'read_ts': self.curr_ts - 27, 'type': u'message' } }) # Re-run long-term for the user enuah.UserCacheHandler.getUserCacheHandler( self.testUserUUID1).moveToLongTerm() # That was stored in error_db, no errors in main body self.assertEqual( edb.get_timeseries_error_db().find({ "user_id": self.testUserUUID1 }).count(), 1) self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30)
def testMoveWhenEmpty(self): # 5 mins of data, every 30 secs = 10 entries per entry type. There are # 3 entry types, so 30 entries # First all the entries are in the usercache self.assertEqual(len(self.uc1.getMessage()), 30) self.assertEqual(len(list(self.ts1.find_entries())), 0) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() # So we end up with all user1 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30) # Add an invalid type edb.get_usercache_db().insert({ 'user_id': self.testUserUUID1, '_id': boi.ObjectId('572d3621d282b8f30def7e85'), 'data': {u'transition': None, 'currState': u'STATE_ONGOING_TRIP'}, 'metadata': {'plugin': 'none', 'write_ts': self.curr_ts - 25, 'time_zone': u'America/Los_Angeles', 'platform': u'ios', 'key': u'statemachine/transition', 'read_ts': self.curr_ts - 27, 'type': u'message'}}) # Re-run long-term for the user enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() # That was stored in error_db, no errors in main body self.assertEqual(edb.get_timeseries_error_db().find({"user_id": self.testUserUUID1}).count(), 1) self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30)
def testTwoLongTermCalls(self): # First all the entries are in the usercache self.assertEqual(len(self.uc1.getMessage()), 30) self.assertEqual(len(list(self.ts1.find_entries())), 0) self.assertEqual(len(self.ucios.getMessage()), 30) self.assertEqual(len(list(self.tsios.find_entries())), 0) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler( self.testUserUUID1).moveToLongTerm() # So we end up with all user1 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30) # Then, we move entries for the ios user into longterm enuah.UserCacheHandler.getUserCacheHandler( self.testUserUUIDios).moveToLongTerm() self.assertEqual(len(self.ucios.getMessage()), 0) self.assertEqual(len(list(self.tsios.find_entries())), 30) # 30 entries from android + 30 entries from ios = 60 self.assertEqual(edb.get_timeseries_db().find().count(), 60) self.assertEqual(edb.get_timeseries_error_db().find().count(), 0) logging.debug("old curr_ts = %d" % self.curr_ts) time.sleep(20) self.curr_ts = int(time.time()) logging.debug("new curr_ts = %d" % self.curr_ts) # Now, re-insert entries for android user1 and ios user1 for offset in range(self.curr_ts - 10, self.curr_ts): for entry in self.entry_list: entry["metadata"]["write_ts"] = offset * 1000 mauc.sync_phone_to_server(self.testUserUUID1, self.entry_list) for offset in range(self.curr_ts - 10, self.curr_ts): for entry in self.ios_entry_list: entry["metadata"]["write_ts"] = offset mauc.sync_phone_to_server(self.testUserUUIDios, self.ios_entry_list) # Now, repeat the above tests to ensure that they get moved again self.assertEqual(len(self.uc1.getMessage()), 30) # We will already have 30 entries in long-term for both android self.assertEqual(len(list(self.ts1.find_entries())), 30) self.assertEqual(len(self.ucios.getMessage()), 30) # and ios self.assertEqual(len(list(self.tsios.find_entries())), 30) # The timequery is 5 secs into the past, to avoid races # So let's sleep here for 5 secs time.sleep(5) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler( self.testUserUUID1).moveToLongTerm() # Now, we have two sets of entries, so we will have 60 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 60) # Then, we move entries for the ios user into longterm enuah.UserCacheHandler.getUserCacheHandler( self.testUserUUIDios).moveToLongTerm() self.assertEqual(len(self.ucios.getMessage()), 0) self.assertEqual(len(list(self.tsios.find_entries())), 60) # 60 entries from android + 60 entries from ios = 120 self.assertEqual(edb.get_timeseries_db().find().count(), 120) self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)
reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting client...") reset_collection(edb.get_client_db(), user.uuid, new_uuid) logging.debug("Resetting client_stats_backup...") reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting server_stats_backup...") reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting result_stats_backup...") reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_place_db...") reset_collection(edb.get_common_place_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_trip_db...") reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_habitica_db...") reset_collection(edb.get_habitica_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_pipeline_state_db...") reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_profile_db...") reset_collection(edb.get_profile_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_db...") reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_error_db...") reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_usercache_db...") reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
edb.get_uuid_db().update({"uuid" : user.uuid}, {"$set": {"uuid" : new_uuid}}) logging.debug("Resetting alternatives...") reset_collection(edb.get_alternatives_db(), user.uuid, new_uuid) logging.debug("Resetting analysis...") reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting client...") reset_collection(edb.get_client_db(), user.uuid, new_uuid) logging.debug("Resetting client_stats_backup...") reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting server_stats_backup...") reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting result_stats_backup...") reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_place_db...") reset_collection(edb.get_common_place_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_trip_db...") reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_habitica_db...") reset_collection(edb.get_habitica_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_pipeline_state_db...") reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_profile_db...") reset_collection(edb.get_profile_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_db...") reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_error_db...") reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_usercache_db...") reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
def testTwoLongTermCalls(self): # First all the entries are in the usercache self.assertEqual(len(self.uc1.getMessage()), 30) self.assertEqual(len(list(self.ts1.find_entries())), 0) self.assertEqual(len(self.ucios.getMessage()), 30) self.assertEqual(len(list(self.tsios.find_entries())), 0) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() # So we end up with all user1 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 30) # Then, we move entries for the ios user into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm() self.assertEqual(len(self.ucios.getMessage()), 0) self.assertEqual(len(list(self.tsios.find_entries())), 30) # 30 entries from android + 30 entries from ios = 60 self.assertEqual(edb.get_timeseries_db().find().count(), 60) self.assertEqual(edb.get_timeseries_error_db().find().count(), 0) logging.debug("old curr_ts = %d" % self.curr_ts) time.sleep(20) self.curr_ts = int(time.time()) logging.debug("new curr_ts = %d" % self.curr_ts) # Now, re-insert entries for android user1 and ios user1 for offset in range(self.curr_ts - 10, self.curr_ts): for entry in self.entry_list: entry["metadata"]["write_ts"] = offset * 1000 mauc.sync_phone_to_server(self.testUserUUID1, self.entry_list) for offset in range(self.curr_ts - 10, self.curr_ts): for entry in self.ios_entry_list: entry["metadata"]["write_ts"] = offset mauc.sync_phone_to_server(self.testUserUUIDios, self.ios_entry_list) # Now, repeat the above tests to ensure that they get moved again self.assertEqual(len(self.uc1.getMessage()), 30) # We will already have 30 entries in long-term for both android self.assertEqual(len(list(self.ts1.find_entries())), 30) self.assertEqual(len(self.ucios.getMessage()), 30) # and ios self.assertEqual(len(list(self.tsios.find_entries())), 30) # The timequery is 5 secs into the past, to avoid races # So let's sleep here for 5 secs time.sleep(5) # Then we move entries for user1 into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm() # Now, we have two sets of entries, so we will have 60 entries in longterm self.assertEqual(len(self.uc1.getMessage()), 0) self.assertEqual(len(list(self.ts1.find_entries())), 60) # Then, we move entries for the ios user into longterm enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm() self.assertEqual(len(self.ucios.getMessage()), 0) self.assertEqual(len(list(self.tsios.find_entries())), 60) # 60 entries from android + 60 entries from ios = 120 self.assertEqual(edb.get_timeseries_db().find().count(), 120) self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)