def copy_to_usercache():
# Step 1: Copy data back to user cache
error_it = edb.get_timeseries_error_db().find()
logging.info("Found %d errors in this round" % error_it.count())
for error in error_it:
logging.debug("Copying entry %s" % error["metadata"])
save_result = edb.get_usercache_db().save(error)
remove_result = edb.get_timeseries_error_db().remove(error["_id"])
logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result))
logging.info("step copy_to_usercache DONE")
def copy_to_usercache():
# Step 1: Copy data back to user cache
error_it = edb.get_timeseries_error_db().find()
uc = edb.get_usercache_db()
te = edb.get_timeseries_error_db()
logging.info("Found %d errors in this round" % error_it.count())
for error in error_it:
logging.debug("Copying entry %s" % error["metadata"])
save_result = uc.save(error)
remove_result = te.remove(error["_id"])
logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result))
logging.info("step copy_to_usercache DONE")
def insert_error(self, entry):
"""
"""
logging.debug("insert_error called")
if "user_id" not in entry:
entry["user_id"] = self.user_id
elif entry["user_id"] != self.user_id:
raise AttributeError("Saving entry for %s in timeseries for %s" % (entry["user_id"], self.user_id))
else:
logging.debug("entry was fine, no need to fix it")
logging.debug("Inserting entry %s into error timeseries" % entry)
edb.get_timeseries_error_db().insert(entry)
def testMoveToLongTerm(self):
# 5 mins of data, every 30 secs = 10 entries per entry type. There are
# 3 entry types, so 30 entries
# First all the entries are in the usercache
self.assertEqual(len(self.uc1.getMessage()), 30)
self.assertEqual(len(list(self.ts1.find_entries())), 0)
self.assertEqual(len(self.uc2.getMessage()), 30)
self.assertEqual(len(list(self.ts2.find_entries())), 0)
self.assertEqual(len(self.ucios.getMessage()), 30)
self.assertEqual(len(list(self.tsios.find_entries())), 0)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm()
# So we end up with all user1 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
# Then, we move entries for the ios user into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm()
self.assertEqual(len(self.ucios.getMessage()), 0)
self.assertEqual(len(list(self.tsios.find_entries())), 30)
# 30 entries from android + 30 entries from ios = 60
self.assertEqual(edb.get_timeseries_db().find().count(), 60)
self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)
# But all existing entries still in usercache for the second user
self.assertEqual(len(self.uc2.getMessage()), 30)
self.assertEqual(len(list(self.ts2.find_entries())), 0)
def testMoveToLongTerm(self):
# 5 mins of data, every 30 secs = 10 entries per entry type. There are
# 3 entry types, so 30 entries
# First all the entries are in the usercache
self.assertEqual(len(self.uc1.getMessage()), 30)
self.assertEqual(len(list(self.ts1.find_entries())), 0)
self.assertEqual(len(self.uc2.getMessage()), 30)
self.assertEqual(len(list(self.ts2.find_entries())), 0)
self.assertEqual(len(self.ucios.getMessage()), 30)
self.assertEqual(len(list(self.tsios.find_entries())), 0)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm()
# So we end up with all user1 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
# Then, we move entries for the ios user into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm()
self.assertEqual(len(self.ucios.getMessage()), 0)
self.assertEqual(len(list(self.tsios.find_entries())), 30)
# 30 entries from android + 30 entries from ios = 60
self.assertEqual(edb.get_timeseries_db().estimated_document_count(), 60)
self.assertEqual(edb.get_timeseries_error_db().estimated_document_count(), 0)
# But all existing entries still in usercache for the second user
self.assertEqual(len(self.uc2.getMessage()), 30)
self.assertEqual(len(list(self.ts2.find_entries())), 0)
def testMoveWhenEmpty(self):
# 5 mins of data, every 30 secs = 10 entries per entry type. There are
# 3 entry types, so 30 entries
# First all the entries are in the usercache
self.assertEqual(len(self.uc1.getMessage()), 30)
self.assertEqual(len(list(self.ts1.find_entries())), 0)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(
self.testUserUUID1).moveToLongTerm()
# So we end up with all user1 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
# Add an invalid type
edb.get_usercache_db().insert_one({
'user_id':
self.testUserUUID1,
'_id':
boi.ObjectId('572d3621d282b8f30def7e85'),
'data': {
u'transition': None,
'currState': u'STATE_ONGOING_TRIP'
},
'metadata': {
'plugin': 'none',
'write_ts': self.curr_ts - 25,
'time_zone': u'America/Los_Angeles',
'platform': u'ios',
'key': u'statemachine/transition',
'read_ts': self.curr_ts - 27,
'type': u'message'
}
})
# Re-run long-term for the user
enuah.UserCacheHandler.getUserCacheHandler(
self.testUserUUID1).moveToLongTerm()
# That was stored in error_db, no errors in main body
self.assertEqual(
edb.get_timeseries_error_db().find({
"user_id": self.testUserUUID1
}).count(), 1)
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
def testMoveWhenEmpty(self):
# 5 mins of data, every 30 secs = 10 entries per entry type. There are
# 3 entry types, so 30 entries
# First all the entries are in the usercache
self.assertEqual(len(self.uc1.getMessage()), 30)
self.assertEqual(len(list(self.ts1.find_entries())), 0)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm()
# So we end up with all user1 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
# Add an invalid type
edb.get_usercache_db().insert({
'user_id': self.testUserUUID1,
'_id': boi.ObjectId('572d3621d282b8f30def7e85'),
'data': {u'transition': None,
'currState': u'STATE_ONGOING_TRIP'},
'metadata': {'plugin': 'none',
'write_ts': self.curr_ts - 25,
'time_zone': u'America/Los_Angeles',
'platform': u'ios',
'key': u'statemachine/transition',
'read_ts': self.curr_ts - 27,
'type': u'message'}})
# Re-run long-term for the user
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm()
# That was stored in error_db, no errors in main body
self.assertEqual(edb.get_timeseries_error_db().find({"user_id": self.testUserUUID1}).count(), 1)
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
def testTwoLongTermCalls(self):
# First all the entries are in the usercache
self.assertEqual(len(self.uc1.getMessage()), 30)
self.assertEqual(len(list(self.ts1.find_entries())), 0)
self.assertEqual(len(self.ucios.getMessage()), 30)
self.assertEqual(len(list(self.tsios.find_entries())), 0)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(
self.testUserUUID1).moveToLongTerm()
# So we end up with all user1 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
# Then, we move entries for the ios user into longterm
enuah.UserCacheHandler.getUserCacheHandler(
self.testUserUUIDios).moveToLongTerm()
self.assertEqual(len(self.ucios.getMessage()), 0)
self.assertEqual(len(list(self.tsios.find_entries())), 30)
# 30 entries from android + 30 entries from ios = 60
self.assertEqual(edb.get_timeseries_db().find().count(), 60)
self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)
logging.debug("old curr_ts = %d" % self.curr_ts)
time.sleep(20)
self.curr_ts = int(time.time())
logging.debug("new curr_ts = %d" % self.curr_ts)
# Now, re-insert entries for android user1 and ios user1
for offset in range(self.curr_ts - 10, self.curr_ts):
for entry in self.entry_list:
entry["metadata"]["write_ts"] = offset * 1000
mauc.sync_phone_to_server(self.testUserUUID1, self.entry_list)
for offset in range(self.curr_ts - 10, self.curr_ts):
for entry in self.ios_entry_list:
entry["metadata"]["write_ts"] = offset
mauc.sync_phone_to_server(self.testUserUUIDios,
self.ios_entry_list)
# Now, repeat the above tests to ensure that they get moved again
self.assertEqual(len(self.uc1.getMessage()), 30)
# We will already have 30 entries in long-term for both android
self.assertEqual(len(list(self.ts1.find_entries())), 30)
self.assertEqual(len(self.ucios.getMessage()), 30)
# and ios
self.assertEqual(len(list(self.tsios.find_entries())), 30)
# The timequery is 5 secs into the past, to avoid races
# So let's sleep here for 5 secs
time.sleep(5)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(
self.testUserUUID1).moveToLongTerm()
# Now, we have two sets of entries, so we will have 60 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 60)
# Then, we move entries for the ios user into longterm
enuah.UserCacheHandler.getUserCacheHandler(
self.testUserUUIDios).moveToLongTerm()
self.assertEqual(len(self.ucios.getMessage()), 0)
self.assertEqual(len(list(self.tsios.find_entries())), 60)
# 60 entries from android + 60 entries from ios = 120
self.assertEqual(edb.get_timeseries_db().find().count(), 120)
self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)
reset_collection(edb.get_analysis_timeseries_db(), user.uuid,
new_uuid)
logging.debug("Resetting client...")
reset_collection(edb.get_client_db(), user.uuid, new_uuid)
logging.debug("Resetting client_stats_backup...")
reset_collection(edb.get_client_stats_db_backup(), user.uuid,
new_uuid)
logging.debug("Resetting server_stats_backup...")
reset_collection(edb.get_server_stats_db_backup(), user.uuid,
new_uuid)
logging.debug("Resetting result_stats_backup...")
reset_collection(edb.get_result_stats_db_backup(), user.uuid,
new_uuid)
logging.debug("Resetting edb.get_common_place_db...")
reset_collection(edb.get_common_place_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_common_trip_db...")
reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_habitica_db...")
reset_collection(edb.get_habitica_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_pipeline_state_db...")
reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_profile_db...")
reset_collection(edb.get_profile_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_db...")
reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_error_db...")
reset_collection(edb.get_timeseries_error_db(), user.uuid,
new_uuid)
logging.debug("Resetting edb.get_usercache_db...")
reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
edb.get_uuid_db().update({"uuid" : user.uuid},
{"$set": {"uuid" : new_uuid}})
logging.debug("Resetting alternatives...")
reset_collection(edb.get_alternatives_db(), user.uuid, new_uuid)
logging.debug("Resetting analysis...")
reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid)
logging.debug("Resetting client...")
reset_collection(edb.get_client_db(), user.uuid, new_uuid)
logging.debug("Resetting client_stats_backup...")
reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid)
logging.debug("Resetting server_stats_backup...")
reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid)
logging.debug("Resetting result_stats_backup...")
reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_common_place_db...")
reset_collection(edb.get_common_place_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_common_trip_db...")
reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_habitica_db...")
reset_collection(edb.get_habitica_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_pipeline_state_db...")
reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_profile_db...")
reset_collection(edb.get_profile_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_db...")
reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_error_db...")
reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_usercache_db...")
reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
def testTwoLongTermCalls(self):
# First all the entries are in the usercache
self.assertEqual(len(self.uc1.getMessage()), 30)
self.assertEqual(len(list(self.ts1.find_entries())), 0)
self.assertEqual(len(self.ucios.getMessage()), 30)
self.assertEqual(len(list(self.tsios.find_entries())), 0)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm()
# So we end up with all user1 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 30)
# Then, we move entries for the ios user into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm()
self.assertEqual(len(self.ucios.getMessage()), 0)
self.assertEqual(len(list(self.tsios.find_entries())), 30)
# 30 entries from android + 30 entries from ios = 60
self.assertEqual(edb.get_timeseries_db().find().count(), 60)
self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)
logging.debug("old curr_ts = %d" % self.curr_ts)
time.sleep(20)
self.curr_ts = int(time.time())
logging.debug("new curr_ts = %d" % self.curr_ts)
# Now, re-insert entries for android user1 and ios user1
for offset in range(self.curr_ts - 10, self.curr_ts):
for entry in self.entry_list:
entry["metadata"]["write_ts"] = offset * 1000
mauc.sync_phone_to_server(self.testUserUUID1, self.entry_list)
for offset in range(self.curr_ts - 10, self.curr_ts):
for entry in self.ios_entry_list:
entry["metadata"]["write_ts"] = offset
mauc.sync_phone_to_server(self.testUserUUIDios, self.ios_entry_list)
# Now, repeat the above tests to ensure that they get moved again
self.assertEqual(len(self.uc1.getMessage()), 30)
# We will already have 30 entries in long-term for both android
self.assertEqual(len(list(self.ts1.find_entries())), 30)
self.assertEqual(len(self.ucios.getMessage()), 30)
# and ios
self.assertEqual(len(list(self.tsios.find_entries())), 30)
# The timequery is 5 secs into the past, to avoid races
# So let's sleep here for 5 secs
time.sleep(5)
# Then we move entries for user1 into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUID1).moveToLongTerm()
# Now, we have two sets of entries, so we will have 60 entries in longterm
self.assertEqual(len(self.uc1.getMessage()), 0)
self.assertEqual(len(list(self.ts1.find_entries())), 60)
# Then, we move entries for the ios user into longterm
enuah.UserCacheHandler.getUserCacheHandler(self.testUserUUIDios).moveToLongTerm()
self.assertEqual(len(self.ucios.getMessage()), 0)
self.assertEqual(len(list(self.tsios.find_entries())), 60)
# 60 entries from android + 60 entries from ios = 120
self.assertEqual(edb.get_timeseries_db().find().count(), 120)
self.assertEqual(edb.get_timeseries_error_db().find().count(), 0)
