def maintain(self):
from webscraper.entities import User
users = User.get_active_users()
#mongodb can have this limit on IN case
MAX_IN_LIMIT = 4000000
#count total deletes
tot = 0
#get celery_db connection
cl = CustomAppSettings.get_mongo()
celery_db = cl.celery_db
# remove old search results from each db
for user in users:
#get old task ids
old_task_ids = user.get_old_task_ids()[:MAX_IN_LIMIT]
count = len(old_task_ids)
tot += count
#db
db = user.get_user_db()
#remove old userdb.task_meta
db.task_meta.remove({'_id': {'$in': old_task_ids}})
#remove old userdb.urls
db.urls.remove({'task_id':{'$in': old_task_ids}})
#remove old celerydb.task_meta
celery_db.task_meta.remove({'_id':{'$in': old_task_ids}})
#write user profile statistics on removed
db = user.get_user_profiles_db()
col = db.cleaning_stats
d = datetime.datetime.utcnow()
doc = {'user': user.get_user_id(), 'date_done': d, 'total_removed': count}
col.insert(doc)
return {'total': tot}
