def history_delete():
"""
Delete searches/sessions/tweets
@searches = list of search ids
@sessions = list of session ids
"""
try:
search_ids = request.args.getlist('searches[]')
session_ids = set(request.args.getlist('sessions[]'))
cursor = _session.find({'search_id': {'$in': search_ids}}, {'_id': 1})
session_ids.update([str(r['_id']) for r in cursor])
session_ids = list(session_ids)
_tweets.remove(
{'session_id': {'$in': session_ids}})
_session.remove(
{'_id': {'$in': [bson.ObjectId(x) for x in session_ids]}})
_search.remove(
{'_id': {'$in': [bson.ObjectId(x) for x in search_ids]}})
return _jsonify(deleted=session_ids)
except Exception, e:
traceback.print_exc()
return _jsonify(error=str(e))
def history_delete():
"""
Delete searches/sessions/tweets
@searches = list of search ids
@sessions = list of session ids
"""
try:
search_ids = request.args.getlist('searches[]')
session_ids = set(request.args.getlist('sessions[]'))
cursor = _session.find({'search_id': {'$in': search_ids}}, {'_id': 1})
session_ids.update([str(r['_id']) for r in cursor])
session_ids = list(session_ids)
_tweets.remove(
{'session_id': {'$in': session_ids}})
_session.remove(
{'_id': {'$in': [bson.ObjectId(x) for x in session_ids]}})
_search.remove(
{'_id': {'$in': [bson.ObjectId(x) for x in search_ids]}})
return _jsonify(deleted=session_ids)
except Exception, e:
traceback.print_exc()
return _jsonify(error=str(e))
def clean_database(n_days):
"""Clean database"""
print 'settings: %s' % settings_module
print 'Clean database: %d days' % n_days
dt = datetime.datetime.utcnow() - datetime.timedelta(days=n_days)
dt_str = dt.isoformat()
print 'Removing sessions before %s' % dt_str
removed = 0
session_list = list(
_session.find({
'saved': {
'$ne': 1
},
'dt': {
'$lt': dt_str
}
}, {
'saved': 1,
'dt': 1
}))
for session_r in session_list:
# Delete tweets associated with session
_tweets.remove({'session_id': str(session_r['_id'])})
# Delete session
_session.remove({'_id': session_r['_id']})
removed += 1
if (removed % 100) == 0:
print '...removed %d sessions...' % removed
print 'Removed %d sessions' % removed
print 'Removing orphan searches'
removed = 0
search_id_list = _session.distinct('search_id')
for search_id in _search.distinct('_id'):
if not str(search_id) in search_id_list:
_search.remove({'_id': search_id})
removed += 1
if (removed % 100) == 0:
print '...removed %d sessions...' % removed
print 'Removed %d orphan searches' % removed
def _get_saved_results(params=None):
"""
Get saved results matching params, grouped by search
"""
search_by_query = []
search_by_list = []
list_map = _get_list_map()
search_q = {'username': session['username']}
search_q.update(params or {})
search_cursor = _search.find(search_q, sort=[('query', pymongo.ASCENDING)])
for search_r in search_cursor:
search_r['_id'] = str(search_r['_id'])
search_r['sessions'] = []
if 'list_id' in search_r:
search_r['list_name'] = list_map.get(search_r['list_id']) \
or search_r.get('list_name') \
or '[unknown]'
session_cursor = _session.find(
{'search_id': search_r['_id'], 'saved': 1},
fields=['_id', 'dt', 'shared'],
sort=[('dt', pymongo.DESCENDING)]
)
for session_r in session_cursor:
session_r['_id'] = str(session_r['_id'])
session_r['dt'] = datetime.datetime \
.strptime(session_r['dt'], '%Y-%m-%dT%H:%M:%S.%f') \
.strftime('%b %d %Y %H:%M:%S')
search_r['sessions'].append(session_r)
if search_r['sessions']:
if 'list_id' in search_r:
search_by_list.append(search_r)
else:
search_by_query.append(search_r)
return (search_by_query, search_by_list)
def _get_saved_results(params=None):
"""
Get saved results matching params, grouped by search
"""
search_by_query = []
search_by_list = []
list_map = _get_list_map()
search_q = {'username': session['username']}
search_q.update(params or {})
search_cursor = _search.find(search_q, sort=[('query', pymongo.ASCENDING)])
for search_r in search_cursor:
search_r['_id'] = str(search_r['_id'])
search_r['sessions'] = []
if 'list_id' in search_r:
search_r['list_name'] = list_map.get(search_r['list_id']) \
or search_r.get('list_name') \
or '[unknown]'
session_cursor = _session.find(
{'search_id': search_r['_id'], 'saved': 1},
fields=['_id', 'dt', 'shared'],
sort=[('dt', pymongo.DESCENDING)]
)
for session_r in session_cursor:
session_r['_id'] = str(session_r['_id'])
session_r['dt'] = datetime.datetime \
.strptime(session_r['dt'], '%Y-%m-%dT%H:%M:%S.%f') \
.strftime('%b %d %Y %H:%M:%S')
search_r['sessions'].append(session_r)
if search_r['sessions']:
if 'list_id' in search_r:
search_by_list.append(search_r)
else:
search_by_query.append(search_r)
return (search_by_query, search_by_list)
def clean_database(n_days):
"""Clean database"""
print 'settings: %s' % settings_module
print 'Clean database: %d days' % n_days
dt = datetime.datetime.utcnow() - datetime.timedelta(days=n_days)
dt_str = dt.isoformat()
print 'Removing sessions before %s' % dt_str
removed = 0
session_list = list(_session.find(
{'saved': {'$ne': 1}, 'dt': {'$lt': dt_str}},
{'saved': 1, 'dt': 1}
))
for session_r in session_list:
# Delete tweets associated with session
_tweets.remove({'session_id': str(session_r['_id'])})
# Delete session
_session.remove({'_id': session_r['_id']})
removed += 1
if (removed % 100) == 0:
print '...removed %d sessions...' % removed
print 'Removed %d sessions' % removed
print 'Removing orphan searches'
removed = 0
search_id_list = _session.distinct('search_id')
for search_id in _search.distinct('_id'):
if not str(search_id) in search_id_list:
_search.remove({'_id': search_id})
removed += 1
if (removed % 100) == 0:
print '...removed %d sessions...' % removed
print 'Removed %d orphan searches' % removed
