def test():
db = connect_db()
data = db['topics'].find_one()
data = data['words']
data = sorted(data, key = lambda x: x['contribution'], reverse = True)
make_demo(data)
return
def __init__(self, s_time, e_time, source_type, num=-1):
'''
s_time: start time
e_time: end time
source_type: list of sources
'''
db = connect_db()
self.cursors = [ db[t].find({'timestamp': {'$gte': s_time, '$lte': e_time}}) for t in source_type]
self.limit = num
return
def __init__(self, s_time, e_time, source_type, keywords, num=-1):
'''
s_time: start time
e_time: end time
source_type: list of sources
'''
db = connect_db()
if keywords == []:
self.cursors = [ db[t].find({'timestamp': {'$gte': s_time, '$lte': e_time}}, timeout=True) for t in source_type]
else:
self.cursors = [ db[t].find({'timestamp': {'$gte': s_time, '$lte': e_time}, 'tokens': {
'$all': keywords}}, timeout=True) for t in source_type]
self.limit = num
return
def __init__(self, s_time, e_time, source_type, num=-1):
'''
s_time: start time
e_time: end time
source_type: list of sources
'''
db = connect_db()
self.cursors = [
db[t].find({'timestamp': {
'$gte': s_time,
'$lte': e_time
}}) for t in source_type
]
self.limit = num
return
def save_model(model):
'''
Given a model, save it to the database
'''
db = connect_db()
for t in model.show_topics(model.num_topics, num_words=100, formatted=False):
topic = {}
topic['words'] = []
topic['period'] ={
'start_time':START_TIME,
'end_time': END_TIME
}
topic['keys'] = KEYS
topic['sources'] = SOURCES
for i in t:
topic['words'].append({'contribution': i[0], 'token': i[1]})
db['topics'].insert(topic)
logging.info('have saved the topics into the database')
return
def __init__(self):
self.new_operation = list()
timer = 1
while True:
try:
self.conn = connect_db()
break
except Exception:
time.sleep(5)
# TODO: may need to restart mysqld
if timer >= 10:
break
timer += 1
# initialize paths
# LOG.info('Initializing paths ...')
for item in NEW_PATHS:
if not os.path.exists(item):
os.makedirs(item)
# get driver
# LOG.info('Initializing drivers ...')
self.driver = get_webdriver()
