def main():
max_id = str(args.max_id)
logger.info("MAX:{}".format(max_id))
query = "select item_id, text from " + sqlconfig.tweet_table_name + \
" where id <= " + max_id + " and success = 1"
logger.info("SQL running...")
start = time.time()
rows = read_table(query)
elapsed_time = time.time() - start
logger.info("sql_time:{0}[sec]".format(elapsed_time))
logger.info("Indexing...")
start = time.time()
p = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(rows)).start()
indexer = Indexer()
for i, row in enumerate(rows):
indexer.add(row[0], noun_list(row[1]))
p.update(i + 1)
p.finish()
elapsed_time = time.time() - start
logger.info("indexing_time:{0}[sec]".format(elapsed_time))
indexer.save("./index.pkl")
def get_and_set_tweet(self, table_name, item_id):
query = 'select * from {0} where item_id = {1}'.format(table_name, item_id)
rows = read_table(query)
if len(rows) is 1:
self.set_tweet(rows[0])
return True
else:
return False
def save_index(file_path):
query = get_query(args.only_reply)
logger.info('query: {}'.format(query))
logger.info("SQL running...")
start = time.time()
rows = read_table(query)
elapsed_time = time.time() - start
logger.info("sql_time:{0}[sec]".format(elapsed_time))
logger.info("Indexing...")
start = time.time()
p = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(rows)).start()
indexer = Indexer()
for i, row in enumerate(rows):
indexer.add(row[0], noun_list(row[1]))
p.update(i + 1)
p.finish()
elapsed_time = time.time() - start
logger.info("indexing_time:{0}[sec]".format(elapsed_time))
logger.info("Saving...")
indexer.save(file_path)
logger.info('Done')
def test_sql_to_table(self):
query = "select * from " + sqlconfig.tweet_table_name + " where id <= 100"
rows = read_table(query)
eq_(len(rows), 100)
