def main(): max_id = str(args.max_id) logger.info("MAX:{}".format(max_id)) query = "select item_id, text from " + sqlconfig.tweet_table_name + \ " where id <= " + max_id + " and success = 1" logger.info("SQL running...") start = time.time() rows = read_table(query) elapsed_time = time.time() - start logger.info("sql_time:{0}[sec]".format(elapsed_time)) logger.info("Indexing...") start = time.time() p = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(rows)).start() indexer = Indexer() for i, row in enumerate(rows): indexer.add(row[0], noun_list(row[1])) p.update(i + 1) p.finish() elapsed_time = time.time() - start logger.info("indexing_time:{0}[sec]".format(elapsed_time)) indexer.save("./index.pkl")
def get_and_set_tweet(self, table_name, item_id): query = 'select * from {0} where item_id = {1}'.format(table_name, item_id) rows = read_table(query) if len(rows) is 1: self.set_tweet(rows[0]) return True else: return False
def save_index(file_path): query = get_query(args.only_reply) logger.info('query: {}'.format(query)) logger.info("SQL running...") start = time.time() rows = read_table(query) elapsed_time = time.time() - start logger.info("sql_time:{0}[sec]".format(elapsed_time)) logger.info("Indexing...") start = time.time() p = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(rows)).start() indexer = Indexer() for i, row in enumerate(rows): indexer.add(row[0], noun_list(row[1])) p.update(i + 1) p.finish() elapsed_time = time.time() - start logger.info("indexing_time:{0}[sec]".format(elapsed_time)) logger.info("Saving...") indexer.save(file_path) logger.info('Done')
def test_sql_to_table(self): query = "select * from " + sqlconfig.tweet_table_name + " where id <= 100" rows = read_table(query) eq_(len(rows), 100)