def get_rows(self): cursor = self._connection.cursor() count_q = 'SELECT count(*) FROM %s;' % (self._table,) cursor.execute(count_q) row_count = next(cursor)[0] consume(cursor) select_q = 'SELECT * FROM %s;' % (self._table,) cursor.execute(select_q) def _inner(): for row in cursor: doc = self._build_doc(row) action = self._build_action(doc) yield self._process_action(action) return count_every(_inner(), total=row_count)
def read_and_index(): json_iter = imap(loads, count_every(stdin, 10000)) bulk(es, json_iter, raise_on_error = True)