def get_rows(self):
cursor = self._connection.cursor()
count_q = 'SELECT count(*) FROM %s;' % (self._table,)
cursor.execute(count_q)
row_count = next(cursor)[0]
consume(cursor)
select_q = 'SELECT * FROM %s;' % (self._table,)
cursor.execute(select_q)
def _inner():
for row in cursor:
doc = self._build_doc(row)
action = self._build_action(doc)
yield self._process_action(action)
return count_every(_inner(), total=row_count)
def read_and_index():
json_iter = imap(loads, count_every(stdin, 10000))
bulk(es, json_iter, raise_on_error = True)
