def _get_query_cell_executions_iter(batch_size=1000, session=None):
offset = 0
while True:
query_cells = get_all_query_cells(
limit=batch_size,
offset=offset,
session=session,
)
query_executions_count = 0
for query_cell in query_cells:
query_cell_executions_offset = 0
while True:
query_cell_executions = get_successful_query_executions_by_data_cell_id(
query_cell.id,
limit=batch_size,
offset=query_cell_executions_offset,
session=session,
)
for query_execution in query_cell_executions:
expand_query_execution = query_execution_to_es(
query_execution,
data_cell=query_cell,
session=session,
)
yield expand_query_execution
query_executions_count += len(query_cell_executions)
if len(query_cell_executions) < batch_size:
break
query_cell_executions_offset += batch_size
LOG.info(
"\n--Query cell count: {}, query cell executions count: {}, offset: {}"
.format(len(query_cells), query_executions_count, offset))
if len(query_cells) < batch_size:
break
offset += batch_size
def get_query_cells_iter(batch_size=1000, session=None):
offset = 0
while True:
query_cells = get_all_query_cells(
limit=batch_size,
offset=offset,
session=session,
)
LOG.info("\n--Query cells count: {}, offset: {}".format(
len(query_cells), offset))
for query_cell in query_cells:
expand_query_cell = query_cell_to_es(query_cell, session=session)
yield expand_query_cell
if len(query_cells) < batch_size:
break
offset += batch_size