def _get_query_cell_executions_iter(batch_size=1000, session=None): offset = 0 while True: query_cells = get_all_query_cells( limit=batch_size, offset=offset, session=session, ) query_executions_count = 0 for query_cell in query_cells: query_cell_executions_offset = 0 while True: query_cell_executions = get_successful_query_executions_by_data_cell_id( query_cell.id, limit=batch_size, offset=query_cell_executions_offset, session=session, ) for query_execution in query_cell_executions: expand_query_execution = query_execution_to_es( query_execution, data_cell=query_cell, session=session, ) yield expand_query_execution query_executions_count += len(query_cell_executions) if len(query_cell_executions) < batch_size: break query_cell_executions_offset += batch_size LOG.info( "\n--Query cell count: {}, query cell executions count: {}, offset: {}" .format(len(query_cells), query_executions_count, offset)) if len(query_cells) < batch_size: break offset += batch_size
def get_query_cells_iter(batch_size=1000, session=None): offset = 0 while True: query_cells = get_all_query_cells( limit=batch_size, offset=offset, session=session, ) LOG.info("\n--Query cells count: {}, offset: {}".format( len(query_cells), offset)) for query_cell in query_cells: expand_query_cell = query_cell_to_es(query_cell, session=session) yield expand_query_cell if len(query_cells) < batch_size: break offset += batch_size