def _filter_stripes( filters, filepath_or_buffer, stripes=None, skip_rows=None, num_rows=None ): # Multiple sources are passed as a list. If a single source is passed, # wrap it in a list for unified processing downstream. if not is_list_like(filepath_or_buffer): filepath_or_buffer = [filepath_or_buffer] # Prepare filters filters = ioutils._prepare_filters(filters) # Get columns relevant to filtering columns_in_predicate = [ col for conjunction in filters for (col, op, val) in conjunction ] # Read and parse file-level and stripe-level statistics file_statistics, stripes_statistics = read_orc_statistics( filepath_or_buffer, columns_in_predicate ) file_stripe_map = [] for file_stat in file_statistics: # Filter using file-level statistics if not ioutils._apply_filters(filters, file_stat): continue # Filter using stripe-level statistics selected_stripes = [] num_rows_scanned = 0 for i, stripe_statistics in enumerate(stripes_statistics): num_rows_before_stripe = num_rows_scanned num_rows_scanned += next(iter(stripe_statistics.values()))[ "number_of_values" ] if stripes is not None and i not in stripes: continue if skip_rows is not None and num_rows_scanned <= skip_rows: continue else: skip_rows = 0 if ( skip_rows is not None and num_rows is not None and num_rows_before_stripe >= skip_rows + num_rows ): continue if ioutils._apply_filters(filters, stripe_statistics): selected_stripes.append(i) file_stripe_map.append(selected_stripes) return file_stripe_map
def _filter_stripes( filters, filepath_or_buffer, stripes=None, skip_rows=None, num_rows=None ): # Prepare filters filters = ioutils._prepare_filters(filters) # Get columns relevant to filtering columns_in_predicate = [ col for conjunction in filters for (col, op, val) in conjunction ] # Read and parse file-level and stripe-level statistics file_statistics, stripes_statistics = read_orc_statistics( filepath_or_buffer, columns_in_predicate ) # Filter using file-level statistics if not ioutils._apply_filters(filters, file_statistics): return [] # Filter using stripe-level statistics selected_stripes = [] num_rows_scanned = 0 for i, stripe_statistics in enumerate(stripes_statistics): num_rows_before_stripe = num_rows_scanned num_rows_scanned += next(iter(stripe_statistics.values()))[ "number_of_values" ] if stripes is not None and i not in stripes: continue if skip_rows is not None and num_rows_scanned <= skip_rows: continue else: skip_rows = 0 if ( skip_rows is not None and num_rows is not None and num_rows_before_stripe >= skip_rows + num_rows ): continue if ioutils._apply_filters(filters, stripe_statistics): selected_stripes.append(i) return selected_stripes