def fill_tile_band(tile_size, tile_coords, set_coords, ar_pred, tx, nodata): ''' Fill an array of zeros of shape tile_size, located at tile_coords with an offset array, ar_pred, located at set_coords ''' # Calc offsets row_off, col_off = calc_offset_from_tile(tile_coords[['ul_x','ul_y']], set_coords[['ul_x','ul_y']], tx) # Get the offset indices of each array tile_inds, set_inds = mosaic.get_offset_array_indices( tile_size, ar_pred.shape, (row_off, col_off)) tile_row_u, tile_row_d, tile_col_l, tile_col_r = tile_inds set_row_u, set_row_d, set_col_l, set_col_r = set_inds # Fill just the part of the array that overlaps ar_tile = np.full(tile_size, np.nan) ar_pred = ar_pred.astype(float) ar_pred[ar_pred == nodata] = np.nan try: ar_tile[tile_row_u:tile_row_d, tile_col_l:tile_col_r] =\ ar_pred[set_row_u:set_row_d, set_col_l:set_col_r] except Exception as e: print e print '\nProblem with offsets' print row_off, col_off, set_coords, tile_coords return ar_tile
def snap_array(ds_in, ds_snap, tx_in, tx_snap, in_nodata, out_nodata, mask_val=None): ar_in = ds_in.ReadAsArray() if mask_val is not None: ar_snap = ds_snap.ReadAsArray() in_shape = ar_in.shape out_shape = ds_snap.RasterYSize, ds_snap.RasterXSize offset = calc_offset((tx_snap[0], tx_snap[3]), tx_in) snap_inds, in_inds = get_offset_array_indices(out_shape, in_shape, offset) np_dtype = ar_in.dtype ar = np.full(out_shape, out_nodata, dtype=np_dtype) ar_in[ar_in == in_nodata] = out_nodata ar[snap_inds[0]:snap_inds[1], snap_inds[2]:snap_inds[3]] = ar_in[in_inds[0]:in_inds[1], in_inds[2]:in_inds[3]] if mask_val is not None: mask_val = int(mask_val) ar[ar_snap == mask_val] = out_nodata return ar
def main(sample_txt, ref_raster, pred_raster, p_nodata, t_nodata, target_col, bins, out_txt, match=None, predict_col=None): p_nodata = int(p_nodata) t_nodata = int(t_nodata) ds_p = gdal.Open(pred_raster) ar_p = ds_p.ReadAsArray() ds_r = gdal.Open(ref_raster) ar_r = ds_r.ReadAsArray() r_xsize = ds_r.RasterXSize r_ysize = ds_r.RasterYSize p_xsize = ds_p.RasterXSize p_ysize = ds_p.RasterYSize tx_r = ds_r.GetGeoTransform() tx_p = ds_p.GetGeoTransform() # If two arrays are different sizes, make prediction array match reference if not r_xsize == p_xsize or r_ysize == p_ysize or tx_r != tx_p: warnings.warn('Prediction and reference rasters do not share the same extent. Snapping prediction raster to reference....') offset = mosaic.calc_offset((tx_r[0], tx_r[3]), tx_p) t_inds, p_inds = mosaic.get_offset_array_indices((r_ysize, r_xsize), (p_ysize, p_xsize), offset) ar_buf = np.full(ar_r.shape, p_nodata, dtype=ar_p.dtype) ar_buf[t_inds[0]:t_inds[1], t_inds[2]:t_inds[3]] = ar_p[p_inds[0]:p_inds[1], p_inds[2]:p_inds[3]] ar_p = ar_buf.copy() del ar_buf bins = parse_bins(bins) sample = pd.read_csv(sample_txt, sep='\t') if target_col in sample.columns: t_sample = sample[target_col] else: raise IndexError('target_col "%s" not in sample' % target_col) if match: t_sample, p_sample = get_samples(ar_p, ar_r, p_nodata, t_nodata, sample, match=match) elif predict_col: p_sample = sample[predict_col] else: p_sample = ar_p[sample.row, sample.col] t_sample = ar_r[sample.row, sample.col] rmse = area_weighted_rmse(ar_p, ar_r, p_sample, t_sample, bins, p_nodata, out_txt=out_txt) return rmse
def get_zone_inds(ar_size, zone_size, tx, feat): ''' Return the array offset indices for pixels overlapping a feature from a vector dataset. Array indices are returned as (upper_row, lower_row, left_col,_right col) to be used to index an array as [upper_row : lower_row, left_col : right_col] ''' geom = feat.GetGeometryRef() x1, x2, y1, y2 = geom.GetEnvelope() # Get the feature ul x and y, and calculate the pixel offset ar_ulx, x_res, x_rot, ar_uly, y_rot, y_res = tx x_sign = x_res/abs(x_res) y_sign = y_res/abs(y_res) f_ulx = min([x0/x_sign for x0 in [x1, x2]])/x_sign f_uly = min([y0/y_sign for y0 in [y1, y2]])/y_sign offset = stem.calc_offset((ar_ulx, ar_uly), (f_ulx, f_uly), tx) # Get the inds for the overlapping portions of each array a_inds, m_inds = mosaic.get_offset_array_indices(ar_size, zone_size, offset) return a_inds, m_inds
def main(params, ar_p=None, out_txt=None, inventory_txt=None, target_col=None, match=False, file_stamp=None): #p_path, t_path, bins, sample_txt, p_nodata, t_nodata, out_dir, inventory_txt=None # Read params and make variables from text inputs = read_params(params) for i in inputs: exec("{0} = str({1})").format(i, inputs[i]) # Check that variables were specified in params try: bins = parse_bins(bins) p_nodata = int(p_nodata) t_nodata = int(t_nodata) str_check = sample_txt #, target_col except NameError as e: print '' missing_var = str(e).split("'")[1] msg = "Variable '%s' not specified in param file:\n%s" % (missing_var, params) raise NameError(msg) #if out_dir_: # then out_dir came from predict_stem call # out_dir = out_dir_ #out_txt = os.path.join(out_dir, 'confusion.txt') if out_txt: out_dir = os.path.dirname(out_txt) if not os.path.exists(out_dir): os.mkdir(out_dir) shutil.copy2(params, out_dir) # If p_path was specified, this call of the function is coming from outside # predict_stem.py. Otherwise, ar_p should be given. if 'p_path' in locals(): print 'Reading in the prediction raster:%s\n' % p_path ds_p = gdal.Open(p_path) ar_p = ds_p.ReadAsArray() ds_t = gdal.Open(t_path) band = ds_t.GetRasterBand(1) ar_t = band.ReadAsArray() #ar_t=ar_t.GetRasterBand(1) #print('read in the truth raster') t_xsize = ds_t.RasterXSize #print('t_xsize is: ', t_xsize) t_ysize = ds_t.RasterYSize #print('tYsize is: ', t_ysize) p_xsize = ds_p.RasterXSize #print('p_xsize is: ', p_xsize) p_ysize = ds_p.RasterYSize #print('p_ysize is: ', p_ysize) tx_t = ds_t.GetGeoTransform() tx_p = ds_p.GetGeoTransform() # If two arrays are different sizes, make prediction array match reference if not t_xsize == p_xsize or t_ysize == p_ysize or tx_t != tx_p: print('entered if statement') warnings.warn( 'Prediction and reference rasters do not share the same extent. Snapping prediction raster to reference....' ) offset = mosaic.calc_offset((tx_t[0], tx_t[3]), tx_p) #print(offset) t_inds, p_inds = mosaic.get_offset_array_indices( (t_ysize, t_xsize), (p_ysize, p_xsize), offset) print(t_inds, p_inds) ar_buf = np.full(ar_t.shape, p_nodata, dtype=ar_p.dtype) print ar_buf.shape ar_buf[t_inds[0]:t_inds[1], t_inds[2]:t_inds[3]] = ar_p[p_inds[0]:p_inds[1], p_inds[2]:p_inds[3]] ar_p = ar_buf.copy() del ar_buf mask = (ar_p == p_nodata) | (ar_t == t_nodata) #''' samples = pd.read_csv(sample_txt, sep='\t', index_col='obs_id') print samples df_adj, df_smp = confusion_matrix_by_area(ar_p, ar_t, samples, p_nodata, t_nodata, mask=mask, bins=bins, out_txt=out_txt, target_col=target_col, match=match) ar_p = None ar_t = None mask = None accuracy = df_adj.ix['producer', 'user'] kappa = df_adj.ix['producer', 'kappa'] if inventory_txt and file_stamp: df_inv = pd.read_csv(inventory_txt, sep='\t', index_col='stamp') if file_stamp in df_inv.index and 'vote' in os.path.basename(out_dir): cols = ['vote_accuracy', 'vote_kappa'] df_inv.ix[file_stamp, cols] = accuracy, kappa df_inv.to_csv(inventory_txt, sep='\t') print 'Vote scores written to inventory_txt: ', inventory_txt if file_stamp in df_inv.index and 'mean' in os.path.basename(out_dir): cols = ['mean_accuracy', 'mean_kappa'] df_inv.ix[file_stamp, cols] = accuracy, kappa df_inv.to_csv(inventory_txt, sep='\t') return df_smp
def main(in_raster, snap_raster, in_nodata, out_nodata, out_path=None, mask_val=None, overwrite=False): t0 = time.time() in_nodata = int(in_nodata) out_nodata = int(out_nodata) print '\nOpening datasets... ' t1 = time.time() ds_in = gdal.Open(in_raster) ar_in = ds_in.ReadAsArray() tx_in = ds_in.GetGeoTransform() #driver = ds_in.GetDriver() ds_in = None ds_snap = gdal.Open(snap_raster) ar_snap = ds_snap.ReadAsArray() tx_snap = ds_snap.GetGeoTransform() prj = ds_snap.GetProjection() ds_snap = None print '%.1f seconds\n' % (time.time() - t1) print 'Snapping input raster...' t1 = time.time() offset = calc_offset((tx_snap[0], tx_snap[3]), tx_in) snap_inds, in_inds = get_offset_array_indices(ar_snap.shape, ar_in.shape, offset) np_dtype = ar_in.dtype ar = np.full(ar_snap.shape, out_nodata, dtype=np_dtype) ar_in[ar_in == in_nodata] = out_nodata ar[snap_inds[0]:snap_inds[1], snap_inds[2]:snap_inds[3]] = ar_in[in_inds[0]:in_inds[1], in_inds[2]:in_inds[3]] if mask_val: mask_val = int(mask_val) ar[ar_snap == mask_val] = out_nodata print '%.1f seconds\n' % (time.time() - t1) if out_path: if ar.max() <= 255 and ar.min() >= 0: gdal_dtype = gdal.GDT_Byte else: gdal_dtype = gdal.GDT_Int16 if os.path.exists(out_path) and not overwrite: sys.exit('out_path already exists') driver = get_gdal_driver(out_path) array_to_raster(ar, tx_snap, prj, driver, out_path, gdal_dtype, out_nodata) # Write metadata desc = ('Input raster %s snapped to the extent of %s.') % (in_raster, snap_raster) if mask_val: desc += ' Data were masked from snap raster with value %s.' % mask_val createMetadata(sys.argv, out_path, description=desc) else: return ar print '\nTotal time to snap raster: %.1f seconds\n' % (time.time() - t0)
def snap_by_tile(ds_in, ds_snap, tiles, tx_snap, tx_in, in_nodata, out_nodata, out_dir, mask_val=None): prj = ds_in.GetProjection() driver = gdal.GetDriverByName('gtiff') if mask_val is not None: mask_val = int(mask_val) row_off, col_off = calc_offset((tx_snap[0], tx_snap[3]), tx_in) in_size = ds_in.RasterYSize, ds_in.RasterXSize n_tiles = float(len(tiles)) t1 = time.time() msg = '\rProccessing tile %d/%d (%.1f%%) || %.1f/~%.1f minutes' template = os.path.join(out_dir, 'tile_%s.pkl') mins = [] maxs = [] for i, (tile_id, coords) in enumerate(tiles.iterrows()): tile_off = row_off - coords.ul_r, col_off - coords.ul_c tile_size = coords.lr_r - coords.ul_r, coords.lr_c - coords.ul_c tile_inds, in_inds = get_offset_array_indices(tile_size, in_size, tile_off) in_ulr, in_lrr, in_ulc, in_lrc = in_inds in_xsize = in_lrc - in_ulc in_ysize = in_lrr - in_ulr if in_xsize <= 0 or in_ysize <= 0: # They don't overlap continue ar_in = ds_in.ReadAsArray(in_ulc, in_ulr, in_xsize, in_ysize) if np.all(ar_in == in_nodata): continue ar_out = np.full(tile_size, out_nodata, dtype=ar_in.dtype) ar_out[tile_inds[0]:tile_inds[1], tile_inds[2]:tile_inds[3]] = ar_in ar_out[ar_out == in_nodata] = out_nodata if mask_val is not None: mask = ds_snap.ReadAsArray(coords.ul_c, coords.ul_r, tile_size[1], tile_size[0]) == mask_val ar_out[mask] = out_nodata out_path = template % tile_id with open(out_path, 'wb') as f: pickle.dump(ar_out, f, protocol=-1) mins.append(ar_out.min()) maxs.append(ar_out.max()) tiles.loc[tile_id, 'file'] = out_path cum_time = (time.time() - t1) / 60. est_time = cum_time / (i + 1) * (n_tiles - i) # estimate remaing time sys.stdout.write(msg % (i + 1, n_tiles, (i + 1) / n_tiles * 100, cum_time, est_time)) sys.stdout.flush() '''ulx, xres, _, uly, _, yres = tx_snap tx = coords.ul_c * xres + ulx, xres, 0, coords.ul_r * yres + uly, 0, yres array_to_raster(ar_out, tx, prj, driver, '/home/server/pi/homes/shooper/delete/tile_%s.tif' % tile_id, gdal.GDT_Int16, out_nodata)''' dtype = get_min_numpy_dtype(np.array(mins + maxs)) return dtype
def main(model_dir, n_tiles, **kwargs): t0 = time.time() n_tiles = [int(n) for n in n_tiles.split(',')] if not os.path.isdir(model_dir): message = 'model directory given does not exist or is not a directory: ', model_dir raise IOError(message) model = os.path.basename(model_dir) dt_dir = os.path.join(model_dir, 'decisiontree_models') set_txt = os.path.join(dt_dir, '%s_support_sets.txt' % model) df_sets = pd.read_csv(set_txt, sep='\t', index_col='set_id') pred_param_path = glob(os.path.join(model_dir, 'predict_stem_*params.txt'))[0] predict_params, df_var = stem.read_params(pred_param_path) train_param_path = glob(os.path.join(model_dir, 'train_stem_*params.txt'))[0] train_params, _ = stem.read_params(train_param_path) df_var.sort_index(inplace=True) nodata = int(predict_params['nodata'].replace('"', '')) if len(kwargs) == 0: var_ids = df_sets.max_importance.unique() var_names = df_var.ix[var_ids].index variables = zip(var_ids, var_names) else: variables = [(variable_id, variable_name) for variable_name, variable_id in kwargs] mask_path = os.path.join(model_dir, '%s_vote.bsq' % model) if not os.path.exists(mask_path): mask_path = mask_path.replace('.bsq', '.tif') mask_ds = gdal.Open(mask_path) mask_tx = mask_ds.GetGeoTransform() xsize = mask_ds.RasterXSize ysize = mask_ds.RasterYSize prj = mask_ds.GetProjection() df_tiles, df_tiles_rc, tile_size = stem.get_tiles(n_tiles, xsize, ysize, mask_tx) total_tiles = len(df_tiles) df_tiles['tile'] = df_tiles.index # Find the tiles that have only nodata values t1 = time.time() print '\nFinding empty tiles...' mask = mask_ds.ReadAsArray() == nodata empty_tiles = stem.find_empty_tiles(df_tiles, ~mask, mask_tx) mask_ds = None print '%s empty tiles found of %s total tiles\n%.1f minutes\n' %\ (len(empty_tiles), total_tiles, (time.time() - t1)/60) # Select only tiles that are not empty df_tiles = df_tiles.select(lambda x: x not in empty_tiles) total_tiles = len(df_tiles) #some_set = df_sets.iloc[0] support_size = [ int(s) for s in train_params['support_size'].replace('"', '').split(',') ] set_size = [int(abs(s / mask_tx[1])) for s in support_size] out_dir = os.path.join(model_dir, 'importance_maps') if not os.path.exists(out_dir): os.mkdir(out_dir) print variables for vi, (v_id, v_name) in enumerate(variables): t1 = time.time() print 'Making map for %s: %s of %s variables\n' % (v_name, vi + 1, len(variables)) ar = np.full((ysize, xsize), nodata, dtype=np.uint8) for i, (t_ind, t_row) in enumerate(df_tiles.iterrows()): t2 = time.time() print 'Aggregating for %s of %s tiles' % (i + 1, total_tiles) # Calculate the size of this tile in case it's at the edge where the # tile size will be slightly different this_size = abs(t_row.lr_y - t_row.ul_y), abs(t_row.lr_x - t_row.ul_x) df_these_sets = stem.get_overlapping_sets(df_sets, t_row, this_size, support_size) rc = df_tiles_rc.ix[t_ind] this_size = rc.lr_r - rc.ul_r, rc.lr_c - rc.ul_c n_sets = len(df_these_sets) # Load overlapping predictions from disk and read them as arrays tile_ul = t_row[['ul_x', 'ul_y']] print n_sets, ' Overlapping sets' importance_bands = [] importance_values = [] for s_ind, s_row in df_these_sets.iterrows(): # Calculate offset and array/tile indices offset = stem.calc_offset(tile_ul, (s_row.ul_x, s_row.ul_y), mask_tx) #if abs(offset[0]) > this_size[0] or abs(offset[1] > this_size[1]): tile_inds, a_inds = mosaic.get_offset_array_indices( tile_size, set_size, offset) # Get feature with maximum importance and fill tile with that val try: with open(s_row.dt_file, 'rb') as f: dt_model = pickle.load(f) importance_value = int( dt_model.feature_importances_[v_id] * 100) importance_values.append(importance_value) #filled = np.full((nrows, ncols), importance_value, dtype=np.uint8) #import_band = stem.fill_tile_band(this_size, filled, tile_inds, nodata) import_band = np.full(this_size, np.nan, dtype=np.float16) import_band[tile_inds[0]:tile_inds[1], tile_inds[2]:tile_inds[3]] = importance_value importance_bands.append(import_band) except Exception as e: print e continue #''' print 'Average importance for this tile: %.1f' % np.mean( importance_values) #Aggregate importance_stack = np.dstack(importance_bands) importance_tile = np.nanmean(importance_stack, axis=2) tile_mask = mask[rc.ul_r:rc.lr_r, rc.ul_c:rc.lr_c] | np.isnan(importance_tile) importance_tile[tile_mask] = nodata ar[rc.ul_r:rc.lr_r, rc.ul_c:rc.lr_c] = np.round(importance_tile).astype(np.uint8) print 'Aggregation time for this tile: %.1f minutes\n' % ( (time.time() - t2) / 60) '''temp_dir = os.path.join(out_dir, 'delete') if not os.path.isdir(temp_dir): os.mkdir(temp_dir) t_tx = tile_ul[0], 30, 0, tile_ul[1], 0, -30 array_to_raster(np.round(importance_tile).astype(np.uint8), t_tx, prj, gdal.GetDriverByName('gtiff'), os.path.join(temp_dir, 'delete_%s.tif' % t_ind), gdal.GDT_Byte, 255, True)''' out_path = os.path.join(out_dir, '%s_importance_%s.tif' % (model, v_name)) try: array_to_raster(ar, mask_tx, prj, gdal.GetDriverByName('gtiff'), out_path, gdal.GDT_Byte, nodata) except Exception as e: print e import pdb pdb.set_trace() print 'Time for this variable: %.1f minutes\n' % ( (time.time() - t1) / 60) print '\nTotal time for %s variables: %.1f hours\n' % (len(variables), ( (time.time() - t0) / 3600))
def main(region_path, tile_path, reference_path, out_dir, id_field='region_id', ref_basename='nlcd'): df = attributes_to_df(region_path) tile_info = attributes_to_df(tile_path) tile_info['ul_x'] = tile_info.xmin tile_info['lr_x'] = tile_info.xmax tile_info['ul_y'] = tile_info.ymax tile_info['lr_y'] = tile_info.ymin _, vector_ext = os.path.splitext(region_path) region_ids = df[id_field].unique() n_regions = len(region_ids) region_ds = ogr.Open(region_path) region_lyr = region_ds.GetLayer() for i, r_id in enumerate(region_ids): print 'Making region dir for %s (%s of %s)' % (r_id, i, n_regions) df_r = df[df.region_id == r_id] id_str = ('0' + str(r_id))[-2:] fid = df_r.index[0] region_feature = region_lyr.GetFeature(fid) xmin, xmax, ymin, ymax = region_feature.GetGeometryRef().GetEnvelope() region_feature.Destroy() df_r['ul_x'] = xmin df_r['lr_x'] = xmax df_r['ul_y'] = ymax df_r['lr_y'] = ymin clip_coords = df_r.loc[fid, ['ul_x', 'lr_x', 'ul_y', 'lr_y']] region_dir = os.path.join(out_dir, 'region_%s' % id_str) if not os.path.exists(region_dir): os.mkdir(region_dir) # Make a shapefile of the tiles out_vector = os.path.join(region_dir, 'tile_{0}{1}'.format(id_str, vector_ext)) if not os.path.exists(out_vector): ''' switch to selection by min/max of coords ''' region_tiles = tile_info[tile_info[id_field] == r_id] coords_to_shp(region_tiles, region_path, out_vector) # Make a map of reference NLCD ds = gdal.Open(out_vector.replace(vector_ext, '.tif')) mask = ds.ReadAsArray() == 255 ds = None nlcd_year = re.search( '\d\d\d\d', reference_path).group() # finds the first one (potentially buggy) out_ref_map = os.path.join( region_dir, '%s_%s_%s.tif' % (ref_basename, nlcd_year, id_str)) if not False: #os.path.exists(out_ref_map): ref_ds = gdal.Open(reference_path) ref_tx = ref_ds.GetGeoTransform() ref_shape = ref_ds.RasterYSize, ref_ds.RasterXSize col_off = (ref_tx[0] - clip_coords.ul_x) / ref_tx[1] row_off = (ref_tx[3] - clip_coords.ul_y) / ref_tx[5] n_cols = abs((clip_coords.ul_x - clip_coords.lr_x) / ref_tx[1]) n_rows = abs((clip_coords.ul_y - clip_coords.lr_y) / ref_tx[1]) ar_inds, ref_inds = get_offset_array_indices( (n_rows, n_cols), ref_shape, (row_off, col_off)) ref_n_cols = ref_inds[1] - ref_inds[0] ref_n_rows = ref_inds[3] - ref_inds[2] ar_ref = ref_ds.ReadAsArray(ref_inds[2], ref_inds[0], ref_n_cols, ref_n_rows) ar = np.full((n_rows, n_cols), 255) ar[ar_inds[0]:ar_inds[1], ar_inds[2]:ar_inds[3]] = ar_ref ar[mask] = 255 tx = clip_coords.ul_x, 30, 0, clip_coords.ul_y, 0, -30 prj = ref_ds.GetProjection() driver = gdal.GetDriverByName('gtiff') array_to_raster(ar, tx, prj, driver, out_ref_map, nodata=255) # Make a clipped raster of the tiles out_raster = out_vector.replace(vector_ext, '.tif') if not os.path.exists(out_raster): tiles = ogr.Open(tile_path) tile_lyr = tiles.GetLayer() tx = clip_coords.ul_x, 30, 0, clip_coords.ul_y, 0, -30 tile_array, _ = kernel_from_shp(tile_lyr, clip_coords, tx, 255, val_field='name') tile_array[ar == 255] = 255 driver = gdal.GetDriverByName('gtiff') prj = tile_lyr.GetSpatialRef().ExportToWkt() array_to_raster(tile_array, tx, prj, driver, out_raster, nodata=255) tiles.Destroy()