def main(gedi_dir, temp_dir, bounds_file, outfile, nproc, res): """ Main function to execute python script """ nproc = int(nproc) - 1 attrib = {'BEAM': 'int', 'FILE': 'str', 'YEAR': 'int', 'JDAY': 'int'} res = float(res) bounds_vec = Vector(bounds_file) bounds_wkt = bounds_vec.wktlist[0] args_list = list((filename, temp_dir, bounds_wkt, res) for filename in Handler(dirname=gedi_dir).find_all('*.h5')) n_files = len(args_list) Opt.cprint('Number of files: {}'.format(str(n_files))) out_res = {} Handler(outfile).file_delete() pool = mp.Pool(processes=nproc) for file_output, err_str in pool.imap_unordered(get_path, args_list): if err_str is None and len(file_output) > 0: add_on = '' count = 0 for geom_wkt, attrs in file_output: if geom_wkt is not None: write_to_txt(geom_wkt, attrs, outfile) out_res[geom_wkt] = attrs count += 1 if count > 0: add_on = ' - FOUND {} BEAMS'.format(str(count)) out_str = str(list(set([attr['FILE'] for _, attr in file_output]))[0]) + ' : READ' + add_on Opt.cprint(out_str) else: if err_str is None: err_str = 'Unknown File I/O Error' Opt.cprint('{}: {}'.format(file_output, err_str)) pool.close() Opt.cprint(outfile)
xpixel = 0.00027 ypixel = 0.00027 x_extent = lon_limits[1] - lon_limits[0] y_extent = lat_limits[1] - lat_limits[0] xsize = np.ceil(x_extent / xpixel) ysize = np.ceil(y_extent / ypixel) print('x extent {} : y extent {}'.format(str(x_extent), str(y_extent))) print('Num of 30m pixels in x {} | Num of 30m pixels in y {}'.format( str(xsize), str(ysize))) wkt_list = list( Vector.wkt_from_coords(coords, geom_type='point') for coords in pos_arr.tolist()) attrib = {'cover': 'float', 'cover_error': 'float'} attr_list = list({ 'cover': cover[0], 'cover_error': cover[1] } for cover in zip(cover_arr, cover_err_arr)) vector = Vector.vector_from_string(wkt_list, geom_string_type='wkt', out_epsg=4326, vector_type='point', attributes=attr_list, attribute_types=attrib,
from geosoup import Vector import numpy as np if __name__ == '__main__': # file1 = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/all_samp_pre_v1.shp" file1 = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/CAN_PSP/PSP_data/CAN_PSP_v1.shp" file2 = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/STUDY_AREA/ABoVE_Study_Domain_geo.shp" lat_x = 52.0 vec = Vector(file1) vec2 = Vector(file2) geom2 = vec2.features[0].GetGeometryRef() lonlat = [] for i in range(vec.nfeat): geom = vec.features[i].GetGeometryRef() pt = geom.GetPoint() lonlat.append(pt[0:2]) print(len(lonlat)) uniq, indices, inverse, count = np.unique(ar=lonlat, axis=0,
start_year = 2000 end_year = 2017 pixel_size = 0.0021 # 250m MODIS resolution fire_year_col = 'FireYear' fire_id_col = 'FIREID' filelist = [ 'D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/FIRE/ak_fire/FireAreaHistory.shp' ] outfolder = 'D:/temp/fire/' for infile in filelist: vec = Vector(infile) sys.stdout.write( '========================\nInput Vector file: {}\n'.format(infile)) attr = vec.attributes[0] attr_dict = dict() for k, v in attr.items(): attr_dict[k] = Vector.string_to_ogr_type(v, 'name') order = np.argsort( np.array([ int(attr_dict[fire_year_col]) for attr_dict in vec.attributes ])).tolist() x_min, x_max, y_min, y_max = vec.layer.GetExtent() sys.stdout.write('Extent: {}\n'.format(' '.join(
def read_gee_extract_data(filename): """ Method to read sample data in the form of a site dictionary with samples dicts by year :param filename: Input data file name :return: dict of list of dicts by year """ lines = Handler(filename).read_from_csv(return_dicts=True) site_dict = dict() line_counter = 0 for j, line in enumerate(lines): include = True for key, val in line.items(): if type(val).__name__ == 'str': if val == 'None': include = False if saturated_bands(line['radsat_qa']) \ or line['GEOMETRIC_RMSE_MODEL'] > 15.0 \ or unclear_value(line['pixel_qa']): include = False if include: line_counter += 1 site_year = str(line['site']) + '_' + str(line['year']) if site_year not in site_dict: geom_wkt = Vector.wkt_from_coords((line['longitude'], line['latitude'])) site_dict[site_year] = {'geom': geom_wkt, 'decid_frac': line['decid_frac'], 'data': dict(), 'site_year': line['year'], 'site': line['site']} temp_dict = dict() sensor_dict = extract_date(line['LANDSAT_ID']) temp_dict['img_jday'] = sensor_dict['date'].timetuple().tm_yday temp_dict['img_year'] = sensor_dict['date'].timetuple().tm_year temp_dict['sensor'] = sensor_dict['sensor'] bands = list('B' + str(ii + 1) for ii in range(7)) + ['slope', 'elevation', 'aspect'] band_dict = dict() for band in bands: if band in line: band_dict[band] = line[band] temp_dict['bands'] = correct_landsat_sr(band_dict, sensor_dict['sensor'], scale=0.0001) site_dict[site_year]['data'].update({'{}_{}'.format(str(temp_dict['img_jday']), str(temp_dict['img_year'])): temp_dict}) # print(line_counter) return site_dict
def get_path(args): """ Method to extract path from a GEDI file args: filename: GEDI filename bounds_wkt: WKT representation of boundary geometry res: bin resolution (degrees) (default : 0.1 degrees) buffer: buffer in degrees :return: (attribute dictionary, geometry WKT, None) if no error is raised while opening file (None, None, error string) if error is raised """ pt_limit = 15 verbose = False filename, temp_dir, boundary_wkt, spatial_resolution = args if verbose: Opt.cprint('Working on - {} '.format(Handler(filename).basename)) Handler(filename).copy_file(temp_dir) temp_filename = temp_dir + Handler(filename).basename date_str = Handler(temp_filename).basename.split('_')[2] year = int(date_str[0:4]) julian_day = int(date_str[4:7]) bounds_geom = ogr.CreateGeometryFromWkt(boundary_wkt) file_keys = [] try: fs = h5py.File(temp_filename, 'r') fs.visit(file_keys.append) except Exception as e: return Handler(temp_filename).basename, ' '.join(e.args) beam_ids = list(set(list(key.split('/')[0].strip() for key in file_keys if 'BEAM' in key))) feat_list = [] err = 'No Keys found' for beam in beam_ids: beam_id = int(beam.replace('BEAM', ''), 2) if verbose: Opt.cprint('\nBEAM - {}'.format(beam_id), newline=' ') try: lat_arr = np.array(fs['{}/geolocation/latitude_bin0'.format(beam)]) lon_arr = np.array(fs['{}/geolocation/longitude_bin0'.format(beam)]) except Exception as e: err = ' '.join(e.args) continue # make an array of lat lon pt_arr = np.vstack([lon_arr, lat_arr]).T # remove NaN values nan_loc_pre = np.where(np.apply_along_axis(lambda x: (not (np.isnan(x[0])) and (not np.isnan(x[1]))), 1, pt_arr)) pt_arr = pt_arr[nan_loc_pre] groups = group_nearby(pt_arr) # find start and end of valid strips chunks = list(pt_arr[elem[0]:(elem[1] + 1), :] for elem in groups) main_geom = ogr.Geometry(ogr.wkbMultiLineString) any_geom = False # find polygons for each strip and add to main_geom for chunk in chunks: if chunk.shape[0] <= pt_limit: if verbose: Opt.cprint('chunk too short size={},'.format(chunk.shape[0]), newline=' ') continue else: if verbose: Opt.cprint('chunk size={},'.format(str(chunk.shape[0])), newline=' ') try: resampled_chunk = resample_chunk(chunk, spatial_resolution) except Exception as e: if verbose: Opt.cprint('invalid chunk({})'.format(e.args[0]), newline=' ') continue part_geom_json = json.dumps({'type': 'Linestring', 'coordinates': resampled_chunk.tolist()}) part_geom = Vector.get_osgeo_geom(part_geom_json, 'json') if part_geom.Intersects(bounds_geom): any_geom = True part_geom_intersection = part_geom.Intersection(bounds_geom) # add to main geometry main_geom.AddGeometryDirectly(part_geom_intersection) attributes = {'BEAM': beam_id, 'FILE': Handler(temp_filename).basename, 'YEAR': year, 'JDAY': julian_day} if any_geom: # Opt.cprint(attributes) wkt = main_geom.ExportToWkt() main_geom = None else: wkt = None feat_list.append((wkt, attributes)) fs.close() Handler(temp_filename).file_delete() if len(feat_list) == 0: return Handler(filename).basename, err else: return feat_list, None
import pandas as pd import json from osgeo import ogr import numpy as np if __name__ == '__main__': infilename = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/CAN_PSP/" \ "CAN_PSPs_Hember-20180207T213138Z-001/CAN_PSPs_Hember/NAFP_L4_SL_ByJur_R16d_ForBrendanRogers1.csv" outfilename = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/CAN_PSP/" \ "CAN_PSPs_Hember-20180207T213138Z-001/CAN_PSPs_Hember/NAFP_L4_SL_ByJur_R16d_ForBrendanRogers1_lat52_ABoVE.shp" bounds = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/STUDY_AREA/ABoVE_Study_Domain_geo.shp" bounds_vec = Vector(bounds) bounds_geom = bounds_vec.features[0].GetGeometryRef() attr = {'ID_Plot': 'str', 'Lat': 'float', 'Lon': 'float'} samp_data = Handler(infilename).read_from_csv(return_dicts=True) wkt_list = list() attr_list = list() spref_str = '+proj=longlat +datum=WGS84' latlon = list() count = 0 for row in samp_data: print('Reading elem: {}'.format(str(count + 1)))
[-113.466796875, 68.13885164925574], [-125.947265625, 70.4073476760681], [-140.09765625, 70.31873847853124], [-156.708984375, 71.63599288330609], [-167.431640625, 69.3493386397765], [-165.146484375, 66.72254132270653], [-168.662109375, 65.47650756256367], [-165.673828125, 63.31268278043484], [-168.837890625, 60.326947742998414], [-166.552734375, 56.218923189166624]] # boundary = ee.Geometry.Polygon(bound_coords) # boundary = ee.FeatureCollection('users/masseyr44/shapefiles/NAboreal_10kmbuffer').first().geometry() boundary_geom = Vector.get_osgeo_geom(Vector.wkt_from_coords( bound_coords, 'polygon'), geom_type='wkt') # values to copy from fusion table/feature collection feat_properties = ['site', 'year', 'decid_frac'] KEYS = ee.List(feat_properties) # properties to retrieve from scene scene_properties = [ 'CLOUD_COVER', 'GEOMETRIC_RMSE_MODEL', 'LANDSAT_ID', 'SOLAR_ZENITH_ANGLE' ] PROPERTY_LIST = ee.List(scene_properties) # Bands to retrieve bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'pixel_qa', 'radsat_qa']
ini_outfile = outdir + "all_samp_pre_v1.shp" outfile = outdir + "all_samp_postbin_v{}.csv".format(version) outshpfile = outdir + "all_samp_postbin_v{}.shp".format(version) trn_outfile = outdir + "all_samp_post_v{}_trn_samp.shp".format(version) val_outfile = outdir + "all_samp_post_v{}_val_samp.shp".format(version) boreal_bounds = "D:/shared/Dropbox/projects/NAU/landsat_deciduous/data/STUDY_AREA/boreal/" \ "NABoreal_simple_10km_buffer_geo.shp" year_bins = [(1984, 1997), (1998, 2002), (2003, 2007), (2008, 2012), (2013, 2018)] # script----------------------------------------------------------------------------------------------- boreal_vec = Vector(boreal_bounds) boreal_geom = Vector.get_osgeo_geom(boreal_vec.wktlist[0]) year_samp = list(list() for _ in range(len(year_bins))) year_samp_reduced = list(list() for _ in range(len(year_bins))) # get data and names file_data = Handler(infile).read_from_csv(return_dicts=True) header = list(file_data[0]) print('\nTotal samples: {}'.format(str(len(file_data)))) boreal_samp_count = 0 # bin all samples based on sample years using year_bins for elem in file_data: