def main(gedi_dir, temp_dir, bounds_file, outfile, nproc, res):
"""
Main function to execute python script
"""
nproc = int(nproc) - 1
attrib = {'BEAM': 'int', 'FILE': 'str', 'YEAR': 'int', 'JDAY': 'int'}
res = float(res)
bounds_vec = Vector(bounds_file)
bounds_wkt = bounds_vec.wktlist[0]
args_list = list((filename, temp_dir, bounds_wkt, res)
for filename in Handler(dirname=gedi_dir).find_all('*.h5'))
n_files = len(args_list)
Opt.cprint('Number of files: {}'.format(str(n_files)))
out_res = {}
Handler(outfile).file_delete()
pool = mp.Pool(processes=nproc)
for file_output, err_str in pool.imap_unordered(get_path, args_list):
if err_str is None and len(file_output) > 0:
add_on = ''
count = 0
for geom_wkt, attrs in file_output:
if geom_wkt is not None:
write_to_txt(geom_wkt, attrs, outfile)
out_res[geom_wkt] = attrs
count += 1
if count > 0:
add_on = ' - FOUND {} BEAMS'.format(str(count))
out_str = str(list(set([attr['FILE'] for _, attr in file_output]))[0]) + ' : READ' + add_on
Opt.cprint(out_str)
else:
if err_str is None:
err_str = 'Unknown File I/O Error'
Opt.cprint('{}: {}'.format(file_output,
err_str))
pool.close()
Opt.cprint(outfile)
xpixel = 0.00027
ypixel = 0.00027
x_extent = lon_limits[1] - lon_limits[0]
y_extent = lat_limits[1] - lat_limits[0]
xsize = np.ceil(x_extent / xpixel)
ysize = np.ceil(y_extent / ypixel)
print('x extent {} : y extent {}'.format(str(x_extent), str(y_extent)))
print('Num of 30m pixels in x {} | Num of 30m pixels in y {}'.format(
str(xsize), str(ysize)))
wkt_list = list(
Vector.wkt_from_coords(coords, geom_type='point')
for coords in pos_arr.tolist())
attrib = {'cover': 'float', 'cover_error': 'float'}
attr_list = list({
'cover': cover[0],
'cover_error': cover[1]
} for cover in zip(cover_arr, cover_err_arr))
vector = Vector.vector_from_string(wkt_list,
geom_string_type='wkt',
out_epsg=4326,
vector_type='point',
attributes=attr_list,
attribute_types=attrib,
from geosoup import Vector
import numpy as np
if __name__ == '__main__':
# file1 = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/all_samp_pre_v1.shp"
file1 = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/CAN_PSP/PSP_data/CAN_PSP_v1.shp"
file2 = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/STUDY_AREA/ABoVE_Study_Domain_geo.shp"
lat_x = 52.0
vec = Vector(file1)
vec2 = Vector(file2)
geom2 = vec2.features[0].GetGeometryRef()
lonlat = []
for i in range(vec.nfeat):
geom = vec.features[i].GetGeometryRef()
pt = geom.GetPoint()
lonlat.append(pt[0:2])
print(len(lonlat))
uniq, indices, inverse, count = np.unique(ar=lonlat,
axis=0,
start_year = 2000
end_year = 2017
pixel_size = 0.0021 # 250m MODIS resolution
fire_year_col = 'FireYear'
fire_id_col = 'FIREID'
filelist = [
'D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/FIRE/ak_fire/FireAreaHistory.shp'
]
outfolder = 'D:/temp/fire/'
for infile in filelist:
vec = Vector(infile)
sys.stdout.write(
'========================\nInput Vector file: {}\n'.format(infile))
attr = vec.attributes[0]
attr_dict = dict()
for k, v in attr.items():
attr_dict[k] = Vector.string_to_ogr_type(v, 'name')
order = np.argsort(
np.array([
int(attr_dict[fire_year_col]) for attr_dict in vec.attributes
])).tolist()
x_min, x_max, y_min, y_max = vec.layer.GetExtent()
sys.stdout.write('Extent: {}\n'.format(' '.join(
def read_gee_extract_data(filename):
"""
Method to read sample data in the form of a site dictionary with samples dicts by year
:param filename: Input data file name
:return: dict of list of dicts by year
"""
lines = Handler(filename).read_from_csv(return_dicts=True)
site_dict = dict()
line_counter = 0
for j, line in enumerate(lines):
include = True
for key, val in line.items():
if type(val).__name__ == 'str':
if val == 'None':
include = False
if saturated_bands(line['radsat_qa']) \
or line['GEOMETRIC_RMSE_MODEL'] > 15.0 \
or unclear_value(line['pixel_qa']):
include = False
if include:
line_counter += 1
site_year = str(line['site']) + '_' + str(line['year'])
if site_year not in site_dict:
geom_wkt = Vector.wkt_from_coords((line['longitude'], line['latitude']))
site_dict[site_year] = {'geom': geom_wkt,
'decid_frac': line['decid_frac'],
'data': dict(),
'site_year': line['year'],
'site': line['site']}
temp_dict = dict()
sensor_dict = extract_date(line['LANDSAT_ID'])
temp_dict['img_jday'] = sensor_dict['date'].timetuple().tm_yday
temp_dict['img_year'] = sensor_dict['date'].timetuple().tm_year
temp_dict['sensor'] = sensor_dict['sensor']
bands = list('B' + str(ii + 1) for ii in range(7)) + ['slope', 'elevation', 'aspect']
band_dict = dict()
for band in bands:
if band in line:
band_dict[band] = line[band]
temp_dict['bands'] = correct_landsat_sr(band_dict,
sensor_dict['sensor'],
scale=0.0001)
site_dict[site_year]['data'].update({'{}_{}'.format(str(temp_dict['img_jday']),
str(temp_dict['img_year'])): temp_dict})
# print(line_counter)
return site_dict
def get_path(args):
"""
Method to extract path from a GEDI file
args:
filename: GEDI filename
bounds_wkt: WKT representation of boundary geometry
res: bin resolution (degrees) (default : 0.1 degrees)
buffer: buffer in degrees
:return: (attribute dictionary, geometry WKT, None) if no error is raised while opening file
(None, None, error string) if error is raised
"""
pt_limit = 15
verbose = False
filename, temp_dir, boundary_wkt, spatial_resolution = args
if verbose:
Opt.cprint('Working on - {} '.format(Handler(filename).basename))
Handler(filename).copy_file(temp_dir)
temp_filename = temp_dir + Handler(filename).basename
date_str = Handler(temp_filename).basename.split('_')[2]
year = int(date_str[0:4])
julian_day = int(date_str[4:7])
bounds_geom = ogr.CreateGeometryFromWkt(boundary_wkt)
file_keys = []
try:
fs = h5py.File(temp_filename, 'r')
fs.visit(file_keys.append)
except Exception as e:
return Handler(temp_filename).basename, ' '.join(e.args)
beam_ids = list(set(list(key.split('/')[0].strip() for key in file_keys if 'BEAM' in key)))
feat_list = []
err = 'No Keys found'
for beam in beam_ids:
beam_id = int(beam.replace('BEAM', ''), 2)
if verbose:
Opt.cprint('\nBEAM - {}'.format(beam_id), newline=' ')
try:
lat_arr = np.array(fs['{}/geolocation/latitude_bin0'.format(beam)])
lon_arr = np.array(fs['{}/geolocation/longitude_bin0'.format(beam)])
except Exception as e:
err = ' '.join(e.args)
continue
# make an array of lat lon
pt_arr = np.vstack([lon_arr, lat_arr]).T
# remove NaN values
nan_loc_pre = np.where(np.apply_along_axis(lambda x: (not (np.isnan(x[0])) and (not np.isnan(x[1]))),
1, pt_arr))
pt_arr = pt_arr[nan_loc_pre]
groups = group_nearby(pt_arr)
# find start and end of valid strips
chunks = list(pt_arr[elem[0]:(elem[1] + 1), :] for elem in groups)
main_geom = ogr.Geometry(ogr.wkbMultiLineString)
any_geom = False
# find polygons for each strip and add to main_geom
for chunk in chunks:
if chunk.shape[0] <= pt_limit:
if verbose:
Opt.cprint('chunk too short size={},'.format(chunk.shape[0]), newline=' ')
continue
else:
if verbose:
Opt.cprint('chunk size={},'.format(str(chunk.shape[0])), newline=' ')
try:
resampled_chunk = resample_chunk(chunk, spatial_resolution)
except Exception as e:
if verbose:
Opt.cprint('invalid chunk({})'.format(e.args[0]), newline=' ')
continue
part_geom_json = json.dumps({'type': 'Linestring', 'coordinates': resampled_chunk.tolist()})
part_geom = Vector.get_osgeo_geom(part_geom_json, 'json')
if part_geom.Intersects(bounds_geom):
any_geom = True
part_geom_intersection = part_geom.Intersection(bounds_geom)
# add to main geometry
main_geom.AddGeometryDirectly(part_geom_intersection)
attributes = {'BEAM': beam_id,
'FILE': Handler(temp_filename).basename,
'YEAR': year,
'JDAY': julian_day}
if any_geom:
# Opt.cprint(attributes)
wkt = main_geom.ExportToWkt()
main_geom = None
else:
wkt = None
feat_list.append((wkt, attributes))
fs.close()
Handler(temp_filename).file_delete()
if len(feat_list) == 0:
return Handler(filename).basename, err
else:
return feat_list, None
import pandas as pd
import json
from osgeo import ogr
import numpy as np
if __name__ == '__main__':
infilename = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/CAN_PSP/" \
"CAN_PSPs_Hember-20180207T213138Z-001/CAN_PSPs_Hember/NAFP_L4_SL_ByJur_R16d_ForBrendanRogers1.csv"
outfilename = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/samples/CAN_PSP/" \
"CAN_PSPs_Hember-20180207T213138Z-001/CAN_PSPs_Hember/NAFP_L4_SL_ByJur_R16d_ForBrendanRogers1_lat52_ABoVE.shp"
bounds = "D:/Shared/Dropbox/projects/NAU/landsat_deciduous/data/STUDY_AREA/ABoVE_Study_Domain_geo.shp"
bounds_vec = Vector(bounds)
bounds_geom = bounds_vec.features[0].GetGeometryRef()
attr = {'ID_Plot': 'str', 'Lat': 'float', 'Lon': 'float'}
samp_data = Handler(infilename).read_from_csv(return_dicts=True)
wkt_list = list()
attr_list = list()
spref_str = '+proj=longlat +datum=WGS84'
latlon = list()
count = 0
for row in samp_data:
print('Reading elem: {}'.format(str(count + 1)))
[-113.466796875, 68.13885164925574],
[-125.947265625, 70.4073476760681],
[-140.09765625, 70.31873847853124],
[-156.708984375, 71.63599288330609],
[-167.431640625, 69.3493386397765],
[-165.146484375, 66.72254132270653],
[-168.662109375, 65.47650756256367],
[-165.673828125, 63.31268278043484],
[-168.837890625, 60.326947742998414],
[-166.552734375, 56.218923189166624]]
# boundary = ee.Geometry.Polygon(bound_coords)
# boundary = ee.FeatureCollection('users/masseyr44/shapefiles/NAboreal_10kmbuffer').first().geometry()
boundary_geom = Vector.get_osgeo_geom(Vector.wkt_from_coords(
bound_coords, 'polygon'),
geom_type='wkt')
# values to copy from fusion table/feature collection
feat_properties = ['site', 'year', 'decid_frac']
KEYS = ee.List(feat_properties)
# properties to retrieve from scene
scene_properties = [
'CLOUD_COVER', 'GEOMETRIC_RMSE_MODEL', 'LANDSAT_ID',
'SOLAR_ZENITH_ANGLE'
]
PROPERTY_LIST = ee.List(scene_properties)
# Bands to retrieve
bands = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'pixel_qa', 'radsat_qa']
ini_outfile = outdir + "all_samp_pre_v1.shp"
outfile = outdir + "all_samp_postbin_v{}.csv".format(version)
outshpfile = outdir + "all_samp_postbin_v{}.shp".format(version)
trn_outfile = outdir + "all_samp_post_v{}_trn_samp.shp".format(version)
val_outfile = outdir + "all_samp_post_v{}_val_samp.shp".format(version)
boreal_bounds = "D:/shared/Dropbox/projects/NAU/landsat_deciduous/data/STUDY_AREA/boreal/" \
"NABoreal_simple_10km_buffer_geo.shp"
year_bins = [(1984, 1997), (1998, 2002), (2003, 2007), (2008, 2012),
(2013, 2018)]
# script-----------------------------------------------------------------------------------------------
boreal_vec = Vector(boreal_bounds)
boreal_geom = Vector.get_osgeo_geom(boreal_vec.wktlist[0])
year_samp = list(list() for _ in range(len(year_bins)))
year_samp_reduced = list(list() for _ in range(len(year_bins)))
# get data and names
file_data = Handler(infile).read_from_csv(return_dicts=True)
header = list(file_data[0])
print('\nTotal samples: {}'.format(str(len(file_data))))
boreal_samp_count = 0
# bin all samples based on sample years using year_bins
for elem in file_data: