version = 11 filelist = list(in_dir + 'year_{}_{}_{}_fire.csv'.format( str(year_edge[0])[2:], str(year_edge[1])[2:], fire_type) for year_edge in year_edges for fire_type in fire_types) zvals_list = list(np.zeros((zcount_lim, leny, lenx)) for _ in years) zcount_list = list(np.zeros((leny, lenx), dtype='int16') for _ in years) for filename in filelist: if 'single' in filename: print(filename) val_dicts = Handler(filename).read_from_csv(return_dicts=True) print(len(val_dicts)) for ii, decid_year in enumerate(years): max_time = 0 decid_band = 'decid{}'.format(str(decid_year)) tc_band = 'tc{}'.format(str(decid_year)) for val_dict in val_dicts: if (type(val_dict[decid_band]).__name__ in ('int', 'float')) and \ (type(val_dict[tc_band]).__name__ in ('int', 'float')): if float(val_dict['longitude']) > -142.0:
plot_file = in_dir + "RF{}_{}_{}_{}_cutoff_{}_deg{}_{}.png".format(ylabel, xlabel, zlabel, str(bin_limit), str(int(z_thresh*scaledown_treecover)), deg, datetime.now().isoformat().split('.')[0] .replace('-', '').replace(':', '')) plot_file2 = in_dir + "RF{}_{}_{}_{}_cutoff_{}_deg{}_{}2.png".format(ylabel, xlabel, zlabel, str(bin_limit), str(int(z_thresh*scaledown_treecover)), deg, datetime.now().isoformat().split('.')[0] .replace('-', '').replace(':', '')) print('Reading file : {}'.format(csv_file)) print('Plot file: {}'.format(plot_file)) val_dicts = Handler(csv_file).read_from_csv(return_dicts=True, read_random=True, line_limit=None, ) for val_dict in val_dicts[0:10]: print(val_dict) basename = Handler(csv_file).basename.split('.csv')[0] elem_list = list() for val_dict in val_dicts: for i in range(3): x, y, z = val_dict[xvar_bands[i]], val_dict[yvar_bands[i]], val_dict[zvar_bands[i]] oz1, oz2 = val_dict[oz_bands[0]], val_dict[oz_bands[1]] if (type(x) in (int, float)) and (type(y) in (int, float)) and (type(z) in (int, float)):
from modules.plots import Plot from modules import Handler, Opt, Sublist from sys import argv if __name__ == '__main__': # script, in_dir = argv in_dir = 'c:/temp/check2/' Handler(dirname=in_dir).dir_create() csv_files = Handler(dirname=in_dir).find_all('.csv') for csv_file in csv_files: Opt.cprint('Reading file : {}'.format(csv_file)) val_dicts = Handler(csv_file).read_from_csv(return_dicts=True, read_random=True, line_limit=None, percent_random=10.0) bandname = Handler(csv_file).basename.split('.csv')[0] plot_file = in_dir + "{}_21.png".format(bandname) xvar = '{}_2'.format(bandname) xlabel = 'Season 2 {}'.format(bandname.upper()) yvar = '{}_1'.format(bandname)
import ee import datetime from modules import Logger, Handler if __name__ == '__main__': ee.Initialize() folder = "c:/temp/decid/" scale = 30 cutoff = 4000 fire_year_limit = 1900 elev_lim = 500 Handler(dirname=folder).dir_create() OUTFILE = folder + \ 'gee_mack_data_extract_{}_v{}.csv'.format(str(scale),datetime.datetime.now().isoformat() .split('.')[0].replace('-', '_').replace(':', '_')) logfile = OUTFILE.split('.csv')[0] + '.log' log = Logger('SAMP', filename=logfile, stream=True) log.lprint('Outfile: {}'.format(OUTFILE)) log.lprint('Logfile: {}'.format(logfile)) decid_layer = ee.Image( "users/masseyr44/decid/decid_mosaic_2000_prediction_vis_nd_3")
'ak_fire_multi_extract_v2019_08_27T20_22_51.csv' ] res = 250 version = 1 fire_year_limit = (1950, 2018) for file_base_name in filelist: filename = folder + file_base_name Opt.cprint('File: {}'.format(file_base_name)) OUTFILE = filename.split('.')[0] + 'reformat_{}_{}.csv'.format( str(res), str(version)) # convert csv files to dicts list_dicts = Handler(filename).read_from_csv(return_dicts=True) Opt.cprint(len(list_dicts)) outcolnames = [ 'ID', 'FIREID', 'SIZE_HA', 'longitude', 'latitude', 'decid1992', 'decid2000', 'decid2005', 'decid2010', 'decid2015', 'decid1992u', 'decid2000u', 'decid2005u', 'decid2010u', 'decid2015u', 'tc1992', 'tc2000', 'tc2005', 'tc2010', 'tc2015', 'tc1992u', 'tc2000u', 'tc2005u', 'tc2010u', 'tc2015u' ] samp_list = list() list_latlon = list() list_years = list() for elem in list_dicts:
module_path = "D:/temp/decid02092020/decid/src/" sys.path.append(module_path) from modules import RFRegressor, Handler ulim = 1.0 llim = 0.0 over_adjust_ = 1.0 clip_ = 0.01 active_dir = "D:/temp/pickle_dir/working/" outdir = active_dir + "out/" rf_file_list = Handler(dirname=active_dir).find_all('*.pickle') ''' rf_file_list = list(active_dir + filename for filename in ["out_tc_2010_samp_v1_summer_RF_772.pickle", "out_tc_2010_samp_v1_RF_59.pickle"]) ''' outfiles = list( filename.replace('.pickle', '.txt') for filename in rf_file_list) for i, filename in enumerate(rf_file_list): print( '-----------------------------------------------------------------' ) print('Random Forest file {}: {}'.format(str(i + 1), filename)) print('Output text file {}'.format(outfiles[i]))
outfile_multiple_list = list(out_dir + year_name + '_multiple_fire.csv' for year_name in year_names) # list of values # initialize list of lists single_burns = list(list() for _ in year_edges) multiple_burns = list(list() for _ in year_edges) # iterate thru files for fi, filename in enumerate(filelist): Opt.cprint('Reading file : {}'.format(filename)) # filepath infile = in_dir + filename # read dictionaries val_dicts = Handler(infile).read_from_csv(return_dicts=True) print('Number of samp in file {}: {}'.format(filename, str(len(val_dicts)))) # iterate thru dictionaries for val_dict in val_dicts: out_dict = dict() # copy all band/column values to output dict for key in decid_bands + tc_bands + decid_uncertainty_bands + tc_uncertainty_bands + fire_cols: out_dict[key] = val_dict[key] year_list = list()
from modules import Vector, Handler if __name__ == '__main__': csvfile = "D:/Shared/Dropbox/projects/NAU/landsat_diva/data2/gee_mack_data_extract_30_v2019_07_09T20_11_02_3_2000.csv" outfile = "D:/Shared/Dropbox/projects/NAU/landsat_diva/data2/gee_mack_data_extract_30_v2019_07_09T20_11_02_3_2000.shp" samp_data = Handler(csvfile).read_from_csv(return_dicts=True) headers = list(samp_data[0]) spref_str = '+proj=longlat +datum=WGS84' wkt_list = list() attr_list = list() attribute_types = dict() for header in headers: attribute_types[header] = Handler.string_to_type(samp_data[0][header], return_type=True) # attribute_types['DecidFracR'] = 'str' # attribute_types['TreeCovR'] = 'str' for k, v in attribute_types.items(): print('{} - {}'.format(str(k), str(v))) for elem in samp_data: wkt_list.append( Vector.wkt_from_coords((elem['longitude'], elem['latitude']), geom_type='point'))
heatmap.draw() # ************************************************** """ # plotting regression heatmap2 val_log = "C:/temp/tree_cover/out_tc_2010_samp_v1_RF_2.txt" ''' pickle_file = "C:/temp/tree_cover/out_tc_2010_samp_v1_RF_2.pickle" classifier = RFRegressor.load_from_pickle(pickle_file) data_dict = classifier.sample_predictions(classifier.data) x = data_dict['obs_y'] y = data_dict['pred_y'] ''' val_lines = Handler(val_log).read_text_by_line() x = list(float(elem.strip()) for elem in val_lines[0].split(',')[1:]) y = list(float(elem.strip()) for elem in val_lines[1].split(',')[1:]) pts = [(x[i], y[i]) for i in range(0, len(x))] print(pts[0:10]) plotfile = 'C:/temp/tree_cover/out_tc_2010_samp_v1_RF_2_val.png' plot_heatmap = { 'type': 'rheatmap2', 'points': pts, 'xlabel': 'Observed Tree Cover', 'ylabel': 'Predicted Tree Cover', 'color_bar_label': 'Data-points per bin',