def get_next_row_dict(filename):
with open(filename, "r") as f:
reader = csv.reader(f)
count = 0
colnames = list()
for file_row in reader:
if count == 0:
colnames = file_row
else:
values = list(
Handler.string_to_type(elem.strip()) for elem in file_row)
if len(values) == len(colnames):
row_dict = dict(zip(colnames, values))
yield row_dict
count += 1
plot_file = in_dir + "RF{}_{}_{}_{}_cutoff_{}_deg{}_{}.png".format(ylabel, xlabel, zlabel, str(bin_limit),
str(int(z_thresh*scaledown_treecover)), deg,
datetime.now().isoformat().split('.')[0]
.replace('-', '').replace(':', ''))
plot_file2 = in_dir + "RF{}_{}_{}_{}_cutoff_{}_deg{}_{}2.png".format(ylabel, xlabel, zlabel, str(bin_limit),
str(int(z_thresh*scaledown_treecover)), deg,
datetime.now().isoformat().split('.')[0]
.replace('-', '').replace(':', ''))
print('Reading file : {}'.format(csv_file))
print('Plot file: {}'.format(plot_file))
val_dicts = Handler(csv_file).read_from_csv(return_dicts=True,
read_random=True,
line_limit=None, )
for val_dict in val_dicts[0:10]:
print(val_dict)
basename = Handler(csv_file).basename.split('.csv')[0]
elem_list = list()
for val_dict in val_dicts:
for i in range(3):
x, y, z = val_dict[xvar_bands[i]], val_dict[yvar_bands[i]], val_dict[zvar_bands[i]]
oz1, oz2 = val_dict[oz_bands[0]], val_dict[oz_bands[1]]
if (type(x) in (int, float)) and (type(y) in (int, float)) and (type(z) in (int, float)):
version = 11
filelist = list(in_dir + 'year_{}_{}_{}_fire.csv'.format(
str(year_edge[0])[2:],
str(year_edge[1])[2:], fire_type) for year_edge in year_edges
for fire_type in fire_types)
zvals_list = list(np.zeros((zcount_lim, leny, lenx)) for _ in years)
zcount_list = list(np.zeros((leny, lenx), dtype='int16') for _ in years)
for filename in filelist:
if 'single' in filename:
print(filename)
val_dicts = Handler(filename).read_from_csv(return_dicts=True)
print(len(val_dicts))
for ii, decid_year in enumerate(years):
max_time = 0
decid_band = 'decid{}'.format(str(decid_year))
tc_band = 'tc{}'.format(str(decid_year))
for val_dict in val_dicts:
if (type(val_dict[decid_band]).__name__ in ('int', 'float')) and \
(type(val_dict[tc_band]).__name__ in ('int', 'float')):
if float(val_dict['longitude']) > -142.0:
import ee
import datetime
from modules import Logger, Handler
if __name__ == '__main__':
ee.Initialize()
folder = "c:/temp/decid/"
scale = 30
cutoff = 4000
fire_year_limit = 1900
elev_lim = 500
Handler(dirname=folder).dir_create()
OUTFILE = folder + \
'gee_mack_data_extract_{}_v{}.csv'.format(str(scale),datetime.datetime.now().isoformat()
.split('.')[0].replace('-', '_').replace(':', '_'))
logfile = OUTFILE.split('.csv')[0] + '.log'
log = Logger('SAMP', filename=logfile, stream=True)
log.lprint('Outfile: {}'.format(OUTFILE))
log.lprint('Logfile: {}'.format(logfile))
decid_layer = ee.Image(
"users/masseyr44/decid/decid_mosaic_2000_prediction_vis_nd_3")
from modules.plots import Plot
from modules import Handler, Opt, Sublist
from sys import argv
if __name__ == '__main__':
# script, in_dir = argv
in_dir = 'c:/temp/check2/'
Handler(dirname=in_dir).dir_create()
csv_files = Handler(dirname=in_dir).find_all('.csv')
for csv_file in csv_files:
Opt.cprint('Reading file : {}'.format(csv_file))
val_dicts = Handler(csv_file).read_from_csv(return_dicts=True,
read_random=True,
line_limit=None,
percent_random=10.0)
bandname = Handler(csv_file).basename.split('.csv')[0]
plot_file = in_dir + "{}_21.png".format(bandname)
xvar = '{}_2'.format(bandname)
xlabel = 'Season 2 {}'.format(bandname.upper())
yvar = '{}_1'.format(bandname)
'Unable to find the server' in e.args[0] or \
'getaddrinfo failed' in e.args[0] or \
'connection attempt failed' in e.args[0]:
log.lprint('Waiting {} secs...'.format(str(wait)))
time.sleep(wait)
continue
else:
log.lprint('Exiting...')
break
# all extracted dictionaries to file
if i == 0:
Handler.write_to_csv(temp_dicts,
header=True,
append=False,
outfile=OUTFILE)
else:
Handler.write_to_csv(temp_dicts,
header=False,
append=True,
outfile=OUTFILE)
time2 = datetime.datetime.now()
log.lprint(
'Time taken for site {s} ({ii} of {nn}): {t} seconds'.format(
s=str(temp_dicts[0]['site']),
ii=str(site_count + 1),
nn=str(n_samp),
'ak_fire_multi_extract_v2019_08_27T20_22_51.csv'
]
res = 250
version = 1
fire_year_limit = (1950, 2018)
for file_base_name in filelist:
filename = folder + file_base_name
Opt.cprint('File: {}'.format(file_base_name))
OUTFILE = filename.split('.')[0] + 'reformat_{}_{}.csv'.format(
str(res), str(version))
# convert csv files to dicts
list_dicts = Handler(filename).read_from_csv(return_dicts=True)
Opt.cprint(len(list_dicts))
outcolnames = [
'ID', 'FIREID', 'SIZE_HA', 'longitude', 'latitude', 'decid1992',
'decid2000', 'decid2005', 'decid2010', 'decid2015', 'decid1992u',
'decid2000u', 'decid2005u', 'decid2010u', 'decid2015u', 'tc1992',
'tc2000', 'tc2005', 'tc2010', 'tc2015', 'tc1992u', 'tc2000u',
'tc2005u', 'tc2010u', 'tc2015u'
]
samp_list = list()
list_latlon = list()
list_years = list()
for elem in list_dicts:
module_path = "D:/temp/decid02092020/decid/src/"
sys.path.append(module_path)
from modules import RFRegressor, Handler
ulim = 1.0
llim = 0.0
over_adjust_ = 1.0
clip_ = 0.01
active_dir = "D:/temp/pickle_dir/working/"
outdir = active_dir + "out/"
rf_file_list = Handler(dirname=active_dir).find_all('*.pickle')
'''
rf_file_list = list(active_dir + filename for filename in
["out_tc_2010_samp_v1_summer_RF_772.pickle",
"out_tc_2010_samp_v1_RF_59.pickle"])
'''
outfiles = list(
filename.replace('.pickle', '.txt') for filename in rf_file_list)
for i, filename in enumerate(rf_file_list):
print(
'-----------------------------------------------------------------'
)
print('Random Forest file {}: {}'.format(str(i + 1), filename))
print('Output text file {}'.format(outfiles[i]))
outfile_multiple_list = list(out_dir + year_name + '_multiple_fire.csv' for year_name in year_names)
# list of values
# initialize list of lists
single_burns = list(list() for _ in year_edges)
multiple_burns = list(list() for _ in year_edges)
# iterate thru files
for fi, filename in enumerate(filelist):
Opt.cprint('Reading file : {}'.format(filename))
# filepath
infile = in_dir + filename
# read dictionaries
val_dicts = Handler(infile).read_from_csv(return_dicts=True)
print('Number of samp in file {}: {}'.format(filename,
str(len(val_dicts))))
# iterate thru dictionaries
for val_dict in val_dicts:
out_dict = dict()
# copy all band/column values to output dict
for key in decid_bands + tc_bands + decid_uncertainty_bands + tc_uncertainty_bands + fire_cols:
out_dict[key] = val_dict[key]
year_list = list()
from modules import Vector, Handler
if __name__ == '__main__':
csvfile = "D:/Shared/Dropbox/projects/NAU/landsat_diva/data2/gee_mack_data_extract_30_v2019_07_09T20_11_02_3_2000.csv"
outfile = "D:/Shared/Dropbox/projects/NAU/landsat_diva/data2/gee_mack_data_extract_30_v2019_07_09T20_11_02_3_2000.shp"
samp_data = Handler(csvfile).read_from_csv(return_dicts=True)
headers = list(samp_data[0])
spref_str = '+proj=longlat +datum=WGS84'
wkt_list = list()
attr_list = list()
attribute_types = dict()
for header in headers:
attribute_types[header] = Handler.string_to_type(samp_data[0][header],
return_type=True)
# attribute_types['DecidFracR'] = 'str'
# attribute_types['TreeCovR'] = 'str'
for k, v in attribute_types.items():
print('{} - {}'.format(str(k), str(v)))
for elem in samp_data:
wkt_list.append(
Vector.wkt_from_coords((elem['longitude'], elem['latitude']),
geom_type='point'))
heatmap.draw()
# **************************************************
"""
# plotting regression heatmap2
val_log = "C:/temp/tree_cover/out_tc_2010_samp_v1_RF_2.txt"
'''
pickle_file = "C:/temp/tree_cover/out_tc_2010_samp_v1_RF_2.pickle"
classifier = RFRegressor.load_from_pickle(pickle_file)
data_dict = classifier.sample_predictions(classifier.data)
x = data_dict['obs_y']
y = data_dict['pred_y']
'''
val_lines = Handler(val_log).read_text_by_line()
x = list(float(elem.strip()) for elem in val_lines[0].split(',')[1:])
y = list(float(elem.strip()) for elem in val_lines[1].split(',')[1:])
pts = [(x[i], y[i]) for i in range(0, len(x))]
print(pts[0:10])
plotfile = 'C:/temp/tree_cover/out_tc_2010_samp_v1_RF_2_val.png'
plot_heatmap = {
'type': 'rheatmap2',
'points': pts,
'xlabel': 'Observed Tree Cover',
'ylabel': 'Predicted Tree Cover',
'color_bar_label': 'Data-points per bin',