nc_var_short = var_short_dict[nc_var]
temp_path += '/' + nc_var_short
zst_temp_path = temp_path + '/zonal_stats'
zst_nc_path_file = '/g/data/ub8/au/FMC/c6/mosaics/deciles/zonal_stats' + '/' + plg_source + '_' + nc_var_short + '_zonal_stat.nc'
zst_csv_path_file = '/g/data/ub8/au/FMC/c6/mosaics/deciles/zonal_stats' + '/' + plg_source + '_' + nc_var_short + '_zonal_stat.csv'
zst_nc_temp_path_file0 = zst_temp_path + '/' + plg_source + '_' + nc_var_short + '_zonal_stat_time_double.nc'
zst_nc_temp_path_file = zst_temp_path + '/' + plg_source + '_' + nc_var_short + '_zonal_stat.nc'
zst_list = []
zst_temp_path_files = glob.glob(zst_temp_path + '/*_dc_zonal_stat.pk')
for zst_temp_path_file in zst_temp_path_files:
zst_list.append(load_pickle(zst_temp_path_file))
df_dict = {}
min_list_all = []
max_list_all = []
avg_list_all = []
dt_list_all = []
plg_list_all = []
for plg_id in zst_list[0].keys():
plg_list, dt_list, min_list, max_list, avg_list = [], [], [], [], []
for zst_dict in zst_list:
for dt, sd in zst_dict[plg_id].items():
dt_list.append(dt)
from copy import copy
tile_path_file = sys.argv[1]
# tile_path_file = '/g/data/xc0/project/FMC_Australia/calc_deciles/data/temp/temp_tiles/flam_1.nc/flammability__1.pk'
tile_path = os.path.dirname(tile_path_file)
tile_file = os.path.basename(tile_path_file)
tile_dc_path = tile_path + '/' + 'deciles'
if not os.path.exists(tile_dc_path):
os.mkdir(tile_dc_path)
tile_dc_file = tile_file.replace('.pk', '_dc.pk')
tile_dc_path_file = tile_dc_path + '/' + tile_dc_file
print('started *****,', tile_file)
tile_dict = load_pickle(tile_path_file)
dc_dict = copy(tile_dict)
del dc_dict['data']
dc_dict['deciles'] = {}
cnt_q = 0
for q in range(10, 100, 10):
dc_dict['deciles'][q] = np.nanpercentile(tile_dict['data'], q, axis=0)
cnt_q += 1
save_pickle(dc_dict, tile_dc_path_file)
print('saved ******', tile_file)
tile_path_files = sorted(glob.glob(tile_path + '/*.pk'))
print('total number of tiles', len(tile_path_files))
calc_dec_concurrent(tile_path_files=tile_path_files)
if step == 'merging':
""" STEP 2: merge and export to nc """
d_arr = np.empty((9, dim_y, dim_x))
d_arr[:] = np.nan
tile_dc_path = tile_path + '/' + 'deciles'
tile_dc_path_files = glob.glob(tile_dc_path + '/*_dc.pk')
for tile_dc_path_file in tile_dc_path_files:
tile_dc_dict = load_pickle(tile_dc_path_file)
st_y = tile_dc_dict['st_y']
en_y = tile_dc_dict['en_y']
st_x = tile_dc_dict['st_x']
en_x = tile_dc_dict['en_x']
print(tile_dc_path_file, st_y, en_y, st_x, en_x)
cnt_q = 0
for q in range(10, 100, 10):
d_arr[cnt_q, st_y:en_y, st_x:en_x] = tile_dc_dict['deciles'][q]
cnt_q += 1
tile_dc_merge_path_file = tile_dc_merge_path + '/' + 'tiles_merge.pk'
if os.path.exists(tile_dc_merge_path_file):
os.remove(tile_dc_merge_path_file)
save_pickle(d_arr, tile_dc_merge_path_file)
import numpy as np
from functions_io import load_pickle
import datetime
pk_path_file = '/g/data/xc0/project/FMC_Australia/calc_deciles/data/temp/fmc/zonal_stats/fmc_c6_2018_dc_zonal_stat.pk'
zst_dict = load_pickle(pk_path_file)
for key, val in zst_dict.items():
print(key, val[datetime.datetime(2018, 8, 1, 0, 0)]['avg'])