num_train[domain] = 0
num_valid[domain] = 0
# output batch filenames
NAME_train = '{}_BATCH_{}_T{}-{}_{}'.format(
var, size, source, domain, sea)
NAME_valid = '{}_BATCH_{}_V{}-{}_{}'.format(
var, size, source, domain, sea)
# random cropping + batch gen
# print('----- Training data process -----')
# FEATURE_train = pu.random_cropping(input_train_3d, keys_3d, input_2d, keys_2d, mask[domain],
# size, gap, ind_train, rnd_range=2, clim=False)
# FEATURE_train = pu.feature_norm(FEATURE_train, method=norm)
# pu.batch_gen(FEATURE_train, batch_size, BATCH_dir, NAME_train, 0);
print('----- Validation data process -----')
FEATURE_valid = pu.random_cropping(input_valid_3d,
keys_3d,
input_2d,
keys_2d,
mask[domain],
size,
gap,
ind_valid,
rnd_range=2,
clim=False)
FEATURE_valid = pu.feature_norm(FEATURE_valid, method=norm)
pu.batch_gen(FEATURE_valid, batch_size, BATCH_dir,
NAME_valid, 0)
input_train_3d[key] = hdf_io[key][pick_train, ...]
input_valid_3d[key] = hdf_io[key][pick_valid, ...]
for i, size in enumerate(sizes):
gap = gaps[i]
for sea in seasons:
# output names
NAME_train = {}; NAME_valid = {}
# indices
ind_train = IND_train['{}_train'.format(sea)]
ind_valid = IND_valid['{}_valid'.format(sea)]
NAME_train = '{}_BATCH_{}_T{}_{}'.format(var, size, source, sea)
NAME_valid = '{}_BATCH_{}_V{}_{}'.format(var, size, source, sea)
# random cropping + batch gen
print('----- Training data process -----')
FEATURE_train = pu.random_cropping_regrid(input_train_3d, keys_3d, input_2d, keys_2d, land_mask, size, gap,
ind_train, var='TMEAN', rnd_range=4)
FEATURE_train = pu.feature_norm(FEATURE_train, method=norm, self_norm=True)
pu.batch_gen(FEATURE_train, batch_size, BATCH_dir, NAME_train, 0);
print('----- Validation data process -----')
FEATURE_valid = pu.random_cropping_regrid(input_valid_3d, keys_3d, input_2d, keys_2d, land_mask, size, gap,
ind_valid, var='TMEAN', rnd_range=4)
FEATURE_valid = pu.feature_norm(FEATURE_valid, method=norm, self_norm=True)
pu.batch_gen(FEATURE_valid, batch_size, BATCH_dir, NAME_valid, 0);
var, size, domain, sea)
# random cropping + batch gen
print('----- Training data process -----')
FEATURE_train = pu.random_cropping(input_train_3d,
keys_3d,
input_2d,
keys_2d,
mask[domain],
size,
gap,
ind_train,
rnd_range=2)
FEATURE_train = pu.feature_norm(FEATURE_train,
method=norm)
pu.batch_gen(FEATURE_train, batch_size, BATCH_dir,
NAME_train, 0)
print('----- Validation data process -----')
FEATURE_valid = pu.random_cropping(input_valid_3d,
keys_3d,
input_2d,
keys_2d,
mask[domain],
size,
gap,
ind_valid,
rnd_range=2)
FEATURE_valid = pu.feature_norm(FEATURE_valid,
method=norm)
pu.batch_gen(FEATURE_valid, batch_size, BATCH_dir,
NAME_valid, 0)
gaps = [2]
sizes = [96]
# clean-up
if del_old:
cmd = 'rm {}ETOPO*npy'.format(BATCH_dir); print(cmd)
subprocess.call(cmd, shell=True)
cmd = 'rm {}ETOPO*npy'.format(BATCH_dir+'temp_batches/'); print(cmd)
subprocess.call(cmd, shell=True)
# etopo fields
input_2d = {}; keys_2d = ['etopo_4km', 'etopo_regrid']
with h5py.File(PRISM_dir+'PRISM_TMAX_features_2015_2020.hdf', 'r') as hdf_io:
for key in keys_2d:
input_2d[key] = hdf_io[key][...]
# land mask
with h5py.File(PRISM_dir+'PRISM_TMAX_features_2015_2020.hdf', 'r') as hdf_io:
land_mask = hdf_io['land_mask'][...]
land_mask[:ind_tune , :] = True # trans + tuning domain
# loop over sizes
for i, size in enumerate(sizes):
gap = gaps[i]
NAME = 'ETOPO_BATCH_{}_'.format(size)
# random cropping + batch gen
FEATURE = pu.random_cropping_2d(input_2d, keys_2d, land_mask, size, gap, rnd_range=1)
FEATURE = pu.feature_norm(FEATURE, method=norm, self_norm=True)
pu.batch_gen(FEATURE, batch_size, BATCH_dir, NAME, 0);