def test_qrnn_datasets(self, backend):
"""
Provide data as dataset object instead of numpy arrays.
"""
set_backend(backend)
backend = get_backend(backend)
data = backend.BatchedDataset((self.x_train, self.y_train), 256)
qrnn = QRNN(self.x_train.shape[1], np.linspace(0.05, 0.95, 10))
qrnn.train(data, maximum_epochs=1)
def read_qrnn(file, inChannels, target):
data = iciData(test_file, inChannels, target, batch_size=batchSize)
qrnn = QRNN.load(file)
y_pre, y_prior, y0, y, y_pos_mean = S.predict(data, qrnn, add_noise=True)
return y_pre, y_prior, y0, y, y_pos_mean
def read_qrnn(qrnn_file, test_file, inChannels, target):
data = mwhsData(test_file, inChannels, target, ocean=False, test_data=True)
qrnn = QRNN.load(qrnn_file)
y_pre, y_prior, y0, y, y_pos_mean = S.predict(data, qrnn, \
add_noise = False)
return y_pre, y_prior, y0, y, y_pos_mean
def read_qrnn(file, inChannels, target):
data = iciData(test_file, inChannels, target, batch_size=batchSize)
# read QRNN
# file = 'qrnn_ici_%s_%s_%s_single.nc'%(depth, width, target)
# print (file)
qrnn = QRNN.load(file)
y_pre, y_prior, y0, y, y_pos_mean = S.predict(data, qrnn, add_noise=True)
return y_pre, y_prior, y0, y, y_pos_mean
def test_save_qrnn(self, backend):
"""
Test saving and loading of QRNNs.
"""
set_backend(backend)
qrnn = QRNN(self.x_train.shape[1], np.linspace(0.05, 0.95, 10))
f = tempfile.NamedTemporaryFile()
qrnn.save(f.name)
qrnn_loaded = QRNN.load(f.name)
x_pred = qrnn.predict(self.x_train)
x_pred_loaded = qrnn.predict(self.x_train)
if not type(x_pred) == np.ndarray:
x_pred = x_pred.detach()
assert np.allclose(x_pred, x_pred_loaded)
binstep = 0.5
bins = np.arange(-20, 15, binstep)
iq = np.argwhere(quantiles == 0.5)[0,0]
#%% Uncertainty plot
plt.rcParams.update({'font.size': 26})
inChannels = np.array([target, 'I5V' , 'I6V', 'I7V', 'I8V', 'I9V', 'I10V', 'I11V'])
i183, = np.argwhere(inChannels == target)[0]
data = iciData("TB_ICI_test.nc",
inChannels, target,
batch_size = batchSize)
file = 'qrnn_ici_%s_%s_%s_single.nc'%(depth, width, target)
print (file)
qrnn = QRNN.load(file)
y_pre, y_prior, y0, y, y_pos_mean = S.predict(data, qrnn, add_noise = True)
fig, ax = plt.subplots(1, 1, figsize = [8, 8])
x = np.arange(-3, 4, 1)
ii = 0
y_all = []
randomList = random.sample(range(0, 24000), 1500)
for i in randomList:
ii +=1
#for i in ind:
y1 = y_pre[i, :] - y_pre[i, 3]
y_all.append(y1)
ax.plot(x, y1, color = colors["grey"], alpha = 0.4)
#%%
y_all = np.stack(y_all)
inChannels = np.concatenate([[target], channels])
# inChannels = np.array(channels)
print(qrnn_dir, channels, inChannels)
qrnn_file = os.path.join(qrnn_path, "qrnn_mwhs_%s.nc"%(target))
print (qrnn_file)
i183, = np.argwhere(inChannels == target)[0]
data = mwhsData(test_file,
inChannels, target, ocean = False, test_data = True)
qrnn = QRNN.load(qrnn_file)
y_pre, y_prior, y0, y, y_pos_mean, x = predict(data, qrnn, \
add_noise = False)
im = (np.abs(y_pre[:, 3] - y_prior[:, i183] )< 5)
# bia, std, ske, mae = S.calculate_statistics(y_prior, y0, y, y_pre[:, 3], im, i183)
SI = np.abs(y_prior[:, 0] - y_prior[:, 1])
im = SI < 5.0
y_pre = y_prior[im, i183]
y0 = y0[im]
y = y[im]
y_prior = y_prior[im , :]
def test_qrnn(self, backend):
"""
Test training of QRNNs using numpy arrays as input.
"""
set_backend(backend)
qrnn = QRNN(self.x_train.shape[1], np.linspace(0.05, 0.95, 10))
qrnn.train((self.x_train, self.y_train), maximum_epochs=1)
qrnn.predict(self.x_train)
x, qs = qrnn.cdf(self.x_train[:2, :])
assert qs[0] == 0.0
assert qs[-1] == 1.0
x, y = qrnn.pdf(self.x_train[:2, :])
assert x.shape == y.shape
mu = qrnn.posterior_mean(self.x_train[:2, :])
assert len(mu.shape) == 1
r = qrnn.sample_posterior(self.x_train[:4, :], n=2)
assert r.shape == (4, 2)
r = qrnn.sample_posterior_gaussian_fit(self.x_train[:4, :], n=2)
assert r.shape == (4, 2)
import argparse
parser = argparse.ArgumentParser(description='Train unet.')
parser.add_argument("training_data", type=str, nargs=1, help="The training data.")
parser.add_argument("levels", type=int, nargs=1, help="Number of downscaling blocks.")
parser.add_argument("n_features", type=int, nargs=1, help="Number of features in network.")
args = parser.parse_args()
training_data = args.training_data[0]
level = args.levels[0]
n_features = args.n_features[0]
################################################################################
# Train network
################################################################################
data = GpmData(training_data)
n = len(data)
training_data, validation_data = torch.utils.data.random_split(data, [int(0.9 * n), n - int(0.9 * n)])
training_loader = DataLoader(training_data, batch_size=16, shuffle=True)
validation_loader = DataLoader(validation_data, batch_size=16, shuffle=True)
quantiles = [0.05, 0.15, 0.25, 0.35, 0.45, 0.5, 0.55, 0.65, 0.75, 0.85, 0.95]
unet = UNet(13, 11, 64, 5)
qrnn = QRNN(13, quantiles, model = unet)
qrnn.train(training_loader, validation_loader, gpu = True, lr = 1e-2, n_epochs=20)
qrnn.train(training_loader, validation_loader, gpu = True, lr = 5e-3, n_epochs=20)
qrnn.train(training_loader, validation_loader, gpu = True, lr = 2e-3, n_epochs=20)
qrnn.model.save("unet.pt")
ci_bins = np.arange(0, 40, 1)
#ci_bins = np.arange(-30, 30, 1)
center_ci_bins = (ci_bins[:-1] + ci_bins[1:]) / 2
bins = np.arange(-40, 0, 1)
center = (bins[:-1] + bins[1:]) / 2
for i, c183 in enumerate(channels):
inChannels = np.array([c183, 'C41', 'C42', 'C43', 'C44'])
test_data = awsTestData(os.path.expanduser(
"~/Dendrite/Projects/AWS-325GHz/AWS/data/TB_AWS_m60_p60_noise_four_test.nc"
),
inChannels,
option=4)
qrnn = QRNN.load('qrnn_4_128_%s.nc' % c183)
y_pre, y_prior, y0, y, y_pos_mean = predict(test_data, qrnn)
mean_ci, bins = bin_uncertainty(y_pre, y_prior[:, 0], y0)
#mean_ci, bins = bin_errors(y_pre, y_prior[:, 0], y0)
ax.plot(center, mean_ci, linewidth='2.5', color=colors[i])
ax.xaxis.set_minor_locator(MultipleLocator(10))
ax.yaxis.set_minor_locator(MultipleLocator(5))
ax.grid(which='both', alpha=0.2)
ci = y_pre[:, 5] - y_pre[:, 1]
hist = histogram(ci, ci_bins)
ax1.plot(center_ci_bins, hist, color=colors[i], linewidth=2.5)
# ax[i].set(ylim = [0, 1])
import argparse
parser = argparse.ArgumentParser(description='Train unet.')
parser.add_argument("training_data", type=str, nargs=1, help="The training data.")
parser.add_argument("levels", type=int, nargs=1, help="Number of downscaling blocks.")
parser.add_argument("n_features", type=int, nargs=1, help="Number of features in network.")
args = parser.parse_args()
training_data = args.training_data[0]
level = args.levels[0]
n_features = args.n_features[0]
################################################################################
# Train network
################################################################################
data = GpmData(training_data)
n = len(data)
training_data, validation_data = torch.utils.data.random_split(data, [int(0.9 * n), n - int(0.9 * n)])
training_loader = DataLoader(training_data, batch_size=32, shuffle=True)
validation_loader = DataLoader(validation_data, batch_size=32, shuffle=True)
skip_connection = "all"
quantiles = np.array([0.01, 0.05, 0.15, 0.25, 0.35, 0.45, 0.5,
0.55, 0.65, 0.75, 0.85, 0.95, 0.99])
unet = UNet(13, n_features=128, quantiles, skip_connection=skip_connection)
qrnn = QRNN(13, model=unet)
qrnn.train(training_loader, gpu = True, lr = 1e-2, momentum=0.99)
qrnn.save("unet.pt")
fig, ax = plt.subplots(1, 5, figsize=[50, 10])
plt.subplots_adjust(wspace=0.001)
for i, target in enumerate(targets):
inChannels_single = np.array(['I1V', 'I2V', 'I3V', 'MWI-15', 'MWI-16'])
file_single = 'qrnn_ici_%s_%s_%s_mwi-alone.nc' % (depth, width, target)
data = iciData(test_file,
inChannels_single,
target,
batch_size=batchSize)
i183, = np.argwhere(inChannels_single == target)[0]
# read QRNN
file_single = 'qrnn_ici_%s_%s_%s_mwi-alone.nc' % (depth, width, target)
print(file_single)
qrnn = QRNN.load(file_single)
y_pre, y_prior, y0, y, y_pos_mean = S.predict(data,
qrnn,
add_noise=True)
hist0, hist1, hist2, bins = PDF_uncertainty_bins(
y_pre, y0, y_prior[:, i183], ulim)
center = (bins[:-1] + bins[1:]) / 2
ax[i].plot(center, hist0, 'k', linewidth=2.5)
ax[i].plot(center, hist1, 'r', linewidth=2.5)
ax[i].plot(center, hist2, 'b', linewidth=2.5)
# ax[i].plot(center, hist3, 'g', linewidth = 2.5)
ax[i].xaxis.set_minor_locator(MultipleLocator(5))
ax[i].grid(which='both', alpha=0.4)
ax[i].set_ylim(0, 1)