def test_partly_pretrained_train(self):
path_to_pretrained_json = os.path.join(get_project_root(),
'res_experiments',
'hps_common_mlp.json')
path_to_pretrained_model = os.path.join(get_project_root(),
'res_experiments',
'trained_models', 'test.pt')
hps_pretrained = HyperParams.from_file(
path_to_json=path_to_pretrained_json)
model_common = Model(hps_pretrained)
model_common.load_model(path_to_pretrained_model)
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model_part = Model(hps)
model_part.load_pretrained_bottom(path_to_pretrained_model,
path_to_pretrained_json)
filename = '../data/small_parameters_base.fits'
history = model_part.train(
filename=filename,
path_to_save='../res_experiments/trained_models/partly_test.pt',
pretrained_bottom=True,
logdir='../res_experiments/')
par_0_com = list(zip(*model_common.net.bottom.mlp.named_parameters())
)[1][0].detach().numpy()
par_0_par = list(zip(*model_part.net.bottom.mlp.named_parameters())
)[1][0].detach().numpy()
assert par_0_com == pytest.approx(par_0_par)
def test_predict_refer(self, common_mlp_rescale_hps):
project_path = get_project_root()
refer_path = get_project_root(
) / 'res_experiments' / 'predictions' / '20170905_030404.fits'
model = Model(common_mlp_rescale_hps)
predicted = model.predict_refer(refer_path)
assert True
def test_partly_pretrained_init(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_common_mlp.json')
path_to_model = os.path.join(get_project_root(), 'res_experiments',
'trained_models', 'test.pt')
hps = HyperParams.from_file(path_to_json=path_to_json)
model_common = Model(hps)
model_common.load_model(path_to_model)
pretrained_dict = model_common.net.state_dict()
pretrained_dict = {
k: v
for k, v in pretrained_dict.items() if 'bottom' in k
}
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model_part = Model(hps)
model_dict = model_part.net.state_dict()
model_dict.update(pretrained_dict)
model_part.net.load_state_dict(model_dict)
par_0_part = list(
zip(*model_part.net.bottom.mlp.named_parameters()))[1][0]
par_0_com = list(
zip(*model_common.net.bottom.mlp.named_parameters()))[1][0]
def test_partly_pretrained_fit_step(self):
path_to_pretrained_json = os.path.join(get_project_root(),
'res_experiments',
'hps_common_mlp.json')
path_to_pretrained_model = os.path.join(get_project_root(),
'res_experiments',
'trained_models', 'test.pt')
hps_pretrained = HyperParams.from_file(
path_to_json=path_to_pretrained_json)
model_common = Model(hps_pretrained)
model_common.load_model(path_to_pretrained_model)
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model_part = Model(hps)
model_part.load_pretrained_bottom(path_to_pretrained_model,
path_to_pretrained_json)
train_loader, val_loader = model_part.make_loader(
filename='../data/small_parameters_base.fits')
loss = model_part.fit_step(train_loader, pretrained_bottom=True)
par_0_com = list(zip(*model_common.net.bottom.mlp.named_parameters())
)[1][0].detach().numpy()
par_0_par = list(zip(*model_part.net.bottom.mlp.named_parameters())
)[1][0].detach().numpy()
assert par_0_com == pytest.approx(par_0_par)
def test_full_mlp_stack(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
filename = get_project_root() / 'data' / 'small_parameters_base.fits'
params = fits.open(filename)[0].data
hps = HyperParams.from_file(path_to_json=path_to_json)
bottom = getattr(models, hps.bottom_net)
top = getattr(models, hps.top_net)
model = Model(hps)
x = model.make_loader(data_arr=params)
x_ = next(iter(x))
net = FullModel(hps, bottom, top)
out = net(x_['X'])
def test_normalize_output_array_angle(self):
project_path = get_project_root()
filename = os.path.join(project_path, 'data/small_parameters_base.fits')
params = fits.open(filename)[0].data[5:11]
sample = normalize_output(params, mode='range', logB=True)
assert sample.min() >= 0
assert sample.max() <= 1
def sample_from_database(self):
project_path = get_project_root()
filename = os.path.join(project_path, 'data/small_parameters_base.fits')
source = 'database'
sobj = SpectrumDataset(param_path=filename, source=source)
sample = sobj[2]
return sample
def test_resnet_train(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_resnet.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
hps.trainset = 5
model = Model(hps)
history = model.train()
assert history[0][0] > 0
def test_generate_spectrum_by_refer(self, common_mlp_rescale_hps):
model = Model(common_mlp_rescale_hps)
project_path = get_project_root()
refer_path = project_path / 'res_experiments' / 'predictions' / '20170905_030404.fits'
refer, names = open_param_file(refer_path, normalize=False)
params = refer.reshape(-1, 11)
spectrum = model.generate_batch_spectrum(params[:10], noise=False)
assert True
def test_flatten(self):
project_path = get_project_root()
filename = project_path / 'data' / 'small_parameters_base.fits'
param_batch = fits.open(filename)[0].data
obj = BatchHinodeME(param_batch)
spectrum = obj.compute_spectrum(with_ff=True, with_noise=True)
sample = FlattenSpectrum()(spectrum[:3, :])
assert True
def test_model_conv_train(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_conv.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model = Model(hps)
x = model.make_loader()
x_ = next(iter(x))
history = model.train()
assert history[0][0] > 0
def test_lm_inversion(self):
line_vector = (6302.5, 2.5, 1)
line_arg = 1000 * (np.linspace(6302.0692255, 6303.2544205, 56) - line_vector[0])
cont_scale = 40000
path_to_data = os.path.join(get_project_root(), 'data', '20170905_030404\\')
full_spectra = read_full_spectra(cont_scale, files_path=path_to_data)[0]
spectrum = full_spectra[0].reshape((512, 56, 4))[0]
x = lm_inversion(spectrum, line_arg=line_arg, line_vec=line_vector)
assert True
def test_normalize_output_array_angle(self):
project_path = get_project_root()
filename = os.path.join(project_path, 'data/small_parameters_base.fits')
params = fits.open(filename)[0].data[5:11]
sample = normalize_output(params, mode='range', logB=True, angle_transformation=True)
assert sample.min() >= 0
assert sample.max() < 1
assert sample[0, 1] == pytest.approx(np.sin(params[0, 1]*np.pi/180), 0.01)
assert True
def test_model_mlp_stack(self, sample_x):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
bottom = getattr(models, hps.bottom_net)
top = getattr(models, hps.top_net)
net = FullModel(hps, bottom, top)
out = net(sample_x)
assert True
def test_read_full_spectra(self):
cont_scale = 40000
path_to_data = os.path.join(get_project_root(), 'data',
'20170905_030404\\')
full_spectra = read_full_spectra(cont_scale, files_path=path_to_data)
flist = [
path_to_data + 'SP3D20170905_030404.4C.fits',
]
full_spectra = read_full_spectra(cont_scale, files_list=flist)
assert True
def flat(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
N = hps.batch_size
d_in = hps.n_input
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
flat = torch.randn(N, d_in, device=device, dtype=torch.float)
self.hps = hps
return flat
def test_model_conv_stack(self, sample_conv_x):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_conv.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
bottom = getattr(models, hps.bottom_net)
top = getattr(models, hps.top_net)
model = Model(hps)
x = model.make_loader()
x_ = next(iter(x))
net = FullModel(hps, bottom, top)
out = net(x_['X'])
assert True
def test_model_partly_ind_train(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model = Model(hps)
filename = '../data/small_parameters_base.fits'
history = model.train(
filename=filename,
path_to_save='../res_experiments/trained_models/partly_test.pt',
logdir='../res_experiments/')
# model.save_model(path_to_save='../res_experiments/trained_models/test.pt')
assert True
def test_common_model_train(self, common_mlp_rescale_hps):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_common_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model = Model(hps)
filename = '../data/small_parameters_base.fits'
history = model.train(
filename=filename,
pregen=True,
path_to_save='../res_experiments/trained_models/test.pt',
logdir='../res_experiments/')
model.save_model(path='../res_experiments/trained_models/common.pt')
assert True
def test_load_pretrained_bottom(self):
path_to_pretrained_json = os.path.join(get_project_root(),
'res_experiments',
'hps_common_mlp.json')
path_to_pretrained_model = os.path.join(get_project_root(),
'res_experiments',
'trained_models', 'test.pt')
hps_pretrained = HyperParams.from_file(
path_to_json=path_to_pretrained_json)
model_common = Model(hps_pretrained)
model_common.load_model(path_to_pretrained_model)
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_partly_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model_part = Model(hps)
model_part.load_pretrained_bottom(path_to_pretrained_model,
path_to_pretrained_json)
par_0_part = list(
zip(*model_part.net.bottom.mlp.named_parameters()))[1][0]
par_0_com = list(
zip(*model_common.net.bottom.mlp.named_parameters()))[1][0]
assert True
def test_continue_training(self, base_mlp_rescale_hps):
base_mlp_rescale_hps.trainset = 1
base_mlp_rescale_hps.valset = 1
base_mlp_rescale_hps.n_epochs = 5
base_mlp_rescale_hps.batch_size = 25
model = Model(base_mlp_rescale_hps)
path_to_save = os.path.join(get_project_root(), 'data', 'test.pt')
history = model.train(path_to_save=path_to_save, save_epoch=[3])
base_mlp_rescale_hps.trainset = 1
base_mlp_rescale_hps.valset = 1
base_mlp_rescale_hps.n_epochs = 2
base_mlp_rescale_hps.batch_size = 25
model.continue_training(path_to_save)
def test_predict_one_pixel_conv(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_conv.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
hps.trainset = 1
hps.valset = 1
hps.n_epochs = 1
hps.batch_size = 1
model = Model(hps)
history = model.train()
filename = Path(os.getcwd()).parent / 'data' / "20170905_030404.fits"
ref = fits.open(filename)
predicted, y, x, _ = model.predict_one_pixel(ref, 3, 4)
assert predicted[0].shape == torch.Size([1, 3])
def test_predict_full_image_conv(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_conv.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
hps.trainset = 1
hps.valset = 1
hps.n_epochs = 1
hps.batch_size = 1
model = Model(hps)
history = model.train()
filename = Path(os.getcwd()).parent / 'data' / "20170905_030404.fits"
ref = fits.open(filename)
predicted, params, lines, cont = model.predict_full_image(ref,
cnn=True)
assert predicted.shape == (ref[1].data.shape + (3, ))
def test_generate_batch_spectrum(self, common_mlp_rescale_hps, params):
model = Model(common_mlp_rescale_hps)
data = model.generate_batch_spectrum(params, noise=False)
project_path = get_project_root()
filename = project_path / 'data' / 'small_parameters_base.fits'
source = 'database'
sobj = SpectrumDataset(param_path=filename,
source=source,
transform=model.transform,
ff=True,
noise=False)
rec0 = sobj[0]
rec0ind = rec0['X'][0].detach().numpy()
rec0batch = data['X'][0][0].detach().numpy()
assert rec0ind == pytest.approx(rec0batch)
assert data['Y'][0].detach().numpy() == pytest.approx(
rec0['Y'].detach().numpy())
def test_model_ind_train(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_independent_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
model = Model(hps)
filename = '../data/small_parameters_base.fits'
history = model.train(
filename=filename,
path_to_save='../res_experiments/trained_models/test.pt',
logdir='../res_experiments/')
# model.save_model(path_to_save='../res_experiments/trained_models/test.pt')
# list(zip(*self.net.top.task_layers[0].named_parameters()))[1][0].grad
params_groups_0 = list(
zip(*model.net.top.task_layers[0].named_parameters()))
params_groups_3 = list(
zip(*model.net.top.task_layers[3].named_parameters()))
assert True
def param_vec_0(self):
project_path = get_project_root()
filename = project_path / 'data' / 'parameters_base.fits'
param_vec = fits.open(filename)[0].data[:2]
return param_vec
def param_batch(self):
project_path = get_project_root()
filename = project_path / 'data' / 'small_parameters_base.fits'
param_batch = fits.open(filename)[0].data
return param_batch
def param_array(self):
project_path = get_project_root()
filename = project_path / 'data' / 'small_parameters_base.fits'
param_array = fits.open(filename)[0].data[:10]
return param_array
def test_hyper_params(self):
path_to_json = os.path.join(get_project_root(), 'res_experiments',
'hps_base_mlp.json')
hps = HyperParams.from_file(path_to_json=path_to_json)
assert hps.activation == 'relu'
def test_predict_full_image(self):
cont_scale = 40000
path_to_data = os.path.join(get_project_root(), 'data', '20170905_030404\\')
full_spectra = read_full_spectra(cont_scale, files_path=path_to_data)[0][:10]
#spectrum = full_spectra[0].reshape((512, 56, 4))[0]
x = predict_full_image(full_spectra, 0)
