def test_linreg(n_epochs):
print('Test: linear regression')
x_values = []
y_values = []
for i in range(5):
x_values.append(i)
y_values.append(5*i + 2 + torch.randn(1).data.item())
x_data = np.array(x_values, dtype=np.float32).reshape(-1, 1)
y_data = np.array(y_values, dtype=np.float32).reshape(-1, 1)
answer = RegressionAnswer(LinearRegression(), lambda model : LBFGS(model.parameters()), torch.nn.MSELoss(), x_data, y_data).run(60)
test = [
RegressionTest('A2Grad-uni', LinearRegression(), lambda model : A2Grad(model.parameters(), 'uni', 1e-1), torch.nn.MSELoss(), x_data, y_data),
RegressionTest('A2Grad-inc', LinearRegression(), lambda model : A2Grad(model.parameters(), 'inc', 1e-1), torch.nn.MSELoss(), x_data, y_data),
RegressionTest('A2Grad-exp', LinearRegression(), lambda model : A2Grad(model.parameters(), 'exp', 1e-1), torch.nn.MSELoss(), x_data, y_data),
RegressionTest('Adam', LinearRegression(), lambda model : Adam(model.parameters()), torch.nn.MSELoss(), x_data, y_data),
RegressionTest('SGD', LinearRegression(), lambda model : SGD(model.parameters(), lr=1e-2), torch.nn.MSELoss(), x_data, y_data),
RegressionTest('LBFGS', LinearRegression(), lambda model : LBFGS(model.parameters()), torch.nn.MSELoss(), x_data, y_data)
]
plt.figure(figsize=(14, 8))
for i in range(len(test)):
test[i].run(n_epochs)
plt.plot(np.arange(1, n_epochs + 1), np.array(test[i].errors) - answer, label=test[i].name)
plt.legend(fontsize=12, loc=1)
plt.title('Linear regression')
plt.xlabel('Epoch')
plt.ylabel('MSE')
plt.savefig('linear.png')
plt.figure(figsize=(14, 8))
for i in range(len(test)):
plt.plot(np.arange(1, n_epochs + 1), np.array(test[i].errors) - answer, label=test[i].name)
plt.legend(fontsize=12, loc=1)
plt.ylim(0, 1e-5)
plt.title('Linear regression')
plt.xlabel('Epoch')
plt.ylabel('MSE')
plt.savefig('linear2.png')
points = np.arange(10, n_epochs, 10)
header = "method "
for i in points:
header += "{} ".format(i)
print(header)
for i in range(len(test)):
test[i].output(points)
print('')
def update_value_net(value_net, states, returns, l2_reg):
optimizer = LBFGS(value_net.parameters(), max_iter=25, history_size=5)
def closure():
optimizer.zero_grad()
values_pred = value_net(states)
value_loss = (values_pred - returns).pow(2).mean()
# weight decay
for param in value_net.parameters():
value_loss += param.pow(2).sum() * l2_reg
value_loss.backward()
return value_loss
optimizer.step(closure)
def train():
content_image, style_image, image_size = load_img_tensor()
# imshow(content_image)
# imshow(style_image)
# input_params = input_image(image_size)
input_params = nn.Parameter(content_image, requires_grad=True)
cnn = cnn_loader()
model, content_losses, style_losses = get_model_losses(
cnn, content_image, style_image, 1, 1000)
epoch = [0]
num_epoches = 100
optimizer = LBFGS([input_params])
content_loss_list = []
style_loss_list = []
while epoch[0] < num_epoches:
def closure():
optimizer.zero_grad()
model(input_params)
content_score = 0
style_score = 0
for cs in content_losses:
content_score += cs.loss
for ss in style_losses:
style_score += ss.loss
loss = content_score + style_score
loss.backward()
epoch[0] += 1
if epoch[0] % 50 == 1:
print('content score: {}, style score: {}'.format(
content_score, style_score))
content_loss_list.append(content_score)
style_loss_list.append(style_score)
return loss
optimizer.step(closure)
return input_params, content_loss_list, style_loss_list
def _fit(self, modules: nn.ModuleDict, train_dl: DeviceDataLoader,
valid_dl: DeviceDataLoader):
r""" Fits \p modules' learners to the training and validation \p DataLoader objects """
self._configure_fit_vars(modules)
for mod_name, module in modules.items():
lr = config.get_learner_val(mod_name,
LearnerParams.Attribute.LEARNING_RATE)
wd = config.get_learner_val(mod_name,
LearnerParams.Attribute.WEIGHT_DECAY)
is_lin_ff = config.DATASET.is_synthetic(
) and module.module.num_hidden_layers == 0
if is_lin_ff:
module.optim = LBFGS(module.parameters(), lr=lr)
else:
module.optim = AdamW(module.parameters(),
lr=lr,
weight_decay=wd,
amsgrad=True)
logging.debug(
f"{mod_name} Optimizer: {module.optim.__class__.__name__}")
for ep in range(1, config.NUM_EPOCH + 1):
# noinspection PyUnresolvedReferences
for _, module in modules.items():
module.epoch_start()
for batch in train_dl:
for _, module in modules.items():
module.process_batch(batch)
for _, module in modules.items():
module.calc_valid_loss(valid_dl)
self._log_epoch(ep, modules)
self._restore_best_model(modules)
self.eval()
def L_BFGS(spec,
transform_fn,
samples=None,
init_x0=None,
maxiter=1000,
tol=1e-6,
verbose=1,
evaiter=10,
metric='sc',
**kwargs):
r"""
Reconstruct spectrogram phase using `Inversion of Auditory Spectrograms, Traditional Spectrograms, and Other
Envelope Representations`_, where I directly use the :class:`torch.optim.LBFGS` optimizer provided in PyTorch.
This method doesn't restrict to traditional short-time Fourier Transform, but any kinds of presentation (ex: Mel-scaled Spectrogram) as
long as the transform function is differentiable.
.. _`Inversion of Auditory Spectrograms, Traditional Spectrograms, and Other Envelope Representations`:
https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=6949659
Args:
spec (Tensor): the input presentation.
transform_fn: a function that has the form ``spec = transform_fn(x)`` where x is an 1d tensor.
samples (int, optional): number of samples in time domain. Default: :obj:`None`
init_x0 (Tensor, optional): an 1d tensor that make use as initial time domain samples. If not provided, will use random
value tensor with length equal to ``samples``.
maxiter (int): maximum number of iterations before timing out.
tol (float): tolerance of the stopping condition base on L2 loss. Default: ``1e-6``.
verbose (bool): whether to be verbose. Default: :obj:`True`
evaiter (int): steps size for evaluation. After each step, the function defined in ``metric`` will evaluate. Default: ``10``
metric (str): evaluation function. Currently available functions: ``'sc'`` (spectral convergence), ``'snr'`` or ``'ser'``. Default: ``'sc'``
**kwargs: other arguments that pass to :class:`torch.optim.LBFGS`.
Returns:
A 1d tensor converted from the given presentation
"""
if init_x0 is None:
init_x0 = spec.new_empty(samples).normal_(std=1e-6)
x = nn.Parameter(init_x0)
T = spec
criterion = nn.MSELoss()
optimizer = LBFGS([x], **kwargs)
def closure():
optimizer.zero_grad()
V = transform_fn(x)
loss = criterion(V, T)
loss.backward()
return loss
bar_dict = {}
if metric == 'snr':
metric_func = SNR
bar_dict['SNR'] = 0
metric = metric.upper()
elif metric == 'ser':
metric_func = SER
bar_dict['SER'] = 0
metric = metric.upper()
else:
metric_func = spectral_convergence
bar_dict['spectral_convergence'] = 0
metric = 'spectral_convergence'
init_loss = None
with tqdm(total=maxiter, disable=not verbose) as pbar:
for i in range(maxiter):
optimizer.step(closure)
if i % evaiter == evaiter - 1:
with torch.no_grad():
V = transform_fn(x)
bar_dict[metric] = metric_func(V, spec).item()
l2_loss = criterion(V, spec).item()
pbar.set_postfix(**bar_dict, loss=l2_loss)
pbar.update(evaiter)
if not init_loss:
init_loss = l2_loss
elif (previous_loss - l2_loss) / init_loss < tol * evaiter:
break
previous_loss = l2_loss
return x.detach()
def default_image_optimizer(input_image: torch.Tensor) -> LBFGS:
return LBFGS([input_image.requires_grad_(True)], lr=1.0, max_iter=1)
def L_BFGS(spec,
transform_fn,
samples=None,
init_x0=None,
max_iter=1000,
tol=1e-6,
verbose=1,
eva_iter=10,
metric='sc',
**kwargs):
r"""
Reconstruct spectrogram phase using `Inversion of Auditory Spectrograms, Traditional Spectrograms, and Other
Envelope Representations`_, where I directly use the :class:`torch.optim.LBFGS` optimizer provided in PyTorch.
This method doesn't restrict to traditional short-time Fourier Transform, but any kinds of presentation (ex: Mel-scaled Spectrogram) as
long as the transform function is differentiable.
.. _`Inversion of Auditory Spectrograms, Traditional Spectrograms, and Other Envelope Representations`:
https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=6949659
Args:
spec (Tensor): the input presentation.
transform_fn: a function that has the form ``spec = transform_fn(x)`` where x is an 1d tensor.
samples (int, optional): number of samples in time domain. Default: :obj:`None`
init_x0 (Tensor, optional): an 1d tensor that make use as initial time domain samples. If not provided, will use random
value tensor with length equal to ``samples``.
max_iter (int): maximum number of iterations before timing out.
tol (float): tolerance of the stopping condition base on L2 loss. Default: ``1e-6``.
verbose (bool): whether to be verbose. Default: :obj:`True`
eva_iter (int): steps size for evaluation. After each step, the function defined in ``metric`` will evaluate. Default: ``10``
metric (str): evaluation function. Currently available functions: ``'sc'`` (spectral convergence), ``'snr'`` or ``'ser'``. Default: ``'sc'``
**kwargs: other arguments that pass to :class:`torch.optim.LBFGS`.
Returns:
A 1d tensor converted from the given presentation
"""
if init_x0 is None:
init_x0 = spec.new_empty(*samples).normal_(std=1e-6)
x = nn.Parameter(init_x0)
T = spec
criterion = nn.MSELoss()
optimizer = LBFGS([x], **kwargs)
def inner_closure():
optimizer.zero_grad()
V = transform_fn(x)
loss = criterion(V, T)
loss.backward()
return loss
def outer_closure(status_dict):
optimizer.step(inner_closure)
with torch.no_grad():
V = transform_fn(x)
return V
_training_loop(outer_closure, {}, T, max_iter, tol, verbose, eva_iter,
metric)
return x.detach()