def __init__(self,
model,
is_cuda=False,
train_step_init=0,
lr=1e-3,
prefix="Variational_Autoencoder",
dashboard_server=None,
checkpoint_path=None,
load=False,
save=False):
self.prefix = prefix
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
if dashboard_server:
protocol, server, port = dashboard_server.split(':')
self.dashboard = Dashboard(prefix,
server=protocol + ':' + server,
port=port)
elif dashboard_server is False: # when False do not create dashboard instance.
pass
else:
self.dashboard = Dashboard(prefix)
self._checkpoint_path = checkpoint_path
self.load = load
self.save = save
self.epoch_number = 0
self.format_dict = dict(prefix=self.prefix,
date=time.strftime("%Y%m%d"),
time=time.strftime("%H%M%S"))
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = Dashboard(
'Grammar-Variational-Autoencoder-experiment')
class Session():
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = Dashboard(
'Grammar-Variational-Autoencoder-experiment')
def train(self, loader, epoch_number):
# built-in method for the nn.module, sets a training flag.
self.model.train()
_losses = []
for batch_idx, data in enumerate(loader):
# have to cast data to FloatTensor. DoubleTensor errors with Conv1D
data = Variable(data)
# do not use CUDA atm
self.optimizer.zero_grad()
recon_batch, mu, log_var = self.model(data)
loss = self.loss_fn(data, mu, log_var, recon_batch)
_losses.append(loss.detach().numpy())
loss.backward()
self.optimizer.step()
self.train_step += 1
loss_value = np.array([loss.data.numpy()])
batch_size = len(data)
self.dashboard.append('training_loss',
'line',
X=np.array([self.train_step]),
Y=loss_value / batch_size)
if batch_idx == 0:
print('batch size', batch_size)
if batch_idx % 40 == 0:
print('training loss: {:.4f}'.format(loss_value[0] /
batch_size))
return _losses
def test(self, loader):
# nn.Module method, sets the training flag to False
self.model.eval()
test_loss = 0
for batch_idx, data in enumerate(loader):
data = Variable(data, volatile=True)
# do not use CUDA atm
recon_batch, mu, log_var = self.model(data)
test_loss += self.loss_fn(data, mu, log_var, recon_batch).item()
test_loss /= len(test_loader.dataset)
print('testset length', len(test_loader.dataset))
print('====> Test set loss: {:.4f}'.format(test_loss))
class Session():
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = Dashboard('Grammar-Variational-Autoencoder-experiment')
def train(self, loader, epoch_number):
# built-in method for the nn.module, sets a training flag.
self.model.train()
for batch_idx, data in enumerate(loader):
# have to cast data to FloatTensor. DoubleTensor errors with Conv1D
data = Variable(data)
# do not use CUDA atm
self.optimizer.zero_grad()
recon_batch, mu, log_var = self.model(data)
loss = self.loss_fn(data, mu, log_var, recon_batch)
loss.backward()
self.optimizer.step()
self.train_step += 1
loss_value = loss.data.numpy()
batch_size = len(data)
self.dashboard.append('training_loss', 'line',
X=np.array([self.train_step]),
Y=loss_value / batch_size)
if batch_idx == 0:
print('batch size', batch_size)
if batch_idx % 40 == 0:
print('training loss: {:.4f}'.format(loss_value[0] / batch_size))
return losses
def test(self, loader):
# nn.Module method, sets the training flag to False
self.model.eval()
test_loss = 0
for batch_idx, data in enumerate(loader):
data = Variable(data, volatile=True)
# do not use CUDA atm
recon_batch, mu, log_var = self.model(data)
test_loss += self.loss_fn(data, mu, log_var, recon_batch).data[0]
test_loss /= len(test_loader.dataset)
print('testset length', len(test_loader.dataset))
print('====> Test set loss: {:.4f}'.format(test_loss))
class Session():
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = Dashboard('Variational-Autoencoder-experiment')
def train(self, loader, epoch_number):
# built-in method for the nn.module, sets a training flag.
self.model.train()
for batch_idx, (data, _) in enumerate(loader):
data = Variable(data)
# do not use CUDA atm
self.optimizer.zero_grad()
recon_batch, mu, log_var = self.model(data)
loss = self.loss_fn(data, mu, log_var, recon_batch)
loss.backward()
self.optimizer.step()
self.train_step += 1
self.dashboard.append('training_loss',
'line',
X=np.array([self.train_step]),
Y=loss.data.numpy())
print(loss.data.numpy())
if loss.data.numpy()[0] > 500:
pass
return losses
def test(self, loader):
# nn.Module method, sets the training flag to False
self.model.eval()
test_loss = 0
for batch_idx, (data, _) in enumerate(loader):
data = Variable(data, volatile=True)
# do not use CUDA atm
recon_batch, mu, logvar = self.model(data)
test_loss += self.loss_fn(data, mu, logvar, recon_batch).data[0]
test_loss /= len(test_loader.dataset)
print('====> Test set loss: {:.4f}'.format(test_loss))
class Session():
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = Dashboard('Variational-Autoencoder-experiment')
def train(self, loader, epoch_number):
# built-in method for the nn.module, sets a training flag.
self.model.train()
for batch_idx, (data, _) in enumerate(loader):
data = Variable(data)
# do not use CUDA atm
self.optimizer.zero_grad()
recon_batch, mu, log_var = self.model(data)
loss = self.loss_fn(data, mu, log_var, recon_batch)
loss.backward()
self.optimizer.step()
self.train_step += 1
self.dashboard.append('training_loss', 'line',
X=np.array([self.train_step]),
Y=loss.data.numpy())
print(loss.data.numpy())
if loss.data.numpy()[0] > 500:
pass
return losses
def test(self, loader):
# nn.Module method, sets the training flag to False
self.model.eval()
test_loss = 0
for batch_idx, (data, _) in enumerate(loader):
data = Variable(data, volatile=True)
# do not use CUDA atm
recon_batch, mu, logvar = self.model(data)
test_loss += self.loss_fn(data, mu, logvar, recon_batch).data[0]
test_loss /= len(test_loader.dataset)
print('====> Test set loss: {:.4f}'.format(test_loss))
def __init__(self, name = None, window_sizes = [30]):
"""
Initialization for the loss tool.
Args: name: loss name, the default is none
window_sizes: the smooth window sizes (default is [30], should be a list)
"""
# raw loss value
self.loss_raw = []
# initialize the dashboard
self.loss_dict = {}
for window_size in window_sizes:
self.loss_dict[window_size] = {}
self.loss_dict[window_size]['loss'] = []
self.loss_dict[window_size]['count'] = 0
self.loss_dict[window_size]['x'] = []
self.loss_dict[window_size]['name'] = "loss-" + str(window_size)
if name:
self.loss_dict[window_size]['name'] = name + "-" + self.loss_dict[window_size]['name']
# init the dashboard
self.dashboard = Dashboard("loss_env")
def __init__(self):
super(VAELoss, self).__init__()
self.bce_loss = nn.BCELoss()
self.bce_loss.size_average = False
self.dashboard = Dashboard('Variational-Autoencoder-experiment')
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = Dashboard('Grammar-Variational-Autoencoder-experiment')
class Session():
def __init__(self,
model,
is_cuda=False,
train_step_init=0,
lr=1e-3,
prefix="Variational_Autoencoder",
dashboard_server=None,
checkpoint_path=None,
load=False,
save=False):
self.prefix = prefix
self.train_step = train_step_init
self.model = model
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
if dashboard_server:
protocol, server, port = dashboard_server.split(':')
self.dashboard = Dashboard(prefix,
server=protocol + ':' + server,
port=port)
elif dashboard_server is False: # when False do not create dashboard instance.
pass
else:
self.dashboard = Dashboard(prefix)
self._checkpoint_path = checkpoint_path
self.load = load
self.save = save
self.epoch_number = 0
self.format_dict = dict(prefix=self.prefix,
date=time.strftime("%Y%m%d"),
time=time.strftime("%H%M%S"))
def train(self,
loader,
epoch_number=None,
verbose=True,
report_interval=100):
if epoch_number is not None:
self.epoch_number = epoch_number
# built-in method for the nn.module, sets a training flag.
self.model.train()
losses = []
for batch_idx, (data, _) in enumerate(loader):
data = Variable(data)
# do not use CUDA atm
self.optimizer.zero_grad()
recon_batch, mu, log_var = self.model(data)
loss = self.loss_fn(data, mu, log_var, recon_batch)
loss.backward()
loss_data = loss.data.numpy()
self.optimizer.step()
self.train_step += 1
self.dashboard.append('training_loss',
'line',
X=np.array([self.train_step]),
Y=loss_data)
losses.append(loss_data)
if verbose and batch_idx % report_interval == 0:
print('batch loss is: {:.4}'.format(
torch.np.mean(losses) / data.size()[0]))
self.epoch_number += 1
return losses
def test(self, test_loader):
# nn.Module method, sets the training flag to False
self.model.eval()
test_loss = 0
for batch_idx, (data, _) in enumerate(test_loader):
data = Variable(data, volatile=True)
# do not use CUDA atm
recon_batch, mu, logvar = self.model(data)
test_loss += self.loss_fn(data, mu, logvar, recon_batch).data[0]
test_loss /= len(test_loader.dataset)
print('====> Test set loss: {:.4f}'.format(test_loss))
@property
def checkpoint_path(self):
if self._checkpoint_path:
return self._checkpoint_path.format(**self.format_dict)
def __enter__(self):
if self.checkpoint_path and self.load:
try:
cp = torch.load(self.checkpoint_path)
self.model.load_state_dict(cp['state_dict'])
self.epoch_number = cp['epoch_number']
except Exception as e:
print('Did not load checkpoint due to exception: {}'.format(e))
return self
def __exit__(self, exc_type, exc_val, exc_tb):
state_dict = self.model.state_dict()
path = os.path.split(self.checkpoint_path)
if len(path) > 1:
try:
os.makedirs(os.path.join(*path[:-1]))
except FileExistsError:
pass
if self.checkpoint_path and self.save:
print('saving state_dict to {}'.format(self.checkpoint_path))
torch.save(
dict(state_dict=state_dict, epoch_number=self.epoch_number),
self.checkpoint_path)
class Session():
def __init__(self, model, train_step_init=0, lr=1e-3, is_cuda=False):
self.train_step = train_step_init
self.model = model.to(device)
# model.to(device)
self.optimizer = optim.Adam(model.parameters(), lr=lr)
self.loss_fn = VAELoss()
self.dashboard = None
if VISUALIZE_DASHBOARD:
self.dashboard = Dashboard(
'Grammar-Variational-Autoencoder-experiment')
def train(self, loader, epoch_number):
# built-in method for the nn.module, sets a training flag.
self.model.train()
_losses = []
for batch_idx, data in enumerate(loader):
# have to cast data to FloatTensor. DoubleTensor errors with Conv1D
data = Variable(data).to(device)
# do not use CUDA atm
self.optimizer.zero_grad()
mu = None
log_var = None
recon_batch = None
if GrammarVariationalAutoEncoder.VAE_MODE:
recon_batch, mu, log_var = self.model(data)
else:
recon_batch = self.model(data)
loss = self.loss_fn(data, recon_batch, mu, log_var)
_losses.append(loss.cpu().detach().numpy())
loss.backward()
self.optimizer.step()
self.train_step += 1
loss_value = loss.cpu().detach().numpy()
batch_size = len(data)
if VISUALIZE_DASHBOARD:
self.dashboard.append('training_loss',
'line',
X=np.array([self.train_step]),
Y=loss_value / batch_size)
if batch_idx == 0:
print('batch size', batch_size)
if batch_idx % 40 == 0:
# print('training loss: {:.4f}'.format(loss_value[0] / batch_size))
print('training loss: {:.20f}'.format(loss_value / batch_size))
self.model.save_model('../save_model/')
return _losses
def test(self, loader):
# nn.Module method, sets the training flag to False
self.model.eval()
test_loss = 0
for batch_idx, data in enumerate(loader):
data = Variable(data, volatile=True)
# do not use CUDA atm
mu = None
log_var = None
recon_batch = None
if GrammarVariationalAutoEncoder.VAE_MODE:
recon_batch, mu, log_var = self.model(data)
else:
recon_batch = self.model(data)
# test_loss += self.loss_fn(data, recon_batch, mu, log_var).data[0]
test_loss += self.loss_fn(data, recon_batch, mu,
log_var).data.detach().cpu().numpy()
test_loss /= len(test_loader.dataset)
print('testset length', len(test_loader.dataset))
print('====> Test set loss: {:.20f}'.format(test_loss))
def test_tangible(self):
eq = ['sin(x*2)', 'exp(x)+x', 'x/3', '3*exp(2/x)']
grammar_weights = "../save_model/grammar_ae_model.pt"
print(grammar_weights)
grammar_model = equation_vae.EquationGrammarModel(grammar_weights,
latent_rep_size=25)
# grammar_model = equation_vae.EquationGrammarModel(grammar_weights, latent_rep_size=LATENT_REP_SIZE)
z, _none = grammar_model.encode(eq)
for i, s in enumerate(grammar_model.decode(z)):
print(eq[i] + " --> " + s)
class Loss_tool(object):
"""
Helper class to help visualize the loss change.
"""
def __init__(self, name = None, window_sizes = [30]):
"""
Initialization for the loss tool.
Args: name: loss name, the default is none
window_sizes: the smooth window sizes (default is [30], should be a list)
"""
# raw loss value
self.loss_raw = []
# initialize the dashboard
self.loss_dict = {}
for window_size in window_sizes:
self.loss_dict[window_size] = {}
self.loss_dict[window_size]['loss'] = []
self.loss_dict[window_size]['count'] = 0
self.loss_dict[window_size]['x'] = []
self.loss_dict[window_size]['name'] = "loss-" + str(window_size)
if name:
self.loss_dict[window_size]['name'] = name + "-" + self.loss_dict[window_size]['name']
# init the dashboard
self.dashboard = Dashboard("loss_env")
def append_loss(self, loss):
"""
Add a new loss value
Args:
loss: the loss value
"""
if type(loss) == torch.Tensor:
self.loss_raw += loss.tolist()
else:
raise ValueError("type: {0} is not support yet, only support torch tensor".format(type(loss)))
# update loss for all different window size
self._update_loss_array()
# plot the loss
self.plot_all()
def _update_loss_array(self):
"""
Update the loss array
"""
for size in self.loss_dict:
if len(self.loss_raw) > size:
self.loss_dict[size]['count'] += 1
self.loss_dict[size]['x'].append(self.loss_dict[size]['count'])
self.loss_dict[size]['loss'].append(
sum(self.loss_raw[-size:]) / size
)
def plot_all(self):
"""
Plot all the curves
"""
for size in self.loss_dict:
name = self.loss_dict[size]['name']
loss_values = self.loss_dict[size]['loss']
x = self.loss_dict[size]['x']
if len(loss_values) > 0:
self.dashboard.plot(name, "line", X=torch.FloatTensor(x), Y=torch.FloatTensor(loss_values))