class Solver:
def __init__(self, params):
# Build data handler
self.data_handler = DataHandler()
self.data_handler.loadData(params['input'])
params['inp_dim'] = self.data_handler.getDataShape()[1]
logging.info("=" * 41)
# Build model
self.model = SPINEModel(params)
self.dtype = torch.FloatTensor
use_cuda = torch.cuda.is_available()
if use_cuda:
self.model.cuda()
self.dtype = torch.cuda.FloatTensor
# Select optimizer
optim_selected = params['optim']
LR = 0.1
if optim_selected == 'sgd':
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=LR)
elif optim_selected == 'adam':
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=LR)
logging.info("=" * 41)
def train(self, params):
num_epochs, batch_size = params['num_epochs'], params['batch_size'],
optimizer = self.optimizer
dtype = self.dtype
STEP_DENOM = 5
scheduler = StepLR(optimizer,
step_size=math.ceil(num_epochs / STEP_DENOM),
gamma=0.3)
for iteration in range(num_epochs):
# lr adjusting
scheduler.step()
# start epoch
self.data_handler.shuffleTrain()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
epoch_losses = np.zeros(4) # rl, asl, psl, total
for batch_idx in range(num_batches):
optimizer.zero_grad()
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(dtype)
out, h, loss, loss_terms = self.model(batch_x, batch_y)
reconstruction_loss, psl_loss, asl_loss = loss_terms
loss.backward()
epoch_losses[0] += reconstruction_loss.data.item()
epoch_losses[1] += asl_loss.data.item()
epoch_losses[2] += psl_loss.data.item()
epoch_losses[3] += loss.data.item()
optimizer.step()
print("After epoch %r, Reconstruction Loss = %.4f, ASL = %.4f, "
"PSL = %.4f, and total = %.4f" %
(iteration + 1, epoch_losses[0], epoch_losses[1],
epoch_losses[2], epoch_losses[3]))
# logging.info("After epoch %r, Sparsity = %.1f"
# %(iteration+1, utils.compute_sparsity(h.cpu().data.numpy())))
# break
# break
def getSpineEmbeddings(self, batch_size, params):
ret = []
self.data_handler.resetDataOrder()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
for batch_idx in range(num_batches):
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(self.dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(self.dtype)
_, h, _, _ = self.model(batch_x, batch_y)
ret.extend(h.cpu().data.numpy())
return np.array(ret)
def getWordsList(self):
return self.data_handler.getWordsList()
class Solver:
def __init__(self, params):
# Build data handler
self.data_handler = DataHandler()
self.data_handler.loadData(params['input'])
params['inp_dim'] = self.data_handler.getDataShape()[1]
logging.info("=" * 41)
# Build model
self.model = SPINEModel(params)
self.dtype = torch.FloatTensor
use_cuda = torch.cuda.is_available()
if use_cuda:
self.model.cuda()
self.dtype = torch.cuda.FloatTensor
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.1)
logging.info("=" * 41)
def train(self, params):
num_epochs, batch_size = params['num_epochs'], params['batch_size'],
optimizer = self.optimizer
dtype = self.dtype
for iteration in range(num_epochs):
self.data_handler.shuffleTrain()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
epoch_losses = np.zeros(5) # rl, asl, psl, Siml, total
sparsity = 0
for batch_idx in range(num_batches):
optimizer.zero_grad()
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(dtype)
out, h, loss, loss_terms, sparsity_ratio = self.model(
batch_x, batch_y)
reconstruction_loss, psl_loss, asl_loss, siml = loss_terms
loss.backward()
optimizer.step()
epoch_losses[0] += reconstruction_loss.data
epoch_losses[1] += asl_loss.data
epoch_losses[2] += psl_loss.data
epoch_losses[3] += loss.data
epoch_losses[4] += siml.data
sparsity += sparsity_ratio.data
print("After epoch %r, Reconstruction Loss = %.4f, ASL = %.4f,"\
"PSL = %.4f, SimL = %.4f , and total = %.4f, sparsity = %.4f"
%(iteration+1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[4], epoch_losses[3], sparsity/num_batches) )
if iteration % 1000 == 0:
spine_embeddings = self.getSpineEmbeddings(512, params)
utils.dump_vectors(spine_embeddings,
'./' + str(iteration / 1000) + '.txt',
self.getWordsList())
#logging.info("After epoch %r, Sparsity = %.1f"
# %(iteration+1, utils.compute_sparsity(h.cpu().data.numpy())))
#break
#break
def getSpineEmbeddings(self, batch_size, params):
ret = []
self.data_handler.resetDataOrder()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
for batch_idx in range(num_batches):
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(self.dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(self.dtype)
_, h, _, _, spars = self.model(batch_x, batch_y)
ret.extend(h.cpu().data.numpy())
return np.array(ret)
def getWordsList(self):
return self.data_handler.getWordsList()
class Solver:
def __init__(self, params):
# Build data handler
self.data_handler = DataHandler()
self.data_handler.loadData(params['input'])
params['inp_dim'] = self.data_handler.getDataShape()[
1] # inp_dim = 1000
logging.info("=" * 41)
# Build model
self.model = SPINEModel(params)
self.dtype = torch.FloatTensor
use_cuda = torch.cuda.is_available()
if use_cuda:
self.model.cuda()
self.dtype = torch.cuda.FloatTensor
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.1)
logging.info("=" * 41)
def train(self, params):
num_epochs, batch_size = params['num_epochs'], params['batch_size']
optimizer = self.optimizer
dtype = self.dtype
for iteration in range(num_epochs): # for each epoch
self.data_handler.shuffleTrain() # 15000 training data shuffled
num_batches = self.data_handler.getNumberOfBatches(
batch_size) # num_batches = number of iterations
epoch_losses = np.zeros(
5) # rl, asl, psl, total !!!!!!!!!!!!!!!!!!
for batch_idx in range(num_batches): # for each iteration
optimizer.zero_grad()
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(
dtype) #dtype = torch.FloatTensor
out, h, loss, loss_terms = self.model(batch_x, batch_y)
reconstruction_loss, psl_loss, asl_loss, local_loss = loss_terms #!!!!!!!!!!!!!!!!!!!!!
loss.backward()
optimizer.step()
#print(local_loss)
epoch_losses[0] += reconstruction_loss.item()
epoch_losses[1] += asl_loss.item()
epoch_losses[2] += psl_loss.item()
epoch_losses[3] += local_loss.item() #!!!!!!!!!!!!!!!!!!!!!!!!
epoch_losses[4] += loss.item()
# epoch_losses[0]+=reconstruction_loss.data[0]
# epoch_losses[1]+=asl_loss.data[0]
# epoch_losses[2]+=psl_loss.data[0]
# epoch_losses[3]+=local_loss.data[0]
# epoch_losses[4]+=loss.data[0]
print("After epoch %r, Reconstruction Loss = %.4f, ASL = %.4f,"\
"PSL = %.4f, Local Loss = %.4f and total = %.4f"
%(iteration+1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[3], epoch_losses[4]) ) #!!!!!!
#logging.info("After epoch %r, Sparsity = %.1f"
# %(iteration+1, utils.compute_sparsity(h.cpu().data.numpy())))
#break
#break
def getSpineEmbeddings(self, batch_size, params):
ret = []
self.data_handler.resetDataOrder()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
for batch_idx in range(num_batches):
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(self.dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(self.dtype)
_, h, _, _ = self.model(batch_x, batch_y)
ret.extend(h.cpu().data.numpy())
return np.array(ret)
def getWordsList(self):
return self.data_handler.getWordsList()
class Solver:
def __init__(self, params):
# build data handler
self.data_handler = DataHandler()
self.data_handler.loadData(params['input'])
params['inp_dim'] = self.data_handler.getDataShape()[1]
print("=" * 41)
# build model
self.model = SPINEModel(params)
self.dtype = torch.FloatTensor
# check if GPU is available
use_cuda = torch.cuda.is_available()
if use_cuda:
self.model.cuda()
self.dtype = torch.cuda.FloatTensor
# set optimizer
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.1)
print("=" * 41)
def train(self, params):
num_epochs, batch_size = params['num_epochs'], params['batch_size'],
optimizer = self.optimizer
dtype = self.dtype
# train for each epoch
for iteration in range(num_epochs):
self.data_handler.shuffleTrain()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
epoch_losses = np.zeros(4) # rl, asl, psl, total
# for each batch
for batch_idx in range(num_batches):
optimizer.zero_grad()
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
# transform batches into tensors
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(dtype)
# calculate losses
out, h, loss, loss_terms = self.model(batch_x, batch_y)
reconstruction_loss, psl_loss, asl_loss = loss_terms
# update loss and optimizer
loss.backward()
optimizer.step()
# assign loss
epoch_losses[0] += reconstruction_loss.data
epoch_losses[1] += asl_loss.data
epoch_losses[2] += psl_loss.data
epoch_losses[3] += loss.data
print("epoch %r: Reconstruction Loss = %.4f, ASL = %.4f, "\
"PSL = %.4f, and total = %.4f"
%(iteration+1, epoch_losses[0], epoch_losses[1], epoch_losses[2], epoch_losses[3]) )
def getSpineEmbeddings(self, batch_size, params):
ret = []
self.data_handler.resetDataOrder()
num_batches = self.data_handler.getNumberOfBatches(batch_size)
# for each batch
for batch_idx in range(num_batches):
batch_x, batch_y = self.data_handler.getBatch(
batch_idx, batch_size, params['noise_level'],
params['denoising'])
# transform batches into tensors
batch_x = Variable(torch.from_numpy(batch_x),
requires_grad=False).type(self.dtype)
batch_y = Variable(torch.from_numpy(batch_y),
requires_grad=False).type(self.dtype)
_, h, _, _ = self.model(batch_x, batch_y)
# append to embeddings
ret.extend(h.cpu().data.numpy())
return np.array(ret)
def getWordsList(self):
return self.data_handler.getWordsList()