def create_new_scheduler(self,
name,
annealer,
annealer_kws,
creation_condition=True):
value_scheduler = None
rsetattr(self, name + '_scheduler', value_scheduler)
if creation_condition:
annealer_kws['device'] = self.device
value_annealer = annealer(annealer_kws)
rsetattr(self, name + '_annealer', value_annealer)
# This is the value that we'll update on each call of
# step_annealers().
rsetattr(self, name,
torch.tensor(value_annealer(0), device=self.device))
dummy_optimizer = optim.Optimizer(
[rgetattr(self, name)],
{'lr': torch.tensor(value_annealer(0), device=self.device)})
rsetattr(self, name + '_optimizer', dummy_optimizer)
value_scheduler = CustomLR(dummy_optimizer, value_annealer)
rsetattr(self, name + '_scheduler', value_scheduler)
self.schedulers.append(value_scheduler)
self.annealed_vars.append(name)
def test_save_optim_defaults(self):
weight = torch.randn(2, 2)
default = torch.randn(2)
opt = optim.Optimizer([weight], {'test_default': default})
filename = 'test_save_optim.pth'
torch.save(opt, filename)
if os.path.exists(filename):
loaded_opt = torch.load(filename)
self.assertEqual(loaded_opt.param_groups[0]['params'][0], weight)
self.assertEqual(loaded_opt.defaults['test_default'], default)
os.remove(filename)
def __init__(self):
#define network
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.net = models.resnet50(pretrained=True)
#reset fc layer
self._num_ftrs = self.net.fc.in_features
self._num_prediction = 2
self.net.fc = nn.Linear(self._num_ftrs, self._num_prediction)
#use gpu if possible
self.net = self.net.to(self.device)
#define loss function
self.criterion = nn.CrossEntropyLoss()
#define optimizer
self.optimizer = optim.Optimizer(self.net.parameters(), {})
#make tbx writer
self.writer = tbx.SummaryWriter()
def create_new_scheduler(obj,
name,
annealer,
annealer_kws,
creation_condition=True):
value_scheduler = None
rsetattr(obj, name + '_scheduler', value_scheduler)
if creation_condition:
annealer_kws['device'] = obj.device
value_annealer = annealer(annealer_kws)
rsetattr(obj, name + '_annealer', value_annealer)
# This is the value that we'll update on each call of
# step_annealers().
rsetattr(obj, name, value_annealer(0).clone().detach())
dummy_optimizer = optim.Optimizer(
[rgetattr(obj, name)], {'lr': value_annealer(0).clone().detach()})
rsetattr(obj, name + '_optimizer', dummy_optimizer)
value_scheduler = CustomLR(dummy_optimizer, value_annealer)
rsetattr(obj, name + '_scheduler', value_scheduler)
obj.schedulers.append(value_scheduler)
obj.annealed_vars.append(name)
def train(arguments,trainData,device,criterion,model):
# Set default logging format
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
def setup_logger(name, log_file, level=logging.INFO):
# """Function setup as many loggers as you want"""
handler = logging.FileHandler(log_file)
handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.setLevel(level)
logger.addHandler(handler)
return logger
print('Defining some tools')
print('************************************\n MODEL IS CUDA:' + str(next(model.parameters()).is_cuda))
# -- Retrieve parameters and gradients:
# -- this extracts and flattens all the trainable parameters of the mode
# -- into a 1-dim vector
# Not needed, added to the optimizer at once
# if model is not None:
# oldparameters,oldgradParameters = model.parameters()
optimizer = arguments.optim_Method.upper()
print('Configuring Optimizer')
if optimizer == 'CG': # No CG model in torch
# Insert Values
maxIter = arguments.max_iter
optimMethod = optim.Optimizer(model.parameters())
elif optimizer == 'LBFGS': # !!!NEEDS CLOSURE FUNCTION
# Insert Values
maxIter = arguments.max_iter
learningRate = arguments.lr
optimMethod = optim.LBFGS(model.parameters(), lr=learningRate, max_iter=maxIter)
elif optimizer == 'SGD':
# Insert Values
weightDecay = arguments.weight_decay
learningRate = arguments.lr
momentum = arguments.momentum
optimMethod = optim.SGD(model.parameters(), lr=learningRate, momentum=momentum, weight_decay=weightDecay)
elif optimizer == 'ASGD':
learningRate = arguments.lr
eta0 = arguments.t0
optimMethod = optim.ASGD(model.parameters(), lr=learningRate, t0=eta0 * trainData.dataset.train_data.size()[0])
else:
raise ValueError('Uknown optimization method')
print(model.parameters())
# !!!!!START TRAINING!!!!
# Since train is called multiple times it is checked if it is loaded in the memory first
# !!!!WORKS LIKE THIS IN LUA, IN PYTHON WE WILL NEED ANOTHER WAY
# Set model to training mode
model = model.train()
print(model)
# do one epoch
print('--->Doing epoch on training data')
print("--->Online epoch # " + str(arguments.epochs) + "[batchSize = " + str(arguments.batch_Size) + "]")
# Begin Fetching batches from Dataloader
# Got this part from https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html
model.cuda().to(device)
criterion.cuda().to(device)
time = datetime.now()
for i in range(arguments.epochs):
# Training
for index, (data, target) in enumerate(trainData):
# Transfer to GPU
data, target = data.cuda().to(device), target.cuda().to(device)
# Forward pass to the NN
if optimizer == 'LBFGS':
# If optimizer needs eval function
# Το τρεχω σωστα??
def closure():
optimMethod.zero_grad()
outputs = model.forward(data)
print(outputs.size())
loss = criterion(outputs, target)
loss.backward()
return loss
loss = optimMethod.step(closure)
print('Loss for batch ' + str(index) + ': ' + str(loss[0]))
else:
# if optimizer does not need eval function
outputs = model.forward(data)
loss = criterion(outputs, target)
# BackProp and optimize
optimMethod.zero_grad()
loss.backward()
optimMethod.step() # Feval)
print('Loss for batch ' + str(index) + ': ' + str(loss.data))
#Time for each epoch
print(datetime.now() - time)
# Save current trained net
torch.save(model.state_dict,
'model_' + arguments.neural_network + '_' + arguments.loss + '_' + optimizer + '.pt') # load with model.load_state_dict and model.eval() to get the correct results
def main(args):
if VERBOSE:
print '***The Replay Buffer currently always returns the most recent experiences (instead of random), so the batches are constant between the tf and torch nets.'
state_dim = 3
action_dim = 1
net = ActorCriticNet(state_dim, action_dim)
target_net = copy.deepcopy(net)
memory = ReplayBuffer(REPLAY_BUFFER_SIZE)
noise = OUNoise(action_dim)
criterion = nn.MSELoss()
optimizer = optim.Adam(net.parameters(), lr=LEARNING_RATE, weight_decay=L2)
target_optim = optim.Optimizer(target_net.parameters(),
{}) # to iterate over target params
if VERBOSE: print '***Making gym env (only used to setup TF net).'
# load tf net (restoring saved parameters)
dtf = ddpg_tf.DDPG_TF(filter_env.makeFilteredEnv(gym.make('Pendulum-v0')),
loadfilename='tf_params-0',
printVars=False)
if VERBOSE: print '***TF net restore complete.'
# load control data (only using a every fourth data), and tf net results
control_states = np.load('control_states.npy')[::4]
control_rewards = np.load('control_rewards.npy')[::4]
tf_record = np.load('tf_control_record.npy')
# replace torch params with tf params, and run control data, collecting torch net results
# first optimization step will occur at i == 50, upon which extra data is recorded to compare tf and torch
# using: no bn, REPLAY_BUFFER_SIZE=200, REPLAY_START_SIZE=50, BATCH_SIZE=50, constant replay_buffer_batches (always the most recent experiences)
replaceNetParams(dtf, net, target_net)
if VERBOSE: print '***Torch net params initialized to TF net params.'
original_net = copy.deepcopy(net) # save original net
original_target_net = copy.deepcopy(target_net)
torch_record = []
loss = -1
first_step = True
for i in xrange(len(control_rewards) - 1):
state = torch.from_numpy(control_states[i].reshape(1,
state_dim)).float()
action = net.getAction(Variable(state)).data
target_action = target_net.getAction(Variable(state)).data
reward = torch.FloatTensor([[control_rewards[i]]]).float()
new_state = torch.from_numpy(control_states[i + 1].reshape(
1, state_dim)).float()
memory.add(state, action, reward, new_state, True)
if memory.count() > REPLAY_START_SIZE:
minibatch = memory.get_batch(BATCH_SIZE)
state_batch = torch.cat([data[0] for data in minibatch], dim=0)
action_batch = torch.cat([data[1] for data in minibatch], dim=0)
reward_batch = torch.cat([data[2] for data in minibatch])
next_state_batch = torch.cat([data[3] for data in minibatch],
dim=0)
done_batch = Tensor([data[4] for data in minibatch])
# calculate y_batch from targets
#next_action_batch = target_net.getAction(Variable(next_state_batch))
value_batch = target_net.getValue(Variable(next_state_batch)).data
y_batch = reward_batch + GAMMA * value_batch * done_batch
if first_step:
if VERBOSE: print '***First Optimization Step complete.'
torch_ys = y_batch
torch_batch = minibatch
torch_outs = net.getValue(Variable(state_batch)).data
# optimize net 1 step
loss = criterion(net.getValue(Variable(state_batch)),
Variable(y_batch))
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss = loss.data[0]
# update targets - using exponential moving averages
for group, target_group in zip(optimizer.param_groups,
target_optim.param_groups):
for param, target_param in zip(group['params'],
target_group['params']):
target_param.data.mul_(1 - TAU)
target_param.data.add_(TAU, param.data)
if first_step:
first_step_net = copy.deepcopy(net)
first_step_target_net = copy.deepcopy(target_net)
first_step = False
torch_record.append(
[action.numpy()[0][0],
target_action.numpy()[0][0], loss])
loss = -1
torch_record = np.array(torch_record)
torch_outs = torch_outs.numpy().T[0]
torch_ys = torch_ys.numpy().T[0]
if VERBOSE: print '***Control Data run complete.'
# compare torch and tf results
# results for each net have 3 columns: [net action prediction, target net action prediction, loss (-1 if there was no training)]
sel = np.arange(45, 55)
#print calc_error(tf_record[sel,:], torch_record[sel,:])
print 'Result comparison:'
print 'control_data_index | tf_net_action | tf_target_net_action | tf_loss | torch_net_action | torch_target_net_action | torch_loss'
print np.hstack(
[sel[:, np.newaxis], tf_record[sel, :], torch_record[sel, :]])
print '\t(a loss of -1 means no training occured in that step)'
# load all tf results from before taking first optimization step
tf_ys = np.load('tf_first_step_y_batch.npy')
tf_rs = np.load('tf_first_step_reward_batch.npy')
tf_ds = np.load('tf_first_step_done_batch.npy')
tf_vs = np.load('tf_first_step_value_batch.npy')
tf_outs = np.load('tf_first_step_output_values.npy')
torch_wd = 1.36607 # weight decay loss of tf net at first optimization step - recorded directly from terminal output of tf net
if VERBOSE:
print '***Comparing first step stats'
# compare tf and torch data from before taking first optimization step
# including calculation of manual loss
print '\terror in ys (between tf and torch)', calc_error(
torch_ys, tf_ys)
print '\terror in predictions (between tf and torch)', calc_error(
torch_outs, tf_outs)
print '\ttorch loss (manually calculated)', np.mean(
(torch_ys - torch_outs)**2)
print '\ttf loss (manually calculated)', np.mean((tf_ys - tf_outs)**2)
print '\ttorch loss', torch_record[50,
2], '(not including weight decay)'
print '\ttf loss', tf_record[
50, 2] - torch_wd, '(not including weight decay)'
return 0
def train(arguments,trainData,device,criterion,model):
# Set default logging format
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
def setup_logger(name, log_file, level=logging.INFO):
# """Function setup as many loggers as you want"""
handler = logging.FileHandler(log_file)
handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.setLevel(level)
logger.addHandler(handler)
return logger
print('Defining some tools')
# This matrix records the current confusion across classes
# In python we will initialize it later
# confusion = confusion_matrix(labels=classes)
# Convert to 2d data
samplesShape = trainData.dataset.data.shape
print(trainData.dataset.data.reshape(samplesShape[0], samplesShape[1] * samplesShape[2] * samplesShape[3]))
# -- Retrieve parameters and gradients:
# -- this extracts and flattens all the trainable parameters of the mode
# -- into a 1-dim vector
# Not needed, added to the optimizer at once
# if model is not None:
# oldparameters,oldgradParameters = model.parameters()
optimizer = arguments.optim_Method.upper()
print('Configuring Optimizer')
if optimizer == 'CG': # No CG model in torch
# Insert Values
maxIter = arguments.max_iter
optimMethod = optim.Optimizer(model.parameters())
elif optimizer == 'LBFGS': # !!!NEEDS CLOSURE FUNCTION
# Insert Values
maxIter = arguments.max_iter
learningRate = arguments.lr
optimMethod = optim.LBFGS(model.parameters(), lr=learningRate, max_iter=maxIter)
elif optimizer == 'SGD':
# Insert Values
weightDecay = arguments.weight_decay
learningRate = arguments.lr
momentum = arguments.momentum
optimMethod = optim.SGD(model.parameters(), lr=learningRate, momentum=momentum, weight_decay=weightDecay)
elif optimizer == 'ASGD':
learningRate = arguments.lr
eta0 = arguments.t0
optimMethod = optim.ASGD(model.parameters(), lr=learningRate, t0=eta0 * trainData.dataset.data.size)
elif optimizer == 'ADAM':
learningRate = arguments.lr
optimMethod = optim.Adam(model.parameters(), lr=learningRate)
else:
raise ValueError('Uknown optimization method')
print(model.parameters())
# !!!!!START TRAINING!!!!
# Since train is called multiple times it is checked if it is loaded in the memory first
# !!!!WORKS LIKE THIS IN LUA, IN PYTHON WE WILL NEED ANOTHER WAY
# Set model to training mode
model = model.train()
print('************************************\n MODEL IS CUDA:' + str(next(model.parameters()).is_cuda) + '************************************\n')
# do one epoch
print('--->Doing epoch on training data')
print("--->Online epoch # " + str(arguments.epochs) + "[batchSize = " + str(arguments.batch_Size) + "]")
# Begin Fetching batches from Dataloader
# Got this part from https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html
time = datetime.now()
for i in range(arguments.epochs):
print('Epoch #' + str(i))
k = 0
# Reduce learning rate on each epoch
# for param_group in optimMethod.param_groups:
# param_group['lr'] = param_group['lr'] * 0.99
# Training
for index, (data, target) in enumerate(trainData):
# Transfer to GPU
data, target = data.cuda(), target.cuda()
# Forward pass to the NN
if optimizer == 'LBFGS':
# If optimizer needs eval function
# Το τρεχω σωστα??
def closure():
optimMethod.zero_grad()
outputs, tsne_results = model.forward(data)
print(outputs.size())
loss = criterion(outputs, target)
loss.backward()
return loss
loss = optimMethod.step(closure)
print('Loss for batch ' + str(index) + ': ' + str(loss[0]))
else:
# if optimizer does not need eval function
outputs, tsne_results, kmeans_data = model.forward(data)
loss = criterion(outputs, target)
# BackProp and optimize
optimMethod.zero_grad()
loss.backward()
optimMethod.step() # Feval)
#print('Loss for batch ' + str(index) + ': ' + str(loss.data))
# Print tsne result at the last batch of each epoch
if (k < 3):
plt.scatter(tsne_results[:,0] , tsne_results[:,1])
plt.scatter(kmeans_data.cluster_centers_[:,0] , kmeans_data.cluster_centers_[:,1], s=250, marker='*', c='red', edgecolor='black', label='centroids')
plt.show()
plt.clf()
k = k +1
print("Features for epoch: " + str(i))
# Clear axes
#Time for each epoch
print(datetime.now() - time)
# Save current trained net
torch.save(model.state_dict(),
'model_' + arguments.neural_network + '_' + arguments.loss + '_' + optimizer + '.pt') # load with model.load_state_dict and model.eval() to get the correct results
print("Model saved with name:" + 'model_' + arguments.neural_network + '_' + arguments.loss + '_' + optimizer + '.pt')
torch.save(criterion.state_dict(), 'optimizer_' + arguments.neural_network + '_' + arguments.loss + '_' + optimizer + '.pt')
print("Optimizer saved with name:" + 'optimizer' + arguments.neural_network + '_' + arguments.loss + '_' + optimizer + '.pt')