def run_single_NN():
task = Task.init(
project_name='MSci-Project',
task_name='Gridworld, n=32, b=4, normal') #init task on ClearML
#load variables from file
open_file = open("NNIRL_param_list.pkl", "rb")
NNIRL_param_list = pickle.load(open_file)
open_file.close()
threshold = NNIRL_param_list[0]
optim_type = NNIRL_param_list[1]
net = NNIRL_param_list[2]
X = NNIRL_param_list[3]
initD = NNIRL_param_list[4]
mu_sa = NNIRL_param_list[5]
muE = NNIRL_param_list[6]
F = NNIRL_param_list[7]
#F = F.type(torch.DoubleTensor)
mdp_data = NNIRL_param_list[8]
configuration_dict = NNIRL_param_list[9]
truep = NNIRL_param_list[10]
NLL_EVD_plots = NNIRL_param_list[11]
example_samples = NNIRL_param_list[12]
noisey_features = NNIRL_param_list[13]
NLL = NLLFunction() # initialise NLL
#assign constants
NLL.F = F
NLL.muE = muE
NLL.mu_sa = mu_sa
NLL.initD = initD
NLL.mdp_data = mdp_data
configuration_dict = task.connect(
configuration_dict) #enabling configuration override by clearml
start_time = time.time() #to time execution
#tester = testers() #to use testing functions
# lists for printing
NLList = []
iterations = []
evdList = []
i = 0 #track iterations
finalOutput = None #store final est R
loss = 1000 #init loss
diff = 1000 #init diff
evd = 10 #init val
#noisey_features=True
if noisey_features:
#add noise to features at states 12, 34 and 64 (when mdp_params.n=8)
#set each states features to all 0
print('\n... adding noise to features at states 12, 34 and 64 ...\n')
X[11, :] = torch.zeros(X.size()[1])
X[33, :] = torch.zeros(X.size()[1])
X[63, :] = torch.zeros(X.size()[1])
#if noisey_paths:
# print('\n... adding noise to paths at states 12, 34 and 64 ...\n')
if (optim_type == 'Adam'):
print('\nOptimising with torch.Adam\n')
optimizer = torch.optim.Adam(
net.parameters(),
lr=configuration_dict.get('base_lr'),
weight_decay=1e-2) #weight decay for l2 regularisation
#while(evd > threshold): #termination criteria: evd threshold
#for p in range(configuration_dict.get('number_of_epochs')): #termination criteria: no of iters in config dict
while diff >= threshold: #termination criteria: loss diff
#for p in range(1): #for testing
prevLoss = loss
#net.zero_grad()
net.zero_grad(set_to_none=True)
output = torch.empty(len(X[0]), 1, dtype=torch.double)
indexer = 0
for j in range(len(X[0])):
thisR = net(X[:, j].view(-1, len(X[:, j])))
output[indexer] = thisR
indexer += 1
finalOutput = output
loss = NLL.apply(output, initD, mu_sa, muE, F,
mdp_data) #use this line for custom gradient
#loss = likelihood(output, initD, mu_sa, muE, F, mdp_data) #use this line for auto gradient
#tester.checkgradients_NN(output, NLL) # check gradients
loss.backward() # propagate grad through network
#nn.utils.clip_grad_norm_(net.parameters(), max_norm=2.0, norm_type=2)
evd = NLL.calculate_EVD(truep, torch.matmul(X, output)) # calc EVD
optimizer.step()
#printline to show est R
#print('{}: output:\n {} | EVD: {} | loss: {} '.format(i, torch.matmul(X, output).repeat(1, 5).detach() , evd, loss.detach() ))
#printline to hide est R
print('{}: | EVD: {} | loss: {} | diff {}'.format(
i, evd, loss, diff))
# store metrics for printing
NLList.append(loss)
iterations.append(i)
evdList.append(evd)
finaloutput = output
tensorboard_writer.add_scalar('loss', loss, i)
tensorboard_writer.add_scalar('evd', evd, i)
tensorboard_writer.add_scalar('diff', diff, i)
i += 1
diff = abs(prevLoss - loss)
else:
print('\implement LBFGS\n')
PATH = './NN_IRL.pth'
torch.save(net.state_dict(), PATH)
tensorboard_writer.close()
if NLL_EVD_plots:
# plot
f, (ax1, ax2) = plt.subplots(1, 2, sharex=True)
ax1.plot(iterations, NLList)
ax1.plot(iterations, NLList, 'r+')
ax1.set_title('NLL')
ax2.plot(iterations, evdList)
ax2.plot(iterations, evdList, 'r+')
ax2.set_title('Expected Value Diff')
#plt.show()
print("\nruntime: --- %s seconds ---\n" % (time.time() - start_time))
return net, finalOutput, (time.time() - start_time)
# Extract full reward function
y_mc_relu_reward = torch.from_numpy(y_mc_relu)
y_mc_relu_reward = y_mc_relu_reward.reshape(len(y_mc_relu_reward), 1)
y_mc_relu_reward = y_mc_relu_reward.repeat((1, 5))
#Solve with learned reward functions
y_mc_relu_v, y_mc_relu_q, y_mc_relu_logp, y_mc_relu_P = linearvalueiteration(
mdp_data, y_mc_relu_reward)
'''
# Print results
print("\nTrue R has:\n - negated likelihood: {}\n - EVD: {}".format(trueNLL, irl_model.NLL.calculate_EVD(truep, r)))
print("\nPred R with ReLU activation has:\n - negated likelihood: {}\n - EVD: {}".format(irl_model.NLL.apply(y_mc_relu_reward, initD, mu_sa, muE, feature_data['splittable'], mdp_data), irl_model.NLL.calculate_EVD(truep, y_mc_relu_reward)))
'''
# Initalise loss function
NLL = NLLFunction()
# Assign loss function constants
NLL.F = feature_data['splittable']
NLL.muE = muE
NLL.mu_sa = mu_sa
NLL.initD = initD
NLL.mdp_data = mdp_data
#Save results
print('\n... saving results ...\n')
# Create path for trained models
RESULTS_PATH = "./noisey_paths/results/dropout/"
for path in [RESULTS_PATH]:
try:
os.makedirs(path)
#Print what benchmark
if (user_input):
if worldtype == "gridworld" or worldtype == "gw" or worldtype == "grid":
print('\n... training on GridWorld benchmark ... \n')
elif worldtype == "objectworld" or worldtype == "ow" or worldtype == "obj":
print('\n... training on ObjectWorld benchmark ... \n')
else:
print('\n... training on GridWorld benchmark ... \n')
#Print true R loss
print('\n... true reward loss is', trueNLL.item(), '... \n')
# Initalise loss function
NLL = NLLFunction()
# Remove chosen states from paths
if states_to_remove[index_states_to_remove] is not None:
N = len(example_samples)
top_index = math.ceil(0.5 * N)
twenty_percent_example_samples = example_samples[0:top_index]
for path in twenty_percent_example_samples:
T = len(path)
pathindex = twenty_percent_example_samples.index(path)
for move in path:
moveindex = twenty_percent_example_samples[pathindex].index(
move)
#remove state
if move[0] in states_to_remove[index_states_to_remove]:
newmove = move
example_samples = NNIRL_param_list[9]
mdp_params = NNIRL_param_list[10]
r = NNIRL_param_list[11]
mdp_solution = NNIRL_param_list[12]
feature_data = NNIRL_param_list[13]
trueNLL = NNIRL_param_list[14]
normalise = NNIRL_param_list[15]
user_input = NNIRL_param_list[16]
worldtype = NNIRL_param_list[17]
torch.manual_seed(mdp_params['seed'])
np.random.seed(seed=mdp_params['seed'])
random.seed(mdp_params['seed'])
# Initalise tester loss function
testNLL = NLLFunction()
# Assign tester loss function constants
testNLL.F = feature_data['splittable']
testNLL.muE = muE
testNLL.mu_sa = mu_sa
testNLL.initD = initD
testNLL.mdp_data = mdp_data
#Print what benchmark
if (user_input):
if worldtype == "gridworld" or worldtype == "gw" or worldtype == "grid":
print('\n... training on GridWorld benchmark ... \n')
elif worldtype == "objectworld" or worldtype == "ow" or worldtype == "obj":
print('\n... training on ObjectWorld benchmark ... \n')
else:
print('\n... training on GridWorld benchmark ... \n')
random.seed(mdp_params['seed'])
#Print what benchmark
if (user_input):
if worldtype == "gridworld" or worldtype == "gw" or worldtype == "grid":
print('\n... training on GridWorld benchmark ... \n')
elif worldtype == "objectworld" or worldtype == "ow" or worldtype == "obj":
print('\n... training on ObjectWorld benchmark ... \n')
else:
print('\n... training on GridWorld benchmark ... \n')
#Print true R loss
print('\n... true reward loss is', trueNLL.item(), '... \n')
# Initalise loss function
NLL = NLLFunction()
# Remove chosen states from paths
if states_to_remove[index_states_to_remove] is not None:
N = len(example_samples)
top_index = math.ceil(0.5 * N)
twenty_percent_example_samples = example_samples[0:top_index]
for path in twenty_percent_example_samples:
T = len(path)
pathindex = twenty_percent_example_samples.index(path)
for move in path:
moveindex = twenty_percent_example_samples[pathindex].index(
move)
#remove state
if move[0] in states_to_remove[index_states_to_remove]:
newmove = move
#Solve MDP
print("\n... performing value iteration for v, q, logp and truep ...")
v, q, logp, truep = linearvalueiteration(mdp_data, r)
mdp_solution = {'v': v, 'q': q, 'p': truep, 'logp': logp}
optimal_policy = torch.argmax(truep, axis=1)
print("\n... done ...")
#Sample paths
if new_paths:
print("\n... sampling paths from true R ...")
example_samples = sampleexamples(N, T, mdp_solution, mdp_data)
print("\n... done sampling", N, "paths ...")
NLL = NLLFunction() # initialise NLL
if new_paths:
initD, mu_sa, muE, F, mdp_data = NLL.calc_var_values(mdp_data, N, T, example_samples, feature_data) # calculate required variables
else:
print("\n... using pre-loaded sampled paths ...")
# Load variables
open_file = open("NNIRL_param_list.pkl", "rb")
NNIRL_param_list = pickle.load(open_file)
open_file.close()
threshold = NNIRL_param_list[0]
optim_type = NNIRL_param_list[1]
net = NNIRL_param_list[2]
initD = NNIRL_param_list[3]
mu_sa = NNIRL_param_list[4]
muE = NNIRL_param_list[5]
mdp_data = NNIRL_param_list[6]
example_samples = NNIRL_param_list[9]
mdp_params = NNIRL_param_list[10]
r = NNIRL_param_list[11]
mdp_solution = NNIRL_param_list[12]
feature_data = NNIRL_param_list[13]
trueNLL = NNIRL_param_list[14]
normalise = NNIRL_param_list[15]
user_input = NNIRL_param_list[16]
worldtype = NNIRL_param_list[17]
torch.manual_seed(mdp_params['seed'])
np.random.seed(seed=mdp_params['seed'])
random.seed(mdp_params['seed'])
# Initalise tester loss function
testNLL = NLLFunction()
# Assign tester loss function constants
testNLL.F = feature_data['splittable']
testNLL.muE = muE
testNLL.mu_sa = mu_sa
testNLL.initD = initD
testNLL.mdp_data = mdp_data
#Print what benchmark
if (user_input):
if worldtype == "gridworld" or worldtype == "gw" or worldtype == "grid":
print('\n... training on GridWorld benchmark ... \n')
elif worldtype == "objectworld" or worldtype == "ow" or worldtype == "obj":
print('\n... training on ObjectWorld benchmark ... \n')
else:
print('\n... training on GridWorld benchmark ... \n')
random.seed(mdp_params['seed'])
#Print what benchmark
if (user_input):
if worldtype == "gridworld" or worldtype == "gw" or worldtype == "grid":
print('\n... training on GridWorld benchmark ... \n')
elif worldtype == "objectworld" or worldtype == "ow" or worldtype == "obj":
print('\n... training on ObjectWorld benchmark ... \n')
else:
print('\n... training on GridWorld benchmark ... \n')
#Print true R loss
print('\n... true reward loss is', trueNLL.item(), '... \n')
# Initalise loss function
NLL = NLLFunction()
# Add noise to features
if states_to_remove[index_states_to_corrupt] is not None:
for state in states_to_remove[index_states_to_corrupt]:
if random.randint(0, 100) < 3: #3% chance of NOT using this state
break
for i in range(len(feature_data['splittable'][state, :])):
if random.randint(
0, 100) < 22: #22% chance of inverting the feature
#invert the feature, works since binary features
feature_data['splittable'][
state, i] = 1 - feature_data['splittable'][state, i]
# Connect configuration dict
configuration_dict = {
random.seed(mdp_params['seed'])
#Print what benchmark
if (user_input):
if worldtype == "gridworld" or worldtype == "gw" or worldtype == "grid":
print('\n... training on GridWorld benchmark ... \n')
elif worldtype == "objectworld" or worldtype == "ow" or worldtype == "obj":
print('\n... training on ObjectWorld benchmark ... \n')
else:
print('\n... training on GridWorld benchmark ... \n')
#Print true R loss
print('\n... true reward loss is', trueNLL.item(), '... \n')
# Initalise loss function
NLL = NLLFunction()
configuration_dict = {
'number_of_epochs': 3,
'base_lr': 0.1,
'p': dropout_val,
'no_hidden_layers': 3,
'no_neurons_in_hidden_layers': len(feature_data['splittable'][0]) * 2
} #set config params for clearml
#configuration_dict = task.connect(configuration_dict) #For when task being logged on clearML
# Assign loss function constants
NLL.F = feature_data['splittable']
NLL.muE = muE
NLL.mu_sa = mu_sa
NLL.initD = initD