def execute(gpu, exp_batch, exp_alias, suppress_output=True, number_of_workers=12):
"""
The main encoder training function.
Args:
gpu: The GPU id number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(os.path.join('configs', exp_batch, exp_alias + '.yaml'))
set_type_of_process('train_encoder')
# Set the process into loading status.
coil_logger.add_message('Loading', {'GPU': os.environ["CUDA_VISIBLE_DEVICES"]})
# we set a seed for this exp
seed_everything(seed=g_conf.MAGICAL_SEED)
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join('_output_logs', exp_alias + '_' +
g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"), "a",
buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
# Preload option
if g_conf.PRELOAD_MODEL_ALIAS is not None:
checkpoint = torch.load(os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS,
'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(os.path.join('_logs', exp_batch, exp_alias,
'checkpoints', str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 1000000000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
# full_dataset = os.path.join(os.environ["SRL_DATASET_PATH"], g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
if len(g_conf.EXPERIENCE_FILE) == 1:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2]
else:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2] + '_' + str(g_conf.EXPERIENCE_FILE[1]).split('/')[-1].split('.')[-2]
dataset = CoILDataset(transform=augmenter,
preload_name=g_conf.PROCESS_NAME + '_' + json_file_name + '_' + g_conf.DATA_USED)
print ("Loaded dataset")
data_loader = select_balancing_strategy(dataset, iteration, number_of_workers)
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE, g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.train()
print(encoder_model)
optimizer = optim.Adam(encoder_model.parameters(), lr=g_conf.LEARNING_RATE)
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
encoder_model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
print ("Before the loss")
if g_conf.ENCODER_MODEL_TYPE in ['ETE']:
criterion = Loss(g_conf.LOSS_FUNCTION)
# Loss time series window
for data in data_loader:
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
capture_time = time.time()
encoder_model.zero_grad()
"""
####################################
ENCODER_MODEL_TYPE can be: one-step-affordances, ETE, stdim, action_prediction
####################################
- one-step-affordances: input RGB images, compute affordances loss.
- ETE: input RGB images and speed, compute action loss (steering, throttle, brake)
- stdim: input two consecutive RGB images, compute the feature loss
- action_prediction: input two consecutive RGB images, compute action classification loss
- forward: input two consecutive RGB images, compute action loss + feature loss
"""
if g_conf.ENCODER_MODEL_TYPE in ['one-step-affordances']:
loss_function_params = {
'classification_gt': dataset.extract_affordances_targets(data, 'classification').cuda(),
# harzard stop, red_light....
'class_weights': g_conf.AFFORDANCES_CLASS_WEIGHT,
'regression_gt': dataset.extract_affordances_targets(data, 'regression').cuda(),
'variable_weights': g_conf.AFFORDANCES_VARIABLE_WEIGHT
}
# we input RGB images, speed and command to train affordances
loss = encoder_model(torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(dataset.extract_commands(data).cuda()),
loss_function_params)
if iteration == 0:
state = {
'iteration': iteration,
'state_dict': encoder_model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(state, os.path.join('_logs', exp_batch, exp_alias
, 'checkpoints', 'inital.pth'))
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['forward']:
# We sample another batch to avoid the superposition
inputs_data = [data['rgb'][0].cuda(), data['rgb'][1].cuda()]
loss, loss_other, loss_ete = encoder_model(inputs_data,
dataset.extract_inputs(data),
# We also add measurements and commands
dataset.extract_commands(data),
dataset.extract_targets(data)[0].cuda()
)
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['ETE']:
branches = encoder_model(torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(dataset.extract_commands(data).cuda()))
loss_function_params = {
'branches': branches,
'targets': dataset.extract_targets(data).cuda(), # steer, throttle, brake
'inputs': dataset.extract_inputs(data).cuda(), # speed
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
loss, _ = criterion(loss_function_params)
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['stdim']:
inputs_data = [data['rgb'][0].cuda(), data['rgb'][1].cuda()]
loss, _, _ = encoder_model(inputs_data,
dataset.extract_inputs(data),
# We also add measurements and commands
dataset.extract_commands(data)
)
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['action_prediction']:
inputs_data = [data['rgb'][0].cuda(), data['rgb'][1].cuda()]
loss, _, _ = encoder_model(inputs_data,
dataset.extract_inputs(data),
# We also add measurements and commands
dataset.extract_commands(data),
dataset.extract_targets(data)[0].cuda()
)
loss.backward()
optimizer.step()
else:
raise ValueError("The encoder model type is not know")
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': encoder_model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(state, os.path.join('_logs', exp_batch, exp_alias
, 'checkpoints', str(iteration) + '.pth'))
iteration += 1
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
if g_conf.ENCODER_MODEL_TYPE in ['stdim', 'action_prediction', 'forward']:
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('f_t', torch.squeeze(data['rgb'][0]), iteration)
coil_logger.add_image('f_ti', torch.squeeze(data['rgb'][1]), iteration)
elif g_conf.ENCODER_MODEL_TYPE in ['one-step-affordances', 'ETE']:
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']), iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
accumulated_time += time.time() - capture_time
coil_logger.add_message('Iterating',
{'Iteration': iteration,
'Loss': loss.data.tolist(),
'Images/s': (iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss': best_loss, 'BestLossIteration': best_loss_iter},
iteration)
loss_window.append(loss.data.tolist())
coil_logger.write_on_error_csv('train', loss.data)
if iteration % 100 == 0:
print('Train Iteration: {} [{}/{} ({:.0f}%)] \t Loss: {:.6f}'.format(
iteration, iteration, g_conf.NUMBER_ITERATIONS,
100. * iteration / g_conf.NUMBER_ITERATIONS, loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu,
exp_batch,
exp_alias,
json_file_path,
suppress_output,
encoder_params=None,
plot_attentions=False):
try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
if json_file_path is not None:
json_file_name = json_file_path.split('/')[-1].split('.')[-2]
else:
raise RuntimeError(
"You need to define the validation json file path")
# At this point the log file with the correct naming is created.
merge_with_yaml(
os.path.join('configs', exp_batch, exp_alias + '.yaml'),
encoder_params)
if plot_attentions:
set_type_of_process('validation',
json_file_name + '_plotAttention')
else:
set_type_of_process('validation', json_file_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
# We create file for saving validation results
summary_file = os.path.join('_logs', exp_batch, g_conf.EXPERIMENT_NAME,
g_conf.PROCESS_NAME + '_csv',
'valid_summary_1camera.csv')
g_conf.immutable(False)
g_conf.DATA_USED = 'central'
g_conf.immutable(True)
if not os.path.exists(summary_file):
csv_outfile = open(summary_file, 'w')
csv_outfile.write(
"%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n" %
('step', 'accumulated_pedestrian_TP',
'accumulated_pedestrian_FP', 'accumulated_pedestrian_FN',
'accumulated_pedestrian_TN', 'accumulated_vehicle_stop_TP',
'accumulated_vehicle_stop_FP', 'accumulated_vehicle_stop_FN',
'accumulated_vehicle_stop_TN', 'accumulated_red_tl_TP',
'accumulated_red_tl_FP', 'accumulated_red_tl_FN',
'accumulated_red_tl_TN', 'MAE_relative_angle'))
csv_outfile.close()
latest = get_latest_evaluated_checkpoint_2(summary_file)
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
#full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name)
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
dataset = CoILDataset(transform=augmenter,
preload_name=g_conf.PROCESS_NAME + '_' +
g_conf.DATA_USED,
process_type='validation',
vd_json_file_path=json_file_path)
print("Loaded Validation dataset")
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
if g_conf.MODEL_TYPE in ['one-step-affordances']:
# one step training, no need to retrain FC layers, we just get the output of encoder model as prediciton
model = EncoderModel(g_conf.ENCODER_MODEL_TYPE,
g_conf.ENCODER_MODEL_CONFIGURATION)
model.cuda()
#print(model)
elif g_conf.MODEL_TYPE in ['separate-affordances']:
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION,
g_conf.ENCODER_MODEL_CONFIGURATION)
model.cuda()
#print(model)
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE,
g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.eval()
# Here we load the pre-trained encoder (not fine-tunned)
if g_conf.FREEZE_ENCODER:
if encoder_params is not None:
encoder_checkpoint = torch.load(
os.path.join(
'_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'], 'checkpoints',
str(encoder_params['encoder_checkpoint']) +
'.pth'))
print(
"Encoder model ",
str(encoder_params['encoder_checkpoint']),
"loaded from ",
os.path.join('_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'],
'checkpoints'))
encoder_model.load_state_dict(
encoder_checkpoint['state_dict'])
encoder_model.eval()
for param_ in encoder_model.parameters():
param_.requires_grad = False
while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE):
latest = get_next_checkpoint_2(g_conf.TEST_SCHEDULE, summary_file)
if os.path.exists(
os.path.join('_logs', exp_batch, g_conf.EXPERIMENT_NAME,
'checkpoints',
str(latest) + '.pth')):
checkpoint = torch.load(
os.path.join('_logs', exp_batch, g_conf.EXPERIMENT_NAME,
'checkpoints',
str(latest) + '.pth'))
checkpoint_iteration = checkpoint['iteration']
model.load_state_dict(checkpoint['state_dict'])
print("Validation checkpoint ", checkpoint_iteration)
model.eval()
for param_ in model.parameters():
param_.requires_grad = False
# Here we load the fine-tunned encoder
if not g_conf.FREEZE_ENCODER and g_conf.MODEL_TYPE not in [
'one-step-affordances'
]:
encoder_checkpoint = torch.load(
os.path.join('_logs', exp_batch,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(latest) + '_encoder.pth'))
print(
"FINE TUNNED encoder model ",
str(latest) + '_encoder.pth', "loaded from ",
os.path.join('_logs', exp_batch,
g_conf.EXPERIMENT_NAME, 'checkpoints'))
encoder_model.load_state_dict(
encoder_checkpoint['state_dict'])
encoder_model.eval()
for param_ in encoder_model.parameters():
param_.requires_grad = False
accumulated_mae_ra = 0
accumulated_pedestrian_TP = 0
accumulated_pedestrian_TN = 0
accumulated_pedestrian_FN = 0
accumulated_pedestrian_FP = 0
accumulated_red_tl_TP = 0
accumulated_red_tl_TN = 0
accumulated_red_tl_FP = 0
accumulated_red_tl_FN = 0
accumulated_vehicle_stop_TP = 0
accumulated_vehicle_stop_TN = 0
accumulated_vehicle_stop_FP = 0
accumulated_vehicle_stop_FN = 0
iteration_on_checkpoint = 0
for data in data_loader:
if g_conf.MODEL_TYPE in ['one-step-affordances']:
c_output, r_output, layers = model.forward_outputs(
torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
dataset.extract_commands(data).cuda())
elif g_conf.MODEL_TYPE in ['separate-affordances']:
if g_conf.ENCODER_MODEL_TYPE in [
'action_prediction', 'stdim', 'ETEDIM',
'FIMBC', 'one-step-affordances'
]:
e, layers = encoder_model.forward_encoder(
torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(
dataset.extract_commands(data).cuda()))
c_output, r_output = model.forward_test(e)
elif g_conf.ENCODER_MODEL_TYPE in [
'ETE', 'ETE_inverse_model', 'forward',
'ETE_stdim'
]:
e, layers = encoder_model.forward_encoder(
torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(
dataset.extract_commands(data).cuda()))
c_output, r_output = model.forward_test(e)
if plot_attentions:
attentions_path = os.path.join(
'_logs', exp_batch, g_conf.EXPERIMENT_NAME,
g_conf.PROCESS_NAME + '_attentions_' + str(latest))
write_attentions(torch.squeeze(data['rgb']), layers,
iteration_on_checkpoint,
attentions_path)
# Accurancy = (TP+TN)/(TP+TN+FP+FN)
# F1-score = 2*TP / (2*TP + FN + FP)
classification_gt = dataset.extract_affordances_targets(
data, 'classification')
regression_gt = dataset.extract_affordances_targets(
data, 'regression')
TP = 0
FN = 0
FP = 0
TN = 0
for i in range(classification_gt.shape[0]):
if classification_gt[i, 0] == (
c_output[0][i, 0] < c_output[0][i, 1]).type(
torch.FloatTensor) == 1:
TP += 1
elif classification_gt[
i, 0] == 1 and classification_gt[i, 0] != (
c_output[0][i, 0] <
c_output[0][i, 1]).type(torch.FloatTensor):
FN += 1
elif classification_gt[
i, 0] == 0 and classification_gt[i, 0] != (
c_output[0][i, 0] <
c_output[0][i, 1]).type(torch.FloatTensor):
FP += 1
if classification_gt[i, 0] == (
c_output[0][i, 0] < c_output[0][i, 1]).type(
torch.FloatTensor) == 0:
TN += 1
accumulated_pedestrian_TP += TP
accumulated_pedestrian_TN += TN
accumulated_pedestrian_FP += FP
accumulated_pedestrian_FN += FN
TP = 0
FN = 0
FP = 0
TN = 0
for i in range(classification_gt.shape[0]):
if classification_gt[i, 1] == (
c_output[1][i, 0] < c_output[1][i, 1]).type(
torch.FloatTensor) == 1:
TP += 1
elif classification_gt[
i, 1] == 1 and classification_gt[i, 1] != (
c_output[1][i, 0] <
c_output[1][i, 1]).type(torch.FloatTensor):
FN += 1
elif classification_gt[
i, 1] == 0 and classification_gt[i, 1] != (
c_output[1][i, 0] <
c_output[1][i, 1]).type(torch.FloatTensor):
FP += 1
if classification_gt[i, 1] == (
c_output[1][i, 0] < c_output[1][i, 1]).type(
torch.FloatTensor) == 0:
TN += 1
accumulated_red_tl_TP += TP
accumulated_red_tl_TN += TN
accumulated_red_tl_FP += FP
accumulated_red_tl_FN += FN
TP = 0
FN = 0
FP = 0
TN = 0
for i in range(classification_gt.shape[0]):
if classification_gt[i, 2] == (
c_output[2][i, 0] < c_output[2][i, 1]).type(
torch.FloatTensor) == 1:
TP += 1
elif classification_gt[i, 2] == 1 and classification_gt[i, 2] !=\
(c_output[2][i, 0] < c_output[2][i, 1]).type(torch.FloatTensor):
FN += 1
elif classification_gt[i, 2] == 0 and classification_gt[i, 2] !=\
(c_output[2][i, 0] < c_output[2][i, 1]).type(torch.FloatTensor):
FP += 1
if classification_gt[i, 2] == (
c_output[2][i, 0] < c_output[2][i, 1]).type(
torch.FloatTensor) == 0:
TN += 1
accumulated_vehicle_stop_TP += TP
accumulated_vehicle_stop_TN += TN
accumulated_vehicle_stop_FP += FP
accumulated_vehicle_stop_FN += FN
# if the data was normalized during training, we need to transform it to its unit
write_regular_output(checkpoint_iteration,
torch.squeeze(r_output[0]),
regression_gt[:, 0])
mae_ra = torch.abs(regression_gt[:, 0] -
torch.squeeze(r_output[0]).type(torch.FloatTensor)).\
numpy()
accumulated_mae_ra += np.sum(mae_ra)
if iteration_on_checkpoint % 100 == 0:
print(
"Validation iteration: %d [%d/%d)] on Checkpoint %d "
%
(iteration_on_checkpoint, iteration_on_checkpoint,
len(data_loader), checkpoint_iteration))
iteration_on_checkpoint += 1
# Here also need a better analysis. TODO divide into curve and other things
MAE_relative_angle = accumulated_mae_ra / (len(dataset))
csv_outfile = open(summary_file, 'a')
csv_outfile.write(
"%s, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f" %
(checkpoint_iteration, accumulated_pedestrian_TP,
accumulated_pedestrian_FP, accumulated_pedestrian_FN,
accumulated_pedestrian_TN, accumulated_vehicle_stop_TP,
accumulated_vehicle_stop_FP, accumulated_vehicle_stop_FN,
accumulated_vehicle_stop_TN, accumulated_red_tl_TP,
accumulated_red_tl_FP, accumulated_red_tl_FN,
accumulated_red_tl_TN, MAE_relative_angle))
csv_outfile.write("\n")
csv_outfile.close()
else:
print('The checkpoint you want to validate is not yet ready ',
str(latest))
coil_logger.add_message('Finished', {})
print('VALIDATION FINISHED !!')
print(' Validation results saved in ==> ', summary_file)
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
except RuntimeError as e:
if latest is not None:
coil_logger.erase_csv(latest)
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
def execute(gpu,
exp_batch,
exp_alias,
suppress_output=True,
number_of_workers=12,
encoder_params=None):
"""
The main training function. This functions loads the latest checkpoint
for a given, exp_batch (folder) and exp_alias (experiment configuration).
With this checkpoint it starts from the beginning or continue some training.
Args:
gpu: The GPU number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(
os.path.join('configs', exp_batch, exp_alias + '.yaml'),
encoder_params)
set_type_of_process('train')
# Set the process into loading status.
coil_logger.add_message('Loading',
{'GPU': os.environ["CUDA_VISIBLE_DEVICES"]})
seed_everything(seed=g_conf.MAGICAL_SEED)
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
if coil_logger.check_finish('train'):
coil_logger.add_message('Finished', {})
return
# Preload option
print(" GOING TO LOAD")
if g_conf.PRELOAD_MODEL_ALIAS is not None:
print(" LOADING A PRELOAD")
checkpoint = torch.load(
os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS, 'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
else:
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
print('loading previous checkpoint ', checkpoint_file)
checkpoint = torch.load(
os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 100000000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
#full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
# We can save preload dataset depends on the json file name, then no need to load dataset for each time with the same dataset
if len(g_conf.EXPERIENCE_FILE) == 1:
json_file_name = str(
g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2]
else:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split(
'/')[-1].split('.')[-2] + '_' + str(
g_conf.EXPERIENCE_FILE[1]).split('/')[-1].split('.')[-2]
dataset = CoILDataset(transform=augmenter,
preload_name=g_conf.PROCESS_NAME + '_' +
json_file_name + '_' + g_conf.DATA_USED)
#dataset = CoILDataset(transform=augmenter, preload_name=str(g_conf.NUMBER_OF_HOURS)+ 'hours_' + g_conf.TRAIN_DATASET_NAME)
print("Loaded Training dataset")
data_loader = select_balancing_strategy(dataset, iteration,
number_of_workers)
if g_conf.MODEL_TYPE in ['separate-affordances']:
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION,
g_conf.ENCODER_MODEL_CONFIGURATION)
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE)
print(model)
# we use the pre-trained encoder model to extract bottleneck Z and train the E-t-E model
if g_conf.MODEL_TYPE in ['separate-affordances']:
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE,
g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.eval()
# To freeze the pre-trained encoder model
if g_conf.FREEZE_ENCODER:
for param_ in encoder_model.parameters():
param_.requires_grad = False
if encoder_params is not None:
encoder_checkpoint = torch.load(
os.path.join(
'_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'], 'checkpoints',
str(encoder_params['encoder_checkpoint']) + '.pth'))
print(
"Encoder model ",
str(encoder_params['encoder_checkpoint']), "loaded from ",
os.path.join('_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'], 'checkpoints'))
encoder_model.load_state_dict(encoder_checkpoint['state_dict'])
if g_conf.FREEZE_ENCODER:
encoder_model.eval()
# To freeze the pre-trained encoder model
for param_ in encoder_model.parameters():
param_.requires_grad = False
else:
optimizer = optim.Adam(list(model.parameters()) +
list(encoder_model.parameters()),
lr=g_conf.LEARNING_RATE)
for name_encoder, param_encoder in encoder_model.named_parameters(
):
if param_encoder.requires_grad:
print(' Unfrozen layers', name_encoder)
else:
print(' Frozen layers', name_encoder)
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
for name, param in model.named_parameters():
if param.requires_grad:
print(' Unfrozen layers', name)
else:
print(' Frozen layers', name)
print("Before the loss")
# Loss time series window
for data in data_loader:
# Basically in this mode of execution, we validate every X Steps, if it goes up 3 times,
# add a stop on the _logs folder that is going to be read by this process
if g_conf.FINISH_ON_VALIDATION_STALE is not None and \
check_loss_validation_stopped(iteration, g_conf.FINISH_ON_VALIDATION_STALE):
break
"""
####################################
Main optimization loop
####################################
"""
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
model.zero_grad()
if not g_conf.FREEZE_ENCODER:
encoder_model.zero_grad()
if g_conf.LABELS_SUPERVISED:
inputs_data = torch.cat(
(data['rgb'], torch.zeros(g_conf.BATCH_SIZE, 1, 88, 200)),
dim=1).cuda()
else:
inputs_data = torch.squeeze(data['rgb'].cuda())
if g_conf.MODEL_TYPE in ['separate-affordances']:
#TODO: for this two encoder models training, we haven't put speed as input to train yet
if g_conf.ENCODER_MODEL_TYPE in [
'action_prediction', 'stdim', 'forward',
'one-step-affordances'
]:
e, inter = encoder_model.forward_encoder(
inputs_data,
dataset.extract_inputs(data).cuda(),
# We also add measurements and commands
torch.squeeze(dataset.extract_commands(data).cuda()))
elif g_conf.ENCODER_MODEL_TYPE in ['ETE']:
e, inter = encoder_model.forward_encoder(
inputs_data,
dataset.extract_inputs(data).cuda(),
torch.squeeze(dataset.extract_commands(data).cuda()))
loss_function_params = {
'classification_gt':
dataset.extract_affordances_targets(
data, 'classification').cuda(),
# harzard stop, red_light....
'class_weights':
g_conf.AFFORDANCES_CLASS_WEIGHT,
'regression_gt':
dataset.extract_affordances_targets(data,
'regression').cuda(),
'variable_weights':
g_conf.AFFORDANCES_VARIABLE_WEIGHT
}
loss = model(e, loss_function_params)
loss.backward()
optimizer.step()
else:
raise RuntimeError(
'Not implement yet, this branch is only work for g_conf.MODEL_TYPE in [separate-affordances]'
)
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
state,
os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(iteration) + '.pth'))
if not g_conf.FREEZE_ENCODER:
encoder_state = {
'iteration': iteration,
'state_dict': encoder_model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
encoder_state,
os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(iteration) + '_encoder.pth'))
iteration += 1
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']),
iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
if iteration % 100 == 0:
print('Train Iteration: {} [{}/{} ({:.0f}%)] \t Loss: {:.6f}'.
format(iteration, iteration, g_conf.NUMBER_ITERATIONS,
100. * iteration / g_conf.NUMBER_ITERATIONS,
loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def gather_data():
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
exp_batch = 'EXP'
exp_alias = 'customized'
number_of_workers = 12
encoder_params = {'encoder_checkpoint': 10000,
'encoder_folder': 'ENCODER',
'encoder_exp': 'customized'}
try:
merge_with_yaml(os.path.join('configs', exp_batch, exp_alias + '.yaml'), encoder_params)
g_conf.PROCESS_NAME = 'analyze_embeddings'
# We can save preload dataset depends on the json file name, then no need to load dataset for each time with the same dataset
if len(g_conf.EXPERIENCE_FILE) == 1:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2]
else:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2] + '_' + str(g_conf.EXPERIENCE_FILE[1]).split('/')[-1].split('.')[-2]
dataset = CoILDataset(transform=None,
preload_name=g_conf.PROCESS_NAME + '_' + json_file_name + '_' + g_conf.DATA_USED)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=g_conf.BATCH_SIZE, sampler=None, num_workers=12, pin_memory=True)
print ("Loaded Training dataset")
# we use the pre-trained encoder model to extract bottleneck Z and train the E-t-E model
iteration = 0
if g_conf.MODEL_TYPE in ['separate-affordances']:
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE, g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.eval()
# To freeze the pre-trained encoder model
if g_conf.FREEZE_ENCODER:
for param_ in encoder_model.parameters():
param_.requires_grad = False
encoder_checkpoint = torch.load(
os.path.join('_logs', encoder_params['encoder_folder'], encoder_params['encoder_exp'], 'checkpoints',
str(encoder_params['encoder_checkpoint']) + '.pth'))
print("Encoder model ", str(encoder_params['encoder_checkpoint']), "loaded from ",
os.path.join('_logs', encoder_params['encoder_folder'], encoder_params['encoder_exp'], 'checkpoints'))
encoder_model.load_state_dict(encoder_checkpoint['state_dict'])
print('g_conf.ENCODER_MODEL_TYPE', g_conf.ENCODER_MODEL_TYPE)
behaviors = []
route_folder = 'customized'
route_files = os.listdir(route_folder)
count = 0
kept_inds = []
total_length = 0
for route_file in sorted(route_files):
print('-'*100, route_file)
route_path = os.path.join(route_folder, route_file)
if os.path.isdir(route_path):
measurements_path = route_path + '/' + 'driving_log.csv'
import pandas as pd
df = pd.read_csv(measurements_path, delimiter=',')
# remove cases which has ego_linear_speed < 0.1 or other_actor_linear_speed < 0
ego_fault = True
events_path = route_path + '/' + 'events.txt'
with open(events_path) as json_file:
events = json.load(json_file)
infractions = events['_checkpoint']['records'][0]['infractions']
import re
infraction_types = ['collisions_layout', 'collisions_pedestrian', 'collisions_vehicle', 'red_light', 'on_sidewalk', 'outside_lane_infraction', 'wrong_lane', 'off_road']
for infraction_type in infraction_types:
for infraction in infractions[infraction_type]:
if 'collisions' in infraction_type:
typ = re.search('.*with type=(.*) and id.*', infraction).group(1)
if 'walker' not in typ:
ego_fault = False
else:
loc = re.search('.*x=(.*), y=(.*), z=(.*), ego_linear_speed=(.*), other_actor_linear_speed=(.*)\)', infraction)
if loc:
ego_linear_speed = float(loc.group(4))
other_actor_linear_speed = float(loc.group(5))
if ego_linear_speed < 0.1 or other_actor_linear_speed < 0:
ego_fault = False
else:
loc = re.search('.*x=(.*), y=(.*), z=(.*)', infraction)
if loc:
ego_fault = False
cur_behaviors = df[['behaviors']].to_numpy()
if ego_fault:
behaviors.append(cur_behaviors)
kept_inds.append(np.array(range(total_length, total_length+cur_behaviors.shape[0])))
count += 1
total_length += cur_behaviors.shape[0]
kept_inds = np.concatenate(kept_inds)
print(f'{count}/{len(route_files)} are used')
misbehavior_labels = np.concatenate(behaviors).squeeze()
print(misbehavior_labels.shape, len(dataset))
print(np.sum(misbehavior_labels==0), np.sum(misbehavior_labels==1))
num_of_iterations = len(dataset) // g_conf.BATCH_SIZE
image_embeddings = []
for data in data_loader:
inputs_data = torch.squeeze(data['rgb'].cuda())
# separate_affordance
e, inter = encoder_model.forward_encoder(inputs_data,
dataset.extract_inputs(data).cuda(),torch.squeeze(dataset.extract_commands(data).cuda()))
# print(len(e), e[0].shape)
image_embeddings.append(e.cpu())
iteration += 1
if iteration == num_of_iterations:
break
image_embeddings = np.concatenate(image_embeddings, axis=0)
image_embeddings = image_embeddings[kept_inds]
print(image_embeddings.shape, misbehavior_labels.shape)
np.savez('misbehavior/embeddings_and_misbehavior_labels', image_embeddings=image_embeddings, misbehavior_labels=misbehavior_labels[:len(image_embeddings)])
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})