def test_speed_reading(self):
if not os.path.exists(self.test_images_write_path + 'normal_steer'):
os.mkdir(self.test_images_write_path + 'normal_steer')
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
'1HoursW1-3-6-8')
# This depends on the number of fused frames. A image could have
# A certain number of fused frames
augmenter = Augmenter(None)
dataset = CoILDataset(full_dataset, transform=augmenter)
keys = range(
0,
len(dataset.measurements[0, :]) - g_conf.NUMBER_IMAGES_SEQUENCE)
sampler = BatchSequenceSampler(
splitter.control_steer_split(dataset.measurements,
dataset.meta_data, keys),
0 * g_conf.BATCH_SIZE, g_conf.BATCH_SIZE,
g_conf.NUMBER_IMAGES_SEQUENCE, g_conf.SEQUENCE_STRIDE)
# data_loader = torch.utils.data.DataLoader(dataset, batch_size=120,
# shuffle=True, num_workers=12, pin_memory=True)
# capture_time = time.time()
data_loader = torch.utils.data.DataLoader(dataset,
batch_sampler=sampler,
shuffle=False,
num_workers=12,
pin_memory=True)
count = 0
print('len ', len(data_loader))
max_steer = 0
for data in data_loader:
print(count)
image, labels = data
count += 1
print("MAX STEER ", max_steer)
parser.add_argument('--gradcam_path',
type=str,
required=True,
help='path to save gradcam heatmap')
parser.add_argument('--type',
type=str,
required=True,
help='type of evaluation')
args = parser.parse_args()
merge_with_yaml(args.config)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
augmenter = Augmenter(None)
dataset = CoILDataset(args.dataset_path,
transform=augmenter,
preload_name=args.preload_name)
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
model = model.cuda()
checkpoint = torch.load(args.checkpoint)
def execute(gpu, exp_batch, exp_alias, validation_dataset, suppress_output):
latest = None
try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
merge_with_yaml(os.path.join('configs', exp_batch,
f'{exp_alias}.yaml'))
# The validation dataset is always fully loaded, so we fix a very high number of hours
g_conf.NUMBER_OF_HOURS = 10000
set_type_of_process(process_type='validation',
param=validation_dataset)
# Save the output to a file if so desired
if suppress_output:
save_output(exp_alias)
# Define the dataset. This structure has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
validation_dataset)
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_name=validation_dataset,
process_type='validation')
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION,
g_conf.SENSORS).cuda()
# The window used to keep track of the trainings
l1_window = []
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(
validation_dataset, None)
# Keep track of the best loss and the iteration where it happens
best_loss = 1000
best_loss_iter = 0
print(20 * '#')
print('Starting validation!')
print(20 * '#')
# Check if the maximum checkpoint for validating has been reached
while not maximum_checkpoint_reached(latest):
# Wait until the next checkpoint is ready (assuming this is run whilst training the model)
if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE):
# Get next checkpoint for validation according to the test schedule and load it
latest = get_next_checkpoint(g_conf.TEST_SCHEDULE)
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
f'{latest}.pth'))
checkpoint_iteration = checkpoint['iteration']
model.load_state_dict(checkpoint['state_dict'])
model.eval() # Turn off dropout and batchnorm (if any)
print(f"Validation loaded, checkpoint {checkpoint_iteration}")
# Main metric will be the used loss for training the network
criterion = Loss(g_conf.LOSS_FUNCTION)
checkpoint_average_loss = 0
# Counter
iteration_on_checkpoint = 0
with torch.no_grad(): # save some computation/memory
for data in data_loader:
# Compute the forward pass on a batch from the validation dataset
controls = data['directions'].cuda()
img = torch.squeeze(data['rgb']).cuda()
speed = dataset.extract_inputs(
data).cuda() # this might not always be speed
# For auxiliary metrics
output = model.forward_branch(img, speed, controls)
# For the loss function
branches = model(img, speed)
loss_function_params = {
'branches': branches,
'targets': dataset.extract_targets(data).cuda(),
'controls': controls,
'inputs': speed,
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
# It could be either waypoints or direct control
if 'waypoint1_angle' in g_conf.TARGETS:
write_waypoints_output(checkpoint_iteration,
output)
else:
write_regular_output(checkpoint_iteration, output)
loss, _ = criterion(loss_function_params)
loss = loss.data.tolist()
# Log a random position
position = random.randint(
0,
len(output.data.tolist()) - 1)
coil_logger.add_message(
'Iterating', {
'Checkpoint':
latest,
'Iteration':
f'{iteration_on_checkpoint * g_conf.BATCH_SIZE}/{len(dataset)}',
f'Validation Loss ({g_conf.LOSS_FUNCTION})':
loss,
'Output':
output[position].data.tolist(),
'GroundTruth':
dataset.extract_targets(
data)[position].data.tolist(),
'Inputs':
dataset.extract_inputs(data)
[position].data.tolist()
}, latest)
# We get the average with a growing list of values
# Thanks to John D. Cook: http://www.johndcook.com/blog/standard_deviation/
iteration_on_checkpoint += 1
checkpoint_average_loss += (
loss -
checkpoint_average_loss) / iteration_on_checkpoint
print(
f"\rProgress: {100 * iteration_on_checkpoint * g_conf.BATCH_SIZE / len(dataset):3.4f}% - "
f"Average Loss ({g_conf.LOSS_FUNCTION}): {checkpoint_average_loss:.16f}",
end='')
"""
########
Finish a round of validation, write results, wait for the next
########
"""
coil_logger.add_scalar(
f'Validation Loss ({g_conf.LOSS_FUNCTION})',
checkpoint_average_loss, latest, True)
# Let's visualize the distribution of the loss
coil_logger.add_histogram(
f'Validation Checkpoint Loss ({g_conf.LOSS_FUNCTION})',
checkpoint_average_loss, latest)
if checkpoint_average_loss < best_loss:
best_loss = checkpoint_average_loss
best_loss_iter = latest
coil_logger.add_message(
'Iterating', {
'Summary': {
'Loss': checkpoint_average_loss,
'BestLoss': best_loss,
'BestLossCheckpoint': best_loss_iter
},
'Checkpoint': latest
}, latest)
l1_window.append(checkpoint_average_loss)
coil_logger.write_on_error_csv(validation_dataset,
checkpoint_average_loss, latest)
# If we are using the finish when validation stops, we check the current checkpoint
if g_conf.FINISH_ON_VALIDATION_STALE is not None:
if dlib.count_steps_without_decrease(l1_window) > 3 and \
dlib.count_steps_without_decrease_robust(l1_window) > 3:
coil_logger.write_stop(validation_dataset, latest)
break
else:
latest = get_latest_evaluated_checkpoint()
time.sleep(1)
coil_logger.add_message('Loading',
{'Message': 'Waiting Checkpoint'})
print("Waiting for the next Validation")
print('\n' + 20 * '#')
print('Finished validation!')
print(20 * '#')
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
except RuntimeError as e:
if latest is not None:
coil_logger.erase_csv(latest)
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
def test_real_data_central_sampler(self):
try:
os.mkdir('_images')
except:
pass
augmenter = Augmenter(g_conf.AUGMENTATION)
dataset = CoILDataset('/home/felipe/Datasets/1HoursW1-3-6-8',
augmenter)
g_conf.NUMBER_IMAGES_SEQUENCE = 1
g_conf.SEQUENCE_STRIDE = 1
#g_conf.LABELS_DIVISION = [[0,2,5], [0,2,5], [0,2,5]]
g_conf.NUMBER_ITERATIONS = 1200
g_conf.BATCH_SIZE = 120
steerings = dataset.measurements[0, :]
# TODO: read meta data and turn into a coool dictionary ?
labels = dataset.measurements[24, :]
print(np.unique(labels))
print('position of camera',
np.where(dataset.meta_data[:, 0] == b'camera'))
camera_names = dataset.measurements[np.where(
dataset.meta_data[:, 0] == b'camera'), :][0][0]
print(" Camera names ")
print(camera_names)
keys = range(0, len(steerings) - g_conf.NUMBER_IMAGES_SEQUENCE)
one_camera_data = splitter.label_split(camera_names, keys, [[0]])
splitted_steer_labels = splitter.control_steer_split(
dataset.measurements, dataset.meta_data, one_camera_data[0])
for split_1 in splitted_steer_labels:
for split_2 in split_1:
for split_3 in split_2:
if split_3 not in one_camera_data[0]:
raise ValueError("not one camera")
#weights = [1.0/len(g_conf.STEERING_DIVISION)]*len(g_conf.STEERING_DIVISION)
#sampler = BatchSequenceSampler(splitted_steer_labels, 0, 120, g_conf.NUMBER_IMAGES_SEQUENCE,
# g_conf.SEQUENCE_STRIDE, False)
sampler = SubsetSampler(one_camera_data[0])
big_steer_vec = []
count = 0
data_loader = torch.utils.data.DataLoader(dataset,
sampler=sampler,
batch_size=120,
num_workers=12,
pin_memory=True)
for data in data_loader:
image, measurements = data
print(image['rgb'].shape)
for i in range(120):
name = '_images/' + str(count) + '.png'
image_to_save = transforms.ToPILImage()(
image['rgb'][i][0].cpu())
image_to_save.save(name)
count += 1
def execute(gpu,
exp_batch,
exp_alias,
suppress_output=True,
number_of_workers=12):
"""
The main training function. This functions loads the latest checkpoint
for a given, exp_batch (folder) and exp_alias (experiment configuration).
With this checkpoint it starts from the beginning or continue some training.
Args:
gpu: gpus ids for training
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(gpu)
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(os.path.join('configs', exp_batch,
exp_alias + '.yaml'))
set_type_of_process('train')
# Set the process into loading status.
coil_logger.add_message('Loading', {'GPU': gpu})
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
if coil_logger.check_finish('train'):
coil_logger.add_message('Finished', {})
return
# Preload option
if g_conf.PRELOAD_MODEL_ALIAS is not None:
checkpoint = torch.load(
os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS, 'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
print('iteration: ', iteration, 'best_loss: ', best_loss)
else:
iteration = 0
best_loss = 10000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
# Instantiate the class used to read the dataset
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_name=str(g_conf.NUMBER_OF_HOURS) +
'hours_' + g_conf.TRAIN_DATASET_NAME)
print("Loaded dataset")
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# define the sampling strategy for mini-batch, different samplers can be found in 'splitter.py'
data_loader = select_balancing_strategy(dataset, iteration,
number_of_workers)
# Instatiate the network architecture
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE
) # adabound and adamio can also be used here
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else:
# We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
# freeze the perception module weights if required
# for m in model.perception.parameters():
# m.requires_grad = False
# total trainable parameters
model_parameters = filter(lambda p: p.requires_grad,
model.parameters())
total_params = sum([np.prod(p.size()) for p in model_parameters])
print('trainable parameters: ', total_params)
# multi-gpu
print('number of gpus: ', torch.cuda.device_count())
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
criterion = Loss(g_conf.LOSS_FUNCTION)
print('Start Training')
st = time.time()
for data in data_loader:
# use this for early stopping if the validation loss is not coming down
if g_conf.FINISH_ON_VALIDATION_STALE is not None and \
check_loss_validation_stopped(iteration, g_conf.FINISH_ON_VALIDATION_STALE):
break
"""
####################################
Main optimization loop
####################################
"""
iteration += 1
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
# additional learning rate scheduler - cyclic cosine annealing (https://arxiv.org/pdf/1704.00109.pdf)
# adjust_learning_rate_cosine_annealing(optimizer, loss_window, iteration)
capture_time = time.time()
controls = data['directions']
model.zero_grad()
branches = model(torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda())
loss_function_params = {
'branches': branches,
'targets': dataset.extract_targets(data).cuda(),
'controls': controls.cuda(),
'inputs': dataset.extract_inputs(data).cuda(),
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
loss, _ = criterion(loss_function_params)
loss.backward()
optimizer.step()
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
if torch.cuda.device_count() > 1:
state_dict_save = model.module.state_dict()
else:
state_dict_save = model.state_dict()
state = {
'iteration': iteration,
'state_dict': state_dict_save,
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
state,
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(iteration) + '.pth'))
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']),
iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
# Log a random position
position = random.randint(0, len(data) - 1)
if torch.cuda.device_count() > 1:
output = model.module.extract_branch(
torch.stack(branches[0:4]), controls)
else:
output = model.extract_branch(torch.stack(branches[0:4]),
controls)
error = torch.abs(output - dataset.extract_targets(data).cuda())
accumulated_time += time.time() - capture_time
coil_logger.add_message(
'Iterating', {
'Iteration':
iteration,
'Loss':
loss.data.tolist(),
'Images/s':
(iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss':
best_loss,
'BestLossIteration':
best_loss_iter,
'Output':
output[position].data.tolist(),
'GroundTruth':
dataset.extract_targets(data)[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(data)[position].data.tolist()
}, iteration)
loss_window.append(loss.data.tolist())
coil_logger.write_on_error_csv('train', loss.data)
print("Iteration: %d Loss: %f" % (iteration, loss.data))
st = time.time()
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu,
exp_batch,
exp_alias,
json_file_path,
suppress_output,
encoder_params=None,
plot_attentions=False):
try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
if json_file_path is not None:
json_file_name = json_file_path.split('/')[-1].split('.')[-2]
else:
raise RuntimeError(
"You need to define the validation json file path")
# At this point the log file with the correct naming is created.
merge_with_yaml(
os.path.join('configs', exp_batch, exp_alias + '.yaml'),
encoder_params)
if plot_attentions:
set_type_of_process('validation',
json_file_name + '_plotAttention')
else:
set_type_of_process('validation', json_file_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
# We create file for saving validation results
summary_file = os.path.join('_logs', exp_batch, g_conf.EXPERIMENT_NAME,
g_conf.PROCESS_NAME + '_csv',
'valid_summary_1camera.csv')
g_conf.immutable(False)
g_conf.DATA_USED = 'central'
g_conf.immutable(True)
if not os.path.exists(summary_file):
csv_outfile = open(summary_file, 'w')
csv_outfile.write(
"%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n" %
('step', 'accumulated_pedestrian_TP',
'accumulated_pedestrian_FP', 'accumulated_pedestrian_FN',
'accumulated_pedestrian_TN', 'accumulated_vehicle_stop_TP',
'accumulated_vehicle_stop_FP', 'accumulated_vehicle_stop_FN',
'accumulated_vehicle_stop_TN', 'accumulated_red_tl_TP',
'accumulated_red_tl_FP', 'accumulated_red_tl_FN',
'accumulated_red_tl_TN', 'MAE_relative_angle'))
csv_outfile.close()
latest = get_latest_evaluated_checkpoint_2(summary_file)
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
#full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name)
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
dataset = CoILDataset(transform=augmenter,
preload_name=g_conf.PROCESS_NAME + '_' +
g_conf.DATA_USED,
process_type='validation',
vd_json_file_path=json_file_path)
print("Loaded Validation dataset")
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
if g_conf.MODEL_TYPE in ['one-step-affordances']:
# one step training, no need to retrain FC layers, we just get the output of encoder model as prediciton
model = EncoderModel(g_conf.ENCODER_MODEL_TYPE,
g_conf.ENCODER_MODEL_CONFIGURATION)
model.cuda()
#print(model)
elif g_conf.MODEL_TYPE in ['separate-affordances']:
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION,
g_conf.ENCODER_MODEL_CONFIGURATION)
model.cuda()
#print(model)
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE,
g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.eval()
# Here we load the pre-trained encoder (not fine-tunned)
if g_conf.FREEZE_ENCODER:
if encoder_params is not None:
encoder_checkpoint = torch.load(
os.path.join(
'_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'], 'checkpoints',
str(encoder_params['encoder_checkpoint']) +
'.pth'))
print(
"Encoder model ",
str(encoder_params['encoder_checkpoint']),
"loaded from ",
os.path.join('_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'],
'checkpoints'))
encoder_model.load_state_dict(
encoder_checkpoint['state_dict'])
encoder_model.eval()
for param_ in encoder_model.parameters():
param_.requires_grad = False
while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE):
latest = get_next_checkpoint_2(g_conf.TEST_SCHEDULE, summary_file)
if os.path.exists(
os.path.join('_logs', exp_batch, g_conf.EXPERIMENT_NAME,
'checkpoints',
str(latest) + '.pth')):
checkpoint = torch.load(
os.path.join('_logs', exp_batch, g_conf.EXPERIMENT_NAME,
'checkpoints',
str(latest) + '.pth'))
checkpoint_iteration = checkpoint['iteration']
model.load_state_dict(checkpoint['state_dict'])
print("Validation checkpoint ", checkpoint_iteration)
model.eval()
for param_ in model.parameters():
param_.requires_grad = False
# Here we load the fine-tunned encoder
if not g_conf.FREEZE_ENCODER and g_conf.MODEL_TYPE not in [
'one-step-affordances'
]:
encoder_checkpoint = torch.load(
os.path.join('_logs', exp_batch,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(latest) + '_encoder.pth'))
print(
"FINE TUNNED encoder model ",
str(latest) + '_encoder.pth', "loaded from ",
os.path.join('_logs', exp_batch,
g_conf.EXPERIMENT_NAME, 'checkpoints'))
encoder_model.load_state_dict(
encoder_checkpoint['state_dict'])
encoder_model.eval()
for param_ in encoder_model.parameters():
param_.requires_grad = False
accumulated_mae_ra = 0
accumulated_pedestrian_TP = 0
accumulated_pedestrian_TN = 0
accumulated_pedestrian_FN = 0
accumulated_pedestrian_FP = 0
accumulated_red_tl_TP = 0
accumulated_red_tl_TN = 0
accumulated_red_tl_FP = 0
accumulated_red_tl_FN = 0
accumulated_vehicle_stop_TP = 0
accumulated_vehicle_stop_TN = 0
accumulated_vehicle_stop_FP = 0
accumulated_vehicle_stop_FN = 0
iteration_on_checkpoint = 0
for data in data_loader:
if g_conf.MODEL_TYPE in ['one-step-affordances']:
c_output, r_output, layers = model.forward_outputs(
torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
dataset.extract_commands(data).cuda())
elif g_conf.MODEL_TYPE in ['separate-affordances']:
if g_conf.ENCODER_MODEL_TYPE in [
'action_prediction', 'stdim', 'ETEDIM',
'FIMBC', 'one-step-affordances'
]:
e, layers = encoder_model.forward_encoder(
torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(
dataset.extract_commands(data).cuda()))
c_output, r_output = model.forward_test(e)
elif g_conf.ENCODER_MODEL_TYPE in [
'ETE', 'ETE_inverse_model', 'forward',
'ETE_stdim'
]:
e, layers = encoder_model.forward_encoder(
torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(
dataset.extract_commands(data).cuda()))
c_output, r_output = model.forward_test(e)
if plot_attentions:
attentions_path = os.path.join(
'_logs', exp_batch, g_conf.EXPERIMENT_NAME,
g_conf.PROCESS_NAME + '_attentions_' + str(latest))
write_attentions(torch.squeeze(data['rgb']), layers,
iteration_on_checkpoint,
attentions_path)
# Accurancy = (TP+TN)/(TP+TN+FP+FN)
# F1-score = 2*TP / (2*TP + FN + FP)
classification_gt = dataset.extract_affordances_targets(
data, 'classification')
regression_gt = dataset.extract_affordances_targets(
data, 'regression')
TP = 0
FN = 0
FP = 0
TN = 0
for i in range(classification_gt.shape[0]):
if classification_gt[i, 0] == (
c_output[0][i, 0] < c_output[0][i, 1]).type(
torch.FloatTensor) == 1:
TP += 1
elif classification_gt[
i, 0] == 1 and classification_gt[i, 0] != (
c_output[0][i, 0] <
c_output[0][i, 1]).type(torch.FloatTensor):
FN += 1
elif classification_gt[
i, 0] == 0 and classification_gt[i, 0] != (
c_output[0][i, 0] <
c_output[0][i, 1]).type(torch.FloatTensor):
FP += 1
if classification_gt[i, 0] == (
c_output[0][i, 0] < c_output[0][i, 1]).type(
torch.FloatTensor) == 0:
TN += 1
accumulated_pedestrian_TP += TP
accumulated_pedestrian_TN += TN
accumulated_pedestrian_FP += FP
accumulated_pedestrian_FN += FN
TP = 0
FN = 0
FP = 0
TN = 0
for i in range(classification_gt.shape[0]):
if classification_gt[i, 1] == (
c_output[1][i, 0] < c_output[1][i, 1]).type(
torch.FloatTensor) == 1:
TP += 1
elif classification_gt[
i, 1] == 1 and classification_gt[i, 1] != (
c_output[1][i, 0] <
c_output[1][i, 1]).type(torch.FloatTensor):
FN += 1
elif classification_gt[
i, 1] == 0 and classification_gt[i, 1] != (
c_output[1][i, 0] <
c_output[1][i, 1]).type(torch.FloatTensor):
FP += 1
if classification_gt[i, 1] == (
c_output[1][i, 0] < c_output[1][i, 1]).type(
torch.FloatTensor) == 0:
TN += 1
accumulated_red_tl_TP += TP
accumulated_red_tl_TN += TN
accumulated_red_tl_FP += FP
accumulated_red_tl_FN += FN
TP = 0
FN = 0
FP = 0
TN = 0
for i in range(classification_gt.shape[0]):
if classification_gt[i, 2] == (
c_output[2][i, 0] < c_output[2][i, 1]).type(
torch.FloatTensor) == 1:
TP += 1
elif classification_gt[i, 2] == 1 and classification_gt[i, 2] !=\
(c_output[2][i, 0] < c_output[2][i, 1]).type(torch.FloatTensor):
FN += 1
elif classification_gt[i, 2] == 0 and classification_gt[i, 2] !=\
(c_output[2][i, 0] < c_output[2][i, 1]).type(torch.FloatTensor):
FP += 1
if classification_gt[i, 2] == (
c_output[2][i, 0] < c_output[2][i, 1]).type(
torch.FloatTensor) == 0:
TN += 1
accumulated_vehicle_stop_TP += TP
accumulated_vehicle_stop_TN += TN
accumulated_vehicle_stop_FP += FP
accumulated_vehicle_stop_FN += FN
# if the data was normalized during training, we need to transform it to its unit
write_regular_output(checkpoint_iteration,
torch.squeeze(r_output[0]),
regression_gt[:, 0])
mae_ra = torch.abs(regression_gt[:, 0] -
torch.squeeze(r_output[0]).type(torch.FloatTensor)).\
numpy()
accumulated_mae_ra += np.sum(mae_ra)
if iteration_on_checkpoint % 100 == 0:
print(
"Validation iteration: %d [%d/%d)] on Checkpoint %d "
%
(iteration_on_checkpoint, iteration_on_checkpoint,
len(data_loader), checkpoint_iteration))
iteration_on_checkpoint += 1
# Here also need a better analysis. TODO divide into curve and other things
MAE_relative_angle = accumulated_mae_ra / (len(dataset))
csv_outfile = open(summary_file, 'a')
csv_outfile.write(
"%s, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f" %
(checkpoint_iteration, accumulated_pedestrian_TP,
accumulated_pedestrian_FP, accumulated_pedestrian_FN,
accumulated_pedestrian_TN, accumulated_vehicle_stop_TP,
accumulated_vehicle_stop_FP, accumulated_vehicle_stop_FN,
accumulated_vehicle_stop_TN, accumulated_red_tl_TP,
accumulated_red_tl_FP, accumulated_red_tl_FN,
accumulated_red_tl_TN, MAE_relative_angle))
csv_outfile.write("\n")
csv_outfile.close()
else:
print('The checkpoint you want to validate is not yet ready ',
str(latest))
coil_logger.add_message('Finished', {})
print('VALIDATION FINISHED !!')
print(' Validation results saved in ==> ', summary_file)
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
except RuntimeError as e:
if latest is not None:
coil_logger.erase_csv(latest)
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
def execute(gpu, exp_batch, exp_alias, dataset_name, validation_set=False):
latest = None
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.immutable(False)
# At this point the log file with the correct naming is created.
merge_with_yaml(os.path.join('configs', exp_batch, exp_alias + '.yaml'))
# If using validation dataset, fix a very high number of hours
if validation_set:
g_conf.NUMBER_OF_HOURS = 10000
g_conf.immutable(True)
# Define the dataset.
full_dataset = [
os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name)
]
augmenter = Augmenter(None)
if validation_set:
# Definition of the dataset to be used. Preload name is just the validation data name
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_names=[dataset_name])
else:
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_names=[
str(g_conf.NUMBER_OF_HOURS) + 'hours_' +
dataset_name
],
train_dataset=True)
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
# Define model
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
"""
######
Run a single driving benchmark specified by the checkpoint were validation is stale
######
"""
if g_conf.FINISH_ON_VALIDATION_STALE is not None:
while validation_stale_point(
g_conf.FINISH_ON_VALIDATION_STALE) is None:
time.sleep(0.1)
validation_state_iteration = validation_stale_point(
g_conf.FINISH_ON_VALIDATION_STALE)
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(validation_state_iteration) + '.pth'))
print("Validation loaded ", validation_state_iteration)
else:
"""
#####
Main Loop , Run a benchmark for each specified checkpoint on the "Test Configuration"
#####
"""
while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE):
# Get the correct checkpoint
# We check it for some task name, all of then are ready at the same time
if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE,
control_filename + '_' + task_list[0]):
latest = get_next_checkpoint(
g_conf.TEST_SCHEDULE,
control_filename + '_' + task_list[0])
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(latest) + '.pth'))
print("Validation loaded ", latest)
else:
time.sleep(0.1)
# Load the model and prepare set it for evaluation
model.load_state_dict(checkpoint['state_dict'])
model.cuda()
model.eval()
first_iter = True
for data in data_loader:
# Compute the forward pass on a batch from the dataset and get the intermediate
# representations of the squeeze network
if "seg" in g_conf.SENSORS.keys():
perception_rep, speed_rep, intentions_rep = \
model.get_intermediate_representations(data,
dataset.extract_inputs(data).cuda(),
dataset.extract_intentions(data).cuda())
perception_rep = perception_rep.data.cpu()
speed_rep = speed_rep.data.cpu()
intentions_rep = intentions_rep.data.cpu()
if first_iter:
perception_rep_all = perception_rep
speed_rep_all = speed_rep
intentions_rep_all = intentions_rep
else:
perception_rep_all = torch.cat(
[perception_rep_all, perception_rep], 0)
speed_rep_all = torch.cat([speed_rep_all, speed_rep], 0)
intentions_rep_all = torch.cat(
[intentions_rep_all, intentions_rep], 0)
first_iter = False
# Save intermediate representations
perception_rep_all = perception_rep_all.tolist()
speed_rep_all = speed_rep_all.tolist()
intentions_rep_all = intentions_rep_all.tolist()
np.save(
os.path.join(
'_preloads', exp_batch + '_' + exp_alias + '_' + dataset_name +
'_representations'),
[perception_rep_all, speed_rep_all, intentions_rep_all])
def execute(gpu,
exp_batch,
exp_alias,
state_dict,
suppress_output=True,
number_of_workers=12):
"""
The main training function. This functions loads the latest checkpoint
for a given, exp_batch (folder) and exp_alias (experiment configuration).
With this checkpoint it starts from the beginning or continue some training.
Args:
gpu: The GPU number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(os.path.join('configs', exp_batch,
exp_alias + '.yaml'))
set_type_of_process('train')
# Set the process into loading status.
coil_logger.add_message('Loading', {'GPU': gpu})
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
if coil_logger.check_finish('train'):
coil_logger.add_message('Finished', {})
return
# Preload option
if g_conf.PRELOAD_MODEL_ALIAS is not None:
checkpoint = torch.load(
os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS, 'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 10000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
# Instantiate the class used to read a dataset. The coil dataset generator
# can be found
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_name=str(g_conf.NUMBER_OF_HOURS) +
'hours_' + g_conf.TRAIN_DATASET_NAME)
print("Loaded dataset")
data_loader = select_balancing_strategy(dataset, iteration,
number_of_workers)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
model.cuda()
if state_dict != '':
seg_model = ERFNet_Fast(2)
seg_model = load_my_state_dict(seg_model, torch.load(state_dict))
seg_model.cuda()
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE)
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
print("Before the loss")
criterion = Loss(g_conf.LOSS_FUNCTION)
color_transforms = Colorizes(2)
board = Dashboard(8097)
# Loss time series window
for data in data_loader:
# Basically in this mode of execution, we validate every X Steps, if it goes up 3 times,
# add a stop on the _logs folder that is going to be read by this process
if g_conf.FINISH_ON_VALIDATION_STALE is not None and \
check_loss_validation_stopped(iteration, g_conf.FINISH_ON_VALIDATION_STALE):
break
"""
####################################
Main optimization loop
####################################
"""
iteration += 1
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
# get the control commands from float_data, size = [120,1]
capture_time = time.time()
controls = data['directions']
# The output(branches) is a list of 5 branches results, each branch is with size [120,3]
model.zero_grad()
if state_dict != '':
with torch.no_grad():
repre = seg_model(torch.squeeze(data['rgb'].cuda()),
only_encode=False)
inputs = repre
imgs = color_transforms(inputs)
inputs = inputs.float().cuda()
else:
inputs = torch.squeeze(data['rgb'].cuda())
# vis
board.image(
torch.squeeze(data['rgb'])[0].cpu().data,
'(train) input iter: ' + str(iteration))
board.image(imgs[0].cpu().data,
'(train) output iter: ' + str(iteration))
branches = model(inputs, dataset.extract_inputs(data).cuda())
loss_function_params = {
'branches': branches,
'targets': dataset.extract_targets(data).cuda(),
'controls': controls.cuda(),
'inputs': dataset.extract_inputs(data).cuda(),
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
loss, _ = criterion(loss_function_params)
loss.backward()
optimizer.step()
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
state,
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(iteration) + '.pth'))
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']),
iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
# Log a random position
position = random.randint(0, len(data) - 1)
output = model.extract_branch(torch.stack(branches[0:4]), controls)
error = torch.abs(output - dataset.extract_targets(data).cuda())
accumulated_time += time.time() - capture_time
coil_logger.add_message(
'Iterating', {
'Iteration':
iteration,
'Loss':
loss.data.tolist(),
'Images/s':
(iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss':
best_loss,
'BestLossIteration':
best_loss_iter,
'Output':
output[position].data.tolist(),
'GroundTruth':
dataset.extract_targets(data)[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(data)[position].data.tolist()
}, iteration)
loss_window.append(loss.data.tolist())
coil_logger.write_on_error_csv('train', loss.data)
print("Iteration: %d Loss: %f" % (iteration, loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu, exp_batch, exp_alias, dataset_name, suppress_output=True, yaml_file=None):
latest = None
# try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
path_to_yaml_file = os.path.join('configs', exp_batch, exp_alias+'.yaml')
if yaml_file is not None:
path_to_yaml_file = os.path.join(yaml_file, exp_alias+'.yaml')
merge_with_yaml(path_to_yaml_file)
# The validation dataset is always fully loaded, so we fix a very high number of hours
# g_conf.NUMBER_OF_HOURS = 10000 # removed to simplify code
"""
# commented out to simplify the code
set_type_of_process('validation', dataset_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join('_output_logs',
exp_alias + '_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
"""
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name)
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
print ('full dataset path: ', full_dataset)
dataset = CoILDataset(full_dataset, transform=augmenter,
preload_name=dataset_name)
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(dataset, batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
""" removing this segment to simplify code
# The window used to keep track of the trainings
l1_window = []
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(dataset_name, None)
"""
model.cuda()
best_mse = 1000
best_error = 1000
best_mse_iter = 0
best_error_iter = 0
# modified validation code from here to run a single model
# checkpoint = torch.load(os.path.join(g_conf.VALIDATION_CHECKPOINT_PATH
# , 'checkpoints', g_conf.VALIDATION_CHECKPOINT_ITERATION + '.pth'))
checkpoint = torch.load(args.checkpoint)
checkpoint_iteration = checkpoint['iteration']
print("model loaded ", checkpoint_iteration)
model.load_state_dict(checkpoint['state_dict'])
model.eval()
accumulated_mse = 0
accumulated_error = 0
iteration_on_checkpoint = 0
# considering steer, throttle & brake so 3x3 matrix
normalized_covariate_shift = torch.zeros(3,3)
print ('data_loader size: ', len(data_loader))
total_output = []
path_names = []
for data in data_loader:
# Compute the forward pass on a batch from the validation dataset
path_names += data[1]
controls = data[0]['directions']
# get prefinal branch activations, only the last layers have dropout
output = model.get_prefinal_layer(torch.squeeze(data[0]['rgb']).cuda(),
dataset.extract_inputs(data[0]).cuda(),
controls)
total_output += output.detach().cpu().tolist()
iteration_on_checkpoint += 1
if iteration_on_checkpoint % 50 == 0:
print ('iter: ', iteration_on_checkpoint)
print (len(total_output), len(path_names))
i = 0
st = time.time()
for act, name in zip(total_output, path_names):
episode_num = name.split('/')[-2]
frame_num = name.split('/')[-1].split('_')[-1].split('.')[0]
if not os.path.isdir(os.path.join(args.save_path, args.dataset_name, episode_num)):
os.mkdir(os.path.join(args.save_path, args.dataset_name, episode_num))
file_name = 'Activation_'+frame_num
i += 1
if i%1000 == 0:
print ('iteration: ', i)
# save activations for each image, to be used for computing the uncertainity later
torch.save(act, os.path.join(args.save_path, args.dataset_name, episode_num, file_name))
print ('time taken: ', time.time()-st)
def execute(gpu, exp_batch, exp_alias, dataset_name, suppress_output):
latest = None
try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
merge_with_yaml(os.path.join('configs', exp_batch,
exp_alias + '.yaml'))
# The validation dataset is always fully loaded, so we fix a very high number of hours
g_conf.NUMBER_OF_HOURS = 10000
set_type_of_process('validation', dataset_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
# Define the dataset.
full_dataset = [
os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name)
]
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_names=[dataset_name])
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
# Create model.
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
# The window used to keep track of the validation loss
l1_window = []
# If we have evaluated a checkpoint, get the validation losses of all the previously
# evaluated checkpoints (validation loss is used for early stopping)
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(dataset_name, None)
model.cuda()
best_mse = 1000
best_error = 1000
best_mse_iter = 0
best_error_iter = 0
# Loop to validate all checkpoints as they are saved during training
while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE):
if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE):
with torch.no_grad():
# Get and load latest checkpoint
latest = get_next_checkpoint(g_conf.TEST_SCHEDULE)
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias,
'checkpoints',
str(latest) + '.pth'))
checkpoint_iteration = checkpoint['iteration']
print("Validation loaded ", checkpoint_iteration)
model.load_state_dict(checkpoint['state_dict'])
model.eval()
accumulated_mse = 0
accumulated_error = 0
iteration_on_checkpoint = 0
if g_conf.USE_REPRESENTATION_LOSS:
accumulated_perception_rep_mse = 0
accumulated_speed_rep_mse = 0
accumulated_intentions_rep_mse = 0
accumulated_rep_mse = 0
accumulated_perception_rep_error = 0
accumulated_speed_rep_error = 0
accumulated_intentions_rep_error = 0
accumulated_rep_error = 0
# Validation loop
for data in data_loader:
# Compute the forward pass on a batch from the validation dataset
controls = data['directions']
# Run model forward and get outputs
# First case corresponds to squeeze network, second case corresponds to driving model without
# mimicking losses, last case corresponds to mimic network
if "seg" in g_conf.SENSORS.keys():
output = model.forward_branch(
data,
dataset.extract_inputs(data).cuda(), controls,
dataset.extract_intentions(data).cuda())
elif not g_conf.USE_REPRESENTATION_LOSS:
output = model.forward_branch(
data,
dataset.extract_inputs(data).cuda(), controls)
else:
output, intermediate_reps = model.forward_branch(
data,
dataset.extract_inputs(data).cuda(), controls)
write_regular_output(checkpoint_iteration, output)
# Compute control loss on current validation batch and accumulate it
targets_to_use = dataset.extract_targets(data)
mse = torch.mean(
(output - targets_to_use.cuda())**2).data.tolist()
mean_error = torch.mean(
torch.abs(output -
targets_to_use.cuda())).data.tolist()
accumulated_error += mean_error
accumulated_mse += mse
error = torch.abs(output - targets_to_use.cuda())
# Compute mimicking losses on current validation batch and accumulate it
if g_conf.USE_REPRESENTATION_LOSS:
expert_reps = dataset.extract_representations(data)
# First L1 losses (seg mask, speed, intention mimicking losses)
if g_conf.USE_PERCEPTION_REP_LOSS:
perception_rep_loss = torch.sum(
torch.abs(intermediate_reps[0] -
expert_reps[0].cuda())
).data.tolist() / (3 * output.shape[0])
else:
perception_rep_loss = 0
if g_conf.USE_SPEED_REP_LOSS:
speed_rep_loss = torch.sum(
torch.abs(intermediate_reps[1] -
expert_reps[1].cuda())
).data.tolist() / (3 * output.shape[0])
else:
speed_rep_loss = 0
if g_conf.USE_INTENTION_REP_LOSS:
intentions_rep_loss = torch.sum(
torch.abs(intermediate_reps[2] -
expert_reps[2].cuda())
).data.tolist() / (3 * output.shape[0])
else:
intentions_rep_loss = 0
rep_error = g_conf.REP_LOSS_WEIGHT * (
perception_rep_loss + speed_rep_loss +
intentions_rep_loss)
accumulated_perception_rep_error += perception_rep_loss
accumulated_speed_rep_error += speed_rep_loss
accumulated_intentions_rep_error += intentions_rep_loss
accumulated_rep_error += rep_error
# L2 losses now
if g_conf.USE_PERCEPTION_REP_LOSS:
perception_rep_loss = torch.sum(
(intermediate_reps[0] -
expert_reps[0].cuda())**
2).data.tolist() / (3 * output.shape[0])
else:
perception_rep_loss = 0
if g_conf.USE_SPEED_REP_LOSS:
speed_rep_loss = torch.sum(
(intermediate_reps[1] -
expert_reps[1].cuda())**
2).data.tolist() / (3 * output.shape[0])
else:
speed_rep_loss = 0
if g_conf.USE_INTENTION_REP_LOSS:
intentions_rep_loss = torch.sum(
(intermediate_reps[2] -
expert_reps[2].cuda())**
2).data.tolist() / (3 * output.shape[0])
else:
intentions_rep_loss = 0
rep_mse = g_conf.REP_LOSS_WEIGHT * (
perception_rep_loss + speed_rep_loss +
intentions_rep_loss)
accumulated_perception_rep_mse += perception_rep_loss
accumulated_speed_rep_mse += speed_rep_loss
accumulated_intentions_rep_mse += intentions_rep_loss
accumulated_rep_mse += rep_mse
# Log a random position
position = random.randint(
0,
len(output.data.tolist()) - 1)
# Logging
if g_conf.USE_REPRESENTATION_LOSS:
total_mse = mse + rep_mse
total_error = mean_error + rep_error
coil_logger.add_message(
'Iterating', {
'Checkpoint':
latest,
'Iteration':
(str(iteration_on_checkpoint * 120) + '/' +
str(len(dataset))),
'MeanError':
mean_error,
'MSE':
mse,
'RepMeanError':
rep_error,
'RepMSE':
rep_mse,
'MeanTotalError':
total_error,
'TotalMSE':
total_mse,
'Output':
output[position].data.tolist(),
'GroundTruth':
targets_to_use[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(
data)[position].data.tolist()
}, latest)
else:
coil_logger.add_message(
'Iterating', {
'Checkpoint':
latest,
'Iteration':
(str(iteration_on_checkpoint * 120) + '/' +
str(len(dataset))),
'MeanError':
mean_error,
'MSE':
mse,
'Output':
output[position].data.tolist(),
'GroundTruth':
targets_to_use[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(
data)[position].data.tolist()
}, latest)
iteration_on_checkpoint += 1
if g_conf.USE_REPRESENTATION_LOSS:
print("Iteration %d on Checkpoint %d : Error %f" %
(iteration_on_checkpoint,
checkpoint_iteration, total_error))
else:
print("Iteration %d on Checkpoint %d : Error %f" %
(iteration_on_checkpoint,
checkpoint_iteration, mean_error))
"""
########
Finish a round of validation, write results, wait for the next
########
"""
# Compute average L1 and L2 losses over whole round of validation and log them
checkpoint_average_mse = accumulated_mse / (
len(data_loader))
checkpoint_average_error = accumulated_error / (
len(data_loader))
coil_logger.add_scalar('L2 Loss', checkpoint_average_mse,
latest, True)
coil_logger.add_scalar('Loss', checkpoint_average_error,
latest, True)
if g_conf.USE_REPRESENTATION_LOSS:
checkpoint_average_perception_rep_mse = accumulated_perception_rep_mse / (
len(data_loader))
checkpoint_average_speed_rep_mse = accumulated_speed_rep_mse / (
len(data_loader))
checkpoint_average_intentions_rep_mse = accumulated_intentions_rep_mse / (
len(data_loader))
checkpoint_average_rep_mse = accumulated_rep_mse / (
len(data_loader))
checkpoint_average_total_mse = checkpoint_average_mse + checkpoint_average_rep_mse
checkpoint_average_perception_rep_error = accumulated_perception_rep_error / (
len(data_loader))
checkpoint_average_speed_rep_error = accumulated_speed_rep_error / (
len(data_loader))
checkpoint_average_intentions_rep_error = accumulated_intentions_rep_error / (
len(data_loader))
checkpoint_average_rep_error = accumulated_rep_error / (
len(data_loader))
checkpoint_average_total_error = checkpoint_average_error + checkpoint_average_rep_mse
# Log L1/L2 loss terms
coil_logger.add_scalar(
'Perception Rep Loss',
checkpoint_average_perception_rep_mse, latest,
True)
coil_logger.add_scalar(
'Speed Rep Loss', checkpoint_average_speed_rep_mse,
latest, True)
coil_logger.add_scalar(
'Intentions Rep Loss',
checkpoint_average_intentions_rep_mse, latest,
True)
coil_logger.add_scalar('Overall Rep Loss',
checkpoint_average_rep_mse,
latest, True)
coil_logger.add_scalar('Total L2 Loss',
checkpoint_average_total_mse,
latest, True)
coil_logger.add_scalar(
'Perception Rep Error',
checkpoint_average_perception_rep_error, latest,
True)
coil_logger.add_scalar(
'Speed Rep Error',
checkpoint_average_speed_rep_error, latest, True)
coil_logger.add_scalar(
'Intentions Rep Error',
checkpoint_average_intentions_rep_error, latest,
True)
coil_logger.add_scalar('Total Rep Error',
checkpoint_average_rep_error,
latest, True)
coil_logger.add_scalar('Total Loss',
checkpoint_average_total_error,
latest, True)
else:
checkpoint_average_total_mse = checkpoint_average_mse
checkpoint_average_total_error = checkpoint_average_error
if checkpoint_average_total_mse < best_mse:
best_mse = checkpoint_average_total_mse
best_mse_iter = latest
if checkpoint_average_total_error < best_error:
best_error = checkpoint_average_total_error
best_error_iter = latest
# Print for logging / to terminal validation results
if g_conf.USE_REPRESENTATION_LOSS:
coil_logger.add_message(
'Iterating', {
'Summary': {
'Control Error': checkpoint_average_error,
'Control Loss': checkpoint_average_mse,
'Rep Error': checkpoint_average_rep_error,
'Rep Loss': checkpoint_average_rep_mse,
'Error': checkpoint_average_total_error,
'Loss': checkpoint_average_total_mse,
'BestError': best_error,
'BestMSE': best_mse,
'BestMSECheckpoint': best_mse_iter,
'BestErrorCheckpoint': best_error_iter
},
'Checkpoint': latest
}, latest)
else:
coil_logger.add_message(
'Iterating', {
'Summary': {
'Error': checkpoint_average_error,
'Loss': checkpoint_average_mse,
'BestError': best_error,
'BestMSE': best_mse,
'BestMSECheckpoint': best_mse_iter,
'BestErrorCheckpoint': best_error_iter
},
'Checkpoint': latest
}, latest)
# Save validation loss history (validation loss is used for early stopping)
l1_window.append(checkpoint_average_total_error)
coil_logger.write_on_error_csv(
dataset_name, checkpoint_average_total_error)
# Early stopping
if g_conf.FINISH_ON_VALIDATION_STALE is not None:
if dlib.count_steps_without_decrease(l1_window) > 3 and \
dlib.count_steps_without_decrease_robust(l1_window) > 3:
coil_logger.write_stop(dataset_name, latest)
break
else:
latest = get_latest_evaluated_checkpoint()
time.sleep(1)
coil_logger.add_message('Loading',
{'Message': 'Waiting Checkpoint'})
print("Waiting for the next Validation")
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
except RuntimeError as e:
if latest is not None:
coil_logger.erase_csv(latest)
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
def execute(gpu,
exp_batch,
exp_alias,
dataset_name,
suppress_output=True,
yaml_file=None):
latest = None
# try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
path_to_yaml_file = os.path.join('configs', exp_batch, exp_alias + '.yaml')
if yaml_file is not None:
path_to_yaml_file = os.path.join(yaml_file, exp_alias + '.yaml')
merge_with_yaml(path_to_yaml_file)
# The validation dataset is always fully loaded, so we fix a very high number of hours
# g_conf.NUMBER_OF_HOURS = 10000 # removed to simplify code
"""
# commenting out this segment to simplify code
set_type_of_process('validation', dataset_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join('_output_logs',
exp_alias + '_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
"""
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
dataset_name) # original code
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
print('full dataset path: ', full_dataset)
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_name=dataset_name)
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
""" removing this segment to simplify code
# The window used to keep track of the trainings
l1_window = []
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(dataset_name, None)
"""
model.cuda()
best_mse = 1000
best_error = 1000
best_mse_iter = 0
best_error_iter = 0
# modified validation code from here to run a single model
# checkpoint = torch.load(os.path.join(g_conf.VALIDATION_CHECKPOINT_PATH
# , 'checkpoints', g_conf.VALIDATION_CHECKPOINT_ITERATION + '.pth'))
checkpoint = torch.load(args.checkpoint)
checkpoint_iteration = checkpoint['iteration']
print("model loaded ", checkpoint_iteration)
model.load_state_dict(checkpoint['state_dict'])
model.train()
accumulated_mse = 0
accumulated_error = 0
iteration_on_checkpoint = 0
print('data_loader size: ', len(data_loader))
total_var = []
for data in data_loader:
# dataloader directly loads the saved activations
# Compute the forward pass on a batch from the validation dataset
controls = data['directions']
curr_var = []
for i in range(100):
output = model.branches(data['activation'].cuda())
output_vec = model.extract_branch(torch.stack(output), controls)
curr_var.append(output_vec.detach().cpu().numpy())
curr_var = np.array(curr_var)
compute_var = np.var(curr_var, axis=0)
total_var += compute_var.tolist()
iteration_on_checkpoint += 1
if iteration_on_checkpoint % 50 == 0:
print('iteration: ', iteration_on_checkpoint)
total_var = np.array(total_var)
print(len(total_var), total_var.shape)
# save the computed variance array, this would be used for uncertainty based sampling in 'tools/filter_dagger_data_var.py'
np.save(
os.path.join(args.save_path, args.dataset_name, 'computed_var.npy'),
total_var)
def execute(gpu,
exp_batch,
exp_alias,
suppress_output=True,
number_of_workers=12):
"""
The main training function. This functions loads the latest checkpoint
for a given, exp_batch (folder) and exp_alias (experiment configuration).
With this checkpoint it starts from the beginning or continue some training.
Args:
gpu: The GPU number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(os.path.join('configs', exp_batch,
exp_alias + '.yaml'))
set_type_of_process('train')
# Set the process into loading status.
coil_logger.add_message('Loading', {'GPU': gpu})
# Seed RNGs
torch.manual_seed(g_conf.MAGICAL_SEED)
random.seed(g_conf.MAGICAL_SEED)
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
if coil_logger.check_finish('train'):
coil_logger.add_message('Finished', {})
return
# Preload option
if g_conf.PRELOAD_MODEL_ALIAS is not None:
checkpoint = torch.load(
os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS, 'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 10000.0
best_loss_iter = 0
# Define the dataset.
# Can specify a list of training datasets or just a single training dataset
if len(g_conf.TRAIN_DATASET_NAMES) == 0:
train_dataset_list = [g_conf.TRAIN_DATASET_NAME]
else:
train_dataset_list = g_conf.TRAIN_DATASET_NAMES
full_dataset = [
os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name)
for dataset_name in train_dataset_list
]
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
# Instantiate the class used to read a dataset. The coil dataset generator
# can be found
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_names=[
str(g_conf.NUMBER_OF_HOURS) + 'hours_' +
dataset_name
for dataset_name in train_dataset_list
],
train_dataset=True)
print("Loaded dataset")
# Create dataloader, model, and optimizer
data_loader = select_balancing_strategy(dataset, iteration,
number_of_workers)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE)
# If we have a previous checkpoint, load model, optimizer, and record of previous
# train loss values (used for the learning rate schedule)
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
print("Before the loss")
# Define control loss function
criterion = Loss(g_conf.LOSS_FUNCTION)
if iteration == 0 and is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
state,
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(iteration) + '.pth'))
# Training loop
for data in data_loader:
# Basically in this mode of execution, we validate every X Steps, if it goes up 3 times,
# add a stop on the _logs folder that is going to be read by this process
if g_conf.FINISH_ON_VALIDATION_STALE is not None and \
check_loss_validation_stopped(iteration, g_conf.FINISH_ON_VALIDATION_STALE):
break
"""
####################################
Main optimization loop
####################################
"""
iteration += 1
# Adjust learning rate based on training loss
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
capture_time = time.time()
model.zero_grad()
controls = data['directions']
# Run model forward and get outputs
# First case corresponds to training squeeze network, second case corresponds to training driving model without
# mimicking losses, last case corresponds to training mimic network
if "seg" in g_conf.SENSORS.keys():
branches = model(data,
dataset.extract_inputs(data).cuda(),
dataset.extract_intentions(data).cuda())
elif not g_conf.USE_REPRESENTATION_LOSS:
branches = model(data, dataset.extract_inputs(data).cuda())
else:
branches, intermediate_reps = model(
data,
dataset.extract_inputs(data).cuda())
# Compute control loss
targets_to_use = dataset.extract_targets(data)
loss_function_params = {
'branches': branches,
'targets': targets_to_use.cuda(),
'controls': controls.cuda(),
'inputs': dataset.extract_inputs(data).cuda(),
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
loss, _ = criterion(loss_function_params)
# Compute mimicking loss
if g_conf.USE_REPRESENTATION_LOSS:
expert_reps = dataset.extract_representations(data)
# Seg mask mimicking loss
if g_conf.USE_PERCEPTION_REP_LOSS:
perception_rep_loss_elementwise = (
intermediate_reps[0] - expert_reps[0].cuda())**2
perception_rep_loss = g_conf.PERCEPTION_REP_WEIGHT * torch.sum(
perception_rep_loss_elementwise) / branches[0].shape[0]
else:
perception_rep_loss = torch.tensor(0.).cuda()
# Speed mimicking loss
if g_conf.USE_SPEED_REP_LOSS:
speed_rep_loss_elementwise = (intermediate_reps[1] -
expert_reps[1].cuda())**2
speed_rep_loss = g_conf.SPEED_REP_WEIGHT * torch.sum(
speed_rep_loss_elementwise) / branches[0].shape[0]
else:
speed_rep_loss = torch.tensor(0.).cuda()
# Stop intentions mimicking loss
if g_conf.USE_INTENTION_REP_LOSS:
intentions_rep_loss_elementwise = (
intermediate_reps[2] - expert_reps[2].cuda())**2
intentions_rep_loss = g_conf.INTENTIONS_REP_WEIGHT * torch.sum(
intentions_rep_loss_elementwise) / branches[0].shape[0]
else:
intentions_rep_loss = torch.tensor(0.).cuda()
rep_loss = g_conf.REP_LOSS_WEIGHT * (
perception_rep_loss + speed_rep_loss + intentions_rep_loss)
overall_loss = loss + rep_loss
else:
overall_loss = loss
overall_loss.backward()
optimizer.step()
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
state,
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(iteration) + '.pth'))
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
coil_logger.add_scalar('Loss', loss.data, iteration)
if g_conf.USE_REPRESENTATION_LOSS:
coil_logger.add_scalar('Perception Rep Loss',
perception_rep_loss.data, iteration)
coil_logger.add_scalar('Speed Rep Loss', speed_rep_loss.data,
iteration)
coil_logger.add_scalar('Intentions Rep Loss',
intentions_rep_loss.data, iteration)
coil_logger.add_scalar('Overall Rep Loss', rep_loss.data,
iteration)
coil_logger.add_scalar('Total Loss', overall_loss.data,
iteration)
if 'rgb' in data:
coil_logger.add_image('Image', torch.squeeze(data['rgb']),
iteration)
if overall_loss.data < best_loss:
best_loss = overall_loss.data.tolist()
best_loss_iter = iteration
# Log a random position
position = random.randint(0, len(data) - 1)
output = model.extract_branch(torch.stack(branches[0:4]), controls)
error = torch.abs(output - targets_to_use.cuda())
accumulated_time += time.time() - capture_time
# Log to terminal and log file
if g_conf.USE_REPRESENTATION_LOSS:
coil_logger.add_message(
'Iterating', {
'Iteration':
iteration,
'Loss':
overall_loss.data.tolist(),
'Control Loss':
loss.data.tolist(),
'Rep Loss':
rep_loss.data.tolist(),
'Images/s':
(iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss':
best_loss,
'BestLossIteration':
best_loss_iter,
'Output':
output[position].data.tolist(),
'GroundTruth':
targets_to_use[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(data)[position].data.tolist()
}, iteration)
else:
coil_logger.add_message(
'Iterating', {
'Iteration':
iteration,
'Loss':
loss.data.tolist(),
'Images/s':
(iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss':
best_loss,
'BestLossIteration':
best_loss_iter,
'Output':
output[position].data.tolist(),
'GroundTruth':
targets_to_use[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(data)[position].data.tolist()
}, iteration)
# Save training loss history (useful for restoring training runs since learning rate is adjusted
# based on training loss)
loss_window.append(overall_loss.data.tolist())
coil_logger.write_on_error_csv('train', overall_loss.data)
print("Iteration: %d Loss: %f" % (iteration, overall_loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu, exp_batch, exp_alias, dataset_name, architecture,
suppress_output):
try:
# We set the visible cuda devices
torch.manual_seed(2)
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# Validation available for:
# coil_unit (UNIT + task combined)
# coil_icra (Also used for finetuned models)
# wgangp_lsd (Our architecture)
architecture_name = architecture
# At this point the log file with the correct naming is created.
if architecture_name == 'coil_unit':
pass
elif architecture_name == 'wgangp_lsd':
merge_with_yaml(
os.path.join('/home/rohitrishabh/CoilWGAN/configs', exp_batch,
exp_alias + '.yaml'))
set_type_of_process('validation', dataset_name)
elif architecture_name == 'coil_icra':
merge_with_yaml(
os.path.join(
'/home/adas/CleanedCode/CoIL_Codes/coil_20-06/configs',
exp_batch, exp_alias + '.yaml'))
set_type_of_process('validation', dataset_name)
if monitorer.get_status(exp_batch, exp_alias + '.yaml',
g_conf.PROCESS_NAME)[0] == "Finished":
# TODO: print some cool summary or not ?
return
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join(
'_output_logs',
g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
#Define the dataset. This structure is has the __get_item__ redefined in a way
#that you can access the HDFILES positions from the root directory as a in a vector.
if dataset_name != []:
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
dataset_name)
else:
full_dataset = os.environ["COIL_DATASET_PATH"]
augmenter = Augmenter(None)
dataset = CoILDataset(full_dataset, transform=augmenter)
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
# TODO: batch size an number of workers go to some configuration file
batchsize = 30
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=batchsize,
shuffle=False,
num_workers=1,
pin_memory=True)
# TODO: here there is clearly a posibility to make a cool "conditioning" system.
if architecture_name == 'coil_unit':
model_task, model_gen = CoILModel('coil_unit')
model_task, model_gen = model_task.cuda(), model_gen.cuda()
else:
model = CoILModel(architecture_name)
model.cuda()
latest = 0
# print (dataset.meta_data)
best_loss = 1000
best_error = 1000
best_loss_mini = 1000
best_loss_iter = 0
best_error_iter = 0
batch_size = 30
best_loss_ckpt = ''
if architecture_name == 'coil_unit':
ckpts = glob.glob('/home/rohitrishabh/UNIT_DA/outputs/' +
exp_alias + '/checkpoints/gen*.pt')
else:
ckpts = glob.glob(
os.path.join(
'/home/adas/CleanedCode/CoIL_Codes/coil_20-06/_logs',
exp_batch, exp_alias) + '/*.pth')
if architecture_name == 'coil_unit':
model_task.eval()
model_gen.eval()
else:
model.eval()
ckpts = sorted(ckpts)
# TODO: refactor on the getting on the checkpoint organization needed
for ckpt in ckpts:
# if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE):
# latest = get_next_checkpoint(g_conf.TEST_SCHEDULE)
# ckpt = os.path.join('/datatmp/Experiments/rohitgan/_logs', exp_batch, exp_alias
# , 'checkpoints', str(latest) + '.pth')
checkpoint = torch.load(ckpt)
print("Validation loaded ", ckpt)
if architecture_name == 'wgangp_lsd':
print(ckpt, checkpoint['best_loss_iter_F'])
model.load_state_dict(checkpoint['stateF_dict'])
model.eval()
elif architecture_name == 'coil_unit':
model_task.load_state_dict(checkpoint['task'])
model_gen.load_state_dict(checkpoint['b'])
model_task.eval()
model_gen.eval()
elif architecture_name == 'coil_icra':
model.load_state_dict(checkpoint['state_dict'])
model.eval()
accumulated_loss = 0
accumulated_error = 0
iteration_on_checkpoint = 0
datacount = 0
for data in data_loader:
input_data, float_data = data
controls = float_data[:, dataset.controls_position(), :]
camera_angle = float_data[:, 26, :]
camera_angle = camera_angle.cuda()
steer = float_data[:, 0, :]
steer = steer.cuda()
speed = float_data[:, 10, :]
speed = speed.cuda()
time_use = 1.0
car_length = 3.0
extra_factor = 2.5
threshold = 1.0
pos = camera_angle > 0.0
pos = pos.type(torch.FloatTensor)
neg = camera_angle <= 0.0
neg = neg.type(torch.FloatTensor)
pos = pos.cuda()
neg = neg.cuda()
rad_camera_angle = math.pi * (torch.abs(camera_angle)) / 180.0
val = extra_factor * (torch.atan(
(rad_camera_angle * car_length) /
(time_use * speed + 0.05))) / 3.1415
steer -= pos * torch.min(val, torch.Tensor([0.6]).cuda())
steer += neg * torch.min(val, torch.Tensor([0.6]).cuda())
steer = steer.cpu()
float_data[:, 0, :] = steer
float_data[:, 0, :][float_data[:, 0, :] > 1.0] = 1.0
float_data[:, 0, :][float_data[:, 0, :] < -1.0] = -1.0
datacount += 1
control_position = 24
speed_position = 10
if architecture_name == 'wgangp_lsd':
embed, output = model(
torch.squeeze(input_data['rgb']).cuda(),
float_data[:, speed_position, :].cuda())
loss = torch.sum(
(output[0] -
dataset.extract_targets(float_data).cuda()
)**2).data.tolist()
mean_error = torch.sum(
torch.abs(output[0] -
dataset.extract_targets(float_data).cuda())
).data.tolist()
elif architecture_name == 'coil_unit':
embed, n_b = model_gen.encode(
torch.squeeze(input_data['rgb']).cuda())
output = model_task(
embed,
Variable(float_data[:, speed_position, :]).cuda())
loss = torch.sum(
(output[0].data -
dataset.extract_targets(float_data).cuda())**2)
mean_error = torch.sum(
torch.abs(output[0].data -
dataset.extract_targets(float_data).cuda()))
elif architecture_name == 'coil_icra':
output = model.forward_branch(
torch.squeeze(input_data['rgb']).cuda(),
float_data[:, speed_position, :].cuda(),
float_data[:, control_position, :].cuda())
loss = torch.sum(
(output - dataset.extract_targets(float_data).cuda()
)**2).data.tolist()
mean_error = torch.sum(
torch.abs(output -
dataset.extract_targets(float_data).cuda())
).data.tolist()
if loss < best_loss_mini:
best_loss_mini = loss
accumulated_error += mean_error
accumulated_loss += loss
# error = torch.abs(output[0] - dataset.extract_targets(float_data).cuda())
# Log a random position
position = random.randint(0, len(float_data) - 1)
iteration_on_checkpoint += 1
print(datacount, len(data_loader), accumulated_loss)
checkpoint_average_loss = accumulated_loss / float(
datacount * batchsize)
checkpoint_average_error = accumulated_error / float(
datacount * batchsize)
if checkpoint_average_loss < best_loss:
best_loss = checkpoint_average_loss
best_loss_iter = latest
best_loss_ckpt = ckpt
if checkpoint_average_error < best_error:
best_error = checkpoint_average_error
best_error_iter = latest
print("current loss", checkpoint_average_loss)
print("best_loss", best_loss)
coil_logger.add_message(
'Iterating', {
'Summary': {
'Error': checkpoint_average_error,
'Loss': checkpoint_average_loss,
'BestError': best_error,
'BestLoss': best_loss,
'BestLossCheckpoint': best_loss_iter,
'BestErrorCheckpoint': best_error_iter
},
'Checkpoint': latest
}, latest)
latest += 2000
coil_logger.add_message('Finished', {})
print("Best Validation Loss ckpt:", best_loss_ckpt)
# TODO: DO ALL THE AMAZING LOGGING HERE, as a way to very the status in paralell.
# THIS SHOULD BE AN INTERELY PARALLEL PROCESS
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu,
exp_batch='nocrash',
exp_alias='resnet34imnet10S1',
suppress_output=True,
yaml_file=None):
latest = None
# try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
path_to_yaml_file = os.path.join('configs', exp_batch, exp_alias + '.yaml')
if yaml_file is not None:
path_to_yaml_file = os.path.join(yaml_file, exp_alias + '.yaml')
merge_with_yaml(path_to_yaml_file)
# The validation dataset is always fully loaded, so we fix a very high number of hours
# g_conf.NUMBER_OF_HOURS = 10000 # removed to simplify code
"""
# commenting this segment to simplify code, uncomment if necessary
set_type_of_process('validation', dataset_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join('_output_logs',
exp_alias + '_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
"""
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
full_dataset = os.path.join(
os.environ["COIL_DATASET_PATH"], g_conf.DART_COVMAT_DATA
) # dataset used for computing dart covariance matrix
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
print('full dataset path: ', full_dataset)
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_name=g_conf.DART_COVMAT_DATA
) # specify DART_COVMAT_DATA in the config file
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
""" removing this segment to simplify code
# The window used to keep track of the trainings
l1_window = []
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(g_conf.DART_COVMAT_DATA, None)
"""
model.cuda()
best_mse = 1000
best_error = 1000
best_mse_iter = 0
best_error_iter = 0
# modified validation code from here to run a single model checkpoint
# used for computing the covariance matrix with the DART model checkpoint
checkpoint = torch.load(
g_conf.DART_MODEL_CHECKPOINT
) # specify DART_MODEL_CHECKPOINT in the config file
checkpoint_iteration = checkpoint['iteration']
print("Validation loaded ", checkpoint_iteration)
model.load_state_dict(checkpoint['state_dict'])
model.eval()
accumulated_mse = 0
accumulated_error = 0
iteration_on_checkpoint = 0
# considering steer, throttle & brake so 3x3 matrix
normalized_covariate_shift = torch.zeros(3, 3)
print('data_loader size: ', len(data_loader))
for data in data_loader:
# Compute the forward pass on a batch from the validation dataset
controls = data['directions']
output = model.forward_branch(
torch.squeeze(data['rgb']).cuda(),
dataset.extract_inputs(data).cuda(), controls)
""" removing this segment to simplify code
# It could be either waypoints or direct control
if 'waypoint1_angle' in g_conf.TARGETS:
write_waypoints_output(checkpoint_iteration, output)
else:
write_regular_output(checkpoint_iteration, output)
"""
mse = torch.mean(
(output - dataset.extract_targets(data).cuda())**2).data.tolist()
mean_error = torch.mean(
torch.abs(output -
dataset.extract_targets(data).cuda())).data.tolist()
accumulated_error += mean_error
accumulated_mse += mse
error = torch.abs(output -
dataset.extract_targets(data).cuda()).data.cpu()
### covariate shift segment starts
error = error.unsqueeze(dim=2)
error_transpose = torch.transpose(error, 1, 2)
# compute covariate shift
covariate_shift = torch.matmul(error, error_transpose)
# expand traj length tensor to Bx3x3 (considering steer, throttle & brake)
traj_lengths = torch.stack([
torch.stack([data['current_traj_length'].squeeze(dim=1)] * 3,
dim=1)
] * 3,
dim=2)
covariate_shift = covariate_shift / traj_lengths
covariate_shift = torch.sum(covariate_shift, dim=0)
# print ('current covariate shift: ', covariate_shift.shape)
normalized_covariate_shift += covariate_shift
### covariate shift segment ends
total_episodes = data['episode_count'][-1].data
iteration_on_checkpoint += 1
if iteration_on_checkpoint % 50 == 0:
print('iteration: ', iteration_on_checkpoint)
print('total episodes: ', total_episodes)
normalized_covariate_shift = normalized_covariate_shift / total_episodes
print('normalized covariate shift: ', normalized_covariate_shift.shape,
normalized_covariate_shift)
# save the matrix to restart directly from the mat file
# np.save(os.path.join(g_conf.COVARIANCE_MATRIX_PATH, 'covariance_matrix_%s.npy'%g_conf.DART_COVMATH_DATA), normalized_covariate_shift)
return normalized_covariate_shift.numpy()
'''
def execute(gpu, exp_batch, exp_alias, dataset_name, suppress_output):
latest = None
try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
merge_with_yaml(os.path.join('configs', exp_batch,
exp_alias + '.yaml'))
# The validation dataset is always fully loaded, so we fix a very high number of hours
g_conf.NUMBER_OF_HOURS = 10000
set_type_of_process('validation', dataset_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
dataset_name)
augmenter = Augmenter(None)
# Definition of the dataset to be used. Preload name is just the validation data name
dataset = CoILDataset(full_dataset,
transform=augmenter,
preload_name=dataset_name)
# Creates the sampler, this part is responsible for managing the keys. It divides
# all keys depending on the measurements and produces a set of keys for each bach.
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
# Set ERFnet for segmentation
model_erf = ERFNet(20)
model_erf = torch.nn.DataParallel(model_erf)
model_erf = model_erf.cuda()
print("LOAD ERFNet - validate")
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model_erf = load_my_state_dict(
model_erf,
torch.load(os.path.join('trained_models/erfnet_pretrained.pth')))
model_erf.eval()
print("ERFNet and weights LOADED successfully")
# The window used to keep track of the trainings
l1_window = []
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(dataset_name, None)
model.cuda()
best_mse = 1000
best_error = 1000
best_mse_iter = 0
best_error_iter = 0
while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE):
if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE):
latest = get_next_checkpoint(g_conf.TEST_SCHEDULE)
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(latest) + '.pth'))
checkpoint_iteration = checkpoint['iteration']
print("Validation loaded ", checkpoint_iteration)
model.load_state_dict(checkpoint['state_dict'])
model.eval()
accumulated_mse = 0
accumulated_error = 0
iteration_on_checkpoint = 0
for data in data_loader:
# Compute the forward pass on a batch from the validation dataset
controls = data['directions']
# Seg batch
rgbs = data['rgb']
with torch.no_grad():
outputs = model_erf(rgbs)
labels = outputs.max(1)[1].byte().cpu().data
seg_road = (labels == 0)
seg_not_road = (labels != 0)
seg = torch.stack((seg_road, seg_not_road), 1).float()
output = model.forward_branch(
torch.squeeze(seg).cuda(),
dataset.extract_inputs(data).cuda(), controls)
# output = model.foward_branch(torch.squeeze(rgbs).cuda(),
# dataset.extract_inputs(data).cuda(),controls)
# It could be either waypoints or direct control
if 'waypoint1_angle' in g_conf.TARGETS:
write_waypoints_output(checkpoint_iteration, output)
else:
write_regular_output(checkpoint_iteration, output)
mse = torch.mean(
(output - dataset.extract_targets(data).cuda()
)**2).data.tolist()
mean_error = torch.mean(
torch.abs(output -
dataset.extract_targets(data).cuda())
).data.tolist()
accumulated_error += mean_error
accumulated_mse += mse
error = torch.abs(output -
dataset.extract_targets(data).cuda())
# Log a random position
position = random.randint(0, len(output.data.tolist()) - 1)
coil_logger.add_message(
'Iterating', {
'Checkpoint':
latest,
'Iteration': (str(iteration_on_checkpoint * 120) +
'/' + str(len(dataset))),
'MeanError':
mean_error,
'MSE':
mse,
'Output':
output[position].data.tolist(),
'GroundTruth':
dataset.extract_targets(
data)[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(data)
[position].data.tolist()
}, latest)
iteration_on_checkpoint += 1
print("Iteration %d on Checkpoint %d : Error %f" %
(iteration_on_checkpoint, checkpoint_iteration,
mean_error))
"""
########
Finish a round of validation, write results, wait for the next
########
"""
checkpoint_average_mse = accumulated_mse / (len(data_loader))
checkpoint_average_error = accumulated_error / (
len(data_loader))
coil_logger.add_scalar('Loss', checkpoint_average_mse, latest,
True)
coil_logger.add_scalar('Error', checkpoint_average_error,
latest, True)
if checkpoint_average_mse < best_mse:
best_mse = checkpoint_average_mse
best_mse_iter = latest
if checkpoint_average_error < best_error:
best_error = checkpoint_average_error
best_error_iter = latest
coil_logger.add_message(
'Iterating', {
'Summary': {
'Error': checkpoint_average_error,
'Loss': checkpoint_average_mse,
'BestError': best_error,
'BestMSE': best_mse,
'BestMSECheckpoint': best_mse_iter,
'BestErrorCheckpoint': best_error_iter
},
'Checkpoint': latest
}, latest)
l1_window.append(checkpoint_average_error)
coil_logger.write_on_error_csv(dataset_name,
checkpoint_average_error)
# If we are using the finish when validation stops, we check the current
if g_conf.FINISH_ON_VALIDATION_STALE is not None:
if dlib.count_steps_without_decrease(l1_window) > 3 and \
dlib.count_steps_without_decrease_robust(l1_window) > 3:
coil_logger.write_stop(dataset_name, latest)
break
else:
latest = get_latest_evaluated_checkpoint()
time.sleep(1)
coil_logger.add_message('Loading',
{'Message': 'Waiting Checkpoint'})
print("Waiting for the next Validation")
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
except RuntimeError as e:
if latest is not None:
coil_logger.erase_csv(latest)
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
# We erase the output that was unfinished due to some process stop.
if latest is not None:
coil_logger.erase_csv(latest)
def execute(gpu, exp_batch, exp_alias, suppress_output=True, number_of_workers=12):
"""
The main training function. This functions loads the latest checkpoint
for a given, exp_batch (folder) and exp_alias (experiment configuration).
With this checkpoint it starts from the beginning or continue some training.
Args:
gpu: The GPU number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(os.path.join('configs', exp_batch, exp_alias + '.yaml'))
set_type_of_process('train')
# Set the process into loading status.
coil_logger.add_message('Loading', {'GPU': gpu})
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join('_output_logs', exp_alias + '_' +
g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"), "a",
buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_'+g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
if coil_logger.check_finish('train'):
coil_logger.add_message('Finished', {})
return
# Preload option
if g_conf.PRELOAD_MODEL_ALIAS is not None:
checkpoint = torch.load(os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS,
'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT)+'.pth'))
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(os.path.join('_logs', exp_batch, exp_alias,
'checkpoints', str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 10000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
# Instantiate the class used to read a dataset. The coil dataset generator
# can be found
dataset = CoILDataset(full_dataset, transform=augmenter,
preload_name=str(g_conf.NUMBER_OF_HOURS)
+ 'hours_' + g_conf.TRAIN_DATASET_NAME)
print ("Loaded dataset")
data_loader = select_balancing_strategy(dataset, iteration, number_of_workers)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE)
# Set ERFnet for segmentation
model_erf = ERFNet(20)
model_erf = torch.nn.DataParallel(model_erf)
model_erf = model_erf.cuda()
print("LOAD ERFNet")
def load_my_state_dict(model, state_dict): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model_erf = load_my_state_dict(model_erf, torch.load(os.path.join('trained_models/erfnet_pretrained.pth')))
model_erf.eval()
print ("ERFNet and weights LOADED successfully")
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
print ("Before the loss")
criterion = Loss(g_conf.LOSS_FUNCTION)
# Loss time series window
for data in data_loader:
# Basically in this mode of execution, we validate every X Steps, if it goes up 3 times,
# add a stop on the _logs folder that is going to be read by this process
if g_conf.FINISH_ON_VALIDATION_STALE is not None and \
check_loss_validation_stopped(iteration, g_conf.FINISH_ON_VALIDATION_STALE):
break
"""
####################################
Main optimization loop
####################################
"""
iteration += 1
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
# get the control commands from float_data, size = [120,1]
capture_time = time.time()
controls = data['directions']
# The output(branches) is a list of 5 branches results, each branch is with size [120,3]
model.zero_grad()
# print("Segmentation")
# use ERFNet to convert RGB to Segmentation
rgbs = data['rgb']
filenames = data['rgb_name']
# # seg one by one
# seg_road = []
# seg_not_road = []
# i = 0
# for inputs in rgbs:
# inputs = inputs.unsqueeze(0)
# # print("inputs ",inputs.shape)
# with torch.no_grad():
# outputs = model_erf(inputs)
# label = outputs[0].max(0)[1].byte().cpu().data
# road = (label == 0)
# not_road = (label != 0)
# seg_road.append(road)
# seg_not_road.append(not_road)
# # # print("label ",label.shape)
# # label_color = Colorize()(label.unsqueeze(0))
# # filename = filenames[i]
# # filenameSave = "./save_color/" + filename.split("CoILTrain/")[1]
# # os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
# # label_save = ToPILImage()(label_color)
# # label_save.save(filenameSave)
# # # print (i, filenameSave)
# # i += 1
# seg_road = torch.stack(seg_road)
# seg_not_road = torch.stack(seg_not_road)
# seg = torch.stack([seg_road,seg_not_road]).transpose(0,1).float()
# # print(seg.shape)
# seg batch
with torch.no_grad():
outputs = model_erf(rgbs)
# print("outputs.shape ",outputs.shape)
labels = outputs.max(1)[1].byte().cpu().data
# print("labels.shape",labels.shape)
# print(np.unique(labels[0]))
seg_road = (labels==0)
seg_not_road = (labels!=0)
seg = torch.stack((seg_road,seg_not_road),1).float()
# save 1st batch's segmentation results
if iteration == 1:
for i in range(120):
label = seg[i,0,:,:]
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/batch_road_mask/%d.png"%(i)
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
label = labels[i,:,:]
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/batch_road/%d.png"%(i)
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
branches = model(torch.squeeze(seg).cuda(),
dataset.extract_inputs(data).cuda())
# branches = model(torch.squeeze(rgbs.cuda()),
# dataset.extract_input(data).cuda())
loss_function_params = {
'branches': branches,
'targets': dataset.extract_targets(data).cuda(),
'controls': controls.cuda(),
'inputs': dataset.extract_inputs(data).cuda(),
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
loss, _ = criterion(loss_function_params)
loss.backward()
optimizer.step()
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(state, os.path.join('_logs', exp_batch, exp_alias
, 'checkpoints', str(iteration) + '.pth'))
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']), iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
# Log a random position
position = random.randint(0, len(data) - 1)
output = model.extract_branch(torch.stack(branches[0:4]), controls)
error = torch.abs(output - dataset.extract_targets(data).cuda())
accumulated_time += time.time() - capture_time
coil_logger.add_message('Iterating',
{'Iteration': iteration,
'Loss': loss.data.tolist(),
'Images/s': (iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss': best_loss, 'BestLossIteration': best_loss_iter,
'Output': output[position].data.tolist(),
'GroundTruth': dataset.extract_targets(data)[
position].data.tolist(),
'Error': error[position].data.tolist(),
'Inputs': dataset.extract_inputs(data)[
position].data.tolist()},
iteration)
loss_window.append(loss.data.tolist())
coil_logger.write_on_error_csv('train', loss.data)
print("Iteration: %d Loss: %f" % (iteration, loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu,
exp_batch,
exp_alias,
suppress_output=True,
number_of_workers=12,
encoder_params=None):
"""
The main training function. This functions loads the latest checkpoint
for a given, exp_batch (folder) and exp_alias (experiment configuration).
With this checkpoint it starts from the beginning or continue some training.
Args:
gpu: The GPU number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(
os.path.join('configs', exp_batch, exp_alias + '.yaml'),
encoder_params)
set_type_of_process('train')
# Set the process into loading status.
coil_logger.add_message('Loading',
{'GPU': os.environ["CUDA_VISIBLE_DEVICES"]})
seed_everything(seed=g_conf.MAGICAL_SEED)
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join(
'_output_logs', exp_alias + '_' + g_conf.PROCESS_NAME + '_' +
str(os.getpid()) + ".out"),
"a",
buffering=1)
sys.stderr = open(os.path.join(
'_output_logs', exp_alias + '_err_' + g_conf.PROCESS_NAME +
'_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
if coil_logger.check_finish('train'):
coil_logger.add_message('Finished', {})
return
# Preload option
print(" GOING TO LOAD")
if g_conf.PRELOAD_MODEL_ALIAS is not None:
print(" LOADING A PRELOAD")
checkpoint = torch.load(
os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS, 'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
else:
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
print('loading previous checkpoint ', checkpoint_file)
checkpoint = torch.load(
os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 100000000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
#full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
# We can save preload dataset depends on the json file name, then no need to load dataset for each time with the same dataset
if len(g_conf.EXPERIENCE_FILE) == 1:
json_file_name = str(
g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2]
else:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split(
'/')[-1].split('.')[-2] + '_' + str(
g_conf.EXPERIENCE_FILE[1]).split('/')[-1].split('.')[-2]
dataset = CoILDataset(transform=augmenter,
preload_name=g_conf.PROCESS_NAME + '_' +
json_file_name + '_' + g_conf.DATA_USED)
#dataset = CoILDataset(transform=augmenter, preload_name=str(g_conf.NUMBER_OF_HOURS)+ 'hours_' + g_conf.TRAIN_DATASET_NAME)
print("Loaded Training dataset")
data_loader = select_balancing_strategy(dataset, iteration,
number_of_workers)
if g_conf.MODEL_TYPE in ['separate-affordances']:
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION,
g_conf.ENCODER_MODEL_CONFIGURATION)
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE)
print(model)
# we use the pre-trained encoder model to extract bottleneck Z and train the E-t-E model
if g_conf.MODEL_TYPE in ['separate-affordances']:
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE,
g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.eval()
# To freeze the pre-trained encoder model
if g_conf.FREEZE_ENCODER:
for param_ in encoder_model.parameters():
param_.requires_grad = False
if encoder_params is not None:
encoder_checkpoint = torch.load(
os.path.join(
'_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'], 'checkpoints',
str(encoder_params['encoder_checkpoint']) + '.pth'))
print(
"Encoder model ",
str(encoder_params['encoder_checkpoint']), "loaded from ",
os.path.join('_logs', encoder_params['encoder_folder'],
encoder_params['encoder_exp'], 'checkpoints'))
encoder_model.load_state_dict(encoder_checkpoint['state_dict'])
if g_conf.FREEZE_ENCODER:
encoder_model.eval()
# To freeze the pre-trained encoder model
for param_ in encoder_model.parameters():
param_.requires_grad = False
else:
optimizer = optim.Adam(list(model.parameters()) +
list(encoder_model.parameters()),
lr=g_conf.LEARNING_RATE)
for name_encoder, param_encoder in encoder_model.named_parameters(
):
if param_encoder.requires_grad:
print(' Unfrozen layers', name_encoder)
else:
print(' Frozen layers', name_encoder)
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
for name, param in model.named_parameters():
if param.requires_grad:
print(' Unfrozen layers', name)
else:
print(' Frozen layers', name)
print("Before the loss")
# Loss time series window
for data in data_loader:
# Basically in this mode of execution, we validate every X Steps, if it goes up 3 times,
# add a stop on the _logs folder that is going to be read by this process
if g_conf.FINISH_ON_VALIDATION_STALE is not None and \
check_loss_validation_stopped(iteration, g_conf.FINISH_ON_VALIDATION_STALE):
break
"""
####################################
Main optimization loop
####################################
"""
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
model.zero_grad()
if not g_conf.FREEZE_ENCODER:
encoder_model.zero_grad()
if g_conf.LABELS_SUPERVISED:
inputs_data = torch.cat(
(data['rgb'], torch.zeros(g_conf.BATCH_SIZE, 1, 88, 200)),
dim=1).cuda()
else:
inputs_data = torch.squeeze(data['rgb'].cuda())
if g_conf.MODEL_TYPE in ['separate-affordances']:
#TODO: for this two encoder models training, we haven't put speed as input to train yet
if g_conf.ENCODER_MODEL_TYPE in [
'action_prediction', 'stdim', 'forward',
'one-step-affordances'
]:
e, inter = encoder_model.forward_encoder(
inputs_data,
dataset.extract_inputs(data).cuda(),
# We also add measurements and commands
torch.squeeze(dataset.extract_commands(data).cuda()))
elif g_conf.ENCODER_MODEL_TYPE in ['ETE']:
e, inter = encoder_model.forward_encoder(
inputs_data,
dataset.extract_inputs(data).cuda(),
torch.squeeze(dataset.extract_commands(data).cuda()))
loss_function_params = {
'classification_gt':
dataset.extract_affordances_targets(
data, 'classification').cuda(),
# harzard stop, red_light....
'class_weights':
g_conf.AFFORDANCES_CLASS_WEIGHT,
'regression_gt':
dataset.extract_affordances_targets(data,
'regression').cuda(),
'variable_weights':
g_conf.AFFORDANCES_VARIABLE_WEIGHT
}
loss = model(e, loss_function_params)
loss.backward()
optimizer.step()
else:
raise RuntimeError(
'Not implement yet, this branch is only work for g_conf.MODEL_TYPE in [separate-affordances]'
)
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
state,
os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(iteration) + '.pth'))
if not g_conf.FREEZE_ENCODER:
encoder_state = {
'iteration': iteration,
'state_dict': encoder_model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(
encoder_state,
os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints',
str(iteration) + '_encoder.pth'))
iteration += 1
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']),
iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
if iteration % 100 == 0:
print('Train Iteration: {} [{}/{} ({:.0f}%)] \t Loss: {:.6f}'.
format(iteration, iteration, g_conf.NUMBER_ITERATIONS,
100. * iteration / g_conf.NUMBER_ITERATIONS,
loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu, exp_batch, exp_alias, dataset_name, suppress_output=True, yaml_file=None):
latest = None
# try:
# We set the visible cuda devices
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
path_to_yaml_file = os.path.join('configs', exp_batch, exp_alias+'.yaml')
if yaml_file is not None:
path_to_yaml_file = os.path.join(yaml_file, exp_alias+'.yaml')
merge_with_yaml(path_to_yaml_file)
# The validation dataset is always fully loaded, so we fix a very high number of hours
# g_conf.NUMBER_OF_HOURS = 10000 # removed to simplify code
"""
# check again if this segment is required or not
set_type_of_process('validation', dataset_name)
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
if suppress_output:
sys.stdout = open(os.path.join('_output_logs',
exp_alias + '_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
"""
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HDFILES positions from the root directory as a in a vector.
dataset_name = dataset_name.split('_')[-1] # since preload file has '<X>hours_' as prefix whereas dataset folder does not
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"], dataset_name) # original code
augmenter = Augmenter(None)
print ('full dataset path: ', full_dataset)
dataset = CoILDataset(full_dataset, transform=augmenter, preload_name=args.dataset_name)
# The data loader is the multi threaded module from pytorch that release a number of
# workers to get all the data.
data_loader = torch.utils.data.DataLoader(dataset, batch_size=g_conf.BATCH_SIZE,
shuffle=False,
num_workers=g_conf.NUMBER_OF_LOADING_WORKERS,
pin_memory=True)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
""" removing this segment to simplify code
# The window used to keep track of the trainings
l1_window = []
latest = get_latest_evaluated_checkpoint()
if latest is not None: # When latest is noe
l1_window = coil_logger.recover_loss_window(dataset_name, None)
"""
model.cuda()
best_mse = 1000
best_error = 1000
best_mse_iter = 0
best_error_iter = 0
# modified validation code from here to run a single model
checkpoint = torch.load(args.checkpoint)
checkpoint_iteration = checkpoint['iteration']
print("model loaded ", checkpoint_iteration)
model.load_state_dict(checkpoint['state_dict'])
model.eval()
accumulated_mse = 0
accumulated_error = 0
iteration_on_checkpoint = 0
print ('data_loader size: ', len(data_loader))
total_error = []
for data in data_loader:
# Compute the forward pass on a batch from the loaded dataset
controls = data['directions']
branches = model(torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda())
output = model.extract_branch(torch.stack(branches[0:4]), controls)
error = torch.abs(output - dataset.extract_targets(data).cuda())
total_error += error.detach().cpu().tolist()
iteration_on_checkpoint += 1
if iteration_on_checkpoint % 50 == 0:
print ('iteration: ', iteration_on_checkpoint)
total_error = np.array(total_error)
print (len(total_error), total_error.shape)
np.save(os.path.join(args.save_path, args.dataset_name, 'computed_error.npy'), total_error)
'''
def execute(gpu, exp_batch, exp_alias, suppress_output=True, number_of_workers=12):
"""
The main encoder training function.
Args:
gpu: The GPU id number
exp_batch: the folder with the experiments
exp_alias: the alias, experiment name
suppress_output: if the output are going to be saved on a file
number_of_workers: the number of threads used for data loading
Returns:
None
"""
try:
# We set the visible cuda devices to select the GPU
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
g_conf.VARIABLE_WEIGHT = {}
# At this point the log file with the correct naming is created.
# You merge the yaml file with the global configuration structure.
merge_with_yaml(os.path.join('configs', exp_batch, exp_alias + '.yaml'))
set_type_of_process('train_encoder')
# Set the process into loading status.
coil_logger.add_message('Loading', {'GPU': os.environ["CUDA_VISIBLE_DEVICES"]})
# we set a seed for this exp
seed_everything(seed=g_conf.MAGICAL_SEED)
# Put the output to a separate file if it is the case
if suppress_output:
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
sys.stdout = open(os.path.join('_output_logs', exp_alias + '_' +
g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"), "a",
buffering=1)
sys.stderr = open(os.path.join('_output_logs',
exp_alias + '_err_' + g_conf.PROCESS_NAME + '_'
+ str(os.getpid()) + ".out"),
"a", buffering=1)
# Preload option
if g_conf.PRELOAD_MODEL_ALIAS is not None:
checkpoint = torch.load(os.path.join('_logs', g_conf.PRELOAD_MODEL_BATCH,
g_conf.PRELOAD_MODEL_ALIAS,
'checkpoints',
str(g_conf.PRELOAD_MODEL_CHECKPOINT) + '.pth'))
# Get the latest checkpoint to be loaded
# returns none if there are no checkpoints saved for this model
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(os.path.join('_logs', exp_batch, exp_alias,
'checkpoints', str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 1000000000.0
best_loss_iter = 0
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the positions from the root directory as a in a vector.
# full_dataset = os.path.join(os.environ["SRL_DATASET_PATH"], g_conf.TRAIN_DATASET_NAME)
# By instantiating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
if len(g_conf.EXPERIENCE_FILE) == 1:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2]
else:
json_file_name = str(g_conf.EXPERIENCE_FILE[0]).split('/')[-1].split('.')[-2] + '_' + str(g_conf.EXPERIENCE_FILE[1]).split('/')[-1].split('.')[-2]
dataset = CoILDataset(transform=augmenter,
preload_name=g_conf.PROCESS_NAME + '_' + json_file_name + '_' + g_conf.DATA_USED)
print ("Loaded dataset")
data_loader = select_balancing_strategy(dataset, iteration, number_of_workers)
encoder_model = EncoderModel(g_conf.ENCODER_MODEL_TYPE, g_conf.ENCODER_MODEL_CONFIGURATION)
encoder_model.cuda()
encoder_model.train()
print(encoder_model)
optimizer = optim.Adam(encoder_model.parameters(), lr=g_conf.LEARNING_RATE)
if checkpoint_file is not None or g_conf.PRELOAD_MODEL_ALIAS is not None:
encoder_model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
accumulated_time = checkpoint['total_time']
loss_window = coil_logger.recover_loss_window('train', iteration)
else: # We accumulate iteration time and keep the average speed
accumulated_time = 0
loss_window = []
print ("Before the loss")
if g_conf.ENCODER_MODEL_TYPE in ['ETE']:
criterion = Loss(g_conf.LOSS_FUNCTION)
# Loss time series window
for data in data_loader:
if iteration % 1000 == 0:
adjust_learning_rate_auto(optimizer, loss_window)
capture_time = time.time()
encoder_model.zero_grad()
"""
####################################
ENCODER_MODEL_TYPE can be: one-step-affordances, ETE, stdim, action_prediction
####################################
- one-step-affordances: input RGB images, compute affordances loss.
- ETE: input RGB images and speed, compute action loss (steering, throttle, brake)
- stdim: input two consecutive RGB images, compute the feature loss
- action_prediction: input two consecutive RGB images, compute action classification loss
- forward: input two consecutive RGB images, compute action loss + feature loss
"""
if g_conf.ENCODER_MODEL_TYPE in ['one-step-affordances']:
loss_function_params = {
'classification_gt': dataset.extract_affordances_targets(data, 'classification').cuda(),
# harzard stop, red_light....
'class_weights': g_conf.AFFORDANCES_CLASS_WEIGHT,
'regression_gt': dataset.extract_affordances_targets(data, 'regression').cuda(),
'variable_weights': g_conf.AFFORDANCES_VARIABLE_WEIGHT
}
# we input RGB images, speed and command to train affordances
loss = encoder_model(torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(dataset.extract_commands(data).cuda()),
loss_function_params)
if iteration == 0:
state = {
'iteration': iteration,
'state_dict': encoder_model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(state, os.path.join('_logs', exp_batch, exp_alias
, 'checkpoints', 'inital.pth'))
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['forward']:
# We sample another batch to avoid the superposition
inputs_data = [data['rgb'][0].cuda(), data['rgb'][1].cuda()]
loss, loss_other, loss_ete = encoder_model(inputs_data,
dataset.extract_inputs(data),
# We also add measurements and commands
dataset.extract_commands(data),
dataset.extract_targets(data)[0].cuda()
)
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['ETE']:
branches = encoder_model(torch.squeeze(data['rgb'].cuda()),
dataset.extract_inputs(data).cuda(),
torch.squeeze(dataset.extract_commands(data).cuda()))
loss_function_params = {
'branches': branches,
'targets': dataset.extract_targets(data).cuda(), # steer, throttle, brake
'inputs': dataset.extract_inputs(data).cuda(), # speed
'branch_weights': g_conf.BRANCH_LOSS_WEIGHT,
'variable_weights': g_conf.VARIABLE_WEIGHT
}
loss, _ = criterion(loss_function_params)
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['stdim']:
inputs_data = [data['rgb'][0].cuda(), data['rgb'][1].cuda()]
loss, _, _ = encoder_model(inputs_data,
dataset.extract_inputs(data),
# We also add measurements and commands
dataset.extract_commands(data)
)
loss.backward()
optimizer.step()
elif g_conf.ENCODER_MODEL_TYPE in ['action_prediction']:
inputs_data = [data['rgb'][0].cuda(), data['rgb'][1].cuda()]
loss, _, _ = encoder_model(inputs_data,
dataset.extract_inputs(data),
# We also add measurements and commands
dataset.extract_commands(data),
dataset.extract_targets(data)[0].cuda()
)
loss.backward()
optimizer.step()
else:
raise ValueError("The encoder model type is not know")
"""
####################################
Saving the model if necessary
####################################
"""
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': encoder_model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'optimizer': optimizer.state_dict(),
'best_loss_iter': best_loss_iter
}
torch.save(state, os.path.join('_logs', exp_batch, exp_alias
, 'checkpoints', str(iteration) + '.pth'))
iteration += 1
"""
################################################
Adding tensorboard logs.
Making calculations for logging purposes.
These logs are monitored by the printer module.
#################################################
"""
if g_conf.ENCODER_MODEL_TYPE in ['stdim', 'action_prediction', 'forward']:
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('f_t', torch.squeeze(data['rgb'][0]), iteration)
coil_logger.add_image('f_ti', torch.squeeze(data['rgb'][1]), iteration)
elif g_conf.ENCODER_MODEL_TYPE in ['one-step-affordances', 'ETE']:
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(data['rgb']), iteration)
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
accumulated_time += time.time() - capture_time
coil_logger.add_message('Iterating',
{'Iteration': iteration,
'Loss': loss.data.tolist(),
'Images/s': (iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss': best_loss, 'BestLossIteration': best_loss_iter},
iteration)
loss_window.append(loss.data.tolist())
coil_logger.write_on_error_csv('train', loss.data)
if iteration % 100 == 0:
print('Train Iteration: {} [{}/{} ({:.0f}%)] \t Loss: {:.6f}'.format(
iteration, iteration, g_conf.NUMBER_ITERATIONS,
100. * iteration / g_conf.NUMBER_ITERATIONS, loss.data))
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except RuntimeError as e:
coil_logger.add_message('Error', {'Message': str(e)})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
def execute(gpu, exp_batch, exp_alias, suppress_output=True):
# We set the visible cuda devices
# TODO: probable race condition, the train has to be started before.
try:
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
# At this point the log file with the correct naming is created.
merge_with_yaml(os.path.join('configs', exp_batch,
exp_alias + '.yaml'))
set_type_of_process('train')
coil_logger.add_message('Loading', {'GPU': gpu})
if not os.path.exists('_output_logs'):
os.mkdir('_output_logs')
# Put the output to a separate file
if suppress_output:
sys.stdout = open(os.path.join(
'_output_logs',
g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"),
"a",
buffering=1)
checkpoint_file = get_latest_saved_checkpoint()
if checkpoint_file is not None:
checkpoint = torch.load(
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(get_latest_saved_checkpoint())))
iteration = checkpoint['iteration']
best_loss = checkpoint['best_loss']
best_loss_iter = checkpoint['best_loss_iter']
else:
iteration = 0
best_loss = 10000.0
best_loss_iter = 0
# TODO: The checkpoint will continue, so it should erase everything up to the iteration on tensorboard
# Define the dataset. This structure is has the __get_item__ redefined in a way
# that you can access the HD_FILES positions from the root directory as a in a vector.
full_dataset = os.path.join(os.environ["COIL_DATASET_PATH"],
g_conf.TRAIN_DATASET_NAME)
# augmenter_cpu = iag.AugmenterCPU(g_conf.AUGMENTATION_SUITE_CPU)
# By instanciating the augmenter we get a callable that augment images and transform them
# into tensors.
augmenter = Augmenter(g_conf.AUGMENTATION)
dataset = CoILDataset(full_dataset, transform=augmenter)
data_loader = select_balancing_strategy(dataset, iteration)
model = CoILModel(g_conf.MODEL_TYPE, g_conf.MODEL_CONFIGURATION)
model.cuda()
if checkpoint_file is not None:
model.load_state_dict(checkpoint['state_dict'])
print(model)
criterion = Loss(g_conf.LOSS_FUNCTION)
optimizer = optim.Adam(model.parameters(), lr=g_conf.LEARNING_RATE)
print(dataset.meta_data)
print(model)
if checkpoint_file is not None:
accumulated_time = checkpoint['total_time']
else:
accumulated_time = 0 # We accumulate iteration time and keep the average speed
#TODO: test experiment continuation. Is the data sampler going to continue were it started.. ?
capture_time = time.time()
for data in data_loader:
input_data, float_data = data
# get the control commands from float_data, size = [120,1]
controls = float_data[:, dataset.controls_position(), :]
# The output(branches) is a list of 5 branches results, each branch is with size [120,3]
model.zero_grad()
branches = model(torch.squeeze(input_data['rgb'].cuda()),
dataset.extract_inputs(float_data).cuda())
loss = criterion(branches,
dataset.extract_targets(float_data).cuda(),
controls.cuda(),
dataset.extract_inputs(float_data).cuda(),
branch_weights=g_conf.BRANCH_LOSS_WEIGHT,
variable_weights=g_conf.VARIABLE_WEIGHT)
# TODO: All these logging things could go out to clean up the main
if loss.data < best_loss:
best_loss = loss.data.tolist()
best_loss_iter = iteration
# Log a random position
position = random.randint(0, len(float_data) - 1)
output = model.extract_branch(torch.stack(branches[0:4]), controls)
error = torch.abs(output -
dataset.extract_targets(float_data).cuda())
# TODO: For now we are computing the error for just the correct branch, it could be multi- branch,
coil_logger.add_scalar('Loss', loss.data, iteration)
coil_logger.add_image('Image', torch.squeeze(input_data['rgb']),
iteration)
loss.backward()
optimizer.step()
accumulated_time += time.time() - capture_time
capture_time = time.time()
# TODO: Get only the float_data that are actually generating output
# TODO: itearation is repeating , and that is dumb
coil_logger.add_message(
'Iterating', {
'Iteration':
iteration,
'Loss':
loss.data.tolist(),
'Images/s':
(iteration * g_conf.BATCH_SIZE) / accumulated_time,
'BestLoss':
best_loss,
'BestLossIteration':
best_loss_iter,
'Output':
output[position].data.tolist(),
'GroundTruth':
dataset.extract_targets(
float_data)[position].data.tolist(),
'Error':
error[position].data.tolist(),
'Inputs':
dataset.extract_inputs(float_data)[position].data.tolist()
}, iteration)
# TODO: For now we are computing the error for just the correct branch, it could be multi-branch,
# TODO: save also the optimizer state dictionary
if is_ready_to_save(iteration):
state = {
'iteration': iteration,
'state_dict': model.state_dict(),
'best_loss': best_loss,
'total_time': accumulated_time,
'best_loss_iter': best_loss_iter
}
# TODO : maybe already summarize the best model ???
torch.save(
state,
os.path.join('_logs', exp_batch, exp_alias, 'checkpoints',
str(iteration) + '.pth'))
iteration += 1
print(iteration)
if iteration % 1000 == 0:
adjust_learning_rate(optimizer, iteration)
del data
coil_logger.add_message('Finished', {})
except KeyboardInterrupt:
coil_logger.add_message('Error', {'Message': 'Killed By User'})
except:
traceback.print_exc()
coil_logger.add_message('Error', {'Message': 'Something Happened'})
