def get_latest_checkpoint_validation():
csv_file_path = os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME,
g_conf.PROCESS_NAME + '_csv')
csv_files = os.listdir(csv_file_path)
if len(csv_files) == 0:
return None
sort_nicely(csv_files)
csv_file_numbers = set(
[float(re.findall('\d+', file)[0]) for file in csv_files])
not_evaluated_logs = list(
set(g_conf.TEST_SCHEDULE).difference(csv_file_numbers))
not_evaluated_logs = sorted(not_evaluated_logs, reverse=False)
if len(not_evaluated_logs) == 0: # Just in case that is the last one
return g_conf.TEST_SCHEDULE[-1]
if g_conf.TEST_SCHEDULE.index(not_evaluated_logs[0]) == 0:
return None
return g_conf.TEST_SCHEDULE[
g_conf.TEST_SCHEDULE.index(not_evaluated_logs[0]) - 1]
def get_latest_saved_checkpoint():
"""
Returns the latest checkpoint number that was saved
"""
checkpoint_files = os.listdir(os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints'))
if len(checkpoint_files) == 0:
return None
else:
sort_nicely(checkpoint_files)
return checkpoint_files[-1]
def print_folder_process_names(exp_batch):
experiments_list = os.listdir(os.path.join('configs', exp_batch))
sort_nicely(experiments_list)
for experiment in experiments_list:
if '.yaml' in experiment:
g_conf.immutable(False)
merge_with_yaml(os.path.join('configs', exp_batch, experiment))
print(
experiment.split('.')[-2] + ': ' +
g_conf.EXPERIMENT_GENERATED_NAME)
def get_latest_saved_checkpoint():
"""
Returns the , latest checkpoint number that was saved
"""
checkpoint_files = glob.glob(os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME, 'checkpoints/*0.pth')
)
if checkpoint_files == []:
return None
else:
sort_nicely(checkpoint_files)
return checkpoint_files[-1].split('/')[-1]
def get_latest_checkpoint_validation():
"""
Get the checkpoint to validate
"""
# Path to the directory containing all the csv files
csv_file_path = os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME,
g_conf.EXPERIMENT_NAME,
f'{g_conf.PROCESS_NAME}_csv')
# List them
csv_files = os.listdir(csv_file_path)
# Return None if there aren't any
if len(csv_files) == 0:
return None
# Otherwise, we should sort them and return the latest one
sort_nicely(csv_files)
# Get the number (as a float?), delete repeated ones
csv_file_numbers = set(
[float(re.findall(r'\d+', file)[0]) for file in csv_files])
# Remove the checkpoints that have already been tested and sort it
not_evaluated_logs = sorted(list(
set(g_conf.TEST_SCHEDULE).difference(csv_file_numbers)),
reverse=False)
if len(not_evaluated_logs) == 0: # Just in case that is the last one
return g_conf.TEST_SCHEDULE[-1]
if g_conf.TEST_SCHEDULE.index(not_evaluated_logs[0]) == 0:
return None
return g_conf.TEST_SCHEDULE[
g_conf.TEST_SCHEDULE.index(not_evaluated_logs[0]) - 1]
def _pre_load_image_folders(self, path, dataset_index=0):
"""
Pre load the image folders for each episode, keep in mind that we only take
the measurements that we think that are interesting for now.
Args
the path for the dataset
Returns
sensor data names: it is a vector with n dimensions being one for each sensor modality
for instance, rgb only dataset will have a single vector with all the image names.
float_data: all the wanted float data is loaded inside a vector, that is a vector
of dictionaries.
"""
episodes_list = glob.glob(os.path.join(path, 'episode_*'))
sort_nicely(episodes_list)
# Do a check if the episodes list is empty
if len(episodes_list) == 0:
raise ValueError(
"There are no episodes on the training dataset folder %s" %
path)
sensor_data_names = []
float_dicts = []
number_of_hours_pre_loaded = 0
# Now we do a check to try to find all the
for episode in episodes_list:
print('Episode ', episode)
available_measurements_dict = data_parser.check_available_measurements(
episode)
if number_of_hours_pre_loaded > g_conf.NUMBER_OF_HOURS:
# The number of wanted hours achieved
break
# Get all the measurements from this episode
measurements_list = glob.glob(os.path.join(episode,
'measurement*'))
sort_nicely(measurements_list)
if len(measurements_list) == 0:
print("EMPTY EPISODE")
continue
# A simple count to keep track how many measurements were added this episode.
count_added_measurements = 0
for measurement in measurements_list:
data_point_number = measurement.split('_')[-1].split('.')[0]
with open(measurement) as f:
measurement_data = json.load(f)
# depending on the configuration file, we eliminated the kind of measurements
# that are not going to be used for this experiment
# We extract the interesting subset from the measurement dict
speed = data_parser.get_speed(measurement_data)
directions = measurement_data['directions']
final_measurement = self._get_final_measurement(
speed, measurement_data, 0, directions,
available_measurements_dict)
if self.is_measurement_partof_experiment(final_measurement):
sensor_data_names.append([])
float_dicts.append(final_measurement)
rgb = 'CentralRGB_' + data_point_number + '.png'
sensor_data_names[-1].append(
os.path.join(episode.split('/')[-1], rgb))
seg = 'SemanticSeg_' + data_point_number + '.png'
sensor_data_names[-1].append(
os.path.join(episode.split('/')[-1], seg))
count_added_measurements += 1
# We do measurements for the left side camera
# We convert the speed to KM/h for the augmentation
# We extract the interesting subset from the measurement dict
final_measurement = self._get_final_measurement(
speed, measurement_data, -30.0, directions,
available_measurements_dict)
if self.is_measurement_partof_experiment(final_measurement):
sensor_data_names.append([])
float_dicts.append(final_measurement)
rgb = 'LeftRGB_' + data_point_number + '.png'
sensor_data_names[-1].append(
os.path.join(episode.split('/')[-1], rgb))
count_added_measurements += 1
# We do measurements augmentation for the right side cameras
final_measurement = self._get_final_measurement(
speed, measurement_data, 30.0, directions,
available_measurements_dict)
if self.is_measurement_partof_experiment(final_measurement):
sensor_data_names.append([])
float_dicts.append(final_measurement)
rgb = 'RightRGB_' + data_point_number + '.png'
sensor_data_names[-1].append(
os.path.join(episode.split('/')[-1], rgb))
count_added_measurements += 1
# Check how many hours were actually added
last_data_point_number = measurements_list[-1].split(
'_')[-1].split('.')[0]
number_of_hours_pre_loaded += (
float(count_added_measurements / 10.0) / 3600.0)
print(" Loaded ", number_of_hours_pre_loaded, " hours of data")
# Make the path to save the pre loaded datasets
if not os.path.exists('_preloads'):
os.mkdir('_preloads')
# If there is a name we saved the preloaded data
if self.preload_names[dataset_index] is not None:
np.save(
os.path.join('_preloads', self.preload_names[dataset_index]),
[sensor_data_names, float_dicts])
return sensor_data_names, float_dicts
def _pre_load_image_folders(self, path):
"""
Pre load the image folders for each episode, keep in mind that we only take
the measurements that we think that are interesting for now.
Args
the path for the dataset
Returns
sensor data names: it is a vector with n dimensions being one for each sensor modality
for instance, rgb only dataset will have a single vector with all the image names.
float_data: all the wanted float data is loaded inside a vector, that is a vector
of dictionaries.
"""
episodes_list = glob.glob(os.path.join(path, 'episode_*'))
sort_nicely(episodes_list)
# Do a check if the episodes list is empty
if len(episodes_list) == 0:
raise ValueError("There are no episodes on the training dataset folder %s" % path)
sensor_data_names = []
float_dicts = []
number_of_hours_pre_loaded = 0
# Now we do a check to try to find all the
for episode in episodes_list:
print('Episode ', episode)
available_measurements_dict = data_parser.check_available_measurements(episode)
if number_of_hours_pre_loaded > g_conf.NUMBER_OF_HOURS:
# The number of wanted hours achieved
break
# Get all the measurements from this episode
measurements_list = glob.glob(os.path.join(episode, 'measurement*'))
sort_nicely(measurements_list)
if len(measurements_list) == 0:
print("EMPTY EPISODE")
continue
# A simple count to keep track how many measurements were added this episode.
count_added_measurements = 0
for measurement in measurements_list[:-3]:
data_point_number = measurement.split('_')[-1].split('.')[0]
with open(measurement) as f:
measurement_data = json.load(f)
# depending on the configuration file, we eliminated the kind of measurements
# that are not going to be used for this experiment
# We extract the interesting subset from the measurement dict
speed = data_parser.get_speed(measurement_data)
directions = measurement_data['directions']
final_measurement = self._get_final_measurement(speed, measurement_data, 0,
directions,
available_measurements_dict)
if self.is_measurement_partof_experiment(final_measurement):
float_dicts.append(final_measurement)
def _pre_load_image_folders_old(self, path):
"""
Pre load the image folders for each episode, keep in mind that we only take
the measurements that we think that are interesting for now.
Args
the path for the dataset
Returns
sensor data names: it is a vector with n dimensions being one for each sensor modality
for instance, rgb only dataset will have a single vector with all the image names.
float_data: all the wanted float data is loaded inside a vector, that is a vector
of dictionaries.
"""
containers_list = glob.glob(os.path.join(path, 'Container_*'))
sort_nicely(containers_list)
# Do a check if the episodes list is empty
if len(containers_list) == 0:
raise ValueError(
f"There are no containers on the training dataset folder: {path}"
)
# We will check one image to see if it matches the size expected by the network
checked_image = False
sensor_data_names = {}
float_dicts = []
number_of_hours_pre_loaded = 0
# Now we do a check to try to find all the
for container in containers_list:
print(f'Container name: {container}')
if number_of_hours_pre_loaded > g_conf.NUMBER_OF_HOURS:
# The number of wanted hours achieved
break
# A simple count to keep track how many measurements were added this episode.
count_added_measurements = 0
# We may have more than one client for each container, so the data_point_number might clash later
client_list = glob.glob(os.path.join(container, '**/Client_*'),
recursive=True)
for client in client_list:
# Get all the measurements from this client
measurements_list = glob.glob(os.path.join(client, 'can_bus*'))
sort_nicely(measurements_list)
if len(measurements_list) == 0:
print("Empty client")
continue
for measurement in tqdm(measurements_list):
data_point_number = os.path.splitext(measurement)[0][
-6:] # /pth/to/can_bus000019.json => 000019
with open(measurement) as f:
measurement_data = json.load(f)
# Delete some non-floatable cases
# depending on the configuration file, we eliminated the kind of measurements
# that are not going to be used for this experiment
del measurement_data['hand_brake']
del measurement_data[
'reverse'] # TODO: not relevant now, but we might be interested later on
del measurement_data['ego_position']
del measurement_data['route_nodes_xyz']
del measurement_data['route_nodes']
# We extract the interesting subset from the measurement dict
speed = data_parser.get_speed(measurement_data)
for sensor in g_conf.SENSORS.keys():
# We will go through each of the cameras
cameras = (('central', 0), ('left', -30.0), ('right',
30.0))
sensor_name = sensor.split('_')[0]
for cam in cameras:
# We do measurements for the three cameras
# We convert the speed to KM/h for the augmentation
# We extract the interesting subset from the measurement dict
final_measurement = self._get_final_measurement(
speed, measurement_data, cam[1])
if self.is_measurement_part_of_experiment(
final_measurement):
float_dicts.append(final_measurement)
sensor_path = glob.glob(os.path.join(
client,
f'**/{sensor_name}_{cam[0]}{data_point_number}.png'
),
recursive=True)
if len(sensor_path) == 0:
continue
if not checked_image:
if not check_size(*sensor_path,
g_conf.SENSORS[sensor]):
raise RuntimeError(
'Unexpected image size for the network!'
)
checked_image = True
if sensor_name in sensor_data_names:
sensor_data_names[sensor_name].append(
*sensor_path)
else:
sensor_data_names[sensor_name] = [
*sensor_path
]
count_added_measurements += 1
# Check how many hours were actually added
number_of_hours_pre_loaded += (
float(count_added_measurements / g_conf.TRAIN_DATA_FPS) /
3600.0)
print(f"Loaded {number_of_hours_pre_loaded} hours of data")
# Make the path to save the pre loaded datasets
if not os.path.exists('_preloads'):
os.mkdir('_preloads')
# If there is a name we saved the preloaded data
if self.preload_name is not None:
np.save(os.path.join('_preloads', self.preload_name),
[sensor_data_names, float_dicts])
return sensor_data_names, float_dicts
