def get_latest_checkpoint_validation(): csv_file_path = os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME, g_conf.EXPERIMENT_NAME, g_conf.PROCESS_NAME + '_csv') csv_files = os.listdir(csv_file_path) if len(csv_files) == 0: return None sort_nicely(csv_files) # data = json_formatter.readJSONlog(open(log_file_path, 'r')) # print ('csv ', [re.findall('\d+', file)[0] for file in csv_files]) csv_file_numbers = set( [float(re.findall('\d+', file)[0]) for file in csv_files]) not_evaluated_logs = list( set(g_conf.TEST_SCHEDULE).difference(csv_file_numbers)) not_evaluated_logs = sorted(not_evaluated_logs, reverse=False) print('csv file numbers ', csv_file_numbers) print("not evaluated logs", not_evaluated_logs) if len(not_evaluated_logs) == 0: # Just in case that is the last one return g_conf.TEST_SCHEDULE[-1] if g_conf.TEST_SCHEDULE.index(not_evaluated_logs[0]) == 0: return None return g_conf.TEST_SCHEDULE[ g_conf.TEST_SCHEDULE.index(not_evaluated_logs[0]) - 1]
def print_folder_process_names(exp_batch): experiments_list = os.listdir(os.path.join('configs', exp_batch)) sort_nicely(experiments_list) for experiment in experiments_list: if '.yaml' in experiment: g_conf.immutable(False) merge_with_yaml(os.path.join('configs', exp_batch, experiment)) print(experiment.split('.')[-2] + ': ' + g_conf.EXPERIMENT_GENERATED_NAME)
def get_latest_saved_checkpoint(): """ Returns the , latest checkpoint number that was saved """ checkpoint_files = os.listdir(os.path.join('/datatmp/Experiments/rohitgan/_logs', g_conf.EXPERIMENT_BATCH_NAME, g_conf.EXPERIMENT_NAME, 'checkpoints')) if checkpoint_files == []: return None else: sort_nicely(checkpoint_files) return checkpoint_files[-1]
def get_latest_checkpoint(): # The path for log csv_file_path = os.path.join('_logs', g_conf.EXPERIMENT_BATCH_NAME, g_conf.EXPERIMENT_NAME, g_conf.PROCESS_NAME + '_csv') csv_files = os.listdir(csv_file_path) if csv_files == []: return None sort_nicely(csv_files) #data = json_formatter.readJSONlog(open(log_file_path, 'r')) return int(re.findall('\d+', csv_files[-1])[0])
def _pre_load_image_folders(self, path): """ Pre load the image folders for each episode, keep in mind that we only take the measurements that we think that are interesting for now. Args the path for the dataset Returns sensor data names: it is a vector with n dimensions being one for each sensor modality for instance, rgb only dataset will have a single vector with all the image names. float_data: all the wanted float data is loaded inside a vector, that is a vector of dictionaries. """ episodes_list = glob.glob(os.path.join(path, 'episode_*')) sort_nicely(episodes_list) # Do a check if the episodes list is empty if len(episodes_list) == 0: raise ValueError( "There are no episodes on the training dataset folder %s" % path) sensor_data_names = [] float_dicts = [] number_of_hours_pre_loaded = 0 # Now we do a check to try to find all the for episode in episodes_list: print('Episode ', episode) available_measurements_dict = data_parser.check_available_measurements( episode) if number_of_hours_pre_loaded > g_conf.NUMBER_OF_HOURS: # The number of wanted hours achieved break # Get all the measurements from this episode measurements_list = glob.glob(os.path.join(episode, 'measurement*')) sort_nicely(measurements_list) if len(measurements_list) == 0: print("EMPTY EPISODE") continue # A simple count to keep track how many measurements were added this episode. count_added_measurements = 0 for measurement in measurements_list[:-3]: data_point_number = measurement.split('_')[-1].split('.')[0] with open(measurement) as f: measurement_data = json.load(f) # depending on the configuration file, we eliminated the kind of measurements # that are not going to be used for this experiment # We extract the interesting subset from the measurement dict speed = data_parser.get_speed(measurement_data) directions = measurement_data['directions'] final_measurement = self._get_final_measurement( speed, measurement_data, 0, directions, available_measurements_dict) if self.is_measurement_partof_experiment(final_measurement): float_dicts.append(final_measurement) rgb = 'CentralRGB_' + data_point_number + '.png' sensor_data_names.append( os.path.join(episode.split('/')[-1], rgb)) count_added_measurements += 1 # We do measurements for the left side camera # We convert the speed to KM/h for the augmentation # We extract the interesting subset from the measurement dict final_measurement = self._get_final_measurement( speed, measurement_data, -30.0, directions, available_measurements_dict) if self.is_measurement_partof_experiment(final_measurement): float_dicts.append(final_measurement) rgb = 'LeftRGB_' + data_point_number + '.png' sensor_data_names.append( os.path.join(episode.split('/')[-1], rgb)) count_added_measurements += 1 # We do measurements augmentation for the right side cameras final_measurement = self._get_final_measurement( speed, measurement_data, 30.0, directions, available_measurements_dict) if self.is_measurement_partof_experiment(final_measurement): float_dicts.append(final_measurement) rgb = 'RightRGB_' + data_point_number + '.png' sensor_data_names.append( os.path.join(episode.split('/')[-1], rgb)) count_added_measurements += 1 # Check how many hours were actually added last_data_point_number = measurements_list[-4].split( '_')[-1].split('.')[0] #print("last and float dicts len", last_data_point_number, count_added_measurements) number_of_hours_pre_loaded += ( float(count_added_measurements / 10.0) / 3600.0) #print(" Added ", ((float(count_added_measurements) / 10.0) / 3600.0)) print(" Loaded ", number_of_hours_pre_loaded, " hours of data") # Make the path to save the pre loaded datasets if not os.path.exists('_preloads'): os.mkdir('_preloads') # If there is a name we saved the preloaded data if self.preload_name is not None: np.save(os.path.join('_preloads', self.preload_name), [sensor_data_names, float_dicts]) return sensor_data_names, float_dicts