def generator(data, batch_size=cfg.BATCH_SIZE): num_records = len(data) while True: #shuffle again for good measure data = shuffle(data) for offset in range(0, num_records, batch_size): batch_data = data[offset:offset + batch_size] if len(batch_data) != batch_size: break b_inputs_img = [] b_labels = [] for seq in batch_data: inputs_img = [] labels = [] for record in seq: #get image data if we don't already have it if record['img_data'] is None: img_arr = load_scaled_image_arr( record['image_path'], cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] inputs_img.append(img_arr) if img_arr is None: continue labels.append(seq[-1]['target_output']) b_inputs_img.append(inputs_img) b_labels.append(labels) X = [np.array(b_inputs_img).reshape(batch_size,\ cfg.SEQUENCE_LENGTH, cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)] y = np.array(b_labels).reshape(batch_size, 2) yield X, y
def generator(save_best, opts, data, batch_size, isTrainSet=True, min_records_to_train=1000): num_records = len(data) while True: if isTrainSet and opts['continuous']: ''' When continuous training, we look for new records after each epoch. This will add new records to the train and validation set. ''' records = gather_records(cfg, tub_names, opts) if len(records) > num_records: collate_records(records, gen_records, opts) new_num_rec = len(data) if new_num_rec > num_records: print('picked up', new_num_rec - num_records, 'new records!') num_records = new_num_rec save_best.reset_best() if num_records < min_records_to_train: print("not enough records to train. need %d, have %d. waiting..." % (min_records_to_train, num_records)) time.sleep(10) continue batch_data = [] keys = list(data.keys()) keys = shuffle(keys) kl = opts['keras_pilot'] if type(kl.model.output) is list: model_out_shape = (2, 1) else: model_out_shape = kl.model.output.shape if type(kl.model.input) is list: model_in_shape = (2, 1) else: model_in_shape = kl.model.input.shape has_imu = type(kl) is KerasIMU has_bvh = type(kl) is KerasBehavioral for key in keys: if not key in data: continue _record = data[key] if _record['train'] != isTrainSet: continue if continuous: #in continuous mode we need to handle files getting deleted filename = _record['image_path'] if not os.path.exists(filename): data.pop(key, None) continue batch_data.append(_record) if len(batch_data) == batch_size: inputs_img = [] inputs_imu = [] inputs_bvh = [] angles = [] throttles = [] for record in batch_data: #get image data if we don't already have it if record['img_data'] is None: filename = record['image_path'] img_arr = load_scaled_image_arr(filename, cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] if has_imu: inputs_imu.append(record['imu_array']) if has_bvh: inputs_bvh.append(record['behavior_arr']) inputs_img.append(img_arr) angles.append(record['angle']) throttles.append(record['throttle']) if img_arr is None: continue img_arr = np.array(inputs_img).reshape(batch_size,\ cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) if has_imu: X = [img_arr, np.array(inputs_imu)] elif has_bvh: X = [img_arr, np.array(inputs_bvh)] else: X = [img_arr] if model_out_shape[1] == 2: y = [np.array([angles, throttles])] else: y = [np.array(angles), np.array(throttles)] yield X, y batch_data = []
def generator(data, opt, batch_size=cfg.BATCH_SIZE): num_records = len(data) while True: #shuffle again for good measure random.shuffle(data) for offset in range(0, num_records, batch_size): batch_data = data[offset:offset + batch_size] if len(batch_data) != batch_size: break b_inputs_img = [] b_vec_in = [] b_labels = [] b_vec_out = [] for seq in batch_data: inputs_img = [] vec_in = [] labels = [] vec_out = [] num_images_target = len(seq) iTargetOutput = -1 if opt['look_ahead']: num_images_target = cfg.SEQUENCE_LENGTH iTargetOutput = cfg.SEQUENCE_LENGTH - 1 for iRec, record in enumerate(seq): #get image data if we don't already have it if len(inputs_img) < num_images_target: if record['img_data'] is None: img_arr = load_scaled_image_arr( record['image_path'], cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] inputs_img.append(img_arr) if iRec >= iTargetOutput: vec_out.append(record['angle']) vec_out.append(record['throttle']) else: vec_in.append(0.0) #record['angle']) vec_in.append(0.0) #record['throttle']) label_vec = seq[iTargetOutput]['target_output'] if look_ahead: label_vec = np.array(vec_out) labels.append(label_vec) b_inputs_img.append(inputs_img) b_vec_in.append(vec_in) b_labels.append(labels) if look_ahead: X = [np.array(b_inputs_img).reshape(batch_size,\ cfg.TARGET_H, cfg.TARGET_W, cfg.SEQUENCE_LENGTH)] X.append(np.array(b_vec_in)) y = np.array(b_labels).reshape( batch_size, (cfg.SEQUENCE_LENGTH + 1) * 2) else: X = [np.array(b_inputs_img).reshape(batch_size,\ cfg.SEQUENCE_LENGTH, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D)] y = np.array(b_labels).reshape(batch_size, 2) yield X, y
def vae_generator(cfg, data, batch_size, isTrainSet=True, min_records_to_train=1000, aug=False, aux=None, pilot=False): num_records = len(data) while True: batch_data = [] keys = list(data.keys()) keys = shuffle(keys) for key in keys: if not key in data: continue _record = data[key] if _record['train'] != isTrainSet: continue batch_data.append(_record) if len(batch_data) == batch_size: inputs_img = [] aux_out = [] steering = [] throttle = [] for record in batch_data: img_arr = None #get image data if we don't already have it if record['img_data'] is None: img_arr = load_scaled_image_arr( record['image_path'], cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] if img_arr is None: continue inputs_img.append(img_arr) if aux is not None: if aux in record['json_data']: aux_out.append(record['json_data'][aux]) else: print("Missing aux data in: {}".format(record)) continue st, th = Tub.get_angle_throttle(record['json_data']) steering.append(st) throttle.append(th) X = np.array(inputs_img).reshape(batch_size, cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) y = {'main_output': X} if pilot: y['steering_output'] = np.array(steering) y['throttle_output'] = np.array(throttle) if aux is not None: aux_out = keras.utils.to_categorical(aux_out, num_classes=7) y['aux_output'] = aux_out yield X, y batch_data = []
def vae_generator(cfg, data, batch_size, isTrainSet=True, min_records_to_train=1000, aug=False, aux=None): """ Returns batches of data for training a VAE, given a dictionary of DonkeyCar inputs. """ num_records = len(data) while True: batch_data = [] keys = list(data.keys()) keys = shuffle(keys) for key in keys: if not key in data: continue _record = data[key] if _record['train'] != isTrainSet: continue batch_data.append(_record) if len(batch_data) == batch_size: inputs_img = [] aux_out = [] for record in batch_data: img_arr = None #get image data if we don't already have it if record['img_data'] is None: img_arr = load_scaled_image_arr( record['image_path'], cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] if aux is not None: if aux in record['json_data']: aux_out.append(record['json_data'][aux]) else: print("Missing aux data in: {}".format(record)) continue if img_arr is None: continue inputs_img.append(img_arr) X = np.array(inputs_img).reshape(batch_size, cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) y = X if aux is not None: aux_out = keras.utils.to_categorical(aux_out, num_classes=7) yield X, {'main_output': y, 'aux_output': aux_out} else: yield X, y batch_data = []
def generator(data, opt, batch_size=cfg.BATCH_SIZE): num_records = len(data) while True: # shuffle again for good measure random.shuffle(data) for offset in range(0, num_records, batch_size): batch_data = data[offset:offset + batch_size] # the current batch for training # TODO: Ignore last batch? if len(batch_data) != batch_size: break # batch input image b_inputs_img = [] b_vec_in = [] b_vec_out = [] b_labels = [] b_labels_2 = [] b_labels_3 = [] for seq in batch_data: inputs_img = [] vec_in = [] vec_out = [] label_vec1 = [] label_vec2 = [] label_vec3 = [] num_images_target = len(seq) iTargetOutput = -1 if opt['look_ahead']: num_images_target = cfg.SEQUENCE_LENGTH # output count iTargetOutput = cfg.SEQUENCE_LENGTH for iRec, record in enumerate(seq): # get image data if we don't already have it if len(inputs_img) < num_images_target: if record['img_data'] is None: # img data loaded here img_arr = load_scaled_image_arr( record['image_path'], cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] # The image array that fit into the network inputs_img.append(img_arr) # Add Imu vector if cfg.model_type == 'rnn_imu' or \ cfg.model_type == 'rnn_imu_linear' or \ cfg.model_type == 'rnn_imu_many2many' or \ cfg.model_type == 'rnn_imu_many2many_imupred' or \ cfg.model_type == 'test': imu_array = [] imu_array.append( record['json_data']['imu1/acl_x']) imu_array.append( record['json_data']['imu1/acl_y']) imu_array.append( record['json_data']['imu1/acl_z']) imu_array.append( record['json_data']['imu1/gyr_x']) imu_array.append( record['json_data']['imu1/gyr_y']) imu_array.append( record['json_data']['imu1/gyr_z']) imu_array.append( record['json_data']['imu2/acl_x']) imu_array.append( record['json_data']['imu2/acl_y']) imu_array.append( record['json_data']['imu2/acl_z']) imu_array.append( record['json_data']['imu2/gyr_x']) imu_array.append( record['json_data']['imu2/gyr_y']) imu_array.append( record['json_data']['imu2/gyr_z']) vec_in.append((imu_array)) if iRec >= iTargetOutput: # for future prediction if cfg.model_type == 'rnn_imu_many2many': angle = float( record['json_data']['user/angle']) throttle = float( record['json_data']["user/throttle"]) angle = dk.utils.linear_bin(angle, N=31, offset=1, R=2) throttle = dk.utils.linear_bin( throttle, N=31, offset=0, R=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE) label_vec1.append(angle) label_vec2.append(throttle) elif cfg.model_type == 'rnn_imu_many2many_imupred': angle = float( record['json_data']['user/angle']) throttle = float( record['json_data']["user/throttle"]) angle = dk.utils.linear_bin(angle, N=31, offset=1, R=2) throttle = dk.utils.linear_bin( throttle, N=31, offset=0, R=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE) label_vec1.append(angle) label_vec2.append(throttle) label_vec3.append(imu_array) else: vec_out.append(record['angle']) vec_out.append(record['throttle']) else: # for future node output continue #vec_in.append(0.0) # record['angle']) #vec_in.append(0.0) # record['throttle']) # Label if cfg.model_type == 'rnn_imu' or cfg.model_type == 'test': angle = float(record['json_data']['user/angle']) throttle = float(record['json_data']["user/throttle"]) # Warning: this is not universal and need to be change manually label_vec1 = dk.utils.linear_bin(angle, N=31, offset=1, R=2) label_vec2 = dk.utils.linear_bin( throttle, N=31, offset=0, R=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE) elif cfg.model_type == 'rnn_imu_linear': angle = float(record['json_data']['user/angle']) throttle = float(record['json_data']["user/throttle"]) label_vec1 = angle label_vec2 = throttle else: label_vec = seq[iTargetOutput]['target_output'] # vec input if look_ahead: label_vec = np.array(vec_out) if cfg.model_type == 'test': inputs_img = inputs_img[0] b_inputs_img.append(inputs_img) b_vec_in.append(vec_in) b_labels.append(label_vec1) b_labels_2.append(label_vec2) if cfg.model_type == 'rnn_imu_many2many_imupred': b_labels_3.append(label_vec3) if cfg.model_type == 'look_ahead': X = [np.array(b_inputs_img).reshape(batch_size, \ cfg.TARGET_H, cfg.TARGET_W, cfg.SEQUENCE_LENGTH)] X.append(np.array(b_vec_in)) y = np.array(b_labels).reshape( batch_size, (cfg.SEQUENCE_LENGTH + 1) * 2) elif cfg.model_type == 'rnn_imu' or cfg.model_type == 'rnn_imu_linear': X1 = np.array(b_inputs_img).reshape( batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D) X2 = np.array(b_vec_in).reshape(batch_size, cfg.SEQUENCE_LENGTH, 12) y1 = np.array(b_labels) y2 = np.array(b_labels_2) elif cfg.model_type == 'rnn_imu_many2many': X1 = np.array(b_inputs_img).reshape( batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D) X2 = np.array(b_vec_in).reshape(batch_size, cfg.SEQUENCE_LENGTH, 12) y1 = np.array(b_labels) y2 = np.array(b_labels_2) elif cfg.model_type == 'rnn_imu_many2many_imupred': X1 = np.array(b_inputs_img).reshape( batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D) X2 = np.array(b_vec_in).reshape(batch_size, cfg.SEQUENCE_LENGTH, 12) y1 = np.array(b_labels) y2 = np.array(b_labels_2) y3 = np.array(b_labels_3) elif cfg.model_type == 'test': X1 = np.array(b_inputs_img).reshape( batch_size, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D) X2 = np.array(b_vec_in).reshape(batch_size, cfg.SEQUENCE_LENGTH, 12) y1 = np.array(b_labels) y2 = np.array(b_labels_2) else: X = [ np.array(b_inputs_img).reshape(batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D) ] y = np.array(b_labels).reshape(batch_size, 2) #print("X ", X.shape) #print("Y ", y.shape) if cfg.model_type == 'rnn_imu' or \ cfg.model_type == 'rnn_imu_linear' or \ cfg.model_type == 'rnn_imu_many2many' or cfg.model_type == 'test': yield [X1, X2], [y1, y2] elif cfg.model_type == 'rnn_imu_many2many_imupred': yield [X1, X2], [y1, y2, y3] else: yield X, y
def generator(save_best, opts, data, batch_size, isTrainSet=True, min_records_to_train=1000): # pdb.set_trace() num_records = len(data) while True: if isTrainSet and opts['continuous']: ''' When continuous training, we look for new records after each epoch. This will add new records to the train and validation set. ''' records = gather_records(cfg, tub_names, opts) if len(records) > num_records: collate_records(records, gen_records, opts) new_num_rec = len(data) if new_num_rec > num_records: print('picked up', new_num_rec - num_records, 'new records!') num_records = new_num_rec save_best.reset_best() if num_records < min_records_to_train: print( "not enough records to train. need %d, have %d. waiting..." % (min_records_to_train, num_records)) time.sleep(10) continue batch_data = [] keys = list(data.keys()) random.shuffle(keys) kl = opts['keras_pilot'] if type(kl.model.output) is list: model_out_shape = (2, 1) else: model_out_shape = kl.model.output.shape if type(kl.model.input) is list: model_in_shape = (2, 1) else: model_in_shape = kl.model.input.shape has_imu = type(kl) is KerasIMU has_bvh = type(kl) is KerasBehavioral img_out = type(kl) is KerasLatent if img_out: import cv2 for key in keys: if not key in data: continue _record = data[key] # format slam points from string representation to numpy array _record["json_data"]["slam_points"] = ast.literal_eval( _record["json_data"]["slam_points"]) _record["json_data"]["slam_points"] = np.array( _record["json_data"]["slam_points"]) if _record['train'] != isTrainSet: continue if continuous: #in continuous mode we need to handle files getting deleted filename = _record['image_path'] if not os.path.exists(filename): data.pop(key, None) continue batch_data.append(_record) if len(batch_data) == batch_size: # pdb.set_trace() inputs_img = [] inputs_imu = [] inputs_bvh = [] angles = [] throttles = [] out_img = [] out = [] # SLAM Points slam_points = [] for record in batch_data: #get image data if we don't already have it if record['img_data'] is None: filename = record['image_path'] img_arr = load_scaled_image_arr(filename, cfg) if img_arr is None: break if aug: img_arr = augment_image(img_arr) if cfg.CACHE_IMAGES: record['img_data'] = img_arr else: img_arr = record['img_data'] if img_out: rz_img_arr = cv2.resize(img_arr, (127, 127)) / 255.0 out_img.append(rz_img_arr[:, :, 0].reshape( (127, 127, 1))) if has_imu: inputs_imu.append(record['imu_array']) if has_bvh: inputs_bvh.append(record['behavior_arr']) inputs_img.append(img_arr) angles.append(record['angle']) throttles.append(record['throttle']) # Add SLAM points slam_points.append(record["json_data"]["slam_points"]) out.append([ record['angle'], record['throttle'], record["json_data"]["slam_points"] ]) # pdb.set_trace() if img_arr is None: continue img_arr = np.array(inputs_img).reshape(batch_size,\ cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D) if has_imu: X = [img_arr, np.array(inputs_imu)] elif has_bvh: X = [img_arr, np.array(inputs_bvh)] else: # Add SLAM point array data X = [img_arr, np.array(slam_points)] if img_out: y = [out_img, np.array(angles), np.array(throttles)] elif model_out_shape[1] == 2: y = [np.array([out]).reshape(batch_size, 2)] else: y = [np.array(angles), np.array(throttles)] yield X, y batch_data = [] return True