def on_model_type(self, obj, model_type): """ Kivy method that is called if self.model_type changes. """ if self.model_type and self.model_type != 'Model type': cfg = tub_screen().ids.config_manager.config if cfg: self.pilot = get_model_by_type(self.model_type, cfg) self.ids.pilot_button.disabled = False
def train(cfg: Config, tub_paths: str, model: str, model_type: str) -> \ tf.keras.callbacks.History: """ Train the model """ model_name, model_ext = os.path.splitext(model) is_tflite = model_ext == '.tflite' if is_tflite: model = f'{model_name}.h5' if not model_type: model_type = cfg.DEFAULT_MODEL_TYPE tubs = tub_paths.split(',') all_tub_paths = [os.path.expanduser(tub) for tub in tubs] output_path = os.path.expanduser(model) train_type = 'linear' if 'linear' in model_type else model_type kl = get_model_by_type(train_type, cfg) if cfg.PRINT_MODEL_SUMMARY: print(kl.model.summary()) dataset = TubDataset(cfg, all_tub_paths) training_records, validation_records = dataset.train_test_split() print(f'Records # Training {len(training_records)}') print(f'Records # Validation {len(validation_records)}') training_pipe = BatchSequence(kl, cfg, training_records, is_train=True) validation_pipe = BatchSequence(kl, cfg, validation_records, is_train=False) dataset_train = training_pipe.create_tf_data().prefetch( tf.data.experimental.AUTOTUNE) dataset_validate = validation_pipe.create_tf_data().prefetch( tf.data.experimental.AUTOTUNE) train_size = len(training_pipe) val_size = len(validation_pipe) assert val_size > 0, "Not enough validation data, decrease the batch " \ "size or add more data." history = kl.train(model_path=output_path, train_data=dataset_train, train_steps=train_size, batch_size=cfg.BATCH_SIZE, validation_data=dataset_validate, validation_steps=val_size, epochs=cfg.MAX_EPOCHS, verbose=cfg.VERBOSE_TRAIN, min_delta=cfg.MIN_DELTA, patience=cfg.EARLY_STOP_PATIENCE) if is_tflite: tf_lite_model_path = f'{os.path.splitext(output_path)[0]}.tflite' keras_model_to_tflite(output_path, tf_lite_model_path) return history
def test_training_pipeline(config: Config, model_type: str, train_filter: Callable[[TubRecord], bool]) -> None: """ Testing consistency of the model interfaces and data used in training pipeline. :param config: donkey config :param model_type: test specification of model type :param train_filter: filter for records :return: None """ kl = get_model_by_type(model_type, config) tub_dir = config.DATA_PATH_ALL if model_type in full_tub else \ config.DATA_PATH # don't shuffle so we can identify data for testing config.TRAIN_FILTER = train_filter dataset = TubDataset(config, [tub_dir], seq_size=kl.seq_size()) training_records, validation_records = \ train_test_split(dataset.get_records(), shuffle=False, test_size=(1. - config.TRAIN_TEST_SPLIT)) seq = BatchSequence(kl, config, training_records, True) data_train = seq.create_tf_data() num_whole_batches = len(training_records) // config.BATCH_SIZE # this takes all batches into one list tf_batch = list(data_train.take(num_whole_batches).as_numpy_iterator()) it = iter(training_records) for xy_batch in tf_batch: # extract x and y values from records, asymmetric in x and y b/c x # requires image manipulations batch_records = [next(it) for _ in range(config.BATCH_SIZE)] records_x = [ kl.x_translate(kl.x_transform_and_process(r, normalize_image)) for r in batch_records ] records_y = [kl.y_translate(kl.y_transform(r)) for r in batch_records] # from here all checks are symmetrical between x and y for batch, o_type, records \ in zip(xy_batch, kl.output_types(), (records_x, records_y)): # check batch dictionary have expected keys assert batch.keys() == o_type.keys(), \ 'batch keys need to match models output types' # convert record values into arrays of batch size values = defaultdict(list) for r in records: for k, v in r.items(): values[k].append(v) # now convert arrays of floats or numpy arrays into numpy arrays np_dict = dict() for k, v in values.items(): np_dict[k] = np.array(v) # compare record values with values from tf.data for k, v in batch.items(): assert np.isclose(v, np_dict[k]).all()
def train(cfg, tub_paths, output_path, model_type): """ Train the model """ # convert single path into list of one element if type(tub_paths) is str: tub_paths = [tub_paths] if 'linear' in model_type: train_type = 'linear' else: train_type = model_type print(f'{train_type} from train function') kl = get_model_by_type(train_type, cfg) kl.compile() if cfg.PRINT_MODEL_SUMMARY: print(kl.model.summary()) batch_size = cfg.BATCH_SIZE dataset = TubDataset(tub_paths, test_size=(1. - cfg.TRAIN_TEST_SPLIT)) training_records, validation_records = dataset.train_test_split() print('Records # Training %s' % len(training_records)) print('Records # Validation %s' % len(validation_records)) training = TubSequence(kl, cfg, training_records) validation = TubSequence(kl, cfg, validation_records) assert len(validation) > 0, "Not enough validation data, decrease the " \ "batch size or add more data." # Setup early stoppage callbacks callbacks = [ EarlyStopping(monitor='val_loss', patience=cfg.EARLY_STOP_PATIENCE), ModelCheckpoint( filepath=output_path, monitor='val_loss', save_best_only=True, verbose=1, ) ] history = kl.model.fit(x=training, steps_per_epoch=len(training), batch_size=batch_size, callbacks=callbacks, validation_data=validation, validation_steps=len(validation), epochs=cfg.MAX_EPOCHS, verbose=cfg.VERBOSE_TRAIN, workers=1, use_multiprocessing=False) return history
def train(cfg: Config, tub_paths: str, model: str = None, model_type: str = None, transfer: str = None, comment: str = None) \ -> tf.keras.callbacks.History: """ Train the model """ database = PilotDatabase(cfg) model_name, model_num, train_type, is_tflite = \ get_model_train_details(cfg, database, model, model_type) output_path = os.path.join(cfg.MODELS_PATH, model_name + '.h5') kl = get_model_by_type(train_type, cfg) if transfer: kl.load(transfer) if cfg.PRINT_MODEL_SUMMARY: print(kl.model.summary()) tubs = tub_paths.split(',') all_tub_paths = [os.path.expanduser(tub) for tub in tubs] dataset = TubDataset(cfg, all_tub_paths) training_records, validation_records = dataset.train_test_split() print(f'Records # Training {len(training_records)}') print(f'Records # Validation {len(validation_records)}') training_pipe = BatchSequence(kl, cfg, training_records, is_train=True) validation_pipe = BatchSequence(kl, cfg, validation_records, is_train=False) dataset_train = training_pipe.create_tf_data().prefetch( tf.data.experimental.AUTOTUNE) dataset_validate = validation_pipe.create_tf_data().prefetch( tf.data.experimental.AUTOTUNE) train_size = len(training_pipe) val_size = len(validation_pipe) assert val_size > 0, "Not enough validation data, decrease the batch " \ "size or add more data." history = kl.train(model_path=output_path, train_data=dataset_train, train_steps=train_size, batch_size=cfg.BATCH_SIZE, validation_data=dataset_validate, validation_steps=val_size, epochs=cfg.MAX_EPOCHS, verbose=cfg.VERBOSE_TRAIN, min_delta=cfg.MIN_DELTA, patience=cfg.EARLY_STOP_PATIENCE, show_plot=cfg.SHOW_PLOT) if is_tflite: tf_lite_model_path = f'{os.path.splitext(output_path)[0]}.tflite' keras_model_to_tflite(output_path, tf_lite_model_path) database_entry = { 'Number': model_num, 'Name': model_name, 'Type': str(kl), 'Tubs': tub_paths, 'Time': time(), 'History': history.history, 'Transfer': os.path.basename(transfer) if transfer else None, 'Comment': comment, 'Config': str(cfg) } database.add_entry(database_entry) database.write() return history
def train(cfg: Config, tub_paths: str, model: str = None, model_type: str = None, transfer: str = None, comment: str = None) \ -> tf.keras.callbacks.History: """ Train the model """ database = PilotDatabase(cfg) if model_type is None: model_type = cfg.DEFAULT_MODEL_TYPE model_path, model_num = \ get_model_train_details(database, model) base_path = os.path.splitext(model_path)[0] kl = get_model_by_type(model_type, cfg) if transfer: kl.load(transfer) if cfg.PRINT_MODEL_SUMMARY: print(kl.interpreter.model.summary()) tubs = tub_paths.split(',') all_tub_paths = [os.path.expanduser(tub) for tub in tubs] dataset = TubDataset(config=cfg, tub_paths=all_tub_paths, seq_size=kl.seq_size()) training_records, validation_records \ = train_test_split(dataset.get_records(), shuffle=True, test_size=(1. - cfg.TRAIN_TEST_SPLIT)) print(f'Records # Training {len(training_records)}') print(f'Records # Validation {len(validation_records)}') # We need augmentation in validation when using crop / trapeze training_pipe = BatchSequence(kl, cfg, training_records, is_train=True) validation_pipe = BatchSequence(kl, cfg, validation_records, is_train=False) tune = tf.data.experimental.AUTOTUNE dataset_train = training_pipe.create_tf_data().prefetch(tune) dataset_validate = validation_pipe.create_tf_data().prefetch(tune) train_size = len(training_pipe) val_size = len(validation_pipe) ### training/validation length limit. Large validation datasets cause memory leaks. train_limit = cfg.TRAIN_LIMIT train_len = len(training_records) if train_limit is not None and train_len > train_limit: train_decrease = train_limit / train_len _train_size = math.ceil(train_size * train_decrease) print(f'train steps decrease from {train_size} to {_train_size}') train_size = _train_size val_limit = cfg.VALIDATION_LIMIT val_len = len(validation_records) if val_limit is not None and val_len > val_limit: val_decrease = val_limit / val_len _val_size = math.ceil(val_size * val_decrease) print(f'val steps decrease from {val_size} to {_val_size}') val_size = _val_size assert val_size > 0, "Not enough validation data, decrease the batch " \ "size or add more data." history = kl.train(model_path=model_path, train_data=dataset_train, train_steps=train_size, batch_size=cfg.BATCH_SIZE, validation_data=dataset_validate, validation_steps=val_size, epochs=cfg.MAX_EPOCHS, verbose=cfg.VERBOSE_TRAIN, min_delta=cfg.MIN_DELTA, use_early_stop=cfg.USE_EARLY_STOP, patience=cfg.EARLY_STOP_PATIENCE, show_plot=cfg.SHOW_PLOT) if getattr(cfg, 'CREATE_TF_LITE', True): tf_lite_model_path = f'{base_path}.tflite' keras_model_to_tflite(model_path, tf_lite_model_path) if getattr(cfg, 'CREATE_TENSOR_RT', False): # load h5 (ie. keras) model model_rt = load_model(model_path) # save in tensorflow savedmodel format (i.e. directory) model_rt.save(f'{base_path}.savedmodel') # pass savedmodel to the rt converter saved_model_to_tensor_rt(f'{base_path}.savedmodel', f'{base_path}.trt') database_entry = { 'Number': model_num, 'Name': os.path.basename(base_path), 'Type': str(kl), 'Tubs': tub_paths, 'Time': time(), 'History': history.history, 'Transfer': os.path.basename(transfer) if transfer else None, 'Comment': comment, 'Config': str(cfg) } database.add_entry(database_entry) database.write() return history
def auto_test(model_type, model_path, dataset): # PLAY model = get_model_by_type(model_type, cfg) model.load(model_path) # cam = PiCamera(image_w=cfg["IMAGE_W"], image_h=cfg["IMAGE_H"], image_d=cfg["IMAGE_DEPTH"], framerate=20, vflip=False, hflip=False) colors = [(0,0,255), (0,255,0), (255,0,0), (0,255,255), (255,255,0), (100,50,200), (200,100,50)] colornames = ["g-", "b-", "r-", "c-", "m-", "y-"] tries = [] for scene in scenes: env = gym.make(scene, conf=conf) print(scene) for k in range(1, 7): print(f"Round {k}") obs = env.reset() speed = 0 Xs = [] Ys = [] for t in range(200): # frame = cam.run() # action = np.array(model.run(frame)) # drive straight with small spee inference = model.run(obs) speed = speed + inference[1] action = np.array([inference[0], speed]) # print(action) # execute the action obs, reward, done, info = env.step(action) # print(info["pos"]) Xs.append(info["pos"][0]) Ys.append(info["pos"][2]) # print(obs) tries.append([scene, k, Xs, Ys]) # Exit the scene env.close() coeffs = { "donkey-generated-track-v0": [3.5, -3.5, 80, 360], "donkey-waveshare-v0": [14, 14, 270, 160], "donkey-warehouse-v0": [-3.5, 3.5, 380, 125] } lastTry = tries[0][0] img = cv2.imread(f"images/{tries[0][0]}.png") plt.figure(1) for n in range(len(tries)): inst = tries[n] if lastTry != inst[0]: cv2.imwrite(f"outputs/{model_type}/{dataset}/track_{lastTry}.png", img) plt.savefig(Path(f"outputs/{model_type}/{dataset}/pos_{lastTry}").with_suffix('.png')) plt.clf() plt.figure(1) lastTry = inst[0] img = cv2.imread(f"images/{inst[0]}.png") plt.plot(Xs, Ys, colornames[n%len(colornames)]) Xs = inst[2] Ys = inst[3] lastX = Xs[0] lastY = Ys[0] offset = coeffs[lastTry] for i in range(1, len(Xs)): rX = round(Xs[i] * offset[0] + offset[2]) rY = round(Ys[i] * offset[1] + offset[3]) cv2.circle(img, (rX, rY), 2, colors[inst[1]%10], 1) import json with open(Path(f"outputs/{model_type}/{dataset}/pos_{inst[0]}_{inst[1]}").with_suffix('.json'), 'w') as outfile: json.dump({"Xs": Xs, "Ys": Ys}, outfile) cv2.imwrite(f"outputs/{model_type}/{dataset}/track_{lastTry}.png", img) plt.savefig(Path(f"outputs/{model_type}/{dataset}/pos_{lastTry}").with_suffix('.png'))