def train(cfg, tub_names, new_model_path, base_model_path=None): """ use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name """ X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] new_model_path = os.path.expanduser(new_model_path) kl = KerasLinear() if base_model_path is not None: base_model_path = os.path.expanduser(base_model_path) kl.load(base_model_path) print('tub_names', tub_names) if not tub_names: tub_names = os.path.join(cfg.DATA_PATH, '*') tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen( X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) print(train_gen[0]) print(type(train_gen)) print(len(train_gen))
def plot_predictions(self, cfg, tub_paths, model_path): """ Plot model predictions for angle and throttle against data from tubs. """ from donkeycar.parts.datastore import TubGroup from donkeycar.parts.keras import KerasLinear #KerasCategorical import pandas as pd import matplotlib.pyplot as plt tg = TubGroup(tub_paths) model_path = os.path.expanduser(model_path) # model = KerasCategorical() model = KerasLinear() model.load(model_path) # gen = tg.get_batch_gen(None, batch_size=len(tg.df),shuffle=False, df=tg.df) # gen = tg.get_batch_gen(None, None, batch_size=len(tg.df),shuffle=False, df=tg.df) # arr = next(gen) user_angles = [] user_throttles = [] pilot_angles = [] pilot_throttles = [] for tub in tg.tubs: num_records = tub.get_num_records() for iRec in tub.get_index(shuffled=False): record = tub.get_record(iRec) img = record["cam/image_array"] user_angle = float(record["user/angle"]) user_throttle = float(record["user/throttle"]) pilot_angle, pilot_throttle = model.run(img) user_angles.append(user_angle) user_throttles.append(user_throttle) pilot_angles.append(pilot_angle) pilot_throttles.append(pilot_throttle) angles_df = pd.DataFrame({'user_angle': user_angles, 'pilot_angle': pilot_angles}) throttles_df = pd.DataFrame({'user_throttle': user_throttles, 'pilot_throttle': pilot_throttles}) fig = plt.figure() title = "Model Predictions\nTubs: {}\nModel: {}".format(tub_paths, model_path) fig.suptitle(title) ax1 = fig.add_subplot(211) ax2 = fig.add_subplot(212) angles_df.plot(ax=ax1) throttles_df.plot(ax=ax2) ax1.legend(loc=4) ax2.legend(loc=4) plt.show()
def get_model_by_type(model_type, cfg): from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN, KerasLocalizer, KerasLatent if model_type is None: model_type = "categorical" input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) roi_crop = (cfg.ROI_CROP_TOP, cfg.ROI_CROP_BOTTOM) if model_type == "localizer" or cfg.TRAIN_LOCALIZER: kl = KerasLocalizer(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), num_locations=cfg.NUM_LOCATIONS, input_shape=input_shape) elif model_type == "behavior" or cfg.TRAIN_BEHAVIORS: kl = KerasBehavioral(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), input_shape=input_shape) elif model_type == "imu": kl = KerasIMU(num_outputs=2, num_imu_inputs=6, input_shape=input_shape) elif model_type == "linear": kl = KerasLinear(input_shape=input_shape, roi_crop=roi_crop) elif model_type == "3d": kl = Keras3D_CNN(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH) elif model_type == "rnn": kl = KerasRNN_LSTM(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH) elif model_type == "categorical": kl = KerasCategorical(input_shape=input_shape, throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE, roi_crop=roi_crop) elif model_type == "latent": kl = KerasLatent(input_shape=input_shape) else: raise Exception("unknown model type: %s" % model_type) return kl
def get_model_by_type(model_type, cfg): from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN if model_type is None: model_type = "categorical" input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) if model_type == "behavior" or cfg.TRAIN_BEHAVIORS: kl = KerasBehavioral(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), input_shape=input_shape) elif model_type == "imu": kl = KerasIMU(num_outputs=2, num_imu_inputs=6, input_shape=input_shape) elif model_type == "linear": kl = KerasLinear(input_shape=input_shape) elif model_type == "3d": kl = Keras3D_CNN(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH) elif model_type == "rnn": kl = KerasRNN_LSTM(seq_length=cfg.SEQUENCE_LENGTH, input_shape=input_shape) elif model_type == "categorical": kl = KerasCategorical(input_shape=input_shape) else: raise Exception("unknown model type: %s" % model_type) return kl
def train(cfg, tub_names, model_name): ''' use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name ''' X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] #use these offsets from the current frame as points to learn the future #steering values. frames = [0, 20, 40, 120] new_y_keys = [] for iFrame in frames: for key in y_keys: new_y_keys.append(key + "_" + str(iFrame)) y_keys = new_y_keys kl = KerasLinear(num_outputs=len(y_keys)) tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT)
def get_model(model_type): if model_type == 'linear': from models import default_linear from donkeycar.parts.keras import KerasLinear return KerasLinear(model=default_linear()) elif model_type == 'linear-dropout': from models import linear_dropout from donkeycar.parts.keras import KerasLinear return KerasLinear(model=linear_dropout()) elif model_type == 'linear-cropped-dropout': from models import linear_cropped_dropout from donkeycar.parts.keras import KerasLinear return KerasLinear(model=linear_cropped_dropout()) elif model_type == 'rnn': from models import KerasRNN_LSTM return KerasRNN_LSTM() else: raise ValueError('Unknown model type')
def get_model_by_type(model_type, cfg): ''' given the string model_type and the configuration settings in cfg create a Keras model and return it. ''' from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN, KerasLocalizer, KerasLatent from donkeycar.parts.tflite import TFLitePilot if model_type is None: model_type = cfg.DEFAULT_MODEL_TYPE print("\"get_model_by_type\" model Type is: {}".format(model_type)) input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) roi_crop = (cfg.ROI_CROP_TOP, cfg.ROI_CROP_BOTTOM) if model_type == "tflite_linear": kl = TFLitePilot() elif model_type == "localizer" or cfg.TRAIN_LOCALIZER: kl = KerasLocalizer(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), num_locations=cfg.NUM_LOCATIONS, input_shape=input_shape) elif model_type == "behavior" or cfg.TRAIN_BEHAVIORS: kl = KerasBehavioral(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), input_shape=input_shape) elif model_type == "imu": kl = KerasIMU(num_outputs=2, num_imu_inputs=6, input_shape=input_shape) elif model_type == "linear": kl = KerasLinear(input_shape=input_shape, roi_crop=roi_crop) elif model_type == "tensorrt_linear": # Aggressively lazy load this. This module imports pycuda.autoinit which causes a lot of unexpected things # to happen when using TF-GPU for training. from donkeycar.parts.tensorrt import TensorRTLinear kl = TensorRTLinear(cfg=cfg) elif model_type == "3d": kl = Keras3D_CNN(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH) elif model_type == "rnn": kl = KerasRNN_LSTM(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH) elif model_type == "categorical": kl = KerasCategorical( input_shape=input_shape, throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE, roi_crop=roi_crop) elif model_type == "latent": kl = KerasLatent(input_shape=input_shape) else: raise Exception("unknown model type: %s" % model_type) return kl
def run(self, args): ''' Start a websocket SocketIO server to talk to a donkey simulator ''' import socketio from donkeycar.parts.simulation import SteeringServer from donkeycar.parts.keras import KerasCategorical, KerasLinear args, parser = self.parse_args(args) cfg = load_config(args.config) if cfg is None: return #TODO: this logic should be in a pilot or modle handler part. if args.type == "categorical": kl = KerasCategorical() elif args.type == "linear": kl = KerasLinear(num_outputs=2) elif args.type == "mxlinear": from donkeycar.parts.mxnetpart import MxnetLinear kl = MxnetLinear() else: print("didn't recognize type:", args.type) return #can provide an optional image filter part img_stack = None #load keras model kl.load(args.model) #start socket server framework sio = socketio.Server() top_speed = float(args.top_speed) #start sim server handler ss = SteeringServer(sio, kpart=kl, top_speed=top_speed, image_part=img_stack) #register events and pass to server handlers @sio.on('telemetry') def telemetry(sid, data): ss.telemetry(sid, data) @sio.on('connect') def connect(sid, environ): ss.connect(sid, environ) ss.go(('0.0.0.0', 9090))
def train(cfg, tub_names, model_name): ''' use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name ''' X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] #use these offsets from the current frame as points to learn the future #steering values. frames = [0, 20, 40, 120] new_y_keys = [] for iFrame in frames: for key in y_keys: new_y_keys.append(key + "_" + str(iFrame)) y_keys = new_y_keys kl = KerasLinear(num_outputs=len(y_keys)) tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen( X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) model_path = os.path.expanduser(model_name) total_records = len(tubgroup.df) total_train = int(total_records * cfg.TRAIN_TEST_SPLIT) total_val = total_records - total_train print('train: %d, validation: %d' % (total_train, total_val)) steps_per_epoch = total_train // cfg.BATCH_SIZE print('steps_per_epoch', steps_per_epoch) kl.train(train_gen, val_gen, saved_model_path=model_path, steps=steps_per_epoch, train_split=cfg.TRAIN_TEST_SPLIT)
def train(cfg, tub_names, new_model_path, base_model_path=None): """ use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name """ X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] new_model_path = os.path.expanduser(new_model_path) kl = KerasLinear() if base_model_path is not None: base_model_path = os.path.expanduser(base_model_path) kl.load(base_model_path) print('tub_names', tub_names) if not tub_names: tub_names = os.path.join(cfg.DATA_PATH, '*') tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) total_records = len(tubgroup.df) total_train = int(total_records * cfg.TRAIN_TEST_SPLIT) total_val = total_records - total_train print('train: %d, validation: %d' % (total_train, total_val)) steps_per_epoch = total_train // cfg.BATCH_SIZE print('steps_per_epoch', steps_per_epoch) kl.train(train_gen, val_gen, saved_model_path=new_model_path, steps=steps_per_epoch, train_split=cfg.TRAIN_TEST_SPLIT)
def train(cfg, tub_names, new_model_path, base_model_path=None): """ 引数 tub_names 似て指定されたパスに格納されている tub データを学習データとして トレーニングを行い、引数 new_model_path にて指定されたパスへ学習済みモデルファイルを格納する。 引数: cfg 個別車両設定オブジェクト、`config.py`がロードされたオブジェクト。 tub_names 学習データとして使用するtubディレクトリのパスを指定する。 new_model_path トレーニング後モデルファイルとして保管するパスを指定する。 base_model_path ファインチューニングを行う場合、ベースとなるモデルファイルを指定する。 戻り値 なし """ # モデルの入力データとなる項目 X_keys = ['cam/image_array'] # モデルの出力データとなる項目 y_keys = ['user/angle', 'user/throttle'] # トレーニング後モデルファイルとして保管するパスをフルパス化 new_model_path = os.path.expanduser(new_model_path) # トレーニング後モデルファイルとして保管するパスをフルパス化 kl = KerasLinear() # ファインチューニングを行う場合は base_model_path にベースモデルファイルパスが指定されている if base_model_path is not None: # ベースモデルファイルパスをフルパス化 base_model_path = os.path.expanduser(base_model_path) # ベースモデルファイルを読み込む kl.load(base_model_path) print('tub_names', tub_names) # 引数tub_names 指定がない場合 if not tub_names: # config.py 上に指定されたデータファイルパスを使用 tub_names = os.path.join(cfg.DATA_PATH, '*') # Tub データ群をあらわすオブジェクトを生成 tubgroup = TubGroup(tub_names) # トレーニングデータGenerator、評価データGeneratorを生成 train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) # 全学習データ件数を取得 total_records = len(tubgroup.df) # トレーニングデータ件数の取得 total_train = int(total_records * cfg.TRAIN_TEST_SPLIT) # 評価データ件数の取得 total_val = total_records - total_train print('train: %d, validation: %d' % (total_train, total_val)) # 1epochごとのステップ数の取得 steps_per_epoch = total_train // cfg.BATCH_SIZE print('steps_per_epoch', steps_per_epoch) # トレーニングの開始 kl.train(train_gen, val_gen, saved_model_path=new_model_path, steps=steps_per_epoch, train_split=cfg.TRAIN_TEST_SPLIT)
def train(cfg, tub_names, model_name): ''' use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name ''' X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] def rt(record): record['user/angle'] = dk.utils.linear_bin(record['user/angle']) return record kl = KerasLinear(num_outputs=2) print('tub_names', tub_names) if not tub_names: tub_names = os.path.join(cfg.DATA_PATH, '*') tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen( X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) model_path = os.path.expanduser(model_name) total_records = len(tubgroup.df) total_train = int(total_records * cfg.TRAIN_TEST_SPLIT) total_val = total_records - total_train print('train: %d, validation: %d' % (total_train, total_val)) steps_per_epoch = total_train // cfg.BATCH_SIZE print('steps_per_epoch', steps_per_epoch) kl.train(train_gen, val_gen, saved_model_path=model_path, steps=steps_per_epoch, train_split=cfg.TRAIN_TEST_SPLIT)
def get_model_by_type(model_type: str, cfg: 'Config') -> 'KerasPilot': ''' given the string model_type and the configuration settings in cfg create a Keras model and return it. ''' from donkeycar.parts.keras import KerasPilot, KerasLinear from donkeycar.parts.tflite import TFLitePilot if model_type is None: model_type = cfg.DEFAULT_MODEL_TYPE print("\"get_model_by_type\" model Type is: {}".format(model_type)) input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) kl: KerasPilot if model_type == "linear": kl = KerasLinear(input_shape=input_shape) else: raise Exception("Unknown model type {:}, supported types are " "linear, categorical, inferred, tflite_linear, " "tensorrt_linear".format(model_type)) return kl
def train(cfg, tub_names, new_model_path, base_model_path=None ): """ use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name """ X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] def train_record_transform(record): """ convert categorical steering to linear and apply image augmentations """ record['user/angle'] = dk.util.data.linear_bin(record['user/angle']) # TODO add augmentation that doesn't use opencv return record def val_record_transform(record): """ convert categorical steering to linear """ record['user/angle'] = dk.util.data.linear_bin(record['user/angle']) return record new_model_path = os.path.expanduser(new_model_path) kl = KerasLinear() if base_model_path is not None: base_model_path = os.path.expanduser(base_model_path) kl.load(base_model_path) print('tub_names', tub_names) if not tub_names: tub_names = os.path.join(cfg.DATA_PATH, '*') tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) total_records = len(tubgroup.df) total_train = int(total_records * cfg.TRAIN_TEST_SPLIT) total_val = total_records - total_train print('train: %d, validation: %d' % (total_train, total_val)) steps_per_epoch = total_train // cfg.BATCH_SIZE print('steps_per_epoch', steps_per_epoch) kl.train(train_gen, val_gen, saved_model_path=new_model_path, steps=steps_per_epoch, train_split=cfg.TRAIN_TEST_SPLIT)
def get_model_by_type(model_type: str, cfg: 'Config') -> 'KerasPilot': ''' given the string model_type and the configuration settings in cfg create a Keras model and return it. ''' from donkeycar.parts.keras import KerasPilot, KerasCategorical, \ KerasLinear, KerasInferred from donkeycar.parts.tflite import TFLitePilot if model_type is None: model_type = cfg.DEFAULT_MODEL_TYPE print("\"get_model_by_type\" model Type is: {}".format(model_type)) input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH) kl: KerasPilot if model_type == "linear": kl = KerasLinear(input_shape=input_shape) elif model_type == "categorical": kl = KerasCategorical( input_shape=input_shape, throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE) elif model_type == 'inferred': kl = KerasInferred(input_shape=input_shape) elif model_type == "tflite_linear": kl = TFLitePilot() elif model_type == "tensorrt_linear": # Aggressively lazy load this. This module imports pycuda.autoinit # which causes a lot of unexpected things to happen when using TF-GPU # for training. from donkeycar.parts.tensorrt import TensorRTLinear kl = TensorRTLinear(cfg=cfg) else: raise Exception("Unknown model type {:}, supported types are " "linear, categorical, inferred, tflite_linear, " "tensorrt_linear".format(model_type)) return kl
def train(cfg, tub_names, model_name, model_type): ''' use the specified data in tub_names to train an artifical neural network saves the output trained model as model_name ''' X_keys = ['cam/image_array'] y_keys = ['user/angle', 'user/throttle'] binning = dk.utils.linear_bin if model_type == "hres_cat": binning = dk.utils.linear_bin_hres def rt(record): record['user/angle'] = binning(record['user/angle']) return record kl = KerasCategorical() if model_type == 'linear': kl = KerasLinear() if model_type == 'categorical': kl = KerasCategorical() if model_type == 'hres_cat': kl = KerasHresCategorical() print('tub_names', tub_names) if not tub_names: tub_names = os.path.join(cfg.DATA_PATH, '*') tubgroup = TubGroup(tub_names) train_gen, val_gen = tubgroup.get_train_val_gen( X_keys, y_keys, record_transform=rt, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) if model_type == 'linear': train_gen, val_gen = tubgroup.get_train_val_gen( X_keys, y_keys, batch_size=cfg.BATCH_SIZE, train_frac=cfg.TRAIN_TEST_SPLIT) model_path = os.path.expanduser(model_name) total_records = len(tubgroup.df) total_train = int(total_records * cfg.TRAIN_TEST_SPLIT) total_val = total_records - total_train print('train: %d, validation: %d' % (total_train, total_val)) steps_per_epoch = total_train // cfg.BATCH_SIZE print('steps_per_epoch', steps_per_epoch) print(val_gen) history, save_best = kl.train(train_gen=train_gen, val_gen=val_gen, saved_model_path=model_path, steps=steps_per_epoch, train_split=cfg.TRAIN_TEST_SPLIT) plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('model loss : %f' % save_best.best) plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.savefig(model_path + '_' + model_type + '_loss_%f.png' % save_best.best)
from donkeycar.parts.keras import KerasLinear from PIL import Image import numpy as np input_shape = (120, 160, 3) roi_crop = (0, 0) kl = KerasLinear(input_shape=input_shape, roi_crop=roi_crop) #kl.load('/home/pi/mycar/models/mypilot20.h5') img = Image.open('./528.jpg') img = np.array(img) #print(img.shape) out = kl.run(img) print(out)
def y_translate(self, y: XY) -> Dict[str, Union[float, np.ndarray]]: """ For now only support keras linear""" return KerasLinear.y_translate(self, y)
def learn(self, total_timesteps, callback=None, log_interval=1, tb_log_name="SAC", print_freq=100, save_path=None): with TensorboardWriter(self.graph, self.tensorboard_log, tb_log_name) as writer: self._setup_learn() # Transform to callable if needed self.learning_rate = get_schedule_fn(self.learning_rate) start_time = time.time() episode_rewards = [0.0] is_teleop_env = hasattr(self.env, "wait_for_teleop_reset") # TeleopEnv if is_teleop_env: print("Waiting for teleop") obs = self.env.wait_for_teleop_reset() else: obs = self.env.reset() self.episode_reward = np.zeros((1, )) ep_info_buf = deque(maxlen=100) ep_len = 0 self.n_updates = 0 infos_values = [] mb_infos_vals = [] model_path = "--Path to model--/myNewModdel.h5" # model_path= None if model_path is not None: cfg = dk.load_config( config_path='--Path to config file inside mycar/config.py') kl = KerasLinear() kl.load(model_path) # vae = self.env.get_vae() self.training_started = False self.start_training = False for step in range(total_timesteps): # Compute current learning_rate frac = 1.0 - step / total_timesteps current_lr = self.learning_rate(frac) if callback is not None: # Only stop training if return value is False, not when it is None. This is for backwards # compatibility with callbacks that have no return statement. if callback(locals(), globals()) is False: break # Before training starts, randomly sample actions # from a uniform distribution for better exploration. # Afterwards, use the learned policy. if step < self.learning_starts and not self.training_started: if model_path is not None: try: img_arr = self.env.get_images() # print(img_arr[0].shape) img_arr = np.asarray(img_arr[0]) img_arr = normalize_and_crop(img_arr, cfg) croppedImgH = img_arr.shape[0] croppedImgW = img_arr.shape[1] if img_arr.shape[2] == 3 and cfg.IMAGE_DEPTH == 1: img_arr = dk.utils.rgb2gray(img_arr).reshape( croppedImgH, croppedImgW, 1) steering, throttle = kl.run(img_arr) action = [steering, throttle / 6.0] action = np.asarray(action) # rescaled_action = action * np.abs(self.action_space.low) rescaled_action = action print('Predicted action :', action) except Exception as e: print(e) action = self.env.action_space.sample() rescaled_action = action else: action = self.env.action_space.sample() rescaled_action = action print(action) # No need to rescale when sampling random action elif not self.training_started: self.start_training = True obs = self.env.reset() else: action = self.policy_tf.step( obs[None], deterministic=False).flatten() # Rescale from [-1, 1] to the correct bounds rescaled_action = action * np.abs(self.action_space.low) assert action.shape == self.env.action_space.shape new_obs, reward, done, info = self.env.step(rescaled_action) ep_len += 1 if print_freq > 0 and ep_len % print_freq == 0 and ep_len > 0: print("{} steps".format(ep_len)) # Store transition in the replay buffer. self.replay_buffer.add(obs, action, reward, new_obs, float(done)) obs = new_obs # Retrieve reward and episode length if using Monitor wrapper maybe_ep_info = info.get('episode') if maybe_ep_info is not None: ep_info_buf.extend([maybe_ep_info]) if writer is not None: # Write reward per episode to tensorboard ep_reward = np.array([reward]).reshape((1, -1)) ep_done = np.array([done]).reshape((1, -1)) self.episode_reward = total_episode_reward_logger( self.episode_reward, ep_reward, ep_done, writer, step) if ep_len > self.train_freq: print("Additional training") self.env.reset() mb_infos_vals = self.optimize(step, writer, current_lr) done = True episode_rewards[-1] += reward if done or self.start_training: self.start_training = False if not (isinstance(self.env, VecEnv) or is_teleop_env): obs = self.env.reset() print("Episode finished. Reward: {:.2f} {} Steps".format( episode_rewards[-1], ep_len)) episode_rewards.append(0.0) ep_len = 0 mb_infos_vals = self.optimize(step, writer, current_lr) # Refresh obs when using TeleopEnv if is_teleop_env: print("Waiting for teleop") obs = self.env.wait_for_teleop_reset() # Log losses and entropy, useful for monitor training if len(mb_infos_vals) > 0: infos_values = np.mean(mb_infos_vals, axis=0) if len(episode_rewards[-101:-1]) == 0: mean_reward = -np.inf else: mean_reward = round( float(np.mean(episode_rewards[-101:-1])), 1) num_episodes = len(episode_rewards) if self.verbose >= 1 and done and log_interval is not None and len( episode_rewards) % log_interval == 0: fps = int(step / (time.time() - start_time)) logger.logkv("episodes", num_episodes) logger.logkv("mean 100 episode reward", mean_reward) logger.logkv( 'ep_rewmean', safe_mean([ep_info['r'] for ep_info in ep_info_buf])) logger.logkv( 'eplenmean', safe_mean([ep_info['l'] for ep_info in ep_info_buf])) logger.logkv("n_updates", self.n_updates) logger.logkv("current_lr", current_lr) logger.logkv("fps", fps) logger.logkv('time_elapsed', "{:.2f}".format(time.time() - start_time)) if len(infos_values) > 0: for (name, val) in zip(self.infos_names, infos_values): logger.logkv(name, val) logger.logkv("total timesteps", step) logger.dumpkvs() # Reset infos: infos_values = [] if is_teleop_env: self.env.is_training = False # Use last batch print("Final optimization before saving") self.env.reset() mb_infos_vals = self.optimize(step, writer, current_lr) plt.figure(1) plt.plot(episode_rewards) plt.title('Episode Rewards') plt.ylabel("Reward") plt.xlabel("Epoch") filename = "training" + str(random.random()) + ".png" plt.savefig(filename) plt.show() return self
def test_linear_with_model(): kc = KerasLinear(default_linear()) assert kc.model is not None
def test_linear(): kl = KerasLinear() assert kl.model is not None
def drive(cfg, model_path=None, use_joystick=False): ''' Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. ''' #Initialisation the CAR V = dk.vehicle.Vehicle() # Initialisation of the CAMERA if cfg.OS == cfg.LINUX_OS: if cfg.CAMERA_ID == cfg.CAMERA_MACBOOK: print("Camera {} can't be used.".format(cfg.CAMERA_MACBOOK)) sys.exit() elif cfg.CAMERA_ID == cfg.CAMERA_MOCK: cam = MockCamera() else: cam = PiCamera(camera=cfg.CAMERA_ID, resolution=cfg.CAMERA_RESOLUTION) elif cfg.OS == cfg.MAC_OS: if cfg.CAMERA_ID == cfg.CAMERA_MACBOOK: cam = MacWebcam() else: cam = VideoStream(camera=cfg.CAMERA_ID, framerate=cfg.DRIVE_LOOP_HZ) V.add(cam, outputs=['cam/image_array'], threaded=True) if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: #modify max_throttle closer to 1.0 to have more power #modify steering_scale lower than 1.0 to have less responsive steering ctr = JoystickController( max_throttle=cfg.JOYSTICK_MAX_THROTTLE, steering_scale=cfg.JOYSTICK_STEERING_SCALE, auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE) else: #This web controller will create a web server that is capable #of managing steering, throttle, modes, and more. ctr = LocalWebController() if (cfg.LANE_DETECTOR): ctr_inputs = ['cam/image_overlaid_lanes', 'ld/runtime'] #Lane Detector #Detects the lane the car is in and outputs a best estimate for the 2 lines lane_detector = LanesDetector(cfg.CAMERA_ID, cfg.LD_PARAMETERS) V.add(lane_detector, inputs=['cam/image_array', 'algorithm/reset'], outputs=['cam/image_overlaid_lanes', 'ld/runtime']) else: ctr_inputs = ['cam/image_array'] V.add( ctr, ctr_inputs, outputs=[ 'user/angle', 'user/throttle', 'user/mode', 'recording' #'algorithm/reset' ], threaded=True) #See if we should even run the pilot module. #This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) #Run the pilot if the mode is not user. kl = KerasLinear() if model_path: kl.load(model_path) V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') #Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) if cfg.OS == cfg.LINUX_OS: steering_controller = PCA9685(cfg.STEERING_CHANNEL) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL) elif cfg.OS == cfg.MAC_OS: steering_controller = MockController() throttle_controller = MockController() steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, center_pulse=cfg.STEERING_CENTER_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) throttle = PWMThrottle( controller=throttle_controller, cal_max_pulse=cfg.THROTTLE_FORWARD_CAL_PWM, max_pulse=cfg.THROTTLE_FORWARD_PWM, cal_min_pulse=cfg.THROTTLE_REVERSE_CAL_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, ) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) #add tub to save data inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode'] types = ['image_array', 'float', 'float', 'str'] th = TubHandler(path=cfg.DATA_PATH) tub = th.new_tub_writer(inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') print("You can now go to <your pi ip address>:8887 to drive your car.") #run the vehicle for 20 seconds V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_joystick=False, use_chaos=False): """ (手動・自動)運転する。 多くの部品(part)から作業用のロボット車両を構築する。 各partはVehicleループ内のジョブとして実行され、コンストラクタフラグ `threaded`に応じて `run` メソッドまたは `run_threaded` メソッドを呼び出す。 すべてのパーツは、 `cfg.DRIVE_LOOP_HZ` で指定されたフレームレートで順次更新され、 各partが適時に処理を終了すると仮定する。 partには名前付きの出力と入力が存在する(どちらかがない場合や複数存在する場合もある)。 Vehicle フレームワークは、名前付き出力を同じ名前の入力を要求するpartに渡すことを処理する。 引数 cfg 個別車両設定オブジェクト、`config.py`がロードされたオブジェクト。 model_path 自動運転時のモデルファイルパスを指定する(デフォルトはNone)。 use_joystick ジョイスティックを使用するかどうかの真偽値(デフォルトはFalse)。 use_chaos 手動運転中に周期的なランダム操舵を加えるかどうかの真偽値(デフォルトはFalse)。 """ # Vehicle オブジェクトの生成 V = dk.vehicle.Vehicle() # Timestamp part の生成 clock = Timestamp() # Timestamp part をVehicleループへ追加 # 入力: # なし # 出力: # 'timestamp' 現在時刻 V.add(clock, outputs=['timestamp']) # PiCamera part の生成 cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION) # 別スレッド実行される PiCamera part をVehicleループへ追加 # 入力: # なし # 出力: # 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ V.add(cam, outputs=['cam/image_array'], threaded=True) # manage.py デフォルトのジョイスティックpart生成 if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: #ctr = JoystickController(max_throttle=cfg.JOYSTICK_MAX_THROTTLE, # steering_scale=cfg.JOYSTICK_STEERING_SCALE, # throttle_axis=cfg.JOYSTICK_THROTTLE_AXIS, # auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE) # ジョイスティック part の生成 from elecom.part import JoystickController ctr = JoystickController(config_path='elecom/jc-u3912t.yml') else: # ステアリング、スロットル、モードなどを管理するWebサーバを作成する # Web Controller part の生成 ctr = LocalWebController(use_chaos=use_chaos) # 別スレッド実行される Web Controller part もしくはジョイスティック part をVehiecleループへ追加 # 入力: # 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ # 出力: # 'user/angle' Web/Joystickにより手動指定した次に取るべきステアリング値 # 'user/throttle' Web/Joystickにより手動指定した次に取るべきスロットル値 # 'user/mode' Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま) # 'recording' tubデータとして保管するかどうかの真偽値 V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) # オートパイロットモジュールを実行すべきかどうかを確認する関数を定義する。 def pilot_condition(mode): ''' オートパイロットモジュール実行判定関数。 引数で指定されたモードを判別して、オートパイロットモジュールを実行するべきかどうか 真偽値を返却する関数。 引数 mode Userモード('user':全手動、'local_angle':操舵のみ自動、'local':全自動) 戻り値 boolean オートパイロットモジュールを実行するかどうかの真偽値 ''' print('mode=' + mode) if mode == 'user': # 全手動時のみ実行しない return False else: return True # オートパイロットモジュール実行判定関数を part 化したオブジェクトを生成 pilot_condition_part = Lambda(pilot_condition) # オートパイロットモジュール実行判定 part を Vehiecle ループへ追加 # 入力: # 'user/mode' Userモード('user':全手動、'local_angle':操舵のみ自動、'local':全自動) # 出力: # 'run_pilot' オートパイロットモジュールを実行するかどうかの真偽値 V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) # Userモードでない場合、オートパイロットを実行する # CNNベースの線形回帰モデル(オートパイロット) part を生成する。 kl = KerasLinear() # 関数driveの引数 model_part 指定がある場合 if model_path: # 学習済みモデルファイルを読み込む kl.load(model_path) # run_condition が真の場合のみ実行されるオートパイロット part をVehicleループへ追加する # 入力: # 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ # 出力: # 'pilot/angle' オートパイロットが指定した次に取るべきステアリング値 # 'pilot/throttle' オートパイロットが指定した次に取るべきスロットル値 V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') # 車両にどの値を入力にするかを判別する def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): ''' 引数で指定された項目から、車両への入力とするステアリング値、スロットル値を確定する関数。 引数 mode Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま) user_angle Web/Joystickにより手動指定した次に取るべきステアリング値 user_throttle Web/Joystickにより手動指定した次に取るべきスロットル値 pilot_angle オートパイロットが指定した次に取るべきステアリング値 pilot_throttle オートパイロットが指定した次に取るべきスロットル値 戻り値 angle 車両への入力とするステアリング値 throttle 車両への入力とするスロットル値 ''' if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle # 車両にどの値を入力にするかを判別する関数を part 化したオブジェクトを生成 drive_mode_part = Lambda(drive_mode) # 車両にどの値を入力にするかを判別する part を Vehicle ループへ追加 # 入力 # 'user/mode' Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま) # 'user/angle' Web/Joystickにより手動指定した次に取るべきステアリング値 # 'user/throttle' Web/Joystickにより手動指定した次に取るべきスロットル値 # 'pilot/angle' オートパイロットが指定した次に取るべきステアリング値 # 'pilot/throttle' オートパイロットが指定した次に取るべきスロットル値 # 戻り値 # 'angle' 車両への入力とするステアリング値 # 'throttle' 車両への入力とするスロットル値 V.add(drive_mode_part, inputs=['user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle'], outputs=['angle', 'throttle']) # 実車両のステアリングサーボを操作するオブジェクトを生成 steering_controller = PCA9685(cfg.STEERING_CHANNEL) # 実車両へステアリング値を指示する part を生成 steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) # 実車両のスロットルECSを操作するオブジェクトを生成 throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL) # 実車両へスロットル値を指示する part を生成 throttle = PWMThrottle(controller=throttle_controller, max_pulse=cfg.THROTTLE_FORWARD_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM) # 実車両へステアリング値を指示する part を Vehiecle ループへ追加 # 入力: # 'angle' 車両への入力とするステアリング値 # 出力: # なし(実車両の操舵へ) V.add(steering, inputs=['angle']) # 実車両へスロットル値を指示する part を Vehiecle ループへ追加 # 入力: # 'throttle' 車両への入力とするスロットル値 # 出力: # なし(実車両のスロットル操作へ) V.add(throttle, inputs=['throttle']) # 保存データを tub ディレクトリに追加 inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'timestamp'] types = ['image_array', 'float', 'float', 'str', 'str'] # 複数 tub ディレクトリの場合 # th = TubHandler(path=cfg.DATA_PATH) # tub = th.new_tub_writer(inputs=inputs, types=types) # 単一 tub ディレクトリの場合 # tub ディレクトリへ書き込む part を生成 tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types) # 'recording'が正であれば tub ディレクトリへ書き込む part を Vehiecle ループへ追加 # 入力 # 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ # 'user/angle' Web/Joystickにより手動指定した次に取るべきステアリング値 # 'user/throttle' Web/Joystickにより手動指定した次に取るべきスロットル値 # 'user/mode' Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま) # 'timestamp' 現在時刻 V.add(tub, inputs=inputs, run_condition='recording') # テレメトリーデータの送信 #tele = PubTelemetry('iotf/emperor.ini', pub_count=20*5) #V.add(tele, inputs=['cam/image_array', 'user/mode', 'user/angle', 'user/throttle', # 'pilot/angle', 'pilot/throttle', 'angle', 'throttle']) # Vehicle ループを開始 V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, model_type='categorical', use_joystick=False): ''' Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. ''' #Initialize car V = dk.vehicle.Vehicle() cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION) V.add(cam, outputs=['cam/image_array'], threaded=True) if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: #modify max_throttle closer to 1.0 to have more power #modify steering_scale lower than 1.0 to have less responsive steering ctr = JoystickController( max_throttle=cfg.JOYSTICK_MAX_THROTTLE, steering_scale=cfg.JOYSTICK_STEERING_SCALE, auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE) else: #This web controller will create a web server that is capable #of managing steering, throttle, and modes, and more. ctr = LocalWebController() V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) #See if we should even run the pilot module. #This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) #Run the pilot if the mode is not user. kl = KerasCategorical() if model_type == 'linear': kl = KerasLinear() if model_type == 'hres_cat': kl = KerasHresCategorical() # Change model type accordingly if (model_type == 'rnn'): kl = KerasRNN_LSTM() if (model_type == 'rnn_bin'): kl = KerasRNN_Categorical() if model_path: #kl.load(model_path) #kl = dk.utils.get_model_by_type(model_type, cfg) kl.load(model_path) V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') #Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) steering_controller = PCA9685(cfg.STEERING_CHANNEL) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL) throttle = PWMThrottle(controller=throttle_controller, max_pulse=cfg.THROTTLE_FORWARD_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) #add tub to save data inputs = [ 'cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'pilot/angle', 'pilot/throttle' ] types = ['image_array', 'float', 'float', 'str', 'float', 'float'] th = TubHandler(path=cfg.DATA_PATH) tub = th.new_tub_writer(inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') #run the vehicle V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS) print("You can now go to <your pi ip address>:8887 to drive your car.")
def drive(cfg, model_path=None): V = dk.vehicle.Vehicle() V.mem.put(['square/angle', 'square/throttle'], (100, 100)) # display square box given by cooridantes. cam = SquareBoxCamera(resolution=cfg.CAMERA_RESOLUTION) V.add(cam, inputs=['square/angle', 'square/throttle'], outputs=['cam/image_array']) # display the image and read user values from a local web controller ctr = LocalWebController() V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) # See if we should even run the pilot module. # This is only needed because the part run_contion only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) # Run the pilot if the mode is not user. kl = KerasLinear() if model_path: kl.load(model_path) V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') # See if we should even run the pilot module. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'pilot_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) clock = Timestamp() V.add(clock, outputs=['timestamp']) # transform angle and throttle values to coordinate values def f(x): return int(x * 100 + 100) l = Lambda(f) V.add(l, inputs=['user/angle'], outputs=['square/angle']) V.add(l, inputs=['user/throttle'], outputs=['square/throttle']) # add tub to save data inputs = [ 'cam/image_array', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle', 'square/angle', 'square/throttle', 'user/mode', 'timestamp' ] types = [ 'image_array', 'float', 'float', 'float', 'float', 'float', 'float', 'str', 'str' ] #multiple tubs #th = TubHandler(path=cfg.DATA_PATH) #tub = th.new_tub_writer(inputs=inputs, types=types) # single tub tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') # run the vehicle for 20 seconds V.start(rate_hz=50, max_loop_count=10000)
def drive(cfg, model_path=None, use_chaos=False): """ Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. """ V = dk.vehicle.Vehicle() clock = Timestamp() V.add(clock, outputs=['timestamp']) cam = JevoisCamera(camera_id=0) V.add(cam, outputs=['cam/image_array'], threaded=True) ctr = XboxGameController(device_search_term='xbox') V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) # See if we should even run the pilot module. # This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) # Run the pilot if the mode is not user. kl = KerasLinear() if model_path: kl.load(model_path) V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') # Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) steering_controller = PCA9685(cfg.STEERING_CHANNEL) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL) throttle = PWMThrottle(controller=throttle_controller, max_pulse=cfg.THROTTLE_FORWARD_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) # add tub to save data inputs = [ 'cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'timestamp' ] types = ['image_array', 'float', 'float', 'str', 'str'] # multiple tubs # th = TubHandler(path=cfg.DATA_PATH) # tub = th.new_tub_writer(inputs=inputs, types=types) # single tub tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') # run the vehicle V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_joystick=False, use_chaos=False): """ Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. """ V = dk.vehicle.Vehicle() clock = Timestamp() V.add(clock, outputs=['timestamp']) cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION) V.add(cam, outputs=['cam/image_array'], threaded=True) if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: ctr = JoystickController( max_throttle=cfg.JOYSTICK_MAX_THROTTLE, steering_scale=cfg.JOYSTICK_STEERING_SCALE, throttle_axis=cfg.JOYSTICK_THROTTLE_AXIS, auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE) else: # This web controller will create a web server that is capable # of managing steering, throttle, and modes, and more. ctr = LocalWebController(use_chaos=use_chaos) V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) # See if we should even run the pilot module. # This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) # Run the pilot if the mode is not user. kl = KerasLinear() if model_path: kl.load(model_path) V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') # Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) steering_controller = PCA9685(cfg.STEERING_CHANNEL) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL) throttle = PWMThrottle(controller=throttle_controller, max_pulse=cfg.THROTTLE_FORWARD_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) # add tub to save data inputs = [ 'cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'timestamp' ] types = ['image_array', 'float', 'float', 'str', 'str'] # multiple tubs # th = TubHandler(path=cfg.DATA_PATH) # tub = th.new_tub_writer(inputs=inputs, types=types) # single tub tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') # run the vehicle V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive_vis(cfg, model_path=None, use_chaos=False): V = dk.vehicle.Vehicle() clock = Timestamp() V.add(clock, outputs=['timestamp']) cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION) V.add(cam, outputs=['cam/image_array'], threaded=True) # Run the pilot if the mode is not user. kl = KerasLinear() if model_path: kl.load(model_path) top_view_transform = TopViewTransform(cfg.CAMERA_RESOLUTION) V.add(Lambda(top_view_transform.wrap), inputs=['cam/image_array'], outputs=['cam/image_array_proj']) V.add(kl, inputs=['cam/image_array_proj'], outputs=['pilot/angle', 'pilot/throttle']) ctr = LocalWebControllerVis(use_chaos=use_chaos) V.add(ctr, inputs=['cam/image_array_proj', 'pilot/angle', 'pilot/throttle'], outputs=['user/mode', 'recording'], threaded=True) # See if we should even run the pilot module. # This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True # Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) driver = ArduinoDriver() V.add(driver, inputs=['user/mode', 'pilot/angle', 'pilot/throttle'], outputs=['user/angle', 'user/throttle']) # add tub to save data inputs = [ 'cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'timestamp' ] types = ['image_array', 'float', 'float', 'str', 'str'] # multiple tubs # th = TubHandler(path=cfg.DATA_PATH) # tub = th.new_tub_writer(inputs=inputs, types=types) # single tub tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') # run the vehicle V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_chaos=False): """ Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. """ V = dk.vehicle.Vehicle() clock = Timestamp() V.add(clock, outputs=['timestamp']) cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION) V.add(cam, outputs=['cam/image_array'], threaded=True) ctr = LocalWebController(use_chaos=use_chaos) V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) # See if we should even run the pilot module. # This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) # Run the pilot if the mode is not user. kl = KerasLinear() if model_path: kl.load(model_path) V.add(kl, inputs=['cam/image_array'], outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') driver = ArduinoDriver() V.add(driver, inputs=['user/mode', 'pilot/angle', 'pilot/throttle'], outputs=['user/angle', 'user/throttle']) # add tub to save data inputs = [ 'cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'timestamp' ] types = ['image_array', 'float', 'float', 'str', 'str'] # multiple tubs # th = TubHandler(path=cfg.DATA_PATH) # tub = th.new_tub_writer(inputs=inputs, types=types) # single tub tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types) V.add(tub, inputs=inputs, run_condition='recording') # run the vehicle V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type='single'): ''' Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. ''' if model_type is None: model_type = "categorical" stereo_cam = camera_type == "stereo" #Initialize car V = dk.vehicle.Vehicle() if stereo_cam: from donkeycar.parts.camera import Webcam camA = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, iCam=0) V.add(camA, outputs=['cam/image_array_a'], threaded=True) camB = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, iCam=1) V.add(camB, outputs=['cam/image_array_b'], threaded=True) def stereo_pair(image_a, image_b): if image_a is not None and image_b is not None: width, height, _ = image_a.shape grey_a = dk.utils.rgb2gray(image_a) grey_b = dk.utils.rgb2gray(image_b) # Added by Felix depth = capture.stereo_depth(image_a, image_b, leftMapX, leftMapY, rightMapX, rightMapY, leftROI, rightROI, imageSize) stereo_image = np.zeros([width, height, 3], dtype=np.dtype('B')) stereo_image[..., 0] = np.reshape(grey_a, (width, height)) stereo_image[..., 1] = np.reshape(grey_b, (width, height)) stereo_image[..., 2] = np.reshape(depth, (width, height)) # -------------- else: stereo_image = [] return np.array(stereo_image) image_sterero_pair_part = Lambda(stereo_pair) V.add(image_sterero_pair_part, inputs=['cam/image_array_a', 'cam/image_array_b'], outputs=['cam/image_array']) else: print("cfg.CAMERA_TYPE", cfg.CAMERA_TYPE) if cfg.CAMERA_TYPE == "PICAM": from donkeycar.parts.camera import PiCamera cam = PiCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H) elif cfg.CAMERA_TYPE == "WEBCAM": from donkeycar.parts.camera import Webcam cam = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H) V.add(cam, outputs=['cam/image_array'], threaded=True) if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: #modify max_throttle closer to 1.0 to have more power #modify steering_scale lower than 1.0 to have less responsive steering ctr = JoystickController( throttle_scale=cfg.JOYSTICK_MAX_THROTTLE, steering_scale=cfg.JOYSTICK_STEERING_SCALE, auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE) else: #This web controller will create a web server that is capable #of managing steering, throttle, and modes, and more. ctr = LocalWebController() V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) #this throttle filter will allow one tap back for esc reverse th_filter = ThrottleFilter() V.add(th_filter, inputs=['user/throttle'], outputs=['user/throttle']) #See if we should even run the pilot module. #This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) def led_cond(mode, recording, num_records, behavior_state): ''' returns a blink rate. 0 for off. -1 for on. positive for rate. ''' if num_records is not None and num_records % 10 == 0: print("recorded", num_records, "records") if behavior_state is not None and model_type == 'behavior': r, g, b = cfg.BEHAVIOR_LED_COLORS[behavior_state] led.set_rgb(r, g, b) return -1 #solid on if recording: return -1 #solid on elif mode == 'user': return 1 elif mode == 'local_angle': return 0.5 elif mode == 'local': return 0.1 return 0 led_cond_part = Lambda(led_cond) V.add( led_cond_part, inputs=['user/mode', 'recording', "tub/num_records", 'behavior/state'], outputs=['led/blink_rate']) if cfg.HAVE_RGB_LED: from donkeycar.parts.led_status import RGB_LED led = RGB_LED(cfg.LED_PIN_R, cfg.LED_PIN_G, cfg.LED_PIN_B, cfg.LED_INVERT) led.set_rgb(cfg.LED_R, cfg.LED_G, cfg.LED_B) V.add(led, inputs=['led/blink_rate']) #IMU if cfg.HAVE_IMU: imu = Mpu6050() V.add(imu, outputs=[ 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z' ], threaded=True) #Behavioral state if model_type == "behavior": bh = BehaviorPart(cfg.BEHAVIOR_LIST) V.add(bh, outputs=[ 'behavior/state', 'behavior/label', "behavior/one_hot_state_array" ]) try: ctr.set_button_down_trigger('L1', bh.increment_state) except: pass kl = KerasBehavioral(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST)) inputs = ['cam/image_array', "behavior/one_hot_state_array"] #IMU elif model_type == "imu": assert (cfg.HAVE_IMU) #Run the pilot if the mode is not user. kl = KerasIMU(num_outputs=2, num_imu_inputs=6) inputs = [ 'cam/image_array', 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z' ] else: if model_type == "linear": kl = KerasLinear() elif model_type == "3d": kl = Keras3D_CNN(seq_length=cfg.SEQUENCE_LENGTH) elif model_type == "rnn": kl = KerasRNN_LSTM(seq_length=cfg.SEQUENCE_LENGTH) else: kl = KerasCategorical() inputs = ['cam/image_array'] if model_path: kl.load(model_path) V.add(kl, inputs=inputs, outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') #Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) steering_controller = PCA9685(cfg.STEERING_CHANNEL, cfg.PCA9685_I2C_ADDR, busnum=cfg.PCA9685_I2C_BUSNUM) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL, cfg.PCA9685_I2C_ADDR, busnum=cfg.PCA9685_I2C_BUSNUM) throttle = PWMThrottle(controller=throttle_controller, max_pulse=cfg.THROTTLE_FORWARD_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) #add tub to save data inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode'] types = ['image_array', 'float', 'float', 'str'] if cfg.TRAIN_BEHAVIORS: inputs += [ 'behavior/state', 'behavior/label', "behavior/one_hot_state_array" ] types += ['int', 'str', 'vector'] if cfg.HAVE_IMU: inputs += [ 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z' ] types += ['float', 'float', 'float', 'float', 'float', 'float'] th = TubHandler(path=cfg.DATA_PATH) tub = th.new_tub_writer(inputs=inputs, types=types) V.add(tub, inputs=inputs, outputs=["tub/num_records"], run_condition='recording') if type(ctr) is LocalWebController: print("You can now go to <your pi ip address>:8887 to drive your car.") elif type(ctr) is JoystickController: print("You can now move your joystick to drive your car.") #tell the controller about the tub ctr.set_tub(tub) #run the vehicle for 20 seconds V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)