def train(cfg, tub_names, new_model_path, base_model_path=None):
"""
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
"""
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
new_model_path = os.path.expanduser(new_model_path)
kl = KerasLinear()
if base_model_path is not None:
base_model_path = os.path.expanduser(base_model_path)
kl.load(base_model_path)
print('tub_names', tub_names)
if not tub_names:
tub_names = os.path.join(cfg.DATA_PATH, '*')
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(
X_keys,
y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
print(train_gen[0])
print(type(train_gen))
print(len(train_gen))
def plot_predictions(self, cfg, tub_paths, model_path):
"""
Plot model predictions for angle and throttle against data from tubs.
"""
from donkeycar.parts.datastore import TubGroup
from donkeycar.parts.keras import KerasLinear #KerasCategorical
import pandas as pd
import matplotlib.pyplot as plt
tg = TubGroup(tub_paths)
model_path = os.path.expanduser(model_path)
# model = KerasCategorical()
model = KerasLinear()
model.load(model_path)
# gen = tg.get_batch_gen(None, batch_size=len(tg.df),shuffle=False, df=tg.df)
# gen = tg.get_batch_gen(None, None, batch_size=len(tg.df),shuffle=False, df=tg.df)
# arr = next(gen)
user_angles = []
user_throttles = []
pilot_angles = []
pilot_throttles = []
for tub in tg.tubs:
num_records = tub.get_num_records()
for iRec in tub.get_index(shuffled=False):
record = tub.get_record(iRec)
img = record["cam/image_array"]
user_angle = float(record["user/angle"])
user_throttle = float(record["user/throttle"])
pilot_angle, pilot_throttle = model.run(img)
user_angles.append(user_angle)
user_throttles.append(user_throttle)
pilot_angles.append(pilot_angle)
pilot_throttles.append(pilot_throttle)
angles_df = pd.DataFrame({'user_angle': user_angles, 'pilot_angle': pilot_angles})
throttles_df = pd.DataFrame({'user_throttle': user_throttles, 'pilot_throttle': pilot_throttles})
fig = plt.figure()
title = "Model Predictions\nTubs: {}\nModel: {}".format(tub_paths, model_path)
fig.suptitle(title)
ax1 = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
angles_df.plot(ax=ax1)
throttles_df.plot(ax=ax2)
ax1.legend(loc=4)
ax2.legend(loc=4)
plt.show()
def get_model_by_type(model_type, cfg):
from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN, KerasLocalizer, KerasLatent
if model_type is None:
model_type = "categorical"
input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)
roi_crop = (cfg.ROI_CROP_TOP, cfg.ROI_CROP_BOTTOM)
if model_type == "localizer" or cfg.TRAIN_LOCALIZER:
kl = KerasLocalizer(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), num_locations=cfg.NUM_LOCATIONS, input_shape=input_shape)
elif model_type == "behavior" or cfg.TRAIN_BEHAVIORS:
kl = KerasBehavioral(num_outputs=2, num_behavior_inputs=len(cfg.BEHAVIOR_LIST), input_shape=input_shape)
elif model_type == "imu":
kl = KerasIMU(num_outputs=2, num_imu_inputs=6, input_shape=input_shape)
elif model_type == "linear":
kl = KerasLinear(input_shape=input_shape, roi_crop=roi_crop)
elif model_type == "3d":
kl = Keras3D_CNN(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH)
elif model_type == "rnn":
kl = KerasRNN_LSTM(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, seq_length=cfg.SEQUENCE_LENGTH)
elif model_type == "categorical":
kl = KerasCategorical(input_shape=input_shape, throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE, roi_crop=roi_crop)
elif model_type == "latent":
kl = KerasLatent(input_shape=input_shape)
else:
raise Exception("unknown model type: %s" % model_type)
return kl
def get_model_by_type(model_type, cfg):
from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN
if model_type is None:
model_type = "categorical"
input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)
if model_type == "behavior" or cfg.TRAIN_BEHAVIORS:
kl = KerasBehavioral(num_outputs=2,
num_behavior_inputs=len(cfg.BEHAVIOR_LIST),
input_shape=input_shape)
elif model_type == "imu":
kl = KerasIMU(num_outputs=2, num_imu_inputs=6, input_shape=input_shape)
elif model_type == "linear":
kl = KerasLinear(input_shape=input_shape)
elif model_type == "3d":
kl = Keras3D_CNN(image_w=cfg.IMAGE_W,
image_h=cfg.IMAGE_H,
image_d=cfg.IMAGE_DEPTH,
seq_length=cfg.SEQUENCE_LENGTH)
elif model_type == "rnn":
kl = KerasRNN_LSTM(seq_length=cfg.SEQUENCE_LENGTH,
input_shape=input_shape)
elif model_type == "categorical":
kl = KerasCategorical(input_shape=input_shape)
else:
raise Exception("unknown model type: %s" % model_type)
return kl
def train(cfg, tub_names, model_name):
'''
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
'''
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
#use these offsets from the current frame as points to learn the future
#steering values.
frames = [0, 20, 40, 120]
new_y_keys = []
for iFrame in frames:
for key in y_keys:
new_y_keys.append(key + "_" + str(iFrame))
y_keys = new_y_keys
kl = KerasLinear(num_outputs=len(y_keys))
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
def get_model(model_type):
if model_type == 'linear':
from models import default_linear
from donkeycar.parts.keras import KerasLinear
return KerasLinear(model=default_linear())
elif model_type == 'linear-dropout':
from models import linear_dropout
from donkeycar.parts.keras import KerasLinear
return KerasLinear(model=linear_dropout())
elif model_type == 'linear-cropped-dropout':
from models import linear_cropped_dropout
from donkeycar.parts.keras import KerasLinear
return KerasLinear(model=linear_cropped_dropout())
elif model_type == 'rnn':
from models import KerasRNN_LSTM
return KerasRNN_LSTM()
else:
raise ValueError('Unknown model type')
def get_model_by_type(model_type, cfg):
'''
given the string model_type and the configuration settings in cfg
create a Keras model and return it.
'''
from donkeycar.parts.keras import KerasRNN_LSTM, KerasBehavioral, KerasCategorical, KerasIMU, KerasLinear, Keras3D_CNN, KerasLocalizer, KerasLatent
from donkeycar.parts.tflite import TFLitePilot
if model_type is None:
model_type = cfg.DEFAULT_MODEL_TYPE
print("\"get_model_by_type\" model Type is: {}".format(model_type))
input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)
roi_crop = (cfg.ROI_CROP_TOP, cfg.ROI_CROP_BOTTOM)
if model_type == "tflite_linear":
kl = TFLitePilot()
elif model_type == "localizer" or cfg.TRAIN_LOCALIZER:
kl = KerasLocalizer(num_outputs=2,
num_behavior_inputs=len(cfg.BEHAVIOR_LIST),
num_locations=cfg.NUM_LOCATIONS,
input_shape=input_shape)
elif model_type == "behavior" or cfg.TRAIN_BEHAVIORS:
kl = KerasBehavioral(num_outputs=2,
num_behavior_inputs=len(cfg.BEHAVIOR_LIST),
input_shape=input_shape)
elif model_type == "imu":
kl = KerasIMU(num_outputs=2, num_imu_inputs=6, input_shape=input_shape)
elif model_type == "linear":
kl = KerasLinear(input_shape=input_shape, roi_crop=roi_crop)
elif model_type == "tensorrt_linear":
# Aggressively lazy load this. This module imports pycuda.autoinit which causes a lot of unexpected things
# to happen when using TF-GPU for training.
from donkeycar.parts.tensorrt import TensorRTLinear
kl = TensorRTLinear(cfg=cfg)
elif model_type == "3d":
kl = Keras3D_CNN(image_w=cfg.IMAGE_W,
image_h=cfg.IMAGE_H,
image_d=cfg.IMAGE_DEPTH,
seq_length=cfg.SEQUENCE_LENGTH)
elif model_type == "rnn":
kl = KerasRNN_LSTM(image_w=cfg.IMAGE_W,
image_h=cfg.IMAGE_H,
image_d=cfg.IMAGE_DEPTH,
seq_length=cfg.SEQUENCE_LENGTH)
elif model_type == "categorical":
kl = KerasCategorical(
input_shape=input_shape,
throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE,
roi_crop=roi_crop)
elif model_type == "latent":
kl = KerasLatent(input_shape=input_shape)
else:
raise Exception("unknown model type: %s" % model_type)
return kl
def run(self, args):
'''
Start a websocket SocketIO server to talk to a donkey simulator
'''
import socketio
from donkeycar.parts.simulation import SteeringServer
from donkeycar.parts.keras import KerasCategorical, KerasLinear
args, parser = self.parse_args(args)
cfg = load_config(args.config)
if cfg is None:
return
#TODO: this logic should be in a pilot or modle handler part.
if args.type == "categorical":
kl = KerasCategorical()
elif args.type == "linear":
kl = KerasLinear(num_outputs=2)
elif args.type == "mxlinear":
from donkeycar.parts.mxnetpart import MxnetLinear
kl = MxnetLinear()
else:
print("didn't recognize type:", args.type)
return
#can provide an optional image filter part
img_stack = None
#load keras model
kl.load(args.model)
#start socket server framework
sio = socketio.Server()
top_speed = float(args.top_speed)
#start sim server handler
ss = SteeringServer(sio,
kpart=kl,
top_speed=top_speed,
image_part=img_stack)
#register events and pass to server handlers
@sio.on('telemetry')
def telemetry(sid, data):
ss.telemetry(sid, data)
@sio.on('connect')
def connect(sid, environ):
ss.connect(sid, environ)
ss.go(('0.0.0.0', 9090))
def train(cfg, tub_names, model_name):
'''
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
'''
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
#use these offsets from the current frame as points to learn the future
#steering values.
frames = [0, 20, 40, 120]
new_y_keys = []
for iFrame in frames:
for key in y_keys:
new_y_keys.append(key + "_" + str(iFrame))
y_keys = new_y_keys
kl = KerasLinear(num_outputs=len(y_keys))
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(
X_keys,
y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
model_path = os.path.expanduser(model_name)
total_records = len(tubgroup.df)
total_train = int(total_records * cfg.TRAIN_TEST_SPLIT)
total_val = total_records - total_train
print('train: %d, validation: %d' % (total_train, total_val))
steps_per_epoch = total_train // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
kl.train(train_gen,
val_gen,
saved_model_path=model_path,
steps=steps_per_epoch,
train_split=cfg.TRAIN_TEST_SPLIT)
def train(cfg, tub_names, new_model_path, base_model_path=None):
"""
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
"""
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
new_model_path = os.path.expanduser(new_model_path)
kl = KerasLinear()
if base_model_path is not None:
base_model_path = os.path.expanduser(base_model_path)
kl.load(base_model_path)
print('tub_names', tub_names)
if not tub_names:
tub_names = os.path.join(cfg.DATA_PATH, '*')
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
total_records = len(tubgroup.df)
total_train = int(total_records * cfg.TRAIN_TEST_SPLIT)
total_val = total_records - total_train
print('train: %d, validation: %d' % (total_train, total_val))
steps_per_epoch = total_train // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
kl.train(train_gen,
val_gen,
saved_model_path=new_model_path,
steps=steps_per_epoch,
train_split=cfg.TRAIN_TEST_SPLIT)
def train(cfg, tub_names, new_model_path, base_model_path=None):
"""
引数 tub_names 似て指定されたパスに格納されている tub データを学習データとして
トレーニングを行い、引数 new_model_path にて指定されたパスへ学習済みモデルファイルを格納する。
引数：
cfg 個別車両設定オブジェクト、`config.py`がロードされたオブジェクト。
tub_names 学習データとして使用するtubディレクトリのパスを指定する。
new_model_path トレーニング後モデルファイルとして保管するパスを指定する。
base_model_path ファインチューニングを行う場合、ベースとなるモデルファイルを指定する。
戻り値
なし
"""
# モデルの入力データとなる項目
X_keys = ['cam/image_array']
# モデルの出力データとなる項目
y_keys = ['user/angle', 'user/throttle']
# トレーニング後モデルファイルとして保管するパスをフルパス化
new_model_path = os.path.expanduser(new_model_path)
# トレーニング後モデルファイルとして保管するパスをフルパス化
kl = KerasLinear()
# ファインチューニングを行う場合は base_model_path にベースモデルファイルパスが指定されている
if base_model_path is not None:
# ベースモデルファイルパスをフルパス化
base_model_path = os.path.expanduser(base_model_path)
# ベースモデルファイルを読み込む
kl.load(base_model_path)
print('tub_names', tub_names)
# 引数tub_names 指定がない場合
if not tub_names:
# config.py 上に指定されたデータファイルパスを使用
tub_names = os.path.join(cfg.DATA_PATH, '*')
# Tub データ群をあらわすオブジェクトを生成
tubgroup = TubGroup(tub_names)
# トレーニングデータGenerator、評価データGeneratorを生成
train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
# 全学習データ件数を取得
total_records = len(tubgroup.df)
# トレーニングデータ件数の取得
total_train = int(total_records * cfg.TRAIN_TEST_SPLIT)
# 評価データ件数の取得
total_val = total_records - total_train
print('train: %d, validation: %d' % (total_train, total_val))
# 1epochごとのステップ数の取得
steps_per_epoch = total_train // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
# トレーニングの開始
kl.train(train_gen,
val_gen,
saved_model_path=new_model_path,
steps=steps_per_epoch,
train_split=cfg.TRAIN_TEST_SPLIT)
def train(cfg, tub_names, model_name):
'''
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
'''
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
def rt(record):
record['user/angle'] = dk.utils.linear_bin(record['user/angle'])
return record
kl = KerasLinear(num_outputs=2)
print('tub_names', tub_names)
if not tub_names:
tub_names = os.path.join(cfg.DATA_PATH, '*')
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(
X_keys,
y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
model_path = os.path.expanduser(model_name)
total_records = len(tubgroup.df)
total_train = int(total_records * cfg.TRAIN_TEST_SPLIT)
total_val = total_records - total_train
print('train: %d, validation: %d' % (total_train, total_val))
steps_per_epoch = total_train // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
kl.train(train_gen,
val_gen,
saved_model_path=model_path,
steps=steps_per_epoch,
train_split=cfg.TRAIN_TEST_SPLIT)
def get_model_by_type(model_type: str, cfg: 'Config') -> 'KerasPilot':
'''
given the string model_type and the configuration settings in cfg
create a Keras model and return it.
'''
from donkeycar.parts.keras import KerasPilot, KerasLinear
from donkeycar.parts.tflite import TFLitePilot
if model_type is None:
model_type = cfg.DEFAULT_MODEL_TYPE
print("\"get_model_by_type\" model Type is: {}".format(model_type))
input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)
kl: KerasPilot
if model_type == "linear":
kl = KerasLinear(input_shape=input_shape)
else:
raise Exception("Unknown model type {:}, supported types are "
"linear, categorical, inferred, tflite_linear, "
"tensorrt_linear".format(model_type))
return kl
def train(cfg, tub_names, new_model_path, base_model_path=None ):
"""
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
"""
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
def train_record_transform(record):
""" convert categorical steering to linear and apply image augmentations """
record['user/angle'] = dk.util.data.linear_bin(record['user/angle'])
# TODO add augmentation that doesn't use opencv
return record
def val_record_transform(record):
""" convert categorical steering to linear """
record['user/angle'] = dk.util.data.linear_bin(record['user/angle'])
return record
new_model_path = os.path.expanduser(new_model_path)
kl = KerasLinear()
if base_model_path is not None:
base_model_path = os.path.expanduser(base_model_path)
kl.load(base_model_path)
print('tub_names', tub_names)
if not tub_names:
tub_names = os.path.join(cfg.DATA_PATH, '*')
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(X_keys, y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
total_records = len(tubgroup.df)
total_train = int(total_records * cfg.TRAIN_TEST_SPLIT)
total_val = total_records - total_train
print('train: %d, validation: %d' % (total_train, total_val))
steps_per_epoch = total_train // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
kl.train(train_gen,
val_gen,
saved_model_path=new_model_path,
steps=steps_per_epoch,
train_split=cfg.TRAIN_TEST_SPLIT)
def get_model_by_type(model_type: str, cfg: 'Config') -> 'KerasPilot':
'''
given the string model_type and the configuration settings in cfg
create a Keras model and return it.
'''
from donkeycar.parts.keras import KerasPilot, KerasCategorical, \
KerasLinear, KerasInferred
from donkeycar.parts.tflite import TFLitePilot
if model_type is None:
model_type = cfg.DEFAULT_MODEL_TYPE
print("\"get_model_by_type\" model Type is: {}".format(model_type))
input_shape = (cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)
kl: KerasPilot
if model_type == "linear":
kl = KerasLinear(input_shape=input_shape)
elif model_type == "categorical":
kl = KerasCategorical(
input_shape=input_shape,
throttle_range=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE)
elif model_type == 'inferred':
kl = KerasInferred(input_shape=input_shape)
elif model_type == "tflite_linear":
kl = TFLitePilot()
elif model_type == "tensorrt_linear":
# Aggressively lazy load this. This module imports pycuda.autoinit
# which causes a lot of unexpected things to happen when using TF-GPU
# for training.
from donkeycar.parts.tensorrt import TensorRTLinear
kl = TensorRTLinear(cfg=cfg)
else:
raise Exception("Unknown model type {:}, supported types are "
"linear, categorical, inferred, tflite_linear, "
"tensorrt_linear".format(model_type))
return kl
def train(cfg, tub_names, model_name, model_type):
'''
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
'''
X_keys = ['cam/image_array']
y_keys = ['user/angle', 'user/throttle']
binning = dk.utils.linear_bin
if model_type == "hres_cat":
binning = dk.utils.linear_bin_hres
def rt(record):
record['user/angle'] = binning(record['user/angle'])
return record
kl = KerasCategorical()
if model_type == 'linear':
kl = KerasLinear()
if model_type == 'categorical':
kl = KerasCategorical()
if model_type == 'hres_cat':
kl = KerasHresCategorical()
print('tub_names', tub_names)
if not tub_names:
tub_names = os.path.join(cfg.DATA_PATH, '*')
tubgroup = TubGroup(tub_names)
train_gen, val_gen = tubgroup.get_train_val_gen(
X_keys,
y_keys,
record_transform=rt,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
if model_type == 'linear':
train_gen, val_gen = tubgroup.get_train_val_gen(
X_keys,
y_keys,
batch_size=cfg.BATCH_SIZE,
train_frac=cfg.TRAIN_TEST_SPLIT)
model_path = os.path.expanduser(model_name)
total_records = len(tubgroup.df)
total_train = int(total_records * cfg.TRAIN_TEST_SPLIT)
total_val = total_records - total_train
print('train: %d, validation: %d' % (total_train, total_val))
steps_per_epoch = total_train // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
print(val_gen)
history, save_best = kl.train(train_gen=train_gen,
val_gen=val_gen,
saved_model_path=model_path,
steps=steps_per_epoch,
train_split=cfg.TRAIN_TEST_SPLIT)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss : %f' % save_best.best)
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.savefig(model_path + '_' + model_type +
'_loss_%f.png' % save_best.best)
from donkeycar.parts.keras import KerasLinear
from PIL import Image
import numpy as np
input_shape = (120, 160, 3)
roi_crop = (0, 0)
kl = KerasLinear(input_shape=input_shape, roi_crop=roi_crop)
#kl.load('/home/pi/mycar/models/mypilot20.h5')
img = Image.open('./528.jpg')
img = np.array(img)
#print(img.shape)
out = kl.run(img)
print(out)
def y_translate(self, y: XY) -> Dict[str, Union[float, np.ndarray]]:
""" For now only support keras linear"""
return KerasLinear.y_translate(self, y)
def learn(self,
total_timesteps,
callback=None,
log_interval=1,
tb_log_name="SAC",
print_freq=100,
save_path=None):
with TensorboardWriter(self.graph, self.tensorboard_log,
tb_log_name) as writer:
self._setup_learn()
# Transform to callable if needed
self.learning_rate = get_schedule_fn(self.learning_rate)
start_time = time.time()
episode_rewards = [0.0]
is_teleop_env = hasattr(self.env, "wait_for_teleop_reset")
# TeleopEnv
if is_teleop_env:
print("Waiting for teleop")
obs = self.env.wait_for_teleop_reset()
else:
obs = self.env.reset()
self.episode_reward = np.zeros((1, ))
ep_info_buf = deque(maxlen=100)
ep_len = 0
self.n_updates = 0
infos_values = []
mb_infos_vals = []
model_path = "--Path to model--/myNewModdel.h5"
# model_path= None
if model_path is not None:
cfg = dk.load_config(
config_path='--Path to config file inside mycar/config.py')
kl = KerasLinear()
kl.load(model_path)
# vae = self.env.get_vae()
self.training_started = False
self.start_training = False
for step in range(total_timesteps):
# Compute current learning_rate
frac = 1.0 - step / total_timesteps
current_lr = self.learning_rate(frac)
if callback is not None:
# Only stop training if return value is False, not when it is None. This is for backwards
# compatibility with callbacks that have no return statement.
if callback(locals(), globals()) is False:
break
# Before training starts, randomly sample actions
# from a uniform distribution for better exploration.
# Afterwards, use the learned policy.
if step < self.learning_starts and not self.training_started:
if model_path is not None:
try:
img_arr = self.env.get_images()
# print(img_arr[0].shape)
img_arr = np.asarray(img_arr[0])
img_arr = normalize_and_crop(img_arr, cfg)
croppedImgH = img_arr.shape[0]
croppedImgW = img_arr.shape[1]
if img_arr.shape[2] == 3 and cfg.IMAGE_DEPTH == 1:
img_arr = dk.utils.rgb2gray(img_arr).reshape(
croppedImgH, croppedImgW, 1)
steering, throttle = kl.run(img_arr)
action = [steering, throttle / 6.0]
action = np.asarray(action)
# rescaled_action = action * np.abs(self.action_space.low)
rescaled_action = action
print('Predicted action :', action)
except Exception as e:
print(e)
action = self.env.action_space.sample()
rescaled_action = action
else:
action = self.env.action_space.sample()
rescaled_action = action
print(action)
# No need to rescale when sampling random action
elif not self.training_started:
self.start_training = True
obs = self.env.reset()
else:
action = self.policy_tf.step(
obs[None], deterministic=False).flatten()
# Rescale from [-1, 1] to the correct bounds
rescaled_action = action * np.abs(self.action_space.low)
assert action.shape == self.env.action_space.shape
new_obs, reward, done, info = self.env.step(rescaled_action)
ep_len += 1
if print_freq > 0 and ep_len % print_freq == 0 and ep_len > 0:
print("{} steps".format(ep_len))
# Store transition in the replay buffer.
self.replay_buffer.add(obs, action, reward, new_obs,
float(done))
obs = new_obs
# Retrieve reward and episode length if using Monitor wrapper
maybe_ep_info = info.get('episode')
if maybe_ep_info is not None:
ep_info_buf.extend([maybe_ep_info])
if writer is not None:
# Write reward per episode to tensorboard
ep_reward = np.array([reward]).reshape((1, -1))
ep_done = np.array([done]).reshape((1, -1))
self.episode_reward = total_episode_reward_logger(
self.episode_reward, ep_reward, ep_done, writer, step)
if ep_len > self.train_freq:
print("Additional training")
self.env.reset()
mb_infos_vals = self.optimize(step, writer, current_lr)
done = True
episode_rewards[-1] += reward
if done or self.start_training:
self.start_training = False
if not (isinstance(self.env, VecEnv) or is_teleop_env):
obs = self.env.reset()
print("Episode finished. Reward: {:.2f} {} Steps".format(
episode_rewards[-1], ep_len))
episode_rewards.append(0.0)
ep_len = 0
mb_infos_vals = self.optimize(step, writer, current_lr)
# Refresh obs when using TeleopEnv
if is_teleop_env:
print("Waiting for teleop")
obs = self.env.wait_for_teleop_reset()
# Log losses and entropy, useful for monitor training
if len(mb_infos_vals) > 0:
infos_values = np.mean(mb_infos_vals, axis=0)
if len(episode_rewards[-101:-1]) == 0:
mean_reward = -np.inf
else:
mean_reward = round(
float(np.mean(episode_rewards[-101:-1])), 1)
num_episodes = len(episode_rewards)
if self.verbose >= 1 and done and log_interval is not None and len(
episode_rewards) % log_interval == 0:
fps = int(step / (time.time() - start_time))
logger.logkv("episodes", num_episodes)
logger.logkv("mean 100 episode reward", mean_reward)
logger.logkv(
'ep_rewmean',
safe_mean([ep_info['r'] for ep_info in ep_info_buf]))
logger.logkv(
'eplenmean',
safe_mean([ep_info['l'] for ep_info in ep_info_buf]))
logger.logkv("n_updates", self.n_updates)
logger.logkv("current_lr", current_lr)
logger.logkv("fps", fps)
logger.logkv('time_elapsed',
"{:.2f}".format(time.time() - start_time))
if len(infos_values) > 0:
for (name, val) in zip(self.infos_names, infos_values):
logger.logkv(name, val)
logger.logkv("total timesteps", step)
logger.dumpkvs()
# Reset infos:
infos_values = []
if is_teleop_env:
self.env.is_training = False
# Use last batch
print("Final optimization before saving")
self.env.reset()
mb_infos_vals = self.optimize(step, writer, current_lr)
plt.figure(1)
plt.plot(episode_rewards)
plt.title('Episode Rewards')
plt.ylabel("Reward")
plt.xlabel("Epoch")
filename = "training" + str(random.random()) + ".png"
plt.savefig(filename)
plt.show()
return self
def test_linear_with_model():
kc = KerasLinear(default_linear())
assert kc.model is not None
def test_linear():
kl = KerasLinear()
assert kl.model is not None
def drive(cfg, model_path=None, use_joystick=False):
'''
Construct a working robotic vehicle from many parts.
Each part runs as a job in the Vehicle loop, calling either
it's run or run_threaded method depending on the constructor flag `threaded`.
All parts are updated one after another at the framerate given in
cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner.
Parts may have named outputs and inputs. The framework handles passing named outputs
to parts requesting the same named input.
'''
#Initialisation the CAR
V = dk.vehicle.Vehicle()
# Initialisation of the CAMERA
if cfg.OS == cfg.LINUX_OS:
if cfg.CAMERA_ID == cfg.CAMERA_MACBOOK:
print("Camera {} can't be used.".format(cfg.CAMERA_MACBOOK))
sys.exit()
elif cfg.CAMERA_ID == cfg.CAMERA_MOCK:
cam = MockCamera()
else:
cam = PiCamera(camera=cfg.CAMERA_ID,
resolution=cfg.CAMERA_RESOLUTION)
elif cfg.OS == cfg.MAC_OS:
if cfg.CAMERA_ID == cfg.CAMERA_MACBOOK:
cam = MacWebcam()
else:
cam = VideoStream(camera=cfg.CAMERA_ID,
framerate=cfg.DRIVE_LOOP_HZ)
V.add(cam, outputs=['cam/image_array'], threaded=True)
if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT:
#modify max_throttle closer to 1.0 to have more power
#modify steering_scale lower than 1.0 to have less responsive steering
ctr = JoystickController(
max_throttle=cfg.JOYSTICK_MAX_THROTTLE,
steering_scale=cfg.JOYSTICK_STEERING_SCALE,
auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE)
else:
#This web controller will create a web server that is capable
#of managing steering, throttle, modes, and more.
ctr = LocalWebController()
if (cfg.LANE_DETECTOR):
ctr_inputs = ['cam/image_overlaid_lanes', 'ld/runtime']
#Lane Detector
#Detects the lane the car is in and outputs a best estimate for the 2 lines
lane_detector = LanesDetector(cfg.CAMERA_ID, cfg.LD_PARAMETERS)
V.add(lane_detector,
inputs=['cam/image_array', 'algorithm/reset'],
outputs=['cam/image_overlaid_lanes', 'ld/runtime'])
else:
ctr_inputs = ['cam/image_array']
V.add(
ctr,
ctr_inputs,
outputs=[
'user/angle', 'user/throttle', 'user/mode', 'recording'
#'algorithm/reset'
],
threaded=True)
#See if we should even run the pilot module.
#This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
#Run the pilot if the mode is not user.
kl = KerasLinear()
if model_path:
kl.load(model_path)
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
#Choose what inputs should change the car.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
if cfg.OS == cfg.LINUX_OS:
steering_controller = PCA9685(cfg.STEERING_CHANNEL)
throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL)
elif cfg.OS == cfg.MAC_OS:
steering_controller = MockController()
throttle_controller = MockController()
steering = PWMSteering(controller=steering_controller,
left_pulse=cfg.STEERING_LEFT_PWM,
center_pulse=cfg.STEERING_CENTER_PWM,
right_pulse=cfg.STEERING_RIGHT_PWM)
throttle = PWMThrottle(
controller=throttle_controller,
cal_max_pulse=cfg.THROTTLE_FORWARD_CAL_PWM,
max_pulse=cfg.THROTTLE_FORWARD_PWM,
cal_min_pulse=cfg.THROTTLE_REVERSE_CAL_PWM,
min_pulse=cfg.THROTTLE_REVERSE_PWM,
zero_pulse=cfg.THROTTLE_STOPPED_PWM,
)
V.add(steering, inputs=['angle'])
V.add(throttle, inputs=['throttle'])
#add tub to save data
inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode']
types = ['image_array', 'float', 'float', 'str']
th = TubHandler(path=cfg.DATA_PATH)
tub = th.new_tub_writer(inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
print("You can now go to <your pi ip address>:8887 to drive your car.")
#run the vehicle for 20 seconds
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_joystick=False, use_chaos=False):
"""
（手動・自動）運転する。
多くの部品(part)から作業用のロボット車両を構築する。
各partはVehicleループ内のジョブとして実行され、コンストラクタフラグ `threaded`に応じて
`run` メソッドまたは `run_threaded` メソッドを呼び出す。
すべてのパーツは、 `cfg.DRIVE_LOOP_HZ` で指定されたフレームレートで順次更新され、
各partが適時に処理を終了すると仮定する。
partには名前付きの出力と入力が存在する(どちらかがない場合や複数存在する場合もある)。
Vehicle フレームワークは、名前付き出力を同じ名前の入力を要求するpartに渡すことを処理する。
引数
cfg 個別車両設定オブジェクト、`config.py`がロードされたオブジェクト。
model_path 自動運転時のモデルファイルパスを指定する（デフォルトはNone）。
use_joystick ジョイスティックを使用するかどうかの真偽値（デフォルトはFalse）。
use_chaos 手動運転中に周期的なランダム操舵を加えるかどうかの真偽値（デフォルトはFalse）。
"""
# Vehicle オブジェクトの生成
V = dk.vehicle.Vehicle()
# Timestamp part の生成
clock = Timestamp()
# Timestamp part をVehicleループへ追加
# 入力：
# なし
# 出力：
# 'timestamp' 現在時刻
V.add(clock, outputs=['timestamp'])
# PiCamera part の生成
cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION)
# 別スレッド実行される PiCamera part をVehicleループへ追加
# 入力：
# なし
# 出力：
# 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ
V.add(cam, outputs=['cam/image_array'], threaded=True)
# manage.py デフォルトのジョイスティックpart生成
if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT:
#ctr = JoystickController(max_throttle=cfg.JOYSTICK_MAX_THROTTLE,
# steering_scale=cfg.JOYSTICK_STEERING_SCALE,
# throttle_axis=cfg.JOYSTICK_THROTTLE_AXIS,
# auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE)
# ジョイスティック part の生成
from elecom.part import JoystickController
ctr = JoystickController(config_path='elecom/jc-u3912t.yml')
else:
# ステアリング、スロットル、モードなどを管理するWebサーバを作成する
# Web Controller part の生成
ctr = LocalWebController(use_chaos=use_chaos)
# 別スレッド実行される Web Controller part もしくはジョイスティック part をVehiecleループへ追加
# 入力：
# 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ
# 出力：
# 'user/angle' Web/Joystickにより手動指定した次に取るべきステアリング値
# 'user/throttle' Web/Joystickにより手動指定した次に取るべきスロットル値
# 'user/mode' Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま)
# 'recording' tubデータとして保管するかどうかの真偽値
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
# オートパイロットモジュールを実行すべきかどうかを確認する関数を定義する。
def pilot_condition(mode):
'''
オートパイロットモジュール実行判定関数。
引数で指定されたモードを判別して、オートパイロットモジュールを実行するべきかどうか
真偽値を返却する関数。
引数
mode Userモード('user':全手動、'local_angle':操舵のみ自動、'local':全自動)
戻り値
boolean オートパイロットモジュールを実行するかどうかの真偽値
'''
print('mode=' + mode)
if mode == 'user':
# 全手動時のみ実行しない
return False
else:
return True
# オートパイロットモジュール実行判定関数を part 化したオブジェクトを生成
pilot_condition_part = Lambda(pilot_condition)
# オートパイロットモジュール実行判定 part を Vehiecle ループへ追加
# 入力：
# 'user/mode' Userモード('user':全手動、'local_angle':操舵のみ自動、'local':全自動)
# 出力：
# 'run_pilot' オートパイロットモジュールを実行するかどうかの真偽値
V.add(pilot_condition_part,
inputs=['user/mode'],
outputs=['run_pilot'])
# Userモードでない場合、オートパイロットを実行する
# CNNベースの線形回帰モデル(オートパイロット) part を生成する。
kl = KerasLinear()
# 関数driveの引数 model_part 指定がある場合
if model_path:
# 学習済みモデルファイルを読み込む
kl.load(model_path)
# run_condition が真の場合のみ実行されるオートパイロット part をVehicleループへ追加する
# 入力：
# 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ
# 出力：
# 'pilot/angle' オートパイロットが指定した次に取るべきステアリング値
# 'pilot/throttle' オートパイロットが指定した次に取るべきスロットル値
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
# 車両にどの値を入力にするかを判別する
def drive_mode(mode,
user_angle, user_throttle,
pilot_angle, pilot_throttle):
'''
引数で指定された項目から、車両への入力とするステアリング値、スロットル値を確定する関数。
引数
mode Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま)
user_angle Web/Joystickにより手動指定した次に取るべきステアリング値
user_throttle Web/Joystickにより手動指定した次に取るべきスロットル値
pilot_angle オートパイロットが指定した次に取るべきステアリング値
pilot_throttle オートパイロットが指定した次に取るべきスロットル値
戻り値
angle 車両への入力とするステアリング値
throttle 車両への入力とするスロットル値
'''
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
# 車両にどの値を入力にするかを判別する関数を part 化したオブジェクトを生成
drive_mode_part = Lambda(drive_mode)
# 車両にどの値を入力にするかを判別する part を Vehicle ループへ追加
# 入力
# 'user/mode' Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま)
# 'user/angle' Web/Joystickにより手動指定した次に取るべきステアリング値
# 'user/throttle' Web/Joystickにより手動指定した次に取るべきスロットル値
# 'pilot/angle' オートパイロットが指定した次に取るべきステアリング値
# 'pilot/throttle' オートパイロットが指定した次に取るべきスロットル値
# 戻り値
# 'angle' 車両への入力とするステアリング値
# 'throttle' 車両への入力とするスロットル値
V.add(drive_mode_part,
inputs=['user/mode', 'user/angle', 'user/throttle',
'pilot/angle', 'pilot/throttle'],
outputs=['angle', 'throttle'])
# 実車両のステアリングサーボを操作するオブジェクトを生成
steering_controller = PCA9685(cfg.STEERING_CHANNEL)
# 実車両へステアリング値を指示する part を生成
steering = PWMSteering(controller=steering_controller,
left_pulse=cfg.STEERING_LEFT_PWM,
right_pulse=cfg.STEERING_RIGHT_PWM)
# 実車両のスロットルECSを操作するオブジェクトを生成
throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL)
# 実車両へスロットル値を指示する part を生成
throttle = PWMThrottle(controller=throttle_controller,
max_pulse=cfg.THROTTLE_FORWARD_PWM,
zero_pulse=cfg.THROTTLE_STOPPED_PWM,
min_pulse=cfg.THROTTLE_REVERSE_PWM)
# 実車両へステアリング値を指示する part を Vehiecle ループへ追加
# 入力：
# 'angle' 車両への入力とするステアリング値
# 出力：
# なし(実車両の操舵へ)
V.add(steering, inputs=['angle'])
# 実車両へスロットル値を指示する part を Vehiecle ループへ追加
# 入力：
# 'throttle' 車両への入力とするスロットル値
# 出力：
# なし(実車両のスロットル操作へ)
V.add(throttle, inputs=['throttle'])
# 保存データを tub ディレクトリに追加
inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode', 'timestamp']
types = ['image_array', 'float', 'float', 'str', 'str']
# 複数 tub ディレクトリの場合
# th = TubHandler(path=cfg.DATA_PATH)
# tub = th.new_tub_writer(inputs=inputs, types=types)
# 単一 tub ディレクトリの場合
# tub ディレクトリへ書き込む part を生成
tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types)
# 'recording'が正であれば tub ディレクトリへ書き込む part を Vehiecle ループへ追加
# 入力
# 'cam/image_array' cfg.CAMERA_RESOLUTION 型式の画像データ
# 'user/angle' Web/Joystickにより手動指定した次に取るべきステアリング値
# 'user/throttle' Web/Joystickにより手動指定した次に取るべきスロットル値
# 'user/mode' Web/Joystickにより手動指定した次に取るべきUserモード(入力なしの場合は前回値のまま)
# 'timestamp' 現在時刻
V.add(tub, inputs=inputs, run_condition='recording')
# テレメトリーデータの送信
#tele = PubTelemetry('iotf/emperor.ini', pub_count=20*5)
#V.add(tele, inputs=['cam/image_array', 'user/mode', 'user/angle', 'user/throttle',
# 'pilot/angle', 'pilot/throttle', 'angle', 'throttle'])
# Vehicle ループを開始
V.start(rate_hz=cfg.DRIVE_LOOP_HZ,
max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, model_type='categorical', use_joystick=False):
'''
Construct a working robotic vehicle from many parts.
Each part runs as a job in the Vehicle loop, calling either
it's run or run_threaded method depending on the constructor flag `threaded`.
All parts are updated one after another at the framerate given in
cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner.
Parts may have named outputs and inputs. The framework handles passing named outputs
to parts requesting the same named input.
'''
#Initialize car
V = dk.vehicle.Vehicle()
cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION)
V.add(cam, outputs=['cam/image_array'], threaded=True)
if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT:
#modify max_throttle closer to 1.0 to have more power
#modify steering_scale lower than 1.0 to have less responsive steering
ctr = JoystickController(
max_throttle=cfg.JOYSTICK_MAX_THROTTLE,
steering_scale=cfg.JOYSTICK_STEERING_SCALE,
auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE)
else:
#This web controller will create a web server that is capable
#of managing steering, throttle, and modes, and more.
ctr = LocalWebController()
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
#See if we should even run the pilot module.
#This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
#Run the pilot if the mode is not user.
kl = KerasCategorical()
if model_type == 'linear':
kl = KerasLinear()
if model_type == 'hres_cat':
kl = KerasHresCategorical()
# Change model type accordingly
if (model_type == 'rnn'):
kl = KerasRNN_LSTM()
if (model_type == 'rnn_bin'):
kl = KerasRNN_Categorical()
if model_path:
#kl.load(model_path)
#kl = dk.utils.get_model_by_type(model_type, cfg)
kl.load(model_path)
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
#Choose what inputs should change the car.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
steering_controller = PCA9685(cfg.STEERING_CHANNEL)
steering = PWMSteering(controller=steering_controller,
left_pulse=cfg.STEERING_LEFT_PWM,
right_pulse=cfg.STEERING_RIGHT_PWM)
throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL)
throttle = PWMThrottle(controller=throttle_controller,
max_pulse=cfg.THROTTLE_FORWARD_PWM,
zero_pulse=cfg.THROTTLE_STOPPED_PWM,
min_pulse=cfg.THROTTLE_REVERSE_PWM)
V.add(steering, inputs=['angle'])
V.add(throttle, inputs=['throttle'])
#add tub to save data
inputs = [
'cam/image_array', 'user/angle', 'user/throttle', 'user/mode',
'pilot/angle', 'pilot/throttle'
]
types = ['image_array', 'float', 'float', 'str', 'float', 'float']
th = TubHandler(path=cfg.DATA_PATH)
tub = th.new_tub_writer(inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
#run the vehicle
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
print("You can now go to <your pi ip address>:8887 to drive your car.")
def drive(cfg, model_path=None):
V = dk.vehicle.Vehicle()
V.mem.put(['square/angle', 'square/throttle'], (100, 100))
# display square box given by cooridantes.
cam = SquareBoxCamera(resolution=cfg.CAMERA_RESOLUTION)
V.add(cam,
inputs=['square/angle', 'square/throttle'],
outputs=['cam/image_array'])
# display the image and read user values from a local web controller
ctr = LocalWebController()
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
# See if we should even run the pilot module.
# This is only needed because the part run_contion only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
# Run the pilot if the mode is not user.
kl = KerasLinear()
if model_path:
kl.load(model_path)
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
# See if we should even run the pilot module.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'pilot_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
clock = Timestamp()
V.add(clock, outputs=['timestamp'])
# transform angle and throttle values to coordinate values
def f(x):
return int(x * 100 + 100)
l = Lambda(f)
V.add(l, inputs=['user/angle'], outputs=['square/angle'])
V.add(l, inputs=['user/throttle'], outputs=['square/throttle'])
# add tub to save data
inputs = [
'cam/image_array', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle', 'square/angle', 'square/throttle', 'user/mode',
'timestamp'
]
types = [
'image_array', 'float', 'float', 'float', 'float', 'float', 'float',
'str', 'str'
]
#multiple tubs
#th = TubHandler(path=cfg.DATA_PATH)
#tub = th.new_tub_writer(inputs=inputs, types=types)
# single tub
tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
# run the vehicle for 20 seconds
V.start(rate_hz=50, max_loop_count=10000)
def drive(cfg, model_path=None, use_chaos=False):
"""
Construct a working robotic vehicle from many parts.
Each part runs as a job in the Vehicle loop, calling either
it's run or run_threaded method depending on the constructor flag `threaded`.
All parts are updated one after another at the framerate given in
cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner.
Parts may have named outputs and inputs. The framework handles passing named outputs
to parts requesting the same named input.
"""
V = dk.vehicle.Vehicle()
clock = Timestamp()
V.add(clock, outputs=['timestamp'])
cam = JevoisCamera(camera_id=0)
V.add(cam, outputs=['cam/image_array'], threaded=True)
ctr = XboxGameController(device_search_term='xbox')
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
# See if we should even run the pilot module.
# This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
# Run the pilot if the mode is not user.
kl = KerasLinear()
if model_path:
kl.load(model_path)
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
# Choose what inputs should change the car.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
steering_controller = PCA9685(cfg.STEERING_CHANNEL)
steering = PWMSteering(controller=steering_controller,
left_pulse=cfg.STEERING_LEFT_PWM,
right_pulse=cfg.STEERING_RIGHT_PWM)
throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL)
throttle = PWMThrottle(controller=throttle_controller,
max_pulse=cfg.THROTTLE_FORWARD_PWM,
zero_pulse=cfg.THROTTLE_STOPPED_PWM,
min_pulse=cfg.THROTTLE_REVERSE_PWM)
V.add(steering, inputs=['angle'])
V.add(throttle, inputs=['throttle'])
# add tub to save data
inputs = [
'cam/image_array', 'user/angle', 'user/throttle', 'user/mode',
'timestamp'
]
types = ['image_array', 'float', 'float', 'str', 'str']
# multiple tubs
# th = TubHandler(path=cfg.DATA_PATH)
# tub = th.new_tub_writer(inputs=inputs, types=types)
# single tub
tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
# run the vehicle
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_joystick=False, use_chaos=False):
"""
Construct a working robotic vehicle from many parts.
Each part runs as a job in the Vehicle loop, calling either
it's run or run_threaded method depending on the constructor flag `threaded`.
All parts are updated one after another at the framerate given in
cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner.
Parts may have named outputs and inputs. The framework handles passing named outputs
to parts requesting the same named input.
"""
V = dk.vehicle.Vehicle()
clock = Timestamp()
V.add(clock, outputs=['timestamp'])
cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION)
V.add(cam, outputs=['cam/image_array'], threaded=True)
if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT:
ctr = JoystickController(
max_throttle=cfg.JOYSTICK_MAX_THROTTLE,
steering_scale=cfg.JOYSTICK_STEERING_SCALE,
throttle_axis=cfg.JOYSTICK_THROTTLE_AXIS,
auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE)
else:
# This web controller will create a web server that is capable
# of managing steering, throttle, and modes, and more.
ctr = LocalWebController(use_chaos=use_chaos)
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
# See if we should even run the pilot module.
# This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
# Run the pilot if the mode is not user.
kl = KerasLinear()
if model_path:
kl.load(model_path)
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
# Choose what inputs should change the car.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
steering_controller = PCA9685(cfg.STEERING_CHANNEL)
steering = PWMSteering(controller=steering_controller,
left_pulse=cfg.STEERING_LEFT_PWM,
right_pulse=cfg.STEERING_RIGHT_PWM)
throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL)
throttle = PWMThrottle(controller=throttle_controller,
max_pulse=cfg.THROTTLE_FORWARD_PWM,
zero_pulse=cfg.THROTTLE_STOPPED_PWM,
min_pulse=cfg.THROTTLE_REVERSE_PWM)
V.add(steering, inputs=['angle'])
V.add(throttle, inputs=['throttle'])
# add tub to save data
inputs = [
'cam/image_array', 'user/angle', 'user/throttle', 'user/mode',
'timestamp'
]
types = ['image_array', 'float', 'float', 'str', 'str']
# multiple tubs
# th = TubHandler(path=cfg.DATA_PATH)
# tub = th.new_tub_writer(inputs=inputs, types=types)
# single tub
tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
# run the vehicle
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive_vis(cfg, model_path=None, use_chaos=False):
V = dk.vehicle.Vehicle()
clock = Timestamp()
V.add(clock, outputs=['timestamp'])
cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION)
V.add(cam, outputs=['cam/image_array'], threaded=True)
# Run the pilot if the mode is not user.
kl = KerasLinear()
if model_path:
kl.load(model_path)
top_view_transform = TopViewTransform(cfg.CAMERA_RESOLUTION)
V.add(Lambda(top_view_transform.wrap),
inputs=['cam/image_array'],
outputs=['cam/image_array_proj'])
V.add(kl,
inputs=['cam/image_array_proj'],
outputs=['pilot/angle', 'pilot/throttle'])
ctr = LocalWebControllerVis(use_chaos=use_chaos)
V.add(ctr,
inputs=['cam/image_array_proj', 'pilot/angle', 'pilot/throttle'],
outputs=['user/mode', 'recording'],
threaded=True)
# See if we should even run the pilot module.
# This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
# Choose what inputs should change the car.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
driver = ArduinoDriver()
V.add(driver,
inputs=['user/mode', 'pilot/angle', 'pilot/throttle'],
outputs=['user/angle', 'user/throttle'])
# add tub to save data
inputs = [
'cam/image_array', 'user/angle', 'user/throttle', 'user/mode',
'timestamp'
]
types = ['image_array', 'float', 'float', 'str', 'str']
# multiple tubs
# th = TubHandler(path=cfg.DATA_PATH)
# tub = th.new_tub_writer(inputs=inputs, types=types)
# single tub
tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
# run the vehicle
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg, model_path=None, use_chaos=False):
"""
Construct a working robotic vehicle from many parts.
Each part runs as a job in the Vehicle loop, calling either
it's run or run_threaded method depending on the constructor flag `threaded`.
All parts are updated one after another at the framerate given in
cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner.
Parts may have named outputs and inputs. The framework handles passing named outputs
to parts requesting the same named input.
"""
V = dk.vehicle.Vehicle()
clock = Timestamp()
V.add(clock, outputs=['timestamp'])
cam = PiCamera(resolution=cfg.CAMERA_RESOLUTION)
V.add(cam, outputs=['cam/image_array'], threaded=True)
ctr = LocalWebController(use_chaos=use_chaos)
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
# See if we should even run the pilot module.
# This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
# Run the pilot if the mode is not user.
kl = KerasLinear()
if model_path:
kl.load(model_path)
V.add(kl,
inputs=['cam/image_array'],
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
driver = ArduinoDriver()
V.add(driver,
inputs=['user/mode', 'pilot/angle', 'pilot/throttle'],
outputs=['user/angle', 'user/throttle'])
# add tub to save data
inputs = [
'cam/image_array', 'user/angle', 'user/throttle', 'user/mode',
'timestamp'
]
types = ['image_array', 'float', 'float', 'str', 'str']
# multiple tubs
# th = TubHandler(path=cfg.DATA_PATH)
# tub = th.new_tub_writer(inputs=inputs, types=types)
# single tub
tub = TubWriter(path=cfg.TUB_PATH, inputs=inputs, types=types)
V.add(tub, inputs=inputs, run_condition='recording')
# run the vehicle
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
def drive(cfg,
model_path=None,
use_joystick=False,
model_type=None,
camera_type='single'):
'''
Construct a working robotic vehicle from many parts.
Each part runs as a job in the Vehicle loop, calling either
it's run or run_threaded method depending on the constructor flag `threaded`.
All parts are updated one after another at the framerate given in
cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner.
Parts may have named outputs and inputs. The framework handles passing named outputs
to parts requesting the same named input.
'''
if model_type is None:
model_type = "categorical"
stereo_cam = camera_type == "stereo"
#Initialize car
V = dk.vehicle.Vehicle()
if stereo_cam:
from donkeycar.parts.camera import Webcam
camA = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, iCam=0)
V.add(camA, outputs=['cam/image_array_a'], threaded=True)
camB = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, iCam=1)
V.add(camB, outputs=['cam/image_array_b'], threaded=True)
def stereo_pair(image_a, image_b):
if image_a is not None and image_b is not None:
width, height, _ = image_a.shape
grey_a = dk.utils.rgb2gray(image_a)
grey_b = dk.utils.rgb2gray(image_b)
# Added by Felix
depth = capture.stereo_depth(image_a, image_b, leftMapX,
leftMapY, rightMapX, rightMapY,
leftROI, rightROI, imageSize)
stereo_image = np.zeros([width, height, 3],
dtype=np.dtype('B'))
stereo_image[..., 0] = np.reshape(grey_a, (width, height))
stereo_image[..., 1] = np.reshape(grey_b, (width, height))
stereo_image[..., 2] = np.reshape(depth, (width, height))
# --------------
else:
stereo_image = []
return np.array(stereo_image)
image_sterero_pair_part = Lambda(stereo_pair)
V.add(image_sterero_pair_part,
inputs=['cam/image_array_a', 'cam/image_array_b'],
outputs=['cam/image_array'])
else:
print("cfg.CAMERA_TYPE", cfg.CAMERA_TYPE)
if cfg.CAMERA_TYPE == "PICAM":
from donkeycar.parts.camera import PiCamera
cam = PiCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H)
elif cfg.CAMERA_TYPE == "WEBCAM":
from donkeycar.parts.camera import Webcam
cam = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H)
V.add(cam, outputs=['cam/image_array'], threaded=True)
if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT:
#modify max_throttle closer to 1.0 to have more power
#modify steering_scale lower than 1.0 to have less responsive steering
ctr = JoystickController(
throttle_scale=cfg.JOYSTICK_MAX_THROTTLE,
steering_scale=cfg.JOYSTICK_STEERING_SCALE,
auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE)
else:
#This web controller will create a web server that is capable
#of managing steering, throttle, and modes, and more.
ctr = LocalWebController()
V.add(ctr,
inputs=['cam/image_array'],
outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'],
threaded=True)
#this throttle filter will allow one tap back for esc reverse
th_filter = ThrottleFilter()
V.add(th_filter, inputs=['user/throttle'], outputs=['user/throttle'])
#See if we should even run the pilot module.
#This is only needed because the part run_condition only accepts boolean
def pilot_condition(mode):
if mode == 'user':
return False
else:
return True
pilot_condition_part = Lambda(pilot_condition)
V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot'])
def led_cond(mode, recording, num_records, behavior_state):
'''
returns a blink rate. 0 for off. -1 for on. positive for rate.
'''
if num_records is not None and num_records % 10 == 0:
print("recorded", num_records, "records")
if behavior_state is not None and model_type == 'behavior':
r, g, b = cfg.BEHAVIOR_LED_COLORS[behavior_state]
led.set_rgb(r, g, b)
return -1 #solid on
if recording:
return -1 #solid on
elif mode == 'user':
return 1
elif mode == 'local_angle':
return 0.5
elif mode == 'local':
return 0.1
return 0
led_cond_part = Lambda(led_cond)
V.add(
led_cond_part,
inputs=['user/mode', 'recording', "tub/num_records", 'behavior/state'],
outputs=['led/blink_rate'])
if cfg.HAVE_RGB_LED:
from donkeycar.parts.led_status import RGB_LED
led = RGB_LED(cfg.LED_PIN_R, cfg.LED_PIN_G, cfg.LED_PIN_B,
cfg.LED_INVERT)
led.set_rgb(cfg.LED_R, cfg.LED_G, cfg.LED_B)
V.add(led, inputs=['led/blink_rate'])
#IMU
if cfg.HAVE_IMU:
imu = Mpu6050()
V.add(imu,
outputs=[
'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x',
'imu/gyr_y', 'imu/gyr_z'
],
threaded=True)
#Behavioral state
if model_type == "behavior":
bh = BehaviorPart(cfg.BEHAVIOR_LIST)
V.add(bh,
outputs=[
'behavior/state', 'behavior/label',
"behavior/one_hot_state_array"
])
try:
ctr.set_button_down_trigger('L1', bh.increment_state)
except:
pass
kl = KerasBehavioral(num_outputs=2,
num_behavior_inputs=len(cfg.BEHAVIOR_LIST))
inputs = ['cam/image_array', "behavior/one_hot_state_array"]
#IMU
elif model_type == "imu":
assert (cfg.HAVE_IMU)
#Run the pilot if the mode is not user.
kl = KerasIMU(num_outputs=2, num_imu_inputs=6)
inputs = [
'cam/image_array', 'imu/acl_x', 'imu/acl_y', 'imu/acl_z',
'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'
]
else:
if model_type == "linear":
kl = KerasLinear()
elif model_type == "3d":
kl = Keras3D_CNN(seq_length=cfg.SEQUENCE_LENGTH)
elif model_type == "rnn":
kl = KerasRNN_LSTM(seq_length=cfg.SEQUENCE_LENGTH)
else:
kl = KerasCategorical()
inputs = ['cam/image_array']
if model_path:
kl.load(model_path)
V.add(kl,
inputs=inputs,
outputs=['pilot/angle', 'pilot/throttle'],
run_condition='run_pilot')
#Choose what inputs should change the car.
def drive_mode(mode, user_angle, user_throttle, pilot_angle,
pilot_throttle):
if mode == 'user':
return user_angle, user_throttle
elif mode == 'local_angle':
return pilot_angle, user_throttle
else:
return pilot_angle, pilot_throttle
drive_mode_part = Lambda(drive_mode)
V.add(drive_mode_part,
inputs=[
'user/mode', 'user/angle', 'user/throttle', 'pilot/angle',
'pilot/throttle'
],
outputs=['angle', 'throttle'])
steering_controller = PCA9685(cfg.STEERING_CHANNEL,
cfg.PCA9685_I2C_ADDR,
busnum=cfg.PCA9685_I2C_BUSNUM)
steering = PWMSteering(controller=steering_controller,
left_pulse=cfg.STEERING_LEFT_PWM,
right_pulse=cfg.STEERING_RIGHT_PWM)
throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL,
cfg.PCA9685_I2C_ADDR,
busnum=cfg.PCA9685_I2C_BUSNUM)
throttle = PWMThrottle(controller=throttle_controller,
max_pulse=cfg.THROTTLE_FORWARD_PWM,
zero_pulse=cfg.THROTTLE_STOPPED_PWM,
min_pulse=cfg.THROTTLE_REVERSE_PWM)
V.add(steering, inputs=['angle'])
V.add(throttle, inputs=['throttle'])
#add tub to save data
inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode']
types = ['image_array', 'float', 'float', 'str']
if cfg.TRAIN_BEHAVIORS:
inputs += [
'behavior/state', 'behavior/label', "behavior/one_hot_state_array"
]
types += ['int', 'str', 'vector']
if cfg.HAVE_IMU:
inputs += [
'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y',
'imu/gyr_z'
]
types += ['float', 'float', 'float', 'float', 'float', 'float']
th = TubHandler(path=cfg.DATA_PATH)
tub = th.new_tub_writer(inputs=inputs, types=types)
V.add(tub,
inputs=inputs,
outputs=["tub/num_records"],
run_condition='recording')
if type(ctr) is LocalWebController:
print("You can now go to <your pi ip address>:8887 to drive your car.")
elif type(ctr) is JoystickController:
print("You can now move your joystick to drive your car.")
#tell the controller about the tub
ctr.set_tub(tub)
#run the vehicle for 20 seconds
V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)
