class ImgPreProcess():
'''
preprocess camera image for inference.
normalize and crop if needed.
'''
def __init__(self, cfg):
self.cfg = cfg
self.ln = None
if cfg.NORM_IMAGES_ILLUMINANCE:
self.ln = LN(model_path, train=False)
def run(self, img_arr):
img = normalize_and_crop(img_arr, self.cfg)
if self.ln is not None:
img = self.ln.normalize_lightness(img_arr)
return img
def __init__(self, cfg):
self.cfg = cfg
self.ln = None
if cfg.NORM_IMAGES_ILLUMINANCE:
self.ln = LN(model_path, train=False)
def train(cfg, tub_names, model_name, transfer_model, model_type, continuous,
aug):
'''
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
'''
verbose = cfg.VERBOSE_TRAIN
if model_type is None:
model_type = cfg.DEFAULT_MODEL_TYPE
if "tflite" in model_type:
#even though we are passed the .tflite output file, we train with an intermediate .h5
#output and then convert to final .tflite at the end.
assert (".tflite" in model_name)
#we only support the linear model type right now for tflite
assert ("linear" in model_type)
model_name = model_name.replace(".tflite", ".h5")
elif "tensorrt" in model_type:
#even though we are passed the .uff output file, we train with an intermediate .h5
#output and then convert to final .uff at the end.
assert (".uff" in model_name)
#we only support the linear model type right now for tensorrt
assert ("linear" in model_type)
model_name = model_name.replace(".uff", ".h5")
if model_name and not '.h5' == model_name[-3:]:
raise Exception("Model filename should end with .h5")
if continuous:
print("continuous training")
gen_records = {}
opts = {'cfg': cfg}
if "linear" in model_type:
train_type = "linear"
else:
train_type = model_type
kl = get_model_by_type(train_type, cfg=cfg)
opts['categorical'] = type(kl) in [KerasCategorical, KerasBehavioral]
print('training with model type', type(kl))
if transfer_model:
print('loading weights from model', transfer_model)
kl.load(transfer_model)
#when transfering models, should we freeze all but the last N layers?
if cfg.FREEZE_LAYERS:
num_to_freeze = len(kl.model.layers) - cfg.NUM_LAST_LAYERS_TO_TRAIN
print('freezing %d layers' % num_to_freeze)
for i in range(num_to_freeze):
kl.model.layers[i].trainable = False
if cfg.OPTIMIZER:
kl.set_optimizer(cfg.OPTIMIZER, cfg.LEARNING_RATE,
cfg.LEARNING_RATE_DECAY)
kl.compile()
if cfg.PRINT_MODEL_SUMMARY:
print(kl.model.summary())
opts['keras_pilot'] = kl
opts['continuous'] = continuous
opts['model_type'] = model_type
extract_data_from_pickles(cfg, tub_names)
records = gather_records(cfg, tub_names, opts, verbose=True)
print('collating %d records ...' % (len(records)))
collate_records(records, gen_records, opts)
def generator(save_best,
opts,
data,
batch_size,
isTrainSet=True,
min_records_to_train=1000,
ln=None):
num_records = len(data)
while True:
if isTrainSet and opts['continuous']:
'''
When continuous training, we look for new records after each epoch.
This will add new records to the train and validation set.
'''
records = gather_records(cfg, tub_names, opts)
if len(records) > num_records:
collate_records(records, gen_records, opts)
new_num_rec = len(data)
if new_num_rec > num_records:
print('picked up', new_num_rec - num_records,
'new records!')
num_records = new_num_rec
save_best.reset_best()
if num_records < min_records_to_train:
print(
"not enough records to train. need %d, have %d. waiting..."
% (min_records_to_train, num_records))
time.sleep(10)
continue
batch_data = []
keys = list(data.keys())
random.shuffle(keys)
kl = opts['keras_pilot']
if type(kl.model.output) is list:
model_out_shape = (2, 1)
else:
model_out_shape = kl.model.output.shape
if type(kl.model.input) is list:
model_in_shape = (2, 1)
else:
model_in_shape = kl.model.input.shape
has_imu = type(kl) is KerasIMU
has_bvh = type(kl) is KerasBehavioral
img_out = type(kl) is KerasLatent
loc_out = type(kl) is KerasLocalizer
il_in = type(kl) is KerasIL
for key in keys:
if not key in data:
continue
_record = data[key]
if _record['train'] != isTrainSet:
continue
if continuous:
#in continuous mode we need to handle files getting deleted
filename = _record['image_path']
if not os.path.exists(filename):
data.pop(key, None)
continue
batch_data.append(_record)
if len(batch_data) == batch_size:
inputs_img = []
inputs_imu = []
inputs_bvh = []
inputs_speed = []
angles = []
throttles = []
out_img = []
out_loc = []
out = []
for record in batch_data:
#get image data if we don't already have it
if record['img_data'] is None:
filename = record['image_path']
img_arr = load_scaled_image_arr(filename, cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if ln is not None:
img_arr = ln.normalize_lightness(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
if il_in:
inputs_speed.append(
record['rotaryencoder/meter_per_second'])
if img_out:
rz_img_arr = cv2.resize(img_arr,
(127, 127)) / 255.0
out_img.append(rz_img_arr[:, :, 0].reshape(
(127, 127, 1)))
if loc_out:
out_loc.append(record['location'])
if has_imu:
inputs_imu.append(record['imu_array'])
if has_bvh:
inputs_bvh.append(record['behavior_arr'])
inputs_img.append(img_arr)
angles.append(record['angle'])
throttles.append(record['throttle'])
out.append([record['angle'], record['throttle']])
if img_arr is None:
continue
img_arr = np.array(inputs_img).reshape(batch_size,\
cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D)
if has_imu:
X = [img_arr, np.array(inputs_imu)]
elif has_bvh:
X = [img_arr, np.array(inputs_bvh)]
elif il_in:
X = [img_arr, np.array(inputs_speed)]
else:
X = [img_arr]
if img_out:
y = [out_img, np.array(angles), np.array(throttles)]
elif out_loc:
y = [
np.array(angles),
np.array(throttles),
np.array(out_loc)
]
elif model_out_shape[1] == 2:
y = [np.array([out]).reshape(batch_size, 2)]
else:
y = [np.array(angles), np.array(throttles)]
yield X, y
batch_data = []
model_path = os.path.expanduser(model_name)
# checkpoint to save model after each epoch and send best to the pi.
send_model_cb = on_best_model if cfg.SEND_BEST_MODEL_TO_PI else None
save_best = MyCPCallback(send_model_cb=send_model_cb,
filepath=model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True,
mode='min',
cfg=cfg)
ln = None
if cfg.NORM_IMAGES_ILLUMINANCE:
ln = LN(model_name, tub_names[0])
train_gen = generator(save_best,
opts,
gen_records,
cfg.BATCH_SIZE,
True,
ln=ln)
val_gen = generator(save_best,
opts,
gen_records,
cfg.BATCH_SIZE,
False,
ln=ln)
total_records = len(gen_records)
num_train = 0
num_val = 0
for key, _record in gen_records.items():
if _record['train'] == True:
num_train += 1
else:
num_val += 1
print("train: %d, val: %d" % (num_train, num_val))
print('total records: %d' % (total_records))
if not continuous:
steps_per_epoch = num_train // cfg.BATCH_SIZE
else:
steps_per_epoch = 100
val_steps = num_val // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
cfg.model_type = model_type
go_train(kl, cfg, train_gen, val_gen, gen_records, model_name,
steps_per_epoch, val_steps, continuous, verbose, save_best)
def train(cfg, tub_names, model_name, transfer_model, model_type, continuous, aug):
'''
use the specified data in tub_names to train an artifical neural network
saves the output trained model as model_name
'''
verbose = cfg.VERBOSE_TRAIN
if model_type is None:
model_type = cfg.DEFAULT_MODEL_TYPE
if continuous:
print("continuous training")
gen_records = {}
opts = { 'cfg' : cfg}
if "linear" in model_type:
train_type = "linear"
else:
train_type = model_type
kl = TorchIL()
print('training with model type', type(kl))
# Do not use
if transfer_model:
print('loading weights from model', transfer_model)
kl.load(transfer_model)
#when transfering models, should we freeze all but the last N layers?
if cfg.FREEZE_LAYERS:
num_to_freeze = len(kl.model.layers) - cfg.NUM_LAST_LAYERS_TO_TRAIN
print('freezing %d layers' % num_to_freeze)
for i in range(num_to_freeze):
kl.model.layers[i].trainable = False
kl.set_optimizer('adam', cfg.LEARNING_RATE, cfg.LEARNING_RATE_DECAY)
opts['torch_pilot'] = kl
opts['continuous'] = continuous
opts['model_type'] = model_type
# Do not use
extract_data_from_pickles(cfg, tub_names)
records = gather_records(cfg, tub_names, opts, verbose=True)
print('collating %d records ...' % (len(records)))
collate_records(records, gen_records, opts)
# make data loader
train_dataset = TubDataset(gen_records, cfg, train=True)
test_dataset = TubDataset(gen_records, cfg, train=False)
train_loader = DataLoader(train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True)
val_loader = DataLoader(test_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True)
model_path = os.path.expanduser(model_name)
# checkpoint to save model after each epoch and send best to the pi.
ln = None
if cfg.NORM_IMAGES_ILLUMINANCE:
ln = LN(model_name, tub_names[0])
total_records = len(gen_records)
num_train = 0
num_val = 0
for key, _record in gen_records.items():
if _record['train'] == True:
num_train += 1
else:
num_val += 1
print("train: %d, val: %d" % (num_train, num_val))
print('total records: %d' %(total_records))
if not continuous:
steps_per_epoch = num_train // cfg.BATCH_SIZE
else:
steps_per_epoch = 100
val_steps = num_val // cfg.BATCH_SIZE
print('steps_per_epoch', steps_per_epoch)
cfg.model_type = model_type
kl.train(train_loader, val_loader, model_name, cfg.MAX_EPOCHS)