def launch_beam_ng(self, mode="manual_mode"):
# Create an ETK800 with the licence plate 'PYTHON'
electrics = Electrics()
# attach to get steering angles
self.vehicle.attach_sensor('electrics', electrics)
# attach to get images from camera
self.vehicle.attach_sensor('front_cam', self.create_camera_sensor())
# Add it to our scenario at this position and rotation
# self.scenario.add_vehicle(self.vehicle)
self.scenario.add_vehicle(
self.vehicle,
pos=tuple(map(float,
config.get("data_collecting.pos").split(","))),
rot_quat=tuple(
map(float,
config.get("data_collecting.rot_quat").split(","))))
# Place files defining our scenario for the simulator to read
self.scenario.make(self.bng)
# Launch BeamNG.research
self.bng.open()
# Load and start our scenario
self.bng.load_scenario(self.scenario)
self.bng.start_scenario()
if mode == "ai_mode":
# Make the vehicle's AI span the map
self.vehicle.ai_drive_in_lane(True)
self.vehicle.ai_set_mode('span')
def collect_data(self, number_of_images: int, mode="manual_mode"):
self.launch_beam_ng(mode)
logger.info("Start after 3 seconds...")
time.sleep(5)
logger.info(
f"Start collecting {config.get('data_collecting.number_of_images')} training images"
)
i = 0
exit_normally = True
try:
while i < number_of_images:
# image is training image and steering is label
img = self.bng.poll_sensors(
self.vehicle)['front_cam']['colour']
steering = self.bng.poll_sensors(
self.vehicle)['electrics']['steering']
logger.info(f"Saving data {i + 1} ...")
self.save_data(img, steering, str(i))
logger.info("Saved data successfully")
i = i + 1
time.sleep(int(config.get("data_collecting.sleep")))
except Exception as ex:
exit_normally = False
logger.info(f"Error while collecting data {ex}")
finally:
self.bng.close()
return exit_normally
def __init__(self, model, threshold=None, excluded_layer=None):
if excluded_layer is None:
excluded_layer = ['pool', 'fc', 'flatten', 'input']
if threshold is None:
self.threshold = float(
config.get("evaluation.neuron_cover_threshold"))
if model is None:
raise RuntimeError('Model needs to be a keras model')
self.model: Model = model
# the layers that are considered in neuron coverage computation
self.included_layers = []
for layer in self.model.layers:
if all(ex not in layer.name for ex in excluded_layer):
self.included_layers.append(layer.name)
# init coverage table
self.coverage_tracker = {}
try:
for layer_name in self.included_layers:
for index in range(
self.model.get_layer(layer_name).output_shape[-1]):
self.coverage_tracker[(layer_name, index)] = False
except Exception as ex:
raise Exception(
f"Error while checking model layer to initialize neuron coverage tracker: {ex}"
)
def clean_data_images():
"""
This function remove images in images folder which do not exist in csv file.
"""
root.file_config("config.yml")
data = pd.read_csv(definitions.ROOT_DIR +
config.get("data_collecting.csv_path"))
img_path = definitions.ROOT_DIR + config.get("data_collecting.data_path")
count = 0
for file in os.listdir(img_path):
if data["image_name"].str.contains(file).any():
pass
else:
count = count + 1
os.remove(img_path + file)
logger.info(f"Image data cleaning successfully! Removed {count} images")
def train_model(self, training_data_path):
validation_accuracy = []
validation_loss = []
train_data = pd.read_csv(training_data_path)
train_data[['steering']] = train_data[['steering']].astype(float)
labels = train_data[['steering']]
kf = KFold(n_splits=10, random_state=7, shuffle=True)
generator = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.3,
rescale=1. / 255)
fold = 1
for train_index, val_index in kf.split(np.zeros(labels.size), labels):
training_data = train_data.iloc[train_index]
validation_data = train_data.iloc[val_index]
train_data_generator = generator.flow_from_dataframe(
training_data,
directory=self.training_images_dir,
x_col="image_name",
y_col="steering",
class_mode="raw",
shuffle=True)
validation_data_generator = generator.flow_from_dataframe(
validation_data,
directory=self.training_images_dir,
x_col="image_name",
y_col="steering",
class_mode="raw",
shuffle=True)
callback1 = tf.keras.callbacks.ModelCheckpoint(
self.save_model_dir + self.get_model_name(fold),
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='max')
callback2 = EarlyStopping(monitor='val_loss',
patience=3,
verbose=0)
callbacks_list = [callback1, callback2]
self.epoch_model.fit(train_data_generator,
epochs=int(config.get("training.epochs")),
callbacks=callbacks_list,
validation_data=validation_data_generator)
self.epoch_model.load_weights(self.save_model_dir + "model_" +
str(fold) + ".h5")
results = self.epoch_model.evaluate(validation_data_generator)
results = dict(zip(self.epoch_model.metrics_names, results))
validation_loss.append(results['loss'])
tf.keras.backend.clear_session()
fold += 1
logger.info("Loss" + validation_loss.__str__())
def __init__(self, model="inception_v3"):
self.save_model_dir = definitions.ROOT_DIR + '/training/models/'
self.training_images_dir = definitions.ROOT_DIR + config.get(
"data_collecting.data_path")
if model == "inception_v3":
self.epoch_model = self.build_inception_v3()
elif model == "cnn":
self.epoch_model = self.build_cnn()
else:
raise Exception("Model is not supported")
def create_camera_sensor(
pos=(-0.3, 2, 1.0), direction=(0, 1, 0), fov=100, res=None):
# Set up camera sensor
resolution = res
if res is None:
resolution = (int(config.get("data_collecting.image_width")),
int(config.get("data_collecting.image_height")))
pos = pos
direction = direction
fov = fov
front_camera = Camera(pos,
direction,
fov,
resolution,
colour=True,
depth=True,
annotation=True)
return front_camera
def save_csv_data(self):
logger.info("Start saving csv file......")
csv_path = definitions.ROOT_DIR + config.get(
'data_collecting.csv_path')
df = pd.DataFrame(self.collected_data,
columns=['image_name', 'steering'])
if not os.path.isfile(csv_path):
df.to_csv(csv_path, index=False, header=True)
else: # else it exists so append without writing the header
df.to_csv(csv_path, index=False, mode='a', header=False)
def save_data(self, img, steering, i: str = "0"):
file_name = str(int(time.time())) + i + ".jpg"
try:
image_path = definitions.ROOT_DIR + config.get(
'data_collecting.data_path') + file_name
imageio.imwrite(image_path, np.asarray(img.convert('RGB')), "jpg")
self.collected_data["image_name"].append(file_name)
self.collected_data["steering"].append(steering)
except Exception as ex:
logger.info(f"Error while saving data -- {ex}")
raise Exception
def run_trainer(model="inception_v3"):
root.file_config("config.yml")
try:
start_time = time.time()
em = EpochModel(model)
em.train_model(definitions.ROOT_DIR +
config.get("data_collecting.csv_path"))
logger.info(
f"--------- Execution time: {time.time() - start_time} seconds ---------"
)
except Exception as ex:
logger.info(f"Error while training model - {ex}")
logger.info(traceback.format_exc())
def clean_data_csv():
"""
One might remove recorded images while images are being collected.
This function synchronizes data in csv files and images in images folder in such cases.
"""
root.file_config("config.yml")
data = pd.read_csv(definitions.ROOT_DIR +
config.get("data_collecting.csv_path"))
img_path = definitions.ROOT_DIR + config.get("data_collecting.data_path")
count = 0
for i, row in data.iterrows():
if not os.path.isfile(img_path + row["image_name"]):
count = count + 1
data.drop(index=i, inplace=True)
elif row["steering"] < -5.0 or row["steering"] > 5:
count = count + 1
data.drop(index=i, inplace=True)
data.to_csv(definitions.ROOT_DIR + config.get("data_collecting.csv_path"),
index=False,
mode='w',
header=True)
logger.info(f"CSV data cleaning successfully!")
def run_collector(number_of_images=0, mode="manual_mode"):
root.file_config("config.yml")
if not number_of_images:
number_of_images = int(config.get("data_collecting.number_of_images"))
collector = DataCollector()
exit_ok = collector.collect_data(number_of_images, mode)
# Save csv data
if exit_ok:
collector.save_csv_data()
logger.info("Data collected successfully!")
else:
logger.info("Saving data...")
collector.save_csv_data()
logger.info(traceback.format_exc())
def __init__(self):
self.collected_data = {"image_name": [], "steering": []}
self.scenario = Scenario(config.get("data_collecting.scenario_level"),
config.get("data_collecting.scenario_name"))
self.vehicle = Vehicle(
'ego_vehicle',
model=config.get("data_collecting.vehicle_model"),
licence='PYTHON')
self.bng = BeamNGpy(config.get("beamNG.host"),
int(config.get("beamNG.port")),
home=config.get("beamNG.home"))
def __init__(self, model_name="model_1.h5"):
self.evaluation_images_dir = definitions.ROOT_DIR + config.get("evaluation.evaluation_data_path")
self.model_path = definitions.ROOT_DIR + config.get("data_collecting.model_path") + model_name
if not os.path.exists(self.model_path):
raise Exception("Please create a model first before evaluating transformations on neuron coverage")
self.model: Model = load_model(self.model_path)