def run_sim(street_1: DecalRoad):
brewer = BeamNGBrewer(street_1.nodes)
waypoint_goal = BeamNGWaypoint('waypoint_goal', get_node_coords(street_1.nodes[-1]))
vehicle = brewer.setup_vehicle()
camera = brewer.setup_scenario_camera()
beamng = brewer.beamng
brewer.setup_road_nodes(street_1.nodes)
maps.beamng_map.generated().write_items(brewer.decal_road.to_json() + '\n' + waypoint_goal.to_json())
cameras = BeamNGCarCameras()
brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose()
#brewer.vehicle_start_pose = BeamNGPose()
sim_data_collector = TrainingDataCollectorAndWriter(vehicle, beamng, street_1, cameras)
brewer.bring_up()
print('bring up ok')
script = calculate_script(brewer.road_points.middle)
# Trick: we start from the road center
vehicle.ai_set_script(script[4:])
#vehicle.ai_drive_in_lane(True)
beamng.pause()
beamng.step(1)
def start():
for idx in range(1000):
if (idx * 0.05 * STEPS) > 3.:
sim_data_collector.collect_and_write_current_data()
dist = distance(sim_data_collector.last_state.pos, waypoint_goal.position)
if dist < 15.0:
beamng.resume()
break
# one step is 0.05 seconds (5/100)
beamng.step(STEPS)
try:
start()
finally:
beamng.close()
def run_sim(nodes):
print(nodes)
brewer = BeamNGBrewer(road_nodes=nodes)
beamng = brewer.beamng
waypoint_goal = BeamNGWaypoint('waypoint_goal', get_node_coords(nodes[-1]))
maps.install_map_if_needed()
maps.beamng_map.generated().write_items(brewer.decal_road.to_json() +
'\n' + waypoint_goal.to_json())
vehicle = brewer.setup_vehicle()
brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose()
#camera = brewer.setup_scenario_camera()
street_1 = brewer.decal_road
sim_data_collector = TrainingDataCollectorAndWriter(
vehicle, beamng, street_1)
brewer.bring_up()
steps = 5
def start():
for idx in range(1000):
if (idx * 0.05 * steps) > 3.:
sim_data_collector.collect_and_write_current_data()
dist = points_distance(sim_data_collector.last_state.pos,
waypoint_goal.position)
if dist < 15.0:
beamng.resume()
break
# one step is 0.05 seconds (5/100)
beamng.step(steps)
try:
start()
finally:
beamng.close()
def _run_simulation(self, nodes) -> SimulationData:
if not self.brewer:
self.brewer = BeamNGBrewer()
self.vehicle = self.brewer.setup_vehicle()
self.camera = self.brewer.setup_scenario_camera()
brewer = self.brewer
brewer.setup_road_nodes(nodes)
beamng = brewer.beamng
waypoint_goal = BeamNGWaypoint('waypoint_goal', get_node_coords(nodes[-1]))
maps.install_map_if_needed()
maps.beamng_map.generated().write_items(brewer.decal_road.to_json() + '\n' + waypoint_goal.to_json())
cameras = BeamNGCarCameras()
vehicle_state_reader = VehicleStateReader(self.vehicle, beamng, additional_sensors=cameras.cameras_array)
brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose()
steps = brewer.params.beamng_steps
simulation_id = time.strftime('%Y-%m-%d--%H-%M-%S', time.localtime())
name = self.config.simulation_name.replace('$(id)', simulation_id)
sim_data_collector = SimulationDataCollector(self.vehicle, beamng, brewer.decal_road, brewer.params,
vehicle_state_reader=vehicle_state_reader,
camera=self.camera,
simulation_name=name)
sim_data_collector.get_simulation_data().start()
try:
brewer.bring_up()
from keras.models import load_model
if not self.model:
self.model = load_model(self.model_file)
predict = NvidiaPrediction(self.model, self.config)
iterations_count = 1000
idx = 0
while True:
idx += 1
if idx >= iterations_count:
sim_data_collector.save()
raise Exception('Timeout simulation ', sim_data_collector.name)
sim_data_collector.collect_current_data(oob_bb=False)
last_state: SimulationDataRecord = sim_data_collector.states[-1]
if points_distance(last_state.pos, waypoint_goal.position) < 15.0:
break
if last_state.is_oob:
break
img = vehicle_state_reader.sensors['cam_center']['colour'].convert('RGB')
steering_angle, throttle = predict.predict(img, last_state)
self.vehicle.control(throttle=throttle, steering=steering_angle, brake=0)
beamng.step(steps)
sim_data_collector.get_simulation_data().end(success=True)
except Exception as ex:
sim_data_collector.get_simulation_data().end(success=False, exception=ex)
traceback.print_exception(type(ex), ex, ex.__traceback__)
finally:
if self.config.simulation_save:
sim_data_collector.save()
try:
sim_data_collector.take_car_picture_if_needed()
except:
pass
self.end_iteration()
return sim_data_collector.simulation_data
class BeamNGNvidiaOob(BeamNGEvaluator):
def __init__(self, config: BeamNGConfig):
self.config = config
self.brewer: BeamNGBrewer = None
self.model_file = str(folders.trained_models_colab.joinpath(config.keras_model_file))
if not os.path.exists(self.model_file):
raise Exception(f'File {self.model_file} does not exist!')
self.model = None
def evaluate(self, members: List[BeamNGMember]):
for member in members:
if not member.needs_evaluation():
log.info(f'{member} is already evaluated. skipping')
continue
counter = 20
attempt = 0
while True:
attempt += 1
if attempt == counter:
raise Exception('Exhausted attempts')
if attempt > 1:
log.info(f'RETRYING TO run simulation {attempt}')
self._close()
else:
log.info(f'{member} BeamNG evaluation start')
if attempt > 2:
time.sleep(5)
sim = self._run_simulation(member.sample_nodes)
if sim.info.success:
break
member.distance_to_boundary = sim.min_oob_distance()
log.info(f'{member} BeamNG evaluation completed')
def _run_simulation(self, nodes) -> SimulationData:
if not self.brewer:
self.brewer = BeamNGBrewer()
self.vehicle = self.brewer.setup_vehicle()
self.camera = self.brewer.setup_scenario_camera()
brewer = self.brewer
brewer.setup_road_nodes(nodes)
beamng = brewer.beamng
waypoint_goal = BeamNGWaypoint('waypoint_goal', get_node_coords(nodes[-1]))
maps.install_map_if_needed()
maps.beamng_map.generated().write_items(brewer.decal_road.to_json() + '\n' + waypoint_goal.to_json())
cameras = BeamNGCarCameras()
vehicle_state_reader = VehicleStateReader(self.vehicle, beamng, additional_sensors=cameras.cameras_array)
brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose()
steps = brewer.params.beamng_steps
simulation_id = time.strftime('%Y-%m-%d--%H-%M-%S', time.localtime())
name = self.config.simulation_name.replace('$(id)', simulation_id)
sim_data_collector = SimulationDataCollector(self.vehicle, beamng, brewer.decal_road, brewer.params,
vehicle_state_reader=vehicle_state_reader,
camera=self.camera,
simulation_name=name)
sim_data_collector.get_simulation_data().start()
try:
brewer.bring_up()
from keras.models import load_model
if not self.model:
self.model = load_model(self.model_file)
predict = NvidiaPrediction(self.model, self.config)
iterations_count = 1000
idx = 0
while True:
idx += 1
if idx >= iterations_count:
sim_data_collector.save()
raise Exception('Timeout simulation ', sim_data_collector.name)
sim_data_collector.collect_current_data(oob_bb=False)
last_state: SimulationDataRecord = sim_data_collector.states[-1]
if points_distance(last_state.pos, waypoint_goal.position) < 15.0:
break
if last_state.is_oob:
break
img = vehicle_state_reader.sensors['cam_center']['colour'].convert('RGB')
steering_angle, throttle = predict.predict(img, last_state)
self.vehicle.control(throttle=throttle, steering=steering_angle, brake=0)
beamng.step(steps)
sim_data_collector.get_simulation_data().end(success=True)
except Exception as ex:
sim_data_collector.get_simulation_data().end(success=False, exception=ex)
traceback.print_exception(type(ex), ex, ex.__traceback__)
finally:
if self.config.simulation_save:
sim_data_collector.save()
try:
sim_data_collector.take_car_picture_if_needed()
except:
pass
self.end_iteration()
return sim_data_collector.simulation_data
def end_iteration(self):
try:
if self.config.beamng_close_at_iteration:
self._close()
else:
if self.brewer:
self.brewer.beamng.stop_scenario()
except Exception as ex:
log.debug('end_iteration() failed:')
traceback.print_exception(type(ex), ex, ex.__traceback__)
def _close(self):
if self.brewer:
try:
self.brewer.beamng.close()
except Exception as ex:
log.debug('beamng.close() failed:')
traceback.print_exception(type(ex), ex, ex.__traceback__)
self.brewer = None
class BeamngExecutor(AbstractTestExecutor):
def __init__(self,
result_folder,
time_budget,
map_size,
oob_tolerance=0.95,
max_speed=70,
beamng_home=None,
beamng_user=None,
road_visualizer=None):
super(BeamngExecutor, self).__init__(result_folder, time_budget,
map_size)
# TODO Is this still valid?
self.test_time_budget = 250000
# TODO This is specific to the TestSubject, we should encapsulate this better
self.risk_value = 0.7
self.oob_tolerance = oob_tolerance
self.maxspeed = max_speed
self.brewer: BeamNGBrewer = None
self.beamng_home = beamng_home
self.beamng_user = beamng_user
# TODO Add checks with default setup. This requires a call to BeamNGpy resolve (~/Documents/BeamNG.research)
if self.beamng_user is not None and not os.path.exists(
os.path.join(self.beamng_user, "research.key")):
log.warning(
"%s is missing but is required to use BeamNG.research", )
# Runtime Monitor about relative movement of the car
self.last_observation = None
# Not sure how to set this... How far can a car move in 250 ms at 5Km/h
self.min_delta_position = 1.0
self.road_visualizer = road_visualizer
def _execute(self, the_test):
# Ensure we do not execute anything longer than the time budget
super()._execute(the_test)
# TODO Show name of the test?
log.info("Executing test %s", the_test.id)
# TODO Not sure why we need to repeat this 2 times...
counter = 2
attempt = 0
sim = None
condition = True
while condition:
attempt += 1
if attempt == counter:
test_outcome = "ERROR"
description = 'Exhausted attempts'
break
if attempt > 1:
self._close()
if attempt > 2:
time.sleep(5)
sim = self._run_simulation(the_test)
if sim.info.success:
if sim.exception_str:
test_outcome = "FAIL"
description = sim.exception_str
else:
test_outcome = "PASS"
description = 'Successful test'
condition = False
execution_data = sim.states
# TODO: report all test outcomes
return test_outcome, description, execution_data
def _is_the_car_moving(self, last_state):
""" Check if the car moved in the past 10 seconds """
# Has the position changed
if self.last_observation is None:
self.last_observation = last_state
return True
# If the car moved since the last observation, we store the last state and move one
if Point(self.last_observation.pos[0],
self.last_observation.pos[1]).distance(
Point(last_state.pos[0],
last_state.pos[1])) > self.min_delta_position:
self.last_observation = last_state
return True
else:
# How much time has passed since the last observation?
if last_state.timer - self.last_observation.timer > 10.0:
return False
else:
return True
def _run_simulation(self, the_test) -> SimulationData:
if not self.brewer:
self.brewer = BeamNGBrewer(beamng_home=self.beamng_home,
beamng_user=self.beamng_user)
self.vehicle = self.brewer.setup_vehicle()
# For the execution we need the interpolated points
nodes = the_test.interpolated_points
brewer = self.brewer
brewer.setup_road_nodes(nodes)
beamng = brewer.beamng
waypoint_goal = BeamNGWaypoint('waypoint_goal',
get_node_coords(nodes[-1]))
# TODO Make sure that maps points to the right folder !
if self.beamng_user is not None:
beamng_levels = LevelsFolder(
os.path.join(self.beamng_user, 'levels'))
maps.beamng_levels = beamng_levels
maps.beamng_map = maps.beamng_levels.get_map('tig')
# maps.print_paths()
maps.install_map_if_needed()
maps.beamng_map.generated().write_items(brewer.decal_road.to_json() +
'\n' + waypoint_goal.to_json())
vehicle_state_reader = VehicleStateReader(self.vehicle,
beamng,
additional_sensors=None)
brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose()
steps = brewer.params.beamng_steps
simulation_id = time.strftime('%Y-%m-%d--%H-%M-%S', time.localtime())
name = 'beamng_executor/sim_$(id)'.replace('$(id)', simulation_id)
sim_data_collector = SimulationDataCollector(
self.vehicle,
beamng,
brewer.decal_road,
brewer.params,
vehicle_state_reader=vehicle_state_reader,
simulation_name=name)
# TODO: Hacky - Not sure what's the best way to set this...
sim_data_collector.oob_monitor.tolerance = self.oob_tolerance
sim_data_collector.get_simulation_data().start()
try:
#start = timeit.default_timer()
brewer.bring_up()
# iterations_count = int(self.test_time_budget/250)
# idx = 0
brewer.vehicle.ai_set_aggression(self.risk_value)
# TODO This does not seem to take any effect...
brewer.vehicle.ai_set_speed(self.maxspeed, mode='limit')
brewer.vehicle.ai_drive_in_lane(True)
brewer.vehicle.ai_set_waypoint(waypoint_goal.name)
while True:
# idx += 1
# assert idx < iterations_count, "Timeout Simulation " + str(sim_data_collector.name)
sim_data_collector.collect_current_data(oob_bb=True)
last_state: SimulationDataRecord = sim_data_collector.states[
-1]
# Target point reached
if points_distance(last_state.pos,
waypoint_goal.position) < 8.0:
break
assert self._is_the_car_moving(
last_state), "Car is not moving fast enough " + str(
sim_data_collector.name)
assert not last_state.is_oob, "Car drove out of the lane " + str(
sim_data_collector.name)
beamng.step(steps)
sim_data_collector.get_simulation_data().end(success=True)
#end = timeit.default_timer()
#run_elapsed_time = end-start
#run_elapsed_time = float(last_state.timer)
self.total_elapsed_time = self.get_elapsed_time()
except AssertionError as aex:
sim_data_collector.save()
# An assertion that trigger is still a successful test execution, otherwise it will count as ERROR
sim_data_collector.get_simulation_data().end(success=True,
exception=aex)
traceback.print_exception(type(aex), aex, aex.__traceback__)
except Exception as ex:
sim_data_collector.save()
sim_data_collector.get_simulation_data().end(success=False,
exception=ex)
traceback.print_exception(type(ex), ex, ex.__traceback__)
finally:
sim_data_collector.save()
try:
sim_data_collector.take_car_picture_if_needed()
except:
pass
self.end_iteration()
return sim_data_collector.simulation_data
def end_iteration(self):
try:
if self.brewer:
self.brewer.beamng.stop_scenario()
except Exception as ex:
traceback.print_exception(type(ex), ex, ex.__traceback__)
def _close(self):
if self.brewer:
try:
self.brewer.beamng.close()
except Exception as ex:
traceback.print_exception(type(ex), ex, ex.__traceback__)
self.brewer = None
def _run_simulation(self, the_test) -> SimulationData:
if not self.brewer:
self.brewer = BeamNGBrewer(beamng_home=self.beamng_home,
beamng_user=self.beamng_user)
self.vehicle = self.brewer.setup_vehicle()
# For the execution we need the interpolated points
nodes = the_test.interpolated_points
brewer = self.brewer
brewer.setup_road_nodes(nodes)
beamng = brewer.beamng
waypoint_goal = BeamNGWaypoint('waypoint_goal',
get_node_coords(nodes[-1]))
# TODO Make sure that maps points to the right folder !
if self.beamng_user is not None:
beamng_levels = LevelsFolder(
os.path.join(self.beamng_user, 'levels'))
maps.beamng_levels = beamng_levels
maps.beamng_map = maps.beamng_levels.get_map('tig')
# maps.print_paths()
maps.install_map_if_needed()
maps.beamng_map.generated().write_items(brewer.decal_road.to_json() +
'\n' + waypoint_goal.to_json())
vehicle_state_reader = VehicleStateReader(self.vehicle,
beamng,
additional_sensors=None)
brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose()
steps = brewer.params.beamng_steps
simulation_id = time.strftime('%Y-%m-%d--%H-%M-%S', time.localtime())
name = 'beamng_executor/sim_$(id)'.replace('$(id)', simulation_id)
sim_data_collector = SimulationDataCollector(
self.vehicle,
beamng,
brewer.decal_road,
brewer.params,
vehicle_state_reader=vehicle_state_reader,
simulation_name=name)
# TODO: Hacky - Not sure what's the best way to set this...
sim_data_collector.oob_monitor.tolerance = self.oob_tolerance
sim_data_collector.get_simulation_data().start()
try:
#start = timeit.default_timer()
brewer.bring_up()
# iterations_count = int(self.test_time_budget/250)
# idx = 0
brewer.vehicle.ai_set_aggression(self.risk_value)
# TODO This does not seem to take any effect...
brewer.vehicle.ai_set_speed(self.maxspeed, mode='limit')
brewer.vehicle.ai_drive_in_lane(True)
brewer.vehicle.ai_set_waypoint(waypoint_goal.name)
while True:
# idx += 1
# assert idx < iterations_count, "Timeout Simulation " + str(sim_data_collector.name)
sim_data_collector.collect_current_data(oob_bb=True)
last_state: SimulationDataRecord = sim_data_collector.states[
-1]
# Target point reached
if points_distance(last_state.pos,
waypoint_goal.position) < 8.0:
break
assert self._is_the_car_moving(
last_state), "Car is not moving fast enough " + str(
sim_data_collector.name)
assert not last_state.is_oob, "Car drove out of the lane " + str(
sim_data_collector.name)
beamng.step(steps)
sim_data_collector.get_simulation_data().end(success=True)
#end = timeit.default_timer()
#run_elapsed_time = end-start
#run_elapsed_time = float(last_state.timer)
self.total_elapsed_time = self.get_elapsed_time()
except AssertionError as aex:
sim_data_collector.save()
# An assertion that trigger is still a successful test execution, otherwise it will count as ERROR
sim_data_collector.get_simulation_data().end(success=True,
exception=aex)
traceback.print_exception(type(aex), aex, aex.__traceback__)
except Exception as ex:
sim_data_collector.save()
sim_data_collector.get_simulation_data().end(success=False,
exception=ex)
traceback.print_exception(type(ex), ex, ex.__traceback__)
finally:
sim_data_collector.save()
try:
sim_data_collector.take_car_picture_if_needed()
except:
pass
self.end_iteration()
return sim_data_collector.simulation_data
