def execute(gpu, exp_batch, exp_alias, city_name='Town01', memory_use=0.2, host='127.0.0.1'): # host,port,gpu_number,path,show_screen,resolution,noise_type,config_path,type_of_driver,experiment_name,city_name,game,drivers_name #drive_config.city_name = city_name # TODO Eliminate drive config. print("Running ", __file__, " On GPU ", gpu, "of experiment name ", exp_alias) os.environ["CUDA_VISIBLE_DEVICES"] = gpu if not os.path.exists('_output_logs'): os.mkdir('_output_logs') sys.stdout = open(os.path.join('_output_logs', g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"), "a", buffering=1) #vglrun - d:7.$GPU $CARLA_PATH / CarlaUE4 / Binaries / Linux / CarlaUE4 / Game / Maps /$TOWN - windowed - benchmark - fps = 10 - world - port =$PORT; #sleep 100000 carla_process, port = start_carla_simulator(gpu, exp_batch, exp_alias, city_name) merge_with_yaml(os.path.join('configs', exp_batch, exp_alias+'.yaml')) set_type_of_process('drive', city_name) log_level = logging.WARNING logging.StreamHandler(stream=None) logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level) # TODO we have some external class that control this weather thing. """ if city_name == 'Town01': experiment_suite = ECCVTrainingSuite() else: experiment_suite = ECCVGeneralizationSuite() """ experiment_suite = TestSuite() coil_logger.add_message('Loading', {'Poses': experiment_suite._poses()}) while True: try: coil_logger.add_message('Loading', {'CARLAClient': host+':'+str(port)}) with make_carla_client(host, port) as client: # Now actually run the driving_benchmark latest = 0 # While the checkpoint is not there while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE): # Get the correct checkpoint if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE): latest = get_next_checkpoint(g_conf.TEST_SCHEDULE) checkpoint = torch.load(os.path.join('_logs', exp_batch, exp_alias , 'checkpoints', str(latest) + '.pth')) coil_agent = CoILAgent(checkpoint) coil_logger.add_message({'Iterating': {"Checkpoint": latest}}) # TODO: Change alias to actual experiment name. run_driving_benchmark(coil_agent, experiment_suite, city_name, exp_batch + '_' + exp_alias + '_' + str(latest) , False, host, port) # Read the resulting dictionary #with open(os.path.join('_benchmark_results', # exp_batch+'_'+exp_alias + 'iteration', 'metrics.json') # , 'r') as f: # summary_dict = json.loads(f.read()) # TODO: When you add the message you need to check if the experiment continues properly # TODO: WRITE AN EFICIENT PARAMETRIZED OUTPUT SUMMARY FOR TEST. #test_agent.finish_model() #test_agent.write(results) else: time.sleep(0.1) break except TCPConnectionError as error: logging.error(error) time.sleep(1) carla_process.kill() break except KeyboardInterrupt: carla_process.kill() coil_logger.add_message('Error', {'Message': 'Killed By User'}) break except: traceback.print_exc() carla_process.kill() coil_logger.add_message('Error', {'Message': 'Something Happened'}) break carla_process.kill()
args = argparser.parse_args() if args.debug: log_level = logging.DEBUG elif args.verbose: log_level = logging.INFO else: log_level = logging.WARNING logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level) logging.info('listening to server %s:%s', args.host, args.port) # We instantiate a forward agent, a simple policy that just set # acceleration as 0.9 and steering as zero agent = L5Agent() # We instantiate an experiment suite. Basically a set of experiments # that are going to be evaluated on this benchmark. if args.corl_2017: experiment_suite = CoRL2017(args.city_name) else: print (' WARNING: running the basic driving benchmark, to run for CoRL 2017' ' experiment suites, you should run' ' python driving_benchmark_example.py --corl-2017') experiment_suite = BasicExperimentSuite(args.city_name) # Now actually run the driving_benchmark run_driving_benchmark(agent, experiment_suite, args.city_name, args.log_name, args.continue_experiment, args.host, args.port)
def execute(gpu, exp_batch, exp_alias, city_name='Town01', memory_use=0.2, host='127.0.0.1'): # host,port,gpu_number,path,show_screen,resolution,noise_type,config_path,type_of_driver,experiment_name,city_name,game,drivers_name #drive_config.city_name = city_name # TODO Eliminate drive config. print("Running ", __file__, " On GPU ", gpu, "of experiment name ", exp_alias) os.environ["CUDA_VISIBLE_DEVICES"] = gpu sys.stdout = open(str(os.getpid()) + ".out", "a", buffering=1) #vglrun - d:7.$GPU $CARLA_PATH / CarlaUE4 / Binaries / Linux / CarlaUE4 / Game / Maps /$TOWN - windowed - benchmark - fps = 10 - world - port =$PORT; #sleep 100000 carla_process, port = start_carla_simulator(gpu, exp_batch, exp_alias, city_name) merge_with_yaml(os.path.join(exp_batch, exp_alias + '.yaml')) set_type_of_process('test') #test_agent = CarlaDrive(experiment_name) # TODO we have some external class that control this weather thing. """ if city_name == 'Town01': experiment_suite = ECCVTrainingSuite() else: experiment_suite = ECCVGeneralizationSuite() """ experiment_suite = TestSuite() while True: try: with make_carla_client(host, port) as client: # Now actually run the driving_benchmark latest = 0 # While the checkpoint is not there while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE): # Get the correct checkpoint if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE): latest = get_next_checkpoint(g_conf.TEST_SCHEDULE) checkpoint = torch.load( os.path.join('_logs', exp_batch, exp_alias, 'checkpoints', str(latest) + '.pth')) coil_agent = CoILAgent(checkpoint) run_driving_benchmark( coil_agent, experiment_suite, city_name, exp_batch + '_' + exp_alias + 'iteration', False, host, port) # Read the resulting dictionary with open( os.path.join( '_benchmark_results', exp_batch + '_' + exp_alias + 'iteration', 'metrics.json'), 'r') as f: summary_dict = json.loads(f.read()) # TODO: When you add the message you need to check if the experiment continues properly coil_logger.add_message( {'Running': { "DBSummary": summary_dict }}) #test_agent.finish_model() #test_agent.write(results) else: time.sleep(0.1) # TODO: is this really needed ??? I believe not. #monitorer.export_results(os.path.join('_benchmark_results', # exp_batch + '_' +exp_alias +'iteration')) break except TCPConnectionError as error: logging.error(error) time.sleep(1) carla_process.kill() except KeyboardInterrupt: carla_process.kill() except: traceback.print_exc() carla_process.kill() carla_process.kill()
def execute(gpu, exp_batch, exp_alias, drive_conditions, memory_use=0.2, host='127.0.0.1', suppress_output=True, no_screen=False): try: print("Running ", __file__, " On GPU ", gpu, "of experiment name ", exp_alias) os.environ["CUDA_VISIBLE_DEVICES"] = gpu if not os.path.exists('_output_logs'): os.mkdir('_output_logs') merge_with_yaml(os.path.join('configs', exp_batch, exp_alias + '.yaml')) print("drive cond", drive_conditions) exp_set_name, town_name = drive_conditions.split('_') if g_conf.USE_ORACLE: control_filename = 'control_output_auto.csv' else: control_filename = 'control_output.csv' if exp_set_name == 'ECCVTrainingSuite': experiment_set = ECCVTrainingSuite() set_type_of_process('drive', drive_conditions) elif exp_set_name == 'ECCVGeneralizationSuite': experiment_set = ECCVGeneralizationSuite() set_type_of_process('drive', drive_conditions) elif exp_set_name == 'TestT1': experiment_set = TestT1() set_type_of_process('drive', drive_conditions) elif exp_set_name == 'TestT2': experiment_set = TestT2() set_type_of_process('drive', drive_conditions) else: raise ValueError(" Exp Set name is not correspondent to a city") if suppress_output: sys.stdout = open(os.path.join( '_output_logs', g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"), "a", buffering=1) #sys.stderr = open(os.path.join('_output_logs', # 'err_'+g_conf.PROCESS_NAME + '_' + str(os.getpid()) + ".out"), # "a", buffering=1) carla_process, port = start_carla_simulator(gpu, town_name, no_screen) coil_logger.add_message( 'Loading', {'Poses': experiment_set.build_experiments()[0].poses}) coil_logger.add_message('Loading', {'CARLAClient': host + ':' + str(port)}) # Now actually run the driving_benchmark latest = get_latest_evaluated_checkpoint() if latest is None: # When nothing was tested, get latest returns none, we fix that. latest = 0 csv_outfile = open( os.path.join('_logs', exp_batch, exp_alias, g_conf.PROCESS_NAME + '_csv', control_filename), 'w') csv_outfile.write( "%s,%s,%s,%s,%s,%s,%s,%s\n" % ('step', 'episodes_completion', 'intersection_offroad', 'intersection_otherlane', 'collision_pedestrians', 'collision_vehicles', 'episodes_fully_completed', 'driven_kilometers')) csv_outfile.close() # Write the header of the summary file used conclusion # While the checkpoint is not there while not maximun_checkpoint_reach(latest, g_conf.TEST_SCHEDULE): try: # Get the correct checkpoint if is_next_checkpoint_ready(g_conf.TEST_SCHEDULE): latest = get_next_checkpoint(g_conf.TEST_SCHEDULE) checkpoint = torch.load( os.path.join('_logs', exp_batch, exp_alias, 'checkpoints', str(latest) + '.pth')) coil_agent = CoILAgent(checkpoint, town_name) coil_logger.add_message('Iterating', {"Checkpoint": latest}, latest) run_driving_benchmark( coil_agent, experiment_set, town_name, exp_batch + '_' + exp_alias + '_' + str(latest) + '_drive_' + control_filename[:-4], True, host, port) path = exp_batch + '_' + exp_alias + '_' + str(latest) \ + '_' + g_conf.PROCESS_NAME.split('_')[0] + '_' + control_filename[:-4] \ + '_' + g_conf.PROCESS_NAME.split('_')[1] + '_' + g_conf.PROCESS_NAME.split('_')[2] print(path) print("Finished") benchmark_json_path = os.path.join(get_latest_path(path), 'metrics.json') with open(benchmark_json_path, 'r') as f: benchmark_dict = json.loads(f.read()) averaged_dict = compute_average_std( [benchmark_dict], experiment_set.weathers, len(experiment_set.build_experiments())) print(averaged_dict) csv_outfile = open( os.path.join('_logs', exp_batch, exp_alias, g_conf.PROCESS_NAME + '_csv', control_filename), 'a') csv_outfile.write( "%d,%f,%f,%f,%f,%f,%f,%f\n" % (latest, averaged_dict['episodes_completion'], averaged_dict['intersection_offroad'], averaged_dict['intersection_otherlane'], averaged_dict['collision_pedestrians'], averaged_dict['collision_vehicles'], averaged_dict['episodes_fully_completed'], averaged_dict['driven_kilometers'])) csv_outfile.close() # TODO: When you add the message you need to check if the experiment continues properly # TODO: WRITE AN EFICIENT PARAMETRIZED OUTPUT SUMMARY FOR TEST. else: time.sleep(0.1) except TCPConnectionError as error: logging.error(error) time.sleep(1) carla_process.kill() coil_logger.add_message('Error', {'Message': 'TCP serious Error'}) exit(1) except KeyboardInterrupt: carla_process.kill() coil_logger.add_message('Error', {'Message': 'Killed By User'}) exit(1) except: traceback.print_exc() carla_process.kill() coil_logger.add_message('Error', {'Message': 'Something Happened'}) exit(1) coil_logger.add_message('Finished', {}) except KeyboardInterrupt: traceback.print_exc() carla_process.kill() coil_logger.add_message('Error', {'Message': 'Killed By User'}) except: traceback.print_exc() carla_process.kill() coil_logger.add_message('Error', {'Message': 'Something happened'}) carla_process.kill()
help='If you want to continue the experiment with the same name' ) args = argparser.parse_args() if args.debug: log_level = logging.DEBUG elif args.verbose: log_level = logging.INFO else: log_level = logging.WARNING logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level) logging.info('listening to server %s:%s', args.host, args.port) if args.hn: agent = AutoPilotAgent050() else: agent = AutoPilotAgent025() if args.experiment == 'C': corl = Fctl2018T(args.city_name, args.times) elif args.experiment == 'S': corl = Fctl2018S(args.city_name, args.times) else: corl = Fctl2018(args.city_name, args.times) # Now actually run the driving_benchmark run_driving_benchmark(agent, corl, args.city_name, args.log_name, args.continue_experiment, args.host, args.port, args.gd)
args = argparser.parse_args() if args.debug: log_level = logging.DEBUG elif args.verbose: log_level = logging.INFO else: log_level = logging.WARNING logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level) logging.info('listening to server %s:%s', args.host, args.port) # We instantiate a forward agent, a simple policy that just set # acceleration as 0.9 and steering as zero agent = ForwardAgent() # We instantiate an experiment suite. Basically a set of experiments # that are going to be evaluated on this benchmark. if args.corl_2017: experiment_suite = CoRL2017(args.city_name) else: print (' WARNING: running the basic driving benchmark, to run for CoRL 2017' ' experiment suites, you should run' ' python driving_benchmark_example.py --corl-2017') experiment_suite = BasicExperimentSuite(args.city_name) # Now actually run the driving_benchmark run_driving_benchmark(agent, experiment_suite, args.city_name, args.log_name, args.continue_experiment, args.host, args.port)
def __init__(self, town='Town01_nemesisA', task='turn-right', weather='ClearNoon', port=2000, save_images=False, gpu_num=0, experiment_name='baseline', adversary_name="adversary_", intersection="42_47"): """ Adversary environment for Carla Simulator """ print("Starting CARLA gym environment") print("Ensure that CARLA is running on port", port) self.town = town self.task = task self.weather = WEATHER_DICT[weather] self.port = port self.save_images = save_images self.gpu_num = gpu_num self.experiment_name = experiment_name self.intersection = intersection self.counter = 0 # counter if more than 1 experiments are run self.agent = None self.avoid_stopping = False self.iterations = 1 self.adversary_name = adversary_name if self.adversary_name == "adversary_": adversary_name_other = "adversarybeta_" else: adversary_name_other = "adversary_" image_label_other = adversary_name_other + self.town + ".png" self._load_agent() # defines what kinds of experiments are going to be run self.experiment_suite = AdversarySuite(self.town, self.task, self.weather, self.iterations, self.intersection) # load the adversary generator self.adversary = Shape(self.town, adversary_name=self.adversary_name) self.log_dir = '_benchmarks_results/' + self.town + '/' self.update_csv_file() print("Running the baseline scenario.") # runs the experiment for the baseline case (no attack) run_driving_benchmark(self.agent, self.experiment_suite, log_name=self.experiment_name, city_name=self.town, port=self.port, save_images=save_images) # some metrics that are collected self.baseline_steer_grad = self.get_steer_gradient() self.baseline_steer = self.get_steer() self.positions = self.get_xy()
def step(self): self.experiment_name = 'adversary_{}'.format(self.opt) self.adversary.lines_rotate(opt=self.opt,color_option=self.color_option) self.adversary_other.lines_rotate(opt=self.opt,color_option="True") self.agent = ImitationLearning(self.town, self.task, self.intersection, self.save_choice,self.avoid_stopping,opt=self.opt,min_frames=self.min_frames,max_frames=self.max_frames, trajectory_no=self.trajectory_no) run_driving_benchmark(agent=self.agent, experiment_suite=self.experiment_suite, city_name=self.town, log_name=self.experiment_name, port=self.port)
targetSteer = targetSteer[: MAX_LEN] # subset steering angles to maximum number of allowed frames env.task = baseline_task env.scene = baseline_scene env.experiment_name = 'baseline' # reset experiment suite with base task + scene env.experiment_suite = AdversarySuite(env.town, env.task, env.weather, env.iterations, env.scene) # run the baseline simulation print("Running the simulation for the baseline path.") run_driving_benchmark(env.agent, env.experiment_suite, log_name=env.experiment_name, city_name=env.town, port=env.port, save_images=False) print("Complete.") baseSteer = env.get_steer() MAX_LEN_B = int(len(baseSteer) * .8) baseSteer = baseSteer[:MAX_LEN_B] def target(pos1, rot1, pos2=0, rot2=0, width=10, length=200, colorR=0,