def main(): screen.set_use_colors(False) parser = argparse.ArgumentParser() parser.add_argument('-pk', '--preset_s3_key', help="(string) Name of a preset to download from S3", type=str, required=False) parser.add_argument( '-ek', '--environment_s3_key', help="(string) Name of an environment file to download from S3", type=str, required=False) parser.add_argument('--model_metadata_s3_key', help="(string) Model Metadata File S3 Key", type=str, required=False) parser.add_argument( '-c', '--checkpoint-dir', help= '(string) Path to a folder containing a checkpoint to write the model to.', type=str, default='./checkpoint') parser.add_argument( '--pretrained-checkpoint-dir', help='(string) Path to a folder for downloading a pre-trained model', type=str, default=PRETRAINED_MODEL_DIR) parser.add_argument('--s3_bucket', help='(string) S3 bucket', type=str, default=os.environ.get( "SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test")) parser.add_argument('--s3_prefix', help='(string) S3 prefix', type=str, default='sagemaker') parser.add_argument('--framework', help='(string) tensorflow or mxnet', type=str, default='tensorflow') parser.add_argument('--pretrained_s3_bucket', help='(string) S3 bucket for pre-trained model', type=str) parser.add_argument('--pretrained_s3_prefix', help='(string) S3 prefix for pre-trained model', type=str, default='sagemaker') parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=os.environ.get("AWS_REGION", "us-east-1")) args, unknown = parser.parse_known_args() s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region) # Load the model metadata model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH, 'model_metadata.json') load_model_metadata(s3_client, args.model_metadata_s3_key, model_metadata_local_path) s3_client.upload_file( os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix), model_metadata_local_path) shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR) success_custom_environment = False if args.environment_s3_key: environment_local_path = "./markov/environments/deepracer_racetrack_env.py" success_custom_environment = s3_client.download_file( s3_key=args.environment_s3_key, local_path=environment_local_path) if not success_custom_environment: print( "Could not download the environment file. Using the default DeepRacer environment." ) else: success_custom_environment = s3_client.upload_file( s3_key=os.path.normpath( "%s/environments/deepracer_racetrack_env.py" % args.s3_prefix), local_path=environment_local_path) if success_custom_environment: print("Using environment: %s" % args.environment_s3_key) # Import to register the environment with Gym import markov.environments success_custom_preset = False if args.preset_s3_key: preset_local_path = "./markov/presets/preset.py" success_custom_preset = s3_client.download_file( s3_key=args.preset_s3_key, local_path=preset_local_path) if not success_custom_preset: print( "Could not download the preset file. Using the default DeepRacer preset." ) else: preset_location = "markov.presets.preset:graph_manager" graph_manager = short_dynamic_import(preset_location, ignore_module_case=True) success_custom_preset = s3_client.upload_file( s3_key=os.path.normpath("%s/presets/preset.py" % args.s3_prefix), local_path=preset_local_path) if success_custom_preset: print("Using preset: %s" % args.preset_s3_key) if not success_custom_preset: from markov.sagemaker_graph_manager import get_graph_manager params_blob = os.environ.get('SM_TRAINING_ENV', '') if params_blob: params = json.loads(params_blob) sm_hyperparams_dict = params["hyperparameters"] else: sm_hyperparams_dict = {} graph_manager, robomaker_hyperparams_json = get_graph_manager( **sm_hyperparams_dict) s3_client.upload_hyperparameters(robomaker_hyperparams_json) print("Uploaded hyperparameters.json to S3") host_ip_address = get_ip_from_host() s3_client.write_ip_config(host_ip_address) print("Uploaded IP address information to S3: %s" % host_ip_address) use_pretrained_model = False if args.pretrained_s3_bucket and args.pretrained_s3_prefix: s3_client_pretrained = SageS3Client( bucket=args.pretrained_s3_bucket, s3_prefix=args.pretrained_s3_prefix, aws_region=args.aws_region) s3_client_pretrained.download_model(args.pretrained_checkpoint_dir) use_pretrained_model = True memory_backend_params = RedisPubSubMemoryBackendParameters( redis_address="localhost", redis_port=6379, run_type='trainer', channel=args.s3_prefix) graph_manager.agent_params.memory.register_var('memory_backend_params', memory_backend_params) ds_params_instance = S3BotoDataStoreParameters( bucket_name=args.s3_bucket, checkpoint_dir=args.checkpoint_dir, aws_region=args.aws_region, s3_folder=args.s3_prefix) graph_manager.data_store_params = ds_params_instance data_store = S3BotoDataStore(ds_params_instance) data_store.graph_manager = graph_manager graph_manager.data_store = data_store training_worker(graph_manager=graph_manager, checkpoint_dir=args.checkpoint_dir, use_pretrained_model=use_pretrained_model, framework=args.framework)
def main(): screen.set_use_colors(False) logger.info("src/training_worker.py - INIZIO MAIN") parser = argparse.ArgumentParser() parser.add_argument('-pk', '--preset_s3_key', help="(string) Name of a preset to download from S3", type=str, required=False) parser.add_argument( '-ek', '--environment_s3_key', help="(string) Name of an environment file to download from S3", type=str, required=False) parser.add_argument('--model_metadata_s3_key', help="(string) Model Metadata File S3 Key", type=str, required=False) parser.add_argument( '-c', '--checkpoint-dir', help= '(string) Path to a folder containing a checkpoint to write the model to.', type=str, default='./checkpoint') parser.add_argument( '--pretrained-checkpoint-dir', help='(string) Path to a folder for downloading a pre-trained model', type=str, default=PRETRAINED_MODEL_DIR) parser.add_argument('--s3_bucket', help='(string) S3 bucket', type=str, default=os.environ.get( "SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test")) parser.add_argument('--s3_prefix', help='(string) S3 prefix', type=str, default='sagemaker') parser.add_argument('--s3_endpoint_url', help='(string) S3 endpoint URL', type=str, default=os.environ.get("S3_ENDPOINT_URL", None)) parser.add_argument('--framework', help='(string) tensorflow or mxnet', type=str, default='tensorflow') parser.add_argument('--pretrained_s3_bucket', help='(string) S3 bucket for pre-trained model', type=str) parser.add_argument('--pretrained_s3_prefix', help='(string) S3 prefix for pre-trained model', type=str, default='sagemaker') parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=os.environ.get("AWS_REGION", "us-east-1")) args, _ = parser.parse_known_args() logger.info("S3 bucket: %s \n S3 prefix: %s \n S3 endpoint URL: %s", args.s3_bucket, args.s3_prefix, args.s3_endpoint_url) s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region, s3_endpoint_url=args.s3_endpoint_url) # Load the model metadata model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH, 'model_metadata.json') utils.load_model_metadata(s3_client, args.model_metadata_s3_key, model_metadata_local_path) s3_client.upload_file( os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix), model_metadata_local_path) shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR) success_custom_preset = False if args.preset_s3_key: preset_local_path = "./markov/presets/preset.py" success_custom_preset = s3_client.download_file( s3_key=args.preset_s3_key, local_path=preset_local_path) if not success_custom_preset: logger.info( "Could not download the preset file. Using the default DeepRacer preset." ) else: preset_location = "markov.presets.preset:graph_manager" graph_manager = short_dynamic_import(preset_location, ignore_module_case=True) success_custom_preset = s3_client.upload_file( s3_key=os.path.normpath("%s/presets/preset.py" % args.s3_prefix), local_path=preset_local_path) if success_custom_preset: logger.info("Using preset: %s" % args.preset_s3_key) if not success_custom_preset: params_blob = os.environ.get('SM_TRAINING_ENV', '') if params_blob: params = json.loads(params_blob) sm_hyperparams_dict = params["hyperparameters"] else: sm_hyperparams_dict = {} #configurazione agente: metadati del modello impostati dall'utente (angolo di sterzo + velocità) + nome #! TODO each agent should have own config agent_config = { 'model_metadata': model_metadata_local_path, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [], ConfigParams.VELOCITY_LIST.value: {}, ConfigParams.STEERING_LIST.value: {}, ConfigParams.CHANGE_START.value: None, ConfigParams.ALT_DIR.value: None, ConfigParams.ACTION_SPACE_PATH.value: 'custom_files/model_metadata.json', ConfigParams.REWARD.value: None, ConfigParams.AGENT_NAME.value: 'racecar' } } agent_list = list() agent_list.append(create_training_agent(agent_config)) logger.info( "src/training_worker.py - ora chiamo la get_graph_manager, che recupera l'agente" ) graph_manager, robomaker_hyperparams_json = get_graph_manager( hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=None) logger.info("src/training_worker.py - ho l'agente") s3_client.upload_hyperparameters(robomaker_hyperparams_json) logger.info("Uploaded hyperparameters.json to S3") # Attach sample collector to graph_manager only if sample count > 0 max_sample_count = int(sm_hyperparams_dict.get("max_sample_count", 0)) if max_sample_count > 0: sample_collector = SampleCollector( s3_client=s3_client, s3_prefix=args.s3_prefix, max_sample_count=max_sample_count, sampling_frequency=int( sm_hyperparams_dict.get("sampling_frequency", 1))) graph_manager.sample_collector = sample_collector host_ip_address = utils.get_ip_from_host() s3_client.write_ip_config(host_ip_address) logger.info("Uploaded IP address information to S3: %s" % host_ip_address) use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix # Handle backward compatibility _, network_type, version = parse_model_metadata(model_metadata_local_path) if use_pretrained_model: if float(version) < float(SIMAPP_VERSION) and \ not utils.has_current_ckpnt_name(args.pretrained_s3_bucket, args.pretrained_s3_prefix, args.aws_region, args.s3_endpoint_url): utils.make_compatible(args.pretrained_s3_bucket, args.pretrained_s3_prefix, args.aws_region, SyncFiles.TRAINER_READY.value) #Select the optimal model for the starting weights utils.do_model_selection(s3_bucket=args.s3_bucket, s3_prefix=args.s3_prefix, region=args.aws_region, s3_endpoint_url=args.s3_endpoint_url) ds_params_instance_pretrained = S3BotoDataStoreParameters( aws_region=args.aws_region, bucket_names={'agent': args.pretrained_s3_bucket}, base_checkpoint_dir=args.pretrained_checkpoint_dir, s3_folders={'agent': args.pretrained_s3_prefix}, s3_endpoint_url=args.s3_endpoint_url) data_store_pretrained = S3BotoDataStore(ds_params_instance_pretrained, graph_manager, True) data_store_pretrained.load_from_store() memory_backend_params = DeepRacerRedisPubSubMemoryBackendParameters( redis_address="localhost", redis_port=6379, run_type=str(RunType.TRAINER), channel=args.s3_prefix, network_type=network_type) graph_manager.memory_backend_params = memory_backend_params ds_params_instance = S3BotoDataStoreParameters( aws_region=args.aws_region, bucket_names={'agent': args.s3_bucket}, base_checkpoint_dir=args.checkpoint_dir, s3_folders={'agent': args.s3_prefix}, s3_endpoint_url=args.s3_endpoint_url) graph_manager.data_store_params = ds_params_instance graph_manager.data_store = S3BotoDataStore(ds_params_instance, graph_manager) task_parameters = TaskParameters() task_parameters.experiment_path = SM_MODEL_OUTPUT_DIR task_parameters.checkpoint_save_secs = 20 if use_pretrained_model: task_parameters.checkpoint_restore_path = args.pretrained_checkpoint_dir task_parameters.checkpoint_save_dir = args.checkpoint_dir #funzione riga 48 #prende in input: # - il grafo (creato con la get_graph_manager) # - robomaker_hyperparams_json (ritornato dalla get_graph_manager) training_worker( graph_manager=graph_manager, task_parameters=task_parameters, user_batch_size=json.loads(robomaker_hyperparams_json)["batch_size"], user_episode_per_rollout=json.loads( robomaker_hyperparams_json)["num_episodes_between_training"])
def main(): screen.set_use_colors(False) parser = argparse.ArgumentParser() parser.add_argument('-pk', '--preset_s3_key', help="(string) Name of a preset to download from S3", type=str, required=False) parser.add_argument('-ek', '--environment_s3_key', help="(string) Name of an environment file to download from S3", type=str, required=False) parser.add_argument('--model_metadata_s3_key', help="(string) Model Metadata File S3 Key", type=str, required=False) parser.add_argument('-c', '--checkpoint-dir', help='(string) Path to a folder containing a checkpoint to write the model to.', type=str, default='./checkpoint') parser.add_argument('--pretrained-checkpoint-dir', help='(string) Path to a folder for downloading a pre-trained model', type=str, default=PRETRAINED_MODEL_DIR) parser.add_argument('--s3_bucket', help='(string) S3 bucket', type=str, default=os.environ.get("SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test")) parser.add_argument('--s3_prefix', help='(string) S3 prefix', type=str, default='sagemaker') parser.add_argument('--framework', help='(string) tensorflow or mxnet', type=str, default='tensorflow') parser.add_argument('--pretrained_s3_bucket', help='(string) S3 bucket for pre-trained model', type=str) parser.add_argument('--pretrained_s3_prefix', help='(string) S3 prefix for pre-trained model', type=str, default='sagemaker') parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=os.environ.get("AWS_REGION", "us-east-1")) start_redis_server() args, _ = parser.parse_known_args() s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region) # Load the model metadata model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH, 'model_metadata.json') utils.load_model_metadata(s3_client, args.model_metadata_s3_key, model_metadata_local_path) s3_client.upload_file(os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix), model_metadata_local_path) shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR) success_custom_preset = False if args.preset_s3_key: preset_local_path = "./markov/presets/preset.py" success_custom_preset = s3_client.download_file(s3_key=args.preset_s3_key, local_path=preset_local_path) if not success_custom_preset: logger.info("Could not download the preset file. Using the default DeepRacer preset.") else: preset_location = "markov.presets.preset:graph_manager" graph_manager = short_dynamic_import(preset_location, ignore_module_case=True) success_custom_preset = s3_client.upload_file( s3_key=os.path.normpath("%s/presets/preset.py" % args.s3_prefix), local_path=preset_local_path) if success_custom_preset: logger.info("Using preset: %s" % args.preset_s3_key) if not success_custom_preset: params_blob = os.environ.get('SM_TRAINING_ENV', '') if params_blob: params = json.loads(params_blob) sm_hyperparams_dict = params["hyperparameters"] else: sm_hyperparams_dict = {} #! TODO each agent should have own config agent_config = {'model_metadata': model_metadata_local_path, 'car_ctrl_cnfig': {ConfigParams.LINK_NAME_LIST.value: [], ConfigParams.VELOCITY_LIST.value : {}, ConfigParams.STEERING_LIST.value : {}, ConfigParams.CHANGE_START.value : None, ConfigParams.ALT_DIR.value : None, ConfigParams.ACTION_SPACE_PATH.value : 'custom_files/model_metadata.json', ConfigParams.REWARD.value : None, ConfigParams.AGENT_NAME.value : 'racecar'}} agent_list = list() agent_list.append(create_training_agent(agent_config)) #agent_list.append(create_training_agent(agent_config)) graph_manager, robomaker_hyperparams_json = get_graph_manager(sm_hyperparams_dict, agent_list) s3_client.upload_hyperparameters(robomaker_hyperparams_json) logger.info("Uploaded hyperparameters.json to S3") host_ip_address = utils.get_ip_from_host() s3_client.write_ip_config(host_ip_address) logger.info("Uploaded IP address information to S3: %s" % host_ip_address) use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix if use_pretrained_model: # Handle backward compatibility _, _, version = parse_model_metadata(model_metadata_local_path) if float(version) < float(utils.SIMAPP_VERSION) and \ not utils.has_current_ckpnt_name(args.pretrained_s3_bucket, args.pretrained_s3_prefix, args.aws_region): utils.make_compatible(args.pretrained_s3_bucket, args.pretrained_s3_prefix, args.aws_region, SyncFiles.TRAINER_READY.value) ds_params_instance_pretrained = S3BotoDataStoreParameters(aws_region=args.aws_region, bucket_name=args.pretrained_s3_bucket, checkpoint_dir=args.pretrained_checkpoint_dir, s3_folder=args.pretrained_s3_prefix) data_store_pretrained = S3BotoDataStore(ds_params_instance_pretrained) data_store_pretrained.load_from_store() memory_backend_params = RedisPubSubMemoryBackendParameters(redis_address="localhost", redis_port=6379, run_type=str(RunType.TRAINER), channel=args.s3_prefix) graph_manager.memory_backend_params = memory_backend_params ds_params_instance = S3BotoDataStoreParameters(aws_region=args.aws_region, bucket_name=args.s3_bucket, checkpoint_dir=args.checkpoint_dir, s3_folder=args.s3_prefix) graph_manager.data_store_params = ds_params_instance data_store = S3BotoDataStore(ds_params_instance) data_store.graph_manager = graph_manager graph_manager.data_store = data_store task_parameters = TaskParameters() task_parameters.experiment_path = SM_MODEL_OUTPUT_DIR task_parameters.checkpoint_save_secs = 20 if use_pretrained_model: task_parameters.checkpoint_restore_path = args.pretrained_checkpoint_dir task_parameters.checkpoint_save_dir = args.checkpoint_dir training_worker( graph_manager=graph_manager, task_parameters=task_parameters, user_batch_size=json.loads(robomaker_hyperparams_json)["batch_size"], user_episode_per_rollout=json.loads(robomaker_hyperparams_json)["num_episodes_between_training"] )
def main(): screen.set_use_colors(False) try: parser = argparse.ArgumentParser() parser.add_argument( '-pk', '--preset_s3_key', help="(string) Name of a preset to download from S3", type=str, required=False) parser.add_argument( '-ek', '--environment_s3_key', help="(string) Name of an environment file to download from S3", type=str, required=False) parser.add_argument('--model_metadata_s3_key', help="(string) Model Metadata File S3 Key", type=str, required=False) parser.add_argument( '-c', '--checkpoint-dir', help= '(string) Path to a folder containing a checkpoint to write the model to.', type=str, default='./checkpoint') parser.add_argument( '--pretrained-checkpoint-dir', help= '(string) Path to a folder for downloading a pre-trained model', type=str, default=PRETRAINED_MODEL_DIR) parser.add_argument('--s3_bucket', help='(string) S3 bucket', type=str, default=os.environ.get( "SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test")) parser.add_argument('--s3_prefix', help='(string) S3 prefix', type=str, default='sagemaker') parser.add_argument('--framework', help='(string) tensorflow or mxnet', type=str, default='tensorflow') parser.add_argument('--pretrained_s3_bucket', help='(string) S3 bucket for pre-trained model', type=str) parser.add_argument('--pretrained_s3_prefix', help='(string) S3 prefix for pre-trained model', type=str, default='sagemaker') parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=os.environ.get("AWS_REGION", "us-east-1")) args, unknown = parser.parse_known_args() start_redis_server() s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region) # Load the model metadata model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH, 'model_metadata.json') load_model_metadata(s3_client, args.model_metadata_s3_key, model_metadata_local_path) s3_client.upload_file( os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix), model_metadata_local_path) shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR) # Register the gym enviroment, this will give clients the ability to creat the enviroment object register(id=defaults.ENV_ID, entry_point=defaults.ENTRY_POINT, max_episode_steps=defaults.MAX_STEPS, reward_threshold=defaults.THRESHOLD) user_batch_size, user_episode_per_rollout = None, None success_custom_preset = False if args.preset_s3_key: preset_local_path = "./markov/presets/preset.py" success_custom_preset = s3_client.download_file( s3_key=args.preset_s3_key, local_path=preset_local_path) if not success_custom_preset: logger.info( "Could not download the preset file. Using the default DeepRacer preset." ) else: preset_location = "markov.presets.preset:graph_manager" graph_manager = short_dynamic_import(preset_location, ignore_module_case=True) success_custom_preset = s3_client.upload_file( s3_key=os.path.normpath("%s/presets/preset.py" % args.s3_prefix), local_path=preset_local_path) if success_custom_preset: agent_param_loc = "markov.presets.preset:agent_params" agent_params = short_dynamic_import( agent_param_loc, ignore_module_case=True) user_batch_size = agent_params.network_wrappers[ 'main'].batch_size user_episode_per_rollout = agent_params.algorithm.num_consecutive_playing_steps.num_steps logger.info("Using preset: %s" % args.preset_s3_key) if not success_custom_preset: from markov.sagemaker_graph_manager import get_graph_manager user_batch_size = json.loads( robomaker_hyperparams_json)["batch_size"], user_episode_per_rollout = json.loads( robomaker_hyperparams_json)["num_episodes_between_training"] params_blob = os.environ.get('SM_TRAINING_ENV', '') if params_blob: params = json.loads(params_blob) sm_hyperparams_dict = params["hyperparameters"] else: sm_hyperparams_dict = {} graph_manager, robomaker_hyperparams_json = get_graph_manager( **sm_hyperparams_dict) s3_client.upload_hyperparameters(robomaker_hyperparams_json) logger.info("Uploaded hyperparameters.json to S3") host_ip_address = get_ip_from_host() s3_client.write_ip_config(host_ip_address) logger.info("Uploaded IP address information to S3: %s" % host_ip_address) use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix if use_pretrained_model: s3_client_pretrained = SageS3Client( bucket=args.pretrained_s3_bucket, s3_prefix=args.pretrained_s3_prefix, aws_region=args.aws_region) s3_client_pretrained.download_model(args.pretrained_checkpoint_dir) memory_backend_params = RedisPubSubMemoryBackendParameters( redis_address="localhost", redis_port=6379, run_type='trainer', channel=args.s3_prefix) ds_params_instance = S3BotoDataStoreParameters( bucket_name=args.s3_bucket, checkpoint_dir=args.checkpoint_dir, aws_region=args.aws_region, s3_folder=args.s3_prefix) graph_manager.data_store_params = ds_params_instance data_store = S3BotoDataStore(ds_params_instance) data_store.graph_manager = graph_manager graph_manager.data_store = data_store training_worker(graph_manager=graph_manager, checkpoint_dir=args.checkpoint_dir, use_pretrained_model=use_pretrained_model, framework=args.framework, memory_backend_params=memory_backend_params, user_batch_size=user_batch_size, user_episode_per_rollout=user_episode_per_rollout) except Exception as ex: utils.json_format_logger( "Training worker exited with exception: {}".format(ex), **utils.build_system_error_dict( utils.SIMAPP_TRAINING_WORKER_EXCEPTION, utils.SIMAPP_EVENT_ERROR_CODE_500)) utils.simapp_exit_gracefully()
def main(): screen.set_use_colors(False) parser = argparse.ArgumentParser() parser.add_argument( '-c', '--checkpoint-dir', help= '(string) Path to a local folder containing a checkpoint to write the model to.', type=str, default='./checkpoint') parser.add_argument( '--pretrained-checkpoint-dir', help= '(string) Path to a local folder for downloading a pre-trained model', type=str, default=PRETRAINED_MODEL_DIR) parser.add_argument('--s3_bucket', help='(string) S3 bucket', type=str, default=os.environ.get( "SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test")) parser.add_argument('--s3_prefix', help='(string) S3 prefix', type=str, default='sagemaker') parser.add_argument('--framework', help='(string) tensorflow or mxnet', type=str, default='tensorflow') parser.add_argument('--pretrained_s3_bucket', help='(string) S3 bucket for pre-trained model', type=str) parser.add_argument('--pretrained_s3_prefix', help='(string) S3 prefix for pre-trained model', type=str, default='sagemaker') parser.add_argument('--RLCOACH_PRESET', help='(string) Default preset to use', type=str, default='object_tracker') parser.add_argument('--aws_region', help='(string) AWS region', type=str, required=True) args, unknown = parser.parse_known_args() s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region) # Import to register the environment with Gym import robomaker.environments preset_location = "robomaker.presets.%s:graph_manager" % args.RLCOACH_PRESET graph_manager = short_dynamic_import(preset_location, ignore_module_case=True) host_ip_address = get_ip_from_host() s3_client.write_ip_config(host_ip_address) print("Uploaded IP address information to S3: %s" % host_ip_address) use_pretrained_model = False if args.pretrained_s3_bucket and args.pretrained_s3_prefix: s3_client_pretrained = SageS3Client( bucket=args.pretrained_s3_bucket, s3_prefix=args.pretrained_s3_prefix, aws_region=args.aws_region) s3_client_pretrained.download_model(PRETRAINED_MODEL_DIR) use_pretrained_model = True memory_backend_params = RedisPubSubMemoryBackendParameters( redis_address="localhost", redis_port=6379, run_type='trainer', channel=args.s3_prefix) graph_manager.agent_params.memory.register_var('memory_backend_params', memory_backend_params) ds_params_instance = S3BotoDataStoreParameters( bucket_name=args.s3_bucket, checkpoint_dir=args.checkpoint_dir, s3_folder=args.s3_prefix, aws_region=args.aws_region) graph_manager.data_store_params = ds_params_instance data_store = S3BotoDataStore(ds_params_instance) data_store.graph_manager = graph_manager graph_manager.data_store = data_store training_worker(graph_manager=graph_manager, checkpoint_dir=args.checkpoint_dir, use_pretrained_model=use_pretrained_model, framework=args.framework)
def main(): screen.set_use_colors(False) parser = argparse.ArgumentParser() parser.add_argument('-c', '--checkpoint-dir', help='(string) Path to a local folder containing a checkpoint to write the model to.', type=str, default='./checkpoint') parser.add_argument('--pretrained-checkpoint-dir', help='(string) Path to a local folder for downloading a pre-trained model', type=str, default=PRETRAINED_MODEL_DIR) parser.add_argument('--s3_bucket', help='(string) S3 bucket', type=str, default=os.environ.get("SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test")) parser.add_argument('--s3_prefix', help='(string) S3 prefix', type=str, default='sagemaker') parser.add_argument('--framework', help='(string) tensorflow or mxnet', type=str, default='tensorflow') parser.add_argument('--pretrained_s3_bucket', help='(string) S3 bucket for pre-trained model', type=str) parser.add_argument('--pretrained_s3_prefix', help='(string) S3 prefix for pre-trained model', type=str, default='sagemaker') parser.add_argument('--RLCOACH_PRESET', help='(string) Default preset to use', type=str, default='deepracer') parser.add_argument('--aws_region', help='(string) AWS region', type=str, required=True) args, unknown = parser.parse_known_args() s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region) # Import to register the environment with Gym import robomaker.environments preset_location = "robomaker.presets.%s:graph_manager" % args.RLCOACH_PRESET graph_manager = short_dynamic_import(preset_location, ignore_module_case=True) host_ip_address = get_ip_from_host() s3_client.write_ip_config(host_ip_address) print("Uploaded IP address information to S3: %s" % host_ip_address) use_pretrained_model = False if args.pretrained_s3_bucket and args.pretrained_s3_prefix: s3_client_pretrained = SageS3Client(bucket=args.pretrained_s3_bucket, s3_prefix=args.pretrained_s3_prefix, aws_region=args.aws_region) s3_client_pretrained.download_model(PRETRAINED_MODEL_DIR) use_pretrained_model = True memory_backend_params = RedisPubSubMemoryBackendParameters(redis_address="localhost", redis_port=6379, run_type='trainer', channel=args.s3_prefix) graph_manager.agent_params.memory.register_var('memory_backend_params', memory_backend_params) ds_params_instance = S3BotoDataStoreParameters(bucket_name=args.s3_bucket, checkpoint_dir=args.checkpoint_dir, s3_folder=args.s3_prefix, aws_region=args.aws_region) graph_manager.data_store_params = ds_params_instance data_store = S3BotoDataStore(ds_params_instance) data_store.graph_manager = graph_manager graph_manager.data_store = data_store training_worker( graph_manager=graph_manager, checkpoint_dir=args.checkpoint_dir, use_pretrained_model=use_pretrained_model, framework=args.framework )