def experiment(args): device = torch.device( "cuda:{}".format(args.device) if args.cuda else "cpu") env = get_env(params['env_name'], params['env']) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) if args.cuda: torch.backends.cudnn.deterministic = True buffer_param = params['replay_buffer'] replay_buffer = BaseReplayBuffer(int(buffer_param['size'])) experiment_name = os.path.split( os.path.splitext( args.config )[0] )[-1] if args.id is None \ else args.id logger = Logger(experiment_name, params['env_name'], args.seed, params, args.log_dir) params['general_setting']['env'] = env params['general_setting']['replay_buffer'] = replay_buffer params['general_setting']['logger'] = logger params['general_setting']['device'] = device params['net']['base_type'] = networks.MLPBase # agent = get_agent( params ) # print(env) # params['general_setting']['collector'] = BaseCollector( # env, pf, replay_buffer # ) pf = policies.GuassianContPolicy( input_shape=env.observation_space.shape[0], output_shape=2 * env.action_space.shape[0], **params['net']) vf = networks.Net(input_shape=env.observation_space.shape[0], output_shape=1, **params['net']) qf = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) pretrain_pf = policies.UniformPolicyContinuous(env.action_space.shape[0]) params['general_setting']['collector'] = BaseCollector(env, pf, replay_buffer, device=device) params['general_setting']['save_dir'] = osp.join(logger.work_dir, "model") agent = SAC(pf=pf, vf=vf, qf=qf, pretrain_pf=pretrain_pf, **params['sac'], **params['general_setting']) agent.train()
def experiment(args): device = torch.device( "cuda:{}".format(args.device) if args.cuda else "cpu") env = get_vec_env(params["env_name"], params["env"], args.vec_env_nums) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) if args.cuda: torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True buffer_param = params['replay_buffer'] experiment_name = os.path.split( os.path.splitext(args.config)[0])[-1] if args.id is None \ else args.id logger = Logger(experiment_name, params['env_name'], args.seed, params, args.log_dir, args.overwrite) params['general_setting']['env'] = env replay_buffer = OnPolicyReplayBuffer( env_nums=args.vec_env_nums, max_replay_buffer_size=int(buffer_param['size']), time_limit_filter=buffer_param['time_limit_filter']) params['general_setting']['replay_buffer'] = replay_buffer params['general_setting']['logger'] = logger params['general_setting']['device'] = device params['net']['base_type'] = networks.MLPBase params['net']['activation_func'] = torch.nn.Tanh pf = policies.GuassianContPolicyBasicBias( input_shape=env.observation_space.shape[0], output_shape=env.action_space.shape[0], **params['net'], **params['policy']) vf = networks.Net(input_shape=env.observation_space.shape, output_shape=1, **params['net']) print(pf) print(vf) params['general_setting']['collector'] = VecOnPolicyCollector( vf, env=env, pf=pf, replay_buffer=replay_buffer, device=device, train_render=False, **params["collector"]) params['general_setting']['save_dir'] = osp.join(logger.work_dir, "model") agent = A2C(pf=pf, vf=vf, **params["a2c"], **params["general_setting"]) agent.train()
def experiment(args): import torch.multiprocessing as mp mp.set_start_method('spawn') device = torch.device("cuda:{}".format(args.device) if args.cuda else "cpu") env = get_env( params['env_name'], params['env']) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) if args.cuda: torch.backends.cudnn.deterministic=True buffer_param = params['replay_buffer'] experiment_name = os.path.split( os.path.splitext( args.config )[0] )[-1] if args.id is None \ else args.id logger = Logger( experiment_name , params['env_name'], args.seed, params, args.log_dir ) params['general_setting']['env'] = env # replay_buffer = OnPolicyReplayBuffer(int(buffer_param['size'])) # example_ob = env.reset() # example_dict = { # "obs": example_ob, # "next_obs": example_ob, # "acts": env.action_space.sample(), # "values": [0], # "rewards": [0], # "terminals": [False] # } replay_buffer = OnPolicyReplayBuffer( int(buffer_param['size'])) # replay_buffer.build_by_example(example_dict) params['general_setting']['replay_buffer'] = replay_buffer params['general_setting']['logger'] = logger params['general_setting']['device'] = device params['net']['base_type']=networks.MLPBase pf = policies.CategoricalDisPolicy( input_shape = env.observation_space.shape[0], output_shape = env.action_space.n, **params['net'], **params['policy'] ) vf = networks.Net( input_shape = env.observation_space.shape, output_shape = 1, **params['net'] ) params['general_setting']['collector'] = OnPlicyCollectorBase( vf, env = env, pf = pf, replay_buffer = replay_buffer, device = "cuda", train_render=False ) # params['general_setting']['collector'] = ParallelOnPlicyCollector( # vf, env = env, pf = pf, replay_buffer = replay_buffer, device=device, worker_nums=2 # ) params['general_setting']['save_dir'] = osp.join(logger.work_dir,"model") agent = PPO( pf = pf, vf = vf, **params["ppo"], **params["general_setting"] ) agent.train()
def get_agent(params): env = params['general_setting']['env'] # params['general_setting']['collector'] = BaseCollector( # env # ) if len(env.observation_space.shape) == 3: params['net']['base_type'] = networks.CNNBase if params['env']['frame_stack']: buffer_param = params['replay_buffer'] efficient_buffer = replay_buffers.MemoryEfficientReplayBuffer( int(buffer_param['size'])) params['general_setting']['replay_buffer'] = efficient_buffer else: params['net']['base_type'] = networks.MLPBase if params['agent'] == 'sac': pf = policies.GuassianContPolicy( input_shape=env.observation_space.shape[0], output_shape=2 * env.action_space.shape[0], **params['net']) vf = networks.Net(input_shape=env.observation_space.shape[0], output_shape=1, **params['net']) qf = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) pretrain_pf = policies.UniformPolicyContinuous( env.action_space.shape[0]) return SAC(pf=pf, vf=vf, qf=qf, pretrain_pf=pretrain_pf, **params['sac'], **params['general_setting']) if params['agent'] == 'twin_sac': pf = policies.GuassianContPolicy( input_shape=env.observation_space.shape[0], output_shape=2 * env.action_space.shape[0], **params['net']) vf = networks.Net(input_shape=env.observation_space.shape[0], output_shape=1, **params['net']) qf1 = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) qf2 = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) pretrain_pf = policies.UniformPolicyContinuous( env.action_space.shape[0]) return TwinSAC(pf=pf, vf=vf, qf1=qf1, qf2=qf2, pretrain_pf=pretrain_pf, **params['twin_sac'], **params['general_setting']) if params['agent'] == 'td3': pf = policies.DetContPolicy(input_shape=env.observation_space.shape[0], output_shape=env.action_space.shape[0], **params['net']) qf1 = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) qf2 = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) pretrain_pf = policies.UniformPolicyContinuous( env.action_space.shape[0]) return TD3(pf=pf, qf1=qf1, qf2=qf2, pretrain_pf=pretrain_pf, **params['td3'], **params['general_setting']) if params['agent'] == 'ddpg': pf = policies.DetContPolicy(input_shape=env.observation_space.shape[0], output_shape=env.action_space.shape[0], **params['net']) qf = networks.FlattenNet(input_shape=env.observation_space.shape[0] + env.action_space.shape[0], output_shape=1, **params['net']) pretrain_pf = policies.UniformPolicyContinuous( env.action_space.shape[0]) return DDPG(pf=pf, qf=qf, pretrain_pf=pretrain_pf, **params['ddpg'], **params['general_setting']) if params['agent'] == 'dqn': qf = networks.Net(input_shape=env.observation_space.shape, output_shape=env.action_space.n, **params['net']) pf = policies.EpsilonGreedyDQNDiscretePolicy( qf=qf, action_shape=env.action_space.n, **params['policy']) pretrain_pf = policies.UniformPolicyDiscrete( action_num=env.action_space.n) params["general_setting"]["optimizer_class"] = optim.RMSprop return DQN(pf=pf, qf=qf, pretrain_pf=pretrain_pf, **params["dqn"], **params["general_setting"]) if params['agent'] == 'bootstrapped dqn': qf = networks.BootstrappedNet( input_shape=env.observation_space.shape, output_shape=env.action_space.n, head_num=params['bootstrapped dqn']['head_num'], **params['net']) pf = policies.BootstrappedDQNDiscretePolicy( qf=qf, head_num=params['bootstrapped dqn']['head_num'], action_shape=env.action_space.n, **params['policy']) pretrain_pf = policies.UniformPolicyDiscrete( action_num=env.action_space.n) params["general_setting"]["optimizer_class"] = optim.RMSprop return BootstrappedDQN(pf=pf, qf=qf, pretrain_pf=pretrain_pf, **params["bootstrapped dqn"], **params["general_setting"]) if params['agent'] == 'qrdqn': qf = networks.Net(input_shape=env.observation_space.shape, output_shape=env.action_space.n * params["qrdqn"]["quantile_num"], **params['net']) pf = policies.EpsilonGreedyQRDQNDiscretePolicy( qf=qf, action_shape=env.action_space.n, **params['policy']) pretrain_pf = policies.UniformPolicyDiscrete( action_num=env.action_space.n) return QRDQN(pf=pf, qf=qf, pretrain_pf=pretrain_pf, **params["qrdqn"], **params["general_setting"]) # On Policy Methods act_space = env.action_space params[params['agent']]['continuous'] = isinstance(act_space, gym.spaces.Box) buffer_param = params['replay_buffer'] buffer = replay_buffers.OnPolicyReplayBuffer(int(buffer_param['size'])) params['general_setting']['replay_buffer'] = buffer if params[params['agent']]['continuous']: pf = policies.GuassianContPolicy( input_shape=env.observation_space.shape, output_shape=2 * env.action_space.shape[0], **params['net']) else: print(params['policy']) print(params['net']) # print(**params['policy']) pf = policies.CategoricalDisPolicy( input_shape=env.observation_space.shape, output_shape=env.action_space.n, **params['net'], **params['policy']) if params['agent'] == 'reinforce': return Reinforce(pf=pf, **params["reinforce"], **params["general_setting"]) # Actor-Critic Frameworks vf = networks.Net(input_shape=env.observation_space.shape, output_shape=1, **params['net']) if params['agent'] == 'a2c': return A2C(pf=pf, vf=vf, **params["a2c"], **params["general_setting"]) if params['agent'] == 'ppo': return PPO(pf=pf, vf=vf, **params["ppo"], **params["general_setting"]) raise Exception("specified algorithm is not implemented")
def experiment(args): device = torch.device("cuda:{}".format(args.device) if args.cuda else "cpu") env = get_env( params['env_name'], params['env']) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) if args.cuda: torch.backends.cudnn.deterministic=True buffer_param = params['replay_buffer'] experiment_name = os.path.split( os.path.splitext( args.config )[0] )[-1] if args.id is None \ else args.id logger = Logger( experiment_name , params['env_name'], args.seed, params, args.log_dir ) params['general_setting']['env'] = env params['general_setting']['logger'] = logger params['general_setting']['device'] = device params['net']['base_type']=networks.MLPBase import torch.multiprocessing as mp mp.set_start_method('spawn') pf = policies.GuassianContPolicy ( input_shape = env.observation_space.shape[0], output_shape = 2 * env.action_space.shape[0], **params['net'] ) vf = networks.Net( input_shape = env.observation_space.shape[0], output_shape = 1, **params['net'] ) qf = networks.FlattenNet( input_shape = env.observation_space.shape[0] + env.action_space.shape[0], output_shape = 1, **params['net'] ) # pretrain_pf = policies.UniformPolicyContinuous(env.action_space.shape[0]) example_ob = env.reset() example_dict = { "obs": example_ob, "next_obs": example_ob, "acts": env.action_space.sample(), "rewards": [0], "terminals": [False] } replay_buffer = SharedBaseReplayBuffer( int(buffer_param['size']), 1 ) replay_buffer.build_by_example(example_dict) params['general_setting']['replay_buffer'] = replay_buffer params['general_setting']['collector'] = ParallelCollector( env, pf, replay_buffer, device=device, worker_nums=1 ) params['general_setting']['save_dir'] = osp.join(logger.work_dir,"model") agent = SAC( pf = pf, vf = vf, qf = qf, **params['sac'], **params['general_setting'] ) agent.train()
def experiment(args): import torch.multiprocessing as mp mp.set_start_method('spawn') device = torch.device( "cuda:{}".format(args.device) if args.cuda else "cpu") # env = get_env(params['env_name'], params['env']) env = VecEnv(4, get_env, [params['env_name'], params['env']]) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) if args.cuda: torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True buffer_param = params['replay_buffer'] experiment_name = os.path.split(os.path.splitext(args.config)[0])[-1] if args.id is None \ else args.id logger = Logger(experiment_name, params['env_name'], args.seed, params, args.log_dir) params['general_setting']['env'] = env # replay_buffer = OnPolicyReplayBuffer(int(buffer_param['size'])) # example_ob = env.reset() # example_dict = { # "obs": example_ob, # "next_obs": example_ob, # "acts": env.action_space.sample(), # "values": [0], # "rewards": [0], # "terminals": [False] # } # replay_buffer = SharedOnPolicyReplayBuffer( int(buffer_param['size']), # args.worker_nums # ) # replay_buffer.build_by_example(example_dict) replay_buffer = VecOnPolicyReplayBuffer( env_nums=2, max_replay_buffer_size=int(buffer_param['size']), time_limit_filter=buffer_param['time_limit_filter']) params['general_setting']['replay_buffer'] = replay_buffer params['general_setting']['logger'] = logger params['general_setting']['device'] = device params['net']['base_type'] = networks.MLPBase params['net']['activation_func'] = torch.tanh pf = policies.GuassianContPolicyBasicBias( input_shape=env.observation_space.shape[0], output_shape=env.action_space.shape[0], init_func=lambda x: init.orthogonal_init( x, scale=np.sqrt(2), constant=0), net_last_init_func=lambda x: init.orthogonal_init( x, scale=0.01, constant=0), **params['net'], **params['policy']) vf = networks.Net(input_shape=env.observation_space.shape, output_shape=1, init_func=lambda x: init.orthogonal_init( x, scale=np.sqrt(2), constant=0), net_last_init_func=lambda x: init.orthogonal_init( x, scale=1, constant=0), **params['net']) # params['general_setting']['collector'] = OnPlicyCollectorBase( # vf, env=env, pf=pf, replay_buffer=replay_buffer, device=device, # train_render=False # ) params['general_setting']['collector'] = VecOnPlicyCollector( vf, env=env, pf=pf, replay_buffer=replay_buffer, device=device, train_render=False, epoch_frames=params["general_setting"]["epoch_frames"] // 2) params['general_setting']['save_dir'] = osp.join(logger.work_dir, "model") agent = PPO(pf=pf, vf=vf, **params["ppo"], **params["general_setting"]) agent.train()
def experiment(args): # import torch.multiprocessing as mp # mp.set_start_method('spawn') device = torch.device( "cuda:{}".format(args.device) if args.cuda else "cpu") env = get_env(params['env_name'], params['env']) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) if args.cuda: torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True buffer_param = params['replay_buffer'] experiment_name = os.path.split(os.path.splitext(args.config)[0])[-1] if args.id is None \ else args.id logger = Logger(experiment_name, params['env_name'], args.seed, params, args.log_dir) params['general_setting']['env'] = env replay_buffer = OnPolicyReplayBuffer( int(buffer_param['size']), time_limit_filter=buffer_param['time_limit_filter']) params['general_setting']['replay_buffer'] = replay_buffer params['general_setting']['logger'] = logger params['general_setting']['device'] = device params['net']['base_type'] = networks.MLPBase params['net']['activation_func'] = nn.Tanh pf = policies.GuassianContPolicyBasicBias( input_shape=env.observation_space.shape[0], output_shape=env.action_space.shape[0], init_func=lambda x: init.orthogonal_init( x, scale=np.sqrt(2), constant=0), net_last_init_func=lambda x: init.orthogonal_init( x, scale=0.01, constant=0), **params['net'], **params['policy']) vf = networks.Net(input_shape=env.observation_space.shape, output_shape=1, init_func=lambda x: init.orthogonal_init( x, scale=np.sqrt(2), constant=0), net_last_init_func=lambda x: init.orthogonal_init( x, scale=1, constant=0), **params['net']) params['general_setting']['collector'] = OnPlicyCollectorBase( vf, env=env, pf=pf, replay_buffer=replay_buffer, device=device, train_render=False, **params["collector"]) params['general_setting']['save_dir'] = osp.join(logger.work_dir, "model") agent = TRPO(pf=pf, vf=vf, **params["trpo"], **params["general_setting"]) print(params["general_setting"]) print(agent.epoch_frames) agent.train()