def __init__(self, state_size, action_size, random_seed): """ Args: ====== state_size (int): state dim action_size (int): action dim random_seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(random_seed) # actor net initialization self.actor_local = Actor(state_size, action_size, random_seed).to(device) self.actor_target = Actor(state_size, action_size, random_seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # critic net initialization self.critic_local = Critic(state_size, action_size, random_seed).to(device) self.critic_target = Critic(state_size, action_size, random_seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Ornstein-Uhlenbeck Exploration Noise Process self.noise = OUNoise(action_space=action_size, seed=random_seed) # Replay memory init self.memory = Memory(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
def __init__(self, hidden_size, env): self.num_states = env.observation_space.shape[0] self.num_actions = env.action_space.shape[0] self.Actor = Actor(input_size=self.num_states, hidden_size=hidden_size, output_size=self.num_actions).cuda() self.Actor_target = Actor(input_size=self.num_states, hidden_size=hidden_size, output_size=self.num_actions).cuda() self.Critic = Critic(input_size=self.num_states, hidden_size=hidden_size, output_size=self.num_actions).cuda() self.Critic_target = Critic(input_size=self.num_states, hidden_size=hidden_size, output_size=self.num_actions).cuda() for target_param, param in zip(self.Actor_target.parameters(), self.Actor.parameters()): target_param.data = param.data for target_param, param in zip(self.Critic_target.parameters(), self.Critic.parameters()): target_param.data = param.data self.Memory = Memory(30000) self.criterion = nn.MSELoss().cuda() self.actor_optimizer = torch.optim.Adam(self.Actor.parameters(), lr=1e-2) self.critic_optimizer = torch.optim.Adam(self.Critic.parameters(), lr=1e-1)
def collect_samples(self, mini_batch_size, size=1): num_step = 0 memory = Memory() while num_step < mini_batch_size: discrete_state, continuous_state = self.reset(size) for walk_step in range(self.max_traj_length - 1): with torch.no_grad(): discrete_action, continuous_action, next_discrete_state, next_continuous_state, old_log_prob = self.step( discrete_state, continuous_state, size) # Currently we assume the exploration step is not done until it reaches max_traj_length. mask = torch.ones((size, 1), device=device) memory.push(discrete_state.type(FloatTensor), continuous_state, discrete_action.type(FloatTensor), continuous_action, next_discrete_state.type(FloatTensor), next_continuous_state, old_log_prob, mask) discrete_state, continuous_state = next_discrete_state, next_continuous_state num_step += 1 if num_step >= mini_batch_size: return memory # one more step for push done with torch.no_grad(): discrete_action, continuous_action, next_discrete_state, next_continuous_state, old_log_prob = self.step( discrete_state, continuous_state, size) mask = torch.zeros((size, 1), device=device) memory.push(discrete_state.type(FloatTensor), continuous_state, discrete_action.type(FloatTensor), continuous_action, next_discrete_state.type(FloatTensor), next_continuous_state, old_log_prob, mask) num_step += 1 return memory
def inference_speed_memory(self, batch_size, seq_length): # input_ids = np.random.randint(0, self.vocab_size, (batch_size, seq_length)) key = jax.random.PRNGKey(0) input_ids = jax.random.randint(key, (batch_size, seq_length), 0, self.vocab_size) @jax.jit def ref_step(): out = self.model(input_ids=input_ids) return out[0] if jax.local_devices()[0].platform == 'gpu': nvml.nvmlInit() ref_step().block_until_ready() handle = nvml.nvmlDeviceGetHandleByIndex(0) meminfo = nvml.nvmlDeviceGetMemoryInfo(handle) max_bytes_in_use = meminfo.used memory = Memory(max_bytes_in_use) # shutdown nvml nvml.nvmlShutdown() else: memory = None timeit.repeat("ref_step().block_until_ready()", repeat=1, number=2,globals=locals()) if self.jit: runtimes = timeit.repeat("ref_step().block_until_ready()", repeat=self.repeat,number=3,globals=locals()) else: with jax.disable_jit(): runtimes = timeit.repeat("ref_step().block_until_ready()",repeat=self.repeat,number=3,globals=locals()) return float(np.min(runtimes)/3.0), memory
def collect_samples(self, batch_size=1): memory = Memory() num_trajs = (batch_size + args.sample_traj_length - 1) // args.sample_traj_length onehot_state, multihot_state, continuous_state = self.reset(num_trajs) for walk_step in range(self.max_traj_length - 1): with torch.no_grad(): onehot_action, multihot_action, continuous_action, next_onehot_state, next_multihot_state, next_continuous_state, old_log_prob = self.step( onehot_state, multihot_state, continuous_state, num_trajs) # Currently we assume the exploration step is not done until it reaches max_traj_length. mask = torch.ones((num_trajs, 1), device=device) memory.push(onehot_state.type(FloatTensor), multihot_state.type(FloatTensor), continuous_state, onehot_action.type(FloatTensor), multihot_action.type(FloatTensor), continuous_action, next_onehot_state.type(FloatTensor), next_multihot_state.type(FloatTensor), next_continuous_state, old_log_prob, mask) onehot_state, multihot_state, continuous_state = next_onehot_state, next_multihot_state, next_continuous_state # one more step for push done with torch.no_grad(): onehot_action, multihot_action, continuous_action, next_onehot_state, next_multihot_state, next_continuous_state, old_log_prob = self.step( onehot_state, multihot_state, continuous_state, num_trajs) mask = torch.zeros((num_trajs, 1), device=device) memory.push(onehot_state.type(FloatTensor), multihot_state.type(FloatTensor), continuous_state, onehot_action.type(FloatTensor), multihot_action.type(FloatTensor), continuous_action, next_onehot_state.type(FloatTensor), next_multihot_state.type(FloatTensor), next_continuous_state, old_log_prob, mask) return memory, num_trajs
def __init__(self, num_actions, checkpoint=None): self.network, self.trainable_parameters = self.init_network( num_actions) self.optimizer = torch.optim.Adam(self.trainable_parameters, lr=1e-4) self.memory = Memory() if checkpoint is not None: load_checkpoint(self.network, self.optimizer, checkpoint)
def __init__(self, model_func, n_way, n_support, jigsaw=False, lbda=0.0, rotation=False, tracking=False, use_bn=True, pretrain=False, image_loader=None, len_dataset=None): super(ProtoNet, self).__init__(model_func, n_way, n_support, use_bn, pretrain) self.loss_fn = nn.CrossEntropyLoss() self.len_dataset = len_dataset self.cuda() self.memory = Memory(size=len_dataset, weight=0.5, device='cuda') self.memory.initialize(self.feature, image_loader) self.jigsaw = jigsaw self.rotation = rotation self.lbda = lbda self.global_count = 0 self.indx = 0 if self.jigsaw: self.projection_transformed_features = nn.Linear( 512 * 9, 512) ### Self-supervision branch #self.fc6 = nn.Sequential() #self.fc6.add_module('fc6_s1',nn.Linear(512, 512))#for resnet #self.fc6.add_module('relu6_s1',nn.ReLU(inplace=True)) #self.fc6.add_module('drop6_s1',nn.Dropout(p=0.5)) #self.fc7 = nn.Sequential() #self.fc7.add_module('fc7',nn.Linear(9*512,4096))#for resnet #self.fc7.add_module('relu7',nn.ReLU(inplace=True)) #self.fc7.add_module('drop7',nn.Dropout(p=0.5)) #self.classifier = nn.Sequential() #self.classifier.add_module('fc8',nn.Linear(4096, 35)) if self.rotation: self.fc6 = nn.Sequential() self.fc6.add_module('fc6_s1', nn.Linear(512, 512)) #for resnet self.fc6.add_module('relu6_s1', nn.ReLU(inplace=True)) self.fc6.add_module('drop6_s1', nn.Dropout(p=0.5)) self.fc7 = nn.Sequential() self.fc7.add_module('fc7', nn.Linear(512, 128)) #for resnet self.fc7.add_module('relu7', nn.ReLU(inplace=True)) self.fc7.add_module('drop7', nn.Dropout(p=0.5)) self.classifier_rotation = nn.Sequential() self.classifier_rotation.add_module('fc8', nn.Linear(128, 4))
def main(): parser = argparse.ArgumentParser() parser.add_argument("--environment", type=str, default='Pendulum-v0') parser.add_argument("--time-steps", type=int, default=30000) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--learning-rate", type=float, default=.9) args = parser.parse_args() env = gym.make(args.environment) unroll = 20 state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] action_bound_high = env.action_space.high action_bound_low = env.action_space.low agent = direct_policy_search(state_dim, action_dim, action_bound_high, action_bound_low, unroll, .9, 5, 'direct_policy_search') # Replay memory memory = Memory(args.replay_mem_size) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) state = env.reset() total_rewards = 0.0 epoch = 1 for time_steps in range(args.time_steps): #env.render() action = agent.act(sess, state) next_state, reward, done, _ = env.step(action) total_rewards += float(reward) # Store tuple in replay memory memory.add([ np.atleast_2d(state), np.atleast_2d(action), reward, np.atleast_2d(next_state), done ]) # Training step batch = np.array(memory.sample(args.batch_size)) assert len(batch) > 0 states = np.concatenate(batch[:, 0], axis=0) # Train the agent agent.train(sess, states) # s <- s' state = np.copy(next_state) if done == True: print 'time steps', time_steps, 'epoch', epoch, 'total rewards', total_rewards, 'unroll', unroll epoch += 1 total_rewards = 0. state = env.reset()
def main2(): # Initialize environment. import gym env = gym.make('Pendulum-v0') state_size = env.observation_space.shape[0] action_size = env.action_space.shape[0] action_bound_high = env.action_space.high action_bound_low = env.action_space.low # Initialize agent. pai = PAI(environment='Pendulum-v0', state_size=state_size, action_size=action_size, hidden_size=20, it_tloop=100, it_dyn=5000, bs_dyn=100, it_policy=1000, bs_policy=50, K=50, T=25, action_bound_low=action_bound_low, action_bound_high=action_bound_high, discount_factor=.9) # Initialize replay memory memory = Memory( 400 * 10 ) #Data from most recent 10 trials (each trial is 400 time steps long). with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(300): total_rewards = 0. state = env.reset() while True: action = pai.act(sess, state) next_state, reward, done, _ = env.step(action) total_rewards += float(reward) # Store tuple in replay memory memory.add([ np.atleast_2d(state), np.atleast_2d(action), reward, np.atleast_2d(next_state), done ]) # s <- s' state = np.copy(next_state) if done == True: print 'epoch', epoch, 'total rewards', total_rewards # Train the agent pai.train(sess, memory) break
def _rollout_with_memory(self, env, network, args, running_state, max_episode_steps, keep_memory=False): memory = Memory() num_steps = 0 reward_list = [] len_list = [] while num_steps < args.batch_size: state = env.reset() if args.state_norm: state = running_state(state) if args.append_time: state = np.append(state, 1.0) reward_sum = 0 for t in range(max_episode_steps): action_mean, action_std, value = network( Tensor(state).unsqueeze(0)) action_mean = action_mean[0] action_std = action_std[0] action, y = network.select_action(action_mean, action_std) action_mean = action_mean.data.numpy() action = action.data.numpy() y = y.data.numpy() next_state, reward, done, info = env.step(action) reward_sum += reward if args.state_norm: next_state = running_state(next_state) if args.append_time: next_state = np.append(next_state, 1 - (t + 1) / max_episode_steps) mask = 0 if (done or ((t + 1) == max_episode_steps)) else 1 memory.push(state, value, action_mean, action, y, mask, next_state, reward) if done: break state = next_state num_steps += (t + 1) reward_list.append(reward_sum) len_list.append(t + 1) meanepreward = np.mean(reward_list) meaneplen = np.mean(len_list) if keep_memory: self.memory = memory self.old_std = network.action_std.data return meanepreward, meaneplen else: return memory, meanepreward, meaneplen, num_steps
def __init__( self, action_dim, filters_C, kernel_size, hidden_R, dropout, dropout_r, Hstep, activation, is_training_mode, ): self.policy_clip = 0.2 self.value_clip = 0.2 self.entropy_coef = 0.0 self.vf_loss_coef = 0.5 self.minibatch = 32 self.PPO_epochs = 10 # TODO use predicted results action_std = 1.0 self.cov_mat = tf.fill([action_dim], action_std ** 2) self.is_training_mode = is_training_mode self.actor = Actor( action_dim, filters_C, kernel_size, hidden_R, dropout, dropout_r, activation, Hstep, ) self.actor_old = Actor( action_dim, filters_C, kernel_size, hidden_R, dropout, dropout_r, activation, Hstep, ) self.critic = Critic( filters_C, kernel_size, hidden_R, dropout, dropout_r, activation, Hstep ) self.critic_old = Critic( filters_C, kernel_size, hidden_R, dropout, dropout_r, activation, Hstep ) self.optimizer = tf.keras.optimizers.Adam(learning_rate=2e-4) self.memory = Memory() self.utils = Utils()
def train_speed_memory(self, batch_size, seq_length): key = jax.random.PRNGKey(0) input_ids = jax.random.randint(key, (batch_size, seq_length), 0, self.vocab_size) targets = jax.random.randint(key, (batch_size, seq_length), 0, self.vocab_size) labels = jax.random.randint(key, (batch_size, seq_length), 0, 2) # input_ids = np.random.randint(0, self.vocab_size, (batch_size, seq_length)) # targets = np.random.randint(0, self.vocab_size, (batch_size, seq_length)) # labels = np.random.randint(0,2, (batch_size, seq_length)) @jax.jit def train_step(): def loss_fn(params): token_mask = jnp.where(labels > 0, 1.0, 0.0).astype(self.dtype) logits = self.model(input_ids=input_ids, train=True, params=params, dropout_rng=jax.random.PRNGKey(0))[0] loss, normalizing_factor = cross_entropy(logits,targets, token_mask) jax.profiler.save_device_memory_profile(f"memory/{workload[0]}_{workload[1]}_memory.prof", "gpu") return loss / normalizing_factor if self.fp16 and jax.local_devices()[0].platform == 'gpu': grad_fn = self.dynamic_scale.value_and_grad(loss_fn) dyn_scale, is_fin, loss, grad = grad_fn(self.model.params) else: grad_fn = jax.value_and_grad(loss_fn) loss, grad = grad_fn(self.model.params) return tree_flatten(grad)[0] if jax.local_devices()[0].platform == 'gpu': nvml.nvmlInit() train_step() handle = nvml.nvmlDeviceGetHandleByIndex(0) meminfo = nvml.nvmlDeviceGetMemoryInfo(handle) max_bytes_in_use = meminfo.used memory = Memory(max_bytes_in_use) # shutdown nvml nvml.nvmlShutdown() else: memory = None # timeit.repeat(train_step,repeat=1,number=2) timeit.repeat("for i in train_step():i.block_until_ready()", repeat=1, number=2,globals=locals()) if self.jit: # runtimes = timeit.repeat(train_step,repeat=self.repeat,number=3) runtimes = timeit.repeat("for i in train_step():i.block_until_ready()", repeat=self.repeat, number=3,globals=locals()) else: with jax.disable_jit(): # runtimes = timeit.repeat(train_step, repeat=self.repeat, number=3) runtimes = timeit.repeat("for i in train_step():i.block_until_ready()", repeat=self.repeat, number=3,globals=locals()) return float(np.min(runtimes)/3.0), memory
def train_feature_extractor(self, sess, replay_buffer, batch_size=100, iterations=1, iterations_left=-1): try: self.buff except: self.buff = Memory(10000) for it in range(iterations): while len(self.buff.mem) < batch_size: state = replay_buffer.sample(1)[0][0] patches = [] for i in range(0, state.shape[1]-self.w+1, self.s): for j in range(0, state.shape[2]-self.w+1, self.s): patches.append(state[0, i:i+self.w, j:j+self.w, :]) assert patches[-1].shape[0] == patches[-1].shape[1] assert patches[-1].shape[0] == self.w from random import shuffle shuffle(patches) self.buff.mem += patches batch = self.buff.mem[:batch_size] self.buff.mem = self.buff.mem[batch_size:] batch = np.concatenate([b[np.newaxis, ...] for b in batch], axis=0) batch = self.process_states(batch) batch = [b.astype(np.float64) / 255. for b in batch] feed_dict = {} for i in range(self.no_inputs): feed_dict[self.params[i]['input']] = batch[i] _, recon_loss, = sess.run([self.update_model_recon, self.recon_loss], feed_dict=feed_dict) print "train_feature_extractor - recon_loss:", recon_loss ######################################################################################################################## if iterations_left <= 10: variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) import pickle recon_x, recon_y, conv1, conv2 = sess.run([self.params[0]['recon'], self.params[1]['recon'], variables[2], variables[4]], feed_dict=feed_dict) pickle.dump( [conv1, conv2, batch, recon_x, recon_y], open( "recons.p", "wb" ) ) ######################################################################################################################## return iterations
def __init__(self, stack_size=256, *args, **kwargs): self.stack = Stack(size=stack_size) self.memory = Memory(memory=kwargs.get('memory', b'')) self.storage = dict() self.args = args self.kwargs = kwargs self.opcode = None self.current_opcode_name = None self.code = None self.msg = None self.code = None self.debug = False self.pc = 0 self.prev_pc = -1 self.stop = False
def __init__(self, env, batch_size=32, gamma=0.99, hidden_units=32, maxlen=10000, tau=0.1, actor_lr=0.001, critic_lr=0.001): self.env=env self.batch_size=batch_size self.gamma=gamma self.maxlen=maxlen self.sess=tf.Session() self.actor=Actor(env, self.sess, hidden_units, tau, actor_lr) self.critic=Critic(env, self.sess, hidden_units, tau, critic_lr) self.memory=Memory(maxlen) self.sess.run(tf.global_variables_initializer()) self.step=0
def train_feature_extractor(self, sess, replay_buffer, batch_size=100, iterations=1): try: self.buff except: self.buff = Memory(10000) for it in range(iterations): while len(self.buff.mem) < batch_size: state = replay_buffer.sample(1)[0][0] state = state.astype(np.float64) state = state / 255. patches = [] for i in range(0, state.shape[1]-self.w+1, self.s): for j in range(0, state.shape[2]-self.w+1, self.s): patches.append(state[0, i:i+self.w, j:j+self.w, :]) assert patches[-1].shape[0] == patches[-1].shape[1] assert patches[-1].shape[0] == self.w from random import shuffle shuffle(patches) self.buff.mem += patches batch = self.buff.mem[:batch_size] self.buff.mem = self.buff.mem[batch_size:] _, recon_loss, = sess.run([self.update_model_recon, self.recon_loss], feed_dict={self.x:batch, self.y:batch}) print "train_feature_extractor - recon_loss:", recon_loss ######################################################################################################################## import pickle recon_x, recon_y = sess.run([self.recon_x_, self.recon_y_], feed_dict={self.x:batch, self.y:batch}) pickle.dump( [batch, recon_x, recon_y], open( "recons.p", "wb" ) ) ######################################################################################################################## return iterations
def main(): parser = argparse.ArgumentParser() parser.add_argument("--environment", type=str, default='CartPole-v0') parser.add_argument("--action-size", type=int, default=2) parser.add_argument("--input-shape", type=list, default=[None, 4]) parser.add_argument("--target-update-freq", type=int, default=200) parser.add_argument("--epsilon-max", type=float, default=1.) parser.add_argument("--epsilon-min", type=float, default=.01) parser.add_argument("--epsilon-decay", type=float, default=.001) parser.add_argument("--learning-rate", type=float, default=.99) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--epochs", type=int, default=30000) parser.add_argument("--replay-mem-size", type=int, default=1000000) args = parser.parse_args() env = gym.make(args.environment) args.action_size = env.action_space.n args.input_shape = [None] + list(env.observation_space.shape) print args # Initialize the controller controller = lstd(state_size=args.input_shape[-1], action_size=args.action_size, learning_rate=args.learning_rate) # Other parameters epsilon = args.epsilon_max # Replay memory memory = Memory(args.replay_mem_size) # Time step time_step = 0. for epoch in range(args.epochs): total_reward = 0 observation = env.reset() for t in range(1000000): env.render() controller.updatew() action = controller.act(observation) if np.random.rand() < epsilon: action = np.random.randint(args.action_size) observation1, reward, done, info = env.step(action) total_reward += reward # Add to memory #memory.add([observation, action, reward, observation1, done]) # Reduce epsilon time_step += 1. epsilon = args.epsilon_min + (args.epsilon_max - args.epsilon_min) * np.exp(-args.epsilon_decay * time_step) # Training step #batch = np.array(memory.sample(args.batch_size)) controller.train([observation, action, reward, observation1, done]) # Set observation observation = observation1 if done: print"Episode finished after {} timesteps".format(t+1), 'epoch', epoch, 'total_rewards', total_reward break
def main(): parser = argparse.ArgumentParser() parser.add_argument("--env-interface", type=str, default='gym!atari') parser.add_argument("--environment", type=str, default='CartPole-v0') parser.add_argument("--action-size", type=int, default=2) parser.add_argument("--input-shape", type=list, default=[None, 4]) parser.add_argument("--target-update-freq", type=int, default=200) parser.add_argument("--epsilon-max", type=float, default=1.) parser.add_argument("--epsilon-min", type=float, default=.01) parser.add_argument("--epsilon-decay", type=float, default=.001) parser.add_argument("--learning-rate", type=float, default=.99) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--epochs", type=int, default=30000) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--K", type=int, default=1, help='The number of steps to train the environment') parser.add_argument( "--L", type=int, default=1, help='The number of Q-learning steps for hypothetical rollouts') parser.add_argument("--latent-size", type=int, default=4, help='Size of vector for Z') args = parser.parse_args() env = env_interface(args.env_interface, args.environment, pixel_feature=False, render=True) #args.action_size = env.action_space.n args.action_size = env.action_size args.input_shape = [None] + list(env.obs_space_shape) print args # Other parameters epsilon = args.epsilon_max # Replay memory memory = Memory(args.replay_mem_size) # Time step time_step = 0. # Initialize the GANs cgan_state = CGAN(input_shape=args.input_shape, action_size=args.action_size, latent_size=args.latent_size, gen_input_shape=args.input_shape) cgan_reward = CGAN(input_shape=args.input_shape, action_size=args.action_size, latent_size=args.latent_size, gen_input_shape=[None, 1]) qnet = qnetwork(input_shape=args.input_shape, action_size=args.action_size, scope='qnet') target_qnet = qnetwork(input_shape=args.input_shape, action_size=args.action_size, scope='target_qnet') update_ops = update_target_graph('qnet', 'target_qnet') rand_no = np.random.rand() #env = gym.wrappers.Monitor(env, '/tmp/cartpole-experiment-' + str(rand_no), force=True, video_callable=False) init = tf.initialize_all_variables() with tf.Session() as sess: sess.run(init) for epoch in range(args.epochs): total_reward = 0 observation = env.reset() for t in range(1000000): #env.render() action = qnet.get_action(sess, observation) if np.random.rand() < epsilon: #action = env.action_space.sample() action = np.random.randint(args.action_size) observation1, reward, done, info = env.step(action) total_reward += reward # Add to memory memory.add([observation, action, reward, observation1, done]) # Reduce epsilon time_step += 1. epsilon = args.epsilon_min + ( args.epsilon_max - args.epsilon_min) * np.exp( -args.epsilon_decay * time_step) # Training step batch = np.array(memory.sample(args.batch_size)) qnet.train(sess, batch, args.learning_rate, target_qnet) # Training step: environment model for k in range(args.K): batch = np.array(memory.sample(args.batch_size)) states = np.vstack(batch[:, 0]) actions = np.array(batch[:, 1]) rewards = batch[:, 2] states1 = np.vstack(batch[:, 3]) _, D_loss_state = sess.run( [cgan_state.D_solver, cgan_state.D_loss], feed_dict={ cgan_state.states: states, cgan_state.actions: actions, cgan_state.Z: sample_z(len(batch), args.latent_size), cgan_state.X: states1 }) _, G_loss_state = sess.run( [cgan_state.G_solver, cgan_state.G_loss], feed_dict={ cgan_state.states: states, cgan_state.actions: actions, cgan_state.Z: sample_z(len(batch), args.latent_size) }) _, D_loss_reward = sess.run( [cgan_reward.D_solver, cgan_reward.D_loss], feed_dict={ cgan_reward.states: states, cgan_reward.actions: actions, cgan_reward.Z: sample_z(len(batch), args.latent_size), cgan_reward.X: rewards[..., np.newaxis] }) _, G_loss_reward = sess.run( [cgan_reward.G_solver, cgan_reward.G_loss], feed_dict={ cgan_reward.states: states, cgan_reward.actions: actions, cgan_reward.Z: sample_z(len(batch), args.latent_size) }) #print D_loss_state, G_loss_state, D_loss_reward, G_loss_state # Training step: imagination rollouts if time_step == 0.: print "time_step 0 here" if time_step >= 0.: for l in range(args.L): batch = np.array(memory.sample(args.batch_size)) assert len(batch) > 0 states1 = np.vstack(batch[:, 3]) actions = np.random.randint(args.action_size, size=len(batch)) dones = np.array([False] * len(batch)) G_sample_state = sess.run(cgan_state.G_sample, feed_dict={ cgan_state.states: states1, cgan_state.actions: actions, cgan_state.Z: sample_z( len(batch), args.latent_size) }) G_sample_reward = sess.run(cgan_reward.G_sample, feed_dict={ cgan_reward.states: states1, cgan_reward.actions: actions, cgan_reward.Z: sample_z( len(batch), args.latent_size) }) qnet.train(sess, None, args.learning_rate, target_qnet, states1, actions, G_sample_reward, G_sample_state, dones) # Set observation observation = observation1 # Update? if int(time_step) % args.target_update_freq == 0: #print "Updating target..." sess.run(update_ops) if done: print "Episode finished after {} timesteps".format( t + 1), 'epoch', epoch, 'total_rewards', total_reward break
def main(): import gym import sys import copy sys.path.append('../..') from utils import Memory #env = gym.make('LunarLander-v2') env = gym.make('Pendulum-v0') #env = gym.make('CartPole-v0') mem = Memory(1000000) batch_size = 32 try: a_size = env.action_space.n a_type = 'discrete' except: try: a_size = env.action_space.shape[0] a_type = 'continuous' except: raise ValueError('Cannot find action size.') emg = gated_env_modeler(s_shape=[None, env.observation_space.shape[0]], a_size=a_size, out_shape=[None, env.observation_space.shape[0]], a_type=a_type, numfactors=256) #emg = gated_env_modeler(s_shape=[None, env.observation_space.shape[0]], a_size=a_size, out_shape=[None, 1], a_type=a_type, numfactors=256) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) while True: s = env.reset() done = False while done == False: #env.render() #a = np.random.randint(a_size) a = random_action(a_size, a_type) s_, r, done, _ = env.step(a) mem.add([s, a, r, s_, done]) batch = mem.sample(batch_size) if len(batch) == batch_size: states = [] actions = [] rewards = [] states_ = [] for i in range(batch_size): states.append(batch[i][0]) actions.append(batch[i][1]) rewards.append(batch[i][2]) states_.append(batch[i][3]) states = np.stack(states, axis=0) actions = np.stack(actions, axis=0) rewards = np.stack(rewards, axis=0) states_ = np.stack(states_, axis=0) #_, loss_s, loss_a, loss_s_, loss = sess.run([emg.update_model, emg.loss_s, emg.loss_a, emg.loss_s_, emg.loss], feed_dict={emg.states:states, emg.states_:rewards[..., np.newaxis], emg.actions_placeholder:actions}) _, loss_s, loss_a, loss_s_, loss = sess.run( [ emg.update_model, emg.loss_s, emg.loss_a, emg.loss_s_, emg.loss ], feed_dict={ emg.states: states, emg.states_: states_, emg.actions_placeholder: actions }) print 'loss_s', loss_s, 'loss_a', loss_a, 'loss_s_', loss_s_, 'loss', loss s = copy.deepcopy(s_) if done == True: break
def main(): parser = argparse.ArgumentParser() parser.add_argument("--env-interface", type=str, default='gym') parser.add_argument("--environment", type=str, default='BreakoutDeterministic-v4') parser.add_argument("--action-size", type=int, default=4) parser.add_argument("--input-shape", type=str, default='None,84,84,4') parser.add_argument("--state-len-max", type=int, default=4) parser.add_argument("--target-update-freq", type=int, default=10000) parser.add_argument("--ep-greedy-speed", type=str, default='slow') parser.add_argument("--epsilon-max", type=float, default=1.) parser.add_argument("--epsilon-min", type=float, default=.01) parser.add_argument("--epsilon-decay-slow", type=int, default=1000000) parser.add_argument("--epsilon-decay-fast", type=float, default=.001) parser.add_argument("--learning-rate", type=float, default=.95) parser.add_argument("--replay-start-size", type=int, default=50000) parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--epochs", type=int, default=30000) parser.add_argument("--pixel-feature", type=int, default=1) parser.add_argument("--padding", type=int, default=0) parser.add_argument("--model", type=str, default='nature') args = parser.parse_args() args.input_shape = str2list(args.input_shape) assert args.model in ['nature', 'gated'] assert args.ep_greedy_speed in ['fast', 'slow'] assert args.env_interface in [ 'gym', 'ale', 'custom_cart', 'custom_cartpole', 'ple' ] if args.env_interface in ['gym', 'ale']: env = env_interface(args.env_interface, args.environment) elif args.env_interface in ['custom_cart', 'custom_cartpole', 'ple']: env = env_interface(args.env_interface, args.environment, bool(args.pixel_feature), bool(args.padding)) args.input_shape = [None] + list(env.obs_space_shape) + [1] args.input_shape[-1] = args.state_len_max args.action_size = env.action_size assert args.state_len_max == args.input_shape[-1] print args #Other other paramters state_old = [] state = [] steps = 0 #Other parameters if args.ep_greedy_speed == 'slow': epsilon = args.epsilon_max epsilon_rate = 0. if args.epsilon_decay_slow != 0: epsilon_rate = ((args.epsilon_max - args.epsilon_min) / float(args.epsilon_decay_slow)) elif args.ep_greedy_speed == 'fast': epsilon = args.epsilon_max #Initialize replay memory memory = Memory(args.replay_mem_size, args.input_shape[1:]) #Initialize neural net qnet, tnet, update_ops = init_network(args.input_shape, args.action_size, args.model) #import time with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(update_ops) for epoch in range(args.epochs): frame = env.reset() total_rewards = 0. total_losses = 0. state_old = [] state = [frame] * args.state_len_max done = False #start = time.time() while done == False: if np.random.rand() < epsilon: action = np.random.randint(args.action_size) else: image_in = np.stack(state, axis=-1)[np.newaxis, ...] action = qnet.get_action(sess, image_in) frame, reward, done, _ = env.step(action) total_rewards += reward state_old = state[:] state.append(frame) if len(state) > args.state_len_max: state = state[1:] #Add to memory memory.add([ np.stack(state_old, axis=-1)[np.newaxis, ...], action, min(1., max(-1., reward)), np.stack(state, axis=-1)[np.newaxis, ...], done ]) #Reduce epsilon if args.ep_greedy_speed == 'slow': epsilon = max(args.epsilon_min, epsilon - epsilon_rate) elif args.ep_greedy_speed == 'fast': epsilon = args.epsilon_min + ( args.epsilon_max - args.epsilon_min) * np.exp( -args.epsilon_decay_fast * float(steps)) if steps > args.replay_start_size: #Training step batch = np.array(memory.sample(args.batch_size)) states = np.concatenate(batch[:, 0], axis=0) actions = batch[:, 1] rewards = batch[:, 2] states1 = np.concatenate(batch[:, 3], axis=0) dones = batch[:, 4] l = qnet.train(sess, states, actions, rewards, states1, dones, args.learning_rate, tnet) total_losses += l #Increase the frame steps counter steps += 1 #Check if target network is to be updated if steps % args.target_update_freq == 0: print "Updating target..." sess.run(update_ops) if done == True: print "epoch:", epoch, "total rewards", total_rewards, "total losses", total_losses, qnet.string #print 'time:', time.time() - start break env.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument("--environment", type=str, default='CartPole-v0') parser.add_argument("--action-size", type=int, default=2) parser.add_argument("--input-shape", type=list, default=[None, 4]) parser.add_argument("--target-update-freq", type=int, default=200) parser.add_argument("--epsilon-max", type=float, default=1.) parser.add_argument("--epsilon-min", type=float, default=.01) parser.add_argument("--epsilon-decay", type=float, default=.001) parser.add_argument("--learning-rate", type=float, default=.99) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--epochs", type=int, default=300) parser.add_argument("--replay-mem-size", type=int, default=1000000) args = parser.parse_args() env = gym.make(args.environment) args.action_size = env.action_space.n args.input_shape = [None] + list(env.observation_space.shape) print args # Epsilon parameter epsilon = args.epsilon_max # Replay memory memory = Memory(args.replay_mem_size) # Time step time_step = 0. # Initialize the agent qnet = qnetwork(input_shape=args.input_shape, action_size=args.action_size, scope='qnet') tnet = qnetwork(input_shape=args.input_shape, action_size=args.action_size, scope='tnet') update_ops = update_target_graph('qnet', 'tnet') with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(args.epochs): total_reward = 0 state = env.reset() while True: #env.render() if np.random.rand() < epsilon: action = np.random.randint(args.action_size) else: action = qnet.act(sess, state) next_state, reward, done, _ = env.step(action) total_reward += reward # Add to memory memory.add([state, action, reward, next_state, done]) # Reduce epsilon time_step += 1. epsilon = args.epsilon_min + ( args.epsilon_max - args.epsilon_min) * np.exp( -args.epsilon_decay * time_step) # Training step batch = np.array(memory.sample(args.batch_size)) qnet.train(sess, batch, args.learning_rate, tnet) # s <- s' state = np.copy(next_state) # Update target network if int(time_step) % args.target_update_freq == 0: sess.run(update_ops) if done: print 'epoch:', epoch, 'total_rewards:', total_reward break
def __init__(self, level_name): self.level_name = level_name # setup environment self.env = gym_super_mario_bros.make(level_name) self.env = JoypadSpace(self.env, SIMPLE_MOVEMENT) # one hot encoded version of our actions self.possible_actions = np.array(np.identity(self.env.action_space.n, dtype=int).tolist()) # resest graph tf.reset_default_graph() # instantiate the DQNetwork self.DQNetwork = DQNetwork(state_size, action_size, learning_rate) # instantiate memory self.memory = Memory(max_size=memory_size) # initialize deque with zero images self.stacked_frames = deque([np.zeros((100, 128), dtype=np.int) for i in range(stack_size)], maxlen=4) for i in range(pretrain_length): # If it's the first step if i == 0: state = self.env.reset() state, self.stacked_frames = stack_frames(self.stacked_frames, state, True) # Get next state, the rewards, done by taking a random action choice = random.randint(1, len(self.possible_actions)) - 1 action = self.possible_actions[choice] next_state, reward, done, _ = self.env.step(choice) # stack the frames next_state, self.stacked_frames = stack_frames(self.stacked_frames, next_state, False) # if the episode is finished (we're dead) if done: # we inished the episode next_state = np.zeros(state.shape) # add experience to memory self.memory.add((state, action, reward, next_state, done)) # start a new episode state = self.env.reset() state, self.stacked_frames = stack_frames(self.stacked_frames, state, True) else: # add experience to memory self.memory.add((state, action, reward, next_state, done)) # our new state is now the next_state state = next_state # saver will help us save our model self.saver = tf.train.Saver() # setup tensorboard writer self.writer = tf.summary.FileWriter("logs/") # losses tf.summary.scalar("Loss", self.DQNetwork.loss) self.write_op = tf.summary.merge_all()
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument("--env", type=str, default='Pendulum-v0') parser.add_argument("--unroll-steps", type=int, default=25) parser.add_argument("--no-samples", type=int, default=20) parser.add_argument("--no-basis", type=int, default=256) parser.add_argument("--discount-factor", type=float, default=.9) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--train-policy-batch-size", type=int, default=32) parser.add_argument("--train-policy-iterations", type=int, default=30) parser.add_argument("--replay-start-size-epochs", type=int, default=2) parser.add_argument("--train-hyperparameters-iterations", type=int, default=50000) parser.add_argument("--goal-position", type=float, default=.45) args = parser.parse_args() print args #env = gym.make(args.env, goal_position=args.goal_position) env = gym.make(args.env) env.seed(seed=args.goal_position) # Gather data to train hyperparameters data = [] rewards = [] dones = [] for _ in range(2): state = env.reset() while True: action = np.random.uniform(env.action_space.low, env.action_space.high, 1) next_state, reward, done, _ = env.step(action) data.append([state, action, next_state]) rewards.append(reward) dones.append(done) state = np.copy(next_state) if done: break states, actions, next_states = [np.stack(d, axis=0) for d in zip(*data)] permutation = np.random.permutation(len(data)) states_actions = np.concatenate([states, actions], axis=-1)[permutation] next_states = next_states[permutation] # Train the hyperparameters hs = [ hyperparameter_search(dim=env.observation_space.shape[0] + env.action_space.shape[0]) for _ in range(env.observation_space.shape[0]) ] hyperparameters = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) batch_size = 32 iterations = args.train_hyperparameters_iterations #idxs = [np.random.randint(len(states_actions), size=batch_size) for _ in range(iterations)] for i in range(len(hs)): hs[i].train_hyperparameters(sess, states_actions, next_states[:, i], iterations, batch_size) hyperparameters.append( sess.run([hs[i].length_scale, hs[i].signal_sd, hs[i].noise_sd])) blr = blr_model(x_dim=env.observation_space.shape[0] + env.action_space.shape[0], y_dim=env.observation_space.shape[0], state_dim=env.observation_space.shape[0], action_dim=env.action_space.shape[0], observation_space_low=env.observation_space.low, observation_space_high=env.observation_space.high, action_bound_low=env.action_space.low, action_bound_high=env.action_space.high, unroll_steps=args.unroll_steps, no_samples=args.no_samples, no_basis=args.no_basis, discount_factor=args.discount_factor, train_policy_batch_size=args.train_policy_batch_size, train_policy_iterations=args.train_policy_iterations, hyperparameters=hyperparameters, debugging_plot=False) # Initialize the memory memory = Memory(args.replay_mem_size) assert len(data) == len(rewards) assert len(data) == len(dones) for dat, reward, done in zip(data, rewards, dones): memory.add([dat[0], dat[1], reward, dat[2], done]) memory2 = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) weights = pickle.load( open( '../custom_environments/weights/mountain_car_continuous_reward' + str(args.goal_position) + '.p', 'rb')) sess.run(blr.assign_ops, feed_dict=dict(zip(blr.placeholders_reward, weights))) # Update the model with data used from training hyperparameters blr.update(sess, states_actions, next_states) blr.train(sess, memory) state = env.reset() total_rewards = 0.0 epoch = 1 for time_steps in range(30000): action = blr.act(sess, state) next_state, reward, done, _ = env.step(action) total_rewards += float(reward) # Append to the batch memory.add([state, action, reward, next_state, done]) memory2.append([state, action, reward, next_state, done]) # s <- s' state = np.copy(next_state) if done == True: print 'time steps', time_steps, 'epoch', epoch, 'total rewards', total_rewards # Update the memory blr.update(sess, memory=memory2) # Train the policy blr.train(sess, memory) epoch += 1 total_rewards = 0. state = env.reset() memory2 = []
def train(input_placeholder, output_data, sess): # build cost function action = tf.placeholder("float", [None, ACTIONS_CHOICE_NUMBER]) y = tf.placeholder("float", [None]) y_action = tf.reduce_sum(tf.multiply(output_data, action), reduction_indices=1) cost = tf.reduce_mean(tf.square(y - y_action)) train_step = tf.train.AdamOptimizer(1e-6).minimize(cost) # start game game_state = game.GameState() global_timestamp = 0 epsilon = EPSILON memory = Memory(MEMORY_SIZE, FRAME_NUM_PER_STACK) # start network sess.run(tf.global_variables_initializer()) # network checkpoint saver and restore loader saver = tf.train.Saver() checkpoint = tf.train.get_checkpoint_state(CHECKPOINT_FOLDER) if checkpoint and checkpoint.model_checkpoint_path and os.path.exists( os.path.join(CHECKPOINT_FOLDER, "global_state.pkl")): data = load_history(os.path.join(CHECKPOINT_FOLDER, "global_state.pkl")) global_timestamp = data['global_timestamp'] epsilon = data['epsilon'] memory = data['memory'] game_state = data['game_state'] saver.restore(sess, checkpoint.model_checkpoint_path) logging.info("restored from checkpoint", extra={ 'stage': get_stage_name(global_timestamp), 'timestamp': global_timestamp, 'epsilon': epsilon, 'reward': "", 'action': "" }) else: image_data, _, _ = game_state.frame_step(actions.NOTHING) memory.initial_stack(image_data) prev_state = memory.get_current_stack() while True: actions_scores = output_data.eval( feed_dict={input_placeholder: [prev_state]})[0] action_name, action_choice = actions.get_next_action( epsilon, actions_scores) image_data, reward, game_terminate = game_state.frame_step( action_choice) memory.stack_frame(image_data) new_state = memory.get_current_stack() memory.remember(prev_state, action_choice, reward, new_state, game_terminate) # anneal if global_timestamp > OBSERVE_DURATION and epsilon > MIN_EPSILON: logging.info("start anneal", extra={ 'stage': get_stage_name(global_timestamp), 'timestamp': global_timestamp, 'epsilon': epsilon, 'reward': reward, 'action': action_name }) epsilon -= float(EPSILON - MIN_EPSILON) / ANNEAL_DURATION # explore + train if global_timestamp > OBSERVE_DURATION: prev_state_batch, action_batch, reward_batch, new_state_batch, game_terminate_batch = memory.get_sample_batches( BATCH_SIZE) y_batch = [] evaluate = output_data.eval( feed_dict={input_placeholder: new_state_batch}) for i, game_terminate in enumerate(game_terminate_batch): # train target to reward if game_terminate: y_batch.append(reward_batch[i]) else: y_batch.append(reward_batch[i] + GAMMA * np.max(evaluate[i])) # gradient train_step.run( feed_dict={ y: y_batch, action: action_batch, input_placeholder: new_state_batch }) if global_timestamp % CHECKPOINT_GAP == 0: saver.save(sess, os.path.join(CHECKPOINT_FOLDER, 'flappy-bird'), global_step=global_timestamp) save_history( os.path.join(CHECKPOINT_FOLDER, 'global_state.pkl'), { 'global_timestamp': global_timestamp, 'epsilon': epsilon, 'memory': memory, 'game_state': game_state }) logging.info("checkpoint saved", extra={ 'stage': get_stage_name(global_timestamp), 'timestamp': global_timestamp, 'epsilon': epsilon, 'reward': reward, 'action': "" }) # update state prev_state = new_state logging.info("finish epoch", extra={ 'stage': get_stage_name(global_timestamp), 'timestamp': global_timestamp, 'epsilon': epsilon, 'reward': reward, 'action': action_name }) global_timestamp += 1
def main(): parser = argparse.ArgumentParser() parser.add_argument("--environment", type=str, default='Pendulum-v0') parser.add_argument("--action-dim", type=int, default=1) parser.add_argument("--state-dim", type=int, default=1) #parser.add_argument("--epochs", type=int, default=30000) parser.add_argument("--time-steps", type=int, default=30000) parser.add_argument('--tau', type=float, help='soft target update parameter', default=0.01) parser.add_argument("--action-bound", type=float, default=1.) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--learning-rate", type=float, default=.9) parser.add_argument("--latent-size", type=int, default=4, help='Size of vector for Z') parser.add_argument("--model", type=str, default='gan') parser.add_argument("--mode", type=str, default='none') args = parser.parse_args() assert args.mode in ['none', 'test', 'transfer'] assert args.model in [ 'mlp', 'gan', 'gated', 'dmlac_mlp', 'dmlac_gan', 'dmlac_gated', 'ddpg_unrolled_pg_mlp', 'dmlac_gp', 'dmlac_truth', 'mpc' ] if args.model == 'dmlac_truth': assert args.environment == 'Pendulum-v0' # Initialize environment env = gym.make(args.environment) args.state_dim = env.observation_space.shape[0] args.action_dim = env.action_space.shape[0] #assert args.action_dim == 1 args.action_bound_high = env.action_space.high args.action_bound_low = env.action_space.low assert len(args.action_bound_high) == len(args.action_bound_low) for i in range(len(args.action_bound_high)): assert args.action_bound_high[i] == -args.action_bound_low[i] print(args) jointddpg, update_target_actor, update_target_critic, copy_target_actor, copy_target_critic = init_model( [None, args.state_dim], args.action_dim, args.latent_size, args.learning_rate, args.action_bound_low, args.action_bound_high, args.tau, args.model) # Replay memory memory = Memory(args.replay_mem_size) # Actor noise exploration_strategy = OUStrategy(jointddpg, env) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) #sess.run(copy_target_critic) #sess.run(copy_target_actor) if args.mode in ['test', 'transfer']: env.seed(1) state = env.reset() total_rewards = 0.0 epoch = 1 for time_steps in range(args.time_steps): env.render() # Choose an action exploration = (float(args.time_steps - time_steps) / float(args.time_steps))**4 action = exploration_strategy.action(sess, state[np.newaxis, ...], exploration) # Execute action state1, reward, done, _ = env.step(action) total_rewards += float(reward) # Store tuple in replay memory memory.add([ state[np.newaxis, ...], action[np.newaxis, ...], reward, state1[np.newaxis, ...], done ]) # Training step batch_B = np.array(memory.sample(args.batch_size)) assert len(batch_B) > 0 states_B = np.concatenate(batch_B[:, 0], axis=0) actions_B = np.concatenate(batch_B[:, 1], axis=0) rewards_B = batch_B[:, 2] states1_B = np.concatenate(batch_B[:, 3], axis=0) dones_B = batch_B[:, 4] #Get another batch batch_M = np.array(memory.sample(args.batch_size)) assert len(batch_M) > 0 states_M = np.vstack(batch_M[:, 0]) actions_M = np.concatenate(batch_M[:, 1], axis=0) if args.model == 'dmlac_gp': jointddpg.update_hist(memory) jointddpg.train(sess, states_B, actions_B, rewards_B, states1_B, dones_B, states_M, actions_M, len(batch_M), args.latent_size) # Update target networks #jointddpg.update(self, sess, update_target_critic, update_target_actor) #sess.run(update_target_critic) #sess.run(update_target_actor) state = np.copy(state1) if done == True: print 'time steps', time_steps, 'epoch', epoch, 'total rewards', total_rewards epoch += 1 total_rewards = 0. if args.mode == 'transfer': if time_steps >= args.time_steps / 3: env.seed(0) else: env.seed(1) elif args.mode == 'test': env.seed(1) state = env.reset() if args.mode == 'transfer': if time_steps == args.time_steps / 3: memory = Memory(args.replay_mem_size)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--environment", type=str, default='MountainCarContinuous-v0') parser.add_argument("--unroll-steps", type=int, default=20) parser.add_argument("--time-steps", type=int, default=30000) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--discount-factor", type=float, default=1.) parser.add_argument("--goal-position", type=float, default=.45) args = parser.parse_args() env = gym.make(args.environment) env.seed(seed=args.goal_position) state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] action_bound_high = env.action_space.high action_bound_low = env.action_space.low agent = direct_policy_search(state_dim, action_dim, action_bound_high, action_bound_low, args.unroll_steps, args.discount_factor, 1, 'direct_policy_search') # Replay memory memory = Memory(args.replay_mem_size) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) #weights = pickle.load(open('../custom_environments/weights/pendulum_reward.p', 'rb')) #weights = pickle.load(open('../custom_environments/weights/mountain_car_continuous_reward'+str(args.goal_position)+'.p', 'rb')) #sess.run(agent.assign_ops0, feed_dict=dict(zip(agent.placeholders_reward, weights))) weights = pickle.load( open( '../custom_environments/weights/mountain_car_continuous_next_state.p', 'rb')) sess.run(agent.assign_ops1, feed_dict=dict(zip(agent.placeholders_state, weights))) state = env.reset() total_rewards = 0.0 epoch = 1 for time_steps in range(args.time_steps): env.render() action = agent.act(sess, state) next_state, reward, done, _ = env.step(action) total_rewards += float(reward) # Store tuple in replay memory memory.add([ np.atleast_2d(state), np.atleast_2d(action), reward, np.atleast_2d(next_state), done ]) # Training step batch = np.array(memory.sample(args.batch_size)) assert len(batch) > 0 states = np.concatenate(batch[:, 0], axis=0) # Train the agent agent.train(sess, states) # s <- s' state = np.copy(next_state) if done == True: print 'time steps', time_steps, 'epoch', epoch, 'total rewards', total_rewards, 'unroll', args.unroll_steps epoch += 1 total_rewards = 0. state = env.reset()
def main(): #Arguments for the q-learner parser = argparse.ArgumentParser() parser.add_argument("--env-interface", type=str, default='gym') parser.add_argument("--environment", type=str, default='BreakoutDeterministic-v4') parser.add_argument("--action-size", type=int, default=4) parser.add_argument("--input-shape", type=str, default='None,84,84,4') parser.add_argument("--state-len-max", type=int, default=4) parser.add_argument("--target-update-freq", type=int, default=10000) parser.add_argument("--epsilon-max", type=float, default=1.) parser.add_argument("--epsilon-min", type=float, default=.01) parser.add_argument("--epsilon-decay", type=int, default=1000000) parser.add_argument("--learning-rate", type=float, default=.95) parser.add_argument("--replay-start-size", type=int, default=50000) parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--epochs", type=int, default=30000) #Arguments for the feature extractor parser.add_argument("--train-fe-shape", type=str, default='None,12,12,4') parser.add_argument("--stop-gradient", type=int, default=0) parser.add_argument("--train-fe-iterations", type=int, default=1000) parser.add_argument("--train-fe-batch-size", type=int, default=100) parser.add_argument("--train-fe-lamb", type=float, default=0.) parser.add_argument("--train-fe-numfactors", type=int, default=200) parser.add_argument("--train-fe-nummap", type=int, default=100) parser.add_argument("--train-fe-learning-rate", type=float, default=.001) parser.add_argument("--train-fe-w", type=int, default=12) parser.add_argument("--train-fe-s", type=int, default=1) parser.add_argument("--use-conv-after-fe", type=int, default=0) parser.add_argument("--ep-greedy-speed", type=str, default='slow') #Arguments for the environment interface parser.add_argument("--pixel-features", type=int, default=1) parser.add_argument("--padding", type=int, default=0) args = parser.parse_args() #Parse arguments wrt other arguments args.input_shape = str2list(args.input_shape) args.train_fe_shape = str2list(args.train_fe_shape) assert args.env_interface in [ 'gym', 'ale', 'custom_cart', 'custom_cartpole' ] assert args.ep_greedy_speed in ['fast', 'slow'] env = env_interface(args.env_interface, args.environment, pixel_feature=bool(args.pixel_features), padding=bool(args.padding), render=True) args.action_size = env.action_size if args.env_interface in ['custom_cart', 'custom_cartpole']: args.input_shape = [None] + list( env.obs_space_shape) + [args.state_len_max] args.train_fe_shape[-1] = args.state_len_max print args #Other other parameters state_old = [] state = [] steps = 0 #Other parameters epsilon_lambda = .001 epsilon = args.epsilon_max epsilon_rate = 0. if args.epsilon_decay != 0: epsilon_rate = ((args.epsilon_max - args.epsilon_min) / float(args.epsilon_decay)) #Initialize replay memory print args.input_shape memory = Memory(args.replay_mem_size, args.input_shape[1:]) #Initialize neural net from gated_qlearning import gated_qlearning qnet = gated_qlearning(shape=args.train_fe_shape,\ nummap=args.train_fe_nummap,\ numfactors=args.train_fe_numfactors,\ learning_rate=args.train_fe_learning_rate,\ frame_shape=args.input_shape,\ a_size=args.action_size,\ stop_gradient=bool(args.stop_gradient),\ lamb=args.train_fe_lamb,\ w=args.train_fe_w,\ s=args.train_fe_s,\ use_conv_after_fe=bool(args.use_conv_after_fe)) qnet_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) tnet = gated_qlearning(shape=args.train_fe_shape,\ nummap=args.train_fe_nummap,\ numfactors=args.train_fe_numfactors,\ learning_rate=args.train_fe_learning_rate,\ frame_shape=args.input_shape,\ a_size=args.action_size,\ stop_gradient=bool(args.stop_gradient),\ lamb=args.train_fe_lamb,\ w=args.train_fe_w,\ s=args.train_fe_s,\ use_conv_after_fe=bool(args.use_conv_after_fe)) tnet_vars = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES)[len(qnet_vars):] update_ops = update_target_graph_vars(qnet_vars, tnet_vars) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) sess.run(update_ops) for epoch in range(args.epochs): frame = env.reset() total_rewards = 0. total_losses = 0. state_old = [] state = [frame] * args.state_len_max done = False while done == False: if np.random.rand() < epsilon: action = np.random.randint(args.action_size) else: image_in = np.stack(state, axis=-1)[np.newaxis, ...] action = qnet.get_action(sess, image_in) frame, reward, done, _ = env.step(action) total_rewards += reward state_old = state[:] state.append(frame) if len(state) > args.state_len_max: state = state[1:] #Add to memory memory.add([np.stack(state_old, axis=-1)[np.newaxis, ...],\ action,\ min(1., max(-1., reward)),\ np.stack(state, axis=-1)[np.newaxis, ...],\ done]) #Reduce epsilon if args.ep_greedy_speed == 'slow': epsilon = max(args.epsilon_min, epsilon - epsilon_rate) elif args.ep_greedy_speed == 'fast': epsilon = args.epsilon_min + ( args.epsilon_max - args.epsilon_min) * np.exp( -epsilon_lambda * float(steps)) #Train the reconstruction loss if args.train_fe_iterations > 0: args.train_fe_iterations -= qnet.train_feature_extractor( sess, memory, args.train_fe_batch_size, 10) print args.train_fe_iterations if steps > args.replay_start_size and args.train_fe_iterations <= 0: #Training step batch = np.array(memory.sample(args.batch_size)) states = np.concatenate(batch[:, 0], axis=0) actions = batch[:, 1] rewards = batch[:, 2] states1 = np.concatenate(batch[:, 3], axis=0) dones = batch[:, 4] Q1 = qnet.get_Q1(sess, states1, tnet) targetQ = rewards + (1. - dones) * args.learning_rate * np.amax( Q1, keepdims=False, axis=1) l, _, _ = qnet.train(sess, states, actions, targetQ[..., np.newaxis]) total_losses += l #Increase the frame steps counter steps += 1 #Check if target network is to be updated if steps % args.target_update_freq == 0: print "Updating target..." sess.run(update_ops) if done == True: print "epoch", epoch, "total rewards", total_rewards, "total losses", total_losses break env.close()
def main2(): import gym import copy from utils import Memory from utils import process_frame2 env = gym.make('BreakoutDeterministic-v4') gc = gated_convolution2(shape=[None, 84, 84, 4], nummap=128, numfactors=128, learning_rate=.001, w=8, s=1, a_size=env.action_space.n) mem = Memory(50000) batch_size = 4 steps = 1 length = 4 action_space = env.action_space.n with tf.Session() as sess: sess.run(tf.global_variables_initializer()) while True: s = env.reset() s = process_frame2(s) state = [s[..., np.newaxis]] * length state_ = [s[..., np.newaxis]] * length action = [-1] * length done = False while done == False: #env.render() a = np.random.randint(env.action_space.n) s_, r, done, _ = env.step(a) s_ = process_frame2(s_) state_.pop(0) action.pop(0) state_.append(s_[..., np.newaxis]) action.append(a) mem.add([ np.concatenate(state, axis=-1)[np.newaxis, ...], np.array(action)[np.newaxis, ...], np.concatenate(state_, axis=-1)[np.newaxis, ...] ]) if len(mem.mem) >= batch_size: batch = mem.sample(batch_size) #Do stuff states = [] actions = [] states_ = [] for i in range(len(batch)): states.append(batch[i][0]) actions.append(batch[i][1]) states_.append(batch[i][2]) states = np.concatenate(states, axis=0).astype( np.float64) / 255. actions = np.concatenate(actions, axis=0) states_ = np.concatenate(states_, axis=0).astype( np.float64) / 255. _, recon_loss, recon_x, recon_y, recon_action_loss = gc.run2( sess, states, actions, states_) print 'steps:', steps, 'recon_loss:', recon_loss, 'recon_action_loss', recon_action_loss, 'main2' steps += 1 if done == True: break
def main(): parser = argparse.ArgumentParser() parser.add_argument("--environment", type=str, default='Pendulum-v0') parser.add_argument("--action-dim", type=int, default=1) parser.add_argument("--state-dim", type=int, default=1) parser.add_argument("--input-shape", type=list, default=[None, 1]) parser.add_argument("--epochs", type=int, default=30000) parser.add_argument('--tau', help='soft target update parameter', default=0.001) parser.add_argument("--action-bound", type=float, default=1.) parser.add_argument("--replay-mem-size", type=int, default=1000000) parser.add_argument("--batch-size", type=int, default=64) parser.add_argument("--gamma", type=float, default=.99) parser.add_argument("--K", type=int, default=1, help='The number of steps to train the environment') parser.add_argument( "--L", type=int, default=1, help='The number of Q-learning steps for hypothetical rollouts') parser.add_argument("--latent-size", type=int, default=4, help='Size of vector for Z') args = parser.parse_args() # Initialize environment env = gym.make(args.environment) args.state_dim = env.observation_space.shape[0] args.input_shape = [None, args.state_dim] args.action_dim = env.action_space.shape[0] #assert args.action_dim == 1 args.action_bound = env.action_space.high print(args) # Networks actor_source = actor(state_shape=[None, args.state_dim],\ action_shape=[None, args.action_dim],\ output_bound=args.action_bound[0],\ scope='actor_source') critic_source = critic(state_shape=[None, args.state_dim],\ action_shape=[None, args.action_dim],\ scope='critic_source') actor_target = actor(state_shape=[None, args.state_dim],\ action_shape=[None, args.action_dim],\ output_bound=args.action_bound[0],\ scope='actor_target') critic_target = critic(state_shape=[None, args.state_dim],\ action_shape=[None, args.action_dim],\ scope='critic_target') # Initialize the GANs cgan_state = CGAN(input_shape=args.input_shape,\ action_size=args.action_dim,\ latent_size=args.latent_size,\ gen_input_shape=args.input_shape,\ continuous_action=True) cgan_reward = CGAN(input_shape=args.input_shape,\ action_size=args.action_dim,\ latent_size=args.latent_size,\ gen_input_shape=[None, 1],\ continuous_action=True) # Update and copy operators update_target_actor = update_target_graph2('actor_source', 'actor_target', args.tau) update_target_critic = update_target_graph2('critic_source', 'critic_target', args.tau) copy_target_actor = update_target_graph2('actor_source', 'actor_target', 1.) copy_target_critic = update_target_graph2('critic_source', 'critic_target', 1.) # Replay memory memory = Memory(args.replay_mem_size) # Actor noise actor_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(args.action_dim)) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(copy_target_critic) sess.run(copy_target_actor) for epoch in range(args.epochs): state = env.reset() total_rewards = 0.0 while True: #env.render() # Choose an action action = sess.run( actor_source.action, feed_dict={actor_source.states: state[np.newaxis, ...] })[0] + actor_noise() # Execute action state1, reward, done, _ = env.step(action) total_rewards += float(reward) # Store tuple in replay memory memory.add([state[np.newaxis, ...],\ action[np.newaxis, ...],\ reward,\ state1[np.newaxis, ...],\ done]) # Training step: update actor critic using real experience batch = np.array(memory.sample(args.batch_size)) assert len(batch) > 0 states = np.concatenate(batch[:, 0], axis=0) actions = np.concatenate(batch[:, 1], axis=0) rewards = batch[:, 2] states1 = np.concatenate(batch[:, 3], axis=0) dones = batch[:, 4] # Update the critic actions1 = sess.run(actor_target.action,\ feed_dict={actor_target.states:states1}) targetQ = np.squeeze(sess.run(critic_target.Q,\ feed_dict={critic_target.states:states1,\ critic_target.actions:actions1}), axis=-1) targetQ = rewards + ( 1. - dones.astype(np.float32)) * args.gamma * targetQ targetQ = targetQ[..., np.newaxis] _, critic_loss = sess.run([critic_source.critic_solver,\ critic_source.loss],\ feed_dict={critic_source.states:states,\ critic_source.actions:actions,\ critic_source.targetQ:targetQ}) # Update the actor critic_grads = sess.run(critic_source.grads,\ feed_dict={critic_source.states:states,\ critic_source.actions:actions})[0]# Grab gradients from critic _ = sess.run(actor_source.opt,\ feed_dict={actor_source.states:states,\ actor_source.dQ_by_da:critic_grads}) # Update target networks sess.run(update_target_critic) sess.run(update_target_actor) # Training step: update the environment model using real experience (i.e., update the conditional GANs) for k in range(args.K): batch = np.array(memory.sample(args.batch_size)) states = np.concatenate(batch[:, 0], axis=0) actions = np.concatenate(batch[:, 1], axis=0) rewards = batch[:, 2] states1 = np.concatenate(batch[:, 3], axis=0) _, D_loss_state = sess.run([cgan_state.D_solver, cgan_state.D_loss],\ feed_dict={cgan_state.states:states,\ cgan_state.actions:actions,\ cgan_state.Z:sample_z(len(batch),\ args.latent_size),\ cgan_state.X:states1}) _, G_loss_state = sess.run([cgan_state.G_solver,\ cgan_state.G_loss],\ feed_dict={cgan_state.states:states,\ cgan_state.actions:actions,\ cgan_state.Z:sample_z(len(batch),\ args.latent_size)}) _, D_loss_reward = sess.run([cgan_reward.D_solver,\ cgan_reward.D_loss],\ feed_dict={cgan_reward.states:states,\ cgan_reward.actions:actions,\ cgan_reward.Z:sample_z(len(batch),\ args.latent_size),\ cgan_reward.X:rewards[..., np.newaxis]}) _, G_loss_reward = sess.run([cgan_reward.G_solver,\ cgan_reward.G_loss],\ feed_dict={cgan_reward.states:states,\ cgan_reward.actions:actions,\ cgan_reward.Z:sample_z(len(batch),\ args.latent_size)}) #print D_loss_state, G_loss_state, D_loss_reward, G_loss_state # Training step: update actor critic using imagination rollouts for l in range(args.L): batch = np.array(memory.sample(args.batch_size)) states_ = np.concatenate(batch[:, 3], axis=0) actions = np.random.uniform(env.action_space.low[0],\ env.action_space.high[0],\ size=(len(batch),\ env.action_space.shape[0])) dones = np.array([False] * len(batch)) G_sample_state = sess.run(cgan_state.G_sample,\ feed_dict={cgan_state.states:states_,\ cgan_state.actions:actions,\ cgan_state.Z:sample_z(len(batch),\ args.latent_size)}) G_sample_reward = sess.run(cgan_reward.G_sample,\ feed_dict={cgan_reward.states:states_,\ cgan_reward.actions:actions,\ cgan_reward.Z:sample_z(len(batch),\ args.latent_size)}) G_sample_reward = np.squeeze(G_sample_reward, axis=-1) # Update the critic actions1 = sess.run(actor_target.action,\ feed_dict={actor_target.states:G_sample_state}) targetQ = np.squeeze(sess.run(critic_target.Q,\ feed_dict={critic_target.states:G_sample_state,\ critic_target.actions:actions1}), axis=-1) targetQ = G_sample_reward + ( 1. - dones.astype(np.float32)) * args.gamma * targetQ targetQ = targetQ[..., np.newaxis] _, critic_loss = sess.run([critic_source.critic_solver,\ critic_source.loss],\ feed_dict={critic_source.states:states_,\ critic_source.actions:actions,\ critic_source.targetQ:targetQ}) # Update the actor critic_grads = sess.run(critic_source.grads,\ feed_dict={critic_source.states:states_,\ critic_source.actions:actions})[0]# Grab gradients from critic _ = sess.run(actor_source.opt,\ feed_dict={actor_source.states:states_,\ actor_source.dQ_by_da:critic_grads}) # Update target networks sess.run(update_target_critic) sess.run(update_target_actor) state = np.copy(state1) if done == True: print 'epoch', epoch, 'total rewards', total_rewards break
print(device) actor = Actor(obs_space=config.obs_space, action_space=config.action_space, hidden_size=config.hidden_size).to(device) critic = Critic(obs_space=config.obs_space, hidden_size=config.hidden_size).to(device) # actor.load_state_dict(torch.load('actor_model.h5')) # critic.load_state_dict(torch.load('critic_model.h5')) wandb.watch(actor) wandb.watch(critic) optimizer_actor = Adam(actor.parameters(), lr=config.actor_lr) optimizer_critic = Adam(critic.parameters(), lr=config.critic_lr) memory = Memory(env.agent_ids) def compute_GAE(rewards, state_values, done, gamma, lamb): """ Computes Generalized Advantage Estimations. """ returns = [rewards[-1] + state_values[-1]] running_sum = rewards[-1] - state_values[-1] for i in reversed(range(len(rewards) - 1)): mask = 0 if done[i + 1] else 1 delta = rewards[i] + gamma * state_values[i + 1] * mask - state_values[i] running_sum = delta + gamma * lamb * running_sum * mask returns.insert(0, running_sum + state_values[i])