def train_net(self):
batch_size = self.hyperparams['batch_size']
sample = self.experience_replay
minibatches = create_minibatch_iterator({'sample': sample}, None,
self.batch_preprocessor,
batch_size)
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
self.recent_loss = deque(
maxlen=int(self.hyperparams['num_recent_steps'] /
self.hyperparams['num_threads']))
self.recent_policy_loss = deque(
maxlen=int(self.hyperparams['num_recent_steps'] /
self.hyperparams['num_threads']))
self.recent_value_loss = deque(
maxlen=int(self.hyperparams['num_recent_steps'] /
self.hyperparams['num_threads']))
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'sample': sample}, None, self.batch_preprocessor,
batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('Update num: %d' % update_num)
self.train_model(batch[0], batch[1], batch[2], batch[3], display)
self.experience_replay.clear()
self.update_learning_rate()
def train_net(self):
batch_size = self.hyperparams['batch_size']
# sample = self.experience_replay.sample(
# len(self.experience_replay))
sample = self.experience_replay
minibatches = create_minibatch_iterator({'sample': sample}, None,
self.batch_preprocessor,
batch_size)
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'sample': sample}, None,
self.batch_preprocessor, batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('Update num: %d' % update_num)
self.train_model(batch[0], batch[1], batch[2], display)
self.experience_replay.clear()
self.update_learning_rate()
def train_net(self, init=False):
batch_size = self.hyperparams['batch_size']
sample = self.experience_replay.get_all()
minibatches = create_minibatch_iterator({'sample': sample}, None,
self.batch_preprocessor,
batch_size)
if init:
num_updates = self.hyperparams['init_updates']
else:
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
avg_recent_loss = deque(maxlen=int(
self.hyperparams['num_recent_steps'] /
self.hyperparams['display_freq']))
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'sample': sample}, None,
self.batch_preprocessor, batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('Update num: %d' % update_num)
self.train_model(batch[0], batch[1], batch[2],
avg_recent_loss, display)
self.experience_replay.clear()
self.update_learning_rate()
def train_net(self):
batch_size = self.hyperparams['batch_size']
# sample = self.experience_replay.sample(
# len(self.experience_replay))
sample = self.experience_replay
minibatches = create_minibatch_iterator({'sample': sample}, None,
self.batch_preprocessor,
batch_size)
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'sample': sample}, None, self.batch_preprocessor,
batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('Update num: %d' % update_num)
self.train_model(batch[0], batch[1], batch[2], display)
self.experience_replay.clear()
self.update_learning_rate()
def train_net(self):
batch_size = self.hyperparams['batch_size']
# sample = self.experience_replay.sample(
# len(self.experience_replay))
train_xs, train_y, train_mask = self.sample_preprocessor(
self.experience_replay)
minibatches = create_minibatch_iterator(
{'state': train_xs['state'],
'train_y': train_y,
'train_mask': train_mask},
None, None, batch_size)
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'state': train_xs['state'],
'train_y': train_y,
'train_mask': train_mask},
None, None, batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('Update num: %d' % update_num)
self.train_model({'state': batch['state']}, batch['train_y'],
batch['train_mask'], display)
del minibatches
del train_xs['state'], train_xs, train_y, train_mask
def train_net(self):
batch_size = self.hyperparams['batch_size']
sample = self.experience_replay
minibatches = create_minibatch_iterator({'sample': sample}, None,
self.batch_preprocessor,
batch_size)
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
self.recent_loss = deque(maxlen=int(
self.hyperparams['num_recent_steps'] /
self.hyperparams['num_threads']))
self.recent_value_loss = deque(maxlen=int(
self.hyperparams['num_recent_steps'] /
self.hyperparams['num_threads']))
self.recent_policy_loss = deque(maxlen=int(
self.hyperparams['num_recent_steps'] /
self.hyperparams['num_threads']))
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'sample': sample}, None,
self.batch_preprocessor, batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('-' * 40 + ' Update num: %d ' % update_num + '-' * 40)
self.train_model(batch[0], batch[1], batch[2], batch[3], display)
self.experience_replay.clear()
self.update_learning_rate()
def train_net(self):
batch_size = self.hyperparams['batch_size']
# sample = self.experience_replay.sample(
# len(self.experience_replay))
train_xs, train_y, train_mask = self.sample_preprocessor(
self.experience_replay)
minibatches = create_minibatch_iterator(
{
'state': train_xs['state'],
'train_y': train_y,
'train_mask': train_mask
}, None, None, batch_size)
num_updates = self.hyperparams['updates_per_iter']
epoch = 0
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update_num in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{
'state': train_xs['state'],
'train_y': train_y,
'train_mask': train_mask
}, None, None, batch_size)
batch = next(minibatches)
epoch += 1
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
display = update_num % self.hyperparams['display_freq'] == 0
if display:
print('Update num: %d' % update_num)
self.train_model({'state': batch['state']}, batch['train_y'],
batch['train_mask'], display)
del minibatches
del train_xs['state'], train_xs, train_y, train_mask
def train_partial_model(self, num_updates,
mega=False, display=False):
batch_size = self.hyperparams['batch_size']
rollout_len = self.hyperparams['value_rollout_length']
if mega:
sample = self.experience_replay.sample(
len(self.experience_replay) - rollout_len - 1, rollout_len,
decompress=False)
else:
sample = self.experience_replay.sample(
batch_size, rollout_len, decompress=False)
minibatches = create_minibatch_iterator({'sample': sample}, None,
self.batch_preprocessor,
batch_size)
epoch = 0
if mega:
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
for update in range(num_updates):
try:
batch = next(minibatches)
except StopIteration:
minibatches = create_minibatch_iterator(
{'sample': sample}, None,
self.batch_preprocessor, batch_size)
batch = next(minibatches)
epoch += 1
if mega:
print('-' * 40 + 'Epoch %d' % epoch + '-' * 40)
feed_dict = self.make_feed_dict({'state': self.state_model,
'reward': self.reward_model,
'value': self.value_model},
{'state': self.state_y,
'reward': self.reward_y,
'value': self.value_y},
batch)
self.sess.run(self.partial_train_step,
feed_dict=feed_dict)
if mega and update % self.hyperparams['update_target_freq'] == 0:
self.update_value_target_weights()
if display and update % self.hyperparams['display_freq'] == 0:
if mega:
print('Partial update %d' % update)
self.state_loss_val = self.state_loss.loss.eval(
feed_dict=feed_dict, session=self.sess)
self.reward_loss_val = self.reward_loss.loss.eval(
feed_dict=feed_dict, session=self.sess)
self.value_loss_val = self.value_loss.loss.eval(
feed_dict=feed_dict, session=self.sess)
self.display_train_update(update)
def recreate_batches():
return create_minibatch_iterator(
train_xs,
train_y,
self.model.train_batch_preprocessor,
batch_size=self.hyperparams['batch_size'],
train_mask=train_mask)
def sample_preprocessor(self, sample):
# batch = batch['sample']
reward_discount = self.hyperparams['reward_discount']
batch_size = self.hyperparams['batch_size']
train_xs = {'state': np.array([s for (s, a, r, n, v) in sample])}
train_y = np.zeros((len(sample),))
train_mask = np.zeros((len(sample), len(self.actions)))
minibatches = create_minibatch_iterator(
{'state': np.array([n for (s, a, r, n, v) in sample])},
None, None, batch_size)
preds = np.zeros((len(sample), len(self.actions)))
for batch_num, batch in enumerate(minibatches):
batch_preds = self.q_model.compute_preds(
batch,
sess=self.sess)
batch_start = batch_size * batch_num
preds[batch_start:batch_start + batch_size] = batch_preds
del minibatches
for experience_num, (state, action, reward, next_state, value) in \
enumerate(sample):
exp_returns = np.max(preds[experience_num])
target = reward + value + reward_discount * exp_returns
train_y[experience_num] = target
train_mask[experience_num, np.argmax(action)] = 1
del state, action, reward, next_state, value
self.target = target
self.exp_returns = exp_returns
# self.mean_state_val = np.mean(np.array(
# [s for (s, a, r, n) in batch]))
# self.mean_preds = np.mean(preds)
return train_xs, train_y, train_mask
def sample_preprocessor(self, sample):
# batch = batch['sample']
reward_discount = self.hyperparams['reward_discount']
batch_size = self.hyperparams['batch_size']
train_xs = {'state': np.array([s for (s, a, r, n, v) in sample])}
train_y = np.zeros((len(sample), ))
train_mask = np.zeros((len(sample), len(self.actions)))
minibatches = create_minibatch_iterator(
{'state': np.array([n for (s, a, r, n, v) in sample])}, None, None,
batch_size)
preds = np.zeros((len(sample), len(self.actions)))
for batch_num, batch in enumerate(minibatches):
batch_preds = self.q_model.compute_preds(batch, sess=self.sess)
batch_start = batch_size * batch_num
preds[batch_start:batch_start + batch_size] = batch_preds
del minibatches
for experience_num, (state, action, reward, next_state, value) in \
enumerate(sample):
exp_returns = np.max(preds[experience_num])
target = reward + value + reward_discount * exp_returns
train_y[experience_num] = target
train_mask[experience_num, np.argmax(action)] = 1
del state, action, reward, next_state, value
self.target = target
self.exp_returns = exp_returns
# self.mean_state_val = np.mean(np.array(
# [s for (s, a, r, n) in batch]))
# self.mean_preds = np.mean(preds)
return train_xs, train_y, train_mask
def compute_val_stats(self, model, val_xs, val_y):
if self.accuracy is not None:
minibatches = create_minibatch_iterator(
val_xs,
val_y,
model.test_batch_preprocessor,
batch_size=self.hyperparams['batch_size'])
def fn(batch):
return self.accuracy.compute(model, batch, self.y_var,
batch['y'])
acc = avg_over_batches(minibatches, fn)
return {'acc': acc}
else:
return None
def compute_val_stats(self,
model,
val_xs, val_y):
if self.accuracy is not None:
minibatches = create_minibatch_iterator(
val_xs, val_y, model.test_batch_preprocessor,
batch_size=self.hyperparams['batch_size'])
def fn(batch):
return self.accuracy.compute(
model, batch, self.y_var, batch['y'])
acc = avg_over_batches(minibatches, fn)
return {'acc': acc}
else:
return None
def recreate_batches():
return create_minibatch_iterator(
train_xs, train_y,
self.model.train_batch_preprocessor,
batch_size=self.hyperparams['batch_size'],
train_mask=train_mask)