def numpy_episodes(train_dir,
test_dir,
shape,
loader,
preprocess_fn=None,
scan_every=10,
num_chunks=None,
**kwargs):
"""Read sequences stored as compressed Numpy files as a TensorFlow dataset.
Args:
train_dir: Directory containing NPZ files of the training dataset.
test_dir: Directory containing NPZ files of the testing dataset.
shape: Tuple of batch size and chunk length for the datasets.
use_cache: Boolean. Set to True to cache episodes in memory. Default is to
read episodes from disk every time.
**kwargs: Keyword arguments to forward to the read episodes implementation.
Returns:
Structured data from numpy episodes as Tensors.
"""
try:
dtypes, shapes = _read_spec(train_dir, **kwargs)
except ZeroDivisionError:
dtypes, shapes = _read_spec(test_dir, **kwargs)
loader = {
'scan': functools.partial(_read_episodes_scan, every=scan_every),
'reload': _read_episodes_reload,
'dummy': _read_episodes_dummy,
}[loader]
train = tf.data.Dataset.from_generator(
functools.partial(loader, train_dir, shape[0], **kwargs), dtypes,
shapes)
test = tf.data.Dataset.from_generator(
functools.partial(loader, test_dir, shape[0], **kwargs), dtypes,
shapes)
chunking = lambda x: tf.data.Dataset.from_tensor_slices(
# Returns dict of image, action, reward, length tensors with num_chunks in 0 dim.
chunk_sequence(x, shape[1], True, num_chunks))
def sequence_preprocess_fn(sequence):
if preprocess_fn:
with tf.device('/cpu:0'):
sequence['image'] = preprocess_fn(sequence['image'])
return sequence
# This transformation (flat_map):
# 1. Chunk each sequence,
# 2. From each sequence one can get variable number of chunks
# (first dim. of a tensor is chunks number, like with batches).
# Flatten to get the dataset of chunks.
train = train.flat_map(chunking)
train = train.shuffle(100 * shape[0])
train = train.batch(shape[0], drop_remainder=True)
train = train.map(sequence_preprocess_fn, 10).prefetch(20)
test = test.flat_map(chunking)
test = test.shuffle(100 * shape[0])
test = test.batch(shape[0], drop_remainder=True)
test = test.map(sequence_preprocess_fn, 10).prefetch(20)
return attr_dict.AttrDict(train=train, test=test)
def optimizer_apply_grads(self, ave_grads, variables):
graph = attr_dict.AttrDict(locals())
summary = tf.cond(self._should_summarize,
lambda: self._define_summaries(graph), str)
optimize = self._optimizer.apply_gradients(zip(ave_grads, variables))
return optimize, summary
def numpy_episodes(train_dir,
test_dir,
shape,
loader,
preprocess_fn=None,
scan_every=10,
num_chunks=None,
**kwargs):
"""Read sequences stored as compressed Numpy files as a TensorFlow dataset.
Args:
train_dir: Directory containing NPZ files of the training dataset.
test_dir: Directory containing NPZ files of the testing dataset.
shape: Tuple of batch size and chunk length for the datasets.
use_cache: Boolean. Set to True to cache episodes in memory. Default is to
read episodes from disk every time.
**kwargs: Keyword arguments to forward to the read episodes implementation.
Returns:
Structured data from numpy episodes as Tensors.
"""
try:
dtypes, shapes = _read_spec(train_dir, **kwargs)
except ZeroDivisionError:
dtypes, shapes = _read_spec(test_dir, **kwargs)
loader = {
'scan': functools.partial(_read_episodes_scan, every=scan_every),
'reload': _read_episodes_reload,
'dummy': _read_episodes_dummy,
}[loader]
train = tf.data.Dataset.from_generator(
functools.partial(loader, train_dir, shape[0], **kwargs), dtypes,
shapes)
test = tf.data.Dataset.from_generator(
functools.partial(loader, test_dir, shape[0], **kwargs), dtypes,
shapes)
chunking = lambda x: tf.data.Dataset.from_tensor_slices(
chunk_sequence(x, shape[1], True, num_chunks))
def sequence_preprocess_fn(sequence):
sequence['image'] = sequence['observ']
return sequence
train = train.flat_map(chunking)
train = train.shuffle(100 * shape[0])
train = train.batch(shape[0], drop_remainder=True)
train = train.map(sequence_preprocess_fn, 10).prefetch(20)
test = test.flat_map(chunking)
test = test.shuffle(100 * shape[0])
test = test.batch(shape[0], drop_remainder=True)
test = test.map(sequence_preprocess_fn, 10).prefetch(20)
return attr_dict.AttrDict(train=train, test=test)
def minimize(self, loss):
summaries = []
gradients, variables = zip(*self._optimizer.compute_gradients(
loss, self._variables, colocate_gradients_with_ops=True))
gradient_norm = tf.global_norm(gradients)
if self._clipping:
gradients, _ = tf.clip_by_global_norm(gradients, self._clipping,
gradient_norm)
graph = attr_dict.AttrDict(locals())
summary = tf.cond(self._should_summarize,
lambda: self._define_summaries(graph), str)
optimize = self._optimizer.apply_gradients(zip(gradients, variables))
return optimize, summary
def numpy_episodes(train_dir,
test_dir,
shape,
reader=None,
loader=None,
num_chunks=None,
preprocess_fn=None):
"""Read sequences stored as compressed Numpy files as a TensorFlow dataset.
Args:
train_dir: Directory containing NPZ files of the training dataset.
test_dir: Directory containing NPZ files of the testing dataset.
shape: Tuple of batch size and chunk length for the datasets.
reader: Callable that reads an episode from a NPZ filename.
loader: Generator that yields episodes.
Returns:
Structured data from numpy episodes as Tensors.
"""
reader = reader or episode_reader
loader = loader or cache_loader
try:
dtypes, shapes = _read_spec(reader, train_dir)
except ZeroDivisionError:
dtypes, shapes = _read_spec(reader, test_dir)
train = tf.data.Dataset.from_generator(
functools.partial(loader, reader, train_dir, shape[0]), dtypes, shapes)
test = tf.data.Dataset.from_generator(
functools.partial(loader, reader, test_dir, shape[0]), dtypes, shapes)
chunking = lambda x: tf.data.Dataset.from_tensor_slices(
chunk_sequence.chunk_sequence(x, shape[1], True, num_chunks))
def sequence_preprocess_fn(sequence):
if preprocess_fn:
sequence['image'] = preprocess_fn(sequence['image'])
return sequence
train = train.flat_map(chunking)
train = train.batch(shape[0], drop_remainder=True)
train = train.map(sequence_preprocess_fn, 10).prefetch(10)
test = test.flat_map(chunking)
test = test.batch(shape[0], drop_remainder=True)
test = test.map(sequence_preprocess_fn, 10).prefetch(10)
return attr_dict.AttrDict(train=train, test=test)
def minimize(self, loss):
summaries = []
gradients, variables = zip(
*self._optimizer.
compute_gradients( # tf.train.AdamOptimizer.compute_gradients(loss, var_list, ...)
loss,
self._variables,
colocate_gradients_with_ops=True)
) # returns a list of (gradient, variable) pairs where "gradient" is the gradient for "variable".
gradient_norm = tf.global_norm(
gradients) # Computes the global norm of multiple tensors.
if self._clipping:
gradients, _ = tf.clip_by_global_norm(gradients, self._clipping,
gradient_norm)
graph = attr_dict.AttrDict(locals())
summary = tf.cond(self._should_summarize,
lambda: self._define_summaries(graph), str)
optimize = self._optimizer.apply_gradients(zip(gradients, variables))
return optimize, summary
def numpy_episodes(train_dir,
test_dir,
shape,
reader=None,
loader=None,
num_chunks=None,
preprocess_fn=None,
aug_fn=None,
simclr=False):
"""Read sequences stored as compressed Numpy files as a TensorFlow dataset.
Args:
train_dir: Directory containing NPZ files of the training dataset.
test_dir: Directory containing NPZ files of the testing dataset.
shape: Tuple of batch size and chunk length for the datasets.
reader: Callable that reads an episode from a NPZ filename.
loader: Generator that yields episodes.
Returns:
Structured data from numpy episodes as Tensors.
"""
reader = reader or episode_reader
loader = loader or cache_loader
try:
dtypes, shapes = _read_spec(reader, train_dir)
except ZeroDivisionError:
dtypes, shapes = _read_spec(reader, test_dir)
train = tf.data.Dataset.from_generator(
functools.partial(loader, reader, train_dir, shape[0]), dtypes, shapes)
test = tf.data.Dataset.from_generator(
functools.partial(loader, reader, test_dir, shape[0]), dtypes, shapes)
chunking = lambda x: tf.data.Dataset.from_tensor_slices(
chunk_sequence.chunk_sequence(x, shape[1], True, num_chunks))
def sequence_preprocess_fn(sequence):
if preprocess_fn:
sequence['image'], noise = preprocess_fn(sequence['image'],
return_noise=True)
if 'ori_img' in sequence.keys():
sequence['ori_img'] = preprocess_fn(sequence['ori_img'],
noise=noise)
if simclr:
print('ccc ', sequence)
sequence['image'] = tf.reshape(sequence['image'],
(shape[0], shape[1]) +
tuple(sequence['image'].shape[2:]))
sequence['action'] = tf.reshape(
sequence['action'],
(shape[0], shape[1], sequence['action'].shape[2]))
sequence['reward'] = tf.reshape(sequence['reward'],
(shape[0], shape[1]))
sequence['aug'] = tf.reshape(sequence['aug'],
(shape[0], shape[1], 2))
sequence['length'] = tf.reshape(sequence['length'], [-1])
print('preprocess ', sequence)
return sequence
bs = shape[0] // 2 if simclr else shape[0]
train = train.flat_map(chunking)
if aug_fn:
print('Training set is augmented')
train = train.map(lambda x: aug_fn(x, phase='train'), 10)
train = train.batch(bs, drop_remainder=True)
train = train.map(sequence_preprocess_fn, 10).prefetch(10)
test = test.flat_map(chunking)
if aug_fn:
print('Test set is augmented')
test = test.map(lambda x: aug_fn(x, phase='test'), 10)
test = test.batch(bs, drop_remainder=True)
test = test.map(sequence_preprocess_fn, 10).prefetch(10)
return attr_dict.AttrDict(train=train, test=test)
def minimize(self, loss, unpacked_losses):
# Auxiliary losses should be helpful to the main loss
# i.e. their gradients should point to similar direction, otherwise they're rescaled by cosine diff
# https://arxiv.org/pdf/1812.02224.pdf
# main_loss = unpacked_losses[0] + unpacked_losses[4] # zs reward and overshooting reward
# gradients, _ = zip(*self._optimizer.compute_gradients(main_loss, self._variables, colocate_gradients_with_ops=True))
# gradients = list(gradients)
# similarities = []
# aux_losses_all = unpacked_losses.copy()
# del aux_losses_all[0]
# del aux_losses_all[4]
# assert len(aux_losses_all) == 4
# # Main loss similarity to itself (debug)
# main_loss_grads, _ = zip(*self._optimizer.compute_gradients(main_loss, self._variables, colocate_gradients_with_ops=True))
# shared_indexes, main_loss_grads_shared, aux_loss_grads_shared = zip(*self.find_shared_gradients(main_loss_grads, main_loss_grads))
# similarity = tf.maximum(self.tensor_list_cosine_similarity(main_loss_grads_shared, aux_loss_grads_shared), 0)
# similarities.append(similarity)
# for aux_loss in aux_losses_all:
# # 1. Find shared parameter gradients (with respect to the main loss). Apply them with weights on auxiliary losses by similarities
# aux_loss_grads_unaltered, _ = zip(*self._optimizer.compute_gradients(aux_loss, self._variables, colocate_gradients_with_ops=True))
# shared_indexes, main_loss_shared_grads_with_aux_loss, aux_loss_grads = zip(*self.find_shared_gradients(main_loss_grads, aux_loss_grads_unaltered))
# similarity = tf.maximum(self.tensor_list_cosine_similarity(main_loss_shared_grads_with_aux_loss, aux_loss_grads), 0)
# similarities.append(similarity)
# aux_loss_grads_weighted = map(lambda auxgrad: auxgrad * similarity, aux_loss_grads)
# # Apply shared grads
# for idx, aux_grad in enumerate(aux_loss_grads_weighted):
# gradients[shared_indexes[idx]] += aux_grad
# # 2. Find task unique gradients (with respect to the main loss). Apply all of them respectively.
# idx_grad_pairs = self.find_task_specific_gradients(main_loss_grads, aux_loss_grads_unaltered)
# if len(idx_grad_pairs):
# diff_indexes, aux_loss_diff_grads = zip(*idx_grad_pairs)
# # Apply aux. task unique grads
# for idx, aux_grad in enumerate(aux_loss_diff_grads):
# if gradients[diff_indexes[idx]] != None:
# gradients[diff_indexes[idx]] += aux_grad
# else:
# gradients[diff_indexes[idx]] = aux_grad
# losses_similarities = []
# main_loss_reward = unpacked_losses[0]
# main_loss_reward_grads = self._optimizer.compute_gradients(main_loss_reward, self._variables, colocate_gradients_with_ops=True)
# main_loss_reward_grads = map(lambda grad: (tf.cast(tf.zeros_like(grad[1]), dtype=tf.float32), grad[1]) if grad[0] == None else grad, main_loss_reward_grads) # filter None grads
# main_loss_reward_grads = map(lambda grad: grad[0], main_loss_reward_grads) # get only gradient tensors
# main_loss_reward_grads = list(main_loss_reward_grads)
# for idx, grad in enumerate(main_loss_reward_grads):
# if grad.shape.as_list() == []:
# main_loss_reward_grads[idx] = tf.expand_dims(grad, -1)
# for unpacked_loss in unpacked_losses:
# grads = self._optimizer.compute_gradients(unpacked_loss, self._variables, colocate_gradients_with_ops=True)
# grads = filter(lambda grad: grad != None, grads)
# # grads = map(lambda grad: (tf.cast(tf.zeros_like(grad[1]), dtype=tf.float32), grad[1]) if grad[0] == None else grad, grads) # filter None grads
# grads = map(lambda grad: grad[0], grads) # get only gradient tensors
# grads = list(grads)
# # Convert all no-shape tensors to shape(1) for l2 normalize
# for idx, grad in enumerate(grads):
# if grad.shape.as_list() == []:
# grads[idx] = tf.expand_dims(grad, -1)
# # Each variable is different size tensor, thus we have to calc similarities variable by variable
# curr_loss_cos_similarity = 0
# for idx, grad in enumerate(grads):
# cos_sim = self.tensor_cosine_similarity(grad, main_loss_reward_grads[idx])
# curr_loss_cos_similarity += (cos_sim / len(grads))
# losses_similarities.append(curr_loss_cos_similarity)
# for idx, unpacked_loss in enumerate(unpacked_losses):
# unpacked_losses[idx] = tf.cond(tf.math.logical_or(tf.math.greater(losses_similarities[idx], 0), tf.math.equal(losses_similarities[idx], 0)),
# lambda: unpacked_loss, lambda: unpacked_loss)
# loss = sum(unpacked_losses)
# summaries = []
# multiloss_gradients, variables = zip(*self._optimizer.compute_gradients(loss, self._variables, colocate_gradients_with_ops=True))
# weighted_gradients = multiloss_gradients
# weighted_gradients = list(map(lambda grad: grad if grad == None else grad * 0, multiloss_gradients))
# for idx, unloss in enumerate(unpacked_losses):
# grads, _ = zip(*self._optimizer.compute_gradients(unloss, self._variables, colocate_gradients_with_ops=True))
# # grads = filter(lambda grad: grad != None, grads)
# for idx2, grad in enumerate(grads):
# if grad == None:
# continue
# weighted_grad = tf.maximum(losses_similarities[idx] * grad, 0)
# weighted_gradients[idx2] = weighted_gradients[idx2] if weighted_gradients[idx2] == None else weighted_gradients[idx2] + weighted_grad
# # grads = map(lambda grad: tf.zeros_like(grad[1]) if grad[0] == None else grad[0], grads) # filter None grads
# # grads = list(map(lambda grad: tf.maximum(losses_similarities[idx] * grad, 0), grads))
# # for idx, wgrad in enumerate(weighted_gradients):
# # weighted_gradients[idx] = wgrad + grads[idx]
#########
# loss = unpacked_losses[0]
#########
gradients, variables = zip(*self._optimizer.compute_gradients(
loss, self._variables, colocate_gradients_with_ops=True))
gradient_norm = tf.global_norm(gradients)
#########
# Apply clipping
if self._clipping:
gradients, _ = tf.clip_by_global_norm(gradients, self._clipping,
gradient_norm)
graph = attr_dict.AttrDict(locals())
summary = tf.cond(self._should_summarize,
lambda: self._define_summaries(graph), str)
optimize = self._optimizer.apply_gradients(zip(gradients, variables))
return optimize, summary, loss, unpacked_losses