def test_iterate(self):
self._populate_replay()
client = tf_client.TFClient(self._client.server_address)
dataset = client.dataset(
table='dist', dtypes=(tf.float32,), shapes=(tf.TensorShape([3, 3]),))
got = self._sample_from(dataset, 10)
for sample in got:
self.assertIsInstance(sample, replay_sample.ReplaySample)
# A single sample is returned so the key should be a scalar int64.
self.assertIsInstance(sample.info.key, np.uint64)
np.testing.assert_array_equal(sample.data[0],
np.zeros((3, 3), dtype=np.float32))
def test_incompatible_dataset_shapes_and_types_without_signature(self):
self._populate_replay()
ds_wrong_shape = timestep_dataset.TimestepDataset(
self._client.server_address,
table='dist',
dtypes=(tf.float32, ),
shapes=(tf.TensorShape([]), ),
max_in_flight_samples_per_worker=100)
with self.assertRaisesRegex(
tf.errors.InvalidArgumentError,
r'Specification has \(dtype, shape\): \(float, \[\]\). '
r'Tensor has \(dtype, shape\): \(float, \[3,3\]\).'):
self._sample_from(ds_wrong_shape, 1)
def _input_fn(params):
"""Convert input into features."""
del params
seq_length = self.max_seq_length
d = tf.data.Dataset.from_generator(
generator,
output_types={
"unique_ids": tf.int32,
"input_ids": tf.int32,
"input_mask": tf.int32,
"segment_ids": tf.int32,
"supporting_mask": tf.int32,
},
output_shapes={
"unique_ids": tf.TensorShape([]),
"input_ids": tf.TensorShape([seq_length]),
"input_mask": tf.TensorShape([seq_length]),
"segment_ids": tf.TensorShape([seq_length]),
"supporting_mask": tf.TensorShape([seq_length]),
})
d = d.batch(batch_size=self.batch_size)
return d
def test_incompatible_signature_shape(self):
self._populate_replay()
client = tf_client.TFClient(self._client.server_address)
dataset = client.dataset(table='signatured',
dtypes=(tf.float32, ),
shapes=(tf.TensorShape([3]), ))
with self.assertRaisesWithPredicateMatch(
tf.errors.InvalidArgumentError,
r'Requested incompatible tensor at flattened index 3 from table '
r'\'signatured\'. Requested \(dtype, shape\): \(float, \[3\]\). '
r'Signature \(dtype, shape\): \(float, \[\?,\?\]\)'):
self._sample_from(dataset, 10)
def __init__(self, name, node_count_key, dtype):
"""Creates a shepherd for data containing a node index.
Args:
name: String name to be used as a prefix for tf.Example keys.
node_count_key: Name of node count field, which is assumed to be present
in the per-instance tf.Examples.
dtype: Type of node data.
"""
self._name = name
self._node_count_key = node_count_key
self._dtype = dtype
self._shape = tf.TensorShape([1])
def return_types_and_shapes(for_trainer):
if for_trainer:
shape = tf.TensorShape([None, None])
label_shape = tf.TensorShape([None, None])
else:
shape = tf.TensorShape([None])
label_shape = tf.TensorShape([None])
output_types = {
"input_ids": tf.int32,
"input_mask": tf.int32,
"token_type_ids": tf.int32,
'label_ids': tf.int32
}
output_shapes = {
"input_ids": shape,
"input_mask": shape,
"token_type_ids": shape,
'label_ids': label_shape
}
return output_types, output_shapes
def _input_fn(params):
"""Convert input into features."""
del params
qry_length = self.max_qry_length
d = tf.data.Dataset.from_generator(
generator,
output_types={
"qas_ids": tf.int32,
"qry_input_ids": tf.int32,
"qry_input_mask": tf.int32,
"qry_entity_id": tf.int32,
"num_entities": tf.int32,
},
output_shapes={
"qas_ids": tf.TensorShape([]),
"qry_input_ids": tf.TensorShape([qry_length]),
"qry_input_mask": tf.TensorShape([qry_length]),
"qry_entity_id": tf.TensorShape([self.num_entities]),
"num_entities": tf.TensorShape([]),
})
d = d.batch(batch_size=self.batch_size)
return d
def input_fn():
"""Create tf.dataset for estimator.
Returns:
tf dataset
"""
tensor_shape = self.num_facts
ds = tf.data.Dataset.from_generator(
generator=fact_follow_generator,
output_types=((tf.float32, tf.float32, tf.float32), tf.int32),
output_shapes=((tf.TensorShape([tensor_shape]),
tf.TensorShape([tensor_shape]),
tf.TensorShape([tensor_shape])),
tf.TensorShape([])))
if n_take > 0:
ds = ds.take(n_take)
if shuffle:
ds = ds.shuffle(SEED)
ds = ds.repeat(epochs)
ds = ds.batch(batch_size)
return ds
def __call__(self, getter, *args, **kwargs):
size = tf.TensorShape(kwargs['shape']).num_elements()
if size < self.small_variable_size_threshold:
device_name = self.device_for_small_variables
else:
device_index, _ = min(enumerate(self.sizes),
key=operator.itemgetter(1))
device_name = self.devices[device_index]
self.sizes[device_index] += size
kwargs['caching_device'] = device_name
var = getter(*args, **kwargs)
return var
def _init_inception():
global softmax
if not os.path.exists(MODEL_DIR):
os.makedirs(MODEL_DIR)
filename = DATA_URL.split('/')[-1]
filepath = os.path.join(MODEL_DIR, filename)
if not os.path.exists(filepath):
def _progress(count, block_size, total_size):
sys.stdout.write('\r>> Downloading %s %.1f%%' %
(filename, float(count * block_size) /
float(total_size) * 100.0))
sys.stdout.flush()
filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath,
_progress)
statinfo = os.stat(filepath)
print('Succesfully downloaded', filename, statinfo.st_size,
'bytes.')
tarfile.open(filepath, 'r:gz').extractall(MODEL_DIR)
with tf.gfile.FastGFile(
os.path.join(MODEL_DIR, 'classify_image_graph_def.pb'),
'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
input_tensor = tf.placeholder(tf.float32,
shape=[None, None, None, 3],
name='InputTensor')
_ = tf.import_graph_def(graph_def,
name='',
input_map={'ExpandDims:0': input_tensor})
#_ = tf.import_graph_def(graph_def, name='')
with tf.Session() as sess:
pool3 = sess.graph.get_tensor_by_name('pool_3:0')
ops = pool3.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
else:
new_shape.append(s)
o.set_shape(tf.TensorShape(new_shape))
#o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
w = sess.graph.get_operation_by_name(
"softmax/logits/MatMul").inputs[1]
logits = tf.matmul(tf.squeeze(pool3, [1, 2]), w)
softmax = tf.nn.softmax(logits)
def set_shapes(self, batch_size, images, labels):
"""Statically set the batch_size dimension."""
if self.transpose_input:
images.set_shape(
images.get_shape().merge_with(
tf.TensorShape([None, None, None, batch_size])
)
)
images = tf.reshape(images, [-1])
labels.set_shape(
labels.get_shape().merge_with(tf.TensorShape([batch_size]))
)
else:
images.set_shape(
images.get_shape().merge_with(
tf.TensorShape([batch_size, None, None, None])
)
)
labels.set_shape(
labels.get_shape().merge_with(tf.TensorShape([batch_size]))
)
return images, labels
def input_fn(params):
"""The actual input function."""
batch_size = params["batch_size"]
d = tf.data.Dataset.from_generator(
functools.partial(convert_examples_to_features,
examples=examples,
window_size=window_size,
stride=stride,
tokenizer=tokenizer),
dict(unique_ids=tf.int32,
input_ids=tf.int32,
input_mask=tf.int32,
input_type_ids=tf.int32,
extract_indices=tf.int32),
dict(unique_ids=tf.TensorShape([]),
input_ids=tf.TensorShape([window_size]),
input_mask=tf.TensorShape([window_size]),
input_type_ids=tf.TensorShape([window_size]),
extract_indices=tf.TensorShape([window_size])))
d = d.batch(batch_size=batch_size, drop_remainder=False)
return d
def __init__(self, name, node_count_key, dtype, shape, mod_paddings=None):
"""Creates a shepherd for data containing node indices.
Args:
name: String name to be used as a prefix for tf.Example keys.
node_count_key: Name of node count field, which is assumed to be present
in the per-instance tf.Examples.
dtype: Type of node data.
shape: The shape.
mod_paddings: If set, pad so dimensions are 0 mod these values.
"""
super(NodeIndicesShepherd, self).__init__(name, node_count_key, dtype)
self._shape = tf.TensorShape(shape)
self._mod_paddings = mod_paddings
def setNodeAttribute(node, tag ,shapeArray):
if(shapeArray is not None):
if(tag=='shapes'): # here we assume always only get first shape in shape list
if(len(shapeArray)==4):
node.attr[tag].list.shape.extend([tf.TensorShape(shapeArray).as_proto()] )
elif( len(shapeArray)==3):
node.attr[tag].list.shape[0].dim[0].size =1
node.attr[tag].list.shape[0].dim[0].size = shapeArray[0]
node.attr[tag].list.shape[0].dim[1].size = shapeArray[1]
node.attr[tag].list.shape[0].dim[2].size = shapeArray[2]
if(tag=='shape'): #TODO Set shape is not working
if(len(shapeArray)==4):
node.attr[tag].shape.CopyFrom(tf.TensorShape(shapeArray).as_proto())
elif( len(shapeArray)==3):
shapeArray4= [None] *4
shapeArray4[0] = 1
shapeArray4[1] = shapeArray[1]
shapeArray4[2] = shapeArray[2]
shapeArray4[3] = shapeArray[3]
node.attr[tag].shape.CopyFrom(tf.TensorShape(shapeArray).as_proto())
def test_checks_sequence_length_when_timesteps_emitted(
self, table_name, actual_sequence_length, provided_sequence_length):
self._populate_replay(actual_sequence_length)
client = tf_client.TFClient(self._client.server_address)
dataset = client.dataset(
table=table_name,
dtypes=(tf.float32,),
shapes=(tf.TensorShape([provided_sequence_length, 3, 3]),),
emit_timesteps=True,
sequence_length=provided_sequence_length)
with self.assertRaises(tf.errors.InvalidArgumentError):
self._sample_from(dataset, 10)
def batching_func(x):
return x.padded_batch(
batch_size,
# The first three entries are the source and target line rows;
# these have unknown-length vectors. The last two entries are
# the source and target row sizes; these are scalars.
padded_shapes=(
tf.TensorShape([src_max_len]), # src
tf.TensorShape([tgt_max_len]), # tgt_input
tf.TensorShape([tgt_max_len]), # tgt_output
tf.TensorShape([]), # src_len
tf.TensorShape([])), # tgt_len
# Pad the source and target sequences with eos tokens.
# (Though notice we don't generally need to do this since
# later on we will be masking out calculations past the true sequence.
padding_values=(
src_eos_id, # src
tgt_eos_id, # tgt_input
tgt_eos_id, # tgt_output
0, # src_len -- unused
0),
# For TPU, must set drop_remainder to True or batch size will be None
drop_remainder=True) # tgt_len -- unused
def test_incompatible_signature_dtype(self, table_name):
self._populate_replay()
dataset = timestep_dataset.TimestepDataset(
self._client.server_address,
table=table_name,
dtypes=(tf.int64,),
shapes=(tf.TensorShape([3, 3]),),
max_in_flight_samples_per_worker=100)
with self.assertRaisesWithPredicateMatch(
tf.errors.InvalidArgumentError,
r'Requested incompatible tensor at flattened index 0 from table '
r'\'{}\'. Requested \(dtype, shape\): \(int64, \[3,3\]\). '
r'Signature \(dtype, shape\): \(float, \[\?,\?\]\)'.format(table_name)):
self._sample_from(dataset, 10)
def __init__(self, shape, filters, kernel, activation=tf.tanh, kernel_initializer=selu_initializer, normalization=None, data_format='channels_last', reuse=None, is_training=False, name='', padding='same', factor_bias_initializer=tf.constant_initializer(0)):
super(ConvGRUCell, self).__init__(_reuse=reuse, name=name)
self._filters = filters
self._kernel = kernel
self._activation = activation
self._kernel_initializer = kernel_initializer
self._normalization = normalization
self._data_format = data_format
self._is_training = is_training
self._regularizer = tf.nn.l2_loss
self._conv = conv2d
self._padding = padding
self.factor_bias_initializer = factor_bias_initializer
if data_format == 'channels_last':
self._size = tf.TensorShape(shape + [self._filters])
self._feature_axis = self._size.ndims
self._data_format_tf = 'NHWC'
elif data_format == 'channels_first':
self._size = tf.TensorShape([self._filters] + shape)
self._feature_axis = 1
self._data_format_tf = 'NCHW'
else:
raise ValueError('Unknown data_format')
def add_op(x, y):
outputs = {
"output_types": [tf.float32],
"output_shapes": [tf.TensorShape([SIZE])],
}
base_path = os.path.dirname(__file__)
lib_path = os.path.join(base_path, "libcustom_op.so")
gp_path = os.path.join(base_path, "custom_codelet.gp")
return ipu.custom_ops.precompiled_user_op([x, y],
lib_path,
gp_path,
outs=outputs)
def return_types_and_shapes(for_trainer, is_multi_label=False):
if for_trainer:
shape = tf.TensorShape([None, None])
label_shape = tf.TensorShape([None])
else:
shape = tf.TensorShape([None])
label_shape = tf.TensorShape([])
output_types = {
"input_ids": tf.int32,
"input_mask": tf.int32,
"token_type_ids": tf.int32,
'label_ids': tf.int32
}
output_shapes = {
"input_ids": shape,
"input_mask": shape,
"token_type_ids": shape,
'label_ids': label_shape
}
if is_multi_label:
output_shapes['label_ids'] = shape
return output_types, output_shapes
def assert_rank_and_shape_compatibility(tensors, rank):
"""Asserts that the tensors have the correct rank and compatible shapes.
Shapes (of equal rank) are compatible if corresponding dimensions are all
equal or unspecified. E.g. `[2, 3]` is compatible with all of `[2, 3]`,
`[None, 3]`, `[2, None]` and `[None, None]`.
Args:
tensors: List of tensors.
rank: A scalar specifying the rank that the tensors passed need to have.
Raises:
ValueError: If the list of tensors is empty or fail the rank and mutual
compatibility asserts.
"""
if not tensors:
raise ValueError("List of tensors should be non-empty.")
union_of_shapes = tf.TensorShape(None)
for tensor in tensors:
tensor_shape = tf.TensorShape(tensor.shape)
tensor_shape.assert_has_rank(rank)
union_of_shapes = union_of_shapes.merge_with(tensor_shape)
def dense_by_sparse_to_dense(d, s, sparsity_mask, blocksize2D, **kwargs):
# The matmul shape (m, n) @ (n, k)
*batch_dims, m, n = d.shape.with_rank_at_least(2).as_list()
assert isinstance(sparsity_mask,
list), "Sparsity mask should be a flat list of 0s and 1s"
blocks_per_inner_group = len(sparsity_mask) // np.prod(
batch_dims) # e.g. [B, A, S, H] there are B*A "inner groups"
blocks_in_dim_n = n // blocksize2D[0]
blocks_in_dim_k = blocks_per_inner_group // blocks_in_dim_n
k = int(blocks_in_dim_k * blocksize2D[1])
# Data-type string has to be float or half
data_type = "half" if d.dtype == tf.float16 else "float"
# The defaults are set here, but can be overridden through kwargs
# for instance the partial_data_type can be overried to float if desired
bs_matmul_args = {
"dim": [m, n, k],
"block_size": [min(128, m)] + blocksize2D,
"sparsity_mask": "".join(str(c) for c in sparsity_mask),
"transposed_rhs": False,
"data_type": data_type,
"partial_data_type": data_type,
"inner_group_size": int(np.prod(
batch_dims)), # how many of the batch dims to run in parallel
"partition_method": "strip",
"memory_cycle_ratio": 1
}
bs_matmul_args.update(
{k: v
for k, v in kwargs.items() if k in bs_matmul_args})
json_attribs = json.dumps(bs_matmul_args)
# Call the custom operator which performs
# [dense x sparse -> dense] matmul with
# a static block-level sparsity mask
y = custom_ops.precompiled_user_op(
[d, s],
get_lib_path("static_block_sparse"),
outs={
"output_types": [d.dtype],
"output_shapes": [tf.TensorShape(list(batch_dims) + [m, k])]
},
op_name="BuildDSD",
separate_gradients=False,
inputs_with_gradients=[0, 1],
attributes=json_attribs,
gradient_attributes=json_attribs)[0]
return y
def testShape(self):
model_rnn = snt.ModelRNN(self.model)
inputs = tf.ones([self.batch_size, 5])
prev_state = tf.ones(dtype=tf.float32,
shape=[self.batch_size, self.hidden_size])
outputs, next_state = model_rnn(inputs, prev_state)
batch_size_shape = tf.TensorShape(self.batch_size)
expected_shape = batch_size_shape.concatenate(self.model.output_size)
self.assertNotEqual(expected_shape, inputs.get_shape())
self.assertEqual(expected_shape, prev_state.get_shape())
self.assertEqual(expected_shape, next_state.get_shape())
self.assertEqual(expected_shape, outputs.get_shape())
def _label_conditioned_variable(name, initializer, labels, num_categories):
"""Label conditioning."""
shape = tf.TensorShape([num_categories]).concatenate(params_shape)
var_collections = slim.utils.get_variable_collections(
variables_collections, name)
var = slim.model_variable(name,
shape=shape,
dtype=dtype,
initializer=initializer,
collections=var_collections,
trainable=trainable)
conditioned_var = tf.gather(var, labels)
conditioned_var = tf.expand_dims(tf.expand_dims(conditioned_var, 1), 1)
return conditioned_var
def output_size(self):
if self._skip_connections and self._concat_final_output_if_skip:
output_size = []
for core_sizes in zip(*tuple(_get_flat_core_sizes(self._cores))):
added_core_size = core_sizes[0]
added_core_size[-1] = sum([size[-1] for size in core_sizes])
output_size.append(tf.TensorShape(added_core_size))
return nest.pack_sequence_as(structure=self._cores[0].output_size,
flat_sequence=output_size)
else:
# Assumes that an element of cores which does not have the output_size
# property does not affect the output shape. Then the 'last' core in the
# sequence with output_size information should be the output_size of the
# DeepRNN. This heuristic is error prone, but we would lose a lot of
# flexibility if we tried to enforce that the final core must have an
# output_size field (e.g. it would be impossible to add a TF nonlinearity
# as the final "core"), but we should at least print a warning if this
# is the case.
final_core = self._cores[-1]
if hasattr(final_core, "output_size"):
# This is definitely the correct value, so no warning needed.
return final_core.output_size
# If we have connected the module at least once, we can get the output
# size of whatever was actually produced.
if self._last_output_size is not None:
tf.logging.warning(
"Final core does not contain .output_size, but the "
"DeepRNN has been connected into the graph, so inferred output "
"size as %s", self._last_output_size)
return self._last_output_size
# If all else fails, iterate backwards through cores and return the
# first one which has an output_size field. This can be incorrect in
# various ways, so warn loudly.
try:
guessed_output_size = next(core.output_size
for core in reversed(self._cores)
if hasattr(core, "output_size"))
except StopIteration:
raise ValueError("None of the 'cores' have output_size information.")
tf.logging.warning(
"Trying to infer output_size of DeepRNN, but the final core %s does "
"not have the .output_size field. The guessed output_size is %s "
"but this may not be correct. If you see shape errors following this "
"warning, you must change the cores used in the DeepRNN so that "
"the final core used has a correct .output_size property.",
final_core, guessed_output_size)
return guessed_output_size
def _get_inception_layer(sess):
"""Prepares inception net for batched usage and returns pool_3 layer. """
layername = 'FID_Inception_Net/pool_3:0'
pool3 = sess.graph.get_tensor_by_name(layername)
ops = pool3.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims is not None:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
else:
new_shape.append(s)
try:
o._shape = tf.TensorShape(new_shape)
except ValueError:
o._shape_val = tf.TensorShape(
new_shape
) # EDIT: added for compatibility with tensorflow 1.6.0
return pool3
def __init__(self,
memory_word_size,
num_read_heads,
top_k=0,
memory_size=None,
name='memory_reader'):
"""Initializes the `MemoryReader`.
Args:
memory_word_size: The dimension of the 1-D read keys this memory reader
should produce. Each row of the memory is of length `memory_word_size`.
num_read_heads: The number of reads to perform.
top_k: Softmax and summation when reading is only over top k most similar
entries in memory. top_k=0 (default) means dense reads, i.e. no top_k.
memory_size: Number of rows in memory.
name: The name for this Sonnet module.
"""
super(MemoryReader, self).__init__(name=name)
self._memory_word_size = memory_word_size
self._num_read_heads = num_read_heads
self._top_k = top_k
# This is not an RNNCore but it is useful to expose the output size.
self._output_size = num_read_heads * memory_word_size
num_read_weights = top_k if top_k > 0 else memory_size
self._read_info_size = ReadInformation(
weights=tf.TensorShape([num_read_heads, num_read_weights]),
indices=tf.TensorShape([num_read_heads, num_read_weights]),
keys=tf.TensorShape([num_read_heads, memory_word_size]),
strengths=tf.TensorShape([num_read_heads]),
)
with self._enter_variable_scope():
# Transforms to value-based read for each read head.
output_dim = (memory_word_size + 1) * num_read_heads
self._keys_and_read_strengths_generator = snt.Linear(output_dim)
def body1_enumerate_over_topo(self, potentials, map_to_indices, core, leafnode_num_record, l_br, r_br, r, r1):
potentials, map_to_indices, core_, leafnode_num_record_, l_br, r_br, r_, r1, r2 = tf.while_loop(
self.cond2_enumerate_over_topo,
self.body2_enumerate_over_topo,
loop_vars = [potentials, map_to_indices, core, leafnode_num_record, l_br, r_br, r, r1, r1+1],
shape_invariants = [tf.TensorShape([None, self.M*self.K]), tf.TensorShape([None, 2]),
core.get_shape(), leafnode_num_record.get_shape(),
tf.TensorShape([None, self.M*self.K]), tf.TensorShape([None, self.M*self.K]),
tf.TensorShape([]), tf.TensorShape([]), tf.TensorShape([])])
r1 = r1 + 1
return potentials, map_to_indices, core, leafnode_num_record, l_br, r_br, r, r1
def get_tensor_shape(self, tensor_name):
"""The tf.TensorShape of a tensor.
Args:
tensor_name: string, the name of a tensor in the graph.
Returns:
a tf.TensorShape
"""
tensor = self._name_to_tensor(tensor_name)
if isinstance(tensor, mtf.Tensor):
return tf.TensorShape(tensor.shape.to_integer_list)
else: # tf.Tensor
return tensor.shape
def test_sampler_parameter_validation(self, **kwargs):
dtypes = (tf.float32, )
shapes = (tf.TensorShape([3, 3]), )
if 'max_in_flight_samples_per_worker' not in kwargs:
kwargs['max_in_flight_samples_per_worker'] = 100
if 'want_error' in kwargs:
error = kwargs.pop('want_error')
with self.assertRaises(error):
reverb_dataset.ReplayDataset(self._client.server_address,
'dist', dtypes, shapes, **kwargs)
else:
reverb_dataset.ReplayDataset(self._client.server_address, 'dist',
dtypes, shapes, **kwargs)
