def test_luongAttention_local():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import LuongAttention
luong_attention_layer = LuongAttention(local=True,
stddev=1.0,
regularizer=None)
assert luong_attention_layer is not None
# Encoded values
encoded_values = np.random.sample((2, 10, 64))
query_values = np.random.sample((2, 64))
position_values = np.random.randint(0, 10, (2, ))
# Get some sample input tensors
encoder_tensor = tf.constant(encoded_values) # BS x SEQLEN x ENC_CELL_SIZE
query_tensor = tf.constant(query_values) # BS x DEC_CELL_SIZE
position_tensor = tf.constant(position_values) # BS
value = luong_attention_layer(
(query_tensor, encoder_tensor, position_tensor))
# Construct the session
output = run_simple_session(value, None)
assert output is not None # Make sure the value is correct
assert output.shape == (2, 64) # Make sure the output shape is correct
# Do regression testing
check_regression('luong_attention_local_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_scaledDotProductSimilarity():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import ScaledDotProductSimilarity
sdp_layer = ScaledDotProductSimilarity()
assert sdp_layer is not None
# Encoded values
query_values = np.random.sample((2, 10, 64))
context_values = np.random.sample((2, 5, 64))
# Get some sample input tensors
query_tensor = tf.constant(query_values)
context_tensor = tf.constant(context_values)
value = sdp_layer((context_tensor, query_tensor))
# Construct the session
output = run_simple_session(value, None)
assert output is not None # Make sure the value is correct
assert output.shape == (2, 5, 10) # Make sure the output shape is correct
# Do regression testing
check_regression('scaled_dot_product_similarity_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_contextQueryAttention():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import ContextQueryAttention
attention_map = ContextQueryAttention(similarity_metric='trilinear')
assert attention_map is not None
# Encoded values
context_values = np.random.sample((2, 8, 12))
query_values = np.random.sample((2, 10, 12))
mask_values = np.random.choice([0, 1], size=(2, 8, 10))
# Get some sample input tensors
context_tensor = tf.constant(context_values)
query_tensor = tf.constant(query_values)
mask_tensor = tf.constant(mask_values)
value = attention_map(inputs=(context_tensor, query_tensor),
mask=mask_tensor)
# Construct the session
output = run_simple_session(value, None)
assert output is not None # Make sure the value is not none
assert output.shape == (2, 8, 4 * 12)
check_regression('context_query_attention_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_gated_tanh():
tf.reset_default_graph()
np.random.seed(256)
tf.random.set_random_seed(256)
# Construct the layer
from rinokeras.core.v1x.common.layers.activations import GatedTanh
gth_layer = GatedTanh(n_units=128)
assert gth_layer is not None
# Encoded values
input_values = np.random.sample((16, 256))
# Get some sample input tensors
input_tensor = tf.constant(input_values)
value = gth_layer(input_tensor)
# Construct the session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
output = sess.run(value)
assert output is not None # Make sure the value is correct
assert output.shape == (16, 128) # Make sure the output shape is correct
# Do regression testing
check_regression('gated_tanh_expected_output',
output,
__file__,
'regression_outputs/test_activation_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_DiagGaussianPd():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.distributions import DiagGaussianPd
layer = DiagGaussianPd(action_shape=(4, 4))
# Make sure that the layer is not None
assert layer is not None
# Encoded values
input_tensor, _ = random_tensor((16, 4, 4))
actions_tensor, _ = random_tensor((16, 4, 4))
# Get the output of the layer
value = layer(input_tensor)
logp_actions = layer.logp_actions(actions_tensor)
entropy = layer.entropy()
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(
inputs=[value, logp_actions, entropy],
feed={},
weights=[layer],
weights_file=weights_file)
# Make sure the value is not null
assert output[0] is not None
assert output[1] is not None
assert output[2] is not None
# Make sure the output shape is correct
assert output[0].shape == (16, 4, 4)
assert output[1].shape == (16, )
assert output[2].shape == ()
# Make sure the output values are correct (If Possible)
pass
# Check loading and restoring
load_restore_test(output=output,
inputs=[value, logp_actions, entropy],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('DiagGaussianPd_expected_output',
output,
__file__,
'regression_outputs/test_distribution_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_transformer_decoder():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.transformer import TransformerDecoder, TransformerInputEmbedding
tie = TransformerInputEmbedding(128, False)
layer = TransformerDecoder(embedding_layer=tie,
output_layer=tf.keras.layers.Dense(128),
n_layers=2,
n_heads=4,
d_model=128,
d_filter=32)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
source_tensor, _ = random_tensor((2,32,128))
target_tensor, _ = random_tensor((2,32,64))
# Get random masking values
source_mask, _ = random_mask_tensor(2, 32)
target_mask, _ = random_mask_tensor(2, 32)
source_mask = convert_to_attention_mask(source_tensor, source_mask)
target_mask = convert_to_attention_mask(target_tensor, target_mask)
# Get the output of the layer
value = layer((source_tensor, target_tensor), mask=(source_mask, target_mask))
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(2,32,128)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('transformer_decoder_output',
output, __file__, 'regression_outputs/test_transformer_decoder_outputs.json', debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(TransformerDecoder, layer)
def test_transformer_base():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.transformer import Transformer
layer = Transformer(discrete=True,
n_symbols_in=16,
n_symbols_out=16,
n_layers=6,
n_heads=4,
d_model=32,
d_filter=16)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
source_tensor, _ = random_sequence_tensor(2, 32, 16)
target_tensor, _ = random_sequence_tensor(2, 32, 16)
source_mask, _ = random_mask_tensor(2, 32)
target_mask, _ = random_mask_tensor(2, 32)
# Get the output of the layer
value = layer((source_tensor, target_tensor),
mask=(source_mask, target_mask))
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(2, 32, 16)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('transformer_base',
output,
__file__,
'regression_outputs/test_transformer_outputs.json',
debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(Transformer, layer)
def test_transformer_encoder_masking_with_conv():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.transformer import TransformerEncoder
layer = TransformerEncoder(embedding_layer=tf.keras.layers.Dense(64),
n_layers=2,
n_heads=4,
d_model=64,
d_filter=128)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
input_tensor, _ = random_tensor((16, 32, 128))
input_mask, _ = random_mask_tensor(16, 32)
input_mask = convert_to_attention_mask(input_tensor, input_mask)
conv_mask, _ = random_mask_tensor(16, 32)
conv_mask = convert_sequence_length_to_sequence_mask(
input_tensor, conv_mask)
# Get the output of the layer
value = layer(input_tensor, mask=(input_mask, conv_mask))
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(16, 32, 64)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression(
'transformer_encoder_output_masking_with_conv',
output,
__file__,
'regression_outputs/test_transformer_encoder_outputs.json',
debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(TransformerEncoder, layer)
def test_qanet_base():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.qanet import QANet
layer = QANet(n_chars=32, n_symbols=32)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
context_tensor, _ = random_sequence_tensor(2, 8, 32)
query_tensor, _ = random_sequence_tensor(2, 8, 32)
context_char_tensor = tf.convert_to_tensor(
np.random.randint(0, 32, (2, 8, 16)))
query_char_tensor = tf.convert_to_tensor(
np.random.randint(0, 32, (2, 8, 16)))
# Get the output of the layer
value = layer(
(context_tensor, query_tensor, context_char_tensor, query_char_tensor))
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(2, 8, 128)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('qanet_base_expected_output',
output,
__file__,
'regression_outputs/test_qanet_outputs.json',
debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(QANet, layer)
def test_transformer_decoder_fast_beam_decode_discrete():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.transformer import TransformerDecoder, TransformerInputEmbedding
tie = TransformerInputEmbedding(128, discrete=True, n_symbols=256)
layer = TransformerDecoder(embedding_layer=tie,
output_layer=tf.keras.layers.Dense(256),
n_layers=2,
n_heads=4,
d_model=128,
d_filter=32)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
source_tensor, _ = random_tensor((2,32,128))
# Get the output of the layer
value, scores = layer.fast_beam_decode(source_tensor, 20, batch_size=2, n_beams=4)
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value, scores],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(2,4, 20), (2,4)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('transformer_decoder_fast_beam_decode',
output, __file__, 'regression_outputs/test_transformer_decoder_outputs.json', debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(TransformerDecoder, layer)
def test_transformer_multi_attention():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.transformer.transformer_attention import TransformerMultiAttention
layer = TransformerMultiAttention(n_heads=4)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
encoder_tensor, _ = random_tensor((16, 32, 64))
decoder_tensor, _ = random_tensor((16, 32, 64))
# Get the output of the layer
value = layer((encoder_tensor, decoder_tensor))
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(16, 32, 64)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression(
'transformer_multi_attention_expected_output',
output,
__file__,
'regression_outputs/test_transformer_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(TransformerMultiAttention, layer)
def test_multiHeadAttentionMap():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import MultiHeadAttentionMap
from rinokeras.core.v1x.common.attention import ScaledDotProductSimilarity
sdp = ScaledDotProductSimilarity()
attention_map = MultiHeadAttentionMap(similarity_metric=sdp,
n_heads=4,
attention_function=tf.nn.softmax)
assert attention_map is not None
assert sdp is not None
# Encoded values
query_values = np.random.sample((2, 8, 12))
key_values = np.random.sample((2, 20, 12))
value_values = np.random.sample((2, 20, 12))
mask_values = np.random.choice([0, 1], size=(2, 8, 20))
# Get some sample input tensors
query_tensor = tf.constant(query_values)
key_tensor = tf.constant(key_values)
value_tensor = tf.constant(value_values)
mask_tensor = tf.constant(mask_values)
value = attention_map(inputs=(query_tensor, key_tensor, value_tensor),
mask=mask_tensor,
return_attention_weights=True)
# Construct the session
output = run_simple_session(value, None)
assert output[0] is not None # Make sure the value is not none
assert output[1] is not None # Make sure the value is not none
assert output[0].shape == (2, 8, 12)
assert output[1].shape == (2, 4, 8, 20)
masked_vals = np.squeeze(output[1][:, 0, :, :])[np.where(mask_values == 0)]
assert np.isclose(masked_vals, np.zeros_like(masked_vals)).all()
check_regression('multihead_attention_map_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_transformer_input_embedding_non_discrete():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.transformer import TransformerInputEmbedding
layer = TransformerInputEmbedding(embed_size=128, discrete=False)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
input_tensor, _ = random_tensor((16, 32, 64))
# Get the output of the layerembedding_layer
value = layer(input_tensor)
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(16, 32, 128)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression(
'transformer_input_embedding_non_discrete_expected_output',
output,
__file__,
'regression_outputs/test_transformer_embedding_outputs.json',
debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(TransformerInputEmbedding, layer)
def test_qanet_encoder_no_mask():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.qanet import QANetEncoderBlock
layer = QANetEncoderBlock(n_conv=2,
n_heads=4,
filter_size=128,
hidden_size=128)
# Make sure that the layer is not None
assert layer is not None
# Encoded values
input_tensor, _ = random_tensor((2,8,128))
# Get the output of the layer
value = layer(input_tensor)
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(2,8,128)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('qanet_encoder_no_mask_expected_output',
output, __file__, 'regression_outputs/test_qanet_encoder_outputs.json', debug=_RK_REBUILD_REGRESSION)
# Do a config test
from_config_test(QANetEncoderBlock, layer)
def test_normed_conv_stack_3d():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.layers.conv import NormedConvStack
ncs_layer = NormedConvStack(dimension=3, filters=12, kernel_size=4)
# Make sure that the layer is not None
assert ncs_layer is not None
# Encoded values
input_tensor, input_values = random_tensor((2, 8, 16, 16, 16))
# Get the output of the layer
value = ncs_layer(input_tensor)
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[ncs_layer],
weights_file=weights_file)
# Make sure the value is not null
assert output[0] is not None
# Make sure the output shape is correct
assert output[0].shape == (2, 8, 16, 16, 12)
# Make sure the output values are correct (If Possible)
pass
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[ncs_layer],
weights_file=weights_file)
# Do regression testing
check_regression('normed_conv_stack_3d_expected_output',
output, __file__, 'regression_outputs/test_conv_outputs.json', debug=_RK_REBUILD_REGRESSION)
def test_applyAttentionMask():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import ApplyAttentionMask
aam_layer = ApplyAttentionMask()
assert aam_layer is not None
# Encoded values
similarity_values = np.ones((2, 10, 10))
similarity_values_heads = np.ones((2, 4, 10, 10))
mask_values = np.random.choice([0, 1], size=(2, 10, 10))
mask_values_heads = np.random.choice([0, 1], size=(2, 10, 10))
# Get some sample input tensors
similarity_tensor = tf.constant(similarity_values)
similarity_heads_tensor = tf.constant(similarity_values_heads)
mask_tensor = tf.constant(mask_values)
mask_heads_tensor = tf.constant(mask_values_heads)
value = aam_layer(inputs=similarity_tensor, mask=mask_tensor)
value_heads = aam_layer(inputs=similarity_heads_tensor,
mask=mask_heads_tensor)
# Construct the session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
output = sess.run([value, value_heads])
assert output[0] is not None # Make sure the value is not none
assert output[1] is not None # Make sure the value is not none
assert output[0].shape == (2, 10, 10) # Make sure the value is not none
# Make sure the value is not none
assert output[1].shape == (2, 4, 10, 10)
check_regression('apply_attention_mask_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_convert_to_attention_mask_2():
reset_session()
# Construct the layer
from rinokeras.core.v1x.utils import convert_to_attention_mask
# Encoded values
input_tensor, _ = random_sequence_tensor(2, 32, 128)
input_mask, _ = random_mask_tensor(2, 32)
# Get the output of the layer
value = convert_to_attention_mask(input_tensor, input_mask)
# Construct the session
output = run_simple_session(inputs=[value], feed={})
assert_not_none(output)
assert_expected_shapes(output, [(2, 32, 32)])
# Do regression testing
check_regression('convert_to_attention_mask_2',
output,
__file__,
'regression_outputs/test_utils_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_grouped_conv():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.layers.conv import GroupedConvolution
rb_layer = GroupedConvolution()
# Make sure that the layer is not None
assert rb_layer is not None
# Encoded values
input_tensor, _ = random_tensor((2, 16, 16, 3))
# Get the output of the layer
value = rb_layer(input_tensor)
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[rb_layer],
weights_file=weights_file)
# Make sure the value is not null
assert output[0] is not None
# Make sure the output shape is correct
assert output[0].shape == (2, 16, 16, 64)
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[rb_layer],
weights_file=weights_file)
# Do regression testing
check_regression('grouped_conv_expected_output',
output, __file__, 'regression_outputs/test_conv_outputs.json', debug=_RK_REBUILD_REGRESSION)
def test_attentionMap():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import AttentionMap
from rinokeras.core.v1x.common.attention import ScaledDotProductSimilarity
sdp = ScaledDotProductSimilarity()
attention_map = AttentionMap(similarity_metric=sdp,
attention_function=tf.nn.softmax)
assert attention_map is not None
assert sdp is not None
# Encoded values
query_values = np.random.sample((2, 8, 12))
key_values = np.random.sample((2, 20, 12))
value_values = np.random.sample((2, 20, 12))
mask_values = np.random.choice([0, 1], size=(2, 8, 20))
# Get some sample input tensors
query_tensor = tf.constant(query_values)
key_tensor = tf.constant(key_values)
value_tensor = tf.constant(value_values)
mask_tensor = tf.constant(mask_values)
value = attention_map(inputs=(query_tensor, key_tensor, value_tensor),
mask=mask_tensor)
# Construct the session
output = run_simple_session(value, None)
assert output[0] is not None # Make sure the value is not none
assert output[1] is not None # Make sure the value is not none
assert output[0].shape == (2, 8, 12)
assert output[1].shape == (2, 8, 20)
check_regression('attention_map_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_selfAttention():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import SelfAttention
attention_map = SelfAttention(similarity_metric='scaled_dot', n_heads=4)
assert attention_map is not None
# Encoded values
sa_values = np.random.sample((4, 64, 12))
mask_values = np.random.choice([0, 1], size=(4, 64, 64))
# Get some sample input tensors
sa_tensor = tf.constant(sa_values)
mask_tensor = tf.constant(mask_values)
value = attention_map(inputs=sa_tensor,
mask=mask_tensor,
return_attention_weights=True)
# Construct the session
output = run_simple_session(value, None)
assert output[0] is not None # Make sure the value is not none
assert output[1] is not None # Make sure the value is not none
assert output[0].shape == (4, 64, 12)
assert output[1].shape == (4, 4, 64, 64)
# WARNING: This might fail because probability
masked_vals = np.squeeze(output[1][:, 0, :, :])[np.where(mask_values == 0)]
assert np.isclose(masked_vals, np.zeros_like(masked_vals)).all()
check_regression('self_attention_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
# Check that you can instantiate a layer from the config
check_from_config(SelfAttention, attention_map)
def test_resnext50_base():
reset_session()
# Construct the layer
from rinokeras.core.v1x.models.resnet.resnet import ResNeXt50
layer = ResNeXt50()
# Make sure that the layer is not None
assert layer is not None
# Encoded values
inputs_tensor, _ = random_tensor((2, 8, 8, 3))
# Get the output of the layer
value = layer(inputs_tensor)
# Create a named temporary file for save/restore testing
with tempfile.TemporaryFile() as weights_file:
# Construct the session
output = run_simple_session_save_weights(inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
assert_not_none(output)
assert_expected_shapes(output, [(2, 2048)])
# Check loading and restoring
load_restore_test(output=output,
inputs=[value],
feed={},
weights=[layer],
weights_file=weights_file)
# Do regression testing
check_regression('resnext50_base_expected_output',
output,
__file__,
'regression_outputs/test_resnet_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_attentionQKVProjection():
reset_session()
# Construct the layer
from rinokeras.core.v1x.common.attention import AttentionQKVProjection
attention_qkv_projection = AttentionQKVProjection(key_depth=8,
value_depth=12)
assert attention_qkv_projection is not None
# Encoded values
query_values = np.random.sample((2, 10, 64))
key_values = np.random.sample((2, 5, 64))
value_values = np.random.sample((2, 5, 64))
# Get some sample input tensors
query_tensor = tf.constant(query_values)
key_tensor = tf.constant(key_values)
value_tensor = tf.constant(value_values)
value = attention_qkv_projection((query_tensor, key_tensor, value_tensor))
# Construct the session
output = run_simple_session(value, None)
assert output is not None # Make sure the value is correct
# Make sure the output shape is correct
assert output[0].shape == (2, 10, 8)
# Make sure the output shape is correct
assert output[1].shape == (2, 5, 8)
# Make sure the output shape is correct
assert output[2].shape == (2, 5, 12)
# Do regression testing
check_regression('attentionqkv_projection_expected_output',
output,
__file__,
'regression_outputs/test_attention_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_coupling_layer():
tf.reset_default_graph()
np.random.seed(256)
tf.random.set_random_seed(256)
# Construct the layer
from rinokeras.core.v1x.common.layers.autoregressive import CouplingLayer
coupling_layer = CouplingLayer(n_units=128,
layer=tf.keras.layers.Dense(128))
assert coupling_layer is not None
# Encoded values
input_values_a = np.random.sample((16, 128))
input_values_b = np.random.sample((16, 128))
# Get some sample input tensors
input_tensor_a = tf.constant(input_values_a)
input_tensor_b = tf.constant(input_values_b)
value, log_s = coupling_layer((input_tensor_a, input_tensor_b))
reverse_value = coupling_layer((input_tensor_a, input_tensor_b),
reverse=True)
# Create a named temporary file for save/restore testing
output_file = tempfile.NamedTemporaryFile()
# Construct the session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
output = sess.run([value, log_s, reverse_value])
# Check weights
weights = coupling_layer.get_weights()
pickle.dump(weights, output_file)
assert output[0] is not None # Make sure the value is not null
assert output[1] is not None # Make sure the value is not null
assert output[2] is not None # Make sure the value is not null
assert output[0].shape == (16, 128
) # Make sure the output shape is correct
assert output[1].shape == (16, 128
) # Make sure the output shape is correct
assert output[2].shape == (16, 128
) # Make sure the output shape is correct
# Load/Restore test
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
output_file.seek(0)
coupling_layer.set_weights(pickle.load(output_file))
restored_output = sess.run([value, log_s, reverse_value])
assert np.isclose(restored_output[0], output[0]).all()
assert np.isclose(restored_output[1], output[1]).all()
assert np.isclose(restored_output[2], output[2]).all()
output_file.close()
# Do regression testing
check_regression('coupling_layer_expected_output',
output,
__file__,
'regression_outputs/test_autoregressive_outputs.json',
debug=_RK_REBUILD_REGRESSION)
def test_random_gauss_noise():
tf.reset_default_graph()
np.random.seed(256)
tf.random.set_random_seed(256)
# Construct the layer
from rinokeras.core.v1x.common.layers.autoregressive import RandomGaussNoise
gaussian_noise_layer = RandomGaussNoise()
assert gaussian_noise_layer is not None
# Encoded values
input_values = np.random.sample((16, 256))
# Get some sample input tensors
input_tensor = tf.constant(input_values)
value = gaussian_noise_layer(input_tensor)
logstd = gaussian_noise_layer.logstd
std = gaussian_noise_layer.std
# Create a named temporary file for save/restore testing
output_file = tempfile.NamedTemporaryFile()
# Construct the session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
output = sess.run([value, logstd, std])
# Check weights
weights = gaussian_noise_layer.get_weights()
pickle.dump(weights, output_file)
assert output[0] is not None # Make sure the value is not null
assert output[1] is not None # Make sure the value is not null
assert output[2] is not None # Make sure the value is not null
assert output[0].shape == (16, 256
) # Make sure the output shape is correct
assert output[1].shape == (256, ) # Make sure the output shape is correct
assert output[2].shape == (256, ) # Make sure the output shape is correct
assert np.isclose(output[1], np.zeros_like(
output[1])).all() # Make sure the output value is correct
assert np.isclose(output[2], np.ones_like(
output[2])).all() # Make sure the output value is correct
assert not np.isclose(output[0],
input_values).all() # Make sure some noise was added
# Load/Restore test
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
output_file.seek(0)
gaussian_noise_layer.set_weights(pickle.load(output_file))
restored_output = sess.run([value, logstd, std])
assert np.isclose(restored_output[0], output[0]).all()
output_file.close()
# Do regression testing
check_regression('random_gauss_noise_expected_output',
output,
__file__,
'regression_outputs/test_autoregressive_outputs.json',
debug=_RK_REBUILD_REGRESSION)
