def setUp(self):
fake_dataset = Dataset.get_fake(self.input_size, 1)
self.model = MADE(fake_dataset,
hidden_sizes=self.hidden_sizes,
batch_size=fake_dataset['train']['data'].shape[0],
hidden_activation=theano.tensor.nnet.sigmoid,
use_cond_mask=self.use_cond_mask,
direct_input_connect=self.direct_input_connect,
direct_output_connect=self.direct_output_connect)
self.nb_shuffle = 50
def setUp(self):
### Testing that the sum of all prob is equal to 1 ###
# This test has to be run in 64bit for accuracy
self._old_theano_config_floatX = theano.config.floatX
theano.config.floatX = 'float64'
self.nb_test = 15
self._shuffling_type = "Full"
fake_dataset = Dataset.get_permutation(self.input_size)
self.model = MADE(fake_dataset,
hidden_sizes=self.hidden_sizes,
batch_size=fake_dataset['train']['data'].shape[0],
hidden_activation=theano.tensor.nnet.sigmoid,
use_cond_mask=self.use_cond_mask,
direct_input_connect=self.direct_input_connect,
direct_output_connect=self.direct_output_connect)
# Train the model to have more accurate results
for i in range(2 * self.input_size):
self.model.shuffle(self._shuffling_type)
self.model.learn(i, True)
def _get_fake_model(self, in_size, hidden_sizes):
fake_dataset = Dataset.get_fake(in_size, 1)
return MADE(
fake_dataset,
# learning_rate=trainingparams['learning_rate'],
# decrease_constant=trainingparams['decrease_constant'],
hidden_sizes=hidden_sizes,
# random_seed=hyperparams['random_seed'],
# batch_size=trainingparams['batch_size'],
hidden_activation=lambda x: x,
use_cond_mask=False,
direct_input_connect="Output",
direct_output_connect=False)
def _get_conditioning_mask_model(dataset, input_size, hidden_sizes):
import theano.tensor as T
return MADE(
dataset,
# learning_rate=trainingparams['learning_rate'],
# decrease_constant=trainingparams['decrease_constant'],
hidden_sizes=hidden_sizes,
# random_seed=hyperparams['random_seed'],
# batch_size=trainingparams['batch_size'],
#hidden_activation=lambda x: x,
hidden_activation=T.nnet.sigmoid,
use_cond_mask=True,
direct_input_connect="None",
direct_output_connect=False,
weights_initialization="Diagonal")
def build_model(dataset, trainingparams, hyperparams, hidden_sizes):
print '\n### Initializing MADE ... ',
start_time = t.time()
model = MADE(dataset,
learning_rate=trainingparams['learning_rate'],
decrease_constant=trainingparams['decrease_constant'],
hidden_sizes=hidden_sizes,
random_seed=hyperparams['random_seed'],
batch_size=trainingparams['batch_size'],
hidden_activation=activation_functions[
hyperparams['hidden_activation']],
use_cond_mask=hyperparams['use_cond_mask'],
direct_input_connect=hyperparams['direct_input_connect'],
direct_output_connect=hyperparams['direct_output_connect'],
update_rule=trainingparams['update_rule'],
dropout_rate=trainingparams['dropout_rate'],
weights_initialization=hyperparams['weights_initialization'],
mask_distribution=hyperparams['mask_distribution'])
print utils.get_done_text(start_time), "###"
return model
def setUp(self):
### Testing that the sum of all prob is equal to 1 ###
# This test has to be run in 64bit for accuracy
self._old_theano_config_floatX = theano.config.floatX
theano.config.floatX = 'float64'
self.nb_test = 15
self._shuffling_type = "Full"
fake_dataset = Dataset.get_permutation(self.input_size)
self.model = MADE(fake_dataset,
hidden_sizes=self.hidden_sizes,
batch_size=fake_dataset['train']['data'].shape[0],
hidden_activation=theano.tensor.nnet.sigmoid,
use_cond_mask=self.use_cond_mask,
direct_input_connect=self.direct_input_connect,
direct_output_connect=self.direct_output_connect)
# Train the model to have more accurate results
for i in range(2 * self.input_size):
self.model.shuffle(self._shuffling_type)
self.model.learn(i, True)
class MasksTests(unittest.TestCase):
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, *args, **kwargs)
# Setup model
self.input_size = 10
self.hidden_sizes = [500]
self.use_cond_mask = False
self.direct_input_connect = "None"
self.direct_output_connect = False
def setUp(self):
fake_dataset = Dataset.get_fake(self.input_size, 1)
self.model = MADE(fake_dataset,
hidden_sizes=self.hidden_sizes,
batch_size=fake_dataset['train']['data'].shape[0],
hidden_activation=theano.tensor.nnet.sigmoid,
use_cond_mask=self.use_cond_mask,
direct_input_connect=self.direct_input_connect,
direct_output_connect=self.direct_output_connect)
self.nb_shuffle = 50
def tearDown(self):
pass
def _shuffles(self, shuffle_type):
for i in range(self.nb_shuffle):
self.model.shuffle(shuffle_type)
def _get_masks(self):
return [layer.weights_mask.get_value() for layer in self.model.layers]
def test_base(self):
for layer in self.model.layers:
mask = layer.weights_mask.get_value()
assert_array_equal(mask, np.ones_like(mask))
def test_shuffle_once(self):
shuffle_type = "Once"
self.model.shuffle("Full") # Initial shuffle, always Full
initial_masks = self._get_masks()
self.model.shuffle(shuffle_type)
shuffled_once_masks = self._get_masks()
# Testing that they are shuffled
for masks in zip(initial_masks, shuffled_once_masks):
assert_false(np.array_equal(masks[0], masks[1]))
self._shuffles(shuffle_type)
shuffled_masks = self._get_masks()
# Testing that they are not shuffled again
for masks in zip(shuffled_once_masks, shuffled_masks):
assert_array_equal(masks[0], masks[1])
def test_reset(self):
shuffle_type = "Full"
self.model.shuffle("Full") # Initial shuffle, always Full
initial_masks = self._get_masks()
self._shuffles(shuffle_type)
shuffled_masks = self._get_masks()
# Testing that they are shuffled
for masks in zip(initial_masks, shuffled_masks):
assert_false(np.array_equal(masks[0], masks[1]))
self.model.reset(shuffle_type)
# Testing that they are resetted
for i in range(len(self.model.layers)):
assert_array_equal(initial_masks[i], self.model.layers[i].weights_mask.get_value())
self._shuffles(shuffle_type) # Shuffling again
# Testing that they are reshuffled exactly at the same state
for i in range(len(self.model.layers)):
assert_array_equal(shuffled_masks[i], self.model.layers[i].weights_mask.get_value())
class AutoregressivePropertyTests(unittest.TestCase):
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, *args, **kwargs)
# Setup model
self.input_size = 12
self.hidden_sizes = [500]
self.use_cond_mask = False
self.direct_input_connect = "None"
self.direct_output_connect = False
def setUp(self):
### Testing that the sum of all prob is equal to 1 ###
# This test has to be run in 64bit for accuracy
self._old_theano_config_floatX = theano.config.floatX
theano.config.floatX = 'float64'
self.nb_test = 15
self._shuffling_type = "Full"
fake_dataset = Dataset.get_permutation(self.input_size)
self.model = MADE(fake_dataset,
hidden_sizes=self.hidden_sizes,
batch_size=fake_dataset['train']['data'].shape[0],
hidden_activation=theano.tensor.nnet.sigmoid,
use_cond_mask=self.use_cond_mask,
direct_input_connect=self.direct_input_connect,
direct_output_connect=self.direct_output_connect)
# Train the model to have more accurate results
for i in range(2 * self.input_size):
self.model.shuffle(self._shuffling_type)
self.model.learn(i, True)
def tearDown(self):
theano.config.floatX = self._old_theano_config_floatX
def test_total_prob(self):
print "Testing on model: input_size={0} hidden_sizes={1}{2}{3}{4} ".format(self.input_size, self.hidden_sizes, " CondMask" if self.use_cond_mask else "", " DirectInputConnect" + self.direct_input_connect if self.direct_input_connect != "None" else "", " DirectOutputConnect" if self.direct_output_connect else ""),
# Test the model on all the data
ps = []
for i in range(self.nb_test):
ps += np.exp(self.model.valid_log_prob(False), dtype=theano.config.floatX).sum(dtype=theano.config.floatX),
self.model.shuffle(self._shuffling_type)
print ".",
assert_almost_equal(ps, 1)
def test_verify_masks(self):
nb_perm_mask = 10
# Set all the parameters to one
for layer in self.model.layers:
for param in layer.params:
param.set_value(np.ones(param.shape.eval(), dtype=theano.config.floatX))
# Make sure that the input do not "see" itself
for perm in range(nb_perm_mask):
base = self.model.use(np.zeros((1, self.input_size), dtype=theano.config.floatX), False)
for i in range(self.input_size):
inp = np.zeros((1, self.input_size), dtype=theano.config.floatX)
inp[0][i] = 1
test = self.model.use(inp, False)
assert_array_equal(base[0][i], test[0][i])
self.model.shuffle(self._shuffling_type)
class AutoregressivePropertyTests(unittest.TestCase):
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, *args, **kwargs)
# Setup model
self.input_size = 12
self.hidden_sizes = [500]
self.use_cond_mask = False
self.direct_input_connect = "None"
self.direct_output_connect = False
def setUp(self):
### Testing that the sum of all prob is equal to 1 ###
# This test has to be run in 64bit for accuracy
self._old_theano_config_floatX = theano.config.floatX
theano.config.floatX = 'float64'
self.nb_test = 15
self._shuffling_type = "Full"
fake_dataset = Dataset.get_permutation(self.input_size)
self.model = MADE(fake_dataset,
hidden_sizes=self.hidden_sizes,
batch_size=fake_dataset['train']['data'].shape[0],
hidden_activation=theano.tensor.nnet.sigmoid,
use_cond_mask=self.use_cond_mask,
direct_input_connect=self.direct_input_connect,
direct_output_connect=self.direct_output_connect)
# Train the model to have more accurate results
for i in range(2 * self.input_size):
self.model.shuffle(self._shuffling_type)
self.model.learn(i, True)
def tearDown(self):
theano.config.floatX = self._old_theano_config_floatX
def test_total_prob(self):
print "Testing on model: input_size={0} hidden_sizes={1}{2}{3}{4} ".format(
self.input_size, self.hidden_sizes,
" CondMask" if self.use_cond_mask else "",
" DirectInputConnect" + self.direct_input_connect
if self.direct_input_connect != "None" else "",
" DirectOutputConnect" if self.direct_output_connect else ""),
# Test the model on all the data
ps = []
for i in range(self.nb_test):
ps += np.exp(
self.model.valid_log_prob(False),
dtype=theano.config.floatX).sum(dtype=theano.config.floatX),
self.model.shuffle(self._shuffling_type)
print ".",
assert_almost_equal(ps, 1)
def test_verify_masks(self):
nb_perm_mask = 10
# Set all the parameters to one
for layer in self.model.layers:
for param in layer.params:
param.set_value(
np.ones(param.shape.eval(), dtype=theano.config.floatX))
# Make sure that the input do not "see" itself
for perm in range(nb_perm_mask):
base = self.model.use(
np.zeros((1, self.input_size), dtype=theano.config.floatX),
False)
for i in range(self.input_size):
inp = np.zeros((1, self.input_size),
dtype=theano.config.floatX)
inp[0][i] = 1
test = self.model.use(inp, False)
assert_array_equal(base[0][i], test[0][i])
self.model.shuffle(self._shuffling_type)