def test_verify_AIS(self):
model = oRBM(input_size=self.input_size,
hidden_size=self.hidden_size,
beta=self.beta)
model.W.set_value(self.W)
model.b.set_value(self.b)
model.c.set_value(self.c)
# Brute force
print "Computing lnZ using brute force (i.e. summing the free energy of all posible $v$)..."
V = theano.shared(
value=cartesian([(0, 1)] * self.input_size, dtype=config.floatX))
brute_force_lnZ = logsumexp(-model.free_energy(V), 0)
f_brute_force_lnZ = theano.function([], brute_force_lnZ)
params_bak = [param.get_value() for param in model.parameters]
print "Approximating lnZ using AIS..."
import time
start = time.time()
try:
ais_working_dir = tempfile.mkdtemp()
result = compute_AIS(model,
M=self.nb_samples,
betas=self.betas,
seed=1234,
ais_working_dir=ais_working_dir,
force=True)
logcummean_Z, logcumstd_Z_down, logcumstd_Z_up = result[
'logcummean_Z'], result['logcumstd_Z_down'], result[
'logcumstd_Z_up']
std_lnZ = result['std_lnZ']
print "{0} sec".format(time.time() - start)
import pylab as plt
plt.gca().set_xmargin(0.1)
plt.errorbar(range(1, self.nb_samples + 1),
logcummean_Z,
yerr=[std_lnZ, std_lnZ],
fmt='or')
plt.errorbar(range(1, self.nb_samples + 1),
logcummean_Z,
yerr=[logcumstd_Z_down, logcumstd_Z_up],
fmt='ob')
plt.plot([1, self.nb_samples], [f_brute_force_lnZ()] * 2, '--g')
plt.ticklabel_format(useOffset=False, axis='y')
plt.show()
AIS_logZ = logcummean_Z[-1]
assert_array_equal(params_bak[0], model.W.get_value())
assert_array_equal(params_bak[1], model.b.get_value())
assert_array_equal(params_bak[2], model.c.get_value())
print np.abs(AIS_logZ - f_brute_force_lnZ())
assert_almost_equal(AIS_logZ, f_brute_force_lnZ(), decimal=2)
finally:
shutil.rmtree(ais_working_dir)
def test_verify_AIS(self):
model = iRBM(input_size=self.input_size,
hidden_size=self.hidden_size,
beta=self.beta)
model.W.set_value(self.W)
model.b.set_value(self.b)
model.c.set_value(self.c)
# Brute force
print "Computing lnZ using brute force (i.e. summing the free energy of all posible $v$)..."
V = theano.shared(value=cartesian([(0, 1)] * self.input_size, dtype=config.floatX))
brute_force_lnZ = logsumexp(-model.free_energy(V), 0)
f_brute_force_lnZ = theano.function([], brute_force_lnZ)
params_bak = [param.get_value() for param in model.parameters]
print "Approximating lnZ using AIS..."
import time
start = time.time()
try:
ais_working_dir = tempfile.mkdtemp()
result = compute_AIS(model, M=self.nb_samples, betas=self.betas, seed=1234, ais_working_dir=ais_working_dir, force=True)
logcummean_Z, logcumstd_Z_down, logcumstd_Z_up = result['logcummean_Z'], result['logcumstd_Z_down'], result['logcumstd_Z_up']
std_lnZ = result['std_lnZ']
print "{0} sec".format(time.time() - start)
import pylab as plt
plt.gca().set_xmargin(0.1)
plt.errorbar(range(1, self.nb_samples+1), logcummean_Z, yerr=[std_lnZ, std_lnZ], fmt='or')
plt.errorbar(range(1, self.nb_samples+1), logcummean_Z, yerr=[logcumstd_Z_down, logcumstd_Z_up], fmt='ob')
plt.plot([1, self.nb_samples], [f_brute_force_lnZ()]*2, '--g')
plt.ticklabel_format(useOffset=False, axis='y')
plt.show()
AIS_logZ = logcummean_Z[-1]
assert_array_equal(params_bak[0], model.W.get_value())
assert_array_equal(params_bak[1], model.b.get_value())
assert_array_equal(params_bak[2], model.c.get_value())
print np.abs(AIS_logZ - f_brute_force_lnZ())
assert_almost_equal(AIS_logZ, f_brute_force_lnZ(), decimal=2)
finally:
shutil.rmtree(ais_working_dir)
def main():
parser = buildArgsParser()
args = parser.parse_args()
# Get experiment folder
experiment_path = args.name
if not os.path.isdir(experiment_path):
# If not a directory, it must be the name of the experiment.
experiment_path = pjoin(".", "experiments", args.name)
if not os.path.isdir(experiment_path):
parser.error('Cannot find experiment: {0}!'.format(args.name))
if not os.path.isfile(pjoin(experiment_path, "model.pkl")):
parser.error(
'Cannot find model for experiment: {0}!'.format(experiment_path))
if not os.path.isfile(pjoin(experiment_path, "hyperparams.json")):
parser.error('Cannot find hyperparams for experiment: {0}!'.format(
experiment_path))
# Load experiments hyperparameters
hyperparams = utils.load_dict_from_json_file(
pjoin(experiment_path, "hyperparams.json"))
with Timer("Loading dataset"):
trainset, validset, testset = dataset.load(
hyperparams['dataset'], hyperparams.get('dataset_percent', 1.))
print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset),
len(testset)),
with Timer("Loading model"):
if hyperparams["model"] == "rbm":
from iRBM.models.rbm import RBM
model_class = RBM
elif hyperparams["model"] == "orbm":
from iRBM.models.orbm import oRBM
model_class = oRBM
elif hyperparams["model"] == "irbm":
from iRBM.models.irbm import iRBM
model_class = iRBM
# Load the actual model.
model = model_class.load(pjoin(experiment_path, "model.pkl"))
if args.irbm_fixed_size:
# Use methods from the oRBM.
import functools
from iRBM.models.orbm import oRBM
setattr(model, "get_base_rate",
functools.partial(oRBM.get_base_rate, model))
setattr(model, "pdf_z_given_v",
functools.partial(oRBM.pdf_z_given_v, model))
setattr(model, "log_z_given_v",
functools.partial(oRBM.log_z_given_v, model))
setattr(model, "free_energy",
functools.partial(oRBM.free_energy, model))
print "({} with {} fixed hidden units)".format(
hyperparams["model"], model.hidden_size)
else:
print "({} with {} hidden units)".format(hyperparams["model"],
model.hidden_size)
# Result files.
if args.irbm_fixed_size:
experiment_path = pjoin(experiment_path, "irbm_fixed_size")
try:
os.makedirs(experiment_path)
except:
pass
ais_result_file = pjoin(experiment_path, "ais_result.json")
result_file = pjoin(experiment_path, "result.json")
if args.lnZ is not None:
lnZ, lnZ_down, lnZ_up = args.lnZ
else:
if not os.path.isfile(ais_result_file) or args.force:
with Timer(
"Estimating model's partition function with AIS({0}) and {1} temperatures."
.format(args.nb_samples, args.nb_temperatures)):
ais_results = compute_AIS(model,
M=args.nb_samples,
betas=np.linspace(
0, 1, args.nb_temperatures),
seed=args.seed,
ais_working_dir=experiment_path,
force=args.force)
ais_results["irbm_fixed_size"] = args.irbm_fixed_size
utils.save_dict_to_json_file(ais_result_file, ais_results)
else:
print "Loading previous AIS results... (use --force to re-run AIS)"
ais_results = utils.load_dict_from_json_file(ais_result_file)
print "AIS({0}) with {1} temperatures".format(
ais_results['nb_samples'], ais_results['nb_temperatures'])
if ais_results['nb_samples'] != args.nb_samples:
print "The number of samples specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(
args.nb_samples, ais_results['nb_samples'])
sys.exit(-1)
if ais_results['nb_temperatures'] != args.nb_temperatures:
print "The number of temperatures specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(
args.nb_temperatures, ais_results['nb_temperatures'])
sys.exit(-1)
if ais_results['seed'] != args.seed:
print "The seed specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(
args.seed, ais_results['seed'])
sys.exit(-1)
if ais_results.get('irbm_fixed_size',
False) != args.irbm_fixed_size:
print "The option '--irbm-fixed' specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(
args.irbm_fixed_size, ais_results['irbm_fixed_size'])
sys.exit(-1)
lnZ = ais_results['logcummean_Z'][-1]
logcumstd_Z_down = ais_results['logcumstd_Z_down'][-1]
logcumstd_Z_up = ais_results['logcumstd_Z_up'][-1]
lnZ_down = lnZ - logcumstd_Z_down
lnZ_up = lnZ + logcumstd_Z_up
print "-> lnZ: {lnZ_down} <= {lnZ} <= {lnZ_up}".format(lnZ_down=lnZ_down,
lnZ=lnZ,
lnZ_up=lnZ_up)
with Timer("\nComputing average NLL on {0} using lnZ={1}.".format(
hyperparams['dataset'], lnZ)):
NLL_train, NLL_valid, NLL_test = compute_AvgStderrNLL(
model, lnZ, trainset, validset, testset)
print "Avg. NLL on trainset: {:.2f} ± {:.2f}".format(
NLL_train.avg, NLL_train.stderr)
print "Avg. NLL on validset: {:.2f} ± {:.2f}".format(
NLL_valid.avg, NLL_valid.stderr)
print "Avg. NLL on testset: {:.2f} ± {:.2f}".format(
NLL_test.avg, NLL_test.stderr)
# Save results JSON file.
if args.lnZ is None:
result = {
'lnZ': float(lnZ),
'lnZ_down': float(lnZ_down),
'lnZ_up': float(lnZ_up),
'trainset': [float(NLL_train.avg),
float(NLL_train.stderr)],
'validset': [float(NLL_valid.avg),
float(NLL_valid.stderr)],
'testset': [float(NLL_test.avg),
float(NLL_test.stderr)],
'irbm_fixed_size': args.irbm_fixed_size,
}
utils.save_dict_to_json_file(result_file, result)
if args.view:
from iRBM.misc import vizu
import matplotlib.pyplot as plt
if hyperparams["dataset"] == "binarized_mnist":
image_shape = (28, 28)
elif hyperparams["dataset"] == "caltech101_silhouettes28":
image_shape = (28, 28)
else:
raise ValueError("Unknown dataset: {0}".format(
hyperparams["dataset"]))
# Display AIS samples.
data = vizu.concatenate_images(ais_results['last_sample_chain'],
shape=image_shape,
border_size=1,
clim=(0, 1))
plt.figure()
plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest')
plt.title("AIS samples")
# Display AIS ~lnZ.
plt.figure()
plt.gca().set_xmargin(0.1)
plt.errorbar(np.arange(ais_results['nb_samples']) + 1,
ais_results["logcummean_Z"],
yerr=[
ais_results['logcumstd_Z_down'],
ais_results['logcumstd_Z_up']
],
fmt='ob',
label='with std ~ln std Z')
plt.legend()
plt.ticklabel_format(useOffset=False, axis='y')
plt.title("~ln mean Z for different number of AIS samples")
plt.ylabel("~lnZ")
plt.xlabel("# AIS samples")
plt.show()
def compute_lnZ(model, nb_chains, temperatures):
ais_results = compute_AIS(model, M=nb_chains, betas=temperatures)
lnZ_est = ais_results['logcummean_Z'][-1]
lnZ_down = ais_results['logcumstd_Z_down'][-1]
lnZ_up = ais_results['logcumstd_Z_up'][-1]
return lnZ_est, lnZ_down, lnZ_up
def main():
parser = buildArgsParser()
args = parser.parse_args()
# Get experiment folder
experiment_path = args.name
if not os.path.isdir(experiment_path):
# If not a directory, it must be the name of the experiment.
experiment_path = pjoin(".", "experiments", args.name)
if not os.path.isdir(experiment_path):
parser.error('Cannot find experiment: {0}!'.format(args.name))
if not os.path.isfile(pjoin(experiment_path, "model.pkl")):
parser.error('Cannot find model for experiment: {0}!'.format(experiment_path))
if not os.path.isfile(pjoin(experiment_path, "hyperparams.json")):
parser.error('Cannot find hyperparams for experiment: {0}!'.format(experiment_path))
# Load experiments hyperparameters
hyperparams = utils.load_dict_from_json_file(pjoin(experiment_path, "hyperparams.json"))
with Timer("Loading dataset"):
trainset, validset, testset = dataset.load(hyperparams['dataset'], hyperparams.get('dataset_percent', 1.))
print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset), len(testset)),
with Timer("Loading model"):
if hyperparams["model"] == "rbm":
from iRBM.models.rbm import RBM
model_class = RBM
elif hyperparams["model"] == "orbm":
from iRBM.models.orbm import oRBM
model_class = oRBM
elif hyperparams["model"] == "irbm":
from iRBM.models.irbm import iRBM
model_class = iRBM
# Load the actual model.
model = model_class.load(pjoin(experiment_path, "model.pkl"))
if args.irbm_fixed_size:
# Use methods from the oRBM.
import functools
from iRBM.models.orbm import oRBM
setattr(model, "get_base_rate", functools.partial(oRBM.get_base_rate, model))
setattr(model, "pdf_z_given_v", functools.partial(oRBM.pdf_z_given_v, model))
setattr(model, "log_z_given_v", functools.partial(oRBM.log_z_given_v, model))
setattr(model, "free_energy", functools.partial(oRBM.free_energy, model))
print "({} with {} fixed hidden units)".format(hyperparams["model"], model.hidden_size)
else:
print "({} with {} hidden units)".format(hyperparams["model"], model.hidden_size)
# Result files.
if args.irbm_fixed_size:
experiment_path = pjoin(experiment_path, "irbm_fixed_size")
try:
os.makedirs(experiment_path)
except:
pass
ais_result_file = pjoin(experiment_path, "ais_result.json")
result_file = pjoin(experiment_path, "result.json")
if args.lnZ is not None:
lnZ, lnZ_down, lnZ_up = args.lnZ
else:
if not os.path.isfile(ais_result_file) or args.force:
with Timer("Estimating model's partition function with AIS({0}) and {1:,} temperatures.".format(args.nb_samples, args.nb_temperatures)):
ais_results = compute_AIS(model, M=args.nb_samples, betas=np.linspace(0, 1, args.nb_temperatures), seed=args.seed, ais_working_dir=experiment_path, force=args.force)
ais_results["irbm_fixed_size"] = args.irbm_fixed_size
utils.save_dict_to_json_file(ais_result_file, ais_results)
else:
print "Loading previous AIS results... (use --force to re-run AIS)"
ais_results = utils.load_dict_from_json_file(ais_result_file)
print "AIS({0}) with {1:,} temperatures".format(ais_results['nb_samples'], ais_results['nb_temperatures'])
if ais_results['nb_samples'] != args.nb_samples:
print "The number of samples specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(args.nb_samples, ais_results['nb_samples'])
sys.exit(-1)
if ais_results['nb_temperatures'] != args.nb_temperatures:
print "The number of temperatures specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(args.nb_temperatures, ais_results['nb_temperatures'])
sys.exit(-1)
if ais_results['seed'] != args.seed:
print "The seed specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(args.seed, ais_results['seed'])
sys.exit(-1)
if ais_results.get('irbm_fixed_size', False) != args.irbm_fixed_size:
print "The option '--irbm-fixed' specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(args.irbm_fixed_size, ais_results['irbm_fixed_size'])
sys.exit(-1)
lnZ = ais_results['logcummean_Z'][-1]
logcumstd_Z_down = ais_results['logcumstd_Z_down'][-1]
logcumstd_Z_up = ais_results['logcumstd_Z_up'][-1]
lnZ_down = lnZ - logcumstd_Z_down
lnZ_up = lnZ + logcumstd_Z_up
print "-> lnZ: {lnZ_down} <= {lnZ} <= {lnZ_up}".format(lnZ_down=lnZ_down, lnZ=lnZ, lnZ_up=lnZ_up)
with Timer("\nComputing average NLL on {0} using lnZ={1}.".format(hyperparams['dataset'], lnZ)):
NLL_train, NLL_valid, NLL_test = compute_AvgStderrNLL(model, lnZ, trainset, validset, testset)
print "Avg. NLL on trainset: {:.2f} ± {:.2f}".format(NLL_train.avg, NLL_train.stderr)
print "Avg. NLL on validset: {:.2f} ± {:.2f}".format(NLL_valid.avg, NLL_valid.stderr)
print "Avg. NLL on testset: {:.2f} ± {:.2f}".format(NLL_test.avg, NLL_test.stderr)
print "---"
Fv_rnd = model.free_energy(np.random.rand(*ais_results['last_sample_chain'].shape)).eval()
print "Avg. F(v) on {:,} random samples: {:.2f} ± {:.2f}".format(args.nb_samples, Fv_rnd.mean(), Fv_rnd.std())
Fv_model = model.free_energy(ais_results['last_sample_chain']).eval()
print "Avg. F(v) on {:,} AIS samples: {:.2f} ± {:.2f}".format(args.nb_samples, Fv_model.mean(), Fv_model.std())
# Save results JSON file.
if args.lnZ is None:
result = {'lnZ': float(lnZ),
'lnZ_down': float(lnZ_down),
'lnZ_up': float(lnZ_up),
'trainset': [float(NLL_train.avg), float(NLL_train.stderr)],
'validset': [float(NLL_valid.avg), float(NLL_valid.stderr)],
'testset': [float(NLL_test.avg), float(NLL_test.stderr)],
'irbm_fixed_size': args.irbm_fixed_size,
}
utils.save_dict_to_json_file(result_file, result)
if args.view:
from iRBM.misc import vizu
import matplotlib.pyplot as plt
if hyperparams["dataset"] == "binarized_mnist":
image_shape = (28, 28)
elif hyperparams["dataset"] == "caltech101_silhouettes28":
image_shape = (28, 28)
else:
raise ValueError("Unknown dataset: {0}".format(hyperparams["dataset"]))
# Display AIS samples.
data = vizu.concatenate_images(ais_results['last_sample_chain'], shape=image_shape, border_size=1, clim=(0, 1))
plt.figure()
plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest')
plt.title("AIS samples")
# Display AIS ~lnZ.
plt.figure()
plt.gca().set_xmargin(0.1)
plt.errorbar(np.arange(ais_results['nb_samples'])+1, ais_results["logcummean_Z"],
yerr=[ais_results['logcumstd_Z_down'], ais_results['logcumstd_Z_up']],
fmt='ob', label='with std ~ln std Z')
plt.legend()
plt.ticklabel_format(useOffset=False, axis='y')
plt.title("~ln mean Z for different number of AIS samples")
plt.ylabel("~lnZ")
plt.xlabel("# AIS samples")
plt.show()
