def setup():
"""
Create pickle file with a simple model.
"""
# tearDown is guaranteed to run pop_load_data.
control.push_load_data(False)
with open('dbm.pkl', 'wb') as f:
dataset = MNIST(which_set='train', start=0, stop=100, binarize=True)
vis_layer = BinaryVector(nvis=784, bias_from_marginals=dataset)
hid_layer1 = BinaryVectorMaxPool(layer_name='h1',
pool_size=1,
irange=.05,
init_bias=-2.,
detector_layer_dim=50)
hid_layer2 = BinaryVectorMaxPool(layer_name='h2',
pool_size=1,
irange=.05,
init_bias=-2.,
detector_layer_dim=10)
model = DBM(batch_size=20,
niter=2,
visible_layer=vis_layer,
hidden_layers=[hid_layer1, hid_layer2])
model.dataset_yaml_src = """
!obj:pylearn2.datasets.binarizer.Binarizer {
raw: !obj:pylearn2.datasets.mnist.MNIST {
which_set: "train",
start: 0,
stop: 100
}
}
"""
model.layer_to_chains = model.make_layer_to_state(1)
cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
def setup():
"""
Create pickle file with a simple model.
"""
# tearDown is guaranteed to run pop_load_data.
control.push_load_data(False)
with open('dbm.pkl', 'wb') as f:
dataset = MNIST(which_set='train', start=0, stop=100, binarize=True)
vis_layer = BinaryVector(nvis=784, bias_from_marginals=dataset)
hid_layer1 = BinaryVectorMaxPool(layer_name='h1', pool_size=1,
irange=.05, init_bias=-2.,
detector_layer_dim=50)
hid_layer2 = BinaryVectorMaxPool(layer_name='h2', pool_size=1,
irange=.05, init_bias=-2.,
detector_layer_dim=10)
model = DBM(batch_size=20, niter=2, visible_layer=vis_layer,
hidden_layers=[hid_layer1, hid_layer2])
model.dataset_yaml_src = """
!obj:pylearn2.datasets.binarizer.Binarizer {
raw: !obj:pylearn2.datasets.mnist.MNIST {
which_set: "train",
start: 0,
stop: 100
}
}
"""
model.layer_to_chains = model.make_layer_to_state(1)
cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
def test_unpickle():
fd, fname = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as f:
d = {'a': 1, 'b': 2}
cPickle.dump(d, f)
loaded = load("{'a': !pkl: '%s'}" % fname)
assert_(loaded['a'] == d)
os.remove(fname)
def test_preproc_pkl():
fd, fname = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as f:
d = ('a', 1)
cPickle.dump(d, f)
environ['TEST_VAR'] = fname
loaded = load('a: !pkl: "${TEST_VAR}"')
assert_(loaded['a'] == d)
del environ['TEST_VAR']
def test_unpickle_key():
fd, fname = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as f:
d = ('a', 1)
cPickle.dump(d, f)
loaded = load("{!pkl: '%s': 50}" % fname)
assert_(first_key(loaded) == d)
assert_(first_value(loaded) == 50)
os.remove(fname)
def test_summarize_model():
"""
Asks the summarize_model.py script to inspect a pickled model and
check that it completes succesfully
"""
skip_if_no_matplotlib()
with open('model.pkl', 'wb') as f:
cPickle.dump(MLP(layers=[Linear(dim=5, layer_name='h0', irange=0.1)],
nvis=10), f, protocol=cPickle.HIGHEST_PROTOCOL)
summarize('model.pkl')
os.remove('model.pkl')
def test_summarize_model():
"""
Asks the summarize_model.py script to inspect a pickled model and
check that it completes succesfully
"""
skip_if_no_matplotlib()
with open('model.pkl', 'wb') as f:
cPickle.dump(MLP(layers=[Linear(dim=5, layer_name='h0', irange=0.1)],
nvis=10),
f,
protocol=cPickle.HIGHEST_PROTOCOL)
summarize('model.pkl')
os.remove('model.pkl')
def test_show_weights():
"""
Create a pickled model and show the weights
"""
skip_if_no_matplotlib()
with open('model.pkl', 'wb') as f:
model = MLP(layers=[Linear(dim=1, layer_name='h0', irange=0.1)],
nvis=784)
model.dataset_yaml_src = """
!obj:pylearn2.datasets.mnist.MNIST {
which_set: 'train'
}
"""
cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
show_weights('model.pkl', rescale='individual',
border=True, out='garbage.png')
os.remove('model.pkl')
os.remove('garbage.png')
def _save(filepath, obj):
"""
.. todo::
WRITEME
"""
try:
import joblib
joblib_available = True
except ImportError:
joblib_available = False
if filepath.endswith('.npy'):
np.save(filepath, obj)
return
# This is dumb
# assert filepath.endswith('.pkl')
save_dir = os.path.dirname(filepath)
# Handle current working directory case.
if save_dir == '':
save_dir = '.'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if os.path.exists(save_dir) and not os.path.isdir(save_dir):
raise IOError("save path %s exists, not a directory" % save_dir)
elif not os.access(save_dir, os.W_OK):
raise IOError("permission error creating %s" % filepath)
try:
if joblib_available and filepath.endswith('.joblib'):
joblib.dump(obj, filepath)
else:
if filepath.endswith('.joblib'):
warnings.warn('Warning: .joblib suffix specified but joblib '
'unavailable. Using ordinary pickle.')
with open(filepath, 'wb') as filehandle:
cPickle.dump(obj, filehandle, get_pickle_protocol())
except Exception as e:
logger.exception("cPickle has failed to write an object to "
"{0}".format(filepath))
if str(e).find('maximum recursion depth exceeded') != -1:
raise
try:
logger.info('retrying with pickle')
with open(filepath, "wb") as f:
pickle.dump(obj, f)
except Exception as e2:
if str(e) == '' and str(e2) == '':
logger.exception('neither cPickle nor pickle could write to '
'{0}'.format(filepath))
logger.exception(
'moreover, neither of them raised an exception that '
'can be converted to a string'
)
logger.exception(
'now re-attempting to write with cPickle outside the '
'try/catch loop so you can see if it prints anything '
'when it dies'
)
with open(filepath, 'wb') as f:
cPickle.dump(obj, f, get_pickle_protocol())
logger.info('Somehow or other, the file write worked once '
'we quit using the try/catch.')
else:
if str(e2) == 'env':
raise
import pdb
tb = pdb.traceback.format_exc()
reraise_as(IOError(str(obj) +
' could not be written to ' +
str(filepath) +
' by cPickle due to ' + str(e) +
' nor by pickle due to ' + str(e2) +
'. \nTraceback ' + tb))
logger.warning('{0} was written by pickle instead of cPickle, due to '
'{1} (perhaps your object'
' is really big?)'.format(filepath, e))
def _save(filepath, obj):
"""
.. todo::
WRITEME
"""
try:
import joblib
joblib_available = True
except ImportError:
joblib_available = False
if filepath.endswith('.npy'):
np.save(filepath, obj)
return
# This is dumb
# assert filepath.endswith('.pkl')
save_dir = os.path.dirname(filepath)
# Handle current working directory case.
if save_dir == '':
save_dir = '.'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if os.path.exists(save_dir) and not os.path.isdir(save_dir):
raise IOError("save path %s exists, not a directory" % save_dir)
elif not os.access(save_dir, os.W_OK):
raise IOError("permission error creating %s" % filepath)
try:
if joblib_available and filepath.endswith('.joblib'):
joblib.dump(obj, filepath)
else:
if filepath.endswith('.joblib'):
warnings.warn('Warning: .joblib suffix specified but joblib '
'unavailable. Using ordinary pickle.')
with open(filepath, 'wb') as filehandle:
cPickle.dump(obj, filehandle, get_pickle_protocol())
except Exception as e:
logger.exception("cPickle has failed to write an object to "
"{0}".format(filepath))
if str(e).find('maximum recursion depth exceeded') != -1:
raise
try:
logger.info('retrying with pickle')
with open(filepath, "wb") as f:
pickle.dump(obj, f)
except Exception as e2:
if str(e) == '' and str(e2) == '':
logger.exception('neither cPickle nor pickle could write to '
'{0}'.format(filepath))
logger.exception(
'moreover, neither of them raised an exception that '
'can be converted to a string'
)
logger.exception(
'now re-attempting to write with cPickle outside the '
'try/catch loop so you can see if it prints anything '
'when it dies'
)
with open(filepath, 'wb') as f:
cPickle.dump(obj, f, get_pickle_protocol())
logger.info('Somehow or other, the file write worked once '
'we quit using the try/catch.')
else:
if str(e2) == 'env':
raise
import pdb
tb = pdb.traceback.format_exc()
reraise_as(IOError(str(obj) +
' could not be written to ' +
str(filepath) +
' by cPickle due to ' + str(e) +
' nor by pickle due to ' + str(e2) +
'. \nTraceback ' + tb))
logger.warning('{0} was written by pickle instead of cPickle, due to '
'{1} (perhaps your object'
' is really big?)'.format(filepath, e))
def gendata(enable, os, downsample, textid=None, seed=2313, verbose=False):
"""
Generate the MNIST+ dataset.
:param enable: dictionary of flags with keys ['texture', 'azimuth',
'rotation', 'elevation'] to enable/disable a given factor of variation.
:param textid: if enable['texture'], id number of the Brodatz texture to
load. If textid is None, we load a random texture for each MNIST image.
:param os: output size (width and height) of MNIST+ images.
:param downsample: factor by which to downsample texture.
:param seed: integer for seeding RNG.
:param verbose: bool
"""
rng = numpy.random.RandomState(seed)
data = mnist.MNIST("train")
test = mnist.MNIST("test")
data.X = numpy.vstack((data.X, test.X))
data.y = numpy.hstack((data.y, test.y))
del test
output = {}
output["data"] = numpy.zeros((len(data.X), os * os))
output["label"] = numpy.zeros(len(data.y))
if enable["azimuth"]:
output["azimuth"] = numpy.zeros(len(data.y))
if enable["elevation"]:
output["elevation"] = numpy.zeros(len(data.y))
if enable["rotation"]:
output["rotation"] = numpy.zeros(len(data.y))
if enable["texture"]:
output["texture_id"] = numpy.zeros(len(data.y))
output["texture_pos"] = numpy.zeros((len(data.y), 2))
for i in xrange(len(data.X)):
# get MNIST image
frgd_img = to_img(data.X[i], 28)
frgd_img = frgd_img.convert("L")
if enable["rotation"]:
rot = rng.randint(0, 360)
output["rotation"][i] = rot
frgd_img = frgd_img.rotate(rot, Image.BILINEAR)
frgd_img = frgd_img.resize((os, os), Image.BILINEAR)
if enable["texture"]:
if textid is None:
# extract patch from texture database. Note that texture #14
# does not exist.
textid = 14
while textid == 14:
textid = rng.randint(1, 113)
patch_img, (px, py) = extract_patch(textid, os, downsample)
patch_arr = to_array(patch_img)
# store output details
output["texture_id"][i] = textid
output["texture_pos"][i] = (px, py)
# generate binary mask for digit outline
frgd_arr = to_array(frgd_img)
mask_arr = frgd_arr > 0.1
# copy contents of masked-MNIST image into background texture
blend_arr = copy(patch_arr)
blend_arr[mask_arr] = frgd_arr[mask_arr]
# this now because the image to emboss
frgd_img = to_img(blend_arr, os)
azi = 45
if enable["azimuth"]:
azi = rng.randint(0, 360)
output["azimuth"][i] = azi
ele = 18.0
if enable["elevation"]:
ele = rng.randint(0, 60)
output["elevation"][i] = ele
mboss_img = emboss(frgd_img, azi=azi, ele=ele)
mboss_arr = to_array(mboss_img)
output["data"][i] = mboss_arr
output["label"][i] = data.y[i]
if verbose:
pl.imshow(mboss_arr.reshape(os, os))
pl.gray()
pl.show()
fname = "mnistplus"
if enable["azimuth"]:
fname += "_azi"
if enable["rotation"]:
fname += "_rot"
if enable["texture"]:
fname += "_tex"
fp = open(fname + ".pkl", "w")
pickle.dump(output, fp, protocol=pickle.HIGHEST_PROTOCOL)
fp.close()
def gendata(enable, os, downsample, textid=None, seed=2313, verbose=False):
"""
Generate the MNIST+ dataset.
:param enable: dictionary of flags with keys ['texture', 'azimuth',
'rotation', 'elevation'] to enable/disable a given factor of variation.
:param textid: if enable['texture'], id number of the Brodatz texture to
load. If textid is None, we load a random texture for each MNIST image.
:param os: output size (width and height) of MNIST+ images.
:param downsample: factor by which to downsample texture.
:param seed: integer for seeding RNG.
:param verbose: bool
"""
rng = numpy.random.RandomState(seed)
data = mnist.MNIST('train')
test = mnist.MNIST('test')
data.X = numpy.vstack((data.X, test.X))
data.y = numpy.hstack((data.y, test.y))
del test
output = {}
output['data'] = numpy.zeros((len(data.X), os*os))
output['label'] = numpy.zeros(len(data.y))
if enable['azimuth']:
output['azimuth'] = numpy.zeros(len(data.y))
if enable['elevation']:
output['elevation'] = numpy.zeros(len(data.y))
if enable['rotation']:
output['rotation'] = numpy.zeros(len(data.y))
if enable['texture']:
output['texture_id'] = numpy.zeros(len(data.y))
output['texture_pos'] = numpy.zeros((len(data.y), 2))
for i in xrange(len(data.X)):
# get MNIST image
frgd_img = to_img(data.X[i], 28)
frgd_img = frgd_img.convert('L')
if enable['rotation']:
rot = rng.randint(0, 360)
output['rotation'][i] = rot
frgd_img = frgd_img.rotate(rot, Image.BILINEAR)
frgd_img = frgd_img.resize((os, os), Image.BILINEAR)
if enable['texture']:
if textid is None:
# extract patch from texture database. Note that texture #14
# does not exist.
textid = 14
while textid == 14:
textid = rng.randint(1, 113)
patch_img, (px, py) = extract_patch(textid, os, downsample)
patch_arr = to_array(patch_img)
# store output details
output['texture_id'][i] = textid
output['texture_pos'][i] = (px, py)
# generate binary mask for digit outline
frgd_arr = to_array(frgd_img)
mask_arr = frgd_arr > 0.1
# copy contents of masked-MNIST image into background texture
blend_arr = copy(patch_arr)
blend_arr[mask_arr] = frgd_arr[mask_arr]
# this now because the image to emboss
frgd_img = to_img(blend_arr, os)
azi = 45
if enable['azimuth']:
azi = rng.randint(0, 360)
output['azimuth'][i] = azi
ele = 18.
if enable['elevation']:
ele = rng.randint(0, 60)
output['elevation'][i] = ele
mboss_img = emboss(frgd_img, azi=azi, ele=ele)
mboss_arr = to_array(mboss_img)
output['data'][i] = mboss_arr
output['label'][i] = data.y[i]
if verbose:
pl.imshow(mboss_arr.reshape(os, os))
pl.gray()
pl.show()
fname = 'mnistplus'
if enable['azimuth']:
fname += "_azi"
if enable['rotation']:
fname += "_rot"
if enable['texture']:
fname += "_tex"
fp = open(fname+'.pkl','w')
pickle.dump(output, fp, protocol=pickle.HIGHEST_PROTOCOL)
fp.close()
