def yaml_file_execution(file_path):
try:
train = yaml_parse.load_path(file_path)
train.algorithm.termination_criterion = EpochCounter(max_epochs=2)
train.main_loop()
except NoDataPathError:
raise SkipTest("PYLEARN2_DATA_PATH environment variable not defined")
def yaml_file_execution(file_path):
try:
train = yaml_parse.load_path(file_path)
train.algorithm.termination_criterion = EpochCounter(max_epochs=2)
train.main_loop()
except NoDataPathError:
raise SkipTest("PYLEARN2_DATA_PATH environment variable not defined")
def load_path(path, environ=None, **kwargs):
"""
Convenience function for loading a YAML configuration from a file
into a `PartialPlus` graph.
Parameters
----------
path : str
The path to the file to load on disk.
environ : dict, optional
A dictionary used for ${FOO} substitutions in addition to
environment variables. If a key appears both in `os.environ`
and this dictionary, the value in this dictionary is used.
Returns
-------
graph : Node
A `PartialPlus` or `Literal` node representing the root
node of the YAML hierarchy.
Notes
-----
Other keyword arguments are passed on to `yaml.load`.
"""
return proxy_to_partialplus(yaml_parse.load_path(path, instantiate=False,
**kwargs),
environ=environ)
def load_path(path, environ=None, **kwargs):
"""
Convenience function for loading a YAML configuration from a file
into a `PartialPlus` graph.
Parameters
----------
path : str
The path to the file to load on disk.
environ : dict, optional
A dictionary used for ${FOO} substitutions in addition to
environment variables. If a key appears both in `os.environ`
and this dictionary, the value in this dictionary is used.
Returns
-------
graph : Node
A `PartialPlus` or `Literal` node representing the root
node of the YAML hierarchy.
Notes
-----
Other keyword arguments are passed on to `yaml.load`.
"""
return proxy_to_partialplus(yaml_parse.load_path(path,
instantiate=False,
**kwargs),
environ=environ)
def test_load_path():
fd, fname = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as f:
f.write("a: 23")
loaded = load_path(fname)
assert_(loaded['a'] == 23)
os.remove(fname)
def load_train_file(config_file_path, environ=None):
"""
Loads and parses a yaml file for a Train object.
Publishes the relevant training environment variables
Parameters
----------
config_file_path : WRITEME
Returns
-------
WRITEME
"""
from pylearn2.config import yaml_parse
suffix_to_strip = '.yaml'
# publish environment variables related to file name
if config_file_path.endswith(suffix_to_strip):
config_file_full_stem = config_file_path[0:-len(suffix_to_strip)]
else:
config_file_full_stem = config_file_path
for varname in ["PYLEARN2_TRAIN_FILE_FULL_STEM"]:
os.environ[varname] = config_file_full_stem
directory = config_file_path.split('/')[:-1]
directory = '/'.join(directory)
if directory != '':
directory += '/'
os.environ["PYLEARN2_TRAIN_DIR"] = directory
os.environ["PYLEARN2_TRAIN_BASE_NAME"] = config_file_path.split('/')[-1]
os.environ["PYLEARN2_TRAIN_FILE_STEM"] = config_file_full_stem.split('/')[-1]
return yaml_parse.load_path(config_file_path, environ=environ)
def load_train_file(config_file_path):
"""Loads and parses a yaml file for a Train object.
Publishes the relevant training environment variables"""
from pylearn2.config import yaml_parse
suffix_to_strip = '.yaml'
# publish environment variables related to file name
if config_file_path.endswith(suffix_to_strip):
config_file_full_stem = config_file_path[0:-len(suffix_to_strip)]
else:
config_file_full_stem = config_file_path
for varname in ["PYLEARN2_TRAIN_FILE_NAME", #this one is deprecated
"PYLEARN2_TRAIN_FILE_FULL_STEM"]: #this is the new, accepted name
environ.putenv(varname, config_file_full_stem)
directory = config_file_path.split('/')[:-1]
directory = '/'.join(directory)
if directory != '':
directory += '/'
environ.putenv("PYLEARN2_TRAIN_DIR", directory)
environ.putenv("PYLEARN2_TRAIN_BASE_NAME", config_file_path.split('/')[-1] )
environ.putenv("PYLEARN2_TRAIN_FILE_STEM", config_file_full_stem.split('/')[-1] )
return yaml_parse.load_path(config_file_path)
def test_IS_cost():
"""
VAE trains properly with the importance sampling cost
"""
yaml_src_path = os.path.join(os.path.dirname(__file__), "test_vae_cost_is_criterion.yaml")
train_object = yaml_parse.load_path(yaml_src_path)
train_object.main_loop()
def test_IS_cost():
"""
VAE trains properly with the importance sampling cost
"""
train_object = yaml_parse.load_path('pylearn2/models/tests/'
'test_vae_cost_is_criterion.yaml')
train_object.main_loop()
def load_train_file(config_file_path):
"""Loads and parses a yaml file for a Train object.
Publishes the relevant training environment variables"""
from pylearn2.config import yaml_parse
suffix_to_strip = '.yaml'
# publish environment variables related to file name
if config_file_path.endswith(suffix_to_strip):
config_file_full_stem = config_file_path[0:-len(suffix_to_strip)]
else:
config_file_full_stem = config_file_path
for varname in [
"PYLEARN2_TRAIN_FILE_NAME", #this one is deprecated
"PYLEARN2_TRAIN_FILE_FULL_STEM"
]: #this is the new, accepted name
environ.putenv(varname, config_file_full_stem)
environ.putenv("PYLEARN2_TRAIN_DIR",
'/'.join(config_file_path.split('/')[:-1]))
environ.putenv("PYLEARN2_TRAIN_BASE_NAME", config_file_path.split('/')[-1])
environ.putenv("PYLEARN2_TRAIN_FILE_STEM",
config_file_full_stem.split('/')[-1])
return yaml_parse.load_path(config_file_path)
def test_VAE_cost():
"""
VAE trains properly with the VAE cost
"""
train_object = yaml_parse.load_path('pylearn2/models/tests/'
'test_vae_cost_vae_criterion.yaml')
train_object.main_loop()
def test_load_path():
fd, fname = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as f:
f.write("a: 23")
loaded = load_path(fname)
assert_(loaded['a'] == 23)
os.remove(fname)
def test_load_from_yaml(self):
"""
Load dataset from an yaml file.
"""
imdset = yaml_parse.load_path(self.yaml_file)
imdset = imdset['dataset']
self.assertEqual(len(imdset.adjusters), 6)
def test_IS_cost():
"""
VAE trains properly with the importance sampling cost
"""
yaml_src_path = os.path.join(os.path.dirname(__file__),
'test_vae_cost_is_criterion.yaml')
train_object = yaml_parse.load_path(yaml_src_path)
train_object.main_loop()
def construct_model(self):
filedir = os.path.join(os.path.dirname(__file__), 'mlps.yaml')
layer_args = yaml_parse.load_path(filedir)[self.modelname]
layers = []
# adapt in case of 2d layer
if (self.conv_class == ConvElemwise):
self.adapt_for_2d_conv(layer_args)
else:
self.adapt_for_time_dim(layer_args)
print layer_args
for i, layer_arg in enumerate(layer_args):
layer = self.construct_layer(layer_arg, i)
layers.append(layer)
input_space = self.create_input_space()
mlp = MLP(input_space=input_space, layers=layers)
return mlp
def load_train_file(config_file_path, environ=None):
"""
Loads and parses a yaml file for a Train object.
Publishes the relevant training environment variables
Parameters
----------
config_file_path : str
Path to a config file containing a YAML string describing a
pylearn2.train.Train object
environ : dict, optional
A dictionary used for ${FOO} substitutions in addition to
environment variables when parsing the YAML file. If a key appears
both in `os.environ` and this dictionary, the value in this
dictionary is used.
Returns
-------
Object described by the YAML string stored in the config file
"""
from pylearn2.config import yaml_parse
suffix_to_strip = '.yaml'
# Publish environment variables related to file name
if config_file_path.endswith(suffix_to_strip):
config_file_full_stem = config_file_path[0:-len(suffix_to_strip)]
else:
config_file_full_stem = config_file_path
os.environ["PYLEARN2_TRAIN_FILE_FULL_STEM"] = config_file_full_stem
directory = config_file_path.split('/')[:-1]
directory = '/'.join(directory)
if directory != '':
directory += '/'
os.environ["PYLEARN2_TRAIN_DIR"] = directory
os.environ["PYLEARN2_TRAIN_BASE_NAME"] = config_file_path.split('/')[-1]
os.environ["PYLEARN2_TRAIN_FILE_STEM"] = config_file_full_stem.split(
'/')[-1]
return yaml_parse.load_path(config_file_path, environ=environ)
def load_train_file(config_file_path, environ=None):
"""
Loads and parses a yaml file for a Train object.
Publishes the relevant training environment variables
Parameters
----------
config_file_path : str
Path to a config file containing a YAML string describing a
pylearn2.train.Train object
environ : dict, optional
A dictionary used for ${FOO} substitutions in addition to
environment variables when parsing the YAML file. If a key appears
both in `os.environ` and this dictionary, the value in this
dictionary is used.
Returns
-------
Object described by the YAML string stored in the config file
"""
from pylearn2.config import yaml_parse
suffix_to_strip = '.yaml'
# Publish environment variables related to file name
if config_file_path.endswith(suffix_to_strip):
config_file_full_stem = config_file_path[0:-len(suffix_to_strip)]
else:
config_file_full_stem = config_file_path
os.environ["PYLEARN2_TRAIN_FILE_FULL_STEM"] = config_file_full_stem
directory = config_file_path.split('/')[:-1]
directory = '/'.join(directory)
if directory != '':
directory += '/'
os.environ["PYLEARN2_TRAIN_DIR"] = directory
os.environ["PYLEARN2_TRAIN_BASE_NAME"] = config_file_path.split('/')[-1]
os.environ[
"PYLEARN2_TRAIN_FILE_STEM"] = config_file_full_stem.split('/')[-1]
return yaml_parse.load_path(config_file_path, environ=environ)
def load_yaml(self, fname):
"""
Slot that loads a YAML file.
"""
if not fname:
return
try:
# publish environment variables relevant to this file
serial.prepare_train_file(fname)
# load the tree of Proxy objects
environ = {}
yaml_tree = yaml_parse.load_path(fname,
instantiate=False,
environ=environ)
yaml_tree = yaml_parse._instantiate(yaml_tree)
self.show_object_tree(yaml_tree)
except Exception, exc:
logger.error('Loading aml file failed', exc_info=True)
QtGui.QMessageBox.warning(self, 'Exception', str(exc))
t6 = time.time()
print(t6 - t1, t2 - t1, t3 - t2, t4 - t3, t5 - t4, t6 - t5)
if self.chunk_size is not None:
assert save_path.endswith('.npy')
save_path_pieces = save_path.split('.npy')
assert len(save_path_pieces) == 2
assert save_path_pieces[1] == ''
save_path = save_path_pieces[0] + '_' + chr(
ord('A') + self.chunk_id) + '.npy'
np.save(save_path, output)
if nan > 0:
warnings.warn(str(nan) + ' features were nan')
if __name__ == '__main__':
assert len(sys.argv) == 2
yaml_path = sys.argv[1]
assert yaml_path.endswith('.yaml')
val = yaml_path[0:-len('.yaml')]
os.environ['FEATURE_EXTRACTOR_YAML_PATH'] = val
os.putenv('FEATURE_EXTRACTOR_YAML_PATH', val)
extractor = yaml_parse.load_path(yaml_path)
extractor()
import os
import shutil
from pylearn2.config import yaml_parse
from pylearn2.utils import serial
from pylearn2.utils import shell
status, rc = shell.run_shell_command("qstat -u goodfell -t @hades")
assert rc == 0
results = open("results.dat", "r")
lines = results.readlines()
results.close()
params = yaml_parse.load_path('params.yaml')
added = 0
print 'Experiment numbers reported by this script start at 0.'
print 'Keep in mind that vim will refer to the first line of results.dat as line 1'
for expnum, line in enumerate(lines):
elems = line.split(' ')
assert elems[-1] == '\n'
obj = elems[0]
if obj == 'P':
# print expnum, 'pending according to results.dat'
expdir = '/RQexec/goodfell/experiment_7/%d' % expnum
if not os.path.exists(expdir):
print 'Experiment not yet configured for experiment',expnum
continue
cluster_info = expdir + '/cluster_info.txt'
if not os.path.exists(cluster_info):
__author__ = "Ian Goodfellow" from pylearn2.config import yaml_parse import sys _, path = sys.argv simulator = yaml_parse.load_path(path) simulator.main_loop()
import os
from pylearn2.utils.serial import mkdir
from pylearn2.config import yaml_parse
from pylearn2.utils import safe_zip
import shutil
results = open("results.dat", "r")
lines = results.readlines()
results.close()
params = yaml_parse.load_path('params.yaml')
validate = open('validate.yaml', 'r')
validate_template = validate.read()
validate.close()
for expnum, line in enumerate(lines):
elems = line.split(' ')
assert elems[-1] == '\n'
obj = elems[0]
if obj == 'P':
expdir = '/RQexec/goodfell/experiment_6/%d' % expnum
if os.path.exists(expdir):
continue
try:
mkdir(expdir)
config = {}
for param, value in safe_zip(params, elems[2:-1]):
if param['type'] == 'float':
value = float(value)
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj = yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj, 'get_batch_topo'):
#obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows * cols)
else:
#obj is a Model
model = obj
def main(options, positional_args):
assert len(positional_args) == 1
path ,= positional_args
out = options.out
rescale = options.rescale
if rescale == 'none':
global_rescale = False
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj =yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj,'get_batch_topo'):
#obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows*cols)
else:
#obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(batch_size = rows * cols, theano_rng = theano_rng)
from theano import function
print 'compiling sampling function'
f = function([],design_examples_var)
print 'sampling'
design_examples = f()
print 'loading dataset'
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = N.asarray( [
N.sqrt(N.sum(N.square(examples[i,:])))
for i in xrange(examples.shape[0])
])
print 'norms of examples: '
print '\tmin: ',norms.min()
print '\tmean: ',norms.mean()
print '\tmax: ',norms.max()
print 'range of elements of examples',(examples.min(),examples.max())
print 'dtype: ', examples.dtype
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= N.abs(examples).max()
if len(examples.shape) != 4:
print 'sorry, view_examples.py only supports image examples for now.'
print 'this dataset has '+str(len(examples.shape)-2)+' topological dimensions'
quit(-1)
is_color = False
assert examples.shape[3] == 2
print examples.shape[1:3]
pv = patch_viewer.PatchViewer( (rows, cols * 2), examples.shape[1:3], is_color = is_color)
for i in xrange(rows*cols):
# Load patches in backwards order for easier cross-eyed viewing
# (Ian can't do the magic eye thing where you focus your eyes past the screen, must
# focus eyes in front of screen)
pv.add_patch(examples[i,:,:,1], activation = 0.0, rescale = patch_rescale)
pv.add_patch(examples[i,:,:,0], activation = 0.0, rescale = patch_rescale)
if out is None:
pv.show()
else:
pv.save(out)
def main(options, positional_args):
assert len(positional_args) == 1
path, = positional_args
out = options.out
rescale = options.rescale
if rescale == "none":
global_rescale = False
patch_rescale = False
elif rescale == "global":
global_rescale = True
patch_rescale = False
elif rescale == "individual":
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith(".pkl"):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith(".yaml"):
print "Building dataset from yaml..."
obj = yaml_parse.load_path(path)
print "...done"
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj, "get_batch_topo"):
# obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows * cols)
else:
# obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(batch_size=rows * cols, theano_rng=theano_rng)
from theano import function
print "compiling sampling function"
f = function([], design_examples_var)
print "sampling"
design_examples = f()
print "loading dataset"
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = N.asarray([N.sqrt(N.sum(N.square(examples[i, :]))) for i in xrange(examples.shape[0])])
print "norms of examples: "
print "\tmin: ", norms.min()
print "\tmean: ", norms.mean()
print "\tmax: ", norms.max()
print "range of elements of examples", (examples.min(), examples.max())
print "dtype: ", examples.dtype
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= N.abs(examples).max()
if len(examples.shape) != 4:
print "sorry, view_examples.py only supports image examples for now."
print "this dataset has " + str(len(examples.shape) - 2) + " topological dimensions"
quit(-1)
#
if examples.shape[3] == 1:
is_color = False
elif examples.shape[3] == 3:
is_color = True
else:
print "got unknown image format with " + str(examples.shape[3]) + " channels"
print "supported formats are 1 channel greyscale or three channel RGB"
quit(-1)
#
print examples.shape[1:3]
pv = patch_viewer.PatchViewer((rows, cols), examples.shape[1:3], is_color=is_color)
for i in xrange(rows * cols):
pv.add_patch(examples[i, :, :, :], activation=0.0, rescale=patch_rescale)
#
if out is None:
pv.show()
else:
pv.save(out)
def __init__(self,
path='train.csv',
task='classification',
expect_labels=True,
expect_headers=True,
delimiter=',',
start=None,
stop=None,
start_fraction=None,
end_fraction=None,
yaml_src=None,
one_hot=True,
num_classes=4,
which_set=None):
"""
.. todo:: ..
WRITEME
"""
self.path = path
self.task = task
self.expect_labels = expect_labels
self.expect_headers = expect_headers
self.delimiter = delimiter
if which_set is not None:
self.start = start
self.stop = stop
self.start_fraction = start_fraction
self.end_fraction = end_fraction
self.view_converter = None
if yaml_src is not None:
self.yaml_src = yaml_parse.load_path(yaml_src)
# self.yaml_src=yaml_parse.load_path("mlp.yaml")
# eventually; triple-quoted yaml...
self.one_hot = one_hot
self.num_classes = num_classes
if which_set is not None and which_set not in[
'train', 'test', 'valid']:
raise ValueError(
'Unrecognized which_set value "%s".' % (which_set,) +
'". Valid values are ["train","test","valid"].')
else:
self.which_set = which_set
if self.start is not None or self.stop is not None:
raise ValueError("Use start/stop or which_set,"
" just not together.")
if task not in ['classification', 'regression']:
raise ValueError('task must be either "classification" or '
'"regression"; got ' + str(task))
if start_fraction is not None:
if end_fraction is not None:
raise ValueError("Use start_fraction or end_fraction, "
" not both.")
if start_fraction <= 0:
raise ValueError("start_fraction should be > 0")
if start_fraction >= 1:
raise ValueError("start_fraction should be < 1")
if end_fraction is not None:
if end_fraction <= 0:
raise ValueError("end_fraction should be > 0")
if end_fraction >= 1:
raise ValueError("end_fraction should be < 1")
if start is not None:
if start_fraction is not None or end_fraction is not None:
raise ValueError("Use start, start_fraction, or end_fraction,"
" just not together.")
if stop is not None:
if start_fraction is not None or end_fraction is not None:
raise ValueError("Use stop, start_fraction, or end_fraction,"
" just not together.")
# and go
self.path = preprocess(self.path)
X, y = self._load_data()
# y=y.astype(int)
# y=map(int, np.rint(y).astype(int))
if self.task == 'regression':
super(CSVDatasetPlus, self).__init__(X=X, y=y)
else:
# , y_labels=4 # y_labels=np.max(y)+1
super(CSVDatasetPlus, self).__init__(
X=X, y=y.astype(int), y_labels=self.num_classes)
import numpy as np import theano.tensor as T from pylearn2.models.s3c import SufficientStatistics from pylearn2.models.s3c import S3C from theano import function import time model_path, e_step_path, data_path, batch_size_str, prefix = sys.argv[1:] print 'loading model' model = serial.load(model_path) model.make_pseudoparams() assert isinstance(model, S3C) print 'building e step' e_step = yaml_parse.load_path(e_step_path) print 'installing e step' model.e_step = e_step e_step.register_model(model) print 'loading data' data = np.load(data_path) m, n = data.shape print 'batch_size: ', batch_size_str batch_size = int(batch_size_str) assert m % batch_size == 0 print 'building energy functional expression' V = T.matrix()
def main(options, positional_args):
"""
.. todo::
WRITEME
"""
assert len(positional_args) == 1
path ,= positional_args
out = options.out
rescale = options.rescale
if rescale == 'none':
global_rescale = False
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj =yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj,'get_batch_topo'):
# obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows*cols)
else:
# obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(batch_size =
rows * cols, theano_rng = theano_rng)
from theano import function
print 'compiling sampling function'
f = function([],design_examples_var)
print 'sampling'
design_examples = f()
print 'loading dataset'
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = N.asarray( [
N.sqrt(N.sum(N.square(examples[i,:])))
for i in xrange(examples.shape[0])
])
print 'norms of examples: '
print '\tmin: ',norms.min()
print '\tmean: ',norms.mean()
print '\tmax: ',norms.max()
print 'range of elements of examples', \
(examples.min(),examples.max())
print 'dtype: ', examples.dtype
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= N.abs(examples).max()
if len(examples.shape) != 4:
print 'sorry, view_examples.py only supports image examples' + \
'for now.'
print 'this dataset has ' + \
str(len(examples.shape)-2)+' topological dimensions'
quit(-1)
if examples.shape[3] == 1:
is_color = False
elif examples.shape[3] == 3:
is_color = True
else:
print 'got unknown image format with ' + str(examples.shape[3]) + \
' channels'
print 'supported formats are 1 channel greyscale or three channel RGB'
quit(-1)
print examples.shape[1:3]
pv = patch_viewer.PatchViewer((rows, cols), examples.shape[1:3],
is_color = is_color)
for i in xrange(rows*cols):
pv.add_patch(examples[i,:,:,:], activation=0.0, rescale=patch_rescale)
if out is None:
pv.show()
else:
pv.save(out)
#!/bin/env python
import numpy as N
import sys
from pylearn2.gui import patch_viewer
from pylearn2.config import yaml_parse
assert len(sys.argv) == 2
path = sys.argv[1]
if path.endswith('.pkl'):
from pylearn2.utils import serial
dataset = serial.load(path)
elif path.endswith('.yaml'):
dataset =yaml_parse.load_path(path)
else:
dataset = yaml_parse.load(path)
rows = 20
cols = 20
examples = dataset.get_batch_topo(rows*cols)
norms = N.asarray( [
N.sqrt(N.sum(N.square(examples[i,:])))
for i in xrange(examples.shape[0])
])
print 'norms of exmaples: '
print '\tmin: ',norms.min()
print '\tmean: ',norms.mean()
print '\tmax: ',norms.max()
from pylearn2.config import yaml_parse import sys dataset = yaml_parse.load_path(sys.argv[1]) var = dataset.X.var(axis=0) print (var.min(), var.mean(), var.max()) mn = dataset.X.mean(axis=0) print (mn.min(), mn.mean(), mn.max())
print((t6-t1, t2-t1, t3-t2, t4-t3, t5-t4, t6-t5))
if self.chunk_size is not None:
assert save_path.endswith('.npy')
save_path_pieces = save_path.split('.npy')
assert len(save_path_pieces) == 2
assert save_path_pieces[1] == ''
save_path = save_path_pieces[0] + '_' + chr(ord('A')+self.chunk_id)+'.npy'
np.save(save_path,output)
if nan > 0:
warnings.warn(str(nan)+' features were nan')
if __name__ == '__main__':
assert len(sys.argv) == 2
yaml_path = sys.argv[1]
assert yaml_path.endswith('.yaml')
val = yaml_path[0:-len('.yaml')]
os.environ['FEATURE_EXTRACTOR_YAML_PATH'] = val
os.putenv('FEATURE_EXTRACTOR_YAML_PATH',val)
val = val.split('/')[-1]
os.environ['FEATURE_EXTRACTOR_YAML_NAME'] = val
os.putenv('FEATURE_EXTRACTOR_YAML_NAME', val)
extractor = yaml_parse.load_path(yaml_path)
extractor()
def main(options, positional_args):
"""
.. todo::
WRITEME
"""
assert len(positional_args) == 1
path, = positional_args
out = options.out
rescale = options.rescale
if rescale == 'none':
global_rescale = False
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj = yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj, 'get_batch_topo'):
#obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows * cols)
else:
#obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(batch_size=rows *
cols,
theano_rng=theano_rng)
from theano import function
print 'compiling sampling function'
f = function([], design_examples_var)
print 'sampling'
design_examples = f()
print 'loading dataset'
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = N.asarray([
N.sqrt(N.sum(N.square(examples[i, :])))
for i in xrange(examples.shape[0])
])
print 'norms of examples: '
print '\tmin: ', norms.min()
print '\tmean: ', norms.mean()
print '\tmax: ', norms.max()
print 'range of elements of examples', (examples.min(), examples.max())
print 'dtype: ', examples.dtype
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= N.abs(examples).max()
if len(examples.shape) != 4:
print 'sorry, view_examples.py only supports image examples for now.'
print 'this dataset has ' + str(len(examples.shape) -
2) + ' topological dimensions'
quit(-1)
is_color = False
assert examples.shape[3] == 2
print examples.shape[1:3]
pv = patch_viewer.PatchViewer((rows, cols * 2),
examples.shape[1:3],
is_color=is_color)
for i in xrange(rows * cols):
# Load patches in backwards order for easier cross-eyed viewing
# (Ian can't do the magic eye thing where you focus your eyes past the screen, must
# focus eyes in front of screen)
pv.add_patch(examples[i, :, :, 1],
activation=0.0,
rescale=patch_rescale)
pv.add_patch(examples[i, :, :, 0],
activation=0.0,
rescale=patch_rescale)
if out is None:
pv.show()
else:
pv.save(out)
def main(options, positional_args):
assert len(positional_args) == 1
path ,= positional_args
out = options.out
rescale = options.rescale
if rescale == 'none':
global_rescale = False
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj =yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj,'get_batch_topo'):
#obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows*cols)
else:
#obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(batch_size = rows * cols, theano_rng = theano_rng)
from theano import function
print 'compiling sampling function'
f = function([],design_examples_var)
print 'sampling'
design_examples = f()
print 'loading dataset'
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = N.asarray( [
N.sqrt(N.sum(N.square(examples[i,:])))
for i in xrange(examples.shape[0])
])
print 'norms of examples: '
print '\tmin: ',norms.min()
print '\tmean: ',norms.mean()
print '\tmax: ',norms.max()
print 'range of elements of examples',(examples.min(),examples.max())
print 'dtype: ', examples.dtype
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= N.abs(examples).max()
if len(examples.shape) != 4:
print 'sorry, view_examples.py only supports image examples for now.'
print 'this dataset has '+str(len(examples.shape)-2)+' topological dimensions'
quit(-1)
#
if examples.shape[3] == 1:
is_color = False
elif examples.shape[3] == 3:
is_color = True
else:
print 'got unknown image format with '+str(examples.shape[3])+' channels'
print 'supported formats are 1 channel greyscale or three channel RGB'
quit(-1)
#
print examples.shape[1:3]
pv = patch_viewer.PatchViewer( (rows, cols), examples.shape[1:3], is_color = is_color)
for i in xrange(rows*cols):
pv.add_patch(examples[i,:,:,:], activation = 0.0, rescale = patch_rescale)
#
if out is None:
pv.show()
else:
pv.save(out)
import gc
import numpy as np
import sys
from pylearn2.config import yaml_parse
from pylearn2.utils import serial
_, config_path = sys.argv
model = yaml_parse.load_path(config_path)
f = model.dump_func()
model.strip_down()
stripped_model_path = config_path.replace('.yaml', '_stripped.pkl')
serial.save(stripped_model_path, model)
srcs = {
'train' : """!obj:pylearn2.datasets.norb_small.FoveatedNORB {
which_set: "train",
scale: 1,
one_hot: 1
}""",
'test' : """!obj:pylearn2.datasets.norb_small.FoveatedNORB {
which_set: "test",
scale: 1,
one_hot: 1
}"""
}
for which_set in srcs:
gc.collect()
#!/bin/env python
import numpy as N
import sys
from pylearn2.gui import patch_viewer
from pylearn2.config import yaml_parse
assert len(sys.argv) == 2
path = sys.argv[1]
if path.endswith('.pkl'):
from pylearn2.utils import serial
dataset = serial.load(path)
elif path.endswith('.yaml'):
dataset = yaml_parse.load_path(path)
else:
dataset = yaml_parse.load(path)
rows = 20
cols = 20
examples = dataset.get_batch_topo(rows * cols)
norms = N.asarray([
N.sqrt(N.sum(N.square(examples[i, :]))) for i in xrange(examples.shape[0])
])
print 'norms of exmaples: '
print '\tmin: ', norms.min()
print '\tmean: ', norms.mean()
print '\tmax: ', norms.max()
print 'range of elements of examples', (examples.min(), examples.max())
import gc
import numpy as np
import sys
from pylearn2.config import yaml_parse
from pylearn2.utils import serial
_, config_path = sys.argv
model = yaml_parse.load_path(config_path)
f = model.dump_func()
model.strip_down()
stripped_model_path = config_path.replace('.yaml', '_stripped.pkl')
serial.save(stripped_model_path, model)
srcs = {
'train':
"""!obj:pylearn2.datasets.norb_small.FoveatedNORB {
which_set: "train",
scale: 1,
one_hot: 1
}""",
'test':
"""!obj:pylearn2.datasets.norb_small.FoveatedNORB {
which_set: "test",
scale: 1,
one_hot: 1
}"""
}
import numpy as np import theano.tensor as T from pylearn2.models.s3c import SufficientStatistics from pylearn2.models.s3c import S3C from theano import function import time model_path, e_step_path, data_path, batch_size_str, prefix = sys.argv[1:] print 'loading model' model = serial.load(model_path) model.make_pseudoparams() assert isinstance(model, S3C) print 'building e step' e_step = yaml_parse.load_path(e_step_path) print 'installing e step' model.e_step = e_step e_step.register_model(model) print 'loading data' data = np.load(data_path) m,n = data.shape print 'batch_size: ',batch_size_str batch_size = int(batch_size_str) assert m % batch_size == 0 print 'building energy functional expression' V = T.matrix()
def show_examples(path, rows, cols, rescale='global', out=None):
"""
.. todo::
WRITEME
Parameters
----------
path : string
The pickle or YAML file to show examples of
rows : int
WRITEME
cols : int
WRITEME
rescale : {'rescale', 'global', 'individual'}
Default is 'rescale', WRITEME
out : string, optional
WRITEME
"""
if rescale == 'none':
global_rescale = False
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj = yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
if hasattr(obj, 'get_batch_topo'):
# obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows * cols)
else:
# obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(batch_size=rows *
cols,
theano_rng=theano_rng)
from theano import function
print 'compiling sampling function'
f = function([], design_examples_var)
print 'sampling'
design_examples = f()
print 'loading dataset'
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = np.asarray([
np.sqrt(np.sum(np.square(examples[i, :])))
for i in xrange(examples.shape[0])
])
print 'norms of examples: '
print '\tmin: ', norms.min()
print '\tmean: ', norms.mean()
print '\tmax: ', norms.max()
print 'range of elements of examples', (examples.min(), examples.max())
print 'dtype: ', examples.dtype
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= np.abs(examples).max()
if len(examples.shape) != 4:
print 'sorry, view_examples.py only supports image examples for now.'
print 'this dataset has ' + str(len(examples.shape) - 2),
print 'topological dimensions'
quit(-1)
if examples.shape[3] == 1:
is_color = False
elif examples.shape[3] == 3:
is_color = True
else:
print 'got unknown image format with ' + str(examples.shape[3]),
print 'channels'
print 'supported formats are 1 channel greyscale or three channel RGB'
quit(-1)
print examples.shape[1:3]
pv = patch_viewer.PatchViewer((rows, cols),
examples.shape[1:3],
is_color=is_color)
for i in xrange(rows * cols):
pv.add_patch(examples[i, :, :, :],
activation=0.0,
rescale=patch_rescale)
if out is None:
pv.show()
else:
pv.save(out)
def show_examples(path, rows, cols, rescale='global', out=None):
"""
.. todo::
WRITEME
Parameters
----------
path : string
The pickle or YAML file to show examples of
rows : int
WRITEME
cols : int
WRITEME
rescale : {'rescale', 'global', 'individual'}
Default is 'rescale', WRITEME
out : string, optional
WRITEME
"""
if rescale == 'none':
global_rescale = False
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print('Building dataset from yaml...')
obj = yaml_parse.load_path(path)
print('...done')
else:
obj = yaml_parse.load(path)
if hasattr(obj, 'get_batch_topo'):
# obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows*cols)
else:
# obj is a Model
model = obj
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
theano_rng = RandomStreams(42)
design_examples_var = model.random_design_matrix(
batch_size=rows * cols, theano_rng=theano_rng
)
from theano import function
print('compiling sampling function')
f = function([], design_examples_var)
print('sampling')
design_examples = f()
print('loading dataset')
dataset = yaml_parse.load(model.dataset_yaml_src)
examples = dataset.get_topological_view(design_examples)
norms = np.asarray([np.sqrt(np.sum(np.square(examples[i, :])))
for i in xrange(examples.shape[0])])
print('norms of examples: ')
print('\tmin: ', norms.min())
print('\tmean: ', norms.mean())
print('\tmax: ', norms.max())
print('range of elements of examples', (examples.min(), examples.max()))
print('dtype: ', examples.dtype)
examples = dataset.adjust_for_viewer(examples)
if global_rescale:
examples /= np.abs(examples).max()
if len(examples.shape) != 4:
print('sorry, view_examples.py only supports image examples for now.')
print('this dataset has ' + str(len(examples.shape) - 2), end='')
print('topological dimensions')
quit(-1)
if examples.shape[3] == 1:
is_color = False
elif examples.shape[3] == 3:
is_color = True
else:
print('got unknown image format with', str(examples.shape[3]), end='')
print('channels')
print('supported formats are 1 channel greyscale or three channel RGB')
quit(-1)
print(examples.shape[1:3])
pv = patch_viewer.PatchViewer((rows, cols), examples.shape[1:3],
is_color=is_color)
for i in xrange(rows*cols):
pv.add_patch(examples[i, :, :, :], activation=0.0,
rescale=patch_rescale)
if out is None:
pv.show()
else:
pv.save(out)
import sys
ignore, N = sys.argv
N = int(N)
dataset_desc_path = 'cifar100_patches.yaml'
from pylearn2.config import yaml_parse
dataset = yaml_parse.load_path(dataset_desc_path)
from pylearn2.models.s3c import S3C, E_Step, Grad_M_Step, E_Step_Scan
NUM_EXAMPLES = 100
OVERKILL = 50
X = dataset.get_batch_design(NUM_EXAMPLES)
D = X.shape[1]
import numpy as np
rng = np.random.RandomState([7, 12, 43])
init_bias_hid = rng.uniform(-5., 0., (N, ))
init_mu = rng.uniform(-5., 5., (N, ))
init_alpha = rng.uniform(.1, 15., (N, ))
init_B = rng.uniform(3.5, 15., (D, ))
from pylearn2.utils.call_check import checked_call
def package_call(to_call, **kwargs):
return checked_call(to_call, kwargs)
from pylearn2.config import yaml_parse
if __name__ == "__main__":
clf = yaml_parse.load_path("conv.yaml")
clf.main_loop()
import sys
ignore, N = sys.argv
N = int(N)
dataset_desc_path = 'cifar100_patches.yaml'
from pylearn2.config import yaml_parse
dataset = yaml_parse.load_path(dataset_desc_path)
from pylearn2.models.s3c import S3C, E_Step, Grad_M_Step, E_Step_Scan
NUM_EXAMPLES = 100
OVERKILL = 50
X = dataset.get_batch_design(NUM_EXAMPLES)
D = X.shape[1]
import numpy as np
rng = np.random.RandomState([7,12,43])
init_bias_hid = rng.uniform(-5.,0.,(N,))
init_mu = rng.uniform(-5.,5.,(N,))
init_alpha = rng.uniform(.1,15.,(N,))
init_B = rng.uniform(3.5,15.,(D,))
from pylearn2.utils.call_check import checked_call
def package_call(to_call, ** kwargs):
return checked_call(to_call, kwargs)
patch_rescale = False
elif rescale == 'global':
global_rescale = True
patch_rescale = False
elif rescale == 'individual':
global_rescale = False
patch_rescale = True
else:
assert False
if path.endswith('.pkl'):
from pylearn2.utils import serial
obj = serial.load(path)
elif path.endswith('.yaml'):
print 'Building dataset from yaml...'
obj =yaml_parse.load_path(path)
print '...done'
else:
obj = yaml_parse.load(path)
rows = options.rows
cols = options.cols
if hasattr(obj,'get_batch_topo'):
#obj is a Dataset
dataset = obj
examples = dataset.get_batch_topo(rows*cols)
else:
#obj is a Model
model = obj
