示例#1
0
def setup():
    """
    Create pickle file with a simple model.
    """
    # tearDown is guaranteed to run pop_load_data.
    control.push_load_data(False)
    with open('dbm.pkl', 'wb') as f:
        dataset = MNIST(which_set='train', start=0, stop=100, binarize=True)
        vis_layer = BinaryVector(nvis=784, bias_from_marginals=dataset)
        hid_layer1 = BinaryVectorMaxPool(layer_name='h1',
                                         pool_size=1,
                                         irange=.05,
                                         init_bias=-2.,
                                         detector_layer_dim=50)
        hid_layer2 = BinaryVectorMaxPool(layer_name='h2',
                                         pool_size=1,
                                         irange=.05,
                                         init_bias=-2.,
                                         detector_layer_dim=10)
        model = DBM(batch_size=20,
                    niter=2,
                    visible_layer=vis_layer,
                    hidden_layers=[hid_layer1, hid_layer2])
        model.dataset_yaml_src = """
!obj:pylearn2.datasets.binarizer.Binarizer {
    raw: !obj:pylearn2.datasets.mnist.MNIST {
        which_set: "train",
        start: 0,
        stop: 100
    }
}
"""
        model.layer_to_chains = model.make_layer_to_state(1)
        cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
示例#2
0
def setup():
    """
    Create pickle file with a simple model.
    """
    # tearDown is guaranteed to run pop_load_data.
    control.push_load_data(False)
    with open('dbm.pkl', 'wb') as f:
        dataset = MNIST(which_set='train', start=0, stop=100, binarize=True)
        vis_layer = BinaryVector(nvis=784, bias_from_marginals=dataset)
        hid_layer1 = BinaryVectorMaxPool(layer_name='h1', pool_size=1,
                                         irange=.05, init_bias=-2.,
                                         detector_layer_dim=50)
        hid_layer2 = BinaryVectorMaxPool(layer_name='h2', pool_size=1,
                                         irange=.05, init_bias=-2.,
                                         detector_layer_dim=10)
        model = DBM(batch_size=20, niter=2, visible_layer=vis_layer,
                    hidden_layers=[hid_layer1, hid_layer2])
        model.dataset_yaml_src = """
!obj:pylearn2.datasets.binarizer.Binarizer {
    raw: !obj:pylearn2.datasets.mnist.MNIST {
        which_set: "train",
        start: 0,
        stop: 100
    }
}
"""
        model.layer_to_chains = model.make_layer_to_state(1)
        cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
示例#3
0
def test_unpickle():
    fd, fname = tempfile.mkstemp()
    with os.fdopen(fd, 'wb') as f:
        d = {'a': 1, 'b': 2}
        cPickle.dump(d, f)
    loaded = load("{'a': !pkl: '%s'}" % fname)
    assert_(loaded['a'] == d)
    os.remove(fname)
示例#4
0
def test_preproc_pkl():
    fd, fname = tempfile.mkstemp()
    with os.fdopen(fd, 'wb') as f:
        d = ('a', 1)
        cPickle.dump(d, f)
    environ['TEST_VAR'] = fname
    loaded = load('a: !pkl: "${TEST_VAR}"')
    assert_(loaded['a'] == d)
    del environ['TEST_VAR']
示例#5
0
def test_unpickle_key():
    fd, fname = tempfile.mkstemp()
    with os.fdopen(fd, 'wb') as f:
        d = ('a', 1)
        cPickle.dump(d, f)
    loaded = load("{!pkl: '%s': 50}" % fname)
    assert_(first_key(loaded) == d)
    assert_(first_value(loaded) == 50)
    os.remove(fname)
def test_summarize_model():
    """
    Asks the summarize_model.py script to inspect a pickled model and
    check that it completes succesfully
    """
    skip_if_no_matplotlib()
    with open('model.pkl', 'wb') as f:
        cPickle.dump(MLP(layers=[Linear(dim=5, layer_name='h0', irange=0.1)],
                         nvis=10), f, protocol=cPickle.HIGHEST_PROTOCOL)
    summarize('model.pkl')
    os.remove('model.pkl')
def test_summarize_model():
    """
    Asks the summarize_model.py script to inspect a pickled model and
    check that it completes succesfully
    """
    skip_if_no_matplotlib()
    with open('model.pkl', 'wb') as f:
        cPickle.dump(MLP(layers=[Linear(dim=5, layer_name='h0', irange=0.1)],
                         nvis=10),
                     f,
                     protocol=cPickle.HIGHEST_PROTOCOL)
    summarize('model.pkl')
    os.remove('model.pkl')
示例#8
0
def test_show_weights():
    """
    Create a pickled model and show the weights
    """
    skip_if_no_matplotlib()
    with open('model.pkl', 'wb') as f:
        model = MLP(layers=[Linear(dim=1, layer_name='h0', irange=0.1)],
                    nvis=784)
        model.dataset_yaml_src = """
!obj:pylearn2.datasets.mnist.MNIST {
        which_set: 'train'
}
"""
        cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
    show_weights('model.pkl', rescale='individual',
                 border=True, out='garbage.png')
    os.remove('model.pkl')
    os.remove('garbage.png')
示例#9
0
def _save(filepath, obj):
    """
    .. todo::

        WRITEME
    """
    try:
        import joblib
        joblib_available = True
    except ImportError:
        joblib_available = False
    if filepath.endswith('.npy'):
        np.save(filepath, obj)
        return
    # This is dumb
    # assert filepath.endswith('.pkl')
    save_dir = os.path.dirname(filepath)
    # Handle current working directory case.
    if save_dir == '':
        save_dir = '.'
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    if os.path.exists(save_dir) and not os.path.isdir(save_dir):
        raise IOError("save path %s exists, not a directory" % save_dir)
    elif not os.access(save_dir, os.W_OK):
        raise IOError("permission error creating %s" % filepath)
    try:
        if joblib_available and filepath.endswith('.joblib'):
            joblib.dump(obj, filepath)
        else:
            if filepath.endswith('.joblib'):
                warnings.warn('Warning: .joblib suffix specified but joblib '
                              'unavailable. Using ordinary pickle.')
            with open(filepath, 'wb') as filehandle:
                cPickle.dump(obj, filehandle, get_pickle_protocol())
    except Exception as e:
        logger.exception("cPickle has failed to write an object to "
                         "{0}".format(filepath))
        if str(e).find('maximum recursion depth exceeded') != -1:
            raise
        try:
            logger.info('retrying with pickle')
            with open(filepath, "wb") as f:
                pickle.dump(obj, f)
        except Exception as e2:
            if str(e) == '' and str(e2) == '':
                logger.exception('neither cPickle nor pickle could write to '
                                 '{0}'.format(filepath))
                logger.exception(
                    'moreover, neither of them raised an exception that '
                    'can be converted to a string'
                )
                logger.exception(
                    'now re-attempting to write with cPickle outside the '
                    'try/catch loop so you can see if it prints anything '
                    'when it dies'
                )
                with open(filepath, 'wb') as f:
                    cPickle.dump(obj, f, get_pickle_protocol())
                logger.info('Somehow or other, the file write worked once '
                            'we quit using the try/catch.')
            else:
                if str(e2) == 'env':
                    raise

                import pdb
                tb = pdb.traceback.format_exc()
                reraise_as(IOError(str(obj) +
                                   ' could not be written to ' +
                                   str(filepath) +
                           ' by cPickle due to ' + str(e) +
                                   ' nor by pickle due to ' + str(e2) +
                                   '. \nTraceback ' + tb))
        logger.warning('{0} was written by pickle instead of cPickle, due to '
                       '{1} (perhaps your object'
                       ' is really big?)'.format(filepath, e))
示例#10
0
def _save(filepath, obj):
    """
    .. todo::
        WRITEME
    """
    try:
        import joblib
        joblib_available = True
    except ImportError:
        joblib_available = False
    if filepath.endswith('.npy'):
        np.save(filepath, obj)
        return
    # This is dumb
    # assert filepath.endswith('.pkl')
    save_dir = os.path.dirname(filepath)
    # Handle current working directory case.
    if save_dir == '':
        save_dir = '.'
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    if os.path.exists(save_dir) and not os.path.isdir(save_dir):
        raise IOError("save path %s exists, not a directory" % save_dir)
    elif not os.access(save_dir, os.W_OK):
        raise IOError("permission error creating %s" % filepath)
    try:
        if joblib_available and filepath.endswith('.joblib'):
            joblib.dump(obj, filepath)
        else:
            if filepath.endswith('.joblib'):
                warnings.warn('Warning: .joblib suffix specified but joblib '
                              'unavailable. Using ordinary pickle.')
            with open(filepath, 'wb') as filehandle:
                cPickle.dump(obj, filehandle, get_pickle_protocol())
    except Exception as e:
        logger.exception("cPickle has failed to write an object to "
                         "{0}".format(filepath))
        if str(e).find('maximum recursion depth exceeded') != -1:
            raise
        try:
            logger.info('retrying with pickle')
            with open(filepath, "wb") as f:
                pickle.dump(obj, f)
        except Exception as e2:
            if str(e) == '' and str(e2) == '':
                logger.exception('neither cPickle nor pickle could write to '
                                 '{0}'.format(filepath))
                logger.exception(
                    'moreover, neither of them raised an exception that '
                    'can be converted to a string'
                )
                logger.exception(
                    'now re-attempting to write with cPickle outside the '
                    'try/catch loop so you can see if it prints anything '
                    'when it dies'
                )
                with open(filepath, 'wb') as f:
                    cPickle.dump(obj, f, get_pickle_protocol())
                logger.info('Somehow or other, the file write worked once '
                            'we quit using the try/catch.')
            else:
                if str(e2) == 'env':
                    raise

                import pdb
                tb = pdb.traceback.format_exc()
                reraise_as(IOError(str(obj) +
                                   ' could not be written to ' +
                                   str(filepath) +
                           ' by cPickle due to ' + str(e) +
                                   ' nor by pickle due to ' + str(e2) +
                                   '. \nTraceback ' + tb))
        logger.warning('{0} was written by pickle instead of cPickle, due to '
                       '{1} (perhaps your object'
                       ' is really big?)'.format(filepath, e))
示例#11
0
def gendata(enable, os, downsample, textid=None, seed=2313, verbose=False):
    """
    Generate the MNIST+ dataset.
    :param enable: dictionary of flags with keys ['texture', 'azimuth',
    'rotation', 'elevation'] to enable/disable a given factor of variation.
    :param textid: if enable['texture'], id number of the Brodatz texture to
    load. If textid is None, we load a random texture for each MNIST image.
    :param os: output size (width and height) of MNIST+ images.
    :param downsample: factor by which to downsample texture.
    :param seed: integer for seeding RNG.
    :param verbose: bool
    """
    rng = numpy.random.RandomState(seed)

    data = mnist.MNIST("train")
    test = mnist.MNIST("test")
    data.X = numpy.vstack((data.X, test.X))
    data.y = numpy.hstack((data.y, test.y))
    del test

    output = {}
    output["data"] = numpy.zeros((len(data.X), os * os))
    output["label"] = numpy.zeros(len(data.y))
    if enable["azimuth"]:
        output["azimuth"] = numpy.zeros(len(data.y))
    if enable["elevation"]:
        output["elevation"] = numpy.zeros(len(data.y))
    if enable["rotation"]:
        output["rotation"] = numpy.zeros(len(data.y))
    if enable["texture"]:
        output["texture_id"] = numpy.zeros(len(data.y))
        output["texture_pos"] = numpy.zeros((len(data.y), 2))

    for i in xrange(len(data.X)):

        # get MNIST image
        frgd_img = to_img(data.X[i], 28)
        frgd_img = frgd_img.convert("L")

        if enable["rotation"]:
            rot = rng.randint(0, 360)
            output["rotation"][i] = rot
            frgd_img = frgd_img.rotate(rot, Image.BILINEAR)

        frgd_img = frgd_img.resize((os, os), Image.BILINEAR)

        if enable["texture"]:

            if textid is None:
                # extract patch from texture database. Note that texture #14
                # does not exist.
                textid = 14
                while textid == 14:
                    textid = rng.randint(1, 113)

            patch_img, (px, py) = extract_patch(textid, os, downsample)
            patch_arr = to_array(patch_img)

            # store output details
            output["texture_id"][i] = textid
            output["texture_pos"][i] = (px, py)

            # generate binary mask for digit outline
            frgd_arr = to_array(frgd_img)
            mask_arr = frgd_arr > 0.1

            # copy contents of masked-MNIST image into background texture
            blend_arr = copy(patch_arr)
            blend_arr[mask_arr] = frgd_arr[mask_arr]

            # this now because the image to emboss
            frgd_img = to_img(blend_arr, os)

        azi = 45
        if enable["azimuth"]:
            azi = rng.randint(0, 360)
            output["azimuth"][i] = azi
        ele = 18.0
        if enable["elevation"]:
            ele = rng.randint(0, 60)
            output["elevation"][i] = ele

        mboss_img = emboss(frgd_img, azi=azi, ele=ele)
        mboss_arr = to_array(mboss_img)

        output["data"][i] = mboss_arr
        output["label"][i] = data.y[i]

        if verbose:
            pl.imshow(mboss_arr.reshape(os, os))
            pl.gray()
            pl.show()

    fname = "mnistplus"
    if enable["azimuth"]:
        fname += "_azi"
    if enable["rotation"]:
        fname += "_rot"
    if enable["texture"]:
        fname += "_tex"
    fp = open(fname + ".pkl", "w")
    pickle.dump(output, fp, protocol=pickle.HIGHEST_PROTOCOL)
    fp.close()
示例#12
0
def gendata(enable, os, downsample, textid=None, seed=2313, verbose=False):
    """
    Generate the MNIST+ dataset.
    :param enable: dictionary of flags with keys ['texture', 'azimuth',
    'rotation', 'elevation'] to enable/disable a given factor of variation.
    :param textid: if enable['texture'], id number of the Brodatz texture to
    load. If textid is None, we load a random texture for each MNIST image.
    :param os: output size (width and height) of MNIST+ images.
    :param downsample: factor by which to downsample texture.
    :param seed: integer for seeding RNG.
    :param verbose: bool
    """
    rng = numpy.random.RandomState(seed)

    data  = mnist.MNIST('train')
    test  = mnist.MNIST('test')
    data.X = numpy.vstack((data.X, test.X))
    data.y = numpy.hstack((data.y, test.y))
    del test

    output = {}
    output['data']  = numpy.zeros((len(data.X), os*os))
    output['label'] = numpy.zeros(len(data.y))
    if enable['azimuth']:
        output['azimuth'] = numpy.zeros(len(data.y))
    if enable['elevation']:
        output['elevation'] = numpy.zeros(len(data.y))
    if enable['rotation']:
        output['rotation'] = numpy.zeros(len(data.y))
    if enable['texture']:
        output['texture_id']  = numpy.zeros(len(data.y))
        output['texture_pos'] = numpy.zeros((len(data.y), 2))

    for i in xrange(len(data.X)):

        # get MNIST image
        frgd_img = to_img(data.X[i], 28)
        frgd_img = frgd_img.convert('L')

        if enable['rotation']:
            rot = rng.randint(0, 360)
            output['rotation'][i] = rot
            frgd_img = frgd_img.rotate(rot, Image.BILINEAR)

        frgd_img = frgd_img.resize((os, os), Image.BILINEAR)

        if enable['texture']:

            if textid is None:
                # extract patch from texture database. Note that texture #14
                # does not exist.
                textid = 14
                while textid == 14:
                    textid = rng.randint(1, 113)

            patch_img, (px, py) = extract_patch(textid, os, downsample)
            patch_arr = to_array(patch_img)

            # store output details
            output['texture_id'][i] = textid
            output['texture_pos'][i] = (px, py)

            # generate binary mask for digit outline
            frgd_arr = to_array(frgd_img)
            mask_arr = frgd_arr > 0.1

            # copy contents of masked-MNIST image into background texture
            blend_arr = copy(patch_arr)
            blend_arr[mask_arr] = frgd_arr[mask_arr]

            # this now because the image to emboss
            frgd_img = to_img(blend_arr, os)

        azi = 45
        if enable['azimuth']:
            azi = rng.randint(0, 360)
            output['azimuth'][i] = azi
        ele = 18.
        if enable['elevation']:
            ele = rng.randint(0, 60)
            output['elevation'][i] = ele

        mboss_img = emboss(frgd_img, azi=azi, ele=ele)
        mboss_arr = to_array(mboss_img)

        output['data'][i] = mboss_arr
        output['label'][i] = data.y[i]

        if verbose:
            pl.imshow(mboss_arr.reshape(os, os))
            pl.gray()
            pl.show()

    fname = 'mnistplus'
    if enable['azimuth']:
        fname += "_azi"
    if enable['rotation']:
        fname += "_rot"
    if enable['texture']:
        fname += "_tex"
    fp = open(fname+'.pkl','w')
    pickle.dump(output, fp, protocol=pickle.HIGHEST_PROTOCOL)
    fp.close()