def test_get_description_with_undescribable_in_describable_raises():
class Foo21(Describable):
pass
f = Foo21()
f.mymod = pytest
with pytest.raises(TypeError) as excinfo:
get_description(f)
assert excinfo.value.args[0].find("module") > -1
assert excinfo.value.args[0].find("[Foo21.mymod]") > -1
def test_get_description_with_undescribable_in_describable_raises():
class Foo21(Describable):
pass
f = Foo21()
f.mymod = pytest
with pytest.raises(TypeError) as excinfo:
get_description(f)
assert excinfo.value.args[0].find("module") > -1
assert excinfo.value.args[0].find("[Foo21.mymod]") > -1
def test_describe_numpy_handler():
nh = NumpyHandler(np.float32)
d = get_description(nh)
assert d == {'@type': 'NumpyHandler', 'dtype': 'float32'}
nh2 = create_from_description(d)
assert isinstance(nh2, NumpyHandler)
assert nh2.dtype == np.float32
def test_describe_pycuda_handler():
from brainstorm.handlers.pycuda_handler import PyCudaHandler
pch = PyCudaHandler()
d = get_description(pch)
assert d == {'@type': 'PyCudaHandler'}
pch2 = create_from_description(d)
assert isinstance(pch2, PyCudaHandler)
def test_describe_trainer():
tr = bs.Trainer(bs.training.SgdStepper(learning_rate=0.7), verbose=False)
tr.add_hook(bs.hooks.StopAfterEpoch(23))
tr.add_hook(bs.hooks.StopOnNan())
d = get_description(tr)
assert d == {
'@type': 'Trainer',
'verbose': False,
'hooks': {
'StopAfterEpoch': {
'@type': 'StopAfterEpoch',
'max_epochs': 23,
'priority': 1},
'StopOnNan': {
'@type': 'StopOnNan',
'check_parameters': True,
'check_training_loss': True,
'logs_to_check': [],
'priority': 2}},
'stepper': {
'@type': 'SgdStepper',
'learning_rate': 0.7},
'train_scorers': []
}
def test_describe_pycuda_handler():
from brainstorm.handlers.pycuda_handler import PyCudaHandler
pch = PyCudaHandler()
d = get_description(pch)
assert d == {'@type': 'PyCudaHandler'}
pch2 = create_from_description(d)
assert isinstance(pch2, PyCudaHandler)
def test_describe_numpy_handler():
nh = NumpyHandler(np.float32)
d = get_description(nh)
assert d == {'@type': 'NumpyHandler', 'dtype': 'float32'}
nh2 = create_from_description(d)
assert isinstance(nh2, NumpyHandler)
assert nh2.dtype == np.float32
def test_describe_trainer():
tr = bs.Trainer(bs.training.SgdStepper(learning_rate=0.7), verbose=False)
tr.add_hook(bs.hooks.StopAfterEpoch(23))
tr.add_hook(bs.hooks.StopOnNan())
d = get_description(tr)
assert d == {
'@type': 'Trainer',
'verbose': False,
'hooks': {
'StopAfterEpoch': {
'@type': 'StopAfterEpoch',
'max_epochs': 23,
'priority': 1
},
'StopOnNan': {
'@type': 'StopOnNan',
'check_parameters': True,
'check_training_loss': True,
'logs_to_check': [],
'priority': 2
}
},
'stepper': {
'@type': 'SgdStepper',
'learning_rate': 0.7
},
'train_scorers': []
}
def test_get_description_basic_types():
assert get_description(13) == 13
assert get_description(0xff) == 0xff
assert get_description(23.5) == 23.5
assert get_description(1.3e-7) == 1.3e-7
assert get_description(True) is True
assert get_description(False) is False
assert get_description('foo') == 'foo'
assert get_description(None) is None
def test_get_description_basic_types():
assert get_description(13) == 13
assert get_description(0xff) == 0xff
assert get_description(23.5) == 23.5
assert get_description(1.3e-7) == 1.3e-7
assert get_description(True) is True
assert get_description(False) is False
assert get_description('foo') == 'foo'
assert get_description(None) is None
def test_get_network_from_description():
net = bs.Network.from_architecture(arch)
net.initialize(1)
d = get_description(net)
net2 = create_from_description(d)
assert isinstance(net2, bs.Network)
assert net2.layers.keys() == net.layers.keys()
assert isinstance(net2.handler, NumpyHandler)
assert net2.handler.dtype == np.float32
assert np.all(net2.buffer.parameters == net.buffer.parameters)
def test_get_network_from_description():
net = bs.Network.from_architecture(arch)
net.initialize(1)
d = get_description(net)
net2 = create_from_description(d)
assert isinstance(net2, bs.Network)
assert net2.layers.keys() == net.layers.keys()
assert isinstance(net2.handler, NumpyHandler)
assert net2.handler.dtype == np.float32
assert np.all(net2.buffer.parameters == net.buffer.parameters)
def test_describe_network():
net = bs.Network.from_architecture(arch)
net.initialize(1)
assert get_description(net) == {
'@type': 'Network',
'architecture': json.loads(json.dumps(arch)),
'handler': {'@type': 'NumpyHandler', 'dtype': 'float32'},
'initializers': {'default': 1},
'weight_modifiers': {},
'gradient_modifiers': {},
'output_name': None
}
def test_recreate_trainer_from_description():
tr = bs.Trainer(bs.training.SgdStepper(learning_rate=0.7), verbose=False)
tr.add_hook(bs.hooks.StopAfterEpoch(23))
tr.add_hook(bs.hooks.StopOnNan())
d = get_description(tr)
tr2 = create_from_description(d)
assert isinstance(tr2, bs.Trainer)
assert tr2.verbose is False
assert list(tr2.hooks.keys()) == ['StopAfterEpoch', 'StopOnNan']
assert tr2.hooks['StopAfterEpoch'].max_epochs == 23
assert isinstance(tr2.stepper, bs.training.SgdStepper)
assert tr2.stepper.learning_rate == 0.7
def test_recreate_trainer_from_description():
tr = bs.Trainer(bs.training.SgdStepper(learning_rate=0.7), verbose=False)
tr.add_hook(bs.hooks.StopAfterEpoch(23))
tr.add_hook(bs.hooks.StopOnNan())
d = get_description(tr)
tr2 = create_from_description(d)
assert isinstance(tr2, bs.Trainer)
assert tr2.verbose is False
assert list(tr2.hooks.keys()) == ['StopAfterEpoch', 'StopOnNan']
assert tr2.hooks['StopAfterEpoch'].max_epochs == 23
assert isinstance(tr2.stepper, bs.training.SgdStepper)
assert tr2.stepper.learning_rate == 0.7
def test_describe_network():
net = bs.Network.from_architecture(arch)
net.initialize(1)
assert get_description(net) == {
'@type': 'Network',
'architecture': json.loads(json.dumps(arch)),
'handler': {
'@type': 'NumpyHandler',
'dtype': 'float32'
},
'initializers': {
'default': 1
},
'weight_modifiers': {},
'gradient_modifiers': {},
'output_name': None
}
def save_as_hdf5(self, filename, comment=''):
"""
Save this network as an HDF5 file.
The file will contain a description of this network and the parameters.
Args:
filename (str):
Name of the file this network should be saved to.
All directories have to exist already.
comment (Optional[str]):
An optional comment that will be saved inside the file.
"""
with h5py.File(filename, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Network file v1.0')
if comment:
f.attrs.create('comment', comment.encode())
description = get_description(self)
f['description'] = json.dumps(description).encode()
f.create_dataset('parameters',
compression='gzip',
data=self.get('parameters'))
def save_as_hdf5(self, filename, comment=''):
"""
Save this network as an HDF5 file.
The file will contain a description of this network and the parameters.
Args:
filename (str):
Name of the file this network should be saved to.
All directories have to exist already.
comment (Optional[str]):
An optional comment that will be saved inside the file.
"""
with h5py.File(filename, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Network file v1.0')
if comment:
f.attrs.create('comment', comment.encode())
description = get_description(self)
f['description'] = json.dumps(description).encode()
f.create_dataset(
'parameters', compression='gzip',
data=self.get('parameters'))
def test_get_description_dict():
assert get_description({}) == {}
assert get_description({'a': 1, 'b': 'bar'}) == {'a': 1, 'b': 'bar'}
def test_get_description_ndarray():
assert get_description(np.array(1.7)) == 1.7
assert get_description(np.array([[1], [2], [3]])) == [[1], [2], [3]]
assert get_description(np.array([1, 2, 3])) == [1, 2, 3]
def test_get_description_with_undescribable_in_dict_raises():
with pytest.raises(TypeError) as excinfo:
get_description({'foo': pytest})
assert excinfo.value.args[0].find("module") > -1
assert excinfo.value.args[0].find("[foo]") > -1
def test_get_description_with_undescribable_in_list_raises():
with pytest.raises(TypeError) as excinfo:
get_description([1, 2, pytest, 4])
assert excinfo.value.args[0].find("module") > -1
assert excinfo.value.args[0].find("[2]") > -1
def test_get_description_for_undescribable_raises():
with pytest.raises(TypeError) as excinfo:
get_description(pytest)
assert excinfo.value.args[0].find("module") > -1
def test_get_description_describable():
class Foo20(Describable):
pass
f = Foo20()
assert get_description(f) == {'@type': 'Foo20'}
def test_get_description_with_undescribable_in_list_raises():
with pytest.raises(TypeError) as excinfo:
get_description([1, 2, pytest, 4])
assert excinfo.value.args[0].find("module") > -1
assert excinfo.value.args[0].find("[2]") > -1
def test_get_description_dict():
assert get_description({}) == {}
assert get_description({'a': 1, 'b': 'bar'}) == {'a': 1, 'b': 'bar'}
def test_get_description_list():
assert get_description([]) == []
assert get_description([1, 2, 3]) == [1, 2, 3]
assert get_description([1, 'b']) == [1, 'b']
def test_get_description_describable():
class Foo20(Describable):
pass
f = Foo20()
assert get_description(f) == {'@type': 'Foo20'}
def test_get_description_for_undescribable_raises():
with pytest.raises(TypeError) as excinfo:
get_description(pytest)
assert excinfo.value.args[0].find("module") > -1
def test_get_description_list():
assert get_description([]) == []
assert get_description([1, 2, 3]) == [1, 2, 3]
assert get_description([1, 'b']) == [1, 'b']
def test_get_description_ndarray():
assert get_description(np.array(1.7)) == 1.7
assert get_description(np.array([[1], [2], [3]])) == [[1], [2], [3]]
assert get_description(np.array([1, 2, 3])) == [1, 2, 3]
def test_get_description_with_undescribable_in_dict_raises():
with pytest.raises(TypeError) as excinfo:
get_description({'foo': pytest})
assert excinfo.value.args[0].find("module") > -1
assert excinfo.value.args[0].find("[foo]") > -1
def test_seedable_description_does_not_include_rnd():
class Foo1(Seedable, Describable):
pass
assert get_description(Foo1()) == {'@type': 'Foo1'}
def initialize(self, default_or_init_dict=None, seed=None, **kwargs):
"""Initialize the weights of the network.
Initialization can be specified in three equivalent ways:
1. just a default initializer:
>>> net.initialize(Gaussian())
Note that this is equivalent to:
>>> net.initialize(default=Gaussian())
2. by passing a dictionary:
>>> net.initialize({'RegularLayer': Uniform(),
... 'LstmLayer': Gaussian()})
3. by using keyword arguments:
>>> net.initialize(RegularLayer=Uniform(),
... LstmLayer=Uniform())
All following explanations will be with regards to the dictionary style
of initialization, because it is the most general one.
Note:
It is not recommended to combine 2. and 3. but if they are,
then keyword arguments take precedence.
Each initialization consists of a layer-pattern and that maps to an
initializer or a weight-pattern dictionary.
Layer patterns can take the following forms:
1. ``{'layer_name': INIT_OR_SUBDICT}``
Matches all the weights of the layer named layer_name
2. ``{'layer_*': INIT_OR_SUBDICT}``
Matches all layers with a name that starts with ``layer_``
The wild-card ``*`` can appear at arbitrary positions and even
multiple times in one path.
There are two special layer patterns:
3. ``{'default': INIT}``
Matches all weights that are not matched by any other
path-pattern
4. ``{'fallback': INIT}``
Set a fallback initializer for every weight. It will only be
evaluated for the weights for which the regular initializer
failed with an InitializationError.
`This is useful for initializers that require a certain shape
of weights and will not work otherwise. The fallback will then
be used for all cases when that initializer failed.`
The weight-pattern sub-dictionary follows the same form as the layer-
pattern:
1) ``{'layer_pattern': {'a': INIT_A, 'b': INIT_B}}``
2) ``{'layer_pattern': {'a*': INIT}``
3) ``{'layer_pattern': {'default': INIT}``
4) ``{'layer_pattern': {'fallback': INIT}``
An initializer can either be a scalar, something that converts to a
numpy array of the correct shape or an :class:`Initializer` object.
So for example:
>>> net.initialize(default=0,
... RnnLayer={'b': [1, 2, 3, 4, 5]},
... ForwardLayer=bs.Gaussian())
Note:
Each view must match exactly one initialization and up to one
fallback to be unambiguous. Otherwise the initialization will fail.
You can specify a seed to make the initialization reproducible:
>>> net.initialize({'default': bs.Gaussian()}, seed=1234)
"""
init_refs = _update_references_with_dict(default_or_init_dict, kwargs)
self.initializers = get_description(init_refs)
all_parameters = {
k: v.parameters
for k, v in self.buffer.items()
if isinstance(v, BufferView) and 'parameters' in v
}
_replace_lists_with_array_initializers(init_refs)
initializers, fallback = resolve_references(all_parameters, init_refs)
init_rnd = self.rnd.create_random_state(seed)
for layer_name, views in sorted(all_parameters.items()):
if views is None:
continue
for view_name, view in sorted(views.items()):
init = initializers[layer_name][view_name]
fb = fallback[layer_name][view_name]
if len(init) > 1:
raise NetworkValidationError(
"Multiple initializers for {}.{}: {}".format(
layer_name, view_name, init))
if len(init) == 0:
raise NetworkValidationError(
"No initializer for {}.{}".format(
layer_name, view_name))
if len(fb) > 1:
raise NetworkValidationError(
"Multiple fallbacks for {}.{}: {}".format(
layer_name, view_name, fb))
fb = fb.pop() if len(fb) else None
self.handler.set_from_numpy(
view,
evaluate_initializer(init.pop(),
view.shape,
fb,
seed=init_rnd.generate_seed()))
def test_seedable_description_does_not_include_rnd():
class Foo1(Seedable, Describable):
pass
assert get_description(Foo1()) == {'@type': 'Foo1'}
def initialize(self, default_or_init_dict=None, seed=None, **kwargs):
"""Initialize the weights of the network.
Initialization can be specified in three equivalent ways:
1. just a default initializer:
>>> net.initialize(Gaussian())
Note that this is equivalent to:
>>> net.initialize(default=Gaussian())
2. by passing a dictionary:
>>> net.initialize({'RegularLayer': Uniform(),
... 'LstmLayer': Gaussian()})
3. by using keyword arguments:
>>> net.initialize(RegularLayer=Uniform(),
... LstmLayer=Uniform())
All following explanations will be with regards to the dictionary style
of initialization, because it is the most general one.
Note:
It is not recommended to combine 2. and 3. but if they are,
then keyword arguments take precedence.
Each initialization consists of a layer-pattern and that maps to an
initializer or a weight-pattern dictionary.
Layer patterns can take the following forms:
1. ``{'layer_name': INIT_OR_SUBDICT}``
Matches all the weights of the layer named layer_name
2. ``{'layer_*': INIT_OR_SUBDICT}``
Matches all layers with a name that starts with ``layer_``
The wild-card ``*`` can appear at arbitrary positions and even
multiple times in one path.
There are two special layer patterns:
3. ``{'default': INIT}``
Matches all weights that are not matched by any other
path-pattern
4. ``{'fallback': INIT}``
Set a fallback initializer for every weight. It will only be
evaluated for the weights for which the regular initializer
failed with an InitializationError.
`This is useful for initializers that require a certain shape
of weights and will not work otherwise. The fallback will then
be used for all cases when that initializer failed.`
The weight-pattern sub-dictionary follows the same form as the layer-
pattern:
1) ``{'layer_pattern': {'a': INIT_A, 'b': INIT_B}}``
2) ``{'layer_pattern': {'a*': INIT}``
3) ``{'layer_pattern': {'default': INIT}``
4) ``{'layer_pattern': {'fallback': INIT}``
An initializer can either be a scalar, something that converts to a
numpy array of the correct shape or an :class:`Initializer` object.
So for example:
>>> net.initialize(default=0,
... RnnLayer={'b': [1, 2, 3, 4, 5]},
... ForwardLayer=bs.Gaussian())
Note:
Each view must match exactly one initialization and up to one
fallback to be unambiguous. Otherwise the initialization will fail.
You can specify a seed to make the initialization reproducible:
>>> net.initialize({'default': bs.Gaussian()}, seed=1234)
"""
init_refs = _update_references_with_dict(default_or_init_dict, kwargs)
self.initializers = get_description(init_refs)
all_parameters = {k: v.parameters
for k, v in self.buffer.items()
if isinstance(v, BufferView) and 'parameters' in v}
_replace_lists_with_array_initializers(init_refs)
initializers, fallback = resolve_references(all_parameters, init_refs)
init_rnd = self.rnd.create_random_state(seed)
for layer_name, views in sorted(all_parameters.items()):
if views is None:
continue
for view_name, view in sorted(views.items()):
init = initializers[layer_name][view_name]
fb = fallback[layer_name][view_name]
if len(init) > 1:
raise NetworkValidationError(
"Multiple initializers for {}.{}: {}".format(
layer_name, view_name, init))
if len(init) == 0:
raise NetworkValidationError("No initializer for {}.{}".
format(layer_name, view_name))
if len(fb) > 1:
raise NetworkValidationError(
"Multiple fallbacks for {}.{}: {}".format(
layer_name, view_name, fb))
fb = fb.pop() if len(fb) else None
self.handler.set_from_numpy(
view,
evaluate_initializer(init.pop(), view.shape, fb,
seed=init_rnd.generate_seed()))
