def pretrainFromConfig(config):
"""
:type config: Config.Config
:rtype: Pretrain | None
"""
pretrainType = config.bool_or_other("pretrain", None)
if pretrainType == "default" or (isinstance(pretrainType, dict)
and pretrainType) or pretrainType is True:
network_init_args = LayerNetwork.init_args_from_config(config)
original_network_json = LayerNetwork.json_from_config(config)
opts = config.get_of_type("pretrain", dict, {})
if config.has("pretrain_copy_output_layer"):
opts.setdefault(
"copy_output_layer",
config.bool_or_other("pretrain_copy_output_layer",
"ifpossible"))
if config.has("pretrain_greedy"):
opts.setdefault("greedy", config.bool("pretrain_greedy", None))
if config.has("pretrain_repetitions"):
if config.is_typed("pretrain_repetitions"):
opts.setdefault("repetitions",
config.typed_value("pretrain_repetitions"))
else:
opts.setdefault("repetitions",
config.int_list("pretrain_repetitions", None))
if config.has("pretrain_construction_algo"):
opts.setdefault("construction_algo",
config.value("pretrain_construction_algo", None))
return Pretrain(original_network_json=original_network_json,
network_init_args=network_init_args,
**opts)
elif not pretrainType:
return None
else:
raise Exception("unknown pretrain type: %s" % pretrainType)
def test_NetworkDescription_to_json_config1():
config = Config()
config.update(config1_dict)
desc = LayerNetworkDescription.from_config(config)
desc_json_content = desc.to_json_content()
pprint(desc_json_content)
assert_in("hidden_0", desc_json_content)
assert_equal(desc_json_content["hidden_0"]["class"], "forward")
assert_in("hidden_1", desc_json_content)
assert_in("output", desc_json_content)
orig_network = LayerNetwork.from_description(desc)
assert_in("hidden_0", orig_network.hidden)
assert_in("hidden_1", orig_network.hidden)
assert_equal(len(orig_network.hidden), 2)
assert_is_instance(orig_network.hidden["hidden_0"], ForwardLayer)
assert_equal(orig_network.hidden["hidden_0"].layer_class, "hidden")
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
assert_in("hidden_0", orig_json_content)
assert_equal(orig_json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", orig_json_content)
assert_in("output", orig_json_content)
new_network = LayerNetwork.from_json(
desc_json_content,
config1_dict["num_inputs"],
{"classes": (config1_dict["num_outputs"], 1)})
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def pretrainFromConfig(config):
"""
:type config: Config.Config
:rtype: Pretrain | None
"""
pretrainType = config.value("pretrain", "")
if pretrainType == "default":
network_init_args = LayerNetwork.init_args_from_config(config)
original_network_json = LayerNetwork.json_from_config(config)
copy_output_layer = config.bool_or_other("pretrain_copy_output_layer", "ifpossible")
greedy = config.bool("pretrain_greedy", None)
if config.is_typed("pretrain_repetitions"):
repetitions = config.typed_value("pretrain_repetitions")
else:
repetitions = config.int_list("pretrain_repetitions", None)
construction_algo = config.value("pretrain_construction_algo", None)
return Pretrain(original_network_json=original_network_json,
network_init_args=network_init_args,
copy_output_layer=copy_output_layer,
greedy=greedy, repetitions=repetitions,
construction_algo=construction_algo)
elif pretrainType == "":
return None
else:
raise Exception("unknown pretrain type: %s" % pretrainType)
def test_NetworkDescription_to_json_config1():
config = Config()
config.update(config1_dict)
desc = LayerNetworkDescription.from_config(config)
desc_json_content = desc.to_json_content()
pprint(desc_json_content)
assert_in("hidden_0", desc_json_content)
assert_equal(desc_json_content["hidden_0"]["class"], "forward")
assert_in("hidden_1", desc_json_content)
assert_in("output", desc_json_content)
orig_network = LayerNetwork.from_description(desc)
assert_in("hidden_0", orig_network.hidden)
assert_in("hidden_1", orig_network.hidden)
assert_equal(len(orig_network.hidden), 2)
assert_is_instance(orig_network.hidden["hidden_0"], ForwardLayer)
assert_equal(orig_network.hidden["hidden_0"].layer_class, "hidden")
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
assert_in("hidden_0", orig_json_content)
assert_equal(orig_json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", orig_json_content)
assert_in("output", orig_json_content)
new_network = LayerNetwork.from_json(
desc_json_content, config1_dict["num_inputs"],
{"classes": (config1_dict["num_outputs"], 1)})
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def init_network_from_config(self, config):
self.pretrain = pretrainFromConfig(config)
self.max_seqs = config.int('max_seqs', -1)
self.compression = config.bool('compression', False)
epoch, model_epoch_filename = self.get_epoch_model(config)
assert model_epoch_filename or self.start_epoch
if model_epoch_filename:
print("loading weights from", model_epoch_filename, file=log.v2)
last_model_hdf = h5py.File(model_epoch_filename, "r")
else:
last_model_hdf = None
if config.bool('initialize_from_model', False):
# That's only about the topology, not the params.
print("initializing network topology from model", file=log.v5)
assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'"
network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
else:
if self.pretrain:
# This would be obsolete if we don't want to load an existing model.
# In self.init_train_epoch(), we initialize a new model.
network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch)
else:
network = LayerNetwork.from_config_topology(config)
# We have the parameters randomly initialized at this point.
# In training, as an initialization, we can copy over the params of an imported model,
# where our topology might slightly differ from the imported model.
if config.value('import_model_train_epoch1', '') and self.start_epoch == 1:
assert last_model_hdf
old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
old_network.load_hdf(last_model_hdf)
last_model_hdf.close()
# Copy params to new network.
from NetworkCopyUtils import intelli_copy_layer
# network.hidden are the input + all hidden layers.
for layer_name, layer in sorted(old_network.hidden.items()):
print("Copy hidden layer %s" % layer_name, file=log.v3)
intelli_copy_layer(layer, network.hidden[layer_name])
for layer_name, layer in sorted(old_network.output.items()):
print("Copy output layer %s" % layer_name, file=log.v3)
intelli_copy_layer(layer, network.output[layer_name])
print("Not copied hidden: %s" % sorted(set(network.hidden.keys()).difference(old_network.hidden.keys())), file=log.v3)
print("Not copied output: %s" % sorted(set(network.output.keys()).difference(old_network.output.keys())), file=log.v3)
# Maybe load existing model parameters.
elif last_model_hdf:
network.load_hdf(last_model_hdf)
last_model_hdf.close()
EngineUtil.maybe_subtract_priors(network, self.train_data, config)
self.network = network
if config.has('dump_json'):
self.network_dump_json(config.value('dump_json', ''))
self.print_network_info()
def test_enc_dec1_init():
config = Config()
config.load_file(StringIO(config_enc_dec1_json))
network_json = LayerNetwork.json_from_config(config)
assert_true(network_json)
network = LayerNetwork.from_json_and_config(network_json, config)
assert_true(network)
def test_enc_dec1_init(): config = Config() config.load_file(StringIO(config_enc_dec1_json)) network_json = LayerNetwork.json_from_config(config) assert_true(network_json) network = LayerNetwork.from_json_and_config(network_json, config) assert_true(network)
def init_network_from_config(self, config):
self.pretrain = pretrainFromConfig(config)
self.max_seqs = config.int('max_seqs', -1)
epoch, model_epoch_filename = self.get_epoch_model(config)
assert model_epoch_filename or self.start_epoch
if model_epoch_filename:
print >> log.v2, "loading weights from", model_epoch_filename
last_model_hdf = h5py.File(model_epoch_filename, "r")
else:
last_model_hdf = None
if config.bool('initialize_from_model', False):
# That's only about the topology, not the params.
print >> log.v5, "initializing network topology from model"
assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'"
network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
else:
if self.pretrain:
# This would be obsolete if we don't want to load an existing model.
# In self.init_train_epoch(), we initialize a new model.
network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch)
else:
network = LayerNetwork.from_config_topology(config)
# We have the parameters randomly initialized at this point.
# In training, as an initialization, we can copy over the params of an imported model,
# where our topology might slightly differ from the imported model.
if config.value('import_model_train_epoch1', '') and self.start_epoch == 1:
assert last_model_hdf
old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
old_network.load_hdf(last_model_hdf)
last_model_hdf.close()
# Copy params to new network.
from NetworkCopyUtils import intelli_copy_layer
# network.hidden are the input + all hidden layers.
for layer_name, layer in sorted(old_network.hidden.items()):
print >> log.v3, "Copy hidden layer %s" % layer_name
intelli_copy_layer(layer, network.hidden[layer_name])
for layer_name, layer in sorted(old_network.output.items()):
print >> log.v3, "Copy output layer %s" % layer_name
intelli_copy_layer(layer, network.output[layer_name])
print >> log.v3, "Not copied hidden: %s" % sorted(set(network.hidden.keys()).difference(old_network.hidden.keys()))
print >> log.v3, "Not copied output: %s" % sorted(set(network.output.keys()).difference(old_network.output.keys()))
# Maybe load existing model parameters.
elif last_model_hdf:
network.load_hdf(last_model_hdf)
last_model_hdf.close()
EngineUtil.maybe_subtract_priors(network, self.train_data, config)
self.network = network
if config.has('dump_json'):
self.network_dump_json(config.value('dump_json', ''))
self.print_network_info()
def test_config1_to_json_network_copy():
config = Config()
config.update(config1_dict)
orig_network = LayerNetwork.from_config_topology(config)
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
new_network = LayerNetwork.from_json(orig_json_content, orig_network.n_in, orig_network.n_out)
assert_equal(orig_network.n_in, new_network.n_in)
assert_equal(orig_network.n_out, new_network.n_out)
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def test_config1_to_json_network_copy():
config = Config()
config.update(config1_dict)
orig_network = LayerNetwork.from_config_topology(config)
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
new_network = LayerNetwork.from_json(orig_json_content, orig_network.n_in, orig_network.n_out)
assert_equal(orig_network.n_in, new_network.n_in)
assert_equal(orig_network.n_out, new_network.n_out)
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def pretrain_from_config(config):
"""
:type config: Config.Config
:rtype: Pretrain | None
"""
import Util
from Config import network_json_from_config
pretrain_type = config.bool_or_other("pretrain", None)
if pretrain_type == "default" or (isinstance(pretrain_type, dict) and pretrain_type) or pretrain_type is True:
if Util.BackendEngine.is_theano_selected():
from Network import LayerNetwork
network_init_args = LayerNetwork.init_args_from_config(config)
else:
network_init_args = None
original_network_json = network_json_from_config(config)
opts = config.get_of_type("pretrain", dict, {})
if config.has("pretrain_copy_output_layer"):
opts.setdefault("copy_output_layer", config.bool_or_other("pretrain_copy_output_layer", "ifpossible"))
if config.has("pretrain_greedy"):
opts.setdefault("greedy", config.bool("pretrain_greedy", None))
if config.has("pretrain_repetitions"):
if config.is_typed("pretrain_repetitions"):
opts.setdefault("repetitions", config.typed_value("pretrain_repetitions"))
else:
opts.setdefault("repetitions", config.int_list("pretrain_repetitions", None))
if config.has("pretrain_construction_algo"):
opts.setdefault("construction_algo", config.value("pretrain_construction_algo", None))
return Pretrain(original_network_json=original_network_json, network_init_args=network_init_args, **opts)
elif not pretrain_type:
return None
else:
raise Exception("unknown pretrain type: %s" % pretrain_type)
def init_network_from_config(self, config):
"""
:param Config.Config config:
"""
self.model_filename = config.value('model', None)
self.pretrain = pretrainFromConfig(config)
self.max_seqs = config.int('max_seqs', -1)
epoch, model_epoch_filename = self.get_epoch_model(config)
assert model_epoch_filename or self.start_epoch
self.epoch = epoch or self.start_epoch
if self.pretrain:
# This would be obsolete if we don't want to load an existing model.
# In self.init_train_epoch(), we initialize a new model.
net_dict = self.pretrain.get_network_json_for_epoch(self.epoch)
else:
net_dict = LayerNetwork.json_from_config(config)
self._init_network(net_desc=net_dict, epoch=self.epoch)
if model_epoch_filename:
print("loading weights from", model_epoch_filename, file=log.v2)
try:
self.network.load_params_from_file(model_epoch_filename, session=self.tf_session)
except tf.errors.NotFoundError:
print("Exiting now because model cannot be loaded.", file=log.v1)
sys.exit(1)
def get_network_for_epoch(self, epoch, mask=None): """ :type epoch: int :rtype: Network.LayerNetwork """ json_content = self.get_network_json_for_epoch(epoch) Layer.rng_seed = epoch return LayerNetwork.from_json(json_content, mask=mask, **self.network_init_args)
def get_network_for_epoch(self, epoch, mask=None): """ :type epoch: int :rtype: Network.LayerNetwork """ json_content = self.get_network_json_for_epoch(epoch) Layer.rng_seed = epoch return LayerNetwork.from_json(json_content, mask=mask, **self.network_init_args)
def get_network_for_epoch(self, epoch, mask=None): """ :type epoch: int :param mask: :rtype: Network.LayerNetwork """ from Network import LayerNetwork from NetworkBaseLayer import Layer json_content = self.get_network_json_for_epoch(epoch) Layer.rng_seed = epoch return LayerNetwork.from_json(json_content, mask=mask, **self.network_init_args)
def saveCrnnNetwork(epoch, layers):
"""
:type epoch: int
:type layers: list[(numpy.ndarray, numpy.ndarray)]
"""
print("Loading Crnn")
from Network import LayerNetwork
from NetworkHiddenLayer import ForwardLayer
from NetworkOutputLayer import OutputLayer
from Pretrain import pretrain_from_config
from EngineBase import EngineBase
pretrain = pretrain_from_config(config)
is_pretrain_epoch = pretrain and epoch <= pretrain.get_train_num_epochs()
modelFilename = config.value("model", None)
assert modelFilename, "need 'model' in config"
filename = EngineBase.epoch_model_filename(modelFilename, epoch,
is_pretrain_epoch)
assert not os.path.exists(filename), "already exists"
if is_pretrain_epoch:
network = pretrain.get_network_for_epoch(epoch)
else:
network = LayerNetwork.from_config_topology(config)
nHiddenLayers = len(network.hidden)
# print network topology
print("Crnn Network layer topology:")
print("input dim:", network.n_in)
print("hidden layer count:", nHiddenLayers)
print("output dim:", network.n_out["classes"])
print("net weights #:", network.num_params())
print("net params:", network.train_params_vars)
print("net output:", network.output["output"])
assert network.n_in == inputDim
#assert network.n_out == outputDim
assert nHiddenLayers + 1 == layerCount # hidden + output layer
assert len(layers) == layerCount
for i, (layerName, hidden) in enumerate(sorted(network.hidden.items())):
# Some checks whether this is a forward-layer.
assert isinstance(hidden, ForwardLayer)
saveCrnnLayer(hidden, *layers[i])
assert isinstance(network.output["output"], OutputLayer)
saveCrnnLayer(network.output["output"], *layers[len(layers) - 1])
import h5py
print(("Save Crnn model under %s" % filename))
model = h5py.File(filename, "w")
network.save_hdf(model, epoch)
model.close()
def num_inputs_outputs_from_config(cls, config):
"""
:type config: Config.Config
:returns (num_inputs, num_outputs),
where num_inputs is like num_outputs["data"][0],
and num_outputs is a dict of data_key -> (dim, ndim),
where data_key is e.g. "classes" or "data",
dim is the feature dimension or the number of classes,
and ndim is the ndim counted without batch-dim,
i.e. ndim=1 means usually sparse data and ndim=2 means dense data.
:rtype: (int,dict[str,(int,int)])
"""
num_inputs = config.int('num_inputs', 0)
target = config.value('target', 'classes')
if config.is_typed('num_outputs'):
num_outputs = config.typed_value('num_outputs')
if not isinstance(num_outputs, dict):
num_outputs = {target: num_outputs}
num_outputs = num_outputs.copy()
from Dataset import convert_data_dims
from Util import BackendEngine
num_outputs = convert_data_dims(num_outputs, leave_dict_as_is=BackendEngine.is_tensorflow_selected())
if "data" in num_outputs:
num_inputs = num_outputs["data"][0]
elif config.has('num_outputs'):
num_outputs = {target: [config.int('num_outputs', 0), 1]}
else:
num_outputs = None
dataset = None
if config.list('train') and ":" not in config.value('train', ''):
dataset = config.list('train')[0]
if not config.is_typed('num_outputs') and dataset:
try:
_num_inputs = hdf5_dimension(dataset, 'inputCodeSize') * config.int('window', 1)
except Exception:
_num_inputs = hdf5_dimension(dataset, 'inputPattSize') * config.int('window', 1)
try:
_num_outputs = {target: [hdf5_dimension(dataset, 'numLabels'), 1]}
except Exception:
_num_outputs = hdf5_group(dataset, 'targets/size')
for k in _num_outputs:
_num_outputs[k] = [_num_outputs[k], len(hdf5_shape(dataset, 'targets/data/' + k))]
if num_inputs: assert num_inputs == _num_inputs
if num_outputs: assert num_outputs == _num_outputs
num_inputs = _num_inputs
num_outputs = _num_outputs
if not num_inputs and not num_outputs and config.has("load"):
from Network import LayerNetwork
import h5py
model = h5py.File(config.value("load", ""), "r")
num_inputs, num_outputs = LayerNetwork._n_in_out_from_hdf_model(model)
assert num_inputs and num_outputs, "provide num_inputs/num_outputs directly or via train"
return num_inputs, num_outputs
def saveCrnnNetwork(epoch, layers):
"""
:type epoch: int
:type layers: list[(numpy.ndarray, numpy.ndarray)]
"""
print("Loading Crnn")
from Network import LayerNetwork
from NetworkHiddenLayer import ForwardLayer
from NetworkOutputLayer import OutputLayer
from Pretrain import pretrain_from_config
from EngineBase import EngineBase
pretrain = pretrain_from_config(config)
is_pretrain_epoch = pretrain and epoch <= pretrain.get_train_num_epochs()
modelFilename = config.value("model", None)
assert modelFilename, "need 'model' in config"
filename = EngineBase.epoch_model_filename(modelFilename, epoch, is_pretrain_epoch)
assert not os.path.exists(filename), "already exists"
if is_pretrain_epoch:
network = pretrain.get_network_for_epoch(epoch)
else:
network = LayerNetwork.from_config_topology(config)
nHiddenLayers = len(network.hidden)
# print network topology
print("Crnn Network layer topology:")
print("input dim:", network.n_in)
print("hidden layer count:", nHiddenLayers)
print("output dim:", network.n_out["classes"])
print("net weights #:", network.num_params())
print("net params:", network.train_params_vars)
print("net output:", network.output["output"])
assert network.n_in == inputDim
#assert network.n_out == outputDim
assert nHiddenLayers + 1 == layerCount # hidden + output layer
assert len(layers) == layerCount
for i, (layerName, hidden) in enumerate(sorted(network.hidden.items())):
# Some checks whether this is a forward-layer.
assert isinstance(hidden, ForwardLayer)
saveCrnnLayer(hidden, *layers[i])
assert isinstance(network.output["output"], OutputLayer)
saveCrnnLayer(network.output["output"], *layers[len(layers) - 1])
import h5py
print(("Save Crnn model under %s" % filename))
model = h5py.File(filename, "w")
network.save_hdf(model, epoch)
model.close()
def get_network_for_epoch(self, epoch, mask=None):
"""
:type epoch: int
:param mask:
:rtype: Network.LayerNetwork
"""
from Network import LayerNetwork
from NetworkBaseLayer import Layer
json_content = self.get_network_json_for_epoch(epoch)
Layer.rng_seed = epoch
return LayerNetwork.from_json(json_content,
mask=mask,
**self.network_init_args)
def pretrainFromConfig(config):
"""
:type config: Config.Config
:rtype: Pretrain | None
"""
pretrainType = config.value("pretrain", "")
if pretrainType == "default":
network_init_args = LayerNetwork.init_args_from_config(config)
original_network_json = LayerNetwork.json_from_config(config)
copy_output_layer = config.bool_or_other("pretrain_copy_output_layer", "ifpossible")
greedy = config.bool("pretrain_greedy", None)
repetitions = config.int_list("pretrain_repetitions", None)
construction_algo = config.value("pretrain_construction_algo", None)
return Pretrain(original_network_json=original_network_json,
network_init_args=network_init_args,
copy_output_layer=copy_output_layer,
greedy=greedy, repetitions=repetitions,
construction_algo=construction_algo)
elif pretrainType == "":
return None
else:
raise Exception("unknown pretrain type: %s" % pretrainType)
def test_enc_dec1_hdf(): filename = tempfile.mktemp(prefix="crnn-model-test") model = h5py.File(filename, "w") config = Config() config.load_file(StringIO(config_enc_dec1_json)) network_json = LayerNetwork.json_from_config(config) assert_true(network_json) network = LayerNetwork.from_json_and_config(network_json, config) assert_true(network) network.save_hdf(model, epoch=42) model.close() loaded_model = h5py.File(filename, "r") loaded_net = LayerNetwork.from_hdf_model_topology(loaded_model) assert_true(loaded_net) assert_equal(sorted(network.hidden.keys()), sorted(loaded_net.hidden.keys())) assert_equal(sorted(network.y.keys()), sorted(loaded_net.y.keys())) assert_equal(sorted(network.j.keys()), sorted(loaded_net.j.keys())) os.remove(filename)
def test_enc_dec1_hdf():
filename = tempfile.mktemp(prefix="crnn-model-test")
model = h5py.File(filename, "w")
config = Config()
config.load_file(StringIO(config_enc_dec1_json))
network_json = LayerNetwork.json_from_config(config)
assert_true(network_json)
network = LayerNetwork.from_json_and_config(network_json, config)
assert_true(network)
network.save_hdf(model, epoch=42)
model.close()
loaded_model = h5py.File(filename, "r")
loaded_net = LayerNetwork.from_hdf_model_topology(loaded_model)
assert_true(loaded_net)
assert_equal(sorted(network.hidden.keys()),
sorted(loaded_net.hidden.keys()))
assert_equal(sorted(network.y.keys()), sorted(loaded_net.y.keys()))
assert_equal(sorted(network.j.keys()), sorted(loaded_net.j.keys()))
os.remove(filename)
def test_config2_bidirect_lstm():
config = Config()
config.update(config2_dict)
desc = LayerNetworkDescription.from_config(config)
assert_true(desc.bidirectional)
network = LayerNetwork.from_config_topology(config)
net_json = network.to_json_content()
pprint(net_json)
assert_in("output", net_json)
assert_in("hidden_0_fw", net_json)
assert_in("hidden_0_bw", net_json)
assert_in("hidden_1_fw", net_json)
assert_in("hidden_1_bw", net_json)
assert_in("hidden_2_fw", net_json)
assert_in("hidden_2_bw", net_json)
assert_equal(net_json["output"]["from"], ["hidden_2_fw", "hidden_2_bw"])
assert_equal(len(net_json), 7)
def test_network_config1_init():
config = Config()
config.update(config1_dict)
network = LayerNetwork.from_config_topology(config)
assert_in("hidden_0", network.hidden)
assert_in("hidden_1", network.hidden)
assert_equal(len(network.hidden), 2)
assert_is_instance(network.hidden["hidden_0"], ForwardLayer)
assert_equal(network.hidden["hidden_0"].layer_class, "hidden")
assert_false(network.recurrent)
json_content = network.to_json_content()
pprint(json_content)
assert_in("hidden_0", json_content)
assert_equal(json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", json_content)
assert_in("output", json_content)
def test_config2_bidirect_lstm():
config = Config()
config.update(config2_dict)
desc = LayerNetworkDescription.from_config(config)
assert_true(desc.bidirectional)
network = LayerNetwork.from_config_topology(config)
net_json = network.to_json_content()
pprint(net_json)
assert_in("output", net_json)
assert_in("hidden_0_fw", net_json)
assert_in("hidden_0_bw", net_json)
assert_in("hidden_1_fw", net_json)
assert_in("hidden_1_bw", net_json)
assert_in("hidden_2_fw", net_json)
assert_in("hidden_2_bw", net_json)
assert_equal(net_json["output"]["from"], ["hidden_2_fw", "hidden_2_bw"])
assert_equal(len(net_json), 7)
def test_network_config1_init():
config = Config()
config.update(config1_dict)
network = LayerNetwork.from_config_topology(config)
assert_in("hidden_0", network.hidden)
assert_in("hidden_1", network.hidden)
assert_equal(len(network.hidden), 2)
assert_is_instance(network.hidden["hidden_0"], ForwardLayer)
assert_equal(network.hidden["hidden_0"].layer_class, "hidden")
assert_false(network.recurrent)
json_content = network.to_json_content()
pprint(json_content)
assert_in("hidden_0", json_content)
assert_equal(json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", json_content)
assert_in("output", json_content)
def main(argv):
argparser = argparse.ArgumentParser(description='Dump network as JSON.')
argparser.add_argument('crnn_config_file')
argparser.add_argument('--epoch', default=1, type=int)
argparser.add_argument('--out', default="/dev/stdout")
args = argparser.parse_args(argv[1:])
init(configFilename=args.crnn_config_file, commandLineOptions=[])
pretrain = pretrainFromConfig(config)
if pretrain:
network = pretrain.get_network_for_epoch(args.epoch)
else:
network = LayerNetwork.from_config_topology(config)
json_data = network.to_json_content()
f = open(args.out, 'w')
print(json.dumps(json_data, indent=2, sort_keys=True), file=f)
f.close()
rnn.finalize()
def main(argv):
argparser = argparse.ArgumentParser(description='Dump network as JSON.')
argparser.add_argument('crnn_config_file')
argparser.add_argument('--epoch', default=1, type=int)
argparser.add_argument('--out', default="/dev/stdout")
args = argparser.parse_args(argv[1:])
init(configFilename=args.crnn_config_file, commandLineOptions=[])
pretrain = pretrainFromConfig(config)
if pretrain:
network = pretrain.get_network_for_epoch(args.epoch)
else:
network = LayerNetwork.from_config_topology(config)
json_data = network.to_json_content()
f = open(args.out, 'w')
print >> f, json.dumps(json_data, indent=2, sort_keys=True)
f.close()
rnn.finalize()
def init_network_from_config(self, config):
self.pretrain = pretrainFromConfig(config)
self.max_seqs = config.int('max_seqs', -1)
epoch, model_epoch_filename = self.get_epoch_model(config)
assert model_epoch_filename or self.start_epoch
if self.pretrain:
# This would be obsolete if we don't want to load an existing model.
# In self.init_train_epoch(), we initialize a new model.
net_dict = self.pretrain.get_network_json_for_epoch(
epoch or self.start_epoch)
else:
net_dict = LayerNetwork.json_from_config(config)
self._init_network(net_desc=net_dict, epoch=epoch or self.start_epoch)
if model_epoch_filename:
print("loading weights from", model_epoch_filename, file=log.v2)
self.network.load_params_from_file(model_epoch_filename,
session=self.tf_session)
def __init__(self, filename): self.network = LayerNetwork.from_hdf(filename, mask="unity", train_flag=False, eval_flag=True) self.f_forwarder = None
def num_inputs_outputs_from_config(cls, config):
"""
:type config: Config.Config
:returns (num_inputs, num_outputs),
where num_inputs is like num_outputs["data"][0],
and num_outputs is a dict of data_key -> (dim, ndim),
where data_key is e.g. "classes" or "data",
dim is the feature dimension or the number of classes,
and ndim is the ndim counted without batch-dim,
i.e. ndim=1 means usually sparse data and ndim=2 means dense data.
:rtype: (int,dict[str,(int,int)])
"""
from Util import BackendEngine
num_inputs = config.int('num_inputs', 0)
target = config.value('target', 'classes')
if config.is_typed('num_outputs'):
num_outputs = config.typed_value('num_outputs')
if not isinstance(num_outputs, dict):
num_outputs = {target: num_outputs}
num_outputs = num_outputs.copy()
from Dataset import convert_data_dims
num_outputs = convert_data_dims(num_outputs, leave_dict_as_is=BackendEngine.is_tensorflow_selected())
if "data" in num_outputs:
num_inputs = num_outputs["data"]
if isinstance(num_inputs, (list, tuple)):
num_inputs = num_inputs[0]
elif isinstance(num_inputs, dict):
if "dim" in num_inputs:
num_inputs = num_inputs["dim"]
else:
num_inputs = num_inputs["shape"][-1]
else:
raise TypeError("data key %r" % num_inputs)
elif config.has('num_outputs'):
num_outputs = {target: [config.int('num_outputs', 0), 1]}
else:
num_outputs = None
dataset = None
if config.list('train') and ":" not in config.value('train', ''):
dataset = config.list('train')[0]
if not config.is_typed('num_outputs') and dataset:
# noinspection PyBroadException
try:
_num_inputs = hdf5_dimension(dataset, 'inputCodeSize') * config.int('window', 1)
except Exception:
_num_inputs = hdf5_dimension(dataset, 'inputPattSize') * config.int('window', 1)
# noinspection PyBroadException
try:
_num_outputs = {target: [hdf5_dimension(dataset, 'numLabels'), 1]}
except Exception:
_num_outputs = hdf5_group(dataset, 'targets/size')
for k in _num_outputs:
_num_outputs[k] = [_num_outputs[k], len(hdf5_shape(dataset, 'targets/data/' + k))]
if num_inputs:
assert num_inputs == _num_inputs
if num_outputs:
assert num_outputs == _num_outputs
num_inputs = _num_inputs
num_outputs = _num_outputs
if not num_inputs and not num_outputs and config.has("load") and BackendEngine.is_theano_selected():
from Network import LayerNetwork
import h5py
model = h5py.File(config.value("load", ""), "r")
# noinspection PyProtectedMember
num_inputs, num_outputs = LayerNetwork._n_in_out_from_hdf_model(model)
assert num_inputs and num_outputs, "provide num_inputs/num_outputs directly or via train"
return num_inputs, num_outputs
def __init__(self, filename):
self.network = LayerNetwork.from_hdf(filename,
mask="unity",
train_flag=False,
eval_flag=True)
self.f_forwarder = None
