def load(self, head):
"""
:type head: h5py.File
"""
try:
grp = head[self.name]
except Exception:
print >> log.v3, "warning: unable to load parameters for layer", self.name
return
grp_class = grp.attrs['class']
if grp_class == "<unknown_softmax>": grp_class = "softmax" # bug in some CRNN version. can be ignored.
if grp_class != self.layer_class:
from NetworkLayer import get_layer_class
if not get_layer_class(grp_class, raise_exception=False) is get_layer_class(self.layer_class):
print >>log.v3, "warning: invalid layer class (expected " + self.layer_class + " got " + grp.attrs['class'] + ")"
for p in self.params:
if p not in grp:
print >> log.v4, "unable to load parameter %s in %s" % (p, self.name)
for p in grp:
if p in self.params:
if self.params[p].get_value(borrow=True, return_internal_type=True).shape == grp[p].shape:
array = grp[p][...]
assert not (numpy.isinf(array).any() or numpy.isnan(array).any())
self.params[p].set_value(array)
else:
print >> log.v2, "warning: invalid layer parameter shape for parameter " + p + " of layer " + self.name + \
" (expected " + str(self.params[p].get_value(borrow=True, return_internal_type=True).shape) + \
" got " + str(grp[p].shape) + ")"
#assert self.params[p].get_value(borrow=True, return_internal_type=True).shape == grp[p].shape, \
# "invalid layer parameter shape for parameter " + p + " of layer " + self.name + \
# " (expected " + str(self.params[p].get_value(borrow=True, return_internal_type=True).shape) + \
# " got " + str(grp[p].shape) + ")"
else:
print >> log.v4, "unable to match parameter %s in %s" % (p, self.name)
def load(self, head):
"""
:type head: h5py.File
"""
try:
grp = head[self.name]
except Exception:
print("warning: unable to load parameters for layer", self.name, file=log.v3)
return
grp_class = as_str(grp.attrs['class'])
if grp_class == "<unknown_softmax>": grp_class = "softmax" # bug in some CRNN version. can be ignored.
if grp_class != self.layer_class:
from NetworkLayer import get_layer_class
if not get_layer_class(grp_class, raise_exception=False) is get_layer_class(self.layer_class):
print("warning: invalid layer class (expected " + self.layer_class + " got " + grp.attrs['class'] + ")", file=log.v3)
for p in self.params:
if p not in grp:
print("unable to load parameter %s in %s" % (p, self.name), file=log.v4)
for p in grp:
if p in self.params:
if self.params[p].get_value(borrow=True, return_internal_type=True).shape == grp[p].shape:
array = grp[p][...]
assert not (numpy.isinf(array).any() or numpy.isnan(array).any())
self.params[p].set_value(array)
else:
print("warning: invalid layer parameter shape for parameter " + p + " of layer " + self.name + \
" (expected " + str(self.params[p].get_value(borrow=True, return_internal_type=True).shape) + \
" got " + str(grp[p].shape) + ")", file=log.v2)
#assert self.params[p].get_value(borrow=True, return_internal_type=True).shape == grp[p].shape, \
# "invalid layer parameter shape for parameter " + p + " of layer " + self.name + \
# " (expected " + str(self.params[p].get_value(borrow=True, return_internal_type=True).shape) + \
# " got " + str(grp[p].shape) + ")"
else:
print("unable to match parameter %s in %s" % (p, self.name), file=log.v4)
def _layer_params(self, info, sources, mask, reverse=False):
"""
:param dict[str] info: self.hidden_info[i]
:param list[str] sources: 'from' entry
:param None | str mask: mask
:param bool reverse: reverse or not
:rtype: dict[str]
"""
import inspect
from NetworkLayer import get_layer_class
params = dict(self.default_layer_info)
params.update(info)
params["from"] = sources
if mask:
params["mask"] = mask
layer_class = get_layer_class(params["layer_class"])
if layer_class.recurrent:
params['truncation'] = self.truncation
if self.bidirectional:
if not reverse:
params['name'] += "_fw"
else:
params['name'] += "_bw"
params['reverse'] = True
if 'sharpgates' in inspect.getargspec(layer_class.__init__).args[1:]:
params['sharpgates'] = self.sharpgates
return params
def _layer_params(self, info, sources, mask, reverse=False):
"""
:param dict[str] info: self.hidden_info[i]
:param list[str] sources: 'from' entry
:param None | str mask: mask
:param bool reverse: reverse or not
:rtype: dict[str]
"""
import Util
if Util.BackendEngine.is_theano_selected():
from NetworkLayer import get_layer_class
elif Util.BackendEngine.is_tensorflow_selected():
from TFNetworkLayer import get_layer_class
else:
raise NotImplementedError
params = dict(self.default_layer_info)
params.update(info)
params["from"] = sources
if mask:
params["mask"] = mask
layer_class = get_layer_class(params["layer_class"])
if layer_class.recurrent:
params['truncation'] = self.truncation
if self.bidirectional:
if not reverse:
params['name'] += "_fw"
else:
params['name'] += "_bw"
params['reverse'] = True
if 'sharpgates' in Util.getargspec(layer_class.__init__).args[1:]:
params['sharpgates'] = self.sharpgates
return params
def _layer_params(self, info, sources, mask, reverse=False):
"""
:param dict[str] info: self.hidden_info[i]
:param list[str] sources: 'from' entry
:param None | str mask: mask
:param bool reverse: reverse or not
:rtype: dict[str]
"""
import inspect
from NetworkLayer import get_layer_class
params = dict(self.default_layer_info)
params.update(info)
params["from"] = sources
if mask:
params["mask"] = mask
layer_class = get_layer_class(params["layer_class"])
if layer_class.recurrent:
params['truncation'] = self.truncation
if self.bidirectional:
if not reverse:
params['name'] += "_fw"
else:
params['name'] += "_bw"
params['reverse'] = True
if 'sharpgates' in inspect.getargspec(layer_class.__init__).args[1:]:
params['sharpgates'] = self.sharpgates
return params
def _layer_params(self, info, sources, mask, reverse=False):
"""
:param dict[str] info: self.hidden_info[i]
:param list[str] sources: 'from' entry
:param None | str mask: mask
:param bool reverse: reverse or not
:rtype: dict[str]
"""
import Util
if Util.BackendEngine.is_theano_selected():
from NetworkLayer import get_layer_class
elif Util.BackendEngine.is_tensorflow_selected():
from TFNetworkLayer import get_layer_class
else:
raise NotImplementedError
params = dict(self.default_layer_info)
params.update(info)
params["from"] = sources
if mask:
params["mask"] = mask
layer_class = get_layer_class(params["layer_class"])
if layer_class.recurrent:
params['truncation'] = self.truncation
if self.bidirectional:
if not reverse:
params['name'] += "_fw"
else:
params['name'] += "_bw"
params['reverse'] = True
if 'sharpgates' in Util.getargspec(layer_class.__init__).args[1:]:
params['sharpgates'] = self.sharpgates
return params
def traverse(model, layer_name, output_index):
index = output_index
mask = network.default_mask
if not mask and 'mask' in model[layer_name].attrs:
mask = model[layer_name].attrs['mask']
if 'from' in model[layer_name].attrs:
x_in = []
for s in model[layer_name].attrs['from'].split(','):
if s == 'data':
x_in.append(SourceLayer(network.n_in, network.x, sparse=sparse_input, name='data', index=network.i))
index = network.i
elif s != "null" and s != "": # this is allowed, recurrent states can be passed as input
if not network.hidden.has_key(s):
index = traverse(model, s, index)
else:
index = network.hidden[s].index
x_in.append(network.hidden[s])
elif s == "":
assert not s
# Fix for old models via NetworkDescription.
s = Layer.guess_source_layer_name(layer_name)
if not s:
# Fix for data input. Just like in NetworkDescription, so that param names are correct.
x_in.append(SourceLayer(n_out=network.n_in, x_out=network.x, name="", index=network.i))
else:
if not network.hidden.has_key(s):
index = traverse(model, s, index)
else:
index = network.hidden[s].index
# Add just like in NetworkDescription, so that param names are correct.
x_in.append(SourceLayer(n_out=network.hidden[s].attrs['n_out'], x_out=network.hidden[s].output, name="", index=network.i))
else:
x_in = [ SourceLayer(network.n_in, network.x, sparse=sparse_input, name='data', index=network.i) ]
if 'encoder' in model[layer_name].attrs:
encoder = []
for s in model[layer_name].attrs['encoder'].split(','):
if s != "":
if not network.hidden.has_key(s):
traverse(model, s, index)
encoder.append(network.hidden[s])
if 'base' in model[layer_name].attrs: # TODO see json
base = []
for s in model[layer_name].attrs['base'].split(','):
if s != "":
if not network.hidden.has_key(s):
traverse(model, s, index)
base.append(network.hidden[s])
for key in ['copy_input', 'copy_output']:
if key in model[layer_name].attrs:
index = traverse(model, model[layer_name].attrs[key], index)
if key == 'copy_input':
copy_input = network.hidden[model[layer_name].attrs[key]]
if key == 'copy_output':
copy_output = network.hidden[model[layer_name].attrs[key]]
if 'encoder' in model[layer_name].attrs and not x_in:
index = output_index
if 'target' in model[layer_name].attrs:
target = model[layer_name].attrs['target']
if target != "null" and target not in network.y:
network.use_target(target, dtype=dtype)
index = network.j[target]
cl = model[layer_name].attrs['class']
if cl == 'softmax':
params = { 'dropout' : 0.0,
'name' : 'output',
'mask' : mask,
'train_flag' : train_flag }
params.update(model[layer_name].attrs)
if 'encoder' in model[layer_name].attrs:
params['encoder'] = encoder #network.hidden[model[layer_name].attrs['encoder']] if model[layer_name].attrs['encoder'] in network.hidden else network.output[model[layer_name].attrs['encoder']]
if 'base' in model[layer_name].attrs:
params['base'] = base
if 'copy_input' in model[layer_name].attrs:
params['copy_input'] = copy_input
if 'copy_output' in model[layer_name].attrs:
params['copy_output'] = copy_output
#if not 'target' in params:
# params['target'] = target
params['index'] = index #output_index
params['sources'] = x_in
params['y_in'] = network.y
params.pop('from', None)
params.pop('class', None)
network.make_classifier(**params)
else:
params = { 'sources': x_in,
'n_out': model[layer_name].attrs['n_out'],
'dropout': model[layer_name].attrs['dropout'] if train_flag else 0.0,
'name': layer_name,
'mask': mask,
'train_flag' : train_flag,
"eval_flag": eval_flag,
'network': network,
'index' : index }
try:
act = model[layer_name].attrs['activation']
params["activation"] = act
except Exception:
pass
params['y_in'] = network.y
layer_class = get_layer_class(cl)
for p in collect_class_init_kwargs(layer_class):
if p in params: continue # don't overwrite existing
if p in model[layer_name].attrs.keys():
params[p] = model[layer_name].attrs[p]
if 'encoder' in model[layer_name].attrs:
params['encoder'] = encoder #network.hidden[model[layer_name].attrs['encoder']] if model[layer_name].attrs['encoder'] in network.hidden else network.output[model[layer_name].attrs['encoder']]
if 'base' in model[layer_name].attrs:
params['base'] = base
if 'target' in model[layer_name].attrs:
params['target'] = model[layer_name].attrs['target']
if layer_class.recurrent:
network.recurrent = True
return network.add_layer(layer_class(**params)).index
def traverse(content, layer_name, target, output_index, inherit=False):
if layer_name in network.hidden:
return network.hidden[layer_name].index
if layer_name in network.output:
return network.output[layer_name].index
source = []
obj = content[layer_name].copy()
if 'inherit' in obj:
if not obj['inherit'] in templates:
traverse(content, obj['inherit'], target, output_index, True)
template = templates[obj['inherit']].copy()
for key in template.keys():
if not key in obj.keys():
obj[key] = template[key]
del obj['inherit']
templates[layer_name] = obj.copy()
if inherit:
return output_index
cl = obj.pop('class', None)
index = output_index
if 'target' in obj:
target = obj['target']
dtype = obj.get("dtype", "int32")
network.use_target(target, dtype=dtype)
if not 'from' in obj and cl is not None:
source = [SourceLayer(network.n_in, network.x, sparse=sparse_input, name='data', index=network.i)]
index = network.i
elif 'from' in obj and obj['from']:
if not isinstance(obj['from'], list):
obj['from'] = [ obj['from'] ]
for prev in obj['from']:
if prev == 'data':
source.append(SourceLayer(network.n_in, network.x, sparse=sparse_input, name='data', index=network.i))
index = network.i
elif not prev in content.keys() and prev != "null":
sparse = obj.pop('sparse_input', False)
dtype = 'int32' if sparse else 'float32'
source.append(SourceLayer(0, None, sparse=sparse, dtype=dtype, name='data', network=network, data_key=prev))
index = source[-1].index
elif prev != "null":
index = traverse(content, prev, target, index)
source.append(network.get_layer(prev))
if 'encoder' in obj:
encoder = []
if not isinstance(obj['encoder'], list):
obj['encoder'] = [obj['encoder']]
for prev in obj['encoder']:
traverse(content, prev, target, index)
encoder.append(network.get_layer(prev))
obj['encoder'] = encoder
if 'base' in obj: # TODO(doetsch) string/layer transform should be smarter
base = []
if not isinstance(obj['base'], list):
if ',' in obj['base']:
obj['base'] = obj['base'].split(',')
else:
obj['base'] = [obj['base']]
for prev in obj['base']:
if prev == 'data':
base.append(SourceLayer(network.n_in, network.x, sparse=sparse_input, name='data', index=network.i))
else:
traverse(content, prev, target, index)
base.append(network.get_layer(prev))
obj['base'] = base
for key in [ 'copy_input', 'copy_output' ]:
if key in obj:
index = traverse(content, obj[key], target, index)
obj[key] = network.get_layer(obj[key])
if 'encoder' in obj and not source:
index = output_index
if 'target' in obj and obj['target'] != "null":
index = network.j[obj['target']]
obj.pop('from', None)
params = { 'sources': source,
'dropout' : 0.0,
'name' : layer_name,
"train_flag": train_flag,
"eval_flag": eval_flag,
'network': network }
params.update(obj)
params["mask"] = mask # overwrite
params['index'] = index
params['y_in'] = network.y
if cl:
templates[layer_name]['class'] = cl
if cl == 'softmax' or cl == 'decoder':
if not 'target' in params:
params['target'] = target
if 'loss' in obj and obj['loss'] in ('ctc','hmm'):
params['index'] = network.i
elif target != "null":
params['index'] = network.j[target] #output_index
return network.make_classifier(**params)
elif cl is not None:
layer_class = get_layer_class(cl)
params.update({'name': layer_name})
if layer_class.recurrent:
network.recurrent = True
return network.add_layer(layer_class(**params)).index
def traverse(content, layer_name, target, output_index, inherit=False):
if layer_name in network.hidden:
return network.hidden[layer_name].index
if layer_name in network.output:
return network.output[layer_name].index
source = []
obj = content[layer_name].copy()
if 'inherit' in obj:
if not obj['inherit'] in templates:
traverse(content, obj['inherit'], target, output_index,
True)
template = templates[obj['inherit']].copy()
for key in template.keys():
if not key in obj.keys():
obj[key] = template[key]
del obj['inherit']
templates[layer_name] = obj.copy()
if inherit:
return output_index
cl = obj.pop('class', None)
index = output_index
if 'target' in obj:
target = obj['target']
dtype = obj.get("dtype", "int32")
network.use_target(target, dtype=dtype)
if not 'from' in obj and cl is not None:
source = [
SourceLayer(network.n_in,
network.x,
sparse=sparse_input,
name='data',
index=network.i)
]
index = network.i
elif 'from' in obj and obj['from']:
if not isinstance(obj['from'], list):
obj['from'] = [obj['from']]
for prev in obj['from']:
if prev == 'data':
source.append(
SourceLayer(network.n_in,
network.x,
sparse=sparse_input,
name='data',
index=network.i))
index = network.i
elif not prev in content.keys() and prev != "null":
sparse = obj.pop('sparse_input', False)
dtype = 'int32' if sparse else 'float32'
source.append(
SourceLayer(0,
None,
sparse=sparse,
dtype=dtype,
name='data',
network=network,
data_key=prev))
index = source[-1].index
elif prev != "null":
index = traverse(content, prev, target, index)
source.append(network.get_layer(prev))
if 'encoder' in obj:
encoder = []
if not isinstance(obj['encoder'], list):
obj['encoder'] = [obj['encoder']]
for prev in obj['encoder']:
traverse(content, prev, target, index)
encoder.append(network.get_layer(prev))
obj['encoder'] = encoder
if 'base' in obj: # TODO(doetsch) string/layer transform should be smarter
base = []
if not isinstance(obj['base'], list):
if ',' in obj['base']:
obj['base'] = obj['base'].split(',')
else:
obj['base'] = [obj['base']]
for prev in obj['base']:
if prev == 'data':
base.append(
SourceLayer(network.n_in,
network.x,
sparse=sparse_input,
name='data',
index=network.i))
else:
traverse(content, prev, target, index)
base.append(network.get_layer(prev))
obj['base'] = base
for key in ['copy_input', 'copy_output', 'aligner']:
if key in obj:
index = traverse(content, obj[key], target, index)
obj[key] = network.get_layer(obj[key])
if 'encoder' in obj and not source:
index = output_index
if 'target' in obj and obj['target'] != "null":
index = network.j[obj['target']]
obj.pop('from', None)
params = {
'sources': source,
'dropout': 0.0,
'name': layer_name,
"train_flag": train_flag,
"eval_flag": eval_flag,
'network': network
}
params.update(obj)
params["mask"] = mask # overwrite
params['index'] = index
params['y_in'] = network.y
if cl:
templates[layer_name]['class'] = cl
if cl == 'softmax' or cl == 'decoder':
if not 'target' in params:
params['target'] = target
if 'loss' in obj and obj['loss'] in ('ctc', 'hmm'):
params['index'] = network.i
elif target != "null":
params['index'] = network.j[target] #output_index
return network.make_classifier(**params)
elif cl is not None:
layer_class = get_layer_class(cl)
params.update({'name': layer_name})
if layer_class.recurrent:
network.recurrent = True
return network.add_layer(layer_class(**params)).index