def check_configs(self):
if self.config['dim'] < 0:
raise xparser_error("dim has invalid value {0}".format(self.config['dim']), self.str())
if self.config['self-repair-scale'] < 0.0 or self.config['self-repair-scale'] > 1.0:
raise xparser_error("self-repair-scale has invalid value {0}".format(self.config['self-repair-scale']), self.str())
if self.config['target-rms'] < 0.0:
raise xparser_error("target-rms has invalid value {0}".format(self.config['target-rms']), self.str())
def check_configs(self):
key = 'cell-dim'
if self.config['cell-dim'] <= 0:
raise xparser_error("cell-dim has invalid value {0}.".format(self.config[key]), self.str())
for key in ['self-repair-scale-nonlinearity']:
if self.config[key] < 0.0 or self.config[key] > 1.0:
raise xparser_error("{0} has invalid value {1}.".format(key, self.config[key]))
def set_configs(self, key_to_value, all_layers):
""" Sets the config variables.
We broke this code out of __init__ for clarity.
the child-class constructor will deal with the configuration values
in a more specific way.
"""
for key, value in key_to_value.items():
if key != 'name':
if not key in self.config:
raise xparser_error(
"Configuration value {0}={1} was not"
" expected in layer of type {2}"
"".format(key, value, self.layer_type), self.str())
self.config[key] = xutils.convert_value_to_type(
key, type(self.config[key]), value)
self.descriptors = dict()
self.descriptor_dims = dict()
# Parse Descriptors and get their dims and their 'final' string form.
# in self.descriptors[key]
for key in self.get_input_descriptor_names():
if not key in self.config:
raise xparser_error(
"{0}: object of type {1} needs to override"
" get_input_descriptor_names()."
"".format(sys.argv[0], str(type(self))), self.str())
descriptor_string = self.config[key] # input string.
assert isinstance(descriptor_string, str)
desc = self.convert_to_descriptor(descriptor_string, all_layers)
desc_dim = self.get_dim_for_descriptor(desc, all_layers)
desc_norm_str = desc.str()
# desc_output_str contains the "final" component names, those that
# appear in the actual config file (i.e. not names like
# 'layer.auxiliary_output'); that's how it differs from desc_norm_str.
# Note: it's possible that the two strings might be the same in
# many, even most, cases-- it depends whether
# output_name(self, auxiliary_output)
# returns self.get_name() + '.' + auxiliary_output
# when auxiliary_output is not None.
# That's up to the designer of the layer type.
desc_output_str = self.get_string_for_descriptor(desc, all_layers)
self.descriptors[key] = {
'string': desc,
'normalized-string': desc_norm_str,
'final-string': desc_output_str,
'dim': desc_dim
}
# the following helps to check the code by parsing it again.
desc2 = self.convert_to_descriptor(desc_norm_str, all_layers)
desc_norm_str2 = desc2.str()
# if the following ever fails we'll have to do some debugging.
if desc_norm_str != desc_norm_str2:
raise xparser_error(
"Likely code error: '{0}' != '{1}'"
"".format(desc_norm_str, desc_norm_str2), self.str())
def set_configs(self, key_to_value, all_layers):
""" Sets the config variables.
We broke this code out of __init__ for clarity.
the child-class constructor will deal with the configuration values
in a more specific way.
"""
for key,value in key_to_value.items():
if key != 'name':
if not key in self.config:
raise xparser_error("Configuration value {0}={1} was not"
" expected in layer of type {2}"
"".format(key, value, self.layer_type),
self.str())
self.config[key] = xutils.convert_value_to_type(key,
type(self.config[key]),
value)
self.descriptors = dict()
self.descriptor_dims = dict()
# Parse Descriptors and get their dims and their 'final' string form.
# in self.descriptors[key]
for key in self.get_input_descriptor_names():
if not key in self.config:
raise xparser_error("{0}: object of type {1} needs to override"
" get_input_descriptor_names()."
"".format(sys.argv[0], str(type(self))),
self.str())
descriptor_string = self.config[key] # input string.
assert isinstance(descriptor_string, str)
desc = self.convert_to_descriptor(descriptor_string, all_layers)
desc_dim = self.get_dim_for_descriptor(desc, all_layers)
desc_norm_str = desc.str()
# desc_output_str contains the "final" component names, those that
# appear in the actual config file (i.e. not names like
# 'layer.auxiliary_output'); that's how it differs from desc_norm_str.
# Note: it's possible that the two strings might be the same in
# many, even most, cases-- it depends whether
# output_name(self, auxiliary_output)
# returns self.get_name() + '.' + auxiliary_output
# when auxiliary_output is not None.
# That's up to the designer of the layer type.
desc_output_str = self.get_string_for_descriptor(desc, all_layers)
self.descriptors[key] = {'string':desc,
'normalized-string':desc_norm_str,
'final-string':desc_output_str,
'dim':desc_dim}
# the following helps to check the code by parsing it again.
desc2 = self.convert_to_descriptor(desc_norm_str, all_layers)
desc_norm_str2 = desc2.str()
# if the following ever fails we'll have to do some debugging.
if desc_norm_str != desc_norm_str2:
raise xparser_error("Likely code error: '{0}' != '{1}'"
"".format(desc_norm_str, desc_norm_str2),
self.str())
def check_configs(self):
for key in ['cell-dim', 'recurrent-projection-dim', 'non-recurrent-projection-dim']:
if self.config[key] <= 0:
raise xparser_error("{0} has invalid value {1}.".format(key, self.config[key]), self.str())
for key in ['self-repair-scale-nonlinearity']:
if self.config[key] < 0.0 or self.config[key] > 1.0:
raise xparser_error("{0} has invalid value {2}.".format(self.layer_type,
key,
self.config[key]))
def check_configs(self):
if self.config['dim'] < 0:
raise xparser_error(
"dim has invalid value {0}".format(self.config['dim']),
self.str())
if self.config['self-repair-scale'] < 0.0 or self.config[
'self-repair-scale'] > 1.0:
raise xparser_error(
"self-repair-scale has invalid value {0}".format(
self.config['self-repair-scale']), self.str())
if self.config['target-rms'] < 0.0:
raise xparser_error(
"target-rms has invalid value {0}".format(
self.config['target-rms']), self.str())
def read_xconfig_file(xconfig_filename):
try:
f = open(xconfig_filename, 'r')
except Exception as e:
sys.exit("{0}: error reading xconfig file '{1}'; error was {2}".format(
sys.argv[0], xconfig_filename, repr(e)))
all_layers = []
while True:
line = f.readline()
if line == '':
break
x = xutils.parse_config_line(line)
if x is None:
continue # line was blank or only comments.
(first_token, key_to_value) = x
# the next call will raise an easy-to-understand exception if
# it fails.
this_layer = parsed_line_to_xconfig_layer(first_token, key_to_value,
all_layers)
all_layers.append(this_layer)
if len(all_layers) == 0:
raise xparser_error("{0}: xconfig file '{1}' is empty".format(
sys.argv[0], xconfig_filename))
f.close()
return all_layers
def read_xconfig_file(xconfig_filename):
try:
f = open(xconfig_filename, 'r')
except Exception as e:
sys.exit("{0}: error reading xconfig file '{1}'; error was {2}".format(
sys.argv[0], xconfig_filename, repr(e)))
all_layers = []
while True:
line = f.readline()
if line == '':
break
x = xutils.parse_config_line(line)
if x is None:
continue # line was blank or only comments.
(first_token, key_to_value) = x
# the next call will raise an easy-to-understand exception if
# it fails.
this_layer = parsed_line_to_xconfig_layer(first_token,
key_to_value,
all_layers)
all_layers.append(this_layer)
if len(all_layers) == 0:
raise xparser_error("{0}: xconfig file '{1}' is empty".format(
sys.argv[0], xconfig_filename))
f.close()
return all_layers
def check_configs(self):
if self.config['dim'] <= -1:
raise xparser_error("In output-layer, dim has invalid value {0}"
"".format(self.config['dim']), self.str())
if self.config['objective-type'] != 'linear' and \
self.config['objective_type'] != 'quadratic':
raise xparser_error("In output-layer, objective-type has"
" invalid value {0}"
"".format(self.config['objective-type']),
self.str())
if self.config['learning-rate-factor'] <= 0.0:
raise xparser_error("In output-layer, learning-rate-factor has"
" invalid value {0}"
"".format(self.config['learning-rate-factor']),
self.str())
def output_name(self, auxiliary_output = None):
node_name = 'm_t'
if auxiliary_output is not None:
if auxiliary_output in self.auxiliary_outputs():
node_name = auxiliary_output
else:
raise xparser_error("Unknown auxiliary output name {0}".format(auxiliary_output), self.str())
return '{0}.{1}'.format(self.name, node_name)
def output_name(self, auxiliary_outputs = None):
# Note: nodes of type output-node in nnet3 may not be accessed in
# Descriptors, so calling this with auxiliary_outputs=None doesn't
# make sense. But it might make sense to make the output of the softmax
# layer and/or the output of the affine layer available as inputs to
# other layers, in some circumstances.
# we'll implement that when it's needed.
raise xparser_error("Outputs of output-layer may not be used by other"
" layers", self.str())
def output_dim(self, auxiliary_output = None):
if auxiliary_output is not None:
if auxiliary_output in self.auxiliary_outputs():
if node_name == 'c_t':
return self.config['cell-dim']
# add code for other auxiliary_outputs here when we decide to expose them
else:
raise xparser_error("Unknown auxiliary output name {0}".format(auxiliary_output), self.str())
return self.config['cell-dim']
def check_configs(self):
if self.config['dim'] <= -1:
raise xparser_error(
"In output-layer, dim has invalid value {0}"
"".format(self.config['dim']), self.str())
if self.config['objective-type'] != 'linear' and \
self.config['objective_type'] != 'quadratic':
raise xparser_error(
"In output-layer, objective-type has"
" invalid value {0}"
"".format(self.config['objective-type']), self.str())
if self.config['learning-rate-factor'] <= 0.0:
raise xparser_error(
"In output-layer, learning-rate-factor has"
" invalid value {0}"
"".format(self.config['learning-rate-factor']), self.str())
def output_name(self, auxiliary_outputs=None):
# Note: nodes of type output-node in nnet3 may not be accessed in
# Descriptors, so calling this with auxiliary_outputs=None doesn't
# make sense. But it might make sense to make the output of the softmax
# layer and/or the output of the affine layer available as inputs to
# other layers, in some circumstances.
# we'll implement that when it's needed.
raise xparser_error(
"Outputs of output-layer may not be used by other"
" layers", self.str())
def __init__(self, first_token, key_to_value, all_layers):
"""
first_token: first token on the xconfig line, e.g. 'affine-layer'.f
key_to_value: dictionary with parameter values
{ 'name':'affine1',
'input':'Append(0, 1, 2, ReplaceIndex(ivector, t, 0))',
'dim=1024' }.
The only required and 'special' values that are dealt with directly
at this level, are 'name' and 'input'. The rest are put in
self.config and are dealt with by the child classes' init functions.
all_layers: An array of objects inheriting XconfigLayerBase for all
previously parsed layers.
"""
self.layer_type = first_token
if not 'name' in key_to_value:
raise xparser_error("Expected 'name' to be specified.", self.str())
self.name = key_to_value['name']
if not xutils.is_valid_line_name(self.name):
raise xparser_error(
"Invalid value: name={0}".format(key_to_value['name']),
self.str())
for prev_layer in all_layers:
if self.name == prev_layer.name:
raise xparser_error("Name '{0}' is used for more than one "
"layer.".format(self.name))
# the following, which should be overridden in the child class, sets
# default config parameters in self.config.
self.set_default_configs()
# The following is not to be reimplemented in child classes;
# it sets the config values to those specified by the user, and
# parses any Descriptors.
self.set_configs(key_to_value, all_layers)
# This method, sets the derived default config values
# i.e., config values when not specified can be derived from
# other values. It can be overridden in the child class.
self.set_derived_configs()
# the following, which should be overridden in the child class, checks
# that the config parameters that have been set are reasonable.
self.check_configs()
def parsed_line_to_xconfig_layer(first_token, key_to_value, prev_names):
conf_line = first_token + ' ' + ' '.join(['{0}={1}'.format(x,y) for x,y in key_to_value.items()])
if not config_to_layer.has_key(first_token):
raise xparser_error("No such layer type.", conf_line)
try:
return config_to_layer[first_token](first_token, key_to_value, prev_names)
except xparser_error as e:
if e.conf_line is None:
# we want to throw informative errors which point to the xconfig line
e.conf_line = conf_line
raise
def __init__(self, first_token, key_to_value, all_layers):
"""
first_token: first token on the xconfig line, e.g. 'affine-layer'.f
key_to_value: dictionary with parameter values
{ 'name':'affine1',
'input':'Append(0, 1, 2, ReplaceIndex(ivector, t, 0))',
'dim=1024' }.
The only required and 'special' values that are dealt with directly
at this level, are 'name' and 'input'. The rest are put in
self.config and are dealt with by the child classes' init functions.
all_layers: An array of objects inheriting XconfigLayerBase for all
previously parsed layers.
"""
self.layer_type = first_token
if not 'name' in key_to_value:
raise xparser_error("Expected 'name' to be specified.", self.str())
self.name = key_to_value['name']
if not xutils.is_valid_line_name(self.name):
raise xparser_error("Invalid value: name={0}".format(
key_to_value['name']), self.str())
# the following, which should be overridden in the child class, sets
# default config parameters in self.config.
self.set_default_configs()
# The following is not to be reimplemented in child classes;
# it sets the config values to those specified by the user, and
# parses any Descriptors.
self.set_configs(key_to_value, all_layers)
# This method, sets the derived default config values
# i.e., config values when not specified can be derived from
# other values. It can be overridden in the child class.
self.set_derived_configs()
# the following, which should be overridden in the child class, checks
# that the config parameters that have been set are reasonable.
self.check_configs()
def parsed_line_to_xconfig_layer(first_token, key_to_value, prev_names):
conf_line = first_token + ' ' + ' '.join(
['{0}={1}'.format(x, y) for x, y in key_to_value.items()])
if not config_to_layer.has_key(first_token):
raise xparser_error("No such layer type.", conf_line)
try:
return config_to_layer[first_token](first_token, key_to_value,
prev_names)
except xparser_error as e:
if e.conf_line is None:
# we want to throw informative errors which point to the xconfig line
e.conf_line = conf_line
raise
def convert_to_descriptor(self, descriptor_string, all_layers):
"""Convenience function intended to be called from child classes,
converts a string representing a descriptor ('descriptor_string')
into an object of type Descriptor, and returns it. It needs 'self' and
'all_layers' (where 'all_layers' is a list of objects of type
XconfigLayerBase) so that it can work out a list of the names of other
layers, and get dimensions from them.
"""
prev_names = xutils.get_prev_names(all_layers, self)
tokens = xutils.tokenize_descriptor(descriptor_string, prev_names)
pos = 0
(descriptor, pos) = xutils.parse_new_descriptor(tokens, pos, prev_names)
# note: 'pos' should point to the 'end of string' marker
# that terminates 'tokens'.
if pos != len(tokens) - 1:
raise xparser_error("Parsing Descriptor, saw junk at end: " +
' '.join(tokens[pos:-1]), self.str())
return descriptor
def convert_to_descriptor(self, descriptor_string, all_layers):
"""Convenience function intended to be called from child classes,
converts a string representing a descriptor ('descriptor_string')
into an object of type Descriptor, and returns it. It needs 'self' and
'all_layers' (where 'all_layers' is a list of objects of type
XconfigLayerBase) so that it can work out a list of the names of other
layers, and get dimensions from them.
"""
prev_names = xutils.get_prev_names(all_layers, self)
tokens = xutils.tokenize_descriptor(descriptor_string, prev_names)
pos = 0
(descriptor,
pos) = xutils.parse_new_descriptor(tokens, pos, prev_names)
# note: 'pos' should point to the 'end of string' marker
# that terminates 'tokens'.
if pos != len(tokens) - 1:
raise xparser_error(
"Parsing Descriptor, saw junk at end: " +
' '.join(tokens[pos:-1]), self.str())
return descriptor
def check_configs(self):
if self.config['splice-indexes'] == '':
raise xparser_error("splice-indexes has to be non-empty",
self.str())
super(XconfigTdnnLayer, self).check_configs()
def output_dim(self, auxiliary_output = None):
# see comment in output_name().
raise xparser_error("Outputs of output-layer may not be used by other"
" layers", self.str())
def get_splice_indexes(self):
try:
return map(lambda x: int(x),
self.config['splice-indexes'].split(","))
except ValueError:
raise xparser_error("Invalid value for splice-indexes.", str(self))
def check_configs(self):
if self.config['dim'] <= 0:
raise xparser_error("Dimension of input-layer '{0}'"
"should be positive.".format(self.name),
self.str())
def check_configs(self):
if self.config['affine-transform-file'] is None:
raise xparser_error("affine-transform-file must be set.", self.str())
def check_configs(self):
if self.config['dim'] <= 0:
raise xparser_error(
"dim specified is invalid".format(self.name, self.layer_type),
self.str())
def check_configs(self):
if self.config['affine-transform-file'] is None:
raise xparser_error("affine-transform-file must be set.",
self.str())
def _add_components(self, input_desc, input_dim, nonlinearities):
output_dim = self.output_dim()
self_repair_scale = self.config['self-repair-scale']
target_rms = self.config['target-rms']
max_change = self.config['max-change']
ng_opt_str = self.config['ng-affine-options']
configs = []
# First the affine node.
line = ('component name={0}.affine'
' type=NaturalGradientAffineComponent'
' input-dim={1}'
' output-dim={2}'
' max-change={3}'
' {4}'
''.format(self.name, input_dim, output_dim, max_change,
ng_opt_str))
configs.append(line)
line = ('component-node name={0}.affine'
' component={0}.affine input={1}'
''.format(self.name, input_desc))
configs.append(line)
cur_node = '{0}.affine'.format(self.name)
for nonlinearity in nonlinearities:
if nonlinearity == 'relu':
line = ('component name={0}.{1}'
' type=RectifiedLinearComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'sigmoid':
line = ('component name={0}.{1}'
' type=SigmoidComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'tanh':
line = ('component name={0}.{1}'
' type=TanhComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'renorm':
line = ('component name={0}.{1}'
' type=NormalizeComponent dim={2}'
' target-rms={3}'
''.format(self.name, nonlinearity, output_dim,
target_rms))
else:
raise xparser_error(
"Unknown nonlinearity type:"
"{0}".format(nonlinearity), self.str())
configs.append(line)
line = ('component-node name={0}.{1}'
' component={0}.{1} input={2}'
''.format(self.name, nonlinearity, cur_node))
configs.append(line)
cur_node = '{0}.{1}'.format(self.name, nonlinearity)
return configs
def check_configs(self):
if self.config['splice-indexes'] == '':
raise xparser_error("splice-indexes has to be non-empty", self.str())
super(XconfigTdnnLayer, self).check_configs()
def _add_components(self, input_desc, input_dim, nonlinearities):
output_dim = self.output_dim()
self_repair_scale = self.config['self-repair-scale']
target_rms = self.config['target-rms']
max_change = self.config['max-change']
ng_opt_str = self.config['ng-affine-options']
configs = []
# First the affine node.
line = ('component name={0}.affine'
' type=NaturalGradientAffineComponent'
' input-dim={1}'
' output-dim={2}'
' max-change={3}'
' {4}'
''.format(self.name, input_dim, output_dim,
max_change, ng_opt_str))
configs.append(line)
line = ('component-node name={0}.affine'
' component={0}.affine input={1}'
''.format(self.name, input_desc))
configs.append(line)
cur_node = '{0}.affine'.format(self.name)
for nonlinearity in nonlinearities:
if nonlinearity == 'relu':
line = ('component name={0}.{1}'
' type=RectifiedLinearComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'sigmoid':
line = ('component name={0}.{1}'
' type=SigmoidComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'tanh':
line = ('component name={0}.{1}'
' type=TanhComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'renorm':
line = ('component name={0}.{1}'
' type=NormalizeComponent dim={2}'
' target-rms={3}'
''.format(self.name, nonlinearity, output_dim,
target_rms))
else:
raise xparser_error("Unknown nonlinearity type:"
"{0}".format(nonlinearity), self.str())
configs.append(line)
line = ('component-node name={0}.{1}'
' component={0}.{1} input={2}'
''.format(self.name, nonlinearity, cur_node))
configs.append(line)
cur_node = '{0}.{1}'.format(self.name, nonlinearity)
return configs
def check_configs(self):
if self.config['dim'] <= 0:
raise xparser_error(
"Dimension of input-layer '{0}'"
"should be positive.".format(self.name), self.str())
def check_configs(self):
if self.config['dim'] <= 0:
raise xparser_error("dim specified is invalid".format(self.name, self.layer_type), self.str())
def output_dim(self, auxiliary_output=None):
# see comment in output_name().
raise xparser_error(
"Outputs of output-layer may not be used by other"
" layers", self.str())
def get_splice_indexes(self):
try:
return map(lambda x: int(x), self.config['splice-indexes'].split(","))
except ValueError:
raise xparser_error("Invalid value for splice-indexes.", str(self))
def get_full_config(self):
ans = []
split_layer_name = self.layer_type.split('-')
assert split_layer_name[-1] == 'layer'
nonlinearities = split_layer_name[:-1]
# by 'descriptor_final_string' we mean a string that can appear in
# config-files, i.e. it contains the 'final' names of nodes.
descriptor_final_string = self.descriptors['input']['final-string']
input_dim = self.descriptors['input']['dim']
output_dim = self.output_dim()
self_repair_scale = self.config['self-repair-scale']
target_rms = self.config['target-rms']
param_stddev = self.config['param-stddev']
bias_stddev = self.config['bias-stddev']
max_change = self.config['max-change']
ng_opt_str = self.config['ng-affine-options']
for config_name in ['ref', 'final']:
# First the affine node.
line = ('component name={0}.affine'
' type=NaturalGradientAffineComponent'
' input-dim={1}'
' output-dim={2}'
' param-stddev={3}'
' bias-stddev={4}'
' max-change={5}'
' {6}'
''.format(self.name, input_dim, output_dim, param_stddev,
bias_stddev, max_change, ng_opt_str))
ans.append((config_name, line))
line = ('component-node name={0}.affine'
' component={0}.affine input={1}'
''.format(self.name, descriptor_final_string))
ans.append((config_name, line))
cur_node = '{0}.affine'.format(self.name)
for nonlinearity in nonlinearities:
if nonlinearity == 'relu':
line = ('component name={0}.{1}'
' type=RectifiedLinearComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'sigmoid':
line = ('component name={0}.{1}'
' type=SigmoidComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'tanh':
line = ('component name={0}.{1}'
' type=TanhComponent dim={2}'
' self-repair-scale={3}'
''.format(self.name, nonlinearity, output_dim,
self_repair_scale))
elif nonlinearity == 'renorm':
line = ('component name={0}.{1}'
' type=NormalizeComponent dim={2}'
' target-rms={3}'
''.format(self.name, nonlinearity, output_dim,
target_rms))
else:
raise xparser_error(
"Unknown nonlinearity type:"
"{0}".format(nonlinearity), self.str())
ans.append((config_name, line))
line = ('component-node name={0}.{1}'
' component={0}.{1} input={2}'
''.format(self.name, nonlinearity, cur_node))
ans.append((config_name, line))
cur_node = '{0}.{1}'.format(self.name, nonlinearity)
return ans
