def Convert(model_file, output_file):
model = util.ReadModel(model_file)
params = {}
for l in model.layer:
for p in l.param:
params['%s_%s' % (l.name, p.name)] = util.ParameterAsNumpy(p)
for e in model.edge:
for p in e.param:
params['%s_%s_%s' %
(e.node1, e.node2, p.name)] = util.ParameterAsNumpy(p)
scipy.io.savemat(output_file, params, oned_as='column')
def InitializeParameter(self, param):
if param.initialization == deepnet_pb2.Parameter.CONSTANT:
return np.zeros(tuple(param.dimensions)) + param.constant
elif param.initialization == deepnet_pb2.Parameter.DENSE_GAUSSIAN:
return param.sigma * np.random.randn(*tuple(param.dimensions))
elif param.initialization == deepnet_pb2.Parameter.PRETRAINED:
node_name = param.pretrained_model_node1
if node_name == '':
node_name = self.proto.name
mat = None
for pretrained_model in param.pretrained_model:
model = util.ReadModel(pretrained_model)
# Find the relevant node in the model.
node = next(n for n in model.layer if n.name == node_name)
# Find the relevant parameter in the node.
pretrained_param = next(p for p in node.param
if p.name == param.name)
assert pretrained_param.mat != '',\
'Pretrained param %s in layer %s of model %s is empty!!' % (
pretrained_param.name, node.name, pretrained_model)
this_mat = util.ParameterAsNumpy(pretrained_param)
if mat is None:
mat = this_mat
else:
mat += this_mat
return mat / len(param.pretrained_model)
else:
raise Exception('Unknown parameter initialization.')
def InitializeParameter(self, param):
if param.initialization == deepnet_pb2.Parameter.CONSTANT:
return np.zeros(tuple(param.dimensions)) + param.constant
elif param.initialization == deepnet_pb2.Parameter.DENSE_GAUSSIAN:
return param.sigma * np.random.randn(*tuple(param.dimensions))
elif param.initialization == deepnet_pb2.Parameter.DENSE_UNIFORM:
return param.sigma * (2 * np.random.rand(*tuple(param.dimensions)) - 1)
elif param.initialization == deepnet_pb2.Parameter.DENSE_GAUSSIAN_SQRT_FAN_IN:
if param.conv:
fan_in = np.prod(param.dimensions[0])
else:
fan_in = np.prod(param.dimensions[1])
stddev = param.sigma / np.sqrt(fan_in)
return stddev * np.random.randn(*tuple(param.dimensions))
elif param.initialization == deepnet_pb2.Parameter.DENSE_UNIFORM_SQRT_FAN_IN:
if param.conv:
fan_in = np.prod(param.dimensions[0])
else:
fan_in = np.prod(param.dimensions[1])
stddev = param.sigma / np.sqrt(fan_in)
return stddev * (2 * np.random.rand(*tuple(param.dimensions)) - 1)
elif param.initialization == deepnet_pb2.Parameter.PRETRAINED:
node1_name = param.pretrained_model_node1
node2_name = param.pretrained_model_node2
if node1_name == '':
node1_name = self.proto.node1
if node2_name == '':
node2_name = self.proto.node2
mat = None
for pretrained_model in param.pretrained_model:
model = util.ReadModel(pretrained_model)
edge = next(e for e in model.edge if e.node1 == node1_name and e.node2 == node2_name)
pretrained_param = next(p for p in edge.param if p.name == param.name)
assert pretrained_param.mat != '',\
'Pretrained param %s in edge %s:%s of model %s is empty!!' % (
pretrained_param.name, edge.node1, edge.node2, pretrained_model)
if param.transpose_pretrained:
assert param.dimensions == pretrained_param.dimensions[::-1],\
'Param has shape %s but transposed pretrained param has shape %s' % (
param.dimensions, reversed(pretrained_param.dimensions))
else:
assert param.dimensions == pretrained_param.dimensions,\
'Param has shape %s but pretrained param has shape %s' % (
param.dimensions, pretrained_param.dimensions)
this_mat = util.ParameterAsNumpy(pretrained_param)
if param.transpose_pretrained:
this_mat = this_mat.T
if mat is None:
mat = this_mat
else:
mat += this_mat
return mat / len(param.pretrained_model)
else:
raise Exception('Unknown parameter initialization.')
def LoadParams(self, tied_to=None):
"""Load the parameters for this edge.
Load the parameters if present in self.proto. Otherwise initialize them
appropriately.
"""
node1 = self.node1
node2 = self.node2
proto = self.proto
self.hyperparams = proto.hyperparams
param_names = [param.name for param in proto.param]
for param in proto.param:
if param.conv or param.local:
n_locs = self.AllocateMemoryForConvolutions(
param, node1, node2)
if not param.dimensions:
if param.conv:
cv = param.conv_params
dims = [cv.num_filters, cv.size**2 * cv.num_colors]
elif param.local:
dims = [
cv.num_filters, n_locs**2 * cv.size**2 * cv.num_colors
]
else:
dims = [
node1.numlabels * node1.dimensions,
node2.numlabels * node2.dimensions
]
param.dimensions.extend(dims)
if tied_to:
if self.transpose:
self.params[param.name] = tied_to.params[param.name].T
else:
self.params[param.name] = tied_to.params[param.name]
mat = self.params[param.name]
else:
if param.mat: # and 'grad' not in param.name:
mat = util.ParameterAsNumpy(param)
else:
mat = self.InitializeParameter(param)
self.params[param.name] = cm.CUDAMatrix(mat)
if param.name == 'weight':
self.temp = cm.empty(mat.shape)
#self.temp2 = cm.empty(mat.shape)
self.gradient = cm.empty(mat.shape)
self.grad_weight = cm.empty(mat.shape)
self.gradient.assign(0)
self.grad_weight.assign(0)
if self.t_op and (self.t_op.optimizer == deepnet_pb2.Operation.PCD or \
self.t_op.optimizer == deepnet_pb2.Operation.CD):
self.suff_stats = cm.empty(
(self.node1.numlabels * self.node1.dimensions,
self.node2.numlabels * self.node2.dimensions))
def LoadParams(self, proto):
self.hyperparams = proto.hyperparams
param_names = [param.name for param in proto.param]
for param in proto.param:
if not param.dimensions:
param.dimensions.extend([proto.numlabels * proto.dimensions])
if param.mat:
mat = util.ParameterAsNumpy(param).reshape(-1, 1)
else:
mat = self.InitializeParameter(param).reshape(-1, 1)
self.params[param.name] = cm.CUDAMatrix(mat)
if param.name == 'bias':
self.grad_bias = cm.empty(mat.shape)
self.grad_bias.assign(0)
self.sample_input = self.hyperparams.sample_input
def LoadParams(self, proto):
self.hyperparams = proto.hyperparams
param_names = [param.name for param in proto.param]
for param in proto.param:
if 'grad_' + param.name not in param_names and not param.name.startswith(
'grad_'):
grad_p = deepnet_pb2.Parameter()
grad_p.name = 'grad_' + param.name
proto.param.extend([grad_p])
for param in proto.param:
if not param.dimensions:
param.dimensions.extend([proto.numlabels * proto.dimensions])
if param.mat and 'grad' not in param.name:
mat = util.ParameterAsNumpy(param).reshape(-1, 1)
else:
mat = self.InitializeParameter(param).reshape(-1, 1)
self.params[param.name] = cm.CUDAMatrix(mat)
def LoadParams(self, proto, t_op=None, tied_to=None):
"""Load the parameters for this edge.
Load the parameters if present in self.proto. Otherwise initialize them
appropriately.
"""
param_names = [param.name for param in proto.param]
for param in proto.param:
assert param.dimensions, 'Empty dimensions'
if tied_to:
if self.transpose:
self.params[param.name] = tied_to.params[param.name].T
else:
self.params[param.name] = tied_to.params[param.name]
mat = self.params[param.name]
else:
if param.mat:
mat = util.ParameterAsNumpy(param)
else:
mat = self.InitializeParameter(param)
self.params[param.name] = cm.CUDAMatrix(mat)
def InitializeParameter(self, param):
if param.initialization == deepnet_pb2.Parameter.CONSTANT:
return np.zeros(tuple(param.dimensions)) + param.constant
elif param.initialization == deepnet_pb2.Parameter.DENSE_GAUSSIAN:
return param.sigma * np.random.randn(*tuple(param.dimensions))
elif param.initialization == deepnet_pb2.Parameter.PRETRAINED:
node_name = param.pretrained_model_node1
if node_name == '':
node_name = self.proto.name
mat = None
for pretrained_model in param.pretrained_model:
if os.path.splitext(pretrained_model)[1] == '.npz':
model_file = os.path.join(self.prefix, pretrained_model)
npzfile = np.load(model_file)
if param.name == 'bias':
this_mat = np.nan_to_num(npzfile['mean'] / npzfile['std'])
elif param.name == 'precision':
this_mat = np.nan_to_num(1. / npzfile['std'])
else:
model_file = os.path.join(self.prefix, pretrained_model)
model = util.ReadModel(model_file)
# Find the relevant node in the model.
node = next(n for n in model.layer if n.name == node_name)
# Find the relevant parameter in the node.
pretrained_param = next(p for p in node.param if p.name == param.name)
assert pretrained_param.mat != '',\
'Pretrained param %s in layer %s of model %s is empty!!' % (
pretrained_param.name, node.name, pretrained_model)
this_mat = util.ParameterAsNumpy(pretrained_param)
if mat is None:
mat = this_mat
else:
mat += this_mat
return mat / len(param.pretrained_model)
else:
raise Exception('Unknown parameter initialization.')
def LoadParams(self, proto, node1, node2):
"""Load the parameters for this edge.
Load the parameters if present in the proto. Otherwise initialize them
appropriately.
Args:
proto: Protocol buffer object that describes this edge.
node1: Layer object at the input to this edge.
node2: Layer object at the output of this edge.
"""
self.hyperparams = proto.hyperparams
param_names = [param.name for param in proto.param]
for param in proto.param:
if 'grad_'+param.name not in param_names and not param.name.startswith(
'grad_') and not param.name.startswith('gradstats_'):
grad_p = deepnet_pb2.Parameter()
grad_p.CopyFrom(param)
grad_p.name = 'grad_' + param.name
proto.param.extend([grad_p])
if 'gradstats_'+param.name not in param_names and not param.name.startswith(
'gradstats_') and not param.name.startswith('grad_'):
grad_stats_p = deepnet_pb2.Parameter()
grad_p.CopyFrom(param)
grad_stats_p.name = 'gradstats_' + param.name
grad_stats_p.constant = 1.0
proto.param.extend([grad_stats_p])
for param in proto.param:
if not param.dimensions:
if param.conv:
self.conv = True
self.conv_params = param.conv_params
param.dimensions.extend(
[self.conv_params.num_filters,
self.conv_params.size**2 * self.conv_params.num_colors])
self.input_t = cm.CUDAMatrix(np.zeros(node1.state.shape[::-1]))
self.output_t = cm.CUDAMatrix(np.zeros(node2.state.shape[::-1]))
self.output_t2 = cm.CUDAMatrix(np.zeros(node2.state.shape[::-1]))
if param.conv_params.max_pool:
numdims, numimages = node1.state.shape
num_colors = param.conv_params.num_colors
num_filters = param.conv_params.num_filters
padding = param.conv_params.padding
size = param.conv_params.size
stride = param.conv_params.stride
assert numdims % num_colors == 0
x = int(np.sqrt(numdims / num_colors))
assert x**2 == numdims / num_colors
n_locs = (x + 2 * padding - size) / stride + 1
self.unpooled_layer = cm.CUDAMatrix(np.zeros((numimages, n_locs**2 * num_filters)))
else:
param.dimensions.extend([node1.numlabels * node1.dimensions,
node2.numlabels * node2.dimensions])
if param.mat and 'grad' not in param.name:
print 'Loading saved parameters'
mat = util.ParameterAsNumpy(param)
else:
mat = self.InitializeParameter(param)
self.params[param.name] = cm.CUDAMatrix(mat)
if param.name == 'weight':
self.temp = cm.empty(mat.shape)
self.temp2 = cm.empty(mat.shape)