def __init__(self, w, config=None):
if type(w) == np.ndarray:
self.w_ = cm.CUDAMatrix(w)
elif type(w) == tuple:
self.w_ = cm.empty(w)
else:
self.w_ = w
self.dw_ = cm.empty_like(self.w_)
self.dw_history_ = cm.empty_like(self.w_)
self.dw_history_.assign(0)
self.dw_.assign(0)
self.t_ = 0
self.rms_prop_ = config.rms_prop
self.rms_prop_factor_ = config.rms_prop_factor
if self.rms_prop_:
self.rms_prop_history_ = cm.empty_like(self.dw_)
self.rms_prop_history_.assign(1)
if config is None:
pass
elif config.init_type == config_pb2.Param.CONSTANT:
self.w_.assign(config.scale)
elif config.init_type == config_pb2.Param.GAUSSIAN:
self.w_.fill_with_randn()
self.w_.mult(config.scale)
elif config.init_type == config_pb2.Param.UNIFORM:
self.w_.fill_with_rand()
self.w_.subtract(0.5)
self.w_.mult(2 * config.scale)
elif config.init_type == config_pb2.Param.LSTM_BIAS:
init_bias = [
config.input_gate_bias, config.forget_gate_bias,
config.input_bias, config.output_gate_bias
]
self.w_.reshape((-1, 4))
for i in xrange(4):
self.w_.slice(i, (i + 1)).assign(init_bias[i])
self.w_.reshape((-1, 1))
elif config.init_type == config_pb2.Param.PRETRAINED:
f = h5py.File(config.file_name)
mat = f[config.dataset_name].value
if len(mat.shape) == 1:
mat = mat.reshape(1, -1)
assert self.w_.shape == mat.shape
self.w_.overwrite(mat)
f.close()
else:
raise Exception('Unknown parameter initialization.')
self.eps_ = config.epsilon
self.momentum_ = config.momentum
self.l2_decay_ = config.l2_decay
self.gradient_clip_ = config.gradient_clip
self.eps_decay_factor = config.eps_decay_factor
self.eps_decay_after = config.eps_decay_after
def __init__(self, w, config=None):
if type(w) == np.ndarray:
self.w_ = cm.CUDAMatrix(w)
elif type(w) == tuple:
self.w_ = cm.empty(w)
else:
self.w_ = w
self.dw_ = cm.empty_like(self.w_)
self.dw_history_ = cm.empty_like(self.w_)
self.dw_history_.assign(0)
self.dw_.assign(0)
self.t_ = 0
self.rms_prop_ = config.rms_prop
self.rms_prop_factor_ = config.rms_prop_factor
if self.rms_prop_:
self.rms_prop_history_ = cm.empty_like(self.dw_)
self.rms_prop_history_.assign(1)
if config is None:
pass
elif config.init_type == config_pb2.Param.CONSTANT:
self.w_.assign(config.scale)
elif config.init_type == config_pb2.Param.GAUSSIAN:
self.w_.fill_with_randn()
self.w_.mult(config.scale)
elif config.init_type == config_pb2.Param.UNIFORM:
self.w_.fill_with_rand()
self.w_.subtract(0.5)
self.w_.mult(2 * config.scale)
elif config.init_type == config_pb2.Param.LSTM_BIAS:
init_bias = [config.input_gate_bias, config.forget_gate_bias, config.input_bias, config.output_gate_bias]
self.w_.reshape((-1, 4))
for i in xrange(4):
self.w_.slice(i, (i+1)).assign(init_bias[i])
self.w_.reshape((-1, 1))
elif config.init_type == config_pb2.Param.PRETRAINED:
f = h5py.File(config.file_name)
mat = f[config.dataset_name].value
if len(mat.shape) == 1:
mat = mat.reshape(1, -1)
assert self.w_.shape == mat.shape
self.w_.overwrite(mat)
f.close()
else:
raise Exception('Unknown parameter initialization.')
self.eps_ = config.epsilon
self.momentum_ = config.momentum
self.l2_decay_ = config.l2_decay
self.gradient_clip_ = config.gradient_clip
self.eps_decay_factor = config.eps_decay_factor
self.eps_decay_after = config.eps_decay_after
def __init__(self, w, config=None):
if type(w) == np.ndarray:
self.w_ = cm.CUDAMatrix(w)
elif type(w) == tuple:
self.w_ = cm.empty(w)
else:
self.w_ = w
self.dw_ = cm.empty_like(self.w_)
self.dw_history_ = cm.empty_like(self.w_)
self.dw_history_sqrt_ = cm.empty_like(self.w_)
self.dw_sqr_ = cm.empty_like(self.w_)
self.dw_history_.assign(0)
self.dw_.assign(0)
self.t_ = 0
if config is None:
pass
elif config.init_type == config_pb2.Param.CONSTANT:
self.w_.assign(config.scale)
elif config.init_type == config_pb2.Param.GAUSSIAN:
self.w_.fill_with_randn()
self.w_.mult(config.scale)
elif config.init_type == config_pb2.Param.UNIFORM:
self.w_.fill_with_rand()
self.w_.subtract(0.5)
self.w_.mult(2 * config.scale)
elif config.init_type == config_pb2.Param.PRETRAINED:
f = h5py.File(config.file_name)
mat = f[config.dataset_name].value
if len(mat.shape) == 1:
mat = mat.reshape(1, -1)
assert self.w_.shape == mat.shape
self.w_.overwrite(mat)
f.close()
else:
raise Exception('Unknown parameter initialization.')
self.eps_ = config.epsilon
self.momentum_ = config.momentum
self.l2_decay_ = config.l2_decay
self.rmsprop_decay_ = 0.9
self.gradient_clip_ = config.gradient_clip
self.eps_decay_factor = config.eps_decay_factor
self.eps_decay_after = config.eps_decay_after
def __init__(self, w, config=None):
if type(w) == np.ndarray:
self.w_ = cm.CUDAMatrix(w)
elif type(w) == tuple:
self.w_ = cm.empty(w)
else:
self.w_ = w
self.dw_ = cm.empty_like(self.w_)
self.dw_history_ = cm.empty_like(self.w_)
self.dw_history_sqrt_ = cm.empty_like(self.w_)
self.dw_sqr_ = cm.empty_like(self.w_)
self.dw_history_.assign(0)
self.dw_.assign(0)
self.t_ = 0
if config is None:
pass
elif config.init_type == config_pb2.Param.CONSTANT:
self.w_.assign(config.scale)
elif config.init_type == config_pb2.Param.GAUSSIAN:
self.w_.fill_with_randn()
self.w_.mult(config.scale)
elif config.init_type == config_pb2.Param.UNIFORM:
self.w_.fill_with_rand()
self.w_.subtract(0.5)
self.w_.mult(2 * config.scale)
elif config.init_type == config_pb2.Param.PRETRAINED:
f = h5py.File(config.file_name)
mat = f[config.dataset_name].value
if len(mat.shape) == 1:
mat = mat.reshape(1, -1)
assert self.w_.shape == mat.shape
self.w_.overwrite(mat)
f.close()
else:
raise Exception('Unknown parameter initialization.')
self.eps_ = config.epsilon
self.momentum_ = config.momentum
self.l2_decay_ = config.l2_decay
self.rmsprop_decay_ = 0.9
self.gradient_clip_ = config.gradient_clip
self.eps_decay_factor = config.eps_decay_factor
self.eps_decay_after = config.eps_decay_after
