def __init__(self, w, config=None): if type(w) == np.ndarray: self.w_ = cm.CUDAMatrix(w) elif type(w) == tuple: self.w_ = cm.empty(w) else: self.w_ = w self.dw_ = cm.empty_like(self.w_) self.dw_history_ = cm.empty_like(self.w_) self.dw_history_.assign(0) self.dw_.assign(0) self.t_ = 0 self.rms_prop_ = config.rms_prop self.rms_prop_factor_ = config.rms_prop_factor if self.rms_prop_: self.rms_prop_history_ = cm.empty_like(self.dw_) self.rms_prop_history_.assign(1) if config is None: pass elif config.init_type == config_pb2.Param.CONSTANT: self.w_.assign(config.scale) elif config.init_type == config_pb2.Param.GAUSSIAN: self.w_.fill_with_randn() self.w_.mult(config.scale) elif config.init_type == config_pb2.Param.UNIFORM: self.w_.fill_with_rand() self.w_.subtract(0.5) self.w_.mult(2 * config.scale) elif config.init_type == config_pb2.Param.LSTM_BIAS: init_bias = [ config.input_gate_bias, config.forget_gate_bias, config.input_bias, config.output_gate_bias ] self.w_.reshape((-1, 4)) for i in xrange(4): self.w_.slice(i, (i + 1)).assign(init_bias[i]) self.w_.reshape((-1, 1)) elif config.init_type == config_pb2.Param.PRETRAINED: f = h5py.File(config.file_name) mat = f[config.dataset_name].value if len(mat.shape) == 1: mat = mat.reshape(1, -1) assert self.w_.shape == mat.shape self.w_.overwrite(mat) f.close() else: raise Exception('Unknown parameter initialization.') self.eps_ = config.epsilon self.momentum_ = config.momentum self.l2_decay_ = config.l2_decay self.gradient_clip_ = config.gradient_clip self.eps_decay_factor = config.eps_decay_factor self.eps_decay_after = config.eps_decay_after
def __init__(self, w, config=None): if type(w) == np.ndarray: self.w_ = cm.CUDAMatrix(w) elif type(w) == tuple: self.w_ = cm.empty(w) else: self.w_ = w self.dw_ = cm.empty_like(self.w_) self.dw_history_ = cm.empty_like(self.w_) self.dw_history_.assign(0) self.dw_.assign(0) self.t_ = 0 self.rms_prop_ = config.rms_prop self.rms_prop_factor_ = config.rms_prop_factor if self.rms_prop_: self.rms_prop_history_ = cm.empty_like(self.dw_) self.rms_prop_history_.assign(1) if config is None: pass elif config.init_type == config_pb2.Param.CONSTANT: self.w_.assign(config.scale) elif config.init_type == config_pb2.Param.GAUSSIAN: self.w_.fill_with_randn() self.w_.mult(config.scale) elif config.init_type == config_pb2.Param.UNIFORM: self.w_.fill_with_rand() self.w_.subtract(0.5) self.w_.mult(2 * config.scale) elif config.init_type == config_pb2.Param.LSTM_BIAS: init_bias = [config.input_gate_bias, config.forget_gate_bias, config.input_bias, config.output_gate_bias] self.w_.reshape((-1, 4)) for i in xrange(4): self.w_.slice(i, (i+1)).assign(init_bias[i]) self.w_.reshape((-1, 1)) elif config.init_type == config_pb2.Param.PRETRAINED: f = h5py.File(config.file_name) mat = f[config.dataset_name].value if len(mat.shape) == 1: mat = mat.reshape(1, -1) assert self.w_.shape == mat.shape self.w_.overwrite(mat) f.close() else: raise Exception('Unknown parameter initialization.') self.eps_ = config.epsilon self.momentum_ = config.momentum self.l2_decay_ = config.l2_decay self.gradient_clip_ = config.gradient_clip self.eps_decay_factor = config.eps_decay_factor self.eps_decay_after = config.eps_decay_after
def __init__(self, w, config=None): if type(w) == np.ndarray: self.w_ = cm.CUDAMatrix(w) elif type(w) == tuple: self.w_ = cm.empty(w) else: self.w_ = w self.dw_ = cm.empty_like(self.w_) self.dw_history_ = cm.empty_like(self.w_) self.dw_history_sqrt_ = cm.empty_like(self.w_) self.dw_sqr_ = cm.empty_like(self.w_) self.dw_history_.assign(0) self.dw_.assign(0) self.t_ = 0 if config is None: pass elif config.init_type == config_pb2.Param.CONSTANT: self.w_.assign(config.scale) elif config.init_type == config_pb2.Param.GAUSSIAN: self.w_.fill_with_randn() self.w_.mult(config.scale) elif config.init_type == config_pb2.Param.UNIFORM: self.w_.fill_with_rand() self.w_.subtract(0.5) self.w_.mult(2 * config.scale) elif config.init_type == config_pb2.Param.PRETRAINED: f = h5py.File(config.file_name) mat = f[config.dataset_name].value if len(mat.shape) == 1: mat = mat.reshape(1, -1) assert self.w_.shape == mat.shape self.w_.overwrite(mat) f.close() else: raise Exception('Unknown parameter initialization.') self.eps_ = config.epsilon self.momentum_ = config.momentum self.l2_decay_ = config.l2_decay self.rmsprop_decay_ = 0.9 self.gradient_clip_ = config.gradient_clip self.eps_decay_factor = config.eps_decay_factor self.eps_decay_after = config.eps_decay_after