def create_rbm(self, visible_num, hidden_num, learning_rate):
'''
Build `RestrictedBoltzmmanMachine`.
Args:
visible_num: The number of units in RBM's visible layer.
hidden_num: The number of units in RBM's hidden layer.
learning_rate: Learning rate.
Returns:
`RestrictedBoltzmmanMachine`.
'''
# `Builder` in `Builder Pattern` for LSTM-RTRBM.
rnnrbm_builder = LSTMRTRBMSimpleBuilder()
# Learning rate.
rnnrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rnnrbm_builder.visible_neuron_part(LogisticFunction(), visible_num)
# Set units in hidden layer.
rnnrbm_builder.hidden_neuron_part(LogisticFunction(), hidden_num)
# Set units in RNN layer.
rnnrbm_builder.rnn_neuron_part(TanhFunction())
# Set graph and approximation function, delegating `SGD` which is-a `OptParams`.
rnnrbm_builder.graph_part(LSTMRTRBMCD(opt_params=SGD()))
# Building.
rbm = rnnrbm_builder.get_result()
return rbm
def learn(self,
sentence_list,
token_master_list,
hidden_neuron_count=1000,
training_count=1,
batch_size=100,
learning_rate=1e-03,
seq_len=5):
'''
Init.
Args:
sentence_list: The `list` of sentences.
token_master_list: Unique `list` of tokens.
hidden_neuron_count: The number of units in hidden layer.
training_count: The number of training.
bath_size: Batch size of Mini-batch.
learning_rate: Learning rate.
seq_len: The length of one sequence.
'''
observed_arr = self.__setup_dataset(sentence_list, token_master_list,
seq_len)
visible_num = observed_arr.shape[-1]
# `Builder` in `Builder Pattern` for LSTM-RTRBM.
rnnrbm_builder = LSTMRTRBMSimpleBuilder()
# Learning rate.
rnnrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rnnrbm_builder.visible_neuron_part(LogisticFunction(), visible_num)
# Set units in hidden layer.
rnnrbm_builder.hidden_neuron_part(LogisticFunction(),
hidden_neuron_count)
# Set units in RNN layer.
rnnrbm_builder.rnn_neuron_part(TanhFunction())
# Set graph and approximation function, delegating `SGD` which is-a `OptParams`.
rnnrbm_builder.graph_part(LSTMRTRBMCD(opt_params=SGD()))
# Building.
rbm = rnnrbm_builder.get_result()
# Learning.
rbm.learn(
# The `np.ndarray` of observed data points.
observed_arr,
# Training count.
training_count=training_count,
# Batch size.
batch_size=batch_size)
self.__rbm = rbm
self.__token_master_list = token_master_list
self.__seq_len = seq_len
def __init__(self,
lstm_model=None,
computable_loss=None,
batch_size=20,
input_neuron_count=100,
hidden_neuron_count=300,
observed_activating_function=None,
input_gate_activating_function=None,
forget_gate_activating_function=None,
output_gate_activating_function=None,
hidden_activating_function=None,
output_activating_function=None,
seq_len=10,
join_io_flag=False,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=50):
'''
Init.
Args:
lstm_model: is-a `lstm_model`.
computable_loss: is-a `ComputableLoss`.
batch_size: Batch size.
This parameters will be refered only when `lstm_model` is `None`.
input_neuron_count: The number of input units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_neuron_count: The number of hidden units.
This parameters will be refered only when `lstm_model` is `None`.
observed_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `TanhFunction`.
input_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
forget_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
output_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
hidden_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
output_activating_function: is-a `ActivatingFunctionInterface` in output layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this model outputs from LSTM's hidden layer in inferencing.
seq_len: The length of sequences.
This means refereed maxinum step `t` in feedforward.
join_io_flag: If this value and value of `output_activating_function` is not `None`,
This model outputs tensors combining observed data points and inferenced data
in a series direction.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
'''
if computable_loss is None:
computable_loss = MeanSquaredError()
if lstm_model is not None:
if isinstance(lstm_model, LSTM) is False:
raise TypeError()
else:
# Init.
graph = LSTMGraph()
# Activation function in LSTM.
if observed_activating_function is None:
graph.observed_activating_function = TanhFunction()
else:
if isinstance(observed_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.observed_activating_function = observed_activating_function
if input_gate_activating_function is None:
graph.input_gate_activating_function = LogisticFunction()
else:
if isinstance(input_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.input_gate_activating_function = input_gate_activating_function
if forget_gate_activating_function is None:
graph.forget_gate_activating_function = LogisticFunction()
else:
if isinstance(forget_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.forget_gate_activating_function = forget_gate_activating_function
if output_gate_activating_function is None:
graph.output_gate_activating_function = LogisticFunction()
else:
if isinstance(output_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.output_gate_activating_function = output_gate_activating_function
if hidden_activating_function is None:
graph.hidden_activating_function = TanhFunction()
else:
if isinstance(hidden_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.hidden_activating_function = hidden_activating_function
if output_activating_function is None:
graph.output_activating_function = TanhFunction()
self.__output_flag = False
output_neuron_count = 1
else:
graph.output_activating_function = output_activating_function
self.__output_flag = True
output_neuron_count = hidden_neuron_count
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=output_neuron_count)
opt_params = SGD()
opt_params.weight_limit = 1e+10
opt_params.dropout_rate = 0.0
lstm_model = LSTM(
# Delegate `graph` to `LSTMModel`.
graph=graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=seq_len,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=computable_loss,
# Optimizer.
opt_params=opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
self.__lstm_model = lstm_model
self.__seq_len = seq_len
self.__learning_rate = learning_rate
self.__join_io_flag = join_io_flag
self.__computable_loss = computable_loss
self.__loss_list = []
self.__epoch_counter = 0
self.__learning_attenuate_rate = learning_attenuate_rate
self.__attenuate_epoch = attenuate_epoch
logger = getLogger("pygan")
self.__logger = logger
def __init__(self,
lstm_model=None,
batch_size=20,
input_neuron_count=100,
hidden_neuron_count=300,
hidden_activating_function=None,
seq_len=10,
learning_rate=1e-05,
verbose_mode=False):
'''
Init.
Args:
lstm_model: is-a `lstm_model`.
batch_size: Batch size.
This parameters will be refered only when `lstm_model` is `None`.
input_neuron_count: The number of input units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_neuron_count: The number of hidden units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
seq_len: The length of sequences.
This means refereed maxinum step `t` in feedforward.
learning_rate: Learning rate.
verbose_mode: Verbose mode or not.
'''
logger = getLogger("pydbm")
handler = StreamHandler()
if verbose_mode is True:
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
else:
handler.setLevel(ERROR)
logger.setLevel(ERROR)
logger.addHandler(handler)
if lstm_model is not None:
if isinstance(lstm_model, LSTM) is False:
raise TypeError()
else:
# Init.
graph = LSTMGraph()
# Activation function in LSTM.
graph.observed_activating_function = TanhFunction()
graph.input_gate_activating_function = LogisticFunction()
graph.forget_gate_activating_function = LogisticFunction()
graph.output_gate_activating_function = LogisticFunction()
if hidden_activating_function is None:
graph.hidden_activating_function = TanhFunction()
else:
if isinstance(hidden_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.hidden_activating_function = hidden_activating_function
graph.output_activating_function = TanhFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1)
opt_params = SGD()
opt_params.weight_limit = 0.5
opt_params.dropout_rate = 0.0
lstm_model = LSTM(
# Delegate `graph` to `LSTMModel`.
graph=graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=seq_len,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
self.__lstm_model = lstm_model
self.__seq_len = seq_len
self.__learning_rate = learning_rate
self.__verbose_mode = verbose_mode
self.__loss_list = []