def __build_encoder_decoder_controller(
self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=0.5,
dropout_rate=0.5,
epochs=1000,
batch_size=20,
learning_rate=1e-05,
attenuate_epoch=50,
learning_attenuate_rate=0.1,
seq_len=8,
bptt_tau=8,
test_size_rate=0.3,
tol=1e-10,
tld=100.0
):
encoder_graph = EncoderGraph()
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
encoder_graph.create_rnn_cells(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1
)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
graph=encoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
bptt_tau=8,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=encoder_opt_params,
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld
)
decoder_graph = DecoderGraph()
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = SoftmaxFunction()
decoder_graph.create_rnn_cells(
input_neuron_count=hidden_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=input_neuron_count
)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
graph=decoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
seq_len=seq_len,
bptt_tau=bptt_tau,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=decoder_opt_params,
verificatable_result=VerificateFunctionApproximation()
)
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld
)
return encoder_decoder_controller
def learn(self,
sentence_list,
token_master_list,
hidden_neuron_count=200,
epochs=100,
batch_size=100,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
bptt_tau=8,
weight_limit=0.5,
dropout_rate=0.5,
test_size_rate=0.3):
'''
Init.
Args:
sentence_list: The `list` of sentences.
token_master_list: Unique `list` of tokens.
hidden_neuron_count: The number of units in hidden layer.
epochs: Epochs of Mini-batch.
bath_size: Batch size of Mini-batch.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
bptt_tau: Refereed maxinum step `t` in Backpropagation Through Time(BPTT).
weight_limit: Regularization for weights matrix
to repeat multiplying the weights matrix and `0.9`
until $\sum_{j=0}^{n}w_{ji}^2 < weight\_limit$.
dropout_rate: The probability of dropout.
test_size_rate: Size of Test data set. If this value is `0`, the
'''
observed_arr = self.__setup_dataset(sentence_list, token_master_list)
self.__logger.debug("Shape of observed data points:")
self.__logger.debug(observed_arr.shape)
# Init.
encoder_graph = EncoderGraph()
# Activation function in LSTM.
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
encoder_graph.create_rnn_cells(
input_neuron_count=observed_arr.shape[-1],
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=observed_arr.shape[-1])
# Init.
decoder_graph = DecoderGraph()
# Activation function in LSTM.
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
decoder_graph.create_rnn_cells(
input_neuron_count=hidden_neuron_count,
hidden_neuron_count=observed_arr.shape[-1],
output_neuron_count=hidden_neuron_count)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
# Delegate `graph` to `LSTMModel`.
graph=encoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=bptt_tau,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=test_size_rate,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=encoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
# Delegate `graph` to `LSTMModel`.
graph=decoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=bptt_tau,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=test_size_rate,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=decoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
# Learning.
encoder_decoder_controller.learn(observed_arr, observed_arr)
self.__controller = encoder_decoder_controller
self.__token_master_list = token_master_list
def __build_encoder_decoder_controller(self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=0.5,
dropout_rate=0.5,
epochs=1000,
batch_size=20,
learning_rate=1e-05,
attenuate_epoch=50,
learning_attenuate_rate=0.1,
seq_len=8,
bptt_tau=8,
test_size_rate=0.3,
tol=1e-10,
tld=100.0):
# Init.
encoder_graph = EncoderGraph()
# Activation function in LSTM.
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
encoder_graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
# Delegate `graph` to `LSTMModel`.
graph=encoder_graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=8,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=encoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation())
# Init.
decoder_graph = DecoderGraph()
# Activation function in LSTM.
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = SoftmaxFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
decoder_graph.create_rnn_cells(input_neuron_count=hidden_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=input_neuron_count)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
# Delegate `graph` to `LSTMModel`.
graph=decoder_graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=bptt_tau,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=decoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation())
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld)
return encoder_decoder_controller