def __init__(self,
learning_rate=0.00001,
hidden_n=100,
hidden_binary_flag=True,
inferancing_training_count=1,
r_batch_size=200):
self.inferancing_training_count = inferancing_training_count
self.r_batch_size = r_batch_size
# `Builder` in `Builder Pattern` for RTRBM.
rtrbm_builder = RTRBMSimpleBuilder()
# Learning rate.
rtrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rtrbm_builder.visible_neuron_part(SoftmaxFunction(), 127)
# Set units in hidden layer.
rtrbm_builder.hidden_neuron_part(
LogisticFunction(normalize_flag=False,
binary_flag=hidden_binary_flag), hidden_n)
# Set units in RNN layer.
rtrbm_builder.rnn_neuron_part(
LogisticFunction(normalize_flag=False, binary_flag=False))
# Set graph and approximation function.
rtrbm_builder.graph_part(RTRBMCD())
# Building.
self.rbm = rtrbm_builder.get_result()
def create_rbm(self, visible_num, hidden_num, learning_rate):
'''
Build `RestrictedBoltzmmanMachine`.
Args:
visible_num: The number of units in RBM's visible layer.
hidden_num: The number of units in RBM's hidden layer.
learning_rate: Learning rate.
Returns:
`RestrictedBoltzmmanMachine`.
'''
# `Builder` in `Builder Pattern` for LSTM-RTRBM.
rnnrbm_builder = LSTMRTRBMSimpleBuilder()
# Learning rate.
rnnrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rnnrbm_builder.visible_neuron_part(LogisticFunction(), visible_num)
# Set units in hidden layer.
rnnrbm_builder.hidden_neuron_part(LogisticFunction(), hidden_num)
# Set units in RNN layer.
rnnrbm_builder.rnn_neuron_part(TanhFunction())
# Set graph and approximation function, delegating `SGD` which is-a `OptParams`.
rnnrbm_builder.graph_part(LSTMRTRBMCD(opt_params=SGD()))
# Building.
rbm = rnnrbm_builder.get_result()
return rbm
def __init__(self, output_layer_flag=False):
'''
Initialize.
'''
if isinstance(output_layer_flag, bool) is False:
raise TypeError()
self.__output_layer_flag = output_layer_flag
self.__logistic_function = LogisticFunction()
def learn(self,
sentence_list,
token_master_list,
hidden_neuron_count=1000,
training_count=1,
batch_size=100,
learning_rate=1e-03,
seq_len=5):
'''
Init.
Args:
sentence_list: The `list` of sentences.
token_master_list: Unique `list` of tokens.
hidden_neuron_count: The number of units in hidden layer.
training_count: The number of training.
bath_size: Batch size of Mini-batch.
learning_rate: Learning rate.
seq_len: The length of one sequence.
'''
observed_arr = self.__setup_dataset(sentence_list, token_master_list,
seq_len)
visible_num = observed_arr.shape[-1]
# `Builder` in `Builder Pattern` for LSTM-RTRBM.
rnnrbm_builder = LSTMRTRBMSimpleBuilder()
# Learning rate.
rnnrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rnnrbm_builder.visible_neuron_part(LogisticFunction(), visible_num)
# Set units in hidden layer.
rnnrbm_builder.hidden_neuron_part(LogisticFunction(),
hidden_neuron_count)
# Set units in RNN layer.
rnnrbm_builder.rnn_neuron_part(TanhFunction())
# Set graph and approximation function, delegating `SGD` which is-a `OptParams`.
rnnrbm_builder.graph_part(LSTMRTRBMCD(opt_params=SGD()))
# Building.
rbm = rnnrbm_builder.get_result()
# Learning.
rbm.learn(
# The `np.ndarray` of observed data points.
observed_arr,
# Training count.
training_count=training_count,
# Batch size.
batch_size=batch_size)
self.__rbm = rbm
self.__token_master_list = token_master_list
self.__seq_len = seq_len
def __build_retrospective_encoder(
self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=0.5,
dropout_rate=0.5,
batch_size=20,
learning_rate=1e-05,
bptt_tau=8
):
encoder_graph = ReEncoderGraph()
encoder_graph.observed_activating_function = TanhFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
encoder_graph.create_rnn_cells(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1
)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = ReEncoder(
graph=encoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
bptt_tau=bptt_tau,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=encoder_opt_params,
verificatable_result=VerificateFunctionApproximation()
)
return encoder
# The `Concrete Builder` in Builder Pattern.
from pydbm.dbm.builders.dbm_multi_layer_builder import DBMMultiLayerBuilder
# Contrastive Divergence for function approximation.
from pydbm.approximation.contrastive_divergence import ContrastiveDivergence
# Logistic Function as activation function.
from pydbm.activation.logistic_function import LogisticFunction
# Observed data points.
observed_arr = np.random.normal(loc=0.5, scale=0.2, size=(10000, 10000))
print("Observed data points:")
print(observed_arr)
# Setting objects for activation function.
activation_list = [
LogisticFunction(binary_flag=False,
normalization_mode="min_max",
normalize_flag=True),
LogisticFunction(binary_flag=False,
normalization_mode="min_max",
normalize_flag=True),
LogisticFunction(binary_flag=False,
normalization_mode="min_max",
normalize_flag=True)
]
# Setting the object for function approximation.
approximaion_list = [ContrastiveDivergence(), ContrastiveDivergence()]
dbm = StackedAutoEncoder(
DBMMultiLayerBuilder(),
[observed_arr.shape[1], 10, observed_arr.shape[1]],
from pprint import pprint
from sklearn.datasets import make_classification
from sklearn.cross_validation import train_test_split
data_tuple = make_classification(n_samples=1000,
n_features=100,
n_informative=10,
n_classes=2,
class_sep=1.0)
data_tuple_x, data_tuple_y = data_tuple
traning_x, test_x, traning_y, test_y = train_test_split(data_tuple_x,
data_tuple_y,
test_size=0.3,
random_state=555)
traning_y = traning_y.reshape(-1, 1)
test_y = test_y.reshape(-1, 1)
nn = NeuralNetwork(
NNMultiLayerBuilder(), [traning_x.shape[1], 10, traning_y.shape[1]],
[LogisticFunction(),
LogisticFunction(),
LogisticFunction()])
nn.learn(traning_x,
traning_y,
traning_count=1000,
learning_rate=0.00001,
momentum_factor=0.00001)
for i in range(10):
pred_arr = nn.predict(test_x[i])
print(pred_arr)
def __init__(self,
lstm_model=None,
computable_loss=None,
batch_size=20,
input_neuron_count=100,
hidden_neuron_count=300,
observed_activating_function=None,
input_gate_activating_function=None,
forget_gate_activating_function=None,
output_gate_activating_function=None,
hidden_activating_function=None,
output_activating_function=None,
seq_len=10,
join_io_flag=False,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=50):
'''
Init.
Args:
lstm_model: is-a `lstm_model`.
computable_loss: is-a `ComputableLoss`.
batch_size: Batch size.
This parameters will be refered only when `lstm_model` is `None`.
input_neuron_count: The number of input units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_neuron_count: The number of hidden units.
This parameters will be refered only when `lstm_model` is `None`.
observed_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `TanhFunction`.
input_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
forget_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
output_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
hidden_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
output_activating_function: is-a `ActivatingFunctionInterface` in output layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this model outputs from LSTM's hidden layer in inferencing.
seq_len: The length of sequences.
This means refereed maxinum step `t` in feedforward.
join_io_flag: If this value and value of `output_activating_function` is not `None`,
This model outputs tensors combining observed data points and inferenced data
in a series direction.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
'''
if computable_loss is None:
computable_loss = MeanSquaredError()
if lstm_model is not None:
if isinstance(lstm_model, LSTM) is False:
raise TypeError()
else:
# Init.
graph = LSTMGraph()
# Activation function in LSTM.
if observed_activating_function is None:
graph.observed_activating_function = TanhFunction()
else:
if isinstance(observed_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.observed_activating_function = observed_activating_function
if input_gate_activating_function is None:
graph.input_gate_activating_function = LogisticFunction()
else:
if isinstance(input_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.input_gate_activating_function = input_gate_activating_function
if forget_gate_activating_function is None:
graph.forget_gate_activating_function = LogisticFunction()
else:
if isinstance(forget_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.forget_gate_activating_function = forget_gate_activating_function
if output_gate_activating_function is None:
graph.output_gate_activating_function = LogisticFunction()
else:
if isinstance(output_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.output_gate_activating_function = output_gate_activating_function
if hidden_activating_function is None:
graph.hidden_activating_function = TanhFunction()
else:
if isinstance(hidden_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.hidden_activating_function = hidden_activating_function
if output_activating_function is None:
graph.output_activating_function = TanhFunction()
self.__output_flag = False
output_neuron_count = 1
else:
graph.output_activating_function = output_activating_function
self.__output_flag = True
output_neuron_count = hidden_neuron_count
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=output_neuron_count)
opt_params = SGD()
opt_params.weight_limit = 1e+10
opt_params.dropout_rate = 0.0
lstm_model = LSTM(
# Delegate `graph` to `LSTMModel`.
graph=graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=seq_len,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=computable_loss,
# Optimizer.
opt_params=opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
self.__lstm_model = lstm_model
self.__seq_len = seq_len
self.__learning_rate = learning_rate
self.__join_io_flag = join_io_flag
self.__computable_loss = computable_loss
self.__loss_list = []
self.__epoch_counter = 0
self.__learning_attenuate_rate = learning_attenuate_rate
self.__attenuate_epoch = attenuate_epoch
logger = getLogger("pygan")
self.__logger = logger
def learn(self,
sentence_list,
token_master_list,
hidden_neuron_count=200,
epochs=100,
batch_size=100,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
bptt_tau=8,
weight_limit=0.5,
dropout_rate=0.5,
test_size_rate=0.3):
'''
Init.
Args:
sentence_list: The `list` of sentences.
token_master_list: Unique `list` of tokens.
hidden_neuron_count: The number of units in hidden layer.
epochs: Epochs of Mini-batch.
bath_size: Batch size of Mini-batch.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
bptt_tau: Refereed maxinum step `t` in Backpropagation Through Time(BPTT).
weight_limit: Regularization for weights matrix
to repeat multiplying the weights matrix and `0.9`
until $\sum_{j=0}^{n}w_{ji}^2 < weight\_limit$.
dropout_rate: The probability of dropout.
test_size_rate: Size of Test data set. If this value is `0`, the
'''
observed_arr = self.__setup_dataset(sentence_list, token_master_list)
self.__logger.debug("Shape of observed data points:")
self.__logger.debug(observed_arr.shape)
# Init.
encoder_graph = EncoderGraph()
# Activation function in LSTM.
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
encoder_graph.create_rnn_cells(
input_neuron_count=observed_arr.shape[-1],
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=observed_arr.shape[-1])
# Init.
decoder_graph = DecoderGraph()
# Activation function in LSTM.
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
decoder_graph.create_rnn_cells(
input_neuron_count=hidden_neuron_count,
hidden_neuron_count=observed_arr.shape[-1],
output_neuron_count=hidden_neuron_count)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
# Delegate `graph` to `LSTMModel`.
graph=encoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=bptt_tau,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=test_size_rate,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=encoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
# Delegate `graph` to `LSTMModel`.
graph=decoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=bptt_tau,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=test_size_rate,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=decoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
# Learning.
encoder_decoder_controller.learn(observed_arr, observed_arr)
self.__controller = encoder_decoder_controller
self.__token_master_list = token_master_list
class NeuralNetworkGraph(Synapse):
'''
The graph for neural networks.
'''
# If True, `Self` consider itself as the neural networks in output layer.
__output_layer_flag = False
# Logistic function.
__logistic_function = None
# Momentum factor.
__momentum_factor_arr = None
def __init__(self, output_layer_flag=False):
'''
Initialize.
'''
if isinstance(output_layer_flag, bool) is False:
raise TypeError()
self.__output_layer_flag = output_layer_flag
self.__logistic_function = LogisticFunction()
def back_propagate(self,
propagated_list,
learning_rate=0.05,
momentum_factor=0.1,
back_nn_list=None,
back_nn_index=0):
'''
Back propagate.
Args:
propagated_list: The list of back propagated feature points.
If this is in output layer, the values of the list are response variable.
learning_rate: Learning rate.
momentum_factor: Momentum factor.
back_nn_list: The list of graph of neural networks.
back_nn_index: The index of graph of neural networks.
Returns:
Tuple(`The back propageted data points`, `The activity`)
'''
if self.__output_layer_flag is True:
if len(self.deeper_neuron_list) != len(propagated_list):
raise IndexError()
diff_list = [
self.deeper_neuron_list[j].activity - propagated_list[j]
for j in range(len(self.deeper_neuron_list))
]
else:
diff_list = propagated_list
diff_list = list(np.nan_to_num(np.array(diff_list)))
diff_arr = np.array([[diff_list[k]] * len(self.shallower_neuron_list)
for k in range(len(diff_list))]).T
if self.__momentum_factor_arr is not None:
momentum_arr = self.__momentum_factor_arr * momentum_factor
else:
momentum_arr = np.ones(diff_arr.shape) * momentum_factor
self.diff_weights_arr = (learning_rate * diff_arr) + momentum_arr
self.__momentum_factor_arr = diff_arr
self.weights_arr = np.nan_to_num(self.weights_arr)
error_arr = diff_arr * self.weights_arr
error_list = error_arr.sum(axis=1)
back_propagated_list = [
self.__logistic_function.derivative(
self.shallower_neuron_list[i].activity) * error_list[i]
for i in range(len(self.shallower_neuron_list))
]
# Normalize.
if len(back_propagated_list) > 1 and sum(back_propagated_list) != 0:
back_propagated_arr = np.array(back_propagated_list)
back_propagated_arr = back_propagated_arr / back_propagated_arr.sum(
)
back_propagated_arr = np.nan_to_num(back_propagated_arr)
back_propagated_list = list(back_propagated_arr)
# Update weights.
self.learn_weights()
[
self.shallower_neuron_list[_i].update_bias(learning_rate)
for _i in range(len(self.shallower_neuron_list))
]
[
self.deeper_neuron_list[_j].update_bias(learning_rate)
for _j in range(len(self.deeper_neuron_list))
]
# Recursive.
if back_nn_list is not None:
if back_nn_index < len(back_nn_list) - 1:
back_nn_list[back_nn_index + 1].back_propagate(
propagated_list=back_propagated_list,
learning_rate=learning_rate,
momentum_factor=momentum_factor,
back_nn_list=back_nn_list,
back_nn_index=back_nn_index + 1)
from pydbm.dbm.deepboltzmannmachine.stacked_auto_encoder import StackedAutoEncoder
# The `Concrete Builder` in Builder Pattern.
from pydbm.dbm.builders.dbm_multi_layer_builder import DBMMultiLayerBuilder
# Contrastive Divergence for function approximation.
from pydbm.approximation.contrastive_divergence import ContrastiveDivergence
# Logistic Function as activation function.
from pydbm.activation.logistic_function import LogisticFunction
# Observed data points.
observed_arr = np.random.normal(loc=0.5, scale=0.2, size=(10000, 10000))
print("Observed data points:")
print(observed_arr)
# Setting objects for activation function.
activation_list = [
LogisticFunction(binary_flag=False, normalize_flag=False),
LogisticFunction(binary_flag=False, normalize_flag=False),
LogisticFunction(binary_flag=False, normalize_flag=False)
]
# Setting the object for function approximation.
approximaion_list = [ContrastiveDivergence(), ContrastiveDivergence()]
dbm = StackedAutoEncoder(
DBMMultiLayerBuilder(),
[observed_arr.shape[1], 10, observed_arr.shape[1]],
activation_list,
approximaion_list,
1e-05, # Setting learning rate.
0.5 # Setting dropout rate.
)
def __build_encoder_decoder_controller(
self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=0.5,
dropout_rate=0.5,
epochs=1000,
batch_size=20,
learning_rate=1e-05,
attenuate_epoch=50,
learning_attenuate_rate=0.1,
seq_len=8,
bptt_tau=8,
test_size_rate=0.3,
tol=1e-10,
tld=100.0
):
encoder_graph = EncoderGraph()
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
encoder_graph.create_rnn_cells(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1
)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
graph=encoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
bptt_tau=8,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=encoder_opt_params,
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld
)
decoder_graph = DecoderGraph()
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = SoftmaxFunction()
decoder_graph.create_rnn_cells(
input_neuron_count=hidden_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=input_neuron_count
)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
graph=decoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
seq_len=seq_len,
bptt_tau=bptt_tau,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=decoder_opt_params,
verificatable_result=VerificateFunctionApproximation()
)
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld
)
return encoder_decoder_controller
def Main(params_dict):
logger = getLogger("pydbm")
handler = StreamHandler()
if params_dict["debug_mode"] is True:
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
else:
handler.setLevel(ERROR)
logger.setLevel(ERROR)
logger.addHandler(handler)
epochs = params_dict["epochs"]
batch_size = params_dict["batch_size"]
seq_len = params_dict["seq_len"]
channel = params_dict["channel"]
height = params_dict["height"]
width = params_dict["width"]
scale = params_dict["scale"]
training_image_dir = params_dict["training_image_dir"]
test_image_dir = params_dict["test_image_dir"]
enc_dim = batch_size * height * width
dec_dim = batch_size * height * width
feature_generator = ImageGenerator(epochs=epochs,
batch_size=batch_size,
training_image_dir=training_image_dir,
test_image_dir=test_image_dir,
seq_len=seq_len,
gray_scale_flag=True,
wh_size_tuple=(height, width),
norm_mode="z_score")
# Init.
encoder_graph = EncoderGraph()
# Activation function in LSTM.
encoder_graph.observed_activating_function = TanhFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = TanhFunction()
encoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
encoder_graph.create_rnn_cells(input_neuron_count=enc_dim,
hidden_neuron_count=200,
output_neuron_count=enc_dim)
# Optimizer for Encoder.
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = 0.5
encoder_opt_params.dropout_rate = 0.5
encoder = Encoder(
# Delegate `graph` to `LSTMModel`.
graph=encoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=8,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=encoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
# Tolerance for the optimization.
# When the loss or score is not improving by at least tol
# for two consecutive iterations, convergence is considered
# to be reached and training stops.
tol=0.0)
# Init.
decoder_graph = DecoderGraph()
# Activation function in LSTM.
decoder_graph.observed_activating_function = TanhFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = TanhFunction()
decoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
decoder_graph.create_rnn_cells(input_neuron_count=200,
hidden_neuron_count=dec_dim,
output_neuron_count=200)
# Optimizer for Decoder.
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = 0.5
decoder_opt_params.dropout_rate = 0.5
decoder = Decoder(
# Delegate `graph` to `LSTMModel`.
graph=decoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=8,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=decoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
# Tolerance for the optimization.
# When the loss or score is not improving by at least tol
# for two consecutive iterations, convergence is considered
# to be reached and training stops.
tol=0.0)
conv1 = ConvolutionLayer1(
ConvGraph1(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
conv2 = ConvolutionLayer2(
ConvGraph2(activation_function=TanhFunction(),
filter_num=batch_size,
channel=batch_size,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
cnn = SpatioTemporalAutoEncoder(
layerable_cnn_list=[conv1, conv2],
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=25,
computable_loss=MeanSquaredError(),
opt_params=Adam(),
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15,
save_flag=False)
cnn.learn_generated(feature_generator)
test_len = 0
test_limit = 1
test_arr_list = []
rec_arr_list = []
for batch_observed_arr, batch_target_arr, test_batch_observed_arr, test_batch_target_arr in feature_generator.generate(
):
test_len += 1
result_arr = cnn.inference(test_batch_observed_arr)
for batch in range(test_batch_target_arr.shape[0]):
for seq in range(test_batch_target_arr[batch].shape[0]):
arr = test_batch_target_arr[batch][seq][0]
arr = (arr - arr.min()) / (arr.max() - arr.min())
arr *= 255
img = Image.fromarray(np.uint8(arr))
img.save("result/" + str(i) + "_" + str(seq) + "_observed.png")
for seq in range(result_arr[batch].shape[0]):
arr = result_arr[batch][seq][0]
arr = (arr - arr.min()) / (arr.max() - arr.min())
arr *= 255
img = Image.fromarray(np.uint8(arr))
img.save("result/" + str(i) + "_" + str(seq) +
"_reconsturcted.png")
if test_len >= test_limit:
break
from pprint import pprint
from sklearn.datasets import make_classification
from sklearn.cross_validation import train_test_split
if __name__ == "__main__":
'''
'''
data_tuple = make_classification(n_samples=20000,
n_features=1000,
n_informative=5,
n_classes=5,
class_sep=1.0,
scale=0.1)
data_tuple_x, data_tuple_y = data_tuple
traning_x, test_x, traning_y, test_y = train_test_split(data_tuple_x,
data_tuple_y,
test_size=0.5,
random_state=888)
dbm = StackedAutoEncoder(DBMMultiLayerBuilder(),
[traning_x.shape[1], 10, traning_x.shape[1]],
LogisticFunction(), ContrastiveDivergence(), 0.05)
dbm.learn(traning_x, traning_count=1)
import pandas as pd
feature_points_df = pd.DataFrame(dbm.feature_points_arr)
print(feature_points_df.shape)
print(feature_points_df.head())
print("-" * 100)
print(feature_points_df.tail())
def __init__(self,
lstm_model=None,
batch_size=20,
input_neuron_count=100,
hidden_neuron_count=300,
hidden_activating_function=None,
seq_len=10,
learning_rate=1e-05,
verbose_mode=False):
'''
Init.
Args:
lstm_model: is-a `lstm_model`.
batch_size: Batch size.
This parameters will be refered only when `lstm_model` is `None`.
input_neuron_count: The number of input units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_neuron_count: The number of hidden units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
seq_len: The length of sequences.
This means refereed maxinum step `t` in feedforward.
learning_rate: Learning rate.
verbose_mode: Verbose mode or not.
'''
logger = getLogger("pydbm")
handler = StreamHandler()
if verbose_mode is True:
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
else:
handler.setLevel(ERROR)
logger.setLevel(ERROR)
logger.addHandler(handler)
if lstm_model is not None:
if isinstance(lstm_model, LSTM) is False:
raise TypeError()
else:
# Init.
graph = LSTMGraph()
# Activation function in LSTM.
graph.observed_activating_function = TanhFunction()
graph.input_gate_activating_function = LogisticFunction()
graph.forget_gate_activating_function = LogisticFunction()
graph.output_gate_activating_function = LogisticFunction()
if hidden_activating_function is None:
graph.hidden_activating_function = TanhFunction()
else:
if isinstance(hidden_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.hidden_activating_function = hidden_activating_function
graph.output_activating_function = TanhFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1)
opt_params = SGD()
opt_params.weight_limit = 0.5
opt_params.dropout_rate = 0.0
lstm_model = LSTM(
# Delegate `graph` to `LSTMModel`.
graph=graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=seq_len,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
self.__lstm_model = lstm_model
self.__seq_len = seq_len
self.__learning_rate = learning_rate
self.__verbose_mode = verbose_mode
self.__loss_list = []
__y = y + _y
if __x < 0 or __y < 0:
vector = 0
else:
try:
vector = map_arr[__y][__x]
except IndexError:
vector = 0
vector_list.append(vector)
vector_list_list.append(vector_list)
vector_arr = np.array(vector_list_list)
vector_arr = vector_arr.astype(float)
dbm = StackedAutoEncoder(
DBMMultiLayerBuilder(),
[vector_arr.shape[1], vector_arr.shape[1], 10],
LogisticFunction(),
ContrastiveDivergence(),
0.005
)
dbm.learn(vector_arr, traning_count=1)
feature_arr = dbm.feature_points_arr
feature_arr = feature_arr[:, 0]
feature_map_arr = feature_arr.reshape(map_d, map_d)
map_arr = map_arr.astype(object)
map_arr[:, 0] = wall_label
map_arr[0, :] = wall_label
map_arr[:, -1] = wall_label
map_arr[-1, :] = wall_label
map_arr[1][1] = start_point_label
map_arr[map_d - 2][map_d - 2] = end_point_label
def __build_encoder_decoder_controller(self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=0.5,
dropout_rate=0.5,
epochs=1000,
batch_size=20,
learning_rate=1e-05,
attenuate_epoch=50,
learning_attenuate_rate=0.1,
seq_len=8,
bptt_tau=8,
test_size_rate=0.3,
tol=1e-10,
tld=100.0):
# Init.
encoder_graph = EncoderGraph()
# Activation function in LSTM.
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
encoder_graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
# Delegate `graph` to `LSTMModel`.
graph=encoder_graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=8,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=encoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation())
# Init.
decoder_graph = DecoderGraph()
# Activation function in LSTM.
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = SoftmaxFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
decoder_graph.create_rnn_cells(input_neuron_count=hidden_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=input_neuron_count)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
# Delegate `graph` to `LSTMModel`.
graph=decoder_graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=bptt_tau,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=decoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation())
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld)
return encoder_decoder_controller
def __init__(self,
midi_path_list,
batch_size=20,
seq_len=8,
time_fraction=1.0,
learning_rate=1e-10,
hidden_dim=15200,
generative_model=None,
discriminative_model=None,
gans_value_function=None):
'''
Init.
Args:
midi_path_list: `list` of paths to MIDI files.
batch_size: Batch size.
seq_len: The length of sequence that LSTM networks will observe.
time_fraction: Time fraction or time resolution (seconds).
learning_rate: Learning rate in `Generator` and `Discriminator`.
hidden_dim: The number of units in hidden layer of `DiscriminativeModel`.
true_sampler: is-a `TrueSampler`.
noise_sampler: is-a `NoiseSampler`.
generative_model: is-a `GenerativeModel`.
discriminative_model: is-a `DiscriminativeModel`.
gans_value_function: is-a `GANsValueFunction`.
'''
self.__midi_controller = MidiController()
self.__midi_df_list = [
self.__midi_controller.extract(midi_path)
for midi_path in midi_path_list
]
bar_gram = BarGram(midi_df_list=self.__midi_df_list,
time_fraction=time_fraction)
self.__bar_gram = bar_gram
dim = self.__bar_gram.dim
true_sampler = BarGramTrueSampler(bar_gram=bar_gram,
midi_df_list=self.__midi_df_list,
batch_size=batch_size,
seq_len=seq_len,
time_fraction=time_fraction)
noise_sampler = BarGramNoiseSampler(bar_gram=bar_gram,
midi_df_list=self.__midi_df_list,
batch_size=batch_size,
seq_len=seq_len,
time_fraction=time_fraction)
if generative_model is None:
conv_activation_function = TanhFunction()
conv_activation_function.batch_norm = BatchNorm()
channel = noise_sampler.channel
convolution_layer_list = [
ConvolutionLayer1(
ConvGraph1(activation_function=conv_activation_function,
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=0.01,
stride=1,
pad=1))
]
deconv_activation_function = SoftmaxFunction()
deconvolution_layer_list = [
DeconvolutionLayer(
DeCNNGraph(activation_function=deconv_activation_function,
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=0.01,
stride=1,
pad=1))
]
opt_params_deconv = Adam()
deconvolution_model = DeconvolutionModel(
deconvolution_layer_list=deconvolution_layer_list,
opt_params=opt_params_deconv)
opt_params = Adam()
opt_params.dropout_rate = 0.0
generative_model = Generator(
batch_size=batch_size,
layerable_cnn_list=convolution_layer_list,
deconvolution_model=deconvolution_model,
condition_noise_sampler=UniformNoiseSampler(
low=-0.1,
high=0.1,
output_shape=(batch_size, channel, seq_len, dim)),
learning_rate=learning_rate,
)
generative_model.noise_sampler = noise_sampler
if discriminative_model is None:
activation_function = LogisticFunction()
activation_function.batch_norm = BatchNorm()
# First convolution layer.
conv2 = ConvolutionLayer2(
# Computation graph for first convolution layer.
ConvGraph2(
# Logistic function as activation function.
activation_function=activation_function,
# The number of `filter`.
filter_num=batch_size,
# The number of channel.
channel=noise_sampler.channel * 2,
# The size of kernel.
kernel_size=3,
# The filter scale.
scale=0.001,
# The nubmer of stride.
stride=1,
# The number of zero-padding.
pad=1))
# Stack.
layerable_cnn_list = [conv2]
opt_params = Adam()
opt_params.dropout_rate = 0.0
if hidden_dim is None:
hidden_dim = channel * seq_len * dim
cnn_output_activating_function = LogisticFunction()
cnn_output_graph = CNNOutputGraph(
hidden_dim=hidden_dim,
output_dim=1,
activating_function=cnn_output_activating_function,
scale=0.01)
discriminative_model = Discriminator(
batch_size=batch_size,
layerable_cnn_list=layerable_cnn_list,
cnn_output_graph=cnn_output_graph,
opt_params=opt_params,
computable_loss=CrossEntropy(),
learning_rate=learning_rate)
if gans_value_function is None:
gans_value_function = MiniMax()
GAN = GenerativeAdversarialNetworks(
gans_value_function=gans_value_function,
feature_matching=FeatureMatching(lambda1=0.01, lambda2=0.99))
self.__noise_sampler = noise_sampler
self.__true_sampler = true_sampler
self.__generative_model = generative_model
self.__discriminative_model = discriminative_model
self.__GAN = GAN
self.__time_fraction = time_fraction
np.seterr(all='raise')
if __name__ == "__main__":
'''
'''
arr = np.arange(333, dtype=np.float64)
arr = np.c_[arr + 0, arr + 1, arr + 2, arr + 3, arr + 4, arr + 5, arr + 6,
arr + 7]
arr = np.array(arr.tolist() * 100)
arr = arr / 333
print(arr.shape)
rtrbm_builder = RTRBMSimpleBuilder()
rtrbm_builder.learning_rate = 0.00001
rtrbm_builder.visible_neuron_part(LogisticFunction(), arr.shape[1])
rtrbm_builder.hidden_neuron_part(LogisticFunction(), 3)
rtrbm_builder.rnn_neuron_part(LogisticFunction())
rtrbm_builder.graph_part(RTRBMCD())
rbm = rtrbm_builder.get_result()
for i in range(arr.shape[0]):
rbm.approximate_learning(arr[i], traning_count=1, batch_size=200)
test_arr = arr[0]
result_list = [None] * arr.shape[0]
for i in range(arr.shape[0]):
rbm.approximate_inferencing(test_arr, traning_count=1, r_batch_size=-1)
result_list[i] = test_arr = rbm.graph.visible_activity_arr
print(rbm.graph.visible_activity_arr)