def create_rbm(self, visible_num, hidden_num, learning_rate):
'''
Build `RestrictedBoltzmmanMachine`.
Args:
visible_num: The number of units in RBM's visible layer.
hidden_num: The number of units in RBM's hidden layer.
learning_rate: Learning rate.
Returns:
`RestrictedBoltzmmanMachine`.
'''
# `Builder` in `Builder Pattern` for LSTM-RTRBM.
rnnrbm_builder = LSTMRTRBMSimpleBuilder()
# Learning rate.
rnnrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rnnrbm_builder.visible_neuron_part(LogisticFunction(), visible_num)
# Set units in hidden layer.
rnnrbm_builder.hidden_neuron_part(LogisticFunction(), hidden_num)
# Set units in RNN layer.
rnnrbm_builder.rnn_neuron_part(TanhFunction())
# Set graph and approximation function, delegating `SGD` which is-a `OptParams`.
rnnrbm_builder.graph_part(LSTMRTRBMCD(opt_params=SGD()))
# Building.
rbm = rnnrbm_builder.get_result()
return rbm
def learn(self,
sentence_list,
token_master_list,
hidden_neuron_count=1000,
training_count=1,
batch_size=100,
learning_rate=1e-03,
seq_len=5):
'''
Init.
Args:
sentence_list: The `list` of sentences.
token_master_list: Unique `list` of tokens.
hidden_neuron_count: The number of units in hidden layer.
training_count: The number of training.
bath_size: Batch size of Mini-batch.
learning_rate: Learning rate.
seq_len: The length of one sequence.
'''
observed_arr = self.__setup_dataset(sentence_list, token_master_list,
seq_len)
visible_num = observed_arr.shape[-1]
# `Builder` in `Builder Pattern` for LSTM-RTRBM.
rnnrbm_builder = LSTMRTRBMSimpleBuilder()
# Learning rate.
rnnrbm_builder.learning_rate = learning_rate
# Set units in visible layer.
rnnrbm_builder.visible_neuron_part(LogisticFunction(), visible_num)
# Set units in hidden layer.
rnnrbm_builder.hidden_neuron_part(LogisticFunction(),
hidden_neuron_count)
# Set units in RNN layer.
rnnrbm_builder.rnn_neuron_part(TanhFunction())
# Set graph and approximation function, delegating `SGD` which is-a `OptParams`.
rnnrbm_builder.graph_part(LSTMRTRBMCD(opt_params=SGD()))
# Building.
rbm = rnnrbm_builder.get_result()
# Learning.
rbm.learn(
# The `np.ndarray` of observed data points.
observed_arr,
# Training count.
training_count=training_count,
# Batch size.
batch_size=batch_size)
self.__rbm = rbm
self.__token_master_list = token_master_list
self.__seq_len = seq_len
def __build_retrospective_encoder(
self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=0.5,
dropout_rate=0.5,
batch_size=20,
learning_rate=1e-05,
bptt_tau=8
):
encoder_graph = ReEncoderGraph()
encoder_graph.observed_activating_function = TanhFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
encoder_graph.create_rnn_cells(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1
)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = ReEncoder(
graph=encoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
bptt_tau=bptt_tau,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=encoder_opt_params,
verificatable_result=VerificateFunctionApproximation()
)
return encoder
def __init__(self,
batch_size,
image_dir,
seq_len=None,
gray_scale_flag=True,
wh_size_tuple=(100, 100),
norm_mode="z_score"):
'''
Init.
Args:
training_image_dir: Dir path which stores image files for training.
test_image_dir: Dir path which stores image files for test.
seq_len: The length of one sequence.
gray_scale_flag: Gray scale or not(RGB).
wh_size_tuple: Tuple(`width`, `height`).
norm_mode: How to normalize pixel values of images.
- `z_score`: Z-Score normalization.
- `min_max`: Min-max normalization.
- `tanh`: Normalization by tanh function.
'''
super().__init__(batch_size=batch_size,
image_dir=image_dir,
seq_len=seq_len,
gray_scale_flag=gray_scale_flag,
wh_size_tuple=wh_size_tuple,
norm_mode=norm_mode)
if gray_scale_flag is True:
channel = 1
else:
channel = 3
self.__conv_layer = ConvolutionLayer(
CNNGraph(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=0.1,
stride=1,
pad=1))
def __init__(self,
midi_path_list,
batch_size=20,
seq_len=8,
time_fraction=1.0,
learning_rate=1e-10,
hidden_dim=15200,
generative_model=None,
discriminative_model=None,
gans_value_function=None):
'''
Init.
Args:
midi_path_list: `list` of paths to MIDI files.
batch_size: Batch size.
seq_len: The length of sequence that LSTM networks will observe.
time_fraction: Time fraction or time resolution (seconds).
learning_rate: Learning rate in `Generator` and `Discriminator`.
hidden_dim: The number of units in hidden layer of `DiscriminativeModel`.
true_sampler: is-a `TrueSampler`.
noise_sampler: is-a `NoiseSampler`.
generative_model: is-a `GenerativeModel`.
discriminative_model: is-a `DiscriminativeModel`.
gans_value_function: is-a `GANsValueFunction`.
'''
self.__midi_controller = MidiController()
self.__midi_df_list = [
self.__midi_controller.extract(midi_path)
for midi_path in midi_path_list
]
bar_gram = BarGram(midi_df_list=self.__midi_df_list,
time_fraction=time_fraction)
self.__bar_gram = bar_gram
dim = self.__bar_gram.dim
true_sampler = BarGramTrueSampler(bar_gram=bar_gram,
midi_df_list=self.__midi_df_list,
batch_size=batch_size,
seq_len=seq_len,
time_fraction=time_fraction)
noise_sampler = BarGramNoiseSampler(bar_gram=bar_gram,
midi_df_list=self.__midi_df_list,
batch_size=batch_size,
seq_len=seq_len,
time_fraction=time_fraction)
if generative_model is None:
conv_activation_function = TanhFunction()
conv_activation_function.batch_norm = BatchNorm()
channel = noise_sampler.channel
convolution_layer_list = [
ConvolutionLayer1(
ConvGraph1(activation_function=conv_activation_function,
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=0.01,
stride=1,
pad=1))
]
deconv_activation_function = SoftmaxFunction()
deconvolution_layer_list = [
DeconvolutionLayer(
DeCNNGraph(activation_function=deconv_activation_function,
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=0.01,
stride=1,
pad=1))
]
opt_params_deconv = Adam()
deconvolution_model = DeconvolutionModel(
deconvolution_layer_list=deconvolution_layer_list,
opt_params=opt_params_deconv)
opt_params = Adam()
opt_params.dropout_rate = 0.0
generative_model = Generator(
batch_size=batch_size,
layerable_cnn_list=convolution_layer_list,
deconvolution_model=deconvolution_model,
condition_noise_sampler=UniformNoiseSampler(
low=-0.1,
high=0.1,
output_shape=(batch_size, channel, seq_len, dim)),
learning_rate=learning_rate,
)
generative_model.noise_sampler = noise_sampler
if discriminative_model is None:
activation_function = LogisticFunction()
activation_function.batch_norm = BatchNorm()
# First convolution layer.
conv2 = ConvolutionLayer2(
# Computation graph for first convolution layer.
ConvGraph2(
# Logistic function as activation function.
activation_function=activation_function,
# The number of `filter`.
filter_num=batch_size,
# The number of channel.
channel=noise_sampler.channel * 2,
# The size of kernel.
kernel_size=3,
# The filter scale.
scale=0.001,
# The nubmer of stride.
stride=1,
# The number of zero-padding.
pad=1))
# Stack.
layerable_cnn_list = [conv2]
opt_params = Adam()
opt_params.dropout_rate = 0.0
if hidden_dim is None:
hidden_dim = channel * seq_len * dim
cnn_output_activating_function = LogisticFunction()
cnn_output_graph = CNNOutputGraph(
hidden_dim=hidden_dim,
output_dim=1,
activating_function=cnn_output_activating_function,
scale=0.01)
discriminative_model = Discriminator(
batch_size=batch_size,
layerable_cnn_list=layerable_cnn_list,
cnn_output_graph=cnn_output_graph,
opt_params=opt_params,
computable_loss=CrossEntropy(),
learning_rate=learning_rate)
if gans_value_function is None:
gans_value_function = MiniMax()
GAN = GenerativeAdversarialNetworks(
gans_value_function=gans_value_function,
feature_matching=FeatureMatching(lambda1=0.01, lambda2=0.99))
self.__noise_sampler = noise_sampler
self.__true_sampler = true_sampler
self.__generative_model = generative_model
self.__discriminative_model = discriminative_model
self.__GAN = GAN
self.__time_fraction = time_fraction
def Main(params_dict):
logger = getLogger("pydbm")
handler = StreamHandler()
if params_dict["debug_mode"] is True:
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
else:
handler.setLevel(ERROR)
logger.setLevel(ERROR)
logger.addHandler(handler)
epochs = params_dict["epochs"]
batch_size = params_dict["batch_size"]
seq_len = params_dict["seq_len"]
channel = params_dict["channel"]
height = params_dict["height"]
width = params_dict["width"]
scale = params_dict["scale"]
training_image_dir = params_dict["training_image_dir"]
test_image_dir = params_dict["test_image_dir"]
enc_dim = batch_size * height * width
dec_dim = batch_size * height * width
feature_generator = ImageGenerator(epochs=epochs,
batch_size=batch_size,
training_image_dir=training_image_dir,
test_image_dir=test_image_dir,
seq_len=seq_len,
gray_scale_flag=True,
wh_size_tuple=(height, width),
norm_mode="z_score")
# Init.
encoder_graph = EncoderGraph()
# Activation function in LSTM.
encoder_graph.observed_activating_function = TanhFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = TanhFunction()
encoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
encoder_graph.create_rnn_cells(input_neuron_count=enc_dim,
hidden_neuron_count=200,
output_neuron_count=enc_dim)
# Optimizer for Encoder.
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = 0.5
encoder_opt_params.dropout_rate = 0.5
encoder = Encoder(
# Delegate `graph` to `LSTMModel`.
graph=encoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=8,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=encoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
# Tolerance for the optimization.
# When the loss or score is not improving by at least tol
# for two consecutive iterations, convergence is considered
# to be reached and training stops.
tol=0.0)
# Init.
decoder_graph = DecoderGraph()
# Activation function in LSTM.
decoder_graph.observed_activating_function = TanhFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = TanhFunction()
decoder_graph.output_activating_function = LogisticFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
decoder_graph.create_rnn_cells(input_neuron_count=200,
hidden_neuron_count=dec_dim,
output_neuron_count=200)
# Optimizer for Decoder.
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = 0.5
decoder_opt_params.dropout_rate = 0.5
decoder = Decoder(
# Delegate `graph` to `LSTMModel`.
graph=decoder_graph,
# The number of epochs in mini-batch training.
epochs=epochs,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=8,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=decoder_opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
# Tolerance for the optimization.
# When the loss or score is not improving by at least tol
# for two consecutive iterations, convergence is considered
# to be reached and training stops.
tol=0.0)
conv1 = ConvolutionLayer1(
ConvGraph1(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
conv2 = ConvolutionLayer2(
ConvGraph2(activation_function=TanhFunction(),
filter_num=batch_size,
channel=batch_size,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
cnn = SpatioTemporalAutoEncoder(
layerable_cnn_list=[conv1, conv2],
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=25,
computable_loss=MeanSquaredError(),
opt_params=Adam(),
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15,
save_flag=False)
cnn.learn_generated(feature_generator)
test_len = 0
test_limit = 1
test_arr_list = []
rec_arr_list = []
for batch_observed_arr, batch_target_arr, test_batch_observed_arr, test_batch_target_arr in feature_generator.generate(
):
test_len += 1
result_arr = cnn.inference(test_batch_observed_arr)
for batch in range(test_batch_target_arr.shape[0]):
for seq in range(test_batch_target_arr[batch].shape[0]):
arr = test_batch_target_arr[batch][seq][0]
arr = (arr - arr.min()) / (arr.max() - arr.min())
arr *= 255
img = Image.fromarray(np.uint8(arr))
img.save("result/" + str(i) + "_" + str(seq) + "_observed.png")
for seq in range(result_arr[batch].shape[0]):
arr = result_arr[batch][seq][0]
arr = (arr - arr.min()) / (arr.max() - arr.min())
arr *= 255
img = Image.fromarray(np.uint8(arr))
img.save("result/" + str(i) + "_" + str(seq) +
"_reconsturcted.png")
if test_len >= test_limit:
break
def __init__(self,
midi_path_list,
target_program=0,
batch_size=10,
seq_len=4,
time_fraction=1.0,
true_sampler=None,
noise_sampler=None,
generative_model=None,
discriminative_model=None,
gans_value_function=None):
'''
Init.
Args:
midi_path_list: `list` of paths to MIDI files.
target_program: Program in generated MIDI.
batch_size: Batch size.
seq_len: The length of sequence that LSTM networks will observe.
time_fraction: Time fraction or time resolution (seconds).
true_sampler: is-a `TrueSampler`.
noise_sampler: is-a `NoiseSampler`.
generative_model: is-a `GenerativeModel`.
discriminative_model: is-a `DiscriminativeModel`.
gans_value_function: is-a `GANsValueFunction`.
'''
self.__midi_controller = MidiController()
self.__midi_df_list = [
self.__midi_controller.extract(midi_path)
for midi_path in midi_path_list
]
# The dimension of observed or feature points.
dim = 12
if true_sampler is None:
true_sampler = MidiTrueSampler(midi_path_list=midi_path_list,
batch_size=batch_size,
seq_len=seq_len)
if noise_sampler is None:
noise_sampler = MidiNoiseSampler(batch_size=batch_size)
if generative_model is None:
hidden_activating_function = TanhFunction()
hidden_activating_function.batch_norm = BatchNorm()
generative_model = Generator(
batch_size=batch_size,
seq_len=seq_len,
input_neuron_count=dim,
hidden_neuron_count=dim,
hidden_activating_function=hidden_activating_function)
generative_model.noise_sampler = noise_sampler
if discriminative_model is None:
discriminative_model = Discriminator(batch_size=batch_size,
seq_len=seq_len,
input_neuron_count=dim,
hidden_neuron_count=dim)
if gans_value_function is None:
gans_value_function = MiniMax()
GAN = GenerativeAdversarialNetworks(
gans_value_function=gans_value_function)
self.__true_sampler = true_sampler
self.__generative_model = generative_model
self.__discriminative_model = discriminative_model
self.__GAN = GAN
self.__time_fraction = time_fraction
self.__target_program = target_program
def __init__(self,
lstm_model=None,
computable_loss=None,
batch_size=20,
input_neuron_count=100,
hidden_neuron_count=300,
observed_activating_function=None,
input_gate_activating_function=None,
forget_gate_activating_function=None,
output_gate_activating_function=None,
hidden_activating_function=None,
output_activating_function=None,
seq_len=10,
join_io_flag=False,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=50):
'''
Init.
Args:
lstm_model: is-a `lstm_model`.
computable_loss: is-a `ComputableLoss`.
batch_size: Batch size.
This parameters will be refered only when `lstm_model` is `None`.
input_neuron_count: The number of input units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_neuron_count: The number of hidden units.
This parameters will be refered only when `lstm_model` is `None`.
observed_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `TanhFunction`.
input_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
forget_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
output_gate_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this value will be `LogisticFunction`.
hidden_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
output_activating_function: is-a `ActivatingFunctionInterface` in output layer.
This parameters will be refered only when `lstm_model` is `None`.
If `None`, this model outputs from LSTM's hidden layer in inferencing.
seq_len: The length of sequences.
This means refereed maxinum step `t` in feedforward.
join_io_flag: If this value and value of `output_activating_function` is not `None`,
This model outputs tensors combining observed data points and inferenced data
in a series direction.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
'''
if computable_loss is None:
computable_loss = MeanSquaredError()
if lstm_model is not None:
if isinstance(lstm_model, LSTM) is False:
raise TypeError()
else:
# Init.
graph = LSTMGraph()
# Activation function in LSTM.
if observed_activating_function is None:
graph.observed_activating_function = TanhFunction()
else:
if isinstance(observed_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.observed_activating_function = observed_activating_function
if input_gate_activating_function is None:
graph.input_gate_activating_function = LogisticFunction()
else:
if isinstance(input_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.input_gate_activating_function = input_gate_activating_function
if forget_gate_activating_function is None:
graph.forget_gate_activating_function = LogisticFunction()
else:
if isinstance(forget_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.forget_gate_activating_function = forget_gate_activating_function
if output_gate_activating_function is None:
graph.output_gate_activating_function = LogisticFunction()
else:
if isinstance(output_gate_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.output_gate_activating_function = output_gate_activating_function
if hidden_activating_function is None:
graph.hidden_activating_function = TanhFunction()
else:
if isinstance(hidden_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.hidden_activating_function = hidden_activating_function
if output_activating_function is None:
graph.output_activating_function = TanhFunction()
self.__output_flag = False
output_neuron_count = 1
else:
graph.output_activating_function = output_activating_function
self.__output_flag = True
output_neuron_count = hidden_neuron_count
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=output_neuron_count)
opt_params = SGD()
opt_params.weight_limit = 1e+10
opt_params.dropout_rate = 0.0
lstm_model = LSTM(
# Delegate `graph` to `LSTMModel`.
graph=graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=learning_rate,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=learning_attenuate_rate,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=attenuate_epoch,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=seq_len,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=computable_loss,
# Optimizer.
opt_params=opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
self.__lstm_model = lstm_model
self.__seq_len = seq_len
self.__learning_rate = learning_rate
self.__join_io_flag = join_io_flag
self.__computable_loss = computable_loss
self.__loss_list = []
self.__epoch_counter = 0
self.__learning_attenuate_rate = learning_attenuate_rate
self.__attenuate_epoch = attenuate_epoch
logger = getLogger("pygan")
self.__logger = logger
def __init__(self,
batch_size=20,
learning_rate=1e-10,
opt_params=None,
convolutional_auto_encoder=None,
deconvolution_layer_list=None,
gray_scale_flag=True,
channel=None):
'''
Init.
Args:
batch_size: Batch size in mini-batch.
learning_rate: Learning rate.
convolutional_auto_encoder: is-a `pydbm.cnn.convolutionalneuralnetwork.convolutional_auto_encoder.ConvolutionalAutoEncoder`.
deconvolution_layer_list: `list` of `DeconvolutionLayer`.
gray_scale_flag: Gray scale or not.
This parameter will be refered when `channel` is None.
If `True`, the channel will be `1`. If `False`, the channel will be `3`.
channel: Channel.
'''
if channel is None:
if gray_scale_flag is True:
channel = 1
else:
channel = 3
if opt_params is None:
opt_params = Adam()
opt_params.dropout_rate = 0.0
if isinstance(opt_params, OptParams) is False:
raise TypeError()
scale = 0.01
if convolutional_auto_encoder is None:
conv1 = ConvolutionLayer1(
ConvGraph1(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
conv2 = ConvolutionLayer2(
ConvGraph2(activation_function=TanhFunction(),
filter_num=batch_size,
channel=batch_size,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
convolutional_auto_encoder = CAE(
layerable_cnn_list=[conv1, conv2],
epochs=100,
batch_size=batch_size,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=25,
computable_loss=MeanSquaredError(),
opt_params=opt_params,
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15,
save_flag=False)
if deconvolution_layer_list is None:
deconvolution_layer_list = [
DeconvolutionLayer(
DeCNNGraph(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
]
self.__convolutional_auto_encoder = convolutional_auto_encoder
self.__deconvolution_layer_list = deconvolution_layer_list
self.__opt_params = opt_params
self.__learning_rate = learning_rate
self.__batch_size = batch_size
self.__saved_img_n = 0
self.__attenuate_epoch = 50
self.__epoch_counter = 0
logger = getLogger("pygan")
self.__logger = logger
def __init__(self,
convolutional_auto_encoder=None,
batch_size=10,
channel=1,
learning_rate=1e-10,
opt_params=None,
feature_matching_layer=0):
'''
Init.
Args:
convolutional_auto_encoder: is-a `pydbm.cnn.convolutionalneuralnetwork.convolutional_auto_encoder.ConvolutionalAutoEncoder`.
learning_rate: Learning rate.
batch_size: Batch size in mini-batch.
learning_rate: Learning rate.
opt_params: is-a `pydbm.optimization.opt_params.OptParams`.
feature_matching_layer: Key of layer number for feature matching forward/backward.
'''
if isinstance(convolutional_auto_encoder,
CAE) is False and convolutional_auto_encoder is not None:
raise TypeError(
"The type of `convolutional_auto_encoder` must be `pydbm.cnn.convolutionalneuralnetwork.convolutional_auto_encoder.ConvolutionalAutoEncoder`."
)
if opt_params is None:
opt_params = Adam()
opt_params.dropout_rate = 0.0
if convolutional_auto_encoder is None:
scale = 0.01
conv1 = ConvolutionLayer1(
ConvGraph1(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
conv2 = ConvolutionLayer2(
ConvGraph2(activation_function=TanhFunction(),
filter_num=batch_size,
channel=batch_size,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
convolutional_auto_encoder = RepellingConvolutionalAutoEncoder(
layerable_cnn_list=[conv1, conv2],
epochs=100,
batch_size=batch_size,
learning_rate=1e-05,
learning_attenuate_rate=0.1,
attenuate_epoch=25,
computable_loss=MeanSquaredError(),
opt_params=opt_params,
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15,
save_flag=False)
self.__convolutional_auto_encoder = convolutional_auto_encoder
self.__learning_rate = learning_rate
self.__epoch_counter = 0
self.__feature_matching_layer = feature_matching_layer
logger = getLogger("pygan")
self.__logger = logger
def __init__(self,
token_list,
epochs=300,
skip_n=1,
batch_size=50,
feature_dim=20,
scale=1e-05,
learning_rate=1e-05,
auto_encoder=None):
'''
Initialize.
Args:
token_list: The list of all tokens in all sentences.
skip_n: N of n-gram.
training_count: The epochs.
batch_size: Batch size.
learning_rate: Learning rate.
feature_dim: The dimension of feature points.
'''
if auto_encoder is not None and isinstance(auto_encoder,
SimpleAutoEncoder) is False:
raise TypeError()
self.__logger = getLogger("pydbm")
self.__token_arr = np.array(token_list)
self.__token_uniquie_arr = np.array(list(set(token_list)))
if auto_encoder is None:
activation_function = TanhFunction()
encoder_graph = EncoderGraph(
activation_function=activation_function,
hidden_neuron_count=self.__token_uniquie_arr.shape[0],
output_neuron_count=feature_dim,
scale=scale,
)
encoder_layer = EncoderLayer(encoder_graph)
opt_params = Adam()
opt_params.dropout_rate = 0.5
encoder = Encoder(
nn_layer_list=[
encoder_layer,
],
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
computable_loss=CrossEntropy(),
opt_params=opt_params,
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15)
decoder_graph = DecoderGraph(
activation_function=SoftmaxFunction(),
hidden_neuron_count=feature_dim,
output_neuron_count=self.__token_uniquie_arr.shape[0],
scale=scale,
)
decoder_layer = DecoderLayer(decoder_graph)
opt_params = Adam()
opt_params.dropout_rate = 0.0
decoder = Decoder(
nn_layer_list=[
decoder_layer,
],
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
computable_loss=CrossEntropy(),
opt_params=opt_params,
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15)
auto_encoder = SimpleAutoEncoder(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
computable_loss=CrossEntropy(),
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15,
)
self.__auto_encoder = auto_encoder
self.__epochs = epochs
self.__batch_size = batch_size
self.__skip_n = skip_n
def __init__(self,
lstm_model=None,
batch_size=20,
input_neuron_count=100,
hidden_neuron_count=300,
hidden_activating_function=None,
seq_len=10,
learning_rate=1e-05,
verbose_mode=False):
'''
Init.
Args:
lstm_model: is-a `lstm_model`.
batch_size: Batch size.
This parameters will be refered only when `lstm_model` is `None`.
input_neuron_count: The number of input units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_neuron_count: The number of hidden units.
This parameters will be refered only when `lstm_model` is `None`.
hidden_activating_function: is-a `ActivatingFunctionInterface` in hidden layer.
This parameters will be refered only when `lstm_model` is `None`.
seq_len: The length of sequences.
This means refereed maxinum step `t` in feedforward.
learning_rate: Learning rate.
verbose_mode: Verbose mode or not.
'''
logger = getLogger("pydbm")
handler = StreamHandler()
if verbose_mode is True:
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
else:
handler.setLevel(ERROR)
logger.setLevel(ERROR)
logger.addHandler(handler)
if lstm_model is not None:
if isinstance(lstm_model, LSTM) is False:
raise TypeError()
else:
# Init.
graph = LSTMGraph()
# Activation function in LSTM.
graph.observed_activating_function = TanhFunction()
graph.input_gate_activating_function = LogisticFunction()
graph.forget_gate_activating_function = LogisticFunction()
graph.output_gate_activating_function = LogisticFunction()
if hidden_activating_function is None:
graph.hidden_activating_function = TanhFunction()
else:
if isinstance(hidden_activating_function,
ActivatingFunctionInterface) is False:
raise TypeError()
graph.hidden_activating_function = hidden_activating_function
graph.output_activating_function = TanhFunction()
# Initialization strategy.
# This method initialize each weight matrices and biases in Gaussian distribution: `np.random.normal(size=hoge) * 0.01`.
graph.create_rnn_cells(input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1)
opt_params = SGD()
opt_params.weight_limit = 0.5
opt_params.dropout_rate = 0.0
lstm_model = LSTM(
# Delegate `graph` to `LSTMModel`.
graph=graph,
# The number of epochs in mini-batch training.
epochs=100,
# The batch size.
batch_size=batch_size,
# Learning rate.
learning_rate=1e-05,
# Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
learning_attenuate_rate=0.1,
# Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
attenuate_epoch=50,
# The length of sequences.
seq_len=seq_len,
# Refereed maxinum step `t` in BPTT. If `0`, this class referes all past data in BPTT.
bptt_tau=seq_len,
# Size of Test data set. If this value is `0`, the validation will not be executed.
test_size_rate=0.3,
# Loss function.
computable_loss=MeanSquaredError(),
# Optimizer.
opt_params=opt_params,
# Verification function.
verificatable_result=VerificateFunctionApproximation(),
tol=0.0)
self.__lstm_model = lstm_model
self.__seq_len = seq_len
self.__learning_rate = learning_rate
self.__verbose_mode = verbose_mode
self.__loss_list = []
def __init__(self,
convolutional_auto_encoder=None,
batch_size=10,
channel=1,
learning_rate=1e-10,
learning_attenuate_rate=0.1,
attenuate_epoch=50,
opt_params=None,
feature_matching_layer=0):
'''
Init.
Args:
convolutional_auto_encoder: is-a `pydbm.cnn.convolutionalneuralnetwork.convolutionalautoencoder.ConvolutionalLadderNetworks`.
learning_rate: Learning rate.
batch_size: Batch size in mini-batch.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
opt_params: is-a `pydbm.optimization.opt_params.OptParams`.
feature_matching_layer: Key of layer number for feature matching forward/backward.
'''
if isinstance(convolutional_auto_encoder,
CLN) is False and convolutional_auto_encoder is not None:
raise TypeError(
"The type of `convolutional_auto_encoder` must be `pydbm.cnn.convolutionalneuralnetwork.convolutional_auto_encoder.ConvolutionalAutoEncoder`."
)
if opt_params is None:
opt_params = Adam()
opt_params.weight_limit = 1e+10
opt_params.dropout_rate = 0.0
if convolutional_auto_encoder is None:
scale = 0.01
conv1 = ConvolutionLayer1(
ConvGraph1(activation_function=TanhFunction(),
filter_num=batch_size,
channel=channel,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
conv2 = ConvolutionLayer2(
ConvGraph2(activation_function=TanhFunction(),
filter_num=batch_size,
channel=batch_size,
kernel_size=3,
scale=scale,
stride=1,
pad=1))
convolutional_auto_encoder = CLN(
layerable_cnn_list=[conv1, conv2],
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
computable_loss=MeanSquaredError(),
opt_params=opt_params,
verificatable_result=VerificateFunctionApproximation(),
test_size_rate=0.3,
tol=1e-15,
save_flag=False)
self.__convolutional_auto_encoder = convolutional_auto_encoder
self.__learning_rate = learning_rate
self.__attenuate_epoch = attenuate_epoch
self.__learning_attenuate_rate = learning_attenuate_rate
self.__epoch_counter = 0
self.__feature_matching_layer = feature_matching_layer
logger = getLogger("pygan")
self.__logger = logger
super().__init__(convolutional_auto_encoder=convolutional_auto_encoder,
batch_size=batch_size,
channel=channel,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
opt_params=opt_params,
feature_matching_layer=feature_matching_layer)
self.__alpha_loss_list = []
self.__sigma_loss_list = []
self.__mu_loss_list = []
def __init__(self,
token_list,
document_list=[],
traning_count=100,
batch_size=20,
learning_rate=1e-05,
feature_dim=100):
'''
Initialize.
Args:
token_list: The list of all tokens in all sentences.
If the input value is a two-dimensional list,
the first-dimensional key represents a sentence number,
and the second-dimensional key represents a token number.
document_list: The list of document composed by tokens.
training_count: The epochs.
batch_size: Batch size.
learning_rate: Learning rate.
feature_dim: The dimension of feature points.
'''
pair_dict = {}
document_dict = {}
self.__token_arr = np.array(token_list)
if self.__token_arr.ndim == 2:
for i in range(self.__token_arr.shape[0]):
for j in range(1, self.__token_arr[i].shape[0] - 1):
pair_dict.setdefault(
(self.__token_arr[i, j], self.__token_arr[i, j - 1]),
0)
pair_dict[(self.__token_arr[i, j],
self.__token_arr[i, j - 1])] += 1
pair_dict.setdefault(
(self.__token_arr[i, j], self.__token_arr[i, j + 1]),
0)
pair_dict[(self.__token_arr[i, j],
self.__token_arr[i, j + 1])] += 1
document_dict.setdefault(self.__token_arr[i], [])
for d in range(len(document_list)):
if self.__token_arr[i, j] in document_list[d]:
document_dict[self.__token_arr[i, j]].append(d)
elif self.__token_arr.ndim == 1:
for i in range(1, self.__token_arr.shape[0] - 1):
pair_dict.setdefault(
(self.__token_arr[i], self.__token_arr[i - 1]), 0)
pair_dict[(self.__token_arr[i], self.__token_arr[i - 1])] += 1
pair_dict.setdefault(
(self.__token_arr[i], self.__token_arr[i + 1]), 0)
pair_dict[(self.__token_arr[i], self.__token_arr[i + 1])] += 1
document_dict.setdefault(self.__token_arr[i], [])
for d in range(len(document_list)):
if self.__token_arr[i] in document_list[d]:
document_dict[self.__token_arr[i]].append(d)
else:
raise ValueError()
token_list = list(set(self.__token_arr.ravel().tolist()))
token_arr = np.zeros((len(token_list), len(token_list)))
pair_arr = np.zeros((len(token_list), len(token_list)))
document_arr = np.zeros((len(token_list), len(document_list)))
for i in range(token_arr.shape[0]):
for j in range(token_arr.shape[0]):
try:
pair_arr[i, j] = pair_dict[(token_list[i], token_list[j])]
token_arr[i, j] = 1.0
except:
pass
if len(document_list) > 0:
if token_list[i] in document_dict:
for d in document_dict[token_list[i]]:
document_arr[i, d] = 1.0
pair_arr = np.exp(pair_arr - pair_arr.max())
pair_arr = pair_arr / pair_arr.sum()
pair_arr = (pair_arr - pair_arr.mean()) / (pair_arr.std() + 1e-08)
if len(document_list) > 0:
document_arr = (document_arr -
document_arr.mean()) / (document_arr.std() + 1e-08)
token_arr = np.c_[pair_arr, document_arr]
token_arr = (token_arr - token_arr.mean()) / (token_arr.std() +
1e-08)
self.__dbm = StackedAutoEncoder(
DBMMultiLayerBuilder(),
[token_arr.shape[1], feature_dim, token_arr.shape[1]],
[TanhFunction(), TanhFunction(),
TanhFunction()],
[ContrastiveDivergence(),
ContrastiveDivergence()],
learning_rate=learning_rate)
self.__dbm.learn(token_arr,
traning_count=traning_count,
batch_size=batch_size,
sgd_flag=True)
feature_points_arr = self.__dbm.feature_points_arr
self.__token_arr = token_arr
self.__token_list = token_list
self.__feature_points_arr = feature_points_arr
