def initialize(self, learning_rate=0.1, corruption_level = 0.0): """docstring for build_model_0""" minibatch_index = Tensor.lscalar('minibatch_index') inputs = Tensor.matrix('denoising_autoencoder_inputs') rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) self.classifier = DenoisingAutoencoder( numpy_rng=rng, theano_rng=theano_rng, n_visible=28 * 28, n_hidden=500 ) self.training_function = self.compiled_training_function( self.classifier, minibatch_index, inputs, learning_rate, corruption_level ) image = Image.fromarray(tile_raster_images(X=self.classifier.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_0.png')
def __init__(self, hparams): super(RoadMapNetwork, self).__init__() self.hparams = hparams dropout = False if self.hparams.DROPOUT == 0 else self.hparams.DROPOUT self.apply_sigmoid = True if self.hparams.LOSS in ["bce", "weighted_bce"]: self.apply_sigmoid = False self.loss_fn = LOSS[self.hparams.LOSS] self.feature_extractor = DenoisingAutoencoder.load_from_checkpoint( FEATURE_EXTRACTOR_PATH) # Output size -> (None, 192, 13, 13) self.feature_extractor.freeze() self.classifier = UNet( num_layers=self.hparams.NUM_LAYERS, features_start=self.hparams.FEATURES_START, dropout=dropout, )
def load_pretrained_layers(self): # Current state of base state_dict = self.state_dict() param_names = list(state_dict.keys()) # Load the pre-trained autoencoder encoder layer model = DenoisingAutoencoder.load_from_checkpoint('denoising.ckpt') temp_enc = list(model.children())[:-1] pretrained_state_dict = temp_enc[0].state_dict() pretrained_param_names = list(pretrained_state_dict.keys()) #We update the first 14 parameters of the base model. for i, param in enumerate(param_names[:14]): state_dict[param] = pretrained_state_dict[ pretrained_param_names[i]] self.load_state_dict(state_dict) print("\nLoaded base model.\n")
def __init__(self, numpy_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10, corruption_levels=[0.1, 0.1]): """ This class is made to support a variable number of layers. :type numpy_rng: numpy.random.RandomState :param numpy_rng: numpy random number generator used to draw initial weights :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` :type n_ins: int :param n_ins: dimension of the input to the sdA :type n_layers_sizes: list of ints :param n_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int :param n_outs: dimension of the output of the network :type corruption_levels: list of float :param corruption_levels: amount of corruption to use for each layer """ self.sigmoid_layers = [] self.dA_layers = [] self.parameters = [] self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 if not theano_rng: theano_rng = RandomStreams(numpy_rng.randint(2**30)) # The SdA is an MLP, for which all weights of intermediate layers # are shared with a different denoising autoencoders # We will first construct the SdA as a deep multilayer perceptron, # and when constructing each sigmoidal layer we also construct a # denoising autoencoder that shares weights with that layer # During pretraining we will train these autoencoders (which will # lead to chainging the weights of the MLP as well) # During finetunining we will finish training the SdA by doing # stochastich gradient descent on the MLP for i in xrange(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden units of # the layer below or the input size if we are on the first layer if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] sigmoid_layer = HiddenLayer(rng=numpy_rng, input_units=input_size, output_units=hidden_layers_sizes[i], nonlinear_function=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... # but we are going to only declare that the parameters of the # sigmoid_layers are parameters of the StackedDAA # the visible biases in the dA are parameters of those # dA, but not the SdA self.parameters.extend(sigmoid_layer.parameters) # Construct a denoising autoencoder that shared weights with this # layer dA_layer = DenoisingAutoencoder(numpy_rng=numpy_rng, theano_rng=theano_rng, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.weights, bhid=sigmoid_layer.biases) self.dA_layers.append(dA_layer) # We now need to add a logistic layer on top of the MLP self.logLayer = LogisticClassifier(input_units=hidden_layers_sizes[-1], output_units=n_outs) self.parameters.extend(self.logLayer.parameters)
] for path in paths: if not os.path.isdir(path): os.makedirs(path) # MNISTデータ(手書き数字画像)を読み込む # 初回はDLするので時間がかかる(53MB) mnist = fetch_mldata('MNIST original') # train, testデータの作成 x_all = mnist.data.astype(np.float32) / 255 x_train, x_test = np.split(x_all, [N]) N_test = x_test.shape[0] # モデルの作成 da = DenoisingAutoencoder(784, 100, noise=SaltAndPepperNoise()) # 学習 all_loss = [] for epoch in xrange(n_epoch): print 'epoch', epoch indexes = np.random.permutation(N) losses = [] sum_loss = 0 for i in xrange(0, N, batchsize): x_batch = x_train[indexes[i:i+batchsize]] loss = da.train(x_batch) sum_loss += float(loss.data) * batchsize if epoch == 0 and i == 0: with open('../output/da/graph.dot', 'w') as o: o.write(computational_graph.build_computational_graph((loss, )).dump())
# 初回はDLするので時間がかかる(53MB) mnist = fetch_mldata('MNIST original') # train, testデータの作成 x_all = mnist.data.astype(np.float32) / 255 y_all = mnist.target.astype(np.int32) x_train, x_test = np.split(x_all, [N]) y_train, y_test = np.split(y_all, [N]) N_test = y_test.size # 1層目 da1_filename = '../output/sda/model_da1.pkl' try: da1 = pickle.load(open(da1_filename)) except IOError: da1 = DenoisingAutoencoder(784, 100, noise=SaltAndPepperNoise()) n_epoch = 30 all_loss = [] for epoch in xrange(n_epoch): print 'epoch', epoch indexes = np.random.permutation(N) losses = [] sum_loss = 0 for i in xrange(0, N, batchsize): x_batch = x_train[indexes[i:i+batchsize]] loss = da1.train(x_batch) sum_loss += float(loss.data) * batchsize print 'train mean loss={}'.format(sum_loss / N) losses += [sum_loss / N] # 評価 sum_loss = 0
if __name__ == "__main__": # テストに使うデータミニバッチ x = T.matrix('x') # ファイルから学習したパラメータをロード f = open("dA.pkl", "rb") state = cPickle.load(f) f.close() # 雑音除去自己符号化器を構築 # 学習時と同様の構成が必要 rng = np.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) dA = DenoisingAutoencoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=28*28, n_hidden=500) # 学習したパラメータをセット dA.__setstate__(state) # テスト用データをロード # 訓練時に使わなかったテストデータで試す datasets = load_data('mnist.pkl.gz') test_set_x = datasets[2][0] # (1) 最初の100枚の画像を描画 # test_set_xは共有変数なのでget_value()で内容を取得できる pos = 1 for i in range(100):
def __init__(self, numpy_rng, n_ins, hidden_layers_sizes, n_outs, corruption_levels): # 隠れ層オブジェクトのリスト self.hidden_layers = [] # 自己符号化器のリスト self.autoencoder_layers = [] # パラメータのリスト self.params = [] # 隠れ層の数 self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) # 学習データのミニバッチ(入力データと正解ラベル)を表すシンボル # これまでの実装と違って複数のメソッド内で使うので属性にしている self.x = T.matrix('x') self.y = T.ivector('y') # ネットワークを構築 # 隠れ層の数だけループして積み上げていく for i in xrange(self.n_layers): # ユニット数 if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # 隠れ層への入力データ if i == 0: layer_input = self.x else: layer_input = self.hidden_layers[-1].output # 多層パーセプトロンの隠れ層 # fine-tuningで重みを更新するため hidden_layer = HiddenLayer(rng=numpy_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) self.hidden_layers.append(hidden_layer) self.params.extend(hidden_layer.params) # 自己符号化器だが重みは多層パーセプトロンの隠れ層と共有 # そのため自己符号化器のparamsはない # 自己符号化器で重みとバイアスの初期値を求めたあとfine-tuningでそのまま重みとバイアスを引き継げる autoencoder_layer = DenoisingAutoencoder(numpy_rng=numpy_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=hidden_layer.W, # 隠れ層の重みを共有 bhid=hidden_layer.b) # 隠れ層のバイアスを共有 self.autoencoder_layers.append(autoencoder_layer) # MNISTの分類ができるように最後にロジスティック回帰層を追加 self.log_layer = LogisticRegression( input=self.hidden_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.log_layer.params) # fine-tuning時のコスト関数を計算するシンボル # 多層パーセプトロンと同じく負の対数尤度 self.finetune_cost = self.log_layer.negative_log_likelihood(self.y) # 分類の誤差率を計算するシンボル self.errors = self.log_layer.errors(self.y)
if __name__ == "__main__": # テストに使うデータミニバッチ x = T.matrix('x') # ファイルから学習したパラメータをロード f = open("dA.pkl", "rb") state = cPickle.load(f) f.close() # 雑音除去自己符号化器を構築 # 学習時と同様の構成が必要 rng = np.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2**30)) dA = DenoisingAutoencoder(numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=28 * 28, n_hidden=500) # 学習したパラメータをセット dA.__setstate__(state) # テスト用データをロード # 訓練時に使わなかったテストデータで試す datasets = load_data('mnist.pkl.gz') test_set_x = datasets[2][0] # (1) 最初の100枚の画像を描画 # test_set_xは共有変数なのでget_value()で内容を取得できる pos = 1 for i in range(100):