def _load_pickle(self, data_path: str, dataset_name: str, mode: str):
imgs_pickle_path = os.path.join(data_path, mode, "imgs.npy")
labels_pickle_path = os.path.join(data_path, mode, "labels.npy")
if os.path.exists(imgs_pickle_path) and os.path.exists(
imgs_pickle_path):
print("Loading from pickle")
imgs = np.load(imgs_pickle_path)
labels = np.load(labels_pickle_path)
if mode == "Train":
self.train_dataset_size = len(labels)
if mode == "Validation":
self.val_dataset_size = len(labels)
self.input_shape = imgs.shape[1:]
else:
if dataset_name == "MNIST":
imgs, labels = load_mnist(data_path, mode)
if dataset_name == "Fashion-MNIST":
imgs, labels = load_fashion_mnist(data_path, mode)
elif dataset_name == "CatVsDog":
imgs, labels = load_cat_vs_dog(data_path, mode)
else:
print("Wrong dataset name")
return -1
np.save(imgs_pickle_path, imgs)
np.save(labels_pickle_path, labels)
return imgs, labels
def get_data():
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, flatten=True, one_hot_label=False)
return x_test, t_test
# coding: utf-8
import sys, os
sys.path.append(os.pardir) # 부모 디렉터리의 파일을 가져올 수 있도록 설정
import numpy as np
import matplotlib.pyplot as plt
from src.dataset.mnist import load_mnist
from src.chap07.simple_convnet import SimpleConvNet
from src.common.trainer import Trainer
# 데이터 읽기
(x_train, t_train), (x_test, t_test) = load_mnist(flatten=False)
# 시간이 오래 걸릴 경우 데이터를 줄인다.
x_train, t_train = x_train[:5000], t_train[:5000]
x_test, t_test = x_test[:1000], t_test[:1000]
max_epochs = 20
network = SimpleConvNet(input_dim=(1, 28, 28),
conv_param={
'filter_num': 30,
'filter_size': 5,
'pad': 0,
'stride': 1
},
hidden_size=100,
output_size=10,
weight_init_std=0.01)
trainer = Trainer(network,
import sys, os
sys.path.append(os.pardir)
import numpy as np
from src.dataset.mnist import load_mnist
#def
(x_train, t_train), (x_test, t_test) = \
load_mnist(normalize=True, one_hot_label=True)
print(x_train.shape)
print(t_train.shape)
train_size = x_train.shape[0]
batch_size = 10
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
print(np.random.choice(60000, 10))
# coding: utf-8
import os
import sys
sys.path.append(os.pardir) # 부모 디렉터리의 파일을 가져올 수 있도록 설정
import numpy as np
import matplotlib.pyplot as plt
from src.dataset.mnist import load_mnist
from src.common.util import smooth_curve
from src.common.multi_layer_net import MultiLayerNet
from src.common.optimizer import SGD
# 0. MNIST 데이터 읽기==========
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)
train_size = x_train.shape[0]
batch_size = 128
max_iterations = 2000
# 1. 실험용 설정==========
weight_init_types = {'std=0.01': 0.01, 'Xavier': 'sigmoid', 'He': 'relu'}
optimizer = SGD(lr=0.01)
networks = {}
train_loss = {}
for key, weight_type in weight_init_types.items():
networks[key] = MultiLayerNet(input_size=784,
hidden_size_list=[100, 100, 100, 100],
output_size=10,
weight_init_std=weight_type)
train_loss[key] = []
def __init__(self,
data_path: str,
dataset_name: str,
batch_size: int,
cache: bool = True,
pickle: bool = True,
use_data_augmentation: bool = True):
""" Creates desired dataset.
Args:
dataset_name: Either MNIST or CatVsDog
"""
self.batch_size: int = batch_size
self.input_shape: int
self.classes_nb: int
data_path: str = os.path.join(data_path, dataset_name)
if dataset_name == "MNIST" or dataset_name == "Fashion-MNIST":
self.classes_nb = 10
elif dataset_name == "CatVsDog":
self.classes_nb = 2
else:
print("Wrong dataset name")
exit()
# Prepare the training dataset.
if pickle:
imgs, labels = self._load_pickle(data_path, dataset_name, "Train")
else:
if dataset_name == "MNIST":
imgs, labels = load_mnist(data_path, "Train")
elif dataset_name == "Fashion-MNIST":
imgs, labels = load_fashion_mnist(data_path, "Train")
elif dataset_name == "CatVsDog":
imgs, labels = load_cat_vs_dog(data_path, "Train")
self.input_shape = imgs.shape[1:]
self.train_dataset_size: int = len(labels)
self.train_dataset = tf.data.Dataset.from_tensor_slices((imgs, labels))
self.train_dataset = DatasetCreator._convert_image_dtype(
self.train_dataset) # Convert to float range
if use_data_augmentation:
self.train_dataset = self.train_dataset.map(color_augment)
self.train_dataset = self.train_dataset.map(
lambda x, y: (tf.image.random_flip_left_right(x), y))
self.train_dataset = self.train_dataset.shuffle(
self.train_dataset_size, reshuffle_each_iteration=False)
self.train_dataset = self.train_dataset.batch(self.batch_size)
print('Train data loaded' + str(' ' *
(os.get_terminal_size()[0] - 17)))
# Prepare the validation dataset.
if pickle:
imgs, labels = self._load_pickle(data_path, dataset_name,
"Validation")
else:
if dataset_name == "MNIST":
imgs, labels = load_mnist(data_path, "Validation")
elif dataset_name == "Fashion-MNIST":
imgs, labels = load_fashion_mnist(data_path, "Validation")
elif dataset_name == "CatVsDog":
imgs, labels = load_cat_vs_dog(data_path, "Validation")
self.val_dataset_size: int = len(labels)
self.val_dataset = tf.data.Dataset.from_tensor_slices((imgs, labels))
self.val_dataset = DatasetCreator._convert_image_dtype(
self.val_dataset)
self.val_dataset = self.val_dataset.shuffle(
self.val_dataset_size, reshuffle_each_iteration=True)
self.val_dataset = self.val_dataset.batch(self.batch_size)
print('Validation data loaded' + str(' ' *
(os.get_terminal_size()[0] - 16)))
if cache:
self.train_dataset = self.train_dataset.cache() # speed things up
self.val_dataset = self.val_dataset.cache()
print(
f"Loaded {self.train_dataset_size} train data and {self.val_dataset_size} val data"
)
import sys, os
sys.path.append(os.pardir)
import numpy as np
from src.dataset.mnist import load_mnist
from PIL import Image
def img_show(img):
pil_img = Image.fromarray(np.uint8(img))
pil_img.show()
(x_train, t_train), (x_test, t_test) = \
load_mnist(flatten=True, normalize=False, one_hot_label=False)
img = x_train[0]
label = t_train[0]
print(label)
print(img.shape)
img = img.reshape(28, 28)
print(img.shape)
img_show(img)
from PIL import Image
import sys
import os
import numpy as np
sys.path.append(os.getcwd())
from src.dataset import mnist
def img_show(img):
pil_img = Image.fromarray(np.uint8(img))
pil_img.show()
(x_train, t_train), (x_test, t_test) = mnist.load_mnist(flatten=True,
normalize=False)
img = x_train[0]
label = t_train[0]
print(label)
print(img.shape)
img = img.reshape(28, 28)
print(img.shape)
img_show(img)
# coding: utf-8
import sys, os
sys.path.append(os.pardir) # 부모 디렉터리의 파일을 가져올 수 있도록 설정
import numpy as np
from src.dataset.mnist import load_mnist
from src.chap05.two_layer_net import TwoLayerNet
# 데이터 읽기
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True,
one_hot_label=True)
network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)
x_batch = x_train[:3]
t_batch = t_train[:3]
grad_numerical = network.numerical_gradient(x_batch, t_batch)
grad_backprop = network.gradient(x_batch, t_batch)
# 각 가중치의 절대 오차의 평균을 구한다.
for key in grad_numerical.keys():
diff = np.average(np.abs(grad_backprop[key] - grad_numerical[key]))
print(key + ":" + str(diff))
def get_data():
(x_train, t_train), (x_test, t_test) = load_mnist(flatten=True,
normalize=True)
return x_test, t_test