def _load_pickle(self, data_path: str, dataset_name: str, mode: str): imgs_pickle_path = os.path.join(data_path, mode, "imgs.npy") labels_pickle_path = os.path.join(data_path, mode, "labels.npy") if os.path.exists(imgs_pickle_path) and os.path.exists( imgs_pickle_path): print("Loading from pickle") imgs = np.load(imgs_pickle_path) labels = np.load(labels_pickle_path) if mode == "Train": self.train_dataset_size = len(labels) if mode == "Validation": self.val_dataset_size = len(labels) self.input_shape = imgs.shape[1:] else: if dataset_name == "MNIST": imgs, labels = load_mnist(data_path, mode) if dataset_name == "Fashion-MNIST": imgs, labels = load_fashion_mnist(data_path, mode) elif dataset_name == "CatVsDog": imgs, labels = load_cat_vs_dog(data_path, mode) else: print("Wrong dataset name") return -1 np.save(imgs_pickle_path, imgs) np.save(labels_pickle_path, labels) return imgs, labels
def get_data(): (x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, flatten=True, one_hot_label=False) return x_test, t_test
# coding: utf-8 import sys, os sys.path.append(os.pardir) # 부모 디렉터리의 파일을 가져올 수 있도록 설정 import numpy as np import matplotlib.pyplot as plt from src.dataset.mnist import load_mnist from src.chap07.simple_convnet import SimpleConvNet from src.common.trainer import Trainer # 데이터 읽기 (x_train, t_train), (x_test, t_test) = load_mnist(flatten=False) # 시간이 오래 걸릴 경우 데이터를 줄인다. x_train, t_train = x_train[:5000], t_train[:5000] x_test, t_test = x_test[:1000], t_test[:1000] max_epochs = 20 network = SimpleConvNet(input_dim=(1, 28, 28), conv_param={ 'filter_num': 30, 'filter_size': 5, 'pad': 0, 'stride': 1 }, hidden_size=100, output_size=10, weight_init_std=0.01) trainer = Trainer(network,
import sys, os sys.path.append(os.pardir) import numpy as np from src.dataset.mnist import load_mnist #def (x_train, t_train), (x_test, t_test) = \ load_mnist(normalize=True, one_hot_label=True) print(x_train.shape) print(t_train.shape) train_size = x_train.shape[0] batch_size = 10 batch_mask = np.random.choice(train_size, batch_size) x_batch = x_train[batch_mask] t_batch = t_train[batch_mask] print(np.random.choice(60000, 10))
# coding: utf-8 import os import sys sys.path.append(os.pardir) # 부모 디렉터리의 파일을 가져올 수 있도록 설정 import numpy as np import matplotlib.pyplot as plt from src.dataset.mnist import load_mnist from src.common.util import smooth_curve from src.common.multi_layer_net import MultiLayerNet from src.common.optimizer import SGD # 0. MNIST 데이터 읽기========== (x_train, t_train), (x_test, t_test) = load_mnist(normalize=True) train_size = x_train.shape[0] batch_size = 128 max_iterations = 2000 # 1. 실험용 설정========== weight_init_types = {'std=0.01': 0.01, 'Xavier': 'sigmoid', 'He': 'relu'} optimizer = SGD(lr=0.01) networks = {} train_loss = {} for key, weight_type in weight_init_types.items(): networks[key] = MultiLayerNet(input_size=784, hidden_size_list=[100, 100, 100, 100], output_size=10, weight_init_std=weight_type) train_loss[key] = []
def __init__(self, data_path: str, dataset_name: str, batch_size: int, cache: bool = True, pickle: bool = True, use_data_augmentation: bool = True): """ Creates desired dataset. Args: dataset_name: Either MNIST or CatVsDog """ self.batch_size: int = batch_size self.input_shape: int self.classes_nb: int data_path: str = os.path.join(data_path, dataset_name) if dataset_name == "MNIST" or dataset_name == "Fashion-MNIST": self.classes_nb = 10 elif dataset_name == "CatVsDog": self.classes_nb = 2 else: print("Wrong dataset name") exit() # Prepare the training dataset. if pickle: imgs, labels = self._load_pickle(data_path, dataset_name, "Train") else: if dataset_name == "MNIST": imgs, labels = load_mnist(data_path, "Train") elif dataset_name == "Fashion-MNIST": imgs, labels = load_fashion_mnist(data_path, "Train") elif dataset_name == "CatVsDog": imgs, labels = load_cat_vs_dog(data_path, "Train") self.input_shape = imgs.shape[1:] self.train_dataset_size: int = len(labels) self.train_dataset = tf.data.Dataset.from_tensor_slices((imgs, labels)) self.train_dataset = DatasetCreator._convert_image_dtype( self.train_dataset) # Convert to float range if use_data_augmentation: self.train_dataset = self.train_dataset.map(color_augment) self.train_dataset = self.train_dataset.map( lambda x, y: (tf.image.random_flip_left_right(x), y)) self.train_dataset = self.train_dataset.shuffle( self.train_dataset_size, reshuffle_each_iteration=False) self.train_dataset = self.train_dataset.batch(self.batch_size) print('Train data loaded' + str(' ' * (os.get_terminal_size()[0] - 17))) # Prepare the validation dataset. if pickle: imgs, labels = self._load_pickle(data_path, dataset_name, "Validation") else: if dataset_name == "MNIST": imgs, labels = load_mnist(data_path, "Validation") elif dataset_name == "Fashion-MNIST": imgs, labels = load_fashion_mnist(data_path, "Validation") elif dataset_name == "CatVsDog": imgs, labels = load_cat_vs_dog(data_path, "Validation") self.val_dataset_size: int = len(labels) self.val_dataset = tf.data.Dataset.from_tensor_slices((imgs, labels)) self.val_dataset = DatasetCreator._convert_image_dtype( self.val_dataset) self.val_dataset = self.val_dataset.shuffle( self.val_dataset_size, reshuffle_each_iteration=True) self.val_dataset = self.val_dataset.batch(self.batch_size) print('Validation data loaded' + str(' ' * (os.get_terminal_size()[0] - 16))) if cache: self.train_dataset = self.train_dataset.cache() # speed things up self.val_dataset = self.val_dataset.cache() print( f"Loaded {self.train_dataset_size} train data and {self.val_dataset_size} val data" )
import sys, os sys.path.append(os.pardir) import numpy as np from src.dataset.mnist import load_mnist from PIL import Image def img_show(img): pil_img = Image.fromarray(np.uint8(img)) pil_img.show() (x_train, t_train), (x_test, t_test) = \ load_mnist(flatten=True, normalize=False, one_hot_label=False) img = x_train[0] label = t_train[0] print(label) print(img.shape) img = img.reshape(28, 28) print(img.shape) img_show(img)
from PIL import Image import sys import os import numpy as np sys.path.append(os.getcwd()) from src.dataset import mnist def img_show(img): pil_img = Image.fromarray(np.uint8(img)) pil_img.show() (x_train, t_train), (x_test, t_test) = mnist.load_mnist(flatten=True, normalize=False) img = x_train[0] label = t_train[0] print(label) print(img.shape) img = img.reshape(28, 28) print(img.shape) img_show(img)
# coding: utf-8 import sys, os sys.path.append(os.pardir) # 부모 디렉터리의 파일을 가져올 수 있도록 설정 import numpy as np from src.dataset.mnist import load_mnist from src.chap05.two_layer_net import TwoLayerNet # 데이터 읽기 (x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True) network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10) x_batch = x_train[:3] t_batch = t_train[:3] grad_numerical = network.numerical_gradient(x_batch, t_batch) grad_backprop = network.gradient(x_batch, t_batch) # 각 가중치의 절대 오차의 평균을 구한다. for key in grad_numerical.keys(): diff = np.average(np.abs(grad_backprop[key] - grad_numerical[key])) print(key + ":" + str(diff))
def get_data(): (x_train, t_train), (x_test, t_test) = load_mnist(flatten=True, normalize=True) return x_test, t_test