def load_data():
"""
Load CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
dirname = 'cifar-10-batches-py'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples,), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000: i * 10000, :, :, :] = data
y_train[(i - 1) * 10000: i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return {'x_train': x_train, 'y_train': y_train,
'x_test': x_test, 'y_test': y_test}
def __load_cifar10():
"""Loads CIFAR10 dataset. """
path = os.path.expanduser("~/.keras/datasets/cifar-10-batches-py")
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
#dirname = 'cifar-10-batches-py'
#origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
#path = get_file(dirname, origin=origin, untar=True, cache_dir="/tmp/keras")
path = "/dev/shm/keras/datasets/cifar-10-batches-py"
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def __load_cifar100():
"""Loads CIFAR100 dataset.
# Arguments
label_mode: one of "fine", "coarse".
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
# Raises
ValueError: in case of invalid `label_mode`.
"""
label_mode = 'fine'
path = os.path.expanduser("~/.keras/datasets/cifar-100-python")
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def get_data(path, num_classes=10):
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
return (x_train, y_train), (x_test, y_test)
def load_cifar10():
# download and extract data
dirname = 'cifar-10-batches-py'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname,
origin,
untar=True,
cache_dir='Z:\\',
cache_subdir="datasets")
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples, ), dtype='uint8')
# load train data
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
# load test data
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if backend.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def input_data(flag):
if flag == 'mnist':
#
path = "D:\\softfiles\\workspace\\data\\tensorflow\\data\\mnist_data\\mnist.npz"
with np.load(path) as f:
x_train, y_train = f['x_train'], f['y_train']
x_test, y_test = f['x_test'], f['y_test']
return (x_train, y_train), (x_test, y_test)
elif flag == 'cifar10':
#'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = "D:\\softfiles\\workspace\\data\\tensorflow\\data\\cifar10\\cifar-10-batches-py"
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_cifar10():
if platform.system() != "Darwin":
(train_x, train_y), (test_x, test_y) = cifar10.load_data()
return (train_x, train_y), (test_x, test_y)
dpath = os.environ["DATASETS"]
if (dpath == None):
print("Missing DATASETS env var.")
exit(-1)
path = os.path.join(dpath, "cifar10")
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def get_data(
origin: str
) -> Tuple[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]:
# This is copied from keras.datasets.cifar10 and modified to support
# a custom origin URL.
dirname = "cifar-10-batches-py"
path = get_file(dirname, origin=origin, untar=True)
num_train_samples = 50000
train_data = np.empty((num_train_samples, 3, 32, 32), dtype="uint8")
train_labels = np.empty((num_train_samples, ), dtype="uint8")
for i in range(1, 6):
fpath = os.path.join(path, "data_batch_" + str(i))
(
train_data[(i - 1) * 10000:i * 10000, :, :, :],
train_labels[(i - 1) * 10000:i * 10000],
) = load_batch(fpath)
fpath = os.path.join(path, "test_batch")
test_data, test_labels = load_batch(fpath)
train_labels = np.reshape(train_labels, (len(train_labels), 1))
test_labels = np.reshape(test_labels, (len(test_labels), 1))
if keras.backend.image_data_format() == "channels_last":
train_data = train_data.transpose(0, 2, 3, 1)
test_data = test_data.transpose(0, 2, 3, 1)
return (train_data, train_labels), (test_data, test_labels)
def load_cifar_data():
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
# dirname = 'E:/dl_data/'
# origin = 'cifar-10-python.tar.gz'
path = get_file(
'3.3_courses/10_dl1/cifar-10-batches-py', '')
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples,), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000: i * 10000, :, :, :] = data
y_train[(i - 1) * 10000: i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
dirname = 'cifar-10-batches-py'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
#/home/mainspring/.keras/datasets/cifar-10-batches-py/data_batch_1
path = os.path.join('/home/mainspring/.keras/datasets', dirname)
#data_utils.get_file(dirname, origin=origin, untar=True)
nb_train_samples = 50000
x_train = np.zeros((nb_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((nb_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = cifar.load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = cifar.load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_dim_ordering() == 'tf':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
dirname = 'cifar-10-batches-py'
origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
# if K.image_data_format() == 'channels_last':
# x_train = x_train.transpose(0, 2, 3, 1)
# x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
path = '/home/zarif/.keras/datasets/cifar-10-batches-py'
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
path = '/Users/zuoyuan/.keras/datasets/cifar-10-batches-py'
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples,), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000: i * 10000, :, :, :],
y_train[(i - 1) * 10000: i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data(label_mode='fine'):
"""Loads CIFAR100 dataset.
# Arguments
label_mode: one of "fine", "coarse".
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
# Raises
ValueError: in case of invalid `label_mode`.
"""
if label_mode not in ['fine', 'coarse']:
raise ValueError('label_mode must be one of "fine" "coarse".')
dirname = 'cifar-100-python'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return {'x_train': x_train, 'y_train': y_train,
'x_test': x_test, 'y_test': y_test}
def load_data(label_mode='fine'):
"""Loads [CIFAR100 dataset](https://www.cs.toronto.edu/~kriz/cifar.html).
This is a dataset of 50,000 32x32 color training images and
10,000 test images, labeled over 100 fine-grained classes that are
grouped into 20 coarse-grained classes. See more info at the
[CIFAR homepage](https://www.cs.toronto.edu/~kriz/cifar.html).
Args:
label_mode: one of "fine", "coarse". If it is "fine" the category labels
are the fine-grained labels, if it is "coarse" the output labels are the
coarse-grained superclasses.
Returns:
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
**x_train, x_test**: uint8 arrays of RGB image data with shape
`(num_samples, 3, 32, 32)` if `tf.keras.backend.image_data_format()` is
`'channels_first'`, or `(num_samples, 32, 32, 3)` if the data format
is `'channels_last'`.
**y_train, y_test**: uint8 arrays of category labels with shape
(num_samples, 1).
Raises:
ValueError: in case of invalid `label_mode`.
"""
if label_mode not in ['fine', 'coarse']:
raise ValueError('`label_mode` must be one of `"fine"`, `"coarse"`.')
dirname = 'cifar-100-python'
origin = 'https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
path = get_file(
dirname,
origin=origin,
untar=True,
file_hash=
'85cd44d02ba6437773c5bbd22e183051d648de2e7d6b014e1ef29b855ba677a7')
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
"""Loads [CIFAR10 dataset](https://www.cs.toronto.edu/~kriz/cifar.html).
This is a dataset of 50,000 32x32 color training images and 10,000 test
images, labeled over 10 categories. See more info at the
[CIFAR homepage](https://www.cs.toronto.edu/~kriz/cifar.html).
Returns:
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
**x_train, x_test**: uint8 arrays of RGB image data with shape
`(num_samples, 3, 32, 32)` if `tf.keras.backend.image_data_format()` is
`'channels_first'`, or `(num_samples, 32, 32, 3)` if the data format
is `'channels_last'`.
**y_train, y_test**: uint8 arrays of category labels
(integers in range 0-9) each with shape (num_samples, 1).
"""
dirname = 'cifar-10-batches-py'
origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(
dirname,
origin=origin,
untar=True,
file_hash=
'6d958be074577803d12ecdefd02955f39262c83c16fe9348329d7fe0b5c001ce')
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
x_test = x_test.astype(x_train.dtype)
y_test = y_test.astype(y_train.dtype)
return (x_train, y_train), (x_test, y_test)
def load_data(config):
"""
Load CIFAR100 dataset.
Parameters
----------
label_mode: one of "fine", "coarse".
Returns
-------
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
Raises
------
ValueError: in case of invalid `label_mode`.
"""
label_mode = 'fine'
if label_mode not in ['fine', 'coarse']:
raise ValueError('label_mode must be one of "fine" "coarse".')
dirname = 'cifar-100-python'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=0.10,
random_state=42,
stratify=y_train)
return {
'x_train': x_train,
'y_train': y_train,
'x_val': x_val,
'y_val': y_val,
'x_test': x_test,
'y_test': y_test
}
def load_data(path=os.path.join(".", "cifar-100-python"), label_mode='fine'):
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_cifar10(raw=False):
"""Loads CIFAR10 dataset from config.CIFAR10_PATH or downloads it if necessary.
:param raw: `True` if no preprocessing should be applied to the data. Otherwise, data is normalized to 1.
:type raw: `bool`
:return: `(x_train, y_train), (x_test, y_test), min, max`
:rtype: `(np.ndarray, np.ndarray), (np.ndarray, np.ndarray), float, float`
"""
import keras.backend as k
from keras.datasets.cifar import load_batch
from keras.utils.data_utils import get_file
from art import DATA_PATH
min_, max_ = 0., 1.
path = get_file(
'cifar-10-batches-py',
untar=True,
cache_subdir=DATA_PATH,
origin='http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype=np.uint8)
y_train = np.zeros((num_train_samples, ), dtype=np.uint8)
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if k.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
min_, max_ = 0, 255
if not raw:
min_, max_ = 0., 1.
x_train, y_train = preprocess(x_train, y_train)
x_test, y_test = preprocess(x_test, y_test)
return (x_train, y_train), (x_test, y_test), min_, max_
def get_in_dist_train_data():
"""
Loads a small batch of CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
dirname = 'cifar-10-batches-py'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
num_batches = 1 # 10K, 10K
num_train_samples = 10000 * num_batches
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples,), dtype='uint8')
# Load only one of the 5 train batches
for i in range(1, 1 + num_batches):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000: i * 10000, :, :, :] = data
# Since in-dist images should have 0 as label
# y_train[(i - 1) * 10000: i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
# Since in-dist images should have 0 as label
# Number of test images is 10000
y_test = np.zeros((10000,), dtype='uint8')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train = x_train[:4000]
y_train = y_train[:4000]
x_test = x_test[:6000]
y_test = y_test[:6000]
return (x_train, y_train), (x_test, y_test)
def load_data(config):
"""
Load CIFAR10 dataset.
Returns
-------
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
dirname = 'cifar-10-batches-py'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=0.10,
random_state=42,
stratify=y_train)
return {
'x_train': x_train,
'y_train': y_train,
'x_val': x_val,
'y_val': y_val,
'x_test': x_test,
'y_test': y_test
}
def load_data(config):
"""
Load CIFAR100 dataset.
Parameters
----------
label_mode: one of "fine", "coarse".
Returns
-------
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
Raises
------
ValueError: in case of invalid `label_mode`.
"""
label_mode = 'fine'
if label_mode not in ['fine', 'coarse']:
raise ValueError('label_mode must be one of "fine" "coarse".')
dirname = 'cifar-100-python'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train,
test_size=0.10,
random_state=42,
stratify=y_train)
return {'x_train': x_train, 'y_train': y_train,
'x_val': x_val, 'y_val': y_val,
'x_test': x_test, 'y_test': y_test}
def load_data(dataset_path: str = None, label_mode='fine'):
if dataset_path:
fpath = os.path.join(dataset_path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(dataset_path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return x_train, y_train, x_test, y_test
else:
(x_train, y_train), (x_test, y_test) = keras.datasets.cifar100.load_data(label_mode='fine')
return x_train, y_train, x_test, y_test
def load_data(dest=None):
"""Loads CIFAR10 dataset.
Returns:
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
dest = HERE + '/DATA'
if dest is None:
dest = '/projects/datascience/username/nas4candle.nasapi/benchmark/cifar10Nas/DATA'
else:
dest = os.path.abspath(os.path.expanduser(dest))
print(f"getfile(origin={origin}, dest={dest})")
path = get_file('cifar-10-batches-py',
origin=origin,
untar=True,
cache_subdir=dest)
num_train_samples = 50000
train_X = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
train_y = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(train_X[(i - 1) * 10000:i * 10000, :, :, :],
train_y[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
test_X, test_y = load_batch(fpath)
train_y = np.reshape(train_y, (len(train_y)))
test_y = np.reshape(test_y, (len(test_y)))
train_X = np.true_divide(train_X, 255)
test_X = np.true_divide(test_X, 255)
if K.image_data_format() == 'channels_last':
train_X = train_X.transpose(0, 2, 3, 1)
test_X = test_X.transpose(0, 2, 3, 1)
return (train_X, train_y), (test_X, test_y)
def load_data(download_path=os.getcwd()):
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
dirname = 'cifar-10-batches-py'
origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
if not os.path.exists(os.path.join(download_path, dirname)):
if not os.path.exists(download_path):
os.mkdir(download_path)
path = get_file(dirname,
origin=origin,
untar=True,
cache_dir=download_path,
cache_subdir='')
else:
path = os.path.join(download_path, dirname)
print("Dataset already exists at: {}".format(path))
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_cifar10():
"""Loads CIFAR10 dataset from config.CIFAR10_PATH or downloads it if necessary.
:return: (x_train, y_train), (x_test, y_test), min, max
:rtype: (tuple of numpy.ndarray), (tuple of numpy.ndarray), float, float
"""
from config import CIFAR10_PATH
from keras.datasets.cifar import load_batch
from keras.utils.data_utils import get_file
min_, max_ = 0., 1.
path = get_file(
'cifar-10-batches-py',
untar=True,
cache_subdir=CIFAR10_PATH,
origin='http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype=np.uint8)
y_train = np.zeros((num_train_samples, ), dtype=np.uint8)
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if k.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
x_train, y_train = preprocess(x_train, y_train)
x_test, y_test = preprocess(x_test, y_test)
return (x_train, y_train), (x_test, y_test), min_, max_
def load_data(dirname):
"""Loads CIFAR10 dataset locally.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
try:
if dirname is not None:
path = os.path.abspath(dirname)
else:
dirname_remote = 'cifar-10-batches-py'
origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname_remote, origin=origin, untar=True)
num_train_samples = 50000
x_train = np.zeros((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.zeros((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
x_train[(i - 1) * 10000:i * 10000, :, :, :] = data
y_train[(i - 1) * 10000:i * 10000] = labels
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
except FileNotFoundError as err:
print(
"ERROR: THERE AREN'T LOCAL FILES, IF YOU WANT TO DOWNLOAD THE DATASET, SET dirname TO None. \n {0}"
.format(err))
def load_data():
dirname = "cifar-10-batches-py"
nb_train_samples = 50000
X_train = np.zeros((nb_train_samples, 3, 32, 32), dtype="uint8")
y_train = np.zeros((nb_train_samples,), dtype="uint8")
for i in range(1, 6):
fpath = os.path.join('/day2/datasets', dirname, 'data_batch_' + str(i))
data, labels = load_batch(fpath)
X_train[(i-1)*10000:i*10000, :, :, :] = data
y_train[(i-1)*10000:i*10000] = labels
fpath = os.path.join('/day2/datasets', dirname, 'test_batch')
X_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
return (X_train, y_train), (X_test, y_test)
def load_data(label_mode='coarse'):
"""Loads CIFAR100 dataset.
# Arguments
label_mode: one of "fine", "coarse".
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
# Raises
ValueError: in case of invalid `label_mode`.
"""
if label_mode not in ['fine', 'coarse']:
raise ValueError('`label_mode` must be one of `"fine"`, `"coarse"`.')
path = './cifar-100-python'
# if you already download data in local path, don't download
# but you don't have data in local path, then download from website
try:
print("already downloaded")
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
except:
dirname = 'cifar-100-python'
origin = 'https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
fpath = os.path.join(path, 'train')
x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
fpath = os.path.join(path, 'test')
x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data(origin, dest):
"""Loads CIFAR10 dataset.
Returns:
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
#origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
if dest is None:
dest = 'datasets'
else:
dest = os.path.abspath(os.path.expanduser(dest))
print(f"getfile(origin={origin}, dest={dest})")
path = get_file('cifar-10-batches-py',
origin='file://' + origin,
untar=True,
cache_subdir=dest)
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def cifar10_load_data():
path = 'cifar10'
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data():
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
Image = np.load('/Users/pc/Downloads/MontgomerySet/CXR_png')
ClinicalReading = np.load(
'/Users/pc/Downloads/MontgomerySet/ClinicalReadings')
dirname = '/Users/pc/Downloads/cifar-10-batches-py'
origin = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
path = get_file(dirname, origin=origin, untar=True)
# create function to transform the data
# create fucntiom to take in the path
num_train_samples = 50000
x_train = np.empty((num_train_samples, 3, 32, 32), dtype='uint8')
y_train = np.empty((num_train_samples, ), dtype='uint8')
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
(x_train[(i - 1) * 10000:i * 10000, :, :, :],
y_train[(i - 1) * 10000:i * 10000]) = load_batch(fpath)
fpath = os.path.join(path, 'test_batch')
x_test, y_test = load_batch(fpath)
y_train = np.reshape(y_train, (len(y_train), 1))
y_test = np.reshape(y_test, (len(y_test), 1))
if K.image_data_format() == 'channels_last':
x_train = x_train.transpose(0, 2, 3, 1)
x_test = x_test.transpose(0, 2, 3, 1)
return (x_train, y_train), (x_test, y_test)
def load_data(data_file):
"""Loads CIFAR10 dataset.
# Returns
Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
"""
num_train_samples = 50000
x_train, y_train = load_batch(data_file)
if K.image_data_format() != 'channels_last':
x_train = x_train.transpose(0, 3, 1, 2)
return (x_train, y_train), (x_train, y_train)
def load_data(label_mode='fine'):
dirname = "cifar-10-batches-py"
origin = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
path = get_file(dirname, origin=origin, untar=True)
nb_test_samples = 10000
nb_train_samples = 50000
X_train2 = np.zeros((nb_train_samples, 3, 32, 32), dtype="uint8")
y_train2 = np.zeros((nb_train_samples,), dtype="uint8")
for i in range(1, 6):
fpath = os.path.join(path, 'data_batch_' + str(i))
data, labels = cifar.load_batch(fpath)
X_train2[(i-1)*10000:i*10000, :, :, :] = data
y_train2[(i-1)*10000:i*10000] = labels
fpath = os.path.join(path, 'test_batch')
X_test2, y_test2 = cifar.load_batch(fpath)
y_train2 = np.reshape(y_train2, (len(y_train2), 1))
y_test2 = np.reshape(y_test2, (len(y_test2), 1))
################################################################
if label_mode not in ['fine', 'coarse']:
raise Exception('label_mode must be one of "fine" "coarse".')
dirname = "cifar-100-python"
origin = "http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
path = get_file(dirname, origin=origin, untar=True)
nb_test_samples = 500
nb_train_samples = 2500
fpath = os.path.join(path, 'train')
X_train1, y_train1 = load_batch(fpath, label_key=label_mode+'_labels')
fpath = os.path.join(path, 'test')
X_test1, y_test1 = load_batch(fpath, label_key=label_mode+'_labels')
y_train1 = np.reshape(y_train1, (len(y_train1), 1))
y_test1 = np.reshape(y_test1, (len(y_test1), 1))
#####################################################################
print(type(X_train1))
print(type(X_train2))
X_train=X_train1.tolist()+X_train2.tolist()
print("X_train transformation worked")
X_test=X_test1.tolist()+X_test2.tolist()
print("X_test transformation worked")
X_test=asarray(X_test)
print("X_test revertion worked")
X_train=asarray(X_train)
print("X_train revertion worked")
print(type(y_test1))
print(type(y_test2))
y_test=y_test1.tolist()+y_test2.tolist()
y_train=y_train1.tolist()+y_train2.tolist()
y_test=asarray(y_test)
y_train=asarray(y_train)
nb_test_samples=len(X_test)
print(nb_test_samples)
nb_train_samples=len(X_train)
print(nb_train_samples)
return (X_train, y_train), (X_test, y_test)