def testPBTKeras(self):
from ray.tune.examples.pbt_tune_cifar10_with_keras import Cifar10Model
from tensorflow.python.keras.datasets import cifar10
cifar10.load_data()
validate_save_restore(Cifar10Model)
validate_save_restore(Cifar10Model, use_object_store=True)
def __init__(self, dataset_type):
self.annot_path = cfg.TRAIN.ANNOT_PATH if dataset_type == 'train' else cfg.TEST.ANNOT_PATH
self.input_sizes = cfg.TRAIN.INPUT_SIZE if dataset_type == 'train' else cfg.TEST.INPUT_SIZE
self.batch_size = cfg.TRAIN.BATCH_SIZE if dataset_type == 'train' else cfg.TEST.BATCH_SIZE
self.data_aug = cfg.TRAIN.DATA_AUG if dataset_type == 'train' else cfg.TEST.DATA_AUG
self.train_input_size = cfg.TRAIN.INPUT_SIZE[0]
self.strides = np.array(cfg.YOLO.STRIDES)
self.classes = utils.read_class_names(cfg.YOLO.CLASSES)
self.num_classes = len(self.classes)
(X_train_orig, Y_train_orig), (X_test_orig,
Y_test_orig) = cifar10.load_data()
X_train = X_train_orig / 255.
X_test = X_test_orig / 255.
Y_train = tf.keras.utils.to_categorical(Y_train_orig, 10)
Y_test = tf.keras.utils.to_categorical(Y_test_orig, 10)
self.trainset = X_train
self.testset = X_test
self.y_train = Y_train
self.y_test = Y_test
self.run_flag = 'train'
# self.annotations = self.load_annotations(dataset_type)
self.num_samples = len(self.trainset)
self.num_samples_test = len(self.testset)
self.num_batchs = int(np.floor(self.num_samples / self.batch_size))
self.num_batchs_test = int(
np.floor(self.num_samples_test / self.batch_size))
self.batch_count = 0
def main(net):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train.astype('float32') / 255, x_test.astype('float32') / 255
y_train, y_test = y_train.astype('int32'), y_test.astype('int32')
mean = np.mean(x_train, axis=0)
x_train -= mean
x_test -= mean
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_train)
resolver = tf.contrib.cluster_resolver.TPUClusterResolver('matthew-rahtz')
tf.contrib.distribute.initialize_tpu_system(resolver)
strategy = tf.contrib.distribute.TPUStrategy(resolver)
with strategy.scope():
model = make_resnet(net)
model.summary()
# model.fit_generator(datagen.flow(x_train, y_train, batch_size=32),
# validation_data=(x_test, y_test),
# epochs=200,
# callbacks=[ReduceLROnPlateau(verbose=1),
# TensorBoard(observer.dir)])
model.fit(x_train, y_train, batch_size=32, epochs=200, steps_per_epoch=390)
def load_cifar10(normalize=True, one_hot=True, label_reshape=True):
r'''
Loading cifar10 by `tf.keras`.
If `label_reshape` is `True`,
the shape of `y_train` and `y_test` will be `(-1,10)`,
else `(-1,1,10)`
Returns:
(x_train, y_train), (x_test, y_test)
'''
(x_train, y_train), (x_test, y_test) = load_data()
if normalize:
x_train = x_train.astype(np.float32)
x_train /= 255.0
x_test = x_test.astype(np.float32)
x_test /= 255.0
if one_hot:
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
y_train = sess.run(tf.one_hot(y_train, NUM_CLASSES))
y_test = sess.run(tf.one_hot(y_test, NUM_CLASSES))
if label_reshape:
y_train = y_train.reshape(-1, NUM_CLASSES)
y_test = y_test.reshape(-1, NUM_CLASSES)
return (x_train, y_train), (x_test, y_test)
def load_CIFAR10_data(data_format, num_val=0):
"""Load the CIFAR-10 from disk, separating validation and training data
Parameters
----------
data_format : string
Whether to Should be 'channels_first' or 'channels_last'
num_val : int
Number of training images to use as a validation set. Defaults to 0.
Returns
-------
Arrays
(x_train, y_train), (x_test, y_test), (x_val, y_val), x_test_raw
"""
(x_training, y_training), (x_test, y_test) = cifar10.load_data()
x_training, x_test = x_training.astype('float32'), x_test.astype('float32')
y_training = np.squeeze(y_training).astype('int32')
y_test = np.squeeze(y_test).astype('int32')
x_test_raw = x_test.copy()
# Sub-sample training data if we want a validation split
if num_val > 0:
num_train = y_training.shape[0] - num_val
train_mask = range(num_train)
val_mask = range(num_train, num_train + num_val)
x_train, y_train = x_training[train_mask], y_training[train_mask]
val = (x_training[val_mask], y_training[val_mask])
else:
x_train, y_train, val = x_training, y_training, None
return (x_train, y_train), (x_test, y_test), val, x_test_raw
def main(net, epochs, batch_size):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train.astype('float32') / 255, x_test.astype(
'float32') / 255
mean = np.mean(x_train, axis=0)
x_train -= mean
x_test -= mean
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_train)
model = make_resnet(net)
model.summary()
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_test, y_test),
epochs=epochs,
callbacks=[
ReduceLROnPlateau(verbose=1, patience=20),
TensorBoard(observer.dir)
])
def get_keras_data(dataname):
"""Get datasets using keras API and return as a Dataset object."""
if dataname == 'cifar10_keras':
train, test = cifar10.load_data()
elif dataname == 'cifar100_coarse_keras':
train, test = cifar100.load_data('coarse')
elif dataname == 'cifar100_keras':
train, test = cifar100.load_data()
elif dataname == 'mnist_keras':
train, test = mnist.load_data()
else:
raise NotImplementedError('dataset not supported')
X = np.concatenate((train[0], test[0]))
y = np.concatenate((train[1], test[1]))
if dataname == 'mnist_keras':
# Add extra dimension for channel
num_rows = X.shape[1]
num_cols = X.shape[2]
X = X.reshape(X.shape[0], 1, num_rows, num_cols)
if K.image_data_format() == 'channels_last':
X = X.transpose(0, 2, 3, 1)
y = y.flatten()
data = Dataset(X, y)
return data
def evaluate_on_cifar10():
total_depth = 36
n_blocks = 3
basic_block_count = total_depth // n_blocks
# region Model
input_layer = Input(shape=[32, 32, 3])
layer = input_layer
kernel_initializer = ResBlock2D.get_fixup_initializer(total_depth)
for k in range(n_blocks):
strides = 2 if k < (n_blocks - 1) else 1
layer = ResBlock2D(filters=16 * (2**k),
basic_block_count=basic_block_count,
strides=strides,
kernel_initializer=kernel_initializer,
use_residual_bias=True)(layer)
if k == (n_blocks - 1):
layer = AveragePooling2D(pool_size=8)(layer)
layer = Flatten()(layer)
layer = Dense(units=10, activation="softmax")(layer)
model = Model(inputs=input_layer, outputs=layer)
model.summary()
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["acc"])
# endregion
# region Data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype(np.float32) / 255.0
x_test = x_test.astype(np.float32) / 255.0
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
generator = ImageDataGenerator(rotation_range=15,
width_shift_range=5. / 32,
height_shift_range=5. / 32,
horizontal_flip=True)
generator.fit(x_train, seed=0)
# endregion
log_dir = "../logs/tests/res_block_cifar10/{}".format(int(time()))
log_dir = os.path.normpath(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, profile_batch=0)
model.fit_generator(generator.flow(x_train, y_train, batch_size=64),
steps_per_epoch=100,
epochs=300,
validation_data=(x_test, y_test),
validation_steps=100,
verbose=1,
callbacks=[tensorboard])
def generate_data():
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_shape = x_train.shape[1:]
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
y_train = y_train.astype('int32')
y_test = y_test.astype('int32')
x_train /= 255
x_test /= 255
return x_train, y_train, x_test, y_test, x_shape
def main():
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
model = ResNet50(input_shape=(32, 32, 3),
include_top=False,
weights='imagenet',
classes=10)
model.summary()
def main(argv):
# get model directory
if len(argv) < 2:
model_dir = model.DEFAULT_DIR
else:
model_dir = argv[1]
train, evaluate = cifar10.load_data()
# modify the image data to have values between 0 and 1
x_train = np.asarray(train[0] / 255.0, dtype=np.float32)
x_eval = np.asarray(evaluate[0] / 255.0, dtype=np.float32)
# modify the label data to have data type int32 and in the correct shape
# e.g. makes [[7],[4],...,[5]] go to [7, 4, 5]
y_train = np.asarray(train[1], dtype=np.int32)
y_train = np.reshape(y_train, [-1])
y_eval = np.asarray(evaluate[1], dtype=np.int32)
y_eval = np.reshape(y_eval, [-1])
classifier = tf.estimator.Estimator(model_fn=model.model_fn,
model_dir=model_dir)
# set up logging to store TBD
log_tensors = {}
logging_hook = tf.train.LoggingTensorHook(tensors=log_tensors,
every_n_iter=50)
# input function for training
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': x_train},
y=y_train,
batch_size=100,
num_epochs=None,
shuffle=True)
# input function for evaluating
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': x_eval},
y=y_eval,
num_epochs=1,
shuffle=False)
while True:
# train model
classifier.train(input_fn=train_input_fn,
steps=STEPS_PER_TEST,
hooks=[logging_hook])
# evaluate model
eval_results = classifier.evaluate(input_fn=eval_input_fn)
print(eval_results)
# save model
estimator.export_savedmodel(MODEL_DIR)
def load_mnist():
(train_x, train_y), (test_x, test_y) = cifar10.load_data()
train_x = train_x.reshape(train_x.shape[0], 32, 32, 3)
test_x = test_x.reshape(test_x.shape[0], 32, 32, 3)
train_x = train_x.astype('float32')
test_x = test_x.astype('float32')
train_x /= 255.0
test_x /= 255.0
train_y = to_categorical(train_y, 10)
test_y = to_categorical(test_y, 10)
return (train_x, train_y), (test_x, test_y)
def load_cifar10():
# Load the CIFAR10 data set.
# Convert it to a Numpy array
# Normalize inputs to [0,1]
# Convert labels to one-hot array
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
train_x = x_train.astype('float32') / 255.0
test_x = x_test.astype('float32') / 255.0
train_y = to_categorical(y_train, 10)
test_y = to_categorical(y_test, 10)
return (train_x, train_y), (test_x, test_y)
def load_image_dataset(dataset_name='mnist'):
"""
Loads the dataset by name.
Args:
dataset_name: string, either "mnist", "cifar10", "cifar100", "fmnist"
Returns:
(X_train, y_train), (X_test, y_test)
"""
allowed_names = ['mnist', 'cifar10', 'cifar100', 'fmnist']
if dataset_name not in allowed_names:
raise ValueError("Dataset name provided is wrong. Must be one of ",
allowed_names)
# print(directory)
if dataset_name == 'mnist':
(X_train, y_train), (X_test, y_test) = mnist.load_data()
elif dataset_name == 'fmnist':
(X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()
elif dataset_name == 'cifar10':
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
elif dataset_name == 'cifar100':
(X_train, y_train), (X_test, y_test) = cifar100.load_data()
else:
raise ValueError(
'%s is not a valid dataset name. Available choices are : %s' %
(dataset_name, str(allowed_names)))
if dataset_name in ['mnist', 'fmnist']:
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
X_train = X_train.astype('float32') / 255.
X_test = X_test.astype('float32') / 255.
elif dataset_name in ['cifar10', 'cifar100']:
X_train = X_train.astype('float32') / 255.
X_test = X_test.astype('float32') / 255.
if dataset_name == 'cifar100':
num_classes = 100
else:
num_classes = 10
y_train = tf.keras.utils.to_categorical(y_train, num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)
return (X_train, y_train), (X_test, y_test)
def _read_data(self):
# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
# Convert class vectors to binary class matrices.
y_train = tf.keras.utils.to_categorical(y_train, num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)
x_train = x_train.astype("float32")
x_train /= 255
x_test = x_test.astype("float32")
x_test /= 255
return (x_train, y_train), (x_test, y_test)
def _read_data(self):
# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
# Convert class vectors to binary class matrices.
y_train = tf.keras.utils.to_categorical(y_train, num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)
x_train = x_train.astype("float32")
x_train /= 255
x_test = x_test.astype("float32")
x_test /= 255
return (x_train, y_train), (x_test, y_test)
def get_cifar10_data(self):
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
if self.num_classes == 2:
# filter classes 0 and 1
x_train = x_train[np.isin(y_train.flatten(), [0, 1])]
y_train = y_train[np.isin(y_train.flatten(), [0, 1])]
x_test = x_test[np.isin(y_test.flatten(), [0, 1])]
y_test = y_test[np.isin(y_test.flatten(), [0, 1])]
cifar10_y_train = to_categorical(y_train, self.num_classes)
cifar10_y_test = to_categorical(y_test, self.num_classes)
cifar10_x_train = x_train.astype('float32')
cifar10_x_test = x_test.astype('float32')
cifar10_x_train /= 255.0
cifar10_x_test /= 255.0
return cifar10_y_train, cifar10_y_test, cifar10_x_train, cifar10_x_test
def __init__(self):
num_classes = 10
# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# Convert class vectors to binary class matrices.
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
super().__init__(x_train, x_test, y_train, y_test, x_train.shape[1:],
num_classes, 'cifar10')
def prepare_dataset(num_classes=10):
# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# Convert class vectors to binary class matrices.
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
return x_train, x_test, y_train, y_test
def generate_data(aug_copy):
(unsup_x, _), (_, _) = cifar10.load_data()
for i in range(aug_copy):
aug, unsup = data_augmentation(unsup_x)
unsup = unsup.reshape(unsup.shape[0],
unsup.shape[1] * unsup.shape[2] * unsup.shape[3])
aug = aug.reshape(aug.shape[0],
aug.shape[1] * aug.shape[2] * aug.shape[3])
unsup_filename = 'unsup_{0}.tfrecord'.format(i)
writer = tf.python_io.TFRecordWriter(path + unsup_filename)
for (x, y) in zip(unsup, aug):
example = tf.train.Example(features=tf.train.Features(
feature={
"unsup": _float_list_feature(x),
"aug": _float_list_feature(y)
}))
writer.write(example.SerializeToString())
def main(args):
if args.dataset == "mnist":
from tensorflow.python.keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
elif args.dataset == "fashion_mnist":
from tensorflow.python.keras.datasets import fashion_mnist
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
elif args.dataset == "cifar10":
from tensorflow.python.keras.datasets import cifar10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
else:
sys.exit("Unknown dataset {}".format(args.dataset))
convert(x_train, y_train, args, "train")
convert(x_test, y_test, args, "test")
def data_init(dataset='cifar10', mode='train', val_rate=0.2):
if dataset == 'cifar10':
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
elif dataset == 'cifar100':
(x_train, y_train), (x_test,
y_test) = cifar100.load_data(label_mode='fine')
else:
raise NotImplementedError
if mode == 'train':
x_train = x_train.astype('float32') / 255
y_train = to_categorical(y_train)
train_index = int((1 - val_rate) * len(x_train))
return (x_train[:train_index], y_train[:train_index]), \
(x_train[train_index:], y_train[train_index:])
elif mode == 'test':
x_test = x_test.astype('float32') / 255
y_test = to_categorical(y_test)
return x_test, y_test
def train():
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
model = classify_model()
hist = model.fit(x_train,
y_train,
batch_size=64,
epochs=20,
validation_data=(x_test, y_test),
shuffle=True)
loss, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', loss)
print('Test accuracy:', acc)
model.save('./CIFAR10_model_else.h5')
def load_task_data(task_name,
DATA_DIR,
TRAIN_SIZE=5000,
TEST_SIZE=1000,
dataset='mnist',
out_dim=10):
if 'permuted' in task_name:
data = input_data.read_data_sets(DATA_DIR, one_hot=True)
shuffle_ids = np.arange(data.train.images.shape[0])
X_TRAIN = data.train.images[shuffle_ids][:TRAIN_SIZE]
Y_TRAIN = data.train.labels[shuffle_ids][:TRAIN_SIZE]
X_TEST = data.test.images[:TEST_SIZE]
Y_TEST = data.test.labels[:TEST_SIZE]
elif 'split' in task_name:
if dataset == 'cifar10':
(X_TRAIN, Y_TRAIN), (X_TEST, Y_TEST) = cifar10.load_data()
# standardize data
X_TRAIN, X_TEST = standardize_flatten(X_TRAIN,
X_TEST,
flatten=False)
print('data shape', X_TRAIN.shape)
Y_TRAIN = one_hot_encoder(Y_TRAIN.reshape(-1), out_dim)
Y_TEST = one_hot_encoder(Y_TEST.reshape(-1), out_dim)
elif 'mnist' in dataset:
data = input_data.read_data_sets(DATA_DIR)
X_TRAIN = np.concatenate(
[data.train.images, data.validation.images], axis=0)
Y_TRAIN = np.concatenate(
[data.train.labels, data.validation.labels], axis=0)
X_TEST = data.test.images
Y_TEST = data.test.labels
return X_TRAIN, Y_TRAIN, X_TEST, Y_TEST
def __init__(self, batch_size=128):
self._batch_size=batch_size
(self._train_x, self._train_y), (self._test_x, self._test_y) = cifar10.load_data()
def main(argv):
del argv # unused
if tf.gfile.Exists(FLAGS.model_dir):
tf.logging.warning("Warning: deleting old log directory at {}".format(
FLAGS.model_dir))
tf.gfile.DeleteRecursively(FLAGS.model_dir)
tf.gfile.MakeDirs(FLAGS.model_dir)
if FLAGS.fake_data:
(x_train, y_train), (x_test, y_test) = build_fake_data()
else:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
(images, labels, handle, training_iterator,
heldout_iterator) = build_input_pipeline(x_train, x_test, y_train, y_test,
FLAGS.batch_size, 500)
if FLAGS.architecture == "resnet":
model_fn = bayesian_resnet
else:
model_fn = bayesian_vgg
model = model_fn(
IMAGE_SHAPE,
num_classes=10,
kernel_posterior_scale_mean=FLAGS.kernel_posterior_scale_mean,
kernel_posterior_scale_constraint=FLAGS.
kernel_posterior_scale_constraint)
logits = model(images)
labels_distribution = tfd.Categorical(logits=logits)
# Perform KL annealing. The optimal number of annealing steps
# depends on the dataset and architecture.
t = tf.Variable(0.0)
kl_regularizer = t / (FLAGS.kl_annealing * len(x_train) / FLAGS.batch_size)
# Compute the -ELBO as the loss. The kl term is annealed from 0 to 1 over
# the epochs specified by the kl_annealing flag.
log_likelihood = labels_distribution.log_prob(labels)
neg_log_likelihood = -tf.reduce_mean(log_likelihood)
kl = sum(model.losses) / len(x_train) * tf.minimum(1.0, kl_regularizer)
loss = neg_log_likelihood + kl
# Build metrics for evaluation. Predictions are formed from a single forward
# pass of the probabilistic layers. They are cheap but noisy
# predictions.
predictions = tf.argmax(logits, axis=1)
with tf.name_scope("train"):
train_accuracy, train_accuracy_update_op = tf.metrics.accuracy(
labels=labels, predictions=predictions)
opt = tf.train.AdamOptimizer(FLAGS.learning_rate)
train_op = opt.minimize(loss)
update_step_op = tf.assign(t, t + 1)
with tf.name_scope("valid"):
valid_accuracy, valid_accuracy_update_op = tf.metrics.accuracy(
labels=labels, predictions=predictions)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
stream_vars_valid = [v for v in tf.local_variables() if "valid/" in v.name]
reset_valid_op = tf.variables_initializer(stream_vars_valid)
with tf.Session() as sess:
sess.run(init_op)
# Run the training loop
train_handle = sess.run(training_iterator.string_handle())
heldout_handle = sess.run(heldout_iterator.string_handle())
training_steps = int(
round(FLAGS.epochs * (len(x_train) / FLAGS.batch_size)))
for step in range(training_steps):
_ = sess.run([train_op, train_accuracy_update_op, update_step_op],
feed_dict={handle: train_handle})
# Manually print the frequency
if step % 100 == 0:
loss_value, accuracy_value, kl_value = sess.run(
[loss, train_accuracy, kl],
feed_dict={handle: train_handle})
print("Step: {:>3d} Loss: {:.3f} Accuracy: {:.3f} KL: {:.3f}".
format(step, loss_value, accuracy_value, kl_value))
if (step + 1) % FLAGS.eval_freq == 0:
# Compute log prob of heldout set by averaging draws from the model:
# p(heldout | train) = int_model p(heldout|model) p(model|train)
# ~= 1/n * sum_{i=1}^n p(heldout | model_i)
# where model_i is a draw from the posterior
# p(model|train).
probs = np.asarray([
sess.run((labels_distribution.probs),
feed_dict={handle: heldout_handle})
for _ in range(FLAGS.num_monte_carlo)
])
mean_probs = np.mean(probs, axis=0)
_, label_vals = sess.run((images, labels),
feed_dict={handle: heldout_handle})
heldout_lp = np.mean(
np.log(mean_probs[np.arange(mean_probs.shape[0]),
label_vals.flatten()]))
print(" ... Held-out nats: {:.3f}".format(heldout_lp))
# Calculate validation accuracy
for _ in range(20):
sess.run(valid_accuracy_update_op,
feed_dict={handle: heldout_handle})
valid_value = sess.run(valid_accuracy,
feed_dict={handle: heldout_handle})
print(" ... Validation Accuracy: {:.3f}".format(valid_value))
sess.run(reset_valid_op)
# Fully Connected Layer 2 - 384개의 특징들(feature)을 10개의 클래스-airplane, automobile, bird...-로 맵핑(maping)합니다.
W_fc2 = tf.Variable(tf.truncated_normal(shape=[384, 10], stddev=5e-2))
b_fc2 = tf.Variable(tf.constant(0.1, shape=[10]))
logits = tf.matmul(h_fc1, W_fc2) + b_fc2
y_pred = tf.nn.softmax(logits)
return y_pred, logits
# 인풋 아웃풋 데이터, 드롭아웃 확률을 입력받기위한 플레이스홀더를 정의합니다.
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
y = tf.placeholder(tf.float32, shape=[None, 10])
keep_prob = tf.placeholder(tf.float32)
# CIFAR-10 데이터를 다운로드하고 데이터를 불러옵니다.
(x_train, y_train), (x_test, y_test) = load_data()
# scalar 형태의 레이블(0~9)을 One-hot Encoding 형태로 변환합니다.
y_train_one_hot = tf.squeeze(tf.one_hot(y_train, 10), axis=1)
y_test_one_hot = tf.squeeze(tf.one_hot(y_test, 10), axis=1)
# Convolutional Neural Networks(CNN) 그래프를 생성합니다.
y_pred, logits = model(x)
# Cross Entropy를 비용함수(loss function)으로 정의하고, RMSPropOptimizer를 이용해서 비용 함수를 최소화합니다.
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits))
train_step = tf.train.RMSPropOptimizer(1e-3).minimize(loss)
# 정확도를 계산하는 연산을 추가합니다.
correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1))
def modelBuild(data, q):
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from tensorflow.python.keras.datasets import cifar10
from tensorflow.python.keras.layers.core import Dense
from tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Flatten, Dropout
from tensorflow.python.keras.layers import Input
from tensorflow.python.keras.models import Sequential
from keras.utils import to_categorical
from tensorflow.python.keras import optimizers
from tensorflow.python.keras.models import Model
from tensorflow.python.keras import backend as K
import csv
##DATA LOADING AND RESHAPNG
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.reshape(50000, 32, 32, 3)
x_test = x_test.reshape(10000, 32, 32, 3)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train = x_train / 255
x_test = x_test / 255
num_classes = 10
print('y data before: ')
print(y_train[0:5])
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
print('\ny data after:')
print(y_train[0:5])
##END DATA LOADING AND RESHAPING
#We need to accept a parameter for Conv. Kernel Sizes, # of Kernels
#Depth of later MLP Network, and # of epochs.
kernelSizes = data[0]
numKernelLayers = data[1]
numKernels = data[2]
depthMLPNetwork = data[3]
numEpochs = data[4]
#For depthMLPNetwork, we'll just add Dense(16, activation='relu') layers in succession before
#the Softmax(10) at the end.
try:
##MODEL SETUP
K.clear_session()
model = Sequential()
firstLayerAdded = False
for i in range(0, numKernelLayers):
if not firstLayerAdded:
model.add(
Conv2D(((i + 1) * numKernels),
(kernelSizes[i], kernelSizes[i]),
activation='relu',
input_shape=(
32,
32,
3,
)))
else:
model.add(
Conv2D(((i + 1) * numKernels),
(kernelSizes[i], kernelSizes[i]),
activation='relu'))
poolName = "pool_" + str(i + 1)
model.add(MaxPooling2D((2, 2), name=poolName))
firstLayerAdded = True
model.add(Flatten())
for j in range(0, depthMLPNetwork):
model.add(Dense(16, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
##END MODEL SETUP
##BEGIN MODEL TRAINING
training_samples = 50000
testing_samples = 10000
batch_size = 128
epochs = numEpochs
history = model.fit(x_train[:training_samples],
y_train[:training_samples],
epochs=epochs,
batch_size=batch_size,
verbose=1,
validation_data=(x_test[:testing_samples],
y_test[:testing_samples]))
##END MODEL TRAINING
##SEND RESULTS BACK TO MAIN TO PRINT TO CSV
res = str(numKernels) + " " + str(numKernelLayers) + " " + str(
kernelSizes) + " " + str(depthMLPNetwork) + " " + str(
numEpochs) + " " + str(
history.history['val_accuracy'][numEpochs - 1])
q.put(res)
return res
except Exception:
import traceback
print(traceback.format_exc())
res = str(numKernels) + " " + str(numKernelLayers) + " " + str(
kernelSizes) + " " + str(depthMLPNetwork) + " " + str(
numEpochs) + " " + "EXCEPTION"
q.put(res)
return res
def evaluate_on_cifar10():
total_depth = 100
n_blocks = 3
depth = (total_depth - 4) // n_blocks
growth_rate = 12
filters = growth_rate * 2
# region Model
input_layer = Input(shape=[32, 32, 3])
layer = input_layer
layer = Conv2D(filters=filters, kernel_size=3, strides=1,
padding="same")(layer)
for k in range(n_blocks):
layer = DenseBlock2D(kernel_size=3,
growth_rate=growth_rate,
depth=depth,
use_batch_normalization=True)(layer)
if k < (n_blocks - 1):
filters += growth_rate * depth // 4
layer = transition_block(layer, filters)
else:
layer = AveragePooling2D(pool_size=8)(layer)
layer = Flatten()(layer)
layer = Dense(units=10, activation="softmax")(layer)
model = Model(inputs=input_layer, outputs=layer)
model.summary()
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["acc"])
# endregion
# region Data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype(np.float32) / 255.0
x_test = x_test.astype(np.float32) / 255.0
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
generator = ImageDataGenerator(rotation_range=15,
width_shift_range=5. / 32,
height_shift_range=5. / 32,
horizontal_flip=True)
generator.fit(x_train, seed=0)
# endregion
log_dir = "../logs/tests/dense_block_cifar10/{}".format(int(time()))
log_dir = os.path.normpath(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, profile_batch=0)
model.fit_generator(generator.flow(x_train, y_train, batch_size=64),
steps_per_epoch=100,
epochs=300,
validation_data=(x_test, y_test),
validation_steps=100,
verbose=1,
callbacks=[tensorboard])
'''
用原始模型model_cifar生成ifgsm对抗样本数据集npy文件new_train
'''
# import keras
import time
from tensorflow.python import keras
import matplotlib.pyplot as plt
import numpy as np
from cleverhans.attacks import LBFGS, FastGradientMethod, ProjectedGradientDescent
from cleverhans.utils_keras import KerasModelWrapper
from keras.applications.imagenet_utils import decode_predictions
from tensorflow.python.keras.datasets import cifar10
import cv2
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
if __name__ == '__main__':
model_keras = keras.models.load_model('../models_test/model_cifar_2.h5')
# model_keras = keras.models.load_model('../models_test/keras_cifar10_trained_model.h5')
batch_size = 512
success = 0
data_size = X_train.shape[0]
adv_train = []
time_st=time.time()
for st in range(0, data_size, batch_size):
sample = np.array(X_train[st : st + batch_size].reshape(-1, 32 * 32 * 3) / 255, dtype=np.float)
# sample = np.array([sample])
sess = keras.backend.get_session()
model = KerasModelWrapper(model_keras)
attack = ProjectedGradientDescent(model, sess=sess)
# print(model.predict(panda.reshape(1, *panda.shape)))
def start_training(tn,vn,ims,bas,epc):
training_num = tn
validation_num = vn
image_size = ims
batch_size = bas
epochs = epc
WEIGHTS_FOLDER = './weights/'
#WEIGHTS_FOLDER = save_path
if not os.path.exists(WEIGHTS_FOLDER):
os.mkdir(WEIGHTS_FOLDER)
# Check if image dimension is correct.
if type(image_size) is list:
val1 = image_size[0]
val2 = image_size[1]
if val1 < 139 or val2 < 139:
print("The size is not ok....")
sys.exit(2)
elif type(image_size) is int:
if image_size <139:
print("The size is not ok...")
sys.exit(2)
# Show the training condition
print("Image size is {}".format(image_size))
print("The batch_size is {}".format(batch_size))
print("The epochs is {}".format(epochs))
# Load images and data from cifar 10
(x_train,y_train),(x_validation,y_validation) = cifar10.load_data()
# Load part of train and test data.
x_train = x_train[:training_num]
x_validation = x_validation[:validation_num]
Y_train = y_train[:training_num]
Y_validation = y_validation[:validation_num]
print("Total Train & Validation Num as shown below")
print("Num of training images : {}".format(x_train.shape[0]))
print("Num of validation images : {}".format(x_validation.shape[0]))
X_train,X_validation = helpResize(x_train,x_validation,image_size)
# Check if both of the list has the correct length.
Y_new_train = np.array([np.zeros(10) for x in range(len(Y_train))],dtype='float32')
for i,x in enumerate(Y_train):
Y_new_train[i][x] = 1
Y_new_val = np.array([np.zeros(10) for x in range(len(Y_validation))],dtype='float32')
for i,x in enumerate(Y_validation):
Y_new_val[i][x] = 1
# This could also be the output of a different Keras model or layer
if type(image_size) is list:
input_shape = tuple(image_size) + (3,)
else:
input_shape = (image_size,image_size,3)
base_model = InceptionV3(weights='imagenet', include_top=False,input_shape=input_shape)
# Get the output of the Inception V3 pretrain model.
x = base_model.output
# Works same as Flatten(), but Flatten has larger dense layers, it might cause worse overfitting.
# However, if the user has a larger dataset then the user can use Flatten() instead of GlobalAveragePooling2D or GlobalMaxPooling2D
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)
# Use SGD as an optimizer
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
datagen = ImageDataGenerator(
rotation_range=0, # Randomly rotate images in the range (degrees, 0 to 180)
# Randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# Randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
zoom_range=0., # set range for random zoom
# Set the mode for filling points outside the input boundaries
horizontal_flip=True, # randomly flip images
)
datagen.fit(X_train)
histories = NCHC_CallBack()
c_time = "{}_{}_{}_{}_{}".format(time.localtime().tm_year,time.localtime().tm_mon,time.localtime().tm_mday,time.localtime().tm_hour,time.localtime().tm_min)
mc = ModelCheckpoint(WEIGHTS_FOLDER+c_time+"_weights.{epoch:02d}-acc-{acc:.2f}-loss-{loss:.2f}.hdf5",
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto', period=1)
# Fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(X_train,
Y_new_train,
batch_size=batch_size),
callbacks = [ histories,mc ], #added here
epochs=epochs,
validation_data=(X_validation, Y_new_val)
)
K.clear_session()
del model
