def load_mnist(classes=10): #MNIST(テスト用)
#the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
#この時点でx_train.shape=(60000,28,28), x_test.shape=(10000,28,28), y_train.shape=(60000,), y_test.shape=(10000,)
x_train = x_train.reshape(-1, 28, 28,
1).astype('float32') #-1はそれ以外に合わせるように合わせるという意味
x_test = x_test.reshape(-1, 28, 28, 1).astype('float32')
#normalization(0〜255の値から0〜1に変換)
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
#convert class vectors to binary class matrices(1 of nb_classesのベクトルに変換)
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
return x_train, y_train, x_test, y_test, 'mnist', classes, 28
def __getitem__(self, idx):
batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size]
batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size]
return np.array([
cv.resize(cv.imread(file_name), (self.image_size, self.image_size))
for file_name in batch_x
]), to_categorical(np.array(batch_y), self.classes)
def load_batch_Data(start=0, end=1000, batch_number=1, batch_size=128):
data = bson.decode_file_iter(open('data/train.bson', 'rb'))
categories = load('data/categories.p')
# print("batch_number %s" % (batch_number))
packs = (end - start) // batch_size
batch_number = max(batch_number % (packs + 1), 1)
start_from = start + (batch_number - 1) * batch_size
end_to = start + (batch_number) * batch_size
# print("packs %s" % (packs))
# print("batch_number %s" % (batch_number))
# print("start_from %s" % (start_from))
# print("end_to %s" % (end_to))
labels = []
features = []
count = 0
for iteration, row in enumerate(data):
category_id = row['category_id']
category_id = categories[category_id]
categorical = np_utils.to_categorical(category_id, len(categories))[0]
for e, pic in enumerate(row['imgs']):
if count >= end_to:
labels = np.array(labels)
features = np.array(features)
return labels, features
if count >= start_from:
im = Image.open(io.BytesIO(pic['picture']))
im.thumbnail((im_w, im_h), Image.ANTIALIAS)
arr = np.array(list(im.getdata())) / 255
arr = np.reshape(arr, (im_w, im_h, 3))
labels.append(categorical)
features.append(arr)
#yield labels, arr
count += 1
labels = np.array(labels)
features = np.array(features)
return labels, features
def inner(batch_size: int, infinite=False):
nonlocal samples
mapping = {'stop': 0, 'warning': 1, 'go': 2, 'stopLeft': 3, 'warningLeft': 4, 'goLeft': 5}
num_samples = len(samples)
samples = shuffle(samples)
while 1:
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset + batch_size]
images = []
labels = batch_samples['Annotation tag'].apply(lambda x: mapping[x]).values
for ind in range(len(batch_samples)):
image = pre_process(batch_samples, ind, crop)
images.append(image)
x_train = np.array(images)
# one hot activation
y_train = to_categorical(labels, num_classes=6)
yield shuffle(x_train, y_train)
if not infinite:
break
def score(self, x, y, **kwargs):
"""Returns the mean accuracy on the given test data and labels.
Arguments:
x: array-like, shape `(n_samples, n_features)`
Test samples where n_samples in the number of samples
and n_features is the number of features.
y: array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
True labels for x.
**kwargs: dictionary arguments
Legal arguments are the arguments of `Sequential.evaluate`.
Returns:
score: float
Mean accuracy of predictions on X wrt. y.
Raises:
ValueError: If the underlying model isn't configured to
compute accuracy. You should pass `metrics=["accuracy"]` to
the `.compile()` method of the model.
"""
y = np.searchsorted(self.classes_, y)
kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
outputs = self.model.evaluate(x, y, **kwargs)
if not isinstance(outputs, list):
outputs = [outputs]
for name, output in zip(self.model.metrics_names, outputs):
if name == 'acc':
return output
raise ValueError('The model is not configured to compute accuracy. '
'You should pass `metrics=["accuracy"]` to '
'the `model.compile()` method.')
def score(self, x, y, **kwargs):
"""Returns the mean accuracy on the given test data and labels.
Arguments:
x: array-like, shape `(n_samples, n_features)`
Test samples where n_samples in the number of samples
and n_features is the number of features.
y: array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
True labels for x.
**kwargs: dictionary arguments
Legal arguments are the arguments of `Sequential.evaluate`.
Returns:
score: float
Mean accuracy of predictions on X wrt. y.
Raises:
ValueError: If the underlying model isn't configured to
compute accuracy. You should pass `metrics=["accuracy"]` to
the `.compile()` method of the model.
"""
y = np.searchsorted(self.classes_, y)
kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
outputs = self.model.evaluate(x, y, **kwargs)
if not isinstance(outputs, list):
outputs = [outputs]
for name, output in zip(self.model.metrics_names, outputs):
if name == 'acc':
return output
raise ValueError('The model is not configured to compute accuracy. '
'You should pass `metrics=["accuracy"]` to '
'the `model.compile()` method.')
def fit(self, x, y, **kwargs):
"""Constructs a new model with `build_fn` & fit the model to `(x, y)`.
Arguments:
x : array-like, shape `(n_samples, n_features)`
Training samples where n_samples in the number of samples
and n_features is the number of features.
y : array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
True labels for X.
**kwargs: dictionary arguments
Legal arguments are the arguments of `Sequential.fit`
Returns:
history : object
details about the training history at each epoch.
"""
if self.build_fn is None:
self.model = self.__call__(**self.filter_sk_params(self.__call__))
elif (not isinstance(self.build_fn, types.FunctionType)
and not isinstance(self.build_fn, types.MethodType)):
self.model = self.build_fn(
**self.filter_sk_params(self.build_fn.__call__))
else:
self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit))
fit_args.update(kwargs)
history = self.model.fit(x, y, **fit_args)
return history
def fit(self, x, y, **kwargs):
"""Constructs a new model with `build_fn` & fit the model to `(x, y)`.
Arguments:
x : array-like, shape `(n_samples, n_features)`
Training samples where n_samples in the number of samples
and n_features is the number of features.
y : array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
True labels for X.
**kwargs: dictionary arguments
Legal arguments are the arguments of `Sequential.fit`
Returns:
history : object
details about the training history at each epoch.
"""
if self.build_fn is None:
self.model = self.__call__(**self.filter_sk_params(self.__call__))
elif (not isinstance(self.build_fn, types.FunctionType) and
not isinstance(self.build_fn, types.MethodType)):
self.model = self.build_fn(
**self.filter_sk_params(self.build_fn.__call__))
else:
self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit))
fit_args.update(kwargs)
history = self.model.fit(x, y, **fit_args)
return history
def provide_images(self, batch_size=1):
image_patch_queue = Queue()
label_patch_queue = Queue()
# Iterate indefinitely over files list
for image_name in chain.from_iterable(repeat(sorted(self.files_list))):
# Read image and its labels
image = imread(
os.path.join(self.image_path, image_name[0]) + image_name[1])
label = imread(os.path.join(self.labels_path, image_name[0] + '.png'))
augmented_image = self.data_augmentation(image)
augmented_label = self.data_augmentation(
to_categorical(label, self.num_classes).reshape(
label.shape + (self.num_classes,)))
assert len(augmented_image) == len(augmented_label)
for aug_image, aug_label in zip(augmented_image, augmented_label):
# Extract image patches
img_patches, lab_patches = self.get_patches(aug_image, aug_label)
# Put patches into queue
[image_patch_queue.put(item) for item in img_patches[:]]
[label_patch_queue.put(item) for item in lab_patches[:]]
assert image_patch_queue.qsize() == label_patch_queue.qsize()
while image_patch_queue.qsize() >= batch_size:
# Pop a batch from the queue
img_batch = []
[img_batch.append(image_patch_queue.get()) for _ in range(batch_size)]
lab_batch = []
[lab_batch.append(label_patch_queue.get()) for _ in range(batch_size)]
yield np.array(img_batch), np.array(lab_batch)
def one_hot(batch_samples):
mapping = {'stop': 0, 'warning': 1, 'go': 2, 'stopLeft': 3, 'warningLeft': 4, 'goLeft': 5}
labels = batch_samples['Annotation tag'].apply(lambda x: mapping[x]).values
y_train = to_categorical(labels, num_classes=6)
return y_train
from tensorflow.contrib.keras.python.keras.layers import Convolution2D, MaxPooling2D, Dropout, Flatten, Dense
from tensorflow.contrib.keras.python.keras.models import Sequential
from tensorflow.contrib.keras.python.keras.utils import np_utils
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# Preprocess
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
# Preprocess labels
Y_train = np_utils.to_categorical(y_train, 10)
Y_test = np_utils.to_categorical(y_test, 10)
# Model
model = Sequential()
#defaultformat: (samples, rows, col
model.add(
Convolution2D(32,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
input_shape=(28, 28, 1)))
model.add(
Convolution2D(32, kernel_size=(3, 3), strides=(1, 1), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
def data_generator(self,
mode='obj',
setting_index=0,
num_classes=24,
dataset_name='train',
batch_size=32,
dataset_loc='hand',
to_shuffle=False,
features_path=None):
for_head = False #True if dataset_loc == 'head' else False
if dataset_name in ['train', 'validation']:
GT_train_labels = load_train_labels(self.zip_ref_labels,
mode,
setting_index=setting_index,
for_head=for_head)
if dataset_loc == 'hand':
image_name_array = self.train_hand_name_array
elif dataset_loc == 'head':
image_name_array = self.train_head_name_array
#GT_train_labels = GT_train_labels[0,:] + GT_train_labels[1,:]*num_classes
else: # load features
train_features = np.load(open(features_path, 'rb'))
GT_train_labels = np.append(
GT_train_labels[:self.num_training_features],
GT_train_labels[self.num_training_data:(
self.num_training_data +
self.num_validation_features)])
else: # test set
GT_test_labels = load_test_labels(self.zip_ref_labels,
mode,
setting_index=setting_index,
for_head=for_head)
if dataset_loc == 'hand':
image_name_array = self.test_hand_name_array
elif dataset_loc == 'head':
image_name_array = self.test_head_name_array
#GT_test_labels = GT_test_labels[0,:] + GT_test_labels[1,:]*num_classes
else: # load features
test_features = np.load(open(features_path, 'rb'))
GT_test_labels = GT_test_labels[:self.num_test_features]
if dataset_name == 'train':
if dataset_loc in ['hand', 'head']:
dataset_size = self.num_training_data
x = image_name_array[:self.num_training_data]
y = GT_train_labels[:self.num_training_data]
else: # load features
dataset_size = self.num_training_features
x = train_features[:self.num_training_features]
y = GT_train_labels[:self.num_training_features]
elif dataset_name == 'validation':
if dataset_loc in ['hand', 'head']:
dataset_size = self.num_validation_data
x = image_name_array[self.num_training_data:]
y = GT_train_labels[self.num_training_data:]
else: # load features
dataset_size = self.num_validation_features
x = train_features[self.num_training_features:]
y = GT_train_labels[self.num_training_features:]
else: # test set
if dataset_loc in ['hand', 'head']:
dataset_size = self.num_test_data
x = image_name_array
y = GT_test_labels
else: # load features
dataset_size = self.num_test_features
x = test_features
y = GT_test_labels
if to_shuffle: # shuffle in mini-batch
if dataset_name in ['validation', 'test']:
prng = np.random.RandomState(1234) # random seed
else: # train set
prng = np.random
rand_perm_array = prng.permutation(dataset_size)
x = x[rand_perm_array]
y = y[rand_perm_array]
if for_head: # multi-label
y_to_categorical = np.zeros((y.size, num_classes))
y_to_categorical[range(y.size), (y % num_classes)] = 1
y_to_categorical[range(y.size), (y // num_classes)] = 1
y = y_to_categorical
else:
y = np_utils.to_categorical(
y, num_classes=num_classes) # labels to one-hot vectors
i = 0
while True:
if i + batch_size > dataset_size:
i = 0
if dataset_loc in ['hand', 'head']:
x_batch = load_images(self.zip_ref_frames,
self.img_height,
self.img_width,
image_name_array=x[i:(i + batch_size)])
else: # load features
x_batch = x[i:(i + batch_size)]
y_batch = y[i:(i + batch_size)]
yield x_batch, y_batch
i += batch_size
def train():
train_dir = './data'
logs_train_dir = './logs/train/'
#logs_val_dir = './logs/val/'
train, train_label, val, val_label = input_train_val_split_keras.get_Data(
train_dir, RATIO)
# Convert class vectors to binary class matrices.
train_label = np_utils.to_categorical(train_label, N_CLASSES)
val_label = np_utils.to_categorical(val_label, N_CLASSES)
# Input image dimensions.
if K.image_data_format() == 'channels_first':
input_shape = (3, IMG_W, IMG_H)
else:
input_shape = (IMG_W, IMG_H, 3)
model = model_keras.inference(input_shape=input_shape, N_CLASSES=N_CLASSES)
model.compile(
loss='categorical_crossentropy',
#optimizer=Adam(lr = model_keras.lr_schedule(0)),
optimizer='Adam',
metrics=['accuracy'])
model.summary()
# Prepare model model saving directory.
save_dir = os.path.join(logs_train_dir, 'saved_models')
#model_name = 'cifar10_%s_model.{epoch:03d}.h5' % 'test'
model_name = 'weights.{epoch:02d}-{val_acc:.2f}.hdf5'
model_name_first = 'first_try.h5'
print(model_name)
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)
filepath_first = os.path.join(save_dir, model_name_first)
# Prepare callbacks for model saving and for learning rate adjustment.
checkpoint = ModelCheckpoint(filepath=filepath,
monitor='val_acc',
verbose=1,
save_best_only=True)
lr_scheduler = LearningRateScheduler(model_keras.lr_schedule)
#lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
# cooldown=0,
# patience=5,
# min_lr=0.5e-6)
#callbacks = [checkpoint, lr_reducer, lr_scheduler]
callbacks = [checkpoint, lr_scheduler]
# Run training, with or without data augmentation.
if not data_augmentation:
print('Not using data augmentation.')
#print('train.shape = {}'.format(train.shape))
#print('train_label.shape = {}'.format(train_label.shape))
#print('val.shape = {}'.format(val.shape))
#print('val_label.shape = {}'.format(val_label.shape))
train_history = model.fit(train,
train_label,
batch_size=BATCH_SIZE,
epochs=epochs,
validation_data=(val, val_label),
shuffle=True,
callbacks=callbacks)
else:
print('Using real-time data augmentation.')
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False)
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(train)
# Fit the model on the batches generated by datagen.flow().
train_history = model.fit_generator(datagen.flow(
train, train_label, batch_size=BATCH_SIZE),
validation_data=(val, val_label),
epochs=epochs,
verbose=1,
workers=4,
callbacks=callbacks)
model.save_weights(filepath_first)
# Score trained model.
scores = model.evaluate(train, train_label, verbose=1)
print('\r\nTrain loss:', scores[0])
print('\r\nTrain accuracy:', scores[1])
scores = model.evaluate(val, val_label, verbose=1)
print('\r\nTest loss:', scores[0])
print('\r\nTest accuracy:', scores[1])
show_train_history(train_history, 'acc', 'val_acc')
def run_test_process(model_name,
batch_size,
epochs,
num_fc_neurons,
dropout_rate,
zip_ref_frames,
zip_ref_labels,
output_path,
mode='obj',
setting_index=0,
num_classes=24,
img_height=None,
img_width=None):
if img_height == None or img_width == None: # default model input shape
img_height, img_width = get_model_input_shape(model_name)
# create DataGenerator
d_gen = DataGenerator(zip_ref_frames,
zip_ref_labels,
img_height,
img_width,
batch_size=1)
for dataset_loc in ['hand', 'head']:
# settings for path
fine_tune_model_weights_path = os.path.join(
output_path, 'fine_tune_weights_' + model_name + '_' +
dataset_loc + '_' + str(epochs) + '.h5')
test_fc_features_path = os.path.join(
output_path,
'test_fc_features_' + model_name + '_' + dataset_loc + '.npy')
probas_pred_test = predict_model_fc_features(
model_name, d_gen, num_fc_neurons, dropout_rate, num_classes,
fine_tune_model_weights_path, test_fc_features_path, img_height,
img_width, dataset_loc)
# concatenate two stream output features
test_fc_hand_features_path = os.path.join(
output_path, 'test_fc_features_' + model_name + '_hand.npy')
test_fc_head_features_path = os.path.join(
output_path, 'test_fc_features_' + model_name + '_head.npy')
concatenate_test_fc_features_path = os.path.join(
output_path, 'concatenate_test_fc_features_' + model_name + '.npy')
concatenate_hand_and_head_features(test_fc_hand_features_path,
test_fc_head_features_path,
concatenate_test_fc_features_path)
two_stream_classifier_weights_path = os.path.join(
output_path, 'two_stream_classifier_weights_' + model_name + '.h5')
probas_pred_test_path = os.path.join(
output_path, 'probas_pred_test_' + model_name + '.npy')
predict_two_stream_classifier(d_gen, num_fc_neurons, dropout_rate,
num_classes,
two_stream_classifier_weights_path,
concatenate_test_fc_features_path,
probas_pred_test_path)
# evaluate on test set
probas_pred_test = np.load(open(
probas_pred_test_path, 'rb')) # load test prediction (probability)
pred_test_labels = np.argmax(probas_pred_test, axis=1).astype(
int
) # change probability to class label, axis=1 : row, determine predicted class number
GT_test_labels = load_test_labels(zip_ref_labels,
mode,
setting_index=setting_index)
GT_test_labels_one_hot = np_utils.to_categorical(
GT_test_labels, num_classes=num_classes) # labels to one-hot vectors
evaluate_acc(GT_test_labels, pred_test_labels)
plot_precision_recall_curve(GT_test_labels_one_hot, probas_pred_test,
num_classes)
obj_classes = [
'free', 'computer', 'cellphone', 'coin', 'ruler', 'thermos-bottle',
'whiteboard-pen', 'whiteboard-eraser', 'pen', 'cup',
'remote-control-TV', 'remote-control-AC', 'switch', 'windows',
'fridge', 'cupboard', 'water-tap', 'toy', 'kettle', 'bottle', 'cookie',
'book', 'magnet', 'lamp-switch'
] # np.arange(num_classes)
plot_confusion_matrix(GT_test_labels,
pred_test_labels,
classes=obj_classes,
normalize=False,
title='Confusion Matrix')