def test_compute_image_resize_params():
# Case-1: Compute median shape for smaller (3-D) images.
data = np.array([np.random.randint(256, size=(5, 5, 5, 3)),
np.random.randint(256, size=(6, 6, 6, 3)),
np.random.randint(256, size=(7, 7, 7, 3)),
np.random.randint(256, size=(8, 8, 8, 3))])
resize_shape = compute_image_resize_params(data)
assert np.array_equal(resize_shape, [6, 6, 6, 3])
modified_data = resize_image_data(data, resize_shape)
for image in modified_data:
assert np.array_equal(image.shape, [6, 6, 6, 3])
# Case-2: Resize to max size for larger (2-D) images.
data = np.array([np.random.randint(256, size=(int(np.sqrt(Constant.MAX_IMAGE_SIZE) + 1),
int(np.sqrt(Constant.MAX_IMAGE_SIZE) + 1),
3))])
resize_shape = compute_image_resize_params(data)
assert np.array_equal(resize_shape, (int(np.sqrt(Constant.MAX_IMAGE_SIZE)),
int(np.sqrt(Constant.MAX_IMAGE_SIZE)),
3))
modified_data = resize_image_data(data, resize_shape)
for image in modified_data:
assert np.array_equal(image.shape, resize_shape)
def test_compute_image_resize_params():
# Case-1: Compute median height and width for smaller images.
data = numpy.array([
numpy.random.randint(256, size=(10, 10, 3)),
numpy.random.randint(256, size=(20, 20, 3)),
numpy.random.randint(256, size=(30, 30, 3)),
numpy.random.randint(256, size=(40, 40, 3))
])
resize_height, resize_width = compute_image_resize_params(data)
assert resize_height == 25
assert resize_width == 25
modified_data = resize_image_data(data, resize_height, resize_width)
for image in modified_data:
assert image.shape == (25, 25, 3)
# Case-2: Resize to max size for larger images.
data = numpy.array([
numpy.random.randint(
256,
size=(int(numpy.sqrt(Constant.MAX_IMAGE_SIZE) + 1),
int(numpy.sqrt(Constant.MAX_IMAGE_SIZE) + 1), 3))
])
resize_height, resize_width = compute_image_resize_params(data)
assert resize_height == int(numpy.sqrt(Constant.MAX_IMAGE_SIZE))
assert resize_width == int(numpy.sqrt(Constant.MAX_IMAGE_SIZE))
modified_data = resize_image_data(data, resize_height, resize_width)
for image in modified_data:
assert image.shape == (resize_height, resize_width, 3)
def final_fit(self, x_train, y_train, x_test, y_test, trainer_args=None, retrain=False):
"""Final training after found the best architecture.
Args:
x_train: A numpy.ndarray of training data.
y_train: A numpy.ndarray of training targets.
x_test: A numpy.ndarray of testing data.
y_test: A numpy.ndarray of testing targets.
trainer_args: A dictionary containing the parameters of the ModelTrainer constructor.
retrain: A boolean of whether reinitialize the weights of the model.
"""
if trainer_args is None:
trainer_args = {'max_no_improvement_num': 30}
if len(x_train.shape) != 0 and len(x_train[0].shape) == 3:
x_train = resize_image_data(x_train, self.resize_height, self.resize_width)
if x_test is not None:
x_test = resize_image_data(x_test, self.resize_height, self.resize_width)
y_train = self.transform_y(y_train)
y_test = self.transform_y(y_test)
train_data = self.data_transformer.transform_train(x_train, y_train)
test_data = self.data_transformer.transform_test(x_test, y_test)
self.cnn.final_fit(train_data, test_data, trainer_args, retrain)
def test_compute_image_resize_params():
# Case-1: Compute median shape for smaller (3-D) images.
data = np.array([
np.random.randint(256, size=(5, 5, 5, 3)),
np.random.randint(256, size=(6, 6, 6, 3)),
np.random.randint(256, size=(7, 7, 7, 3)),
np.random.randint(256, size=(8, 8, 8, 3))
])
resize_shape = compute_image_resize_params(data)
assert np.array_equal(resize_shape, [6, 6, 6, 3])
modified_data = resize_image_data(data, resize_shape)
for image in modified_data:
assert np.array_equal(image.shape, [6, 6, 6, 3])
# Case-2: Resize to max size for larger (2-D) images.
data = np.array([
np.random.randint(256,
size=(int(np.sqrt(Constant.MAX_IMAGE_SIZE) + 1),
int(np.sqrt(Constant.MAX_IMAGE_SIZE) + 1), 3))
])
resize_shape = compute_image_resize_params(data)
assert np.array_equal(resize_shape, (int(np.sqrt(
Constant.MAX_IMAGE_SIZE)), int(np.sqrt(Constant.MAX_IMAGE_SIZE)), 3))
modified_data = resize_image_data(data, resize_shape)
for image in modified_data:
assert np.array_equal(image.shape, resize_shape)
def fit(self, x, y, x_test=None, y_test=None, time_limit=None):
x = np.array(x)
y = np.array(y).flatten()
if self.verbose:
print("Preprocessing the images.")
self.resize_shape = compute_image_resize_params(x)
x = resize_image_data(x, self.resize_shape)
x_test = resize_image_data(x_test, self.resize_shape)
if self.verbose:
print("Preprocessing finished.")
super().fit(x, y, x_test, y_test, time_limit)
def preprocess(self, x):
"""THIS FUNCTION NEEDS A DESCRIPTION.
Args:
x: DATA TO RESIZE, IS IT A LIST OR A SINGLE IMAGE?
"""
return resize_image_data(x, self.resize_shape)
def fit(self, x, y, time_limit=None):
"""Find the best neural architecture for classifying the training data and train it.
Based on the given dataset, the function will find the best neural architecture for it.
The dataset must be in numpy.ndarray format.
The training and validation data should be passed through `x`, `y`. This method will automatically split
the training and validation data into training and validation sets.
Args:
x: A numpy.ndarray instance containing the training data or the training data combined with the
validation data.
y: A numpy.ndarray instance containing the labels of the training data. or the label of the training data
combined with the validation label.
time_limit: The time limit for the search in seconds. (optional, default = None, which turns into 24 hours in method)
Effects:
Trains a model that fits the data using the best neural architecture
"""
x = np.array(x)
y = np.array(y)
if self.verbose:
print("Preprocessing the images.")
self.resize_shape = compute_image_resize_params(x)
x = resize_image_data(x, self.resize_shape)
if self.verbose:
print("Preprocessing finished.")
super().fit(x, y, time_limit)
def evaluate(self, x_test, y_test):
"""Return the accuracy score between predict value and `y_test`."""
if len(x_test.shape) != 0 and len(x_test.shape) == 3:
x_test = resize_image_data(x_test, self.resize_height,
self.resize_width)
y_predict = self.predict(x_test)
return self.metric().evaluate(y_test, y_predict)
def fit(self, x, y, x_test=None, y_test=None, time_limit=None):
x = np.array(x)
if len(x.shape) != 0 and len(x[0].shape) == 3:
if self.verbose:
print("Preprocessing the images.")
self.resize_height, self.resize_width = compute_image_resize_params(
x)
x = resize_image_data(x, self.resize_height, self.resize_width)
if x_test is not None:
x_test = resize_image_data(x_test, self.resize_height,
self.resize_width)
if self.verbose:
print("Preprocessing finished.")
y = np.array(y).flatten()
validate_xy(x, y)
y = self.transform_y(y)
if x_test is None or y_test is None:
# Divide training data into training and testing data.
validation_set_size = int(len(y) * Constant.VALIDATION_SET_SIZE)
validation_set_size = min(validation_set_size, 500)
validation_set_size = max(validation_set_size, 1)
x_train, x_test, y_train, y_test = train_test_split(
x, y, test_size=validation_set_size, random_state=42)
else:
x_train = x
y_train = y
# Transform x_train
if self.data_transformer is None:
self.data_transformer = ImageDataTransformer(x,
augment=self.augment)
# Wrap the data into DataLoaders
train_data = self.data_transformer.transform_train(x_train, y_train)
test_data = self.data_transformer.transform_test(x_test, y_test)
# Save the classifier
pickle_to_file(self, os.path.join(self.path, 'classifier'))
if time_limit is None:
time_limit = 24 * 60 * 60
self.cnn.fit(self.get_n_output_node(), x_train.shape, train_data,
test_data, time_limit)
def fit(self, x, y, x_test=None, y_test=None, time_limit=None):
x = np.array(x)
y = np.array(y).flatten()
if self.verbose:
print("Preprocessing the images.")
if x is not None and (len(x.shape) == 4 or len(x.shape) == 1 and len(x[0].shape) == 3):
self.resize_height, self.resize_width = compute_image_resize_params(x)
if self.resize_height is not None:
x = resize_image_data(x, self.resize_height, self.resize_width)
print("x is ", x.shape)
if self.resize_height is not None:
x_test = resize_image_data(x_test, self.resize_height, self.resize_width)
if self.verbose:
print("Preprocessing finished.")
super().fit(x, y, x_test, y_test, time_limit)
def fit(self, x, y, time_limit=None):
x = np.array(x)
y = np.array(y)
if self.verbose:
print("Preprocessing the images.")
self.resize_shape = compute_image_resize_params(x)
x = resize_image_data(x, self.resize_shape)
if self.verbose:
print("Preprocessing finished.")
super().fit(x, y, time_limit)
def predict(self, x_test):
"""Return predict results for the testing data.
Args:
x_test: An instance of numpy.ndarray containing the testing data.
Returns:
A numpy.ndarray containing the results.
"""
if Constant.LIMIT_MEMORY:
pass
x_test = resize_image_data(x_test, self.resize_shape)
test_loader = self.data_transformer.transform_test(x_test)
model = self.graph.produce_model()
model.eval()
outputs = []
with torch.no_grad():
for index, inputs in enumerate(test_loader):
outputs.append(model(inputs).numpy())
output = reduce(lambda x, y: np.concatenate((x, y)), outputs)
return self.inverse_transform_y(output)
def preprocess(self, x):
return resize_image_data(x, self.resize_shape)
def preprocess(self, x):
return resize_image_data(x, self.resize_shape)
def evaluate(self, x_test, y_test):
"""Return the accuracy score between predict value and `y_test`."""
x_test = resize_image_data(x_test, self.resize_shape)
y_predict = self.predict(x_test)
return self.metric().evaluate(y_test, y_predict)
def preprocess(self, x):
if len(x.shape) != 0 and len(x[0].shape) == 3:
if self.resize_height is not None:
return resize_image_data(x, self.resize_height, self.resize_width)
return x