def build_recognition_system(vgg16,num_workers=2):
'''
Creates a trained recognition system by generating training features from all training images.
[input]
* vgg16: prebuilt VGG-16 network.
* num_workers: number of workers to process in parallel
[saved]
* features: numpy.ndarray of shape (N,K)
* labels: numpy.ndarray of shape (N)
'''
# ----- TODO -----
num_workers=util.get_num_CPU()
vgg16_weights=util.get_VGG16_weights()
train_data = np.load("../data/train_data.npz")
train_name,labels = train_data['image_names'],train_data['labels']
lenth=len(train_name)
vgg16=torchvision.models.vgg16(pretrained=True)
features=[]
for i in range(0,lenth):
image_path = os.path.join('../data/',train_name[i][0])
args=[i,image_path,vgg16]
single_feature=get_image_feature(args)
features.append(single_feature)
features=np.asarray(features)
#np.savez('../code/trained_system_deep.npz', features=features, labels=labels)
print('saved train_system_deep.npz')
return features,labels
def evaluate_recognition_system(vgg16, num_workers=2):
'''
Evaluates the recognition system for all test images and returns the confusion matrix.
[input]
* vgg16: prebuilt VGG-16 network.
* num_workers: number of workers to process in parallel
[output]
* conf: numpy.ndarray of shape (8,8)
* accuracy: accuracy of the evaluated system
'''
trained_system = None
if DEBUGGING:
trained_system = np.load("trained_system_deep_torch.npz")
else:
trained_system = np.load("trained_system_deep.npz") # testing
test_data = np.load("../data/test_data.npz", allow_pickle=True)
T = test_data['image_names'].shape[0]
# test
T = T
paths = test_data['image_names']
features = trained_system['features']
print(features.shape)
labels = test_data['labels']
C = np.zeros((8, 8))
if DEBUGGING:
for i in range(0, T):
feature = get_image_feature(
(i, '../data/'.strip() + paths[i][0], vgg16))
near_feature = np.argmax(distance_to_set(feature, features))
print(near_feature, trained_system['labels'][near_feature])
predict = trained_system['labels'][near_feature]
C[labels[i], predict] += 1
else:
pool = multiprocessing.Pool(num_workers)
results = []
weight = util.get_VGG16_weights()
for i in range(0, T):
args = [(i, '../data/'.strip() + paths[i][0], weight)]
results.append(pool.apply_async(get_image_feature, args))
i = 0
for result in results:
feature = result.get()
dist = distance_to_set(feature, features) # neg
near_feature = np.argmax(dist)
print(i, trained_system['labels'][near_feature])
predict = trained_system['labels'][near_feature]
C[labels[i], predict] += 1
i += 1
accuracy = np.trace(C) / np.sum(C)
return C, accuracy
def extract_deep_feature(x, vgg16_weights):
weights = util.get_VGG16_weights()
#preprocessing:
x = x.astype('float') / 255
x = skimage.transform.resize(x, (224, 224, 3), preserve_range=True)
mean = np.array([0.485, 0.456, 0.406]).reshape(1, 1, 3)
std = np.array([0.229, 0.224, 0.225]).reshape(1, 1, 3)
x = (x - mean) / std
count = 0
for l in weights:
# =============================================================================
# if c ==10:
# break
# =============================================================================
if l[0] == 'conv2d':
print("Convolution Time")
x = multichannel_conv2d(x, l[1], l[2])
if count == 2:
print("breakinnn")
break
if l[0] == 'relu':
print("Relu Time!")
x = relu(x)
if l[0] == 'maxpool2d':
print("Pool Time!")
x = max_pool2d(x, l[1])
if l[0] == 'linear':
print("Linear Time! entering once")
count += 1
if count == 1:
x = x.T
x = np.einsum('kli->kil', x)
x = linear(x, l[1], l[2])
print("image is : ", x)
print(x.shape)
# np.save("convo",x)
return x
def evaluate_one_image(args):
train_features = np.load("trained_system_deep.npz")['features']
labels = np.load("trained_system_deep.npz")['labels']
i, image_path = args
img_path = "../data/"+image_path[0]
image = imageio.imread(img_path)
img = preprocess_image(image)
weights = util.get_VGG16_weights()
feature = network_layers.extract_deep_feature(img, weights)
label = labels[np.argmax(distance_to_set(feature, train_features))]
print("Evaluation done for image ", i)
np.save("../temp/"+"predicted_label_deep_"+str(i)+".npy", label)
def evaluate_recognition_system(vgg16,num_workers=2):
'''
Evaluates the recognition system for all test images and returns the confusion matrix.
[input]
* vgg16: prebuilt VGG-16 network.
* num_workers: number of workers to process in parallel
[output]
* conf: numpy.ndarray of shape (8,8)
* accuracy: accuracy of the evaluated system
'''
# ----- TODO -----
test_data = np.load("../data/test_data.npz")
test_data, predicted_labels = test_data['image_names'], test_data['labels']
trained_system = np.load("../code/trained_system_deep.npz")
train_features,train_labels = trained_system['features'],trained_system['labels']
vgg16_weights=util.get_VGG16_weights()
labels = []
i=0
for i in range(0,len(test_data)):
print('vgg recognization:',i)
image_path = os.path.join('../data/',test_data[i][0])
args=[i,image_path,vgg16]
feature = get_image_feature1(args)
dist = distance_to_set(feature,train_features)
distance_index=np.argmin(dist)
recognized_deep_labels=train_labels[distance_index]
labels.append(recognized_deep_labels)
recognized_deep_labels = np.array(labels)
conf = np.zeros((8,8))
all_num=len(predicted_labels)
for x in range(0,all_num):
i=predicted_labels[x]
j=labels[x]
conf[i,j] = conf[i,j]+1
correct_num=np.trace(conf)
accuracy = correct_num/all_num
print('Accuracy is', accuracy*100,'%')
print(conf)
return conf,accuracy
pass
def test_conv():
vgg16_weights = util.get_VGG16_weights()
x = np.random.rand(227, 227, 3).astype(np.float32)
weight = vgg16_weights[0][1]
bias = vgg16_weights[0][2]
response_mine = multichannel_conv2d(x, weight, bias)
t_x = torch.Tensor(np.transpose(x, [2, 0, 1])[np.newaxis, :, :, :])
t_w = torch.Tensor(weight)
t_b = torch.Tensor(bias)
t_response_pt = torch.nn.functional.conv2d(t_x, t_w, t_b, padding=1)
response_pt = t_response_pt.numpy()
response_pt = np.transpose(response_pt[0, :, :, :], [1, 2, 0])
error = np.sum(np.abs(response_mine - response_pt)) / np.sum(
np.abs(response_pt))
print('conv: %f' % error)
def build_recognition_system(vgg16, num_workers=2):
'''
Creates a trained recognition system by generating training features from all training images.
[input]
* vgg16: prebuilt VGG-16 network.
* num_workers: number of workers to process in parallel
[saved]
* features: numpy.ndarray of shape (N,K)
* labels: numpy.ndarray of shape (N)
'''
train_data = np.load("../data/train_data.npz", allow_pickle=True)
T = train_data['image_names'].shape[0]
paths = train_data['image_names']
features = None
labels = train_data['labels']
# one processer, vgg version
# vgg does not work after copy??
if DEBUGGING:
features = []
for i in range(T):
features.append(
get_image_feature(
(i, '../data/'.strip() + paths[i][0], vgg16)))
features = np.array(features)
# np.savez('trained_system_deep_torch.npz', features=features, labels=labels)
else:
pool = multiprocessing.Pool(num_workers)
results = []
weight = util.get_VGG16_weights()
for i in range(0, T):
args = [(i, '../data/'.strip() + paths[i][0], weight)]
results.append(pool.apply_async(get_image_feature, args))
features = []
for result in results:
feature = result.get()
features.append(feature)
features = np.array(features)
def get_image_feature1(args):
'''
Extracts deep features from the prebuilt VGG-16 network.
This is a function run by a subprocess.
[input]
* i: index of training image
* image_path: path of image file
* vgg16: prebuilt VGG-16 network.
[saved]
* x: evaluated deep feature
'''
i,image_path,vgg16 = args
# ----- TODO -----
vgg16_weights=util.get_VGG16_weights()
path = os.path.join('../data/',image_path)
image = imageio.imread(path)
image=preprocess_image(image)
x = vgg16.features(image.double()).detach().numpy() #x is the conv output of first 30 laryers
## go through the rest layers of Vgg16
layer_num=len(vgg16_weights)
for j in range(31,layer_num):
layer_name=vgg16_weights[j][0]
if layer_name=='relu':
x = network_layers.relu(x)
x = np.asarray(x)
if layer_name=='linear':
weights=vgg16_weights[j][1]
bias=vgg16_weights[j][2]
x=np.ndarray.flatten(x)
x = network_layers.linear(x,weights,bias)
print('get deep feature:',i)
del image
return x
pass
def predict_image(args):
'''
Predicts the label using the trained system and extracted VGG-16 features
This is a function run by a subprocess.
[input]
* image_path: path of image file
* features: trained features from VGG-16
* train_labels: trained set of labels
* vgg16: prebuilt VGG-16 network
[output]
* predicted_label: int representing the predicted label
'''
global PROGRESS
with PROGRESS_LOCK:
PROGRESS += NPROC
image_path, features, train_labels, vgg16 = args
print('Processing: %03d/160 | Image: %s' % (PROGRESS, image_path))
# Read and preprocess image
image = imageio.imread('../data/' + image_path)
image = preprocess_image(image)
# Extract deep features
if USE_PYTORCH:
feat = vgg16.features(image).flatten() # VGG-16 Features
feat = vgg16.classifier[:5](
feat).detach().numpy().flatten() # VGG-16 Classifiers
else:
feat = network_layers.extract_deep_feature(image,
util.get_VGG16_weights())
# Find the predicted label
predicted_label = train_labels[np.argmin(distance_to_set(feat, features))]
return predicted_label
def get_image_feature(args):
'''
Extracts deep features from the prebuilt VGG-16 network
This is a function run by a subprocess
[input]
* i: index of training image
* image_path: path of image file
* vgg16: prebuilt VGG-16 network
[saved]
* feat: evaluated deep feature
'''
global PROGRESS
with PROGRESS_LOCK:
PROGRESS += NPROC
i, image_path, vgg16 = args
print('Processing: %04d/1440 | Index: %04d | Image: %s' %
(PROGRESS, i, image_path))
# Read and preprocess image
image = imageio.imread('../data/' + image_path)
image = preprocess_image(image)
# Extract deep features
if USE_PYTORCH:
feat = vgg16.features(image).flatten() # VGG-16 Features
feat = vgg16.classifier[:5](
feat).detach().numpy().flatten() # VGG-16 Classifiers
else:
feat = network_layers.extract_deep_feature(image,
util.get_VGG16_weights())
# Save the extracted features
np.save('%s/%d' % (TEMP_PATH, i), feat)
return feat
def get_image_feature(args):
'''
Extracts deep features from the prebuilt VGG-16 network.
This is a function run by a subprocess.
[input]
* i: index of training image
* image_path: path of image file
* vgg16: prebuilt VGG-16 network.
* time_start: time stamp of start time
[saved]
* feat: evaluated deep feature
'''
i, image_path = args
img_path = "../data/"+image_path[0]
image = imageio.imread(img_path)
img = preprocess_image(image)
weights = util.get_VGG16_weights()
feature = network_layers.extract_deep_feature(img, weights)
np.save("../temp/"+"image_feature_"+str(i)+".npy", feature)
import torch
import skimage.transform
import torchvision.transforms
import util
import network_layers
import deep_recog
import skimage.transform
from skimage import data
from skimage.transform import resize
from skimage import io
# image = data.camera()
# img = resize(image, (20, 20))
# print(img)
vgg16_weights = util.get_VGG16_weights()
# print(len(vgg16_weights))
# # print(vgg16_weights[0])
# for i in range(len(vgg16_weights)):
# name = vgg16_weights[i][0]
# if name == 'maxpool2d':
# print(vgg16_weights[i][1].shape)
# print(vgg16_weights[i][2].shape)
path_img = "../data/aquarium/sun_aztvjgubyrgvirup.jpg"
image = io.imread(path_img)
image = image.astype('float') / 255
img = deep_recog.preprocess_image(image)
diff = deep_recog.evaluate_deep_extractor(img, vgg16_weights)