def fit_output_pca(self, raw_images, dim=20):
if self._X is None:
send_msg(self._log,
"return a empty dict, since the model is not fitted yet")
return dict()
# convolution
I_layer1 = dict()
I_freq = dict()
for i, one_image in enumerate(raw_images):
I_i = conv_layer(one_image, self._filters)
I_layer1[i] = I_i
for j, one_I in enumerate(I_i):
if not j in I_freq:
I_freq[j] = []
I_freq[j].append(one_I.reshape(one_I.size, ))
send_msg(self._log, "conv stage finished")
self._pca_freq = dict()
for i in I_freq:
I_freq[i] = np.asarray(I_freq[i])
# de-mean
I_freq[i] = I_freq[i] - (I_freq[i].mean(axis=1)).reshape(-1, 1)
pca = PCA(n_components=dim)
pca.fit(I_freq[i])
self._pca_freq[i] = pca
def pipeline(train_imgs,
train_labels,
test_imgs,
test_labels,
param: namedtuple,
log=False,
rgb=False):
global RANDOM_STATE
net = PCANet2(l1=param.l1,
patch_size1=param.patch_size1,
stride1=param.stride1,
method1=param.method1,
fourier_basis_sel1=param.fourier_basis_sel1,
l2=param.l2,
patch_size2=param.patch_size2,
stride2=param.stride2,
method2=param.method2,
fourier_basis_sel2=param.fourier_basis_sel2,
reduction_method=param.reduction_method,
block_size=param.block_size,
block_stride=param.block_stride,
feature_method=param.feature_method,
log=log,
rgb=rgb)
train_feats = net.fit_transform(train_imgs, train_labels)
test_feats = net.transform(test_imgs)
train_feats, test_feats = dict_feats_to_np_feats(
train_feats), dict_feats_to_np_feats(test_feats)
scaler = StandardScaler(with_std=False)
zero_mean_train_feats = scaler.fit_transform(train_feats)
X_train = zero_mean_train_feats
y_train = train_labels
zero_mean_test_feats = scaler.fit_transform(test_feats)
X_test = zero_mean_test_feats
y_test = test_labels
# X_train = train_feats
# y_train = train_labels
# X_test = test_feats
# y_test = test_labels
send_msg(log, "training a {} clf ...".format(param.clf))
CLF = clf_dict.get(param.clf, LinearSVC)
clf = CLF(random_state=RANDOM_STATE, max_iter=1000)
# X_train, X_test = sparse.csr_matrix(X_train), sparse.csr_matrix(X_test)
send_msg(
log, "train_feats.shape: {}, test_feats.shape: {}".format(
X_train.shape, X_test.shape))
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)
send_msg(log, net)
send_msg(log, 'Mean Accuracy Score: {}'.format(accuracy))
return accuracy, net, clf
def transform_output_pca(self, raw_images):
if self._pca_freq is None:
send_msg(self._log,
"return a empty dict, since the model is not fitted yet")
return dict()
# convolution
I_layer1 = dict()
feats = [-1] * len(raw_images)
for i, one_image in enumerate(raw_images):
I_i = conv_layer(one_image, self._filters)
feats_one_img = []
for j, one_I in enumerate(I_i):
reduced_feats = self._pca_freq[j].transform(
one_I.reshape(1, -1))
feats_one_img.extend(reduced_feats)
feats[i] = np.asarray(feats_one_img).reshape(-1, )
send_msg(self._log, "conv & output pca stage finished")
return np.asarray(feats)
def pipeline(train_imgs,
train_labels,
test_imgs,
test_labels,
param: namedtuple,
log=False):
global RANDOM_STATE
net = PCANet(l=param.l,
num_images=param.num_images,
patch_size=param.patch_size,
stride=param.stride,
block_size=param.block_size,
block_stride=param.block_stride,
method=param.method,
fourier_basis_sel=param.fourier_basis_sel,
reduction_method=param.reduction_method,
feature_method=param.feature_method,
quantization_method=param.quantization_method,
wavelet=param.wavelet,
wavelet_decomp_level=param.wavelet_decomp_level,
log=log)
train_feats = net.fit_transform(train_imgs, train_labels)
test_feats = net.transform(test_imgs)
train_feats, test_feats = dict_feats_to_np_feats(
train_feats), dict_feats_to_np_feats(test_feats)
send_msg(
log, "train_feats.shape: {}, test_feats.shape: {}".format(
train_feats.shape, test_feats.shape))
scaler = StandardScaler(with_std=False)
zero_mean_train_feats = scaler.fit_transform(train_feats)
X_train = zero_mean_train_feats
y_train = train_labels
zero_mean_test_feats = scaler.fit_transform(test_feats)
X_test = zero_mean_test_feats
y_test = test_labels
CLF = clf_dict.get(param.clf, LinearSVC)
clf = CLF(random_state=RANDOM_STATE, max_iter=2000)
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)
send_msg(log, net)
send_msg(log, 'Mean Accuracy Score: {}'.format(accuracy))
return accuracy, net, clf
def transform(self, raw_images) -> np.ndarray:
send_msg(self._log, "begin transforming")
if self._X is None:
send_msg(self._log,
"return a empty dict, since the model is not fitted yet")
return dict()
send_msg(self._log, "conv stage")
# convolution
I_layer1 = []
kwargs = {}
kwargs['wavelet'] = self._wavelet
kwargs['level'] = self._wavelet_decomp_level
kwargs['patch_size'] = self._patch_size
for i, one_image in enumerate(raw_images):
I_i = conv_layer(one_image, self._filters, self._select_coeffs_idx,
**kwargs)
I_layer1.append(I_i)
send_msg(self._log, "conv stage finished")
send_msg(self._log,
"\tI_layer1.shape: {}".format(np.array(I_layer1).shape))
send_msg(self._log, "reduction stage")
# int_img_dict = dict()
int_img_list = []
# output layer
# idx refers to img idx
for idx in range(len(I_layer1)):
curr_images = I_layer1[idx]
integer_image = None
# quantization method
if self._quantization_method == 'binarize':
curr_images = [
binarize(one_image) for one_image in curr_images
]
elif self._quantization_method == 'relu':
curr_images = [relu(one_image) for one_image in curr_images]
elif self._quantization_method == 'identity':
pass
else:
raise NotImplementedError
if self._reduction_method == 'concat': # concat method offer a upper bound
# int_img_dict[idx] = np.stack(curr_images, 2)
# int_img_list.append(curr_images)
int_img_list.append(np.stack(curr_images, 2))
else:
# FIXME: for exponent method, reverse's behavior is opposite from others, but resembles original paper
for j, one_image in enumerate(reversed(curr_images)):
# reduction method
if integer_image is None:
if self._reduction_method == 'exponent':
integer_image = one_image * (2**0)
elif self._reduction_method == 'add':
integer_image = one_image * 1
elif self._reduction_method == 'linear_add':
integer_image = one_image * 1
elif self._reduction_method == 'square_add':
integer_image = one_image * (1**2)
elif self._reduction_method == 'cube_add':
integer_image = one_image * (1**3)
else:
raise NotImplementedError
else:
if self._reduction_method == 'exponent':
integer_image = integer_image * 2 + one_image
elif self._reduction_method == 'add':
integer_image = integer_image + one_image
elif self._reduction_method == 'linear_add':
integer_image = integer_image + one_image * (j + 1)
elif self._reduction_method == 'square_add':
integer_image = integer_image + np.round(
one_image * (j + 1)**2 / 2).astype(int)
elif self._reduction_method == 'cube_add':
integer_image = integer_image + np.round(
one_image * (j + 1)**3 / 3).astype(int)
else:
raise NotImplementedError
# int_img_dict[idx] = integer_image
int_img_list.append(integer_image)
send_msg(self._log, "reduction stage finished")
send_msg(
self._log,
"\tint_img_list.shape: {}".format(np.array(int_img_list).shape))
send_msg(self._log, "feature stage")
# feats = dict()
# feats = [-1] * len(int_img_list)
feats = []
for idx in range(len(int_img_list)):
# integer_image = int_img_dict[idx]
integer_image = int_img_list[idx]
all_vec_bhist = []
for i in range(0, integer_image.shape[0] - self._block_size[0] + 1,
self._block_stride):
for j in range(
0, integer_image.shape[1] - self._block_size[1] + 1,
self._block_stride):
curr_block = integer_image[i:i + self._block_size[0], j:j + self._block_size[1]]\
.reshape(*self._block_size, -1)
# feature method
if self._feature_method == 'histogram':
counter = Counter(
curr_block.reshape((curr_block.size, )))
block_hist = dict(counter)
# vectorize bhist
if self._reduction_method == 'exponent':
vec_bhist = np.zeros((1 << self._l, ))
elif self._reduction_method == 'add':
vec_bhist = np.zeros((self._l + 1, ))
elif self._reduction_method == 'linear_add':
vec_bhist = np.zeros([
round((self._l * (self._l + 1)) / 2) + 1,
])
elif self._reduction_method == 'square_add':
vec_bhist = np.zeros([
round((self._l * (self._l + 1) *
(self._l + 0.5)) / 9) + self._l**2 + 1,
])
elif self._reduction_method == 'cube_add':
vec_bhist = np.zeros([
int((self._l * (self._l + 1)) / 2)**2 + 1,
])
for hist_bin, val in block_hist.items():
vec_bhist[hist_bin] = val
all_vec_bhist.extend(vec_bhist)
elif self._feature_method == 'avg_pooling':
block_avg = np.mean(curr_block)
all_vec_bhist.append(block_avg)
elif self._feature_method == 'max_pooling':
block_max = np.max(curr_block)
all_vec_bhist.append(block_max)
elif self._feature_method == 'min_pooling':
block_min = np.min(curr_block)
all_vec_bhist.append(block_min)
elif self._feature_method == 'variance_pooling':
block_var = np.var(curr_block)
all_vec_bhist.append(block_var)
elif self._feature_method == 'max_avg_min_pooling':
block_avg = np.mean(curr_block)
block_max = np.max(curr_block)
block_min = np.min(curr_block)
all_vec_bhist.extend([block_avg, block_max, block_min])
elif self._feature_method == 'quantile':
quantile_list = np.linspace(0, 1, self._l)
block_quantile = np.quantile(curr_block,
quantile_list).tolist()
all_vec_bhist.extend(
block_quantile +
[np.mean(curr_block),
np.var(curr_block)])
elif self._feature_method == 'naive_quantile': # min, max, mean, var
for dim in range(curr_block.shape[2]):
block_2d = curr_block[..., dim]
quantile_list = np.linspace(0, 1, 2)
block_quantile = np.quantile(
block_2d, quantile_list).tolist()
all_vec_bhist.extend(
block_quantile +
[np.mean(block_2d),
np.var(block_2d)])
elif self._feature_method == 'exponent_quantile':
quantile_list = np.linspace(0, 1, 2**self._l)
block_quantile = np.quantile(curr_block,
quantile_list).tolist()
all_vec_bhist.extend(
block_quantile +
[np.mean(curr_block),
np.var(curr_block)])
else:
raise NotImplementedError
# feats[idx] = all_vec_bhist
feats.append(all_vec_bhist)
return np.array(feats)
def fit(self, imgs: (list, np.ndarray), labels: (list, np.ndarray)):
send_msg(self._log, "begin fitting")
assert len(imgs) == len(
labels), "len(imgs) = {} while len(labels) = {}".format(
len(imgs), len(labels))
send_msg(self._log, 'processing images...')
X = process_images(imgs, self._patch_size, self._stride)
send_msg(self._log, "X.shape: {}".format(X.shape))
# as matrix transform representation is currently not supported, Wavelets based filter
# get_w will give the selected indices of selected coefficients instead of convolution filters
if self._method == 'Wavelet':
send_msg(self._log, "constructing filters")
l_a = int(self._wavelet_la * self._l)
l_b = self._l - l_a
select_coeffs_idx, _ = get_w(
X, (l_a, l_b),
patch_size=self._patch_size,
method=self._method,
wavelet=self._wavelet,
wavelet_decomp_level=self._wavelet_decomp_level,
labels=labels)
self._select_coeffs_idx = select_coeffs_idx
else:
send_msg(self._log, "constructing filters")
w_l1 = get_w(X,
self._l,
patch_size=self._patch_size,
method=self._method,
fourier_basis_sel=self._fourier_basis_sel,
labels=labels)
# construct stage 1 filters
self._filters = w_l1
send_msg(
self._log,
'\tfilter.shape: {}'.format(np.array(self._filters).shape))
self._X = X
def transform(self, raw_images) -> (np.ndarray, sparse.csr_matrix):
send_msg(self._log, "raw_images.shape: {}".format(raw_images.shape))
send_msg(self._log, "begin transforming")
if self._filters1 is None or self._filters2 is None:
send_msg(
self._log,
"return a empty feats, since the model is not fitted yet")
return []
# convolution, every img corresponds to l1l2 feature maps
O = [] # N x L1 x L2 x h x w
send_msg(self._log, "\tconv stage")
for i, one_image in enumerate(raw_images): # i corresponds to img idx
# stage 1 conv
I_i = conv_layer(
one_image, self._filters1, complex=self._complex
) # L1 x h x w, each img generates L1 stage-1 feature maps
O_i = [] # L1 x L2 x h x w
assert len(I_i) == self._l1, len(I_i)
for l in range(self._l1):
stage1_feature_map_l = I_i[l]
O_i_l = conv_layer(
stage1_feature_map_l,
self._filters2,
complex=self._complex
) # L2 x h x w, each stage-1 map generates L2 stage-2 feature maps
O_i.append(O_i_l)
O.append(O_i)
O = np.array(O)
# send_msg(self._log, "\tO.shape: {}".format(np.array(O).shape))
# # FIXME: test code, pool before redecution
# O = block_reduce(O, block_size=(1, 1, 1, 2, 2), func=np.max)
# send_msg(self._log, "\tafter max pooling: O.shape: {}".format(np.array(O).shape))
send_msg(self._log, "\treduction stage")
T = [] # N x L1 x h x w
for i in range(len(raw_images)):
O_i = O[i] # L1 x L2 x h x w
# quantalization phase
O_i = np.array(O_i)
O_i = np.where(O_i > 0, np.ones_like(O_i),
np.zeros_like(O_i)) # binarize
# reduction phase
O_i_0 = O_i[0] # L2 x h x w
l2, h, w = O_i_0.shape
assert l2 == self._l2
T_i = [] # L1 x h x w
for l in range(self._l1):
O_i_l = O_i[l] # L2 x h x w
exponent_mat = (2**np.arange(0, self._l2,
1)).reshape([-1, 1,
1]) # L2 x 1 x 1
integer_image_l = np.sum(exponent_mat * O_i_l,
axis=0).astype(int) # h x w
T_i.append(integer_image_l)
T.append(T_i)
T = np.array(T)
send_msg(self._log,
"\tT.shape: {}".format(np.array(T).shape)) # N x L1 x h x w
send_msg(self._log, "\tfeature stage")
h, w = T.shape[2:4]
# FIXME: only for cifar method
if self._feature_method == 'spp_histogram':
n_pooled_feature = sum([item**2 for item in self._levels])
feats = np.zeros(
[T.shape[0], n_pooled_feature * self._l1 * (2**self._l2)])
# pre-compute B
B = int((1 + (h - self._block_size[0]) / self._block_stride) *
(1 + (w - self._block_size[1]) / self._block_stride))
send_msg(self._log,
"\tnumber blocks in each feature map: {}".format(B),
'warn')
for i in range(len(raw_images)):
T_i = T[i] # L1 x h x w
T_i_idx = np.zeros_like(T_i, dtype=int)
hist_i = np.zeros([B * self._l1, (2**self._l2)],
dtype=int) # B * L1, 2^L2
final_hist_i = np.zeros(
[n_pooled_feature,
self._l1 * (2**self._l2)]) # n_pooled, (l1 * 2 ^ l2)
cnt = 0
# histogram phase
block_stage_for_spp(self._l1, self._block_size,
self._block_stride, self._l2, T_i, hist_i,
T_i_idx)
# print(view_as_blocks(T_i_idx[1], (4, 4)))
for level in self._levels: # pyramid levels
spatial_bin_size = (int(h // level), int(w // level))
for bin_idx1 in range(level):
for bin_idx2 in range(level):
# corresponding blocks idx of this bin
valid_T_i_idx = list(
set(T_i_idx[:, bin_idx1 *
spatial_bin_size[1]:(bin_idx1 +
1) *
spatial_bin_size[1], bin_idx2 *
spatial_bin_size[0]:(bin_idx2 +
1) *
spatial_bin_size[0]].reshape([-1
])))
# fetch all local histogram corresponding to this bin
valid_hist_i = hist_i[valid_T_i_idx, :].reshape(
[-1, self._l1 * 2**self._l2])
valid_hist_i = np.amax(valid_hist_i,
axis=0) # l1 * 2^l2
final_hist_i[cnt] = valid_hist_i
cnt += 1
final_hist_i = final_hist_i.reshape([-1
]) # n_pooled * l1 * 2^l2
# feats[i] = final_hist_i / np.linalg.norm(final_hist_i, ord=2)
feats[i] = final_hist_i
else:
n_img, _, h, w, *_ = T.shape
B = int((1 + (h - self._block_size[0]) / self._block_stride) *
(1 + (w - self._block_size[1]) / self._block_stride))
send_msg(self._log,
"\tnumber blocks in each feature map: {}".format(B),
'warn')
feats = np.empty([n_img, B * self._l1 * 2**self._l2
]) # n, (B * L1 * 2^L2)
for i in range(len(raw_images)):
T_i = T[i] # L1 x h x w
# histogram phase
hist_i = np.zeros([B * self._l1 * 2**self._l2],
dtype=int) # 2^L2 x L1 x B
# hist_i = np.zeros([2 ** self._l2 * self._l1 * B], dtype=int)
if self._feature_method == 'histogram':
block_stage(self._l1, self._block_size, self._block_stride,
self._l2, T_i, hist_i)
else: # pool_histogram (stage-2 pool)
pooled_T_i = np.zeros([self._l1, h // 2, w // 2],
dtype=int)
pool_stage(self._l1, T_i, pooled_T_i)
block_stage(self._l1, self._block_size, self._block_stride,
self._l2, pooled_T_i, hist_i)
feats[i] = hist_i
send_msg(self._log, "done")
return np.array(feats) # N x 2^L2 x L1 x B
def fit(self, imgs: (list, np.ndarray), labels: (list, np.ndarray)):
assert len(imgs) == len(
labels), "len(imgs) = {} while len(labels) = {}".format(
len(imgs), len(labels))
send_msg(self._log, "begin fitting")
'''
stage 1
'''
send_msg(self._log, '\tstage-1')
send_msg(self._log, '\tprocessing images...')
X = process_images(imgs, self._patch_size1, self._stride1) # 提取X
send_msg(self._log, "\tX.shape: {}".format(X.shape))
# construct stage 1 filters
send_msg(self._log, "\tconstructing filters 1")
w_l1 = get_w(X,
self._l1,
patch_size=self._patch_size1,
method=self._method1,
fourier_basis_sel=self._fourier_basis_sel1,
labels=labels,
rgb=self._rgb)
del X
self._filters1 = w_l1
send_msg(self._log,
'\tfilter1.shape: {}'.format(np.array(self._filters1).shape))
'''
stage 2
'''
send_msg(self._log, '\tstage-2')
send_msg(self._log, "\tconstruct conved images...")
conved_images = []
for img in imgs:
conved_img = conv_layer(
img, # no need to pad by hand, scipy.ndimage.convolve do pad for us
self._filters1,
complex=self._complex) # a list, len=len(filters)
conved_images.extend(conved_img)
send_msg(
self._log,
"\tlength of conved images: {}, each element's shape: {}".format(
len(conved_images), conved_images[0].shape))
# another pca stage
send_msg(self._log, '\tprocessing images...')
Y = process_images(conved_images,
patch_size=self._patch_size2,
stride=self._stride2)
send_msg(self._log, "\tY.shape: {}".format(Y.shape))
send_msg(self._log, "\tconstructing filters 2")
# construct stage 2 filters
w_l2 = get_w(Y,
self._l2,
patch_size=self._patch_size2,
method=self._method2,
fourier_basis_sel=self._fourier_basis_sel2,
labels=labels)
del imgs
del conved_images
del Y
self._filters2 = w_l2
send_msg(self._log,
'\tfilter2.shape: {}'.format(np.array(self._filters2).shape))
send_msg(self._log, "done")
gc.collect()