def featureSelector_binMethod(X,Y):
'''
Select 50% features that has the lowest value of cross entropy
input:
X: output from max-pooling. Typically it has (numOfImages,pooled dimension, pooled dimension, K) size. e.g., (50000,14,14,41) at unit 1
Y: labels corresponding to X
output:
selectedFeatures: selected 50% features
selectedFeaturesIndeces: indeces of the selected features
'''
reshapedPooledX = X.reshape((len(X),-1))
crossEntropyObj = Cross_Entropy(num_class = 10, num_bin = 10)
crossEntropyValStorage = np.zeros(reshapedPooledX.shape[-1])
for storageIndex in range(reshapedPooledX.shape[-1]): #12 minutes for 100 images and 8036(14*14*41) dimension
crossEntropyValStorage[storageIndex] = crossEntropyObj.compute(reshapedPooledX[:,storageIndex].reshape(-1,1), Y)
print("calculation is done \n")
sortedIndex = np.argsort(crossEntropyValStorage) #return indeces that would be the indeces when the array is sorted
numOfNs = int(sortedIndex.shape[0]/2) #select 50 persent of lowest values
#numOfNs = int(1000) #Select 1000 lowest features
selectedFeaturesIndex = sortedIndex[0:numOfNs]
numOfImgs = int(reshapedPooledX.shape[0]) #e.g.50000 images
reducedFeatureImgs = np.zeros((numOfImgs,numOfNs))
#Select features for each image
for imgIndex in range(numOfImgs):
for selectedFeatureCount in range(numOfNs):
curr_selectedFeatureIndex = selectedFeaturesIndex[selectedFeatureCount]
reducedFeatureImgs[imgIndex,selectedFeatureCount] = reshapedPooledX[imgIndex,curr_selectedFeatureIndex]
return selectedFeaturesIndex, reducedFeatureImgs, numOfNs
def train_single(self, input_vector, target_vector):
""""
Forward Propagation
input: input_vector [784], target_vector [10]
H1: output_vector [100], weights_in_hidden[100, 784], output_hidden[100]
output: output_vector2 [10], weight_hidden_output[10, 100], output_network[10]
loss scalar 2.3
Backward Propagation
gradient [10], derived [10], tmp2 [10], hidden_errors [100,10], tmp5 [100,10]
"""
input_vector = np.array(input_vector, ndmin=2).T
target_vector = np.array(target_vector, ndmin=2).T
output_vector1 = np.dot(self.weights_in_hidden, input_vector)
output_hidden = Activation.leakyReLU(output_vector1)
output_vector2 = np.dot(self.weights_hidden_output, output_hidden)
output_network = Activation.leakyReLU(output_vector2)
loss = Cross_Entropy.calc(output_network, target_vector)
gradient = Cross_Entropy.derived_calc(output_network, target_vector)
# update the weights:
derived1 = Derivative.leakyReLU(gradient)
tmp2 = derived1 * (loss * gradient)
# calculate hidden errors:
hidden_errors = np.dot(self.weights_hidden_output.T, loss * derived1)
# update the weights:
tmp5 = hidden_errors * Derivative.leakyReLU(output_hidden)
self.weights_hidden_output += self.learning_rate * np.dot(
tmp2, output_hidden.T)
self.weights_in_hidden += self.learning_rate * np.dot(
tmp5, input_vector.T)
def cal_CE_layer(features, trainLabel):
kernelNum = features.shape[1]
#Compute the Cross_Entropy of Layer 1
CE = []
for i in range(kernelNum):
tempData = features[:, i]
tempData = tempData.reshape((len(tempData), -1))
ce = Cross_Entropy(num_class=10, num_bin=10)
ce_value = ce.compute(tempData, trainLabel)
CE.append(ce_value)
return CE
class Cal_ce:
def __init__(self, output_train_sub):
self.ce = Cross_Entropy(num_class=10, num_bin=5)
def ce_cal(self, each_out, label):
each_out = each_out.reshape(-1, 1)
return self.ce.compute(each_out, label)
def extract_feature(re_output, re_output_test, N, train_label):
my_CrossEntropy = Cross_Entropy(num_class=10, num_bin=5)
feat_ce = np.zeros(re_output.shape[-1])
print("re_output.shape[-1]", re_output.shape[-1])
print("re_output_test.shape[-1]", re_output_test.shape[-1])
for k in range(re_output.shape[-1]):
feat_ce[k] = my_CrossEntropy.KMeans_Cross_Entropy(
re_output[:, k].reshape(-1, 1), train_label)
# print(" --> KMeans cross entropy: %s" % str(feat_ce[k]))
sorted_index = np.argsort(feat_ce) # increasing
final_output = np.zeros([re_output.shape[0], N], np.float32)
print("re_output.shape[0]", re_output.shape[0])
final_output_test = np.zeros([re_output_test.shape[0], N], np.float32)
print("re_output_test.shape[0]", re_output_test.shape[0])
for i in range(0, N):
final_output[:, i] = re_output[:, sorted_index[i]]
final_output_test[:, i] = re_output_test[:, sorted_index[i]]
return final_output, final_output_test
def feature_selection(layer1_o, layer2_o, layer3_o, Y):
x_train1 = layer1_o.reshape(len(layer1_o), -1)
x_train2 = layer2_o.reshape(len(layer2_o), -1)
x_train3 = layer3_o.reshape(len(layer3_o), -1)
input_list = [x_train1, x_train2, x_train3]
res_list = []
for i in range(3):
print("Currently doing feature selection for hop {}".format(i+1))
x_train = input_list[i]
# Calculate loss for each features
ce = Cross_Entropy(num_class=10, num_bin=5)
feat_ce = np.zeros(x_train.shape[-1])
start = time.time()
for i in range(x_train.shape[-1]): #35280 features:
if i % 1000 == 999:
print("{}/{}".format(i+1, x_train.shape[1]))
temp_vec = x_train1[:,i]
feat_ce[i] = ce.KMeans_Cross_Entropy(x_train[:,i].reshape(-1,1), Y)
end = time.time()
print("time elapse: {}s".format(end-start))
# Order the feature by its cross entropy loss
feat_ce_tuplelist = []
[feat_ce_tuplelist.append((feat_ce[i],i)) for i in range(x_train.shape[-1])]
feat_ce_tuplelist.sort()
# Select top 1000 features
if len(feat_ce_tuplelist) > 1000:
feat_ce_tuplelist_sel = feat_ce_tuplelist[0:1000]
feat_num_sel = []
[feat_num_sel.append(feat_ce_tuplelist_sel[i][1]) for i in range(1000)]
x_train_sel = x_train[:,feat_num_sel]
res_list.append(x_train_sel)
else:
res_list.append(x_train)
return res_list[0], res_list[1], res_list[2]
def __init__(self, output_train_sub):
self.ce = Cross_Entropy(num_class=10, num_bin=5)
each_out = each_out.reshape(-1, 1)
return self.ce.compute(each_out, label)
# parallel
# for each in output_train_avg:
# each = each.reshape(each.shape[0], -1)
# ce = Cross_Entropy(num_class=10, num_bin=5)
# cal_ce = Cal_ce(ce)
# feat_ce = Parallel(n_jobs=mp.cpu_count(), backend='multiprocessing')(delayed(cal_ce.ce_cal)(each[:,k], y_train_later) for k in range(each.shape[-1]))
# feat_ce = np.array(feat_ce)
# feature_set.append(np.argpartition(feat_ce, np.min( (Ns, each.shape[-1] - 1) ))[:np.min( (Ns, each.shape[-1]) )])
for each in output_train_avg:
each = each.reshape(each.shape[0], -1)
ce = Cross_Entropy(num_class=10, num_bin=10)
feat_ce = np.zeros(each.shape[-1])
for k in range(each.shape[-1]):
feat_ce[k] = ce.compute(each[:, k].reshape(-1, 1), y_train_later)
# print(" --> KMeans ce: %s"%str(feat_ce[k]))
feature_set.append(
np.argpartition(feat_ce, np.min(
(Ns, each.shape[-1] - 1)))[:np.min((Ns, each.shape[-1]))])
print("------- DONE -------\n")
# for each in output_train_max:
# each = each.reshape(each.shape[0], -1)
# ce = Cross_Entropy(num_class=10, num_bin=5)
# cal_ce = Cal_ce(ce)
# feat_ce = Parallel(n_jobs=mp.cpu_count(), backend='multiprocessing')(delayed(cal_ce.ce_cal)(each[:,k], y_train_later) for k in range(each.shape[-1]))
# feat_ce = np.array(feat_ce)
output = new_p2.transform(X)
print(output[0].shape, output[1].shape, output[2].shape)
# max pooling
for i in range(0, 3):
output[i] = skimage.measure.block_reduce(output[i], (1, 2, 2, 1),
np.max)
# reshaping
for i in range(0, 3):
output[i] = output[i].reshape(
10, output[i].shape[1] * output[i].shape[2] * output[i].shape[3])
# ------------------ cross entropy ------------------ #
ce = Cross_Entropy(num_class=10, num_bin=5)
entropy = np.zeros(output[2].shape[1])
rank = []
for j in range(0, output[2].shape[1]):
entropy[j] = ce.KMeans_Cross_Entropy(output[2][:, j].reshape(-1, 1),
y_train[0:10])
rank = np.argsort(-entropy)
output[2] = output[2][:, rank[0:15]]
# ------------------ LAG ------------------ #
lag = LAG(encode='distance',
num_clusters=[5, 5, 5, 5, 5, 5, 5, 5, 5, 5],
alpha=10,
learner=myLLSR(onehot=False))
X_train[0:nti].shape
"""# Module 2"""
start = time.time()
output = phop.transform(X_train[0:nti])
print(output[0].shape,output[1].shape,output[2].shape)
from cross_entropy import Cross_Entropy
X_reshape0 = output[0].reshape(len(output[0]), -1,output[0].shape[-1])
X_reshape1 = output[1].reshape(len(output[1]), -1,output[1].shape[-1])
X_reshape2 = output[2].reshape(len(output[2]), -1,output[2].shape[-1])
kce = Cross_Entropy(num_class=10, num_bin=5)
kce_fd0 = np.zeros(X_reshape0.shape[-1])
kce_fd1 = np.zeros(X_reshape1.shape[-1])
kce_fd2 = np.zeros(X_reshape2.shape[-1])
for k in range(X_reshape0.shape[-1]):
kce_fd0[k] = kce.KMeans_Cross_Entropy(X_reshape0[:,:,k].reshape(X_reshape0.shape[0],-1), y_train[0:nti])
print('finished 0')
for k in range(X_reshape1.shape[-1]):
kce_fd1[k] = kce.KMeans_Cross_Entropy(X_reshape1[:,:,k].reshape(X_reshape1.shape[0],-1), y_train[0:nti])
print('finished 1')
for k in range(X_reshape2.shape[-1]):
kce_fd2[k] = kce.KMeans_Cross_Entropy(X_reshape2[:,:,k].reshape(X_reshape2.shape[0],-1), y_train[0:nti])
print('finished 2')
Ns = 1500
feature_set = []
class Cal_ce:
def __init__(self, output_train_sub):
self.ce = Cross_Entropy(num_class=10, num_bin=5)
def ce_cal(self, each_out, label):
each_out = each_out.reshape(-1, 1)
return self.ce.compute(each_out, label)
for each in output_train_avg:
each = each.reshape(each.shape[0], -1)
ce = Cross_Entropy(num_class=10, num_bin=5)
cal_ce = Cal_ce(ce)
feat_ce = Parallel(n_jobs=mp.cpu_count(), backend='multiprocessing')(
delayed(cal_ce.ce_cal)(each[:, k], y_train_later)
for k in range(each.shape[-1]))
feat_ce = np.array(feat_ce)
feature_set.append(
np.argpartition(feat_ce, np.min(
(Ns, each.shape[-1] - 1)))[:np.min((Ns, each.shape[-1]))])
for each in output_train_max:
each = each.reshape(each.shape[0], -1)
ce = Cross_Entropy(num_class=10, num_bin=5)
cal_ce = Cal_ce(ce)
feat_ce = Parallel(n_jobs=mp.cpu_count(), backend='multiprocessing')(
delayed(cal_ce.ce_cal)(each[:, k], y_train_later)
(output1[2], output2[2], output3[2], output4[2], output5[2]))
print(output_1.shape, output_2.shape, output_3.shape)
# print(output[0].shape, output[1].shape, output[2].shape)
print("------- DONE -------\n")
#Cross Entropy1
from cross_entropy import Cross_Entropy
temp1 = output_1.reshape((len(output_1), -1))
print("input feature shape is: %s" % str(temp1.shape))
fit_data_train, data_val, fit_label_train, label_val = train_test_split(
data_train,
label_train,
test_size=0.8,
random_state=0,
stratify=label_train)
CE = Cross_Entropy(num_class=10, num_bin=5)
output1Feature = np.zeros((temp1.shape[-1]))
for i in range((temp1.shape[-1])):
output1Feature[i] = CE.KMeans_Cross_Entropy(temp1[:, i].reshape(-1, 1),
label_train)
print("--> KMeans CE: %s" % str(output1Feature[i]))
print("------DONE-----\n")
#Cross Entropy2
from cross_entropy import Cross_Entropy
temp2 = output_2.reshape((len(output_2), -1))
print("input feature shape is: %s" % str(temp2.shape))
fit_data_train, data_val, fit_label_train, label_val = train_test_split(
data_train,
label_train,
output = new_p2.transform(X)
print(output[0].shape, output[1].shape, output[2].shape)
# max pooling
for i in range(0, 3):
output[i] = skimage.measure.block_reduce(output[i], (1, 2, 2, 1),
np.max)
# reshaping
for i in range(0, 3):
output[i] = output[i].reshape(
train_num,
output[i].shape[1] * output[i].shape[2] * output[i].shape[3])
# ------------------ cross entropy ------------------ #
ce = Cross_Entropy(num_class=10, num_bin=5)
print('Ns_1 - %d Ns_2 - %d Ns_3 - %d' % (Ns_1, Ns_2, Ns_3))
entropy = np.zeros(output[0].shape[1])
rank_1 = []
for j in range(0, output[0].shape[1]):
entropy[j] = ce.compute(output[0][:, j].reshape(-1, 1),
y_train[0:train_num])
rank_1 = np.argsort(-entropy)
output[0] = output[0][:, rank_1[0:Ns_1]]
# ------------------------------------ #
entropy = np.zeros(output[1].shape[1])
rank_2 = []
y_train_later = data_set[:, -1][:, None]
else:
x_train_later = np.reshape(x_train_later,
(num_train_later * 10, 32, 32, 3))
x_train_later = x_train_later.astype('float32')
x_train_later /= 255
output_train = p2.transform(x_train_later)
Ns = 1500
feature_set = []
for each in output_train:
each = each.reshape(each.shape[0], -1)
ce = Cross_Entropy(num_class=10, num_bin=5)
feat_ce = np.zeros(each.shape[-1])
for k in range(each.shape[-1]):
feat_ce[k] = ce.KMeans_Cross_Entropy(each[:, k].reshape(-1, 1),
y_train_later)
# print(" --> KMeans ce: %s"%str(feat_ce[k]))
feature_set.append(
np.argpartition(feat_ce, np.min(
(Ns, each.shape[-1] - 1)))[:np.min((Ns, each.shape[-1]))])
print("------- DONE -------\n")
f = open('feature_set.pkl', 'wb')
pickle.dump(feature_set, f)
f.close()
print()
