def create_label(keywords, image):
for i, key in enumerate(keywords):
label = image.split('_')[0]
if key.lower() == label.lower():
ohl = np.array([i])
return (ohl)
def getComment(user, image):
imagePath = './data/' + user + '/' + image
assert os.path.exists(imagePath)
if '.jpg' in image:
shortCode = image.split('.jpg')[0]
elif '.jpeg' in image:
shortCode = image.split('.jpeg')[0]
else:
print("Error: not jpg or jpeg")
exit()
#while os.path.exists('./data/' + user + '/' + shortCode + '.comment'):
commentPath = './data/' + user + '/' + shortCode + '.0'
with open(commentPath, 'r') as f:
line = eval(f.readline())
comment = unicode2str(line[0])
#comment = line[0]
return comment
def populate_mapping(self):
f = open(self.map_file, 'r')
for line in f:
line = line.rstrip()
image, year = line.split("\t")
gender, imfilename = image.split("/")
if gender is 'M':
encodeGender = 1
elif gender is 'F':
encodeGender = 0
self.fnameToGender[image] = encodeGender
self.map[image] = year
def load_single_image(source_image, img_size=(224, 224)):
assert os.path.exists(source_image)
single_image = np.zeros((1, img_size[0], img_size[1], 3))
image = Image.open(source_image)
image = image.resize(img_size)
image.load()
image_background = Image.new("RGB", image.size, (255, 255, 255))
image_background.paste(image, mask=image.split()[3]) # Remove any alpha channel
image = np.array(image_background) / 255.
single_image[0] = image
return single_image
def blue_score_evaluation_with_sentence_bleu_greedy(imageid_nd_caption_cleaned,test_encodedfeatures):
predictedscore=0
len=0
for image in test_encodedfeatures.keys():
machinecaption=[]
allcaptionforblue=[]
input_image=test_encodedfeatures[image].reshape((1,2048))
machinecaption=greedy_search(input_image)
machinecaption=machinecaption.split(' ')
# machinepredictions.append(machinecaption)
imagenwoutextension=image.split('.')[0]
# print(imagenwoutextension)
# print(image)
# referencetoeach_caption=[]
for each_caption in imageid_nd_caption_cleaned[imagenwoutextension]:
tempList = each_caption.split(' ')
allcaptionforblue.append(tempList)
# print(allcaptionforblue)
# print(machinecaption)
predictedscore = predictedscore+sentence_bleu(allcaptionforblue,machinecaption)
len=len+1
return predictedscore/len
def strip_alpha_channel(self, image, fill_color='#ffffff'):
''' Strip the alpha channel of an image and fill with fill color
'''
background = Image.new(image.mode[:-1], image.size, fill_color)
background.paste(image, image.split()[-1])
return background
dest='val/Whole/'+classes[str(val_label.iloc[i].label)]
if(os.path.exists(dest+'/'+source.split('/')[-1])==False):
shutil.move(source,dest)
limit=num_val_samples
for folder in tqdm(os.listdir(val_dir)):
#os.shuffle(val_dir+'/'+folder)
count=0
for image in os.listdir(val_dir+'/'+folder):
img = cv2.imread(val_dir+'/'+folder+'/'+image,0)
height = img.shape[0]
width = img.shape[1]
header = img[0:(int(height*0.33)), 0:width]
footer = img[int(height*0.67):height, 0:width]
left_body = img[int(height*0.33):int(height*0.67), 0:int(width*0.5)]
right_body = img[int(height*0.33):int(height*0.67), int(width*0.5):width]
header_path='val/header/'+folder+'/'+image.split('/')[-1]
footer_path='val/footer/'+folder+'/'+image.split('/')[-1]
left_path='val/left_body/'+folder+'/'+image.split('/')[-1]
right_path='val/right_body/'+folder+'/'+image.split('/')[-1]
if(os.path.exists(header_path)==False):
cv2.imwrite(header_path, header)
if(os.path.exists(footer_path)==False):
cv2.imwrite(footer_path, footer)
if(os.path.exists(left_path)==False):
cv2.imwrite(left_path, left_body)
if(os.path.exists(right_path)==False):
cv2.imwrite(right_path, right_body)
count+=1
if(count==limit):
break
limit,alpha=num_train_samples,0.037
print("BLEU score of the predicted caption_for_beamcount_3: "+str(blue_score_evaluation_with_sentence_bleu(imageid_nd_caption_cleaned, test_encodedfeatures,3)))
training_m.load_weights('/content/gdrive/My Drive/Machine Learning/Flickr_Data/Flickr_Data/weights_512/weights0.h5')
print("BLEU score of the predicted caption_greedy: "+str(blue_score_evaluation_with_sentence_bleu_greedy(imageid_nd_caption_cleaned, test_encodedfeatures)))
print("BLEU score of the predicted caption_for_beamcount_3: "+str(blue_score_evaluation_with_sentence_bleu(imageid_nd_caption_cleaned, test_encodedfeatures,3)))
training_m.load_weights('/content/gdrive/My Drive/Machine Learning/Flickr_Data/Flickr_Data/weights_512/weights25.h5')
for image in test_encodedfeatures.keys():
input_image=test_encodedfeatures[image].reshape((1,2048))
img=mpimg.imread('/content/gdrive/My Drive/Machine Learning/Flickr_Data/Flickr_Data/Images/'+image)
imgplot = plt.imshow(img)
plt.show()
inputimage=image.split('.')[0]
actualCaptions = list()
for caption in imageid_nd_caption_cleaned[inputimage]:
tempList = caption.split(' ')
actualCaptions.append(tempList)
# print(actualCaptions)
pred=prediction_using_beamsearch(input_image,5)
predictionscore = sentence_bleu(actualCaptions,pred.split(' '))
print("Image_Name: "+image)
print("Predicted caption:"+pred)
print("blue score when used batch size 512 and with >200 epochs: "+str(predictionscore))
print("--------------------------------------")
print("BLEU score of the predicted caption_greedy_for_512_49: "+str(blue_score_evaluation_with_sentence_bleu_greedy(imageid_nd_caption_cleaned, test_encodedfeatures)))
training_m.load_weights('/content/gdrive/My Drive/Machine Learning/Flickr_Data/Flickr_Data/weights_600_after_512/weights_900_2_49.h5')
dataset.head()
# In[9]:
x_train, y_train, x_test, y_test = [], [], [], []
# In[10]:
for i in range(1, numInstances): # loop from 1 to number of instances in dataset
try: #try catch
emotion, image, usage = lines[i].split(",") #split dataset by comma (csv file)
imgval = image.split(" ") #remove white space
pixels = np.array(imgval, 'float32') #convert image pixels to float32 as expected by CNN
emotion = keras.utils.to_categorical(emotion, num_classes) #stores the emotion value as a binary matrix
if 'Training' in usage: #filter values marked as training
y_train.append(emotion) #add emotion value as y train
x_train.append(pixels) #add image pixel value as x train
elif 'PublicTest' in usage: #filter values marked as testing
y_test.append(emotion) #add emotion value as y test
x_test.append(pixels) #add image pixel value as x test
except:
print("", end="")
train_id = load_set(path_to_train_id_file)
# 1. -------- Xử lý ảnh dùng để train -----------------------------------------------
# Lấy lấy các ảnh jpg trong thư mục chứa toàn bộ ảnh Flickr
all_train_images_in_folder = glob.glob(path_to_image_folder + '*.jpg')
# Đọc toàn bộ nội dung file danh sách các file ảnh dùng để train
train_images = set(open(path_to_train_id_file, 'r').read().strip().split('\n'))
# Danh sách ấy sẽ lưu full path vào biến train_img
train_img = []
for image in all_train_images_in_folder: # Duyệt qua tất cả các file trong folder
# Nếu tên file của nó thuộc training set
if image.split('/')[-1] in train_images:
train_img.append(image) # Thì thêm vào danh sách ảnh sẽ dùng để train
# 2. -------- Xử lý ảnh dùng để test (xử lý tương tự) -------------------------------------
test_images = set(open(path_to_train_id_file, 'r').read().strip().split('\n'))
test_img = []
for image in all_train_images_in_folder:
if image.split('/')[-1] in test_images:
test_img.append(image)
############################ III. PHẦN XỬ LÝ DỮ LIỆU MÔ TẢ ############################
# Hàm đọc mô tả từ file và Thêm 'startseq', 'endseq' cho chuỗi
# Get each image and preprocess the image
print("Predicting " + image)
x_batch = np.array(
Image.open(datapath + directory + "/" + image))
x_batch = resize(x_batch, output_shape=(64, 64))
x_batch = np.rint(x_batch * 255)
x_batch = np.expand_dims(x_batch, -1)
x_batch = np.tile(x_batch, 3)
x_batch = np.expand_dims(x_batch, 0)
x_batch = preprocess_input(x_batch,
data_format='channels_last')
# Extract feature map and write features to file
pred = model.predict(x_batch)
pred = np.squeeze(pred)
pred = np.max(pred, axis=(0, 1))
id = image.split(".tif")[0]
outputfile = output_fold + layer + "_green.txt"
output = open(outputfile, "a")
output.write(directory)
output.write("\t")
for feat in pred:
output.write(str(feat))
output.write("\t")
output.write("\n")
output.close()
def train():
x_train, y_train, x_test, y_test = [], [], [], []
df = pd.read_csv('dataset/fer2013.csv')
for i, row in df.iterrows():
emotion, image, usage = row['emotion'], row['pixels'], row['usage']
pixels = np.array(image.split(' '), 'float32')
emotion = keras.utils.to_categorical(emotion, num_expressions)
if 'Training' in usage:
y_train.append(emotion)
x_train.append(pixels)
if 'PublicTest' in usage:
y_test.append(emotion)
x_test.append(pixels)
print('Length: ', len(x_train))
# data transformation to train and test sets
x_train = np.array(x_train, 'float32')
y_train = np.array(y_train, 'float32')
x_test = np.array(x_test, 'float32')
y_test = np.array(y_test, 'float32')
x_train /= 255 #normalize inputs between [0, 1]
x_test /= 255
x_train = x_train.reshape(x_train.shape[0], 48, 48, 1)
x_train = x_train.astype('float32')
x_test = x_test.reshape(x_test.shape[0], 48, 48, 1)
x_test = x_test.astype('float32')
print('Train Sample: {}'.format(x_train.shape[0]))
print('Test Sample: {}'.format(x_test.shape[0]))
# constructing the cnn structure
model = Sequential()
# 1st convolution layer
'''
relu : (Rectified Linear Unit) Activation Function
https://towardsdatascience.com/activation-functions-neural-networks-1cbd9f8d91d6
'''
model.add(Conv2D(64, (5, 5), activation='relu', input_shape=(48, 48, 1)))
model.add(MaxPooling2D(pool_size=(5, 5), strides=(2, 2)))
# 2nd Convolution Layer
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2)))
#3rd convolution layer
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2)))
model.add(Flatten())
# fully connected neural network
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(num_expressions, activation='softmax'))
# batch process
gen = ImageDataGenerator()
train_generator = gen.flow(x_train, y_train, batch_size=batch_size)
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
model.fit_generator(train_generator,
steps_per_epoch=batch_size,
epochs=epochs)
train_score = model.evaluate(x_train, y_train, verbose=0)
print('Train loss:', train_score[0])
print('Train accuracy:', 100 * train_score[1])
test_score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', test_score[0])
print('Test accuracy:', 100 * test_score[1])
model.save('my_model.hdf5')
