forked from endgameinc/homoglyph
/
run_siamese.py
241 lines (185 loc) · 8.89 KB
/
run_siamese.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
"""Sample code for siamese neural net for detecting spoofing attacks"""
import matplotlib
matplotlib.use('Agg')
import cPickle as pickle
import editdistance
import matplotlib.pyplot as plt
import numpy as np
import os
import random
import strSimilarity
from keras.layers import Dense, Input, Lambda, Flatten, Convolution2D, MaxPooling2D
from keras.layers.advanced_activations import LeakyReLU
from keras.models import Sequential, Model
from keras import backend as K
from keras.optimizers import RMSprop
from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont
from sklearn.metrics import roc_curve, auc, roc_auc_score
isFast = True # If True, then it runs on a very small dataset (and results won't be that great)
dataset_type = 'process'
#dataset_type = 'domain'
OUTPUT_DIR = 'output'
if not os.path.isdir(OUTPUT_DIR):
os.mkdir(OUTPUT_DIR)
if dataset_type == 'domain':
OUTPUT_FILE = os.path.join(OUTPUT_DIR, 'domain_results.pkl')
INPUT_FILE = os.path.join('data', 'domains_spoof.pkl')
IMAGE_FILE = os.path.join(OUTPUT_DIR, 'domains_roc_curve.png')
OUTPUT_NAME = 'Domain Spoofing'
elif dataset_type == 'process':
OUTPUT_FILE = os.path.join(OUTPUT_DIR, 'process_results.pkl')
INPUT_FILE = os.path.join('data', 'process_spoof.pkl')
IMAGE_FILE = os.path.join(OUTPUT_DIR, 'process_roc_curve.png')
OUTPUT_NAME = 'Process Spoofing'
else:
raise Exception('Unknown dataset type: %s' % (dataset_type,))
def generate_imgs(strings, font_location, font_size, image_size, text_location):
font = ImageFont.truetype(font_location, font_size)
str_imgs = []
for st in strings:
# Create a single channel image of floats
img1 = Image.new('F', image_size)
dimg = ImageDraw.Draw(img1)
dimg.text(text_location, st.lower(), font=font)
img1 = np.expand_dims(img1, axis=0)
str_imgs.append(img1)
return np.array(str_imgs, dtype=np.float32)
def euclidean_distance(vects):
x, y = vects
return K.sqrt(K.sum(K.square(x - y)+np.random.rand()*.0001, axis=1, keepdims=True))
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return (shape1[0], 1)
def contrastive_loss(y_true, y_pred):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
'''
margin = 1
return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)), axis=-1, keepdims=False)
def build_model(data_shape):
model = Sequential()
model.add(Convolution2D(128, 5, 5, input_shape=data_shape))
model.add(LeakyReLU(alpha=.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3))
model.add(LeakyReLU(alpha=.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(32))
input_a = Input(shape=data_shape)
input_b = Input(shape=data_shape)
processed_a = model(input_a)
processed_b = model(input_b)
distance = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([processed_a, processed_b])
model = Model(input=[input_a, input_b], output=distance)
# # train
rms = RMSprop()
model.compile(loss=contrastive_loss, optimizer=rms)
return model
def initialize_encoder(self):
"""Initialize encoder for translating images to features."""
# Set locations of models, weights, and feature parameters
model_file = os.path.join(OUTPUT_DIR, dataset_type + '_cnn.json')
weight_file = os.path.join(OUTPUT_DIR, dataset_type + '_cnn.h5')
# Load model
model = model_from_json(open(model_file).read())
model.load_weights(weight_file)
# Set up encoder to convert images to features
encoder = self._tm.layers[2]
input_shape = tuple(model.get_layer(model.layers[0].name).input_shape[1:])
input_a = Input(shape=input_shape)
encoder(input_a)
return encoder
if not os.path.isfile(OUTPUT_FILE):
font_location = "Arial.ttf"
font_size = 10
image_size = (150, 12)
text_location = (0, 0)
max_epochs = 25
data = pickle.load(open(INPUT_FILE))
if isFast:
data['train'] = random.sample(data['train'], 20000)
data['validate'] = random.sample(data['validate'], 100)
data['test'] = random.sample(data['test'], 1000)
max_epochs = 10
# organize data and translate from th to tf image ordering via .transpose( (0,2,3,1) )
X1_train = generate_imgs([x[0] for x in data['train']], font_location, font_size, image_size, text_location).transpose( (0,2,3,1) )
X2_train = generate_imgs([x[1] for x in data['train']], font_location, font_size, image_size, text_location).transpose( (0,2,3,1) )
y_train = [x[2] for x in data['train']]
X1_valid = generate_imgs([x[0] for x in data['validate']], font_location, font_size, image_size, text_location).transpose( (0,2,3,1) )
X2_valid = generate_imgs([x[1] for x in data['validate']], font_location, font_size, image_size, text_location).transpose( (0,2,3,1) )
y_valid = [x[2] for x in data['validate']]
X1_test = generate_imgs([x[0] for x in data['test']], font_location, font_size, image_size, text_location).transpose( (0,2,3,1) )
X2_test = generate_imgs([x[1] for x in data['test']], font_location, font_size, image_size, text_location).transpose( (0,2,3,1) )
y_test = [x[2] for x in data['test']]
model = build_model((12, 150, 1))
# First figure out how many epochs we need
max_auc = 0
max_idx = 0
for i in range(max_epochs):
model.fit([X1_train, X2_train], y_train, batch_size=8, nb_epoch=1)
scores = [-x[0] for x in model.predict([X1_valid, X2_valid])]
t_auc = roc_auc_score(y_valid, scores)
if t_auc > max_auc:
print('Updated best AUC from %f to %f' % (max_auc, t_auc))
max_auc = t_auc
max_idx = i+1
# Train on the correct number of epochs
model = build_model((12, 150, 1))
model.fit([X1_train, X2_train], y_train, batch_size=8, nb_epoch=max_idx)
# Save the NN
json_string = model.to_json()
model.save_weights(os.path.join(OUTPUT_DIR, dataset_type + '_cnn.h5'), overwrite=True)
open(os.path.join(OUTPUT_DIR, dataset_type + '_cnn.json'), 'wb').write(json_string)
scores = [-x[0] for x in model.predict([X1_test, X2_test])]
fpr_siamese, tpr_siamese, _ = roc_curve(y_test, scores)
roc_auc_siamese = auc(fpr_siamese, tpr_siamese)
#
# Run Edit distance
#
scores = [(editdistance.eval(x[0].lower(), x[1].lower()), len(x[0]), 1.0-x[2]) for x in data['test']]
y_percent_score = [float(x[0])/x[1] for x in scores]
y_score, _, y_test = zip(*scores)
fpr_ed, tpr_ed, _ = roc_curve(y_test, y_score)
roc_auc_ed = auc(fpr_ed, tpr_ed)
fpr_ps, tpr_ps, _ = roc_curve(y_test, y_percent_score)
roc_auc_ps = auc(fpr_ps, tpr_ps)
#
# Run editdistance visual similarity
#
scores = [(strSimilarity.howConfusableAre(x[0].lower(), x[1].lower()), 1.0-x[2]) for x in data['test']]
y_score, y_test = zip(*scores)
fpr_edvs, tpr_edvs, _ = roc_curve(y_test, [-x for x in y_score])
roc_auc_edvs = auc(fpr_edvs, tpr_edvs)
#
# Store results
#
results = {}
results['editdistance_vs'] = {'fpr': fpr_edvs, 'tpr': tpr_edvs, 'auc':roc_auc_edvs}
results['editdistance'] = {'fpr': fpr_ed, 'tpr': tpr_ed, 'auc':roc_auc_ed}
results['editdistance_percent'] = {'fpr': fpr_ps, 'tpr': tpr_ps, 'auc':roc_auc_ps}
results['siamese'] = {'fpr': fpr_siamese, 'tpr': tpr_siamese, 'auc':roc_auc_siamese}
pickle.dump(results, open(OUTPUT_FILE, 'w'))
results = pickle.load(open(OUTPUT_FILE))
#
# Make Figures
#
fig = plt.figure()
plt.plot(results['siamese']['fpr'], results['siamese']['tpr'], 'b',
label='Siamese CNN (AUC=%0.2f)' % results['siamese']['auc'])
plt.plot(results['editdistance_vs']['fpr'], results['editdistance_vs']['tpr'], 'g',
label='Visual edit distance (AUC=%0.2f)' % results['editdistance_vs']['auc'])
plt.plot(results['editdistance']['fpr'], results['editdistance']['tpr'], 'r',
label='Edit distance (AUC=%0.2f)' % results['editdistance']['auc'])
plt.plot(results['editdistance_percent']['fpr'], results['editdistance_percent']['tpr'],
label='Percent edit distance (AUC=%0.2f)' % results['editdistance_percent']['auc'])
plt.plot([0, 1], [0, 1], 'k', lw=3, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('{} - Receiver Operating Characteristic'.format(OUTPUT_NAME))
plt.legend(loc="lower right")
fig.savefig(IMAGE_FILE)