def train_and_score(network):
"""Train the model, return test loss.
Args:
network (dict): the parameters of the network
"""
# create the model
model = create_model(input_size=input_size,
output_size=output_size,
n_layers=network['n_layers'],
n_neurons=network['n_neurons'],
activation_function=network['activation'],
learning_rate=network['learning_rate'],
dropout_rate=network['dropout_rate'],
optimizer=network['optimizer'])
# train the model
results = model.fit(x=train_X, y=train_Y, epochs=network['epochs'])
preds = model.predict(test_X)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
if len(preds) == len(preds[preds == 0]):
f1_score = 0.0
else:
_, _, f1_score = scores(preds, test_Y)
f1_score = f1_score[0]
return f1_score
def score_one_at_time(model, files, test_Y, nlp_x):
preds = np.empty((0,test_Y.shape[1]))
right_test_Y = np.empty((0,test_Y.shape[1]))
tot = 0
with tqdm(total=len(files)) as pbar:
for i, img_file in enumerate(files):
try:
image = Image.open(img_file, 'r')
image_X = np.asarray(image).reshape(1, IMG_SIZE, IMG_SIZE, levels)
nlp_X = nlp_x[get_txt_file(img_file)].reshape(1, NLP_SIZE)
Y = model.predict([image_X, nlp_X], verbose=0)
preds = np.append(preds, Y, axis=0)
right_test_Y = np.append(right_test_Y, test_Y[i:i+1], axis=0)
except:
logging.debug("error: " + img_file)
pbar.update(1)
test_Y = right_test_Y
#std_dev = np.std(preds)
preds[preds>=0.5] = 1
preds[preds<0.5] = 0
_, _, f1 = scores(preds, test_Y)
print("F-scores: " + str(f1) + " -----------------------")
return np.mean(f1)
def test(model, test_X, test_Y):
preds = model.predict(test_X)
#print(preds)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
print("Other F1 score: ", f1_score(test_Y, preds, average='micro'))
precision, recall, f1 = scores(preds, test_Y)
print("Precision: ", precision)
print("Recall: ", recall)
print("f1score: ", f1)
def train_and_score_in_memory(network):
model = create_model(network['layers_and_filters'],
network['kernel_size'],
network['activation'], (IMG_SIZE, IMG_SIZE, levels),
network['dropout_rate'],
network['optimizer'],
network['learning_rate'],
output_size=train_Y.shape[1])
result = model.fit(x=train_X, y=train_Y, epochs=network['epochs'])
preds = model.predict(test_X)
std_dev = np.std(preds)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
_, _, f1 = scores(preds, test_Y)
return np.mean(f1) + 2 * std_dev
def predict_and_score(model, test_X, test_Y):
preds = model.predict(test_X)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
print(len(preds), len(preds[preds == 0]), len(test_Y),
len(test_Y[test_Y == 0]))
precision, recall, f1 = scores(preds, test_Y)
print("Test-precision: ", precision)
print("Test-recall: ", recall)
print("Test-f1score: ", f1)
test_Y = test_Y.reshape(-1)
preds = preds.reshape(-1)
def score_one_at_time(model, files, test_Y):
preds = np.empty((0, test_Y.shape[1]))
with tqdm(total=len(files)) as pbar:
for i, img_file in enumerate(files):
image = Image.open(img_file, 'r')
X = np.asarray(image).reshape((-1, IMG_SIZE, IMG_SIZE, levels))
Y = model.predict(X, verbose=0)
preds = np.append(preds, Y, axis=0)
pbar.update(1)
std_dev = np.std(preds)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
_, _, f1 = scores(preds, test_Y)
return np.mean(
f1
) + 2 * std_dev # std deviation to avoid results near 0.5 or all equals
def score_one_at_time(model, files, test_Y):
preds = np.empty((0, test_Y.shape[1]))
with tqdm(total=len(files)) as pbar:
for i, img_file in enumerate(files):
try:
image = Image.open(img_file, 'r')
X = np.asarray(image).reshape((-1, IMG_SIZE, IMG_SIZE, levels))
Y = model.predict(X, verbose=0)
preds = np.append(preds, Y, axis=0)
except:
print('Error scoring: ' + img_file)
pbar.update(1)
print(preds, test_Y)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
print("Other F1 score: ", f1_score(test_Y, preds, average='micro'))
precision, recall, f1 = scores(preds, test_Y)
print("Precision: ", precision)
print("Recall: ", recall)
print("f1score: ", f1)
def main_in_memory():
dataset = load_dataset(images_root_dir)
train_X, train_Y, test_X, test_Y = prepare_dataset(dataset, target=target)
model = create_model(network['layers_and_filters'],
network['kernel_size'],
network['activation'], (IMG_SIZE, IMG_SIZE, levels),
network['dropout_rate'],
network['optimizer'],
network['learning_rate'],
output_size=train_Y.shape[1])
result = model.fit(x=train_X, y=train_Y, epochs=network['epochs'])
preds = model.predict(test_X)
print(preds, test_Y)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
print("Other F1 score: ", f1_score(test_Y, preds, average='micro'))
precision, recall, f1 = scores(preds, test_Y)
print("precision: ", precision)
print("recall: ", recall)
print("f1score: ", f1)
def score_one_at_time(model, files, test_Y, nlp_x, entropy_x, androdet_x):
preds = np.empty((0, test_Y.shape[1]))
right_test_Y = np.empty((0, test_Y.shape[1]))
tot = 0
with tqdm(total=len(files)) as pbar:
for i, img_file in enumerate(files):
try:
image = Image.open(img_file, 'r')
image_X = np.asarray(image).reshape(1, IMG_SIZE, IMG_SIZE,
levels)
androdet_X = androdet_x[get_apk_file(img_file)].reshape(
1, ANDRODET_SIZE)
nlp_X = nlp_x[get_original_old_file(img_file) +
'.txt'].reshape(1, NLP_SIZE)
entropy_X = entropy_x[get_original_file(img_file)].reshape(
1, 1)
Y = model.predict([androdet_X, image_X, nlp_X, entropy_X],
verbose=0)
preds = np.append(preds, Y, axis=0)
right_test_Y = np.append(right_test_Y, test_Y[i:i + 1], axis=0)
except:
logging.debug("error: " + img_file)
pbar.update(1)
test_Y = right_test_Y
print(preds, test_Y)
preds[preds >= 0.5] = 1
preds[preds < 0.5] = 0
print("Other F1 score: ", f1_score(test_Y, preds, average='micro'))
precision, recall, f1 = scores(preds, test_Y)
print("Precision: ", precision)
print("Recall: ", recall)
print("f1score: ", f1)