def lipo_lstm_test(r,threshold_SC,threshold_BC,symbols_TC,seq,TestCaseNum,Mutation,CoverageStop):
r.resetTime()
seeds = 3
random.seed(seeds)
# set oracle radius
oracleRadius = 0.2
# load model
lipo = lipoClass()
lipo.load_data()
lipo.load_model()
sme = SmilesEnumerator()
# test layer
layer = 1
# choose time steps to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
# calculate mean and std for z-norm
h_train = lipo.cal_hidden_keras(lipo.X_train, layer)
mean_TC, std_TC, max_SC, min_SC, max_BC, min_BC = aggregate_inf(h_train, indices)
n_seeds = 200
# get the seeds pool
smiles_seeds = []
for item in lipo.X_orig_train:
if sme.randomize_smiles(item,0) != None:
smiles_seeds.append(item)
if len(smiles_seeds) == n_seeds:
break
smiles_seeds = np.array(smiles_seeds)
X_seeds = lipo.smile_vect(smiles_seeds)
# predict logD value from smiles representation
smiles = np.array(['SCC(=O)O[C@@]1(SC[NH+](C[C@H]1SC=C)C)c2SCSCc2'])
test = np.squeeze(lipo.smile_vect(smiles))
[h_t, c_t, f_t] = lipo.cal_hidden_state(test,layer)
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
threshold_nc = 0
nctoe.testObjective.setParamters(lipo.model, layer, threshold_nc, test)
# test objective KMNC
kmnctoe = KMNCTestObjectiveEvaluation(r)
k_sec = 10
kmnctoe.testObjective.setParamters(lipo.model, layer, k_sec, test)
# test objective NBC
nbctoe = NBCTestObjectiveEvaluation(r)
ub = 0.7
lb = -0.7
nbctoe.testObjective.setParamters(lipo.model, layer, ub, lb, test)
# test objective SNAC
snactoe = NCTestObjectiveEvaluation(r)
threshold_snac = 0.7
snactoe.testObjective.setParamters(lipo.model, layer, threshold_snac, test)
# test objective SC
SCtoe = SCTestObjectiveEvaluation(r)
SC_test_obj = 'h'
act_SC = SCtoe.get_activations(np.array([h_t]))
SCtoe.testObjective.setParamters(lipo.model, SC_test_obj, layer, float(threshold_SC), indices, max_SC, min_SC, np.squeeze(act_SC))
# test objective BC
BCtoe = BCTestObjectiveEvaluation(r)
BC_test_obj = 'h'
act_BC = BCtoe.get_activations(np.array([h_t]))
BCtoe.testObjective.setParamters(lipo.model, BC_test_obj, layer, float(threshold_BC), indices, max_BC, min_BC, np.squeeze(act_BC))
# test objective TC
TCtoe = TCTestObjectiveEvaluation(r)
seq_len = 5
TC_test_obj = 'h'
TCtoe.testObjective.setParamters(lipo.model, TC_test_obj, layer, int(symbols_TC), seq_len, indices, mean_TC, std_TC)
y_seeds = np.squeeze(lipo.model.predict(X_seeds))
X_test = []
r_t = 400 // len(X_seeds)
while lipo.numSamples < int(TestCaseNum):
# generate test cases
unique_test = np.repeat(np.arange(len(X_seeds)), r_t, axis=0)
smiles_test1 = np.repeat(smiles_seeds, r_t, axis=0)
y_test1 = np.repeat(y_seeds, r_t, axis=0)
new_smiles = np.array([sme.randomize_smiles(smiles_test1[i], i+lipo.numSamples) for i in range(len(smiles_test1))])
test2 = lipo.smile_vect(new_smiles)
if lipo.numSamples > 0 and Mutation == 'genetic':
y_test1 = np.concatenate((y_test1, np.array([sc_test_1]), np.array([bc_test_1]), np.array([tc_test_1])))
test2 = np.concatenate((test2, np.array([sc_test_2]), np.array([bc_test_2]), np.array([tc_test_2])))
unique_test = np.concatenate((unique_test, np.array([seed_id_sc]), np.array([seed_id_bc]), np.array([seed_id_tc])))
y_test2 = np.squeeze(lipo.model.predict(test2))
# # display statistics of adv.
lipo.displayInfo(y_test1, y_test2, unique_test)
# calculate the hidden state
h_test = lipo.cal_hidden_keras(test2, layer)
# update the coverage
# update NC coverage
nctoe.update_features(test2)
# update KMNC coverage
kmnctoe.update_features(test2)
# update NBC coverage
nbctoe.update_features(test2)
# update SNAC coverage
snactoe.update_features(test2)
# update SC coverage
SCtoe.update_features(h_test, len(X_test))
# update BC coverage
BCtoe.update_features(h_test, len(X_test))
# update TC coverage
TCtoe.update_features(h_test, len(X_test))
X_test = X_test + test2.tolist()
if Mutation == 'genetic':
num_generation = 10
sc_test_record = SCtoe.testObjective.test_record
bc_test_record = BCtoe.testObjective.test_record
tc_test_record = TCtoe.testObjective.test_record
if len(sc_test_record) != 0:
print('boost coverage for SC')
sc_feature, sc_cov_fit = random.choice(list(sc_test_record.items()))
seed_id_sc = sc_cov_fit[0] % len(X_seeds)
sc_test_1 = y_seeds[seed_id_sc]
# boost coverage with GA
sc_test_2 = getNextInputByGA(lipo, SCtoe, sc_feature, np.array(X_test[sc_cov_fit[0]]), num_generation,
lipo.numSamples)
print('\n')
else:
sc_test_1 = y_seeds[0]
sc_test_2 = X_seeds[0]
seed_id_sc = 0
if len(bc_test_record) != 0:
print('boost coverage for BC')
bc_feature, bc_cov_fit = random.choice(list(bc_test_record.items()))
seed_id_bc = bc_cov_fit[0] % len(X_seeds)
bc_test_1 = y_seeds[seed_id_bc]
# boost coverage with GA
bc_test_2 = getNextInputByGA(lipo, BCtoe, bc_feature, np.array(X_test[bc_cov_fit[0]]), num_generation,
lipo.numSamples)
print('\n')
else:
bc_test_1 = y_seeds[0]
bc_test_2 = X_seeds[0]
seed_id_bc = 0
if len(tc_test_record) != 0:
print('boost coverage for TC')
tc_feature, tc_cov_fit = random.choice(list(tc_test_record.items()))
seed_id_tc = tc_cov_fit[1] % len(X_seeds)
tc_test_1 = y_seeds[seed_id_tc]
# boost coverage with GA
tc_test_2 = getNextInputByGA(lipo, TCtoe, tc_feature, np.array(X_test[tc_cov_fit[1]]), num_generation,
lipo.numSamples)
else:
tc_test_1 = y_seeds[0]
tc_test_2 = X_seeds[0]
seed_id_tc = 0
# write information to file
writeInfo(r, lipo.numSamples, lipo.numAdv, lipo.perturbations, nctoe.coverage, kmnctoe.coverage, nbctoe.coverage,
snactoe.coverage, SCtoe.coverage, BCtoe.coverage, TCtoe.coverage, len(lipo.unique_adv))
print("statistics: \n")
nctoe.displayCoverage()
kmnctoe.displayCoverage()
nbctoe.displayCoverage()
snactoe.displayCoverage()
SCtoe.displayCoverage()
BCtoe.displayCoverage()
TCtoe.displayCoverage()
print('unique adv.', len(lipo.unique_adv))
lipo.displaySuccessRate()
def mnist_lstm_adv_test(r, threshold_SC, threshold_BC, symbols_TC, seq,
TestCaseNum, Mutation, CoverageStop):
r.resetTime()
seeds = 1
np.random.seed(seeds)
random.seed(seeds)
# set up oracle radius
oracleRadius = 0.01
# load model
mn = mnistclass()
mn.load_model()
# test layer
layer = 1
mean = 0
# choose time steps to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
# calculate mean and std for z-norm
h_train = mn.cal_hidden_keras(mn.X_train, layer)
mean_TC, std_TC, max_SC, min_SC, max_BC, min_BC = aggregate_inf(
h_train, indices)
# get the seeds pool
X_seeds = []
# input seeds
for label_idx in range(10):
x_class = mn.X_train[mn.y_train_org == label_idx]
for i in range(89, 99):
X_seeds.append(x_class[i])
# X_seeds = mn.X_train[mn.y_train_org == 2]
# X_seeds = mn.X_train[:50000]
# test case
test = mn.X_test[11]
h_t = mn.cal_hidden_keras(np.array([test]), layer)[0]
# h_t, c_t, f_t = mn.cal_hidden_state(test, layer)
# output digit image
# img = test.reshape((28, 28, 1))
# pred_img = image.array_to_img(img)
# pred_img.save('2.jpg')
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
threshold_nc = 0
nctoe.testObjective.setParamters(mn.model, layer, threshold_nc, test)
# test objective KMNC
kmnctoe = KMNCTestObjectiveEvaluation(r)
k_sec = 10
kmnctoe.testObjective.setParamters(mn.model, layer, k_sec, test)
# test objective NBC
nbctoe = NBCTestObjectiveEvaluation(r)
ub = 0.7
lb = -0.7
nbctoe.testObjective.setParamters(mn.model, layer, ub, lb, test)
# test objective SNAC
snactoe = NCTestObjectiveEvaluation(r)
threshold_snac = 0.7
snactoe.testObjective.setParamters(mn.model, layer, threshold_snac, test)
# test objective SC
SCtoe = SCTestObjectiveEvaluation(r)
SC_test_obj = 'h'
act_SC = SCtoe.get_activations(np.array([h_t]))
SCtoe.testObjective.setParamters(mn.model, SC_test_obj, layer,
float(threshold_SC), indices, max_SC,
min_SC, np.squeeze(act_SC))
# test objective BC
BCtoe = BCTestObjectiveEvaluation(r)
BC_test_obj = 'h'
act_BC = BCtoe.get_activations(np.array([h_t]))
BCtoe.testObjective.setParamters(mn.model, BC_test_obj, layer,
float(threshold_BC), indices, max_BC,
min_BC, np.squeeze(act_BC))
# test objective TC
TCtoe = TCTestObjectiveEvaluation(r)
seq_len = 5
TC_test_obj = 'h'
act_TC = TCtoe.get_activations(np.array([h_t]))
TCtoe.testObjective.setParamters(mn.model, TC_test_obj, layer,
int(symbols_TC), seq_len, indices,
mean_TC, std_TC)
# visualize internal structure information
# act_TC = Z_ScoreNormalization(np.squeeze(act_TC), mean_TC, std_TC)
# act_BC = np.sum(f_t, axis=1) / float(f_t.shape[1])
# act_SC = (np.squeeze(act_SC) -min_SC) / (max_SC - min_SC)
# plt.figure(1)
# plot_x = np.arange(len(act_TC))
# plt.plot(plot_x, act_TC)
# plt.ylabel('$\\xi_t^{h}$', fontsize=14)
# plt.xticks(fontsize=14)
# plt.yticks(fontsize=14)
# plt.figure(2)
# plot_x = np.arange(len(act_BC))
# plt.bar(plot_x, act_BC)
# plt.ylabel('$\\xi_t^{f, avg}$', fontsize=14)
# plt.xticks(fontsize=14)
# plt.yticks(fontsize=14)
# plt.figure(3)
# plot_x = np.arange(len(act_SC))
# plt.bar(plot_x, act_SC)
# plt.xlabel('Input', fontsize=14)
# plt.ylabel('$\Delta\\xi_t^{h}$', fontsize=14)
# plt.xticks(fontsize=14)
# plt.yticks(fontsize=14)
# plt.show()
X_test = []
r_t = 1000 // len(X_seeds)
while mn.numSamples < int(TestCaseNum):
# generate test cases
test1 = np.repeat(X_seeds, r_t, axis=0)
unique_test = np.repeat(np.arange(len(X_seeds)), r_t, axis=0)
var = np.random.uniform(0.005, 0.02)
noise = np.random.normal(mean, var**0.5, test1.shape)
test2 = test1 + noise
test2 = np.clip(test2, 0.0, 1.0)
if mn.numSamples > 0 and Mutation == 'genetic':
test1 = np.concatenate(
(test1, np.array([sc_test_1]), np.array([bc_test_1]),
np.array([tc_test_1])))
test2 = np.concatenate(
(test2, np.array([sc_test_2]), np.array([bc_test_2]),
np.array([tc_test_2])))
unique_test = np.concatenate(
(unique_test, np.array([seed_id_sc]), np.array([seed_id_bc]),
np.array([seed_id_tc])))
# display statistics of adv.
o, m = oracle(test1, test2, 2, oracleRadius)
mn.displayInfo(test1, test2, o, m, unique_test)
# calculate the hidden state
h_test = mn.cal_hidden_keras(test2, layer)
# update the coverage
# update NC coverage
nctoe.update_features(test2)
# update KMNC coverage
kmnctoe.update_features(test2)
# update NBC coverage
nbctoe.update_features(test2)
# update SNAC coverage
snactoe.update_features(test2)
# update SC coverage
SCtoe.update_features(h_test, len(X_test))
# update BC coverage
BCtoe.update_features(h_test, len(X_test))
# update TC coverage
TCtoe.update_features(h_test, len(X_test))
X_test = X_test + test2.tolist()
if Mutation == 'genetic':
num_generation = 10
sc_test_record = SCtoe.testObjective.test_record
bc_test_record = BCtoe.testObjective.test_record
tc_test_record = TCtoe.testObjective.test_record
if len(sc_test_record) != 0:
print('boost coverage for SC')
sc_feature, sc_cov_fit = random.choice(
list(sc_test_record.items()))
seed_id_sc = sc_cov_fit[0] % len(X_seeds)
sc_test_1 = X_seeds[seed_id_sc]
# boost coverage with GA
sc_test_2 = getNextInputByGA(mn, SCtoe, sc_feature,
np.array(X_test[sc_cov_fit[0]]),
num_generation, mn.numSamples)
print('\n')
if len(bc_test_record) != 0:
print('boost coverage for BC')
bc_feature, bc_cov_fit = random.choice(
list(bc_test_record.items()))
seed_id_bc = bc_cov_fit[0] % len(X_seeds)
bc_test_1 = X_seeds[seed_id_bc]
# boost coverage with GA
bc_test_2 = getNextInputByGA(mn, BCtoe, bc_feature,
np.array(X_test[bc_cov_fit[0]]),
num_generation, mn.numSamples)
print('\n')
if len(tc_test_record) != 0:
print('boost coverage for TC')
tc_feature, tc_cov_fit = random.choice(
list(tc_test_record.items()))
seed_id_tc = tc_cov_fit[1] % len(X_seeds)
tc_test_1 = X_seeds[seed_id_tc]
# boost coverage with GA
tc_test_2 = getNextInputByGA(mn, TCtoe, tc_feature,
np.array(X_test[tc_cov_fit[1]]),
num_generation, mn.numSamples)
# write information to file
writeInfo(r, mn.numSamples, mn.numAdv, mn.perturbations,
nctoe.coverage, kmnctoe.coverage, nbctoe.coverage,
snactoe.coverage, SCtoe.coverage, BCtoe.coverage,
TCtoe.coverage, len(mn.unique_adv))
print("statistics: \n")
nctoe.displayCoverage()
kmnctoe.displayCoverage()
nbctoe.displayCoverage()
snactoe.displayCoverage()
SCtoe.displayCoverage()
BCtoe.displayCoverage()
TCtoe.displayCoverage()
print('unique adv.', len(mn.unique_adv))
mn.displaySuccessRate()
def vgg16_lstm_test(r, threshold_SC, threshold_BC, symbols_TC, seq,
TestCaseNum, Mutation, CoverageStop):
r.resetTime()
random.seed(2)
# set up oracle radius
oracleRadius = 0.1 * 255
# load model
uvlc = ucf101_vgg16_lstm_class()
uvlc.model.summary()
# test layer
layer = 0
# preload the data
# X = []
# Y_real = []
# Y_pred = []
# images_X = []
# for index in range(500):
# images, test, label, pred_label = uvlc.predict(index)
# X.append(test)
# Y_real.append(label)
# Y_pred.append(pred_label)
# images_X.append(images)
# np.savez('dataset/very_large_data/ucf101_test.npz', data1 = X, data2 = Y_real, data3 = Y_pred, data4 =images_X)
with np.load('dataset/very_large_data/ucf101_test.npz') as data:
X = data['data1']
Y_real = data['data2']
Y_pred = data['data3']
images_X = data['data4']
acc = 0
for i in range(len(Y_real)):
if Y_pred[i] == Y_real[i]:
acc = acc + 1
acc = acc / len(Y_real)
# choose time step to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
# calculate mean and std for z-norm
h_train = uvlc.cal_hidden_keras(X, layer)
mean_TC, std_TC, max_SC, min_SC, max_BC, min_BC = aggregate_inf(
h_train, indices)
mode = "gaussian"
# noise type "gaussian", "localvar", "poisson", "salt", "pepper", "s&p", "speckle"
# test case
image, test, label, pred_label = uvlc.predict(11)
h_t = uvlc.cal_hidden_keras(np.array([test]), layer)[0]
# get the seeds pool
X_seeds = X[:200]
images_X_seeds = images_X[:200]
Y_seeds = Y_pred[:200]
# X_seeds = X[Y_pred == "Basketball"]
# Y_seeds = Y_pred[Y_pred == "Basketball"]
# images_X_seeds = images_X[Y_pred == "Basketball"]
# X_seeds = X_seeds[:100]
# Y_seeds = Y_seeds[:100]
# images_X_seeds = images_X_seeds[:100]
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
threshold_nc = 0
nctoe.testObjective.setParamters(uvlc.model, layer, threshold_nc, test)
# test objective KMNC
kmnctoe = KMNCTestObjectiveEvaluation(r)
k_sec = 10
kmnctoe.testObjective.setParamters(uvlc.model, layer, k_sec, test)
# test objective NBC
nbctoe = NBCTestObjectiveEvaluation(r)
ub = 0.7
lb = -0.7
nbctoe.testObjective.setParamters(uvlc.model, layer, ub, lb, test)
# test objective SNAC
snactoe = NCTestObjectiveEvaluation(r)
threshold_snac = 0.7
snactoe.testObjective.setParamters(uvlc.model, layer, threshold_snac, test)
# test objective SC
SCtoe = SCTestObjectiveEvaluation(r)
SC_test_obj = 'h'
act_SC = SCtoe.get_activations(np.array([h_t]))
SCtoe.testObjective.setParamters(uvlc.model, SC_test_obj, layer,
float(threshold_SC), indices, max_SC,
min_SC, np.squeeze(act_SC))
# test objective BC
BCtoe = BCTestObjectiveEvaluation(r)
BC_test_obj = 'h'
act_BC = BCtoe.get_activations(np.array([h_t]))
BCtoe.testObjective.setParamters(uvlc.model, BC_test_obj, layer,
float(threshold_BC), indices, max_BC,
min_BC, np.squeeze(act_BC))
# test objective TC
TCtoe = TCTestObjectiveEvaluation(r)
seq_len = 5
TC_test_obj = 'h'
TCtoe.testObjective.setParamters(uvlc.model, TC_test_obj, layer,
int(symbols_TC), seq_len, indices,
mean_TC, std_TC)
images_X_test = []
X_test = []
r_t = 400 // len(X_seeds)
while uvlc.numSamples < int(TestCaseNum):
test1 = np.repeat(X_seeds, r_t, axis=0)
label1 = np.repeat(Y_seeds, r_t, axis=0)
unique_test = np.repeat(np.arange(len(X_seeds)), r_t, axis=0)
var = random.uniform(0.0001, 0.001)
# new_images, test2, label2, conf2 = uvlc.predict_imgs(images, mode, uvlc.numSamples, var)
noise = np.random.normal(0, var**0.5, test1.shape)
test2 = test1 + noise
label2 = uvlc.predict_test(np.array(test2))
if uvlc.numSamples > 0 and Mutation == 'genetic':
label1 = np.concatenate(
(label1, np.array([sc_label_1]), np.array([bc_label_1]),
np.array([tc_label_1])))
label2 = np.concatenate(
(label2, np.array([sc_label_2]), np.array([bc_label_2]),
np.array([tc_label_2])))
unique_test = np.concatenate(
(unique_test, np.array([seed_id_sc]), np.array([seed_id_bc]),
np.array([seed_id_tc])))
test2 = np.concatenate(
(test2, np.array([sc_test_2]), np.array([bc_test_2]),
np.array([tc_test_2])))
o, m = oracle_uvlc(test1, test2, 2, oracleRadius)
uvlc.displayInfo(label1, label2, o, m, unique_test)
# calculate hidden values
h_test = uvlc.cal_hidden_keras(np.array(test2), layer)
# update the coverage
# update NC coverage
nctoe.update_features(test2)
# update KMNC coverage
kmnctoe.update_features(test2)
# update NBC coverage
nbctoe.update_features(test2)
# update SNAC coverage
snactoe.update_features(test2)
# update SC coverage
SCtoe.update_features(h_test, len(X_test))
# update BC coverage
BCtoe.update_features(h_test, len(X_test))
# update TC coverage
TCtoe.update_features(h_test, len(X_test))
X_test = X_test + test2.tolist()
if Mutation == 'genetic':
num_generation = 10
sc_test_record = SCtoe.testObjective.test_record
bc_test_record = BCtoe.testObjective.test_record
tc_test_record = TCtoe.testObjective.test_record
if len(sc_test_record) != 0:
print('boost coverage for SC')
sc_feature, sc_cov_fit = random.choice(
list(sc_test_record.items()))
seed_id_sc = sc_cov_fit[0] % len(X_seeds)
sc_label_1 = Y_seeds[seed_id_sc]
# boost coverage with GA
sc_test_2 = getNextInputByGA(uvlc, SCtoe, sc_feature,
np.array(X_test[sc_cov_fit[0]]),
num_generation, uvlc.numSamples)
sc_label_2 = uvlc.predict_test(np.array([sc_test_2]))[0]
print('\n')
if len(bc_test_record) != 0:
print('boost coverage for BC')
bc_feature, bc_cov_fit = random.choice(
list(bc_test_record.items()))
seed_id_bc = bc_cov_fit[0] % len(X_seeds)
bc_label_1 = Y_seeds[seed_id_bc]
# boost coverage with GA
bc_test_2 = getNextInputByGA(uvlc, BCtoe, bc_feature,
np.array(X_test[bc_cov_fit[0]]),
num_generation, uvlc.numSamples)
bc_label_2 = uvlc.predict_test(np.array([bc_test_2]))[0]
print('\n')
if len(tc_test_record) != 0:
print('boost coverage for TC')
tc_feature, tc_cov_fit = random.choice(
list(tc_test_record.items()))
seed_id_tc = tc_cov_fit[1] % len(X_seeds)
tc_label_1 = Y_seeds[seed_id_tc]
# boost coverage with GA
tc_test_2 = getNextInputByGA(uvlc, TCtoe, tc_feature,
np.array(X_test[tc_cov_fit[1]]),
num_generation, uvlc.numSamples)
tc_label_2 = uvlc.predict_test(np.array([tc_test_2]))[0]
# write information to file
writeInfo(r, uvlc.numSamples, uvlc.numAdv, uvlc.perturbations,
nctoe.coverage, kmnctoe.coverage, nbctoe.coverage,
snactoe.coverage, SCtoe.coverage, BCtoe.coverage,
TCtoe.coverage, len(uvlc.unique_adv))
print("statistics: \n")
nctoe.displayCoverage()
kmnctoe.displayCoverage()
nbctoe.displayCoverage()
snactoe.displayCoverage()
SCtoe.displayCoverage()
BCtoe.displayCoverage()
TCtoe.displayCoverage()
uvlc.displaySamples()
print('unique adv.', len(uvlc.unique_adv))
uvlc.displaySuccessRate()
def ts_lstm_test(r, threshold_CC, threshold_MC, symbols_SQ, seq, TestCaseNum,
minimalTest, TargMetri, CoverageStop):
r.resetTime()
random.seed(1)
# set oracle radius
oracleRadius = 0.2
# load model
ts = tsClass()
ts.load_data('dataset/sp500.csv', 50, True)
model = ts.load_model()
if not model:
ts.train_model()
# minimal test dataset generation
if minimalTest != '0':
ncdata = []
ccdata = []
mcdata = []
sqpdata = []
sqndata = []
# test layer
layer = 1
termin = 0
test_data = [
1455.219971, 1399.420044, 1402.109985, 1403.449951, 1441.469971,
1457.599976, 1438.560059, 1432.25, 1449.680054, 1465.150024,
1455.140015, 1455.900024, 1445.569946, 1441.359985, 1401.530029,
1410.030029, 1404.089966, 1398.560059, 1360.160034, 1394.459961,
1409.280029, 1409.119995, 1424.969971, 1424.369995, 1424.23999,
1441.719971, 1411.709961, 1416.829956, 1387.119995, 1389.939941,
1402.050049, 1387.670044, 1388.26001, 1346.089966, 1352.170044,
1360.689941, 1353.430054, 1333.359985, 1348.050049, 1366.420044,
1379.189941, 1381.76001, 1409.170044, 1391.280029, 1355.619995,
1366.699951, 1401.689941, 1395.069946, 1383.619995, 1359.150024
]
# predict logD value from smiles representation
test_array = np.array(test_data).reshape((1, 50, 1))
test = np.array(ts.create_sequence(test_array))
h_t, c_t, f_t = ts.cal_hidden_state(test)
# input seeds
X_train = ts.X_train[random.sample(range(3100), 3000)]
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
nctoe.model = ts.model
nctoe.testObjective.layer = layer
nctoe.testCase = test
activations_nc = nctoe.get_activations()
nctoe.testObjective.feature = (np.argwhere(
activations_nc >= np.min(activations_nc))).tolist()
nctoe.testObjective.setOriginalNumOfFeature()
# test objective CC
cctoe = CCTestObjectiveEvaluation(r)
cctoe.model = ts.model
cctoe.testObjective.layer = layer
cctoe.hidden = h_t
cctoe.threshold = float(threshold_CC)
activations_cc = cctoe.get_activations()
total_features_cc = (np.argwhere(
activations_cc >= np.min(activations_cc))).tolist()
cctoe.testObjective.feature = total_features_cc
cctoe.testObjective.setOriginalNumOfFeature()
cctoe.testObjective.setfeaturecount()
# test objective MC
mctoe = MCTestObjectiveEvaluation(r)
mctoe.model = ts.model
mctoe.testObjective.layer = layer
mctoe.hidden = f_t
mctoe.threshold = float(threshold_MC)
activations_mc = mctoe.get_activations()
total_features_mc = (np.argwhere(
activations_mc >= np.min(activations_mc))).tolist()
mctoe.testObjective.feature = total_features_mc
mctoe.testObjective.setOriginalNumOfFeature()
mctoe.testObjective.setfeaturecount()
# test objective SQ
sqtoe = SQTestObjectiveEvaluation(r)
sqtoe.model = ts.model
sqtoe.testObjective.layer = layer
sqtoe.symbols = int(symbols_SQ)
# generate all the features
# choose time steps to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
#slice(70, 75)
# characters to represent time series
alpha_list = [chr(i) for i in range(97, 97 + int(symbols_SQ))]
symb = ''.join(alpha_list)
sqtoe.testObjective.feature_p = list(iter.product(symb,
repeat=t2 - t1 + 1))
sqtoe.testObjective.feature_n = list(iter.product(symb,
repeat=t2 - t1 + 1))
sqtoe.testObjective.setOriginalNumOfFeature()
for test in X_train:
for i in range(4):
pred1 = ts.displayInfo(test)
# get next input test2 from the current input test
smiles = ts.vect_smile(np.array([test]))
new_smiles = np.array([sme.randomize_smiles(smiles[0], i)])
test2 = np.squeeze(ts.smile_vect(new_smiles))
if not (test2 is None):
pred2 = ts.displayInfo(test2)
h_t, c_t, f_t = ts.cal_hidden_state(test2)
cctoe.hidden = h_t
ts.updateSample(pred1, pred2, 0, True)
# update NC coverage
nctoe.testCase = test2
nctoe.update_features()
# update CC coverage
cctoe.hidden = h_t
cctoe.update_features()
# update MC coverage
mctoe.hidden = f_t
mctoe.update_features()
# update SQ coverage
sqtoe.hidden = h_t
sqtoe.update_features(indices)
# write information to file
writeInfo(r, ts.numSamples, ts.numAdv, ts.perturbations,
nctoe.coverage, cctoe.coverage, mctoe.coverage,
sqtoe.coverage_p, sqtoe.coverage_n)
# terminate condition
if TargMetri == 'CC':
termin = cctoe.coverage
elif TargMetri == 'GC':
termin = mctoe.coverage
elif TargMetri == 'SQN':
termin = sqtoe.coverage_n
elif TargMetri == 'SQP':
termin = sqtoe.coverage_p
# output test cases and adversarial example
if minimalTest == '0':
f = open('output/smiles_test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if abs(pred1 - pred2) >= 1:
f = open('adv_output/adv_smiles_test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
else:
if nctoe.minimal == 1:
ncdata.append(test2)
f = open('minimal_nc/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if cctoe.minimal == 1:
ccdata.append(test2)
f = open('minimal_cc/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if mctoe.minimal == 1:
mcdata.append(test2)
f = open('minimal_mc/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if sqtoe.minimalp == 1:
sqpdata.append(test2)
f = open('minimal_sqp/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if sqtoe.minimaln == 1:
sqndata.append(test2)
f = open('minimal_sqn/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
# check termination condition
if ts.numSamples < int(TestCaseNum) and termin < float(
CoverageStop):
continue
else:
io.savemat(
'log_folder/feature_count_CC.mat',
{'feature_count_CC': cctoe.testObjective.feature_count})
io.savemat(
'log_folder/feature_count_GC.mat',
{'feature_count_GC': mctoe.testObjective.feature_count})
# if minimalTest != '0':
# np.save('minimal_nc/ncdata', ncdata)
# np.save('minimal_cc/ccdata', ccdata)
# np.save('minimal_mc/mcdata', mcdata)
# np.save('minimal_sqp/sqpdata', sqpdata)
# np.save('minimal_sqn/sqndata', sqndata)
break
if ts.numSamples < int(TestCaseNum) and termin < float(CoverageStop):
continue
else:
break
print("statistics: \n")
nctoe.displayCoverage()
cctoe.displayCoverage()
mctoe.displayCoverage()
sqtoe.displayCoverage1()
sqtoe.displayCoverage2()
ts.displaySamples()
ts.displaySuccessRate()
def mnist_lstm_test(r, threshold_CC, threshold_MC, symbols_SQ, seq,
TestCaseNum, minimalTest, TargMetri, CoverageStop):
r.resetTime()
# epsilon value range (a, b]
random.seed(3)
a = 0.05
b = 0.1
step_bound = 5
# set up oracle radius
oracleRadius = 0.005
# load model
mn = mnistclass()
mn.load_model()
# test layer
layer = 1
termin = 0
# test case
test = mn.X_test[15]
h_t, c_t, f_t = mn.cal_hidden_state(test, layer)
# input seeds
X_train = mn.X_train[random.sample(range(20000), 5000)]
# minimal test dataset generation
if minimalTest != '0':
ncdata = []
ccdata = []
mcdata = []
sqpdata = []
sqndata = []
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
nctoe.model = mn.model
nctoe.testObjective.layer = layer
nctoe.testCase = test
activations_nc = nctoe.get_activations()
nctoe.testObjective.feature = (np.argwhere(
activations_nc >= np.min(activations_nc))).tolist()
nctoe.testObjective.setOriginalNumOfFeature()
# test objective CC
cctoe = CCTestObjectiveEvaluation(r)
cctoe.model = mn.model
cctoe.testObjective.layer = layer
cctoe.hidden = h_t
cctoe.threshold = float(threshold_CC)
activations_cc = cctoe.get_activations()
total_features_cc = (np.argwhere(
activations_cc >= np.min(activations_cc))).tolist()
cctoe.testObjective.feature = total_features_cc
cctoe.testObjective.setOriginalNumOfFeature()
cctoe.testObjective.setfeaturecount()
# test objective MC
mctoe = MCTestObjectiveEvaluation(r)
mctoe.model = mn.model
mctoe.testObjective.layer = layer
mctoe.hidden = f_t
mctoe.threshold = float(threshold_MC)
activations_mc = mctoe.get_activations()
total_features_mc = (np.argwhere(
activations_mc >= np.min(activations_mc))).tolist()
mctoe.testObjective.feature = total_features_mc
mctoe.testObjective.setOriginalNumOfFeature()
mctoe.testObjective.setfeaturecount()
# test objective SQ
sqtoe = SQTestObjectiveEvaluation(r)
sqtoe.model = mn.model
sqtoe.testObjective.layer = layer
sqtoe.symbols = int(symbols_SQ)
# generate all the features
# choose time steps to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
# characters to represent time series
alpha_list = [chr(i) for i in range(97, 97 + int(symbols_SQ))]
symb = ''.join(alpha_list)
sqtoe.testObjective.feature_p = list(iter.product(symb,
repeat=t2 - t1 + 1))
sqtoe.testObjective.feature_n = list(iter.product(symb,
repeat=t2 - t1 + 1))
sqtoe.testObjective.setOriginalNumOfFeature()
# get gradient function for the mnist
f, nodes_names = get_gradients_function(mn.model, mn.layerName(0))
for test in X_train:
for i in range(4):
o = oracle(test, 2, oracleRadius)
last_activation = np.squeeze(mn.model.predict(test[np.newaxis, :]))
(label1, conf1) = mn.displayInfo(test)
epsilon = random.uniform(a, b)
# get next input test2 from the current input test
step = random.randint(1, step_bound)
test2 = getNextInputByGradient(f, nodes_names, mn, epsilon, test,
last_activation, step)
if not (test2 is None):
(label2, conf2) = mn.displayInfo(test2)
h_t, c_t, f_t = mn.cal_hidden_state(test2, layer)
mn.updateSample(label2, label1, o.measure(test2),
o.passOracle(test2))
# update NC coverage
nctoe.testCase = test2
nctoe.update_features()
# update CC coverage
cctoe.hidden = h_t
cctoe.update_features()
# update MC coverage
mctoe.hidden = f_t
mctoe.update_features()
# update SQ coverage
sqtoe.hidden = h_t
sqtoe.update_features(indices)
# write information to file
writeInfo(r, mn.numSamples, mn.numAdv, mn.perturbations,
nctoe.coverage, cctoe.coverage, mctoe.coverage,
sqtoe.coverage_p, sqtoe.coverage_n)
# terminate condition
if TargMetri == 'CC':
termin = cctoe.coverage
elif TargMetri == 'GC':
termin = mctoe.coverage
elif TargMetri == 'SQN':
termin = sqtoe.coverage_n
elif TargMetri == 'SQP':
termin = sqtoe.coverage_p
# output test cases and adversarial example
if minimalTest == '0':
img = test2.reshape((28, 28, 1))
pred_img = image.array_to_img(img)
pred_img.save('output/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
if label2 != label1 and o.passOracle(test2) == True:
pred_img.save('adv_output/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
else:
img = test2.reshape((28, 28, 1))
pred_img = image.array_to_img(img)
if nctoe.minimal == 1:
ncdata.append(test2)
pred_img.save('minimal_nc/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
if cctoe.minimal == 1:
ccdata.append(test2)
pred_img.save('minimal_cc/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
if mctoe.minimal == 1:
mcdata.append(test2)
pred_img.save('minimal_mc/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
if sqtoe.minimalp == 1:
sqpdata.append(test2)
pred_img.save('minimal_sqp/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
if sqtoe.minimaln == 1:
sqndata.append(test2)
pred_img.save('minimal_sqn/output_%d_%d_%d.jpg' %
(mn.numSamples, label1, label2))
# check termination condition
if mn.numSamples < int(TestCaseNum) and termin < float(
CoverageStop):
continue
else:
io.savemat(
'log_folder/feature_count_CC.mat',
{'feature_count_CC': cctoe.testObjective.feature_count})
io.savemat(
'log_folder/feature_count_GC.mat',
{'feature_count_GC': mctoe.testObjective.feature_count})
# if minimalTest != '0':
# np.save('minimal_nc/ncdata', ncdata)
# np.save('minimal_cc/ccdata', ccdata)
# np.save('minimal_mc/mcdata', mcdata)
# np.save('minimal_sqp/sqpdata', sqpdata)
# np.save('minimal_sqn/sqndata', sqndata)
break
if mn.numSamples < int(TestCaseNum) and termin < float(CoverageStop):
continue
else:
break
print("statistics: \n")
nctoe.displayCoverage()
cctoe.displayCoverage()
mctoe.displayCoverage()
sqtoe.displayCoverage1()
sqtoe.displayCoverage2()
mn.displaySamples()
mn.displaySuccessRate()
def sentimentGenerateTestSuite(r, threshold_SC, threshold_BC, symbols_TC, seq,
TestCaseNum, Mutation, CoverageStop):
r.resetTime()
seeds = 3
random.seed(seeds)
# set oracle radius
oracleRadius = 0.2
# load model
sm = Sentiment()
sm.load_model()
# test layer
layer = 1
#choose time step to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
# calculate mean and std for z-norm
h_train = sm.cal_hidden_keras(sm.X_train, layer)
mean_TC, std_TC, max_SC, min_SC, max_BC, min_BC = aggregate_inf(
h_train, indices)
# get the seeds pool
X_seeds = []
# input seeds
for label_idx in range(2):
x_class = sm.X_train[sm.y_train == label_idx]
for i in range(100, 200):
X_seeds.append(x_class[i])
# X_seeds = sm.X_train[sm.y_train == 0]
# X_seeds = X_seeds[:100]
# predict sentiment from reviews
review = "really good film to watch and highly recommended"
# review = "movie is horrible and watching experience is terrible"
tmp = sm.fromTextToID(review)
test = np.squeeze(sm.pre_processing_x([tmp]))
[h_t, c_t, f_t] = sm.cal_hidden_state(test, layer)
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
threshold_nc = 0
nctoe.testObjective.setParamters(sm.model, layer, threshold_nc, test)
# test objective KMNC
kmnctoe = KMNCTestObjectiveEvaluation(r)
k_sec = 10
kmnctoe.testObjective.setParamters(sm.model, layer, k_sec, test)
# test objective NBC
nbctoe = NBCTestObjectiveEvaluation(r)
ub = 0.7
lb = -0.7
nbctoe.testObjective.setParamters(sm.model, layer, ub, lb, test)
# test objective SNAC
snactoe = NCTestObjectiveEvaluation(r)
threshold_snac = 0.7
snactoe.testObjective.setParamters(sm.model, layer, threshold_snac, test)
# test objective SC
SCtoe = SCTestObjectiveEvaluation(r)
SC_test_obj = 'h'
act_SC = SCtoe.get_activations(np.array([h_t]))
SCtoe.testObjective.setParamters(sm.model, SC_test_obj, layer,
float(threshold_SC), indices, max_SC,
min_SC, np.squeeze(act_SC))
# test objective BC
BCtoe = BCTestObjectiveEvaluation(r)
BC_test_obj = 'h'
act_BC = BCtoe.get_activations(np.array([h_t]))
BCtoe.testObjective.setParamters(sm.model, BC_test_obj, layer,
float(threshold_BC), indices, max_BC,
min_BC, np.squeeze(act_BC))
# test objective TC
TCtoe = TCTestObjectiveEvaluation(r)
seq_len = 5
TC_test_obj = 'h'
act_TC = TCtoe.get_activations(np.array([h_t]))
TCtoe.testObjective.setParamters(sm.model, TC_test_obj, layer,
int(symbols_TC), seq_len, indices,
mean_TC, std_TC)
# visualize internal structure information
# act_TC = np.squeeze(act_TC)[-8:]
# act_SC = np.squeeze(act_SC)[-8:]
# act_TC = Z_ScoreNormalization(act_TC, mean_TC, std_TC)
# act_BC = np.sum(f_t, axis=1) / float(f_t.shape[1])
# act_BC = act_BC[-8:]
# act_SC = (act_SC - min_SC) / (max_SC - min_SC)
#
# plt.figure(1)
# plot_x = np.arange(len(act_TC))
# plt.plot(plot_x, act_TC)
# plt.ylabel('$\\xi_t^{h}$', fontsize=14)
# plt.xticks(fontsize=14)
# plt.yticks(fontsize=14)
# plt.figure(2)
# plot_x = np.arange(len(act_BC))
# plt.bar(plot_x, act_BC)
# plt.ylabel('$\\xi_t^{f, avg}$', fontsize=14)
# plt.xticks(fontsize=14)
# plt.yticks(fontsize=14)
# plt.figure(3)
# plot_x = np.arange(len(act_SC))
# plt.bar(plot_x, act_SC)
# plt.xlabel('Input', fontsize=14)
# plt.ylabel('$\Delta\\xi_t^{h}$', fontsize=14)
# plt.xticks(fontsize=14)
# plt.yticks(fontsize=14)
# plt.show()
text_seeds = [sm.fromIDToText(item) for item in X_seeds]
y_seeds = sm.getOutputResult(X_seeds)
X_test = []
r_t = 400 // len(X_seeds)
while sm.numSamples < int(TestCaseNum):
# generate test cases
unique_test = np.repeat(np.arange(len(X_seeds)), r_t, axis=0)
y_test1 = np.repeat(y_seeds, r_t, axis=0)
alpha = random.uniform(0.01, oracleRadius)
aug_text = []
for text in text_seeds:
out = eda(text,
sm.numSamples,
alpha_sr=alpha,
alpha_ri=alpha,
alpha_rs=alpha,
p_rd=alpha,
num_aug=r_t)
aug_text = aug_text + out
tmp = [sm.fromTextToID(text) for text in aug_text]
test2 = sm.pre_processing_x(tmp)
if sm.numSamples > 0 and Mutation == 'genetic':
y_test1 = np.concatenate(
(y_test1, np.array([sc_test_1]), np.array([bc_test_1]),
np.array([tc_test_1])))
test2 = np.concatenate(
(test2, np.array([sc_test_2]), np.array([bc_test_2]),
np.array([tc_test_2])))
unique_test = np.concatenate(
(unique_test, np.array([seed_id_sc]), np.array([seed_id_bc]),
np.array([seed_id_tc])))
y_test2 = sm.getOutputResult(test2)
# # display statistics of adv.
sm.displayInfo(y_test1, y_test2, alpha, unique_test)
# calculate the hidden state
h_test = sm.cal_hidden_keras(test2, layer)
# update the coverage
# update NC coverage
nctoe.update_features(test2)
# update KMNC coverage
kmnctoe.update_features(test2)
# update NBC coverage
nbctoe.update_features(test2)
# update SNAC coverage
snactoe.update_features(test2)
# update SC coverage
SCtoe.update_features(h_test, len(X_test))
# update BC coverage
BCtoe.update_features(h_test, len(X_test))
# update TC coverage
TCtoe.update_features(h_test, len(X_test))
X_test = X_test + test2.tolist()
if Mutation == 'genetic':
num_generation = 10
sc_test_record = SCtoe.testObjective.test_record
bc_test_record = BCtoe.testObjective.test_record
tc_test_record = TCtoe.testObjective.test_record
if len(sc_test_record) != 0:
print('boost coverage for SC')
sc_feature, sc_cov_fit = random.choice(
list(sc_test_record.items()))
seed_id_sc = sc_cov_fit[0] % len(X_seeds)
sc_test_1 = y_seeds[seed_id_sc]
# boost coverage with GA
sc_test_2 = getNextInputByGA(sm, SCtoe, sc_feature,
np.array(X_test[sc_cov_fit[0]]),
num_generation, sm.numSamples)
print('\n')
if len(bc_test_record) != 0:
print('boost coverage for BC')
bc_feature, bc_cov_fit = random.choice(
list(bc_test_record.items()))
seed_id_bc = bc_cov_fit[0] % len(X_seeds)
bc_test_1 = y_seeds[seed_id_bc]
# boost coverage with GA
bc_test_2 = getNextInputByGA(sm, BCtoe, bc_feature,
np.array(X_test[bc_cov_fit[0]]),
num_generation, sm.numSamples)
print('\n')
if len(tc_test_record) != 0:
print('boost coverage for TC')
tc_feature, tc_cov_fit = random.choice(
list(tc_test_record.items()))
seed_id_tc = tc_cov_fit[1] % len(X_seeds)
tc_test_1 = y_seeds[seed_id_tc]
# boost coverage with GA
tc_test_2 = getNextInputByGA(sm, TCtoe, tc_feature,
np.array(X_test[tc_cov_fit[1]]),
num_generation, sm.numSamples)
# write information to file
writeInfo(r, sm.numSamples, sm.numAdv, sm.perturbations,
nctoe.coverage, kmnctoe.coverage, nbctoe.coverage,
snactoe.coverage, SCtoe.coverage, BCtoe.coverage,
TCtoe.coverage, len(sm.unique_adv))
print("statistics: \n")
nctoe.displayCoverage()
kmnctoe.displayCoverage()
nbctoe.displayCoverage()
snactoe.displayCoverage()
SCtoe.displayCoverage()
BCtoe.displayCoverage()
TCtoe.displayCoverage()
print('unique adv.', len(sm.unique_adv))
sm.displaySuccessRate()
def lipo_lstm_test(r, threshold_CC, threshold_MC, symbols_SQ, seq, TestCaseNum,
minimalTest, TargMetri, CoverageStop):
r.resetTime()
random.seed(1)
# set oracle radius
oracleRadius = 0.2
# load model
lipo = lipoClass()
lipo.load_data()
lipo.load_model()
# minimal test dataset generation
if minimalTest != '0':
ncdata = []
ccdata = []
mcdata = []
sqpdata = []
sqndata = []
# test layer
layer = 1
termin = 0
# predict logD value from smiles representation
smiles = np.array(['CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'])
test = np.squeeze(lipo.smile_vect(smiles))
h_t, c_t, f_t = lipo.cal_hidden_state(test)
# input seeds
X_train = lipo.X_train[random.sample(range(3100), 3000)]
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
nctoe.model = lipo.model
nctoe.testObjective.layer = layer
nctoe.testCase = test
activations_nc = nctoe.get_activations()
nctoe.testObjective.feature = (np.argwhere(
activations_nc >= np.min(activations_nc))).tolist()
nctoe.testObjective.setOriginalNumOfFeature()
# test objective CC
cctoe = CCTestObjectiveEvaluation(r)
cctoe.model = lipo.model
cctoe.testObjective.layer = layer
cctoe.hidden = h_t
cctoe.threshold = float(threshold_CC)
activations_cc = cctoe.get_activations()
total_features_cc = (np.argwhere(
activations_cc >= np.min(activations_cc))).tolist()
cctoe.testObjective.feature = total_features_cc
cctoe.testObjective.setOriginalNumOfFeature()
cctoe.testObjective.setfeaturecount()
# test objective MC
mctoe = MCTestObjectiveEvaluation(r)
mctoe.model = lipo.model
mctoe.testObjective.layer = layer
mctoe.hidden = f_t
mctoe.threshold = float(threshold_MC)
activations_mc = mctoe.get_activations()
total_features_mc = (np.argwhere(
activations_mc >= np.min(activations_mc))).tolist()
mctoe.testObjective.feature = total_features_mc
mctoe.testObjective.setOriginalNumOfFeature()
mctoe.testObjective.setfeaturecount()
# test objective SQ
sqtoe = SQTestObjectiveEvaluation(r)
sqtoe.model = lipo.model
sqtoe.testObjective.layer = layer
sqtoe.symbols = int(symbols_SQ)
# generate all the features
# choose time steps to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
#slice(70, 75)
# characters to represent time series
alpha_list = [chr(i) for i in range(97, 97 + int(symbols_SQ))]
symb = ''.join(alpha_list)
sqtoe.testObjective.feature_p = list(iter.product(symb,
repeat=t2 - t1 + 1))
sqtoe.testObjective.feature_n = list(iter.product(symb,
repeat=t2 - t1 + 1))
sqtoe.testObjective.setOriginalNumOfFeature()
# define smile enumerator of a molecule
sme = SmilesEnumerator()
for test in X_train:
for i in range(4):
pred1 = lipo.displayInfo(test)
# get next input test2 from the current input test
smiles = lipo.vect_smile(np.array([test]))
new_smiles = np.array([sme.randomize_smiles(smiles[0], i)])
test2 = np.squeeze(lipo.smile_vect(new_smiles))
if not (test2 is None):
pred2 = lipo.displayInfo(test2)
h_t, c_t, f_t = lipo.cal_hidden_state(test2)
cctoe.hidden = h_t
lipo.updateSample(pred1, pred2, 0, True)
# update NC coverage
nctoe.testCase = test2
nctoe.update_features()
# update CC coverage
cctoe.hidden = h_t
cctoe.update_features()
# update MC coverage
mctoe.hidden = f_t
mctoe.update_features()
# update SQ coverage
sqtoe.hidden = h_t
sqtoe.update_features(indices)
# write information to file
writeInfo(r, lipo.numSamples, lipo.numAdv, lipo.perturbations,
nctoe.coverage, cctoe.coverage, mctoe.coverage,
sqtoe.coverage_p, sqtoe.coverage_n)
# terminate condition
if TargMetri == 'CC':
termin = cctoe.coverage
elif TargMetri == 'GC':
termin = mctoe.coverage
elif TargMetri == 'SQN':
termin = sqtoe.coverage_n
elif TargMetri == 'SQP':
termin = sqtoe.coverage_p
# output test cases and adversarial example
if minimalTest == '0':
f = open('output/smiles_test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if abs(pred1 - pred2) >= 1:
f = open('adv_output/adv_smiles_test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
else:
if nctoe.minimal == 1:
ncdata.append(test2)
f = open('minimal_nc/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if cctoe.minimal == 1:
ccdata.append(test2)
f = open('minimal_cc/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if mctoe.minimal == 1:
mcdata.append(test2)
f = open('minimal_mc/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if sqtoe.minimalp == 1:
sqpdata.append(test2)
f = open('minimal_sqp/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
if sqtoe.minimaln == 1:
sqndata.append(test2)
f = open('minimal_sqn/test_set.txt', 'a')
f.write(new_smiles[0])
f.write('\n')
f.close()
# check termination condition
if lipo.numSamples < int(TestCaseNum) and termin < float(
CoverageStop):
continue
else:
io.savemat(
'log_folder/feature_count_CC.mat',
{'feature_count_CC': cctoe.testObjective.feature_count})
io.savemat(
'log_folder/feature_count_GC.mat',
{'feature_count_GC': mctoe.testObjective.feature_count})
# if minimalTest != '0':
# np.save('minimal_nc/ncdata', ncdata)
# np.save('minimal_cc/ccdata', ccdata)
# np.save('minimal_mc/mcdata', mcdata)
# np.save('minimal_sqp/sqpdata', sqpdata)
# np.save('minimal_sqn/sqndata', sqndata)
break
if lipo.numSamples < int(TestCaseNum) and termin < float(CoverageStop):
continue
else:
break
print("statistics: \n")
nctoe.displayCoverage()
cctoe.displayCoverage()
mctoe.displayCoverage()
sqtoe.displayCoverage1()
sqtoe.displayCoverage2()
lipo.displaySamples()
lipo.displaySuccessRate()
def sentimentGenerateTestSuite(r,threshold_CC,threshold_MC,symbols_SQ,seq,TestCaseNum,minimalTest,TargMetri,CoverageStop):
r.resetTime()
random.seed(1)
# set oracle radius
oracleRadius = 0.2
# load model
sm = Sentiment()
sm.load_model()
# test layer
layer = 1
termin = 0
# minimal test dataset generation
if minimalTest != '0':
ncdata = []
ccdata = []
mcdata = []
sqpdata = []
sqndata = []
# predict sentiment from reviews
review = "i really dislike the movie"
tmp = sm.fromTextToID(review)
test = np.squeeze(sm.pre_processing_x(tmp))
h_t, c_t, f_t = sm.cal_hidden_state(test)
# input seeds
X_train = sm.X_train[random.sample(range(20000),5000)]
# test objective NC
nctoe = NCTestObjectiveEvaluation(r)
nctoe.model = sm.model
nctoe.testObjective.layer = layer
nctoe.testCase = test
activations_nc = nctoe.get_activations()
nctoe.testObjective.feature = (np.argwhere(activations_nc >= np.min(activations_nc))).tolist()
nctoe.testObjective.setOriginalNumOfFeature()
# test objective CC
cctoe = CCTestObjectiveEvaluation(r)
cctoe.model = sm.model
cctoe.testObjective.layer = layer
cctoe.hidden = h_t
cctoe.threshold = float(threshold_CC)
activations_cc = cctoe.get_activations()
total_features_cc = (np.argwhere(activations_cc >= np.min(activations_cc))).tolist()
cctoe.testObjective.feature = total_features_cc
cctoe.testObjective.setOriginalNumOfFeature()
cctoe.testObjective.setfeaturecount()
# test objective MC
mctoe = MCTestObjectiveEvaluation(r)
mctoe.model = sm.model
mctoe.testObjective.layer = layer
mctoe.hidden = f_t
mctoe.threshold = float(threshold_MC)
activations_mc = mctoe.get_activations()
total_features_mc = (np.argwhere(activations_mc >= np.min(activations_mc))).tolist()
mctoe.testObjective.feature = total_features_mc
mctoe.testObjective.setOriginalNumOfFeature()
mctoe.testObjective.setfeaturecount()
# test objective SQ
sqtoe = SQTestObjectiveEvaluation(r)
sqtoe.model = sm.model
sqtoe.testObjective.layer = layer
sqtoe.symbols = int(symbols_SQ)
# generate all the features
# choose time steps to cover
t1 = int(seq[0])
t2 = int(seq[1])
indices = slice(t1, t2 + 1)
# slice(480, 485)
# characters to represent time series
alpha_list = [chr(i) for i in range(97, 97 + int(symbols_SQ))]
symb = ''.join(alpha_list)
sqtoe.testObjective.feature_p = list(iter.product(symb, repeat=t2-t1+1))
sqtoe.testObjective.feature_n = list(iter.product(symb, repeat=t2-t1+1))
sqtoe.testObjective.setOriginalNumOfFeature()
for test in X_train:
for i in range(4):
text = sm.fromIDToText(test)
(label1, conf1) = sm.displayInfo(test)
# get next input test2
# test case pertubations
alpha = random.uniform(0.001, oracleRadius)
aug_text = eda(text, alpha_sr=alpha, alpha_ri=alpha, alpha_rs=alpha, p_rd=alpha, num_aug=1)
tmp = sm.fromTextToID(str(aug_text[0]))
test2 = np.squeeze(sm.pre_processing_x(tmp))
if not (test2 is None):
(label2, conf2) = sm.displayInfo(test2)
h_t, c_t, f_t = sm.cal_hidden_state(test2)
cctoe.hidden = h_t
sm.updateSample(label2, label1, alpha, True)
# update NC coverage
nctoe.testCase = test2
nctoe.update_features()
# update CC coverage
cctoe.hidden = h_t
cctoe.update_features()
# update MC coverage
mctoe.hidden = f_t
mctoe.update_features()
# update SQ coverage
sqtoe.hidden = h_t
sqtoe.update_features(indices)
# write information to file
writeInfo(r, sm.numSamples, sm.numAdv, sm.perturbations,nctoe.coverage,cctoe.coverage, mctoe.coverage, sqtoe.coverage_p,sqtoe.coverage_n)
# terminate condition
if TargMetri == 'CC':
termin = cctoe.coverage
elif TargMetri == 'GC':
termin = mctoe.coverage
elif TargMetri == 'SQN':
termin = sqtoe.coverage_n
elif TargMetri == 'SQP':
termin = sqtoe.coverage_p
# output test cases and adversarial examples
if minimalTest == '0':
f = open('output/test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
if label2 != label1 :
f = open('adv_output/adv_test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.write(str(label2))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
else:
if nctoe.minimal == 1 :
ncdata.append(test2)
f = open('minimal_nc/test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
if cctoe.minimal == 1 :
ccdata.append(test2)
f = open('minimal_cc/test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
if mctoe.minimal == 1 :
mcdata.append(test2)
f = open('minimal_mc/test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
if sqtoe.minimalp == 1 :
sqpdata.append(test2)
f = open('minimal_sqp/test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
if sqtoe.minimaln == 1 :
sqndata.append(test2)
f = open('minimal_sqn/test_set.txt', 'a')
f.write(str(label1))
f.write('\t')
f.writelines(str(aug_text[0]))
f.write('\n')
f.close()
# check termination condition
if sm.numSamples < int(TestCaseNum) and termin < float(CoverageStop):
continue
else:
io.savemat('log_folder/feature_count_CC.mat', {'feature_count_CC': cctoe.testObjective.feature_count})
io.savemat('log_folder/feature_count_GC.mat', {'feature_count_GC': mctoe.testObjective.feature_count})
# if minimalTest != '0':
# np.save('minimal_nc/ncdata', ncdata)
# np.save('minimal_cc/ccdata', ccdata)
# np.save('minimal_mc/mcdata', mcdata)
# np.save('minimal_sqp/sqpdata', sqpdata)
# np.save('minimal_sqn/sqndata', sqndata)
break
if sm.numSamples < int(TestCaseNum) and termin < float(CoverageStop):
continue
else:
break
print("statistics: \n")
nctoe.displayCoverage()
cctoe.displayCoverage()
mctoe.displayCoverage()
sqtoe.displayCoverage1()
sqtoe.displayCoverage2()
sm.displaySamples()
sm.displaySuccessRate()
