def test_using_gurobi(point, n, net_path):
method = lambda x: np.argsort(x)[-2]
gurobi_ptr = partial(GurobiSingleLayer, polyhedron_max_dim=1, use_relu=True, add_alpha_constraint=True,
use_counter_example=True)
idx_max, other_idx = get_out_idx(point, n, net_path, method)
print(idx_max, other_idx)
res, queries_stats, alpha_history = adversarial_query(point, 0.01, idx_max, other_idx, net_path,
gurobi_ptr, n)
assert res
def test_multilayer_large_n():
# point = np.array([1.0] * 40)
point = points[-1]
net_path = multi_layer_paths[1]
n = 12
gurobi_ptr = partial(GurobiMultiLayer,
polyhedron_max_dim=1,
use_relu=True,
add_alpha_constraint=True,
use_counter_example=True)
method = lambda x: np.argsort(x)[-2]
idx_max, other_idx = get_out_idx(point, n, net_path, method)
res, _, _ = adversarial_query(point, 0.01, idx_max, other_idx, net_path,
gurobi_ptr, n)
assert res
return
def test_specific_multilayer():
point = points[3]
net_path = multi_layer_paths[0]
n = 2
print(point)
print(net_path)
print("n=", n)
gurobi_ptr = partial(GurobiMultiLayer,
polyhedron_max_dim=1,
use_relu=True,
add_alpha_constraint=True,
use_counter_example=True,
debug=True,
max_steps=1)
method = lambda x: np.argsort(x)[-2]
idx_max, other_idx = get_out_idx(point, n, net_path, method)
res, _, _ = adversarial_query(point, 0.01, idx_max, other_idx, net_path,
gurobi_ptr, n)
assert res
def run_experiment(in_tensor, radius, idx_max, other_idx, h5_file, gurobi_ptr,
n_iterations, steps):
queries_stats = {}
start = timer()
try:
res, queries_stats, alpha_history = adversarial_query(
in_tensor, radius, idx_max, other_idx, h5_file, gurobi_ptr,
n_iterations, steps)
except ValueError as e:
# row_result = {'point': in_tensor, 'error': e, 'error_traceback': traceback.format_exc(), 'result' : False}
res = False
except TimeoutError as e:
res = False
queries_stats['FFNN_Timeout'] = True
except AssertionError as e:
res = False
end = timer()
if queries_stats is not None:
queries_stats['total_time'] = end - start
return {'time': end - start, 'result': res, 'stats': queries_stats}
def run_exp_signle_time(points, radius, h5_file, t, only_rns=False, pbar=None, save_results=False):
our_raw, rns_raw = [], []
for j, point in enumerate(points):
idx_max, other_idx = get_out_idx(point, t, h5_file, lambda x: np.argsort(x)[-2])
# idx_max, other_idx = None, None
rnsverify_time = -1
rnsverify_time = rns_verify_query(h5_file, point, idx_max, other_idx, t, radius)
our_time = -1
if not only_rns:
gurobi_ptr = partial(GurobiMultiLayer, use_relu=True, add_alpha_constraint=True,
use_counter_example=True)
try:
start = timer()
res, _, _ = adversarial_query(point, radius, idx_max, other_idx, h5_file, gurobi_ptr, t)
our_time = timer() - start
except ValueError:
res = False
our_time = -1
assert res
if pbar:
pbar.update(1)
our_raw.append(our_time)
rns_raw.append(rnsverify_time)
print('time: {}, point: {} our: {}, rns: {}'.format(t, j, our_time, rnsverify_time))
if save_results:
exp_name = 'verification time as a function of iterations, one rnn cell over {} points, time: {}'.format(
len(points), t)
file_name = "rns_{}time{}_{}.pkl".format('' if only_rns else 'ours_', t, time.strftime("%Y%m%d-%H%M%S"))
pickle_path = os.path.join(PICKLE_DIR, file_name)
print("#" * 100)
print(" " * 20 + "PICKLE PATH: {}".format(pickle_path))
print("#" * 100)
pickle.dump({'our_raw': our_raw, 'rns_raw': rns_raw, 'exp_name': exp_name}, open(pickle_path, "wb"))
return our_raw, rns_raw
def test_temp():
# Note that we use use_relu = False
import tempfile
import tensorflow.keras as k
pass_counter = 0
total_tests = 100
rnn_dim = 4
# for _ in range(total_tests):
with tempfile.NamedTemporaryFile(suffix=".h5") as fp:
net_path = fp.name
model = k.Sequential()
# model.add(k.layers.SimpleRNN(2, input_shape=(None, 1), activation='relu', return_sequences=True))
model.add(k.layers.SimpleRNN(rnn_dim, input_shape=(None, 1), activation='relu', return_sequences=False))
model.add(k.layers.Dense(2, activation='relu'))
w_h = np.random.uniform(-0.5, 0.5, (rnn_dim, rnn_dim))
w_in = np.random.random(rnn_dim)[None, :]
b = np.random.random(rnn_dim)
# w_h = np.array([[0, 1.0], [1., 0.]])
# model.layers[0].set_weights([np.array([1.0, 1.0])[None, :], w_h, np.array([0., 0.])])
model.layers[0].set_weights([w_in, w_h, b])
w_in_1 = np.random.random((rnn_dim, 2))
# model.layers[1].set_weights([np.array([[2.0, 0], [0, 1.0]]), np.array([0., 0.])])
model.layers[1].set_weights([w_in_1, np.array([0., 0.])])
model.save(net_path)
point = np.array([1.0])
# net_path = './FMCAD_EXP/models/model_20classes_rnn8_fc32_fc32_fc32_0050.ckpt'
n = 3
method = lambda x: np.argsort(x)[-2]
idx_max, other_idx = get_out_idx(point, n, net_path, method)
gurobi_ptr = partial(GurobiMultiLayer, polyhedron_max_dim=1, use_relu=True, add_alpha_constraint=True,
use_counter_example=True)
res, _, _ = adversarial_query(point, 0.01, idx_max, other_idx, net_path, gurobi_ptr, n)
pass_counter += res