def yolo_non_max_suppression(scores, boxes, classes, max_boxes = 10, iou_threshold = 0.5):
"""
Applies Non-max suppression (NMS) to set of boxes
Arguments:
scores -- tensor of shape (None,), output of yolo_filter_boxes()
boxes -- tensor of shape (None, 4), output of yolo_filter_boxes() that have been scaled to the image size (see later)
classes -- tensor of shape (None,), output of yolo_filter_boxes()
max_boxes -- integer, maximum number of predicted boxes you'd like
iou_threshold -- real value, "intersection over union" threshold used for NMS filtering
Returns:
scores -- tensor of shape (, None), predicted score for each box
boxes -- tensor of shape (4, None), predicted box coordinates
classes -- tensor of shape (, None), predicted class for each box
"""
max_boxes_tensor = K.variable(max_boxes, dtype='int32') # tensor to be used in tf.image.non_max_suppression()
K.get_session().run(tf.variables_initializer([max_boxes_tensor])) # initialize variable max_boxes_tensor
# Use tf.image.non_max_suppression() to get the list of indices corresponding to boxes you keep
nms_indices = tf.image.non_max_suppression(boxes,scores,max_boxes_tensor,iou_threshold=iou_threshold)
# Use K.gather() to select only nms_indices from scores, boxes and classes
scores = K.gather(scores,nms_indices)
boxes = K.gather(boxes,nms_indices)
classes = K.gather(classes,nms_indices)
return scores, boxes, classes
def yolo_eval(yolo_outputs,
image_shape,
max_boxes=10,
score_threshold=.6,
iou_threshold=.5):
"""Evaluate YOLO model on given input batch and return filtered boxes."""
box_xy, box_wh, box_confidence, box_class_probs = yolo_outputs
boxes = yolo_boxes_to_corners(box_xy, box_wh)
boxes, scores, classes = yolo_filter_boxes(
boxes, box_confidence, box_class_probs, threshold=score_threshold)
# Scale boxes back to original image shape.
height = image_shape[0]
width = image_shape[1]
image_dims = K.stack([height, width, height, width])
image_dims = K.reshape(image_dims, [1, 4])
boxes = boxes * image_dims
# TODO: Something must be done about this ugly hack!
max_boxes_tensor = K.variable(max_boxes, dtype='int32')
K.get_session().run(tf.variables_initializer([max_boxes_tensor]))
nms_index = tf.image.non_max_suppression(
boxes, scores, max_boxes_tensor, iou_threshold=iou_threshold)
boxes = K.gather(boxes, nms_index)
scores = K.gather(scores, nms_index)
classes = K.gather(classes, nms_index)
return boxes, scores, classes
def start_session_get_args_and_model(intra_ops, inter_ops, semantics_json, weights_hd5=None, tensor_type=None):
K.clear_session()
K.get_session().close()
cfg = K.tf.ConfigProto(intra_op_parallelism_threads=intra_ops, inter_op_parallelism_threads=inter_ops)
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
return args_and_model_from_semantics(semantics_json, weights_hd5, tensor_type)
def clear_session():
try:
K.clear_session()
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
except AttributeError as e:
print('Could not clear session. Maybe you are using Theano backend?')
def mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.05):
y_pred_ = tf.to_int32(y_pred > t)
score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)
K.get_session().run(tf.local_variables_initializer())
with tf.control_dependencies([up_opt]):
score = tf.identity(score)
prec.append(score)
return K.mean(K.stack(prec), axis=0)
def VGG_16(weights_path=None):
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=(3, 224, 224)))
model.add(Convolution2D(64, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation="relu"))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation="relu"))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation="relu"))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation="relu"))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation="relu"))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation="relu"))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(Flatten())
model.add(Dense(4096, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(4096, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(1000, activation="softmax"))
if weights_path:
model.load_weights(weights_path)
ops = []
for layer in model.layers:
if layer.__class__.__name__ in ["Convolution1D", "Convolution2D"]:
original_w = K.get_value(layer.W)
converted_w = convert_kernel(original_w)
ops.append(tf.assign(layer.W, converted_w).op)
K.get_session().run(ops)
return model
def set_keras_backend(backend):
if K.backend() != backend:
os.environ["KERAS_BACKEND"] = backend
importlib.reload(K)
assert K.backend() == backend
if backend == "tensorflow":
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
K.clear_session()
def __init__(self, rate, noise_shape=None, seed=None, dropoutEnabled = False, **kwargs):
super(DropoutEverytime, self).__init__(**kwargs)
self.rate = min(1., max(0., rate))
self.noise_shape = noise_shape
self.seed = seed
self.supports_masking = True
#self.dropoutEnabled = tf.constant(int(dropoutEnabled))
self.dropoutEnabled = self._getDropoutEnabled()
assignment = tf.assign(self.dropoutEnabled, [int(dropoutEnabled)])
tf.global_variables_initializer().run(session=K.get_session())
K.get_session().run(assignment)
def run(Data, Model, path):
sess = K.get_session()
K.set_learning_phase(False)
data, model = Data(), Model(path)
if Data == MNIST:
attack = CarliniL2(sess, model, batch_size=100, max_iterations=2000,
binary_search_steps=5, initial_const=1., learning_rate=1e-1,
targeted=False)
else:
attack = CarliniL2(sess, model, batch_size=100, max_iterations=200,
binary_search_steps=3, initial_const=.01, learning_rate=1e-2,
targeted=True, confidence=2)
now = time.time()
for name,X,y in [["test",data.test_data, data.test_labels]]:
print("OKAY",name)
for k in range(0,len(y),5000):
#if os.path.exists("tmp/"+path.split("/")[1]+"."+name+".adv.X."+str(k)+".npy"):
# print('skip',k)
# continue
now = time.time()
adv = attack.attack(X[k:k+100], y[k:k+100])
#print('time',time.time()-now)
#print('accuracy',np.mean(np.argmax(model.model.predict(adv),axis=1)==np.argmax(y[k:k+5000],axis=1)))
#print('mean distortion',np.mean(np.sum((adv-X[k:k+5000])**2,axis=(1,2,3))**.5))
np.save("/tmp/"+path.split("/")[1]+"."+name+".adv.X."+str(k),adv)
def run_test(Data, Model, path):
sess = K.get_session()
K.set_learning_phase(False)
data = Data()
model = Model(path)
N = 1000
X = data.train_data[np.random.choice(np.arange(len(data.train_data)), N, replace=False)].reshape((N,-1))
#Y = data.train_data[np.random.choice(np.arange(len(data.train_data)), N, replace=False)].reshape((N,-1))
Y = data.test_data[np.random.choice(np.arange(len(data.test_data)), N, replace=False)].reshape((N,-1))
#attack = FGS(sess, model, N, .275)
attack = CarliniL2(sess, model, batch_size=100, binary_search_steps=2, initial_const=1, targeted=False, max_iterations=500)
idx = np.random.choice(np.arange(len(data.test_data)), N, replace=False)
Y = attack.attack(data.test_data[idx], data.test_labels[idx]).reshape((N,-1))
iterations = 1000
sigma2 = 100
mmd2u, mmd2u_null, p_value = kernel_two_sample_test(X, Y, iterations=iterations,
kernel_function='rbf',
gamma=1.0/sigma2,
verbose=True)
def __init__(self, **kwargs):
self.__dict__.update(self._defaults) # set up default values
self.__dict__.update(kwargs) # and update with user overrides
self.class_names = self._get_class()
self.anchors = self._get_anchors()
self.sess = K.get_session()
self.boxes, self.scores, self.classes = self.generate()
def main():
models = build_or_load()
style_layer = models[0].get_layer('style')
print('Creating input')
style_in = tf.placeholder(tf.float32, shape=(NUM_STYLES, NUM_STYLES))
style_out = style_layer(style_in)
# All possible styles
all_styles = np.identity(NUM_STYLES)
with K.get_session() as sess:
embedding = sess.run(style_out, { style_in: all_styles })
print('Writing to out directory')
np.savetxt(os.path.join(OUT_DIR, 'style_embedding_vec.tsv'), embedding, delimiter='\t')
labels = [[g] * len(styles[i]) for i, g in enumerate(genre)]
# Flatten
labels = [y for x in labels for y in x]
# Retreive specific artists
styles_labels = [y for x in styles for y in x]
styles_labels = np.reshape(styles_labels, [-1, 1])
labels = np.reshape(labels, [-1, 1])
labels = np.hstack([labels, styles_labels])
# Add metadata header
header = ['Genre', 'Artist']
labels = np.vstack([header, labels])
np.savetxt(os.path.join(OUT_DIR, 'style_embedding_labels.tsv'), labels, delimiter='\t', fmt='%s')
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
if (layer.name=='block1_conv1'):
print(np.shape(layer.input[:, :, :, 0]))
abs_img = tf.expand_dims(layer.input[:, :, :, 0], axis=-1)
phase_img = tf.expand_dims(layer.input[:, :, :, 1], axis=-1)
tf.summary.image('{}_in_abs'.format(layer.name),
abs_img, max_outputs=self.config.max_image_summary)
tf.summary.image('{}_in_phase'.format(layer.name), phase_img, max_outputs=self.config.max_image_summary)
if (layer.name=='depth_output'):
tf.summary.image('{}_estimated_dem'.format(layer.name),
layer.output, max_outputs=self.config.max_image_summary)
tf.summary.image('{}_gt_dem'.format(layer.name),
self.Y_test[0], max_outputs=self.config.max_image_summary)
self.merged = tf.summary.merge_all()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
def __init__(self, graph=None, using_keras=False):
self.graph = graph or tf.Graph()
self.sess = tf.Session(graph=self.graph)
if using_keras:
self.keras_prev_sess = K.get_session()
else:
self.keras_prev_sess = None
def _run_initsync(self):
# tparams = [list(chain(*tp)) for tp in self._tower_params]
tparams = self._tower_params
# Check to prevent from unnecessarily re-initializing and
# synchronizing, i.e. when the model loads the weights.
for v in chain.from_iterable(tparams):
if getattr(v, '_keras_initialized', False):
return
KB.manual_variable_initialization(True)
sess = KB.get_session()
KB.manual_variable_initialization(False)
# glob_variables = tf.global_variables()
# sess.run(tf.variables_initializer(glob_variables))
# Initialize on GPU0 and sync to other GPUs
init_op = tf.variables_initializer(tparams[0])
# init_op = tf.variables_initializer(self._tower_params[0])
# init_op = tf.variables_initializer(self.trainable_weights)
sess.run(init_op)
# Important if using model_creator. Not necessary of model instance is
# reused in which case the model layers are shared between slices
# and are automatically sync'd.
sync_op = all_sync_params(tparams, self._gdev_list,
usenccl=self._usenccl)
sess.run(sync_op)
for v in chain.from_iterable(tparams):
v._keras_initialized = True
def predict(self):
weights_name = '{}/phase_2_best.h5'.format(self.root_dir)
self.model_body.load_weights(weights_name)
yolo_outputs = yolo_head(self.model_body.output, self.anchors, len(self.class_names))
input_image_shape = K.placeholder(shape=(2,))
boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=0.5, iou_threshold=0)
sess = K.get_session()
results = []
for i_path in self.images:
im = Image.open(i_path)
image_data = np.array(im.resize((416, 416), Image.BICUBIC), dtype=np.float) / 255.
if len(image_data.shape) >= 3:
image_data = np.expand_dims(image_data, 0)
feed_dict = {self.model_body.input: image_data,input_image_shape: [im.size[1], im.size[0]], K.learning_phase(): 0}
out_boxes, out_scores, out_classes = sess.run([boxes, scores, classes],feed_dict=feed_dict)
for i, c in list(enumerate(out_classes)):
box_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{}'.format(box_class)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(im.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(im.size[0], np.floor(right + 0.5).astype('int32'))
results.append((i_path,box_class,score,top, left, bottom, right))
else:
logging.warning("skipping {} contains less than 3 channels".format(i_path))
return results
def test_attention_layer_by_run(self):
input_shape = (7, 8, 5, 6, 9)
# record, document,section,sentence,word
input_feature_dims = (20, 10, 50, 60, 30)
# document, section, sentence, word
attention_output_dims = (45, 35, 25, 65)
# document, section, sentence, word
attention_weight_vector_dims = (82, 72, 62, 52)
# embedding
embedding_rows = 200
embedding_dim = 50
# classifier
initial_embedding = np.random.random((embedding_rows, embedding_dim))
inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims)
hierarchical_attention = HierarchicalAttention(input_feature_dims[0], attention_output_dims, attention_weight_vector_dims, embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder=False)
output = hierarchical_attention(inputs)
total = 2
output_dim = 10
timesteps = input_shape[0]
x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim)
# build feed_dic
feed_dict = {}
for i in range(len(inputs)):
feed_dict[inputs[i]] = x_train[i]
feed_dict[K.learning_phase()] = 1
# tf.initialize_all_variables()
# y_out is fine 2,7, 110
y_out = K.get_session().run(output, feed_dict=feed_dict)
self.assertEquals(y_out.shape , (2, 7, 110), "y_out")
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
for weight in layer.weights:
mapped_weight_name = weight.name.replace(':', '_')
tf.summary.histogram(mapped_weight_name, weight)
if self.write_grads:
grads = self.model.optimizer.get_gradients(self.model.total_loss,
weight)
def is_indexed_slices(grad):
return type(grad).__name__ == 'IndexedSlices'
grads = [
grad.values if is_indexed_slices(grad) else grad
for grad in grads]
for grad in grads:
tf.summary.histogram('{}_grad'.format(mapped_weight_name), grad)
if hasattr(layer, 'output'):
tf.summary.histogram('{}_out'.format(layer.name),
layer.output)
self.merged = tf.summary.merge_all()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
def submit(model,args,test_dir,size=[100,100]):
import scipy.misc
files = os.listdir(test_dir)
group_size = int(len(files) / args.groups)
with open("results.txt","w") as result_file:
for i in range(args.groups):
print("Running on group",i)
test_im = []
filenames = []
for j in range(group_size * i,min(len(files),group_size*(i+1))):
filepath = os.path.join(test_dir, files[j])
image = scipy.misc.imread(filepath)
image = scipy.misc.imresize(image, (size[0], size[1],3))
test_im.append(image)
filenames.append(files[j])
test_im = np.array(test_im)
test_im = test_im.reshape(-1, 100, 100, 3).astype('float32') / 255.
print("predicting on ",test_im.shape[0]," images")
y_pred = model.predict(test_im)
top_values, top_indices = K.get_session().run(tf.nn.top_k(y_pred, k=5,sorted=True))
print("writing to file")
for l in range(len(filenames)):
fn = filenames[l]
vals = " ".join(map(str,top_indices[l]))
result_file.write("test/"+fn + " "+vals+"\n")
def get_config(self):
config = {'rate': self.rate,
'noise_shape': self.noise_shape,
'seed': self.seed,
'dropoutEnabled': self.dropoutEnabled.eval(session=K.get_session())
}
base_config = super(DropoutEverytime, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
for weight in layer.weights:
if hasattr(tf, 'histogram_summary'):
tf.histogram_summary(weight.name, weight)
else:
tf.summary.histogram(weight.name, weight)
if self.write_images:
w_img = tf.squeeze(weight)
shape = w_img.get_shape()
if len(shape) > 1 and shape[0] > shape[1]:
w_img = tf.transpose(w_img)
if len(shape) == 1:
w_img = tf.expand_dims(w_img, 0)
w_img = tf.expand_dims(tf.expand_dims(w_img, 0), -1)
if hasattr(tf, 'image_summary'):
tf.image_summary(weight.name, w_img)
else:
tf.summary.image(weight.name, w_img)
if hasattr(layer, 'output'):
if hasattr(tf, 'histogram_summary'):
tf.histogram_summary('{}_out'.format(layer.name),
layer.output)
else:
tf.summary.histogram('{}_out'.format(layer.name),
layer.output)
if hasattr(tf, 'merge_all_summaries'):
self.merged = tf.merge_all_summaries()
else:
self.merged = tf.summary.merge_all()
if self.write_graph:
if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'):
self.writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
elif parse_version(tf.__version__) >= parse_version('0.8.0'):
self.writer = tf.train.SummaryWriter(self.log_dir,
self.sess.graph)
else:
self.writer = tf.train.SummaryWriter(self.log_dir,
self.sess.graph_def)
else:
if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'):
self.writer = tf.summary.FileWriter(self.log_dir)
else:
self.writer = tf.train.SummaryWriter(self.log_dir)
def update_task_metrics(self, X, y, batch_size):
# Reset metric accumulators
n_batch = len(X) // batch_size
sess = K.get_session()
sess.run(self._reset_task_metrics_op)
for i in range(n_batch):
xi, yi, sample_weights = self.model._standardize_user_data(X[i * batch_size:(i+1) * batch_size], y[i*batch_size:(i+1)*batch_size], batch_size=batch_size)
sess.run(self._update_task_metrics_op, {self.model.input:xi[0], self.model.targets[0]:yi[0], self.model.sample_weights[0]:sample_weights[0]})
def broadcast_global_variables(root_rank):
"""Broadcasts all global variables from root rank to all other processes.
Arguments:
root_rank: Rank of the process from which global variables will be broadcasted
to all other processes.
"""
bcast_op = hvd.broadcast_global_variables(root_rank)
return K.get_session().run(bcast_op)
def preprocess_image(im, width, height, train=True):
size = min(im.shape[:2])
im = tf.constant(im)
if train:
im = tf.random_crop(im, (size, size, 3))
im = tf.image.resize_images(im, (width, height))
else:
im = tf.image.resize_image_with_crop_or_pad(im, height, width)
im = K.get_session().run(im)
return preprocess_input(im)
def reset_weights(model):
session = K.get_session()
for layer in model.layers:
if isinstance(layer, Dense):
old = layer.get_weights()
layer.W.initializer.run(session=session)
layer.b.initializer.run(session=session)
print(np.array_equal(old, layer.get_weights())," after initializer run")
else:
print(layer, "not reinitialized")
def __init__(self):
self.model_path = 'model_data/yolo.h5' # model path or trained weights path
self.anchors_path = 'model_data/yolo_anchors.txt'
self.classes_path = 'model_data/coco_classes.txt'
self.score = 0.3
self.iou = 0.45
self.class_names = self._get_class()
self.anchors = self._get_anchors()
self.sess = K.get_session()
self.model_image_size = (416, 416) # fixed size or (None, None), hw
self.boxes, self.scores, self.classes = self.generate()
def run_pca(Data, num_components=10, invert=False):
data = Data()
sess = K.get_session()
K.set_learning_phase(False)
shape = (-1, 784)
pca = sklearn.decomposition.PCA(n_components=num_components)
pca.fit(data.train_data.reshape(shape)) # [:10000]
if invert:
model = MNISTModel("models/mnist-pca-cnn-top-"+str(num_components))
else:
model = make_model(num_components)
model.load_weights("models/mnist-pca-top-"+str(num_components))
model = Wrap(model,pca)
tf_mean = tf.constant(pca.mean_,dtype=tf.float32)
tf_components = tf.constant(pca.components_.T,dtype=tf.float32)
def new_predict(xs):
# map to PCA space
xs = tf.reshape(xs,(-1,784))
xs -= tf_mean
xs = tf.matmul(xs, tf_components)
# map back
xs = tf.matmul(xs, tf.transpose(tf_components))
xs += tf_mean
xs = tf.reshape(xs, (-1, 28, 28, 1))
return model.model(xs)
if invert:
model.predict = new_predict
attack = CarliniL2(sess, model, batch_size=100, max_iterations=3000,
binary_search_steps=6, targeted=False,
initial_const=1)
N = 100
test_adv = attack.attack(data.test_data[:N], data.test_labels[:N])
print('accuracy',np.mean(np.argmax(sess.run(model.predict(tf.constant(data.test_data,dtype=np.float32))),axis=1)==np.argmax(data.test_labels,axis=1)))
print(list(test_adv[0].flatten()))
print('dist',np.mean(np.sum((test_adv-data.test_data[:N])**2,axis=(1,2,3))**.5))
it = np.argmax(sess.run(model.predict(tf.constant(test_adv))),axis=1)
print('success',np.mean(it==np.argmax(data.test_labels,axis=1)[:N]))
def compare_baseline():
data = MNIST()
model = MNISTModel("models/mnist")
sess = K.get_session()
attack = CarliniL2(sess, model, batch_size=100, max_iterations=3000,
binary_search_steps=4, targeted=False,
initial_const=10)
N = 100
test_adv = attack.attack(data.test_data[:N], data.test_labels[:N])
print('dist',np.mean(np.sum((test_adv-data.test_data[:N])**2,axis=(1,2,3))**.5))
def _initialize_variables():
"""Utility to initialize uninitialized variables on the fly.
"""
variables = tf.local_variables()
uninitialized_variables = []
for v in variables:
if not hasattr(v, '_keras_initialized') or not v._keras_initialized:
uninitialized_variables.append(v)
v._keras_initialized = True
if uninitialized_variables:
sess = K.get_session()
sess.run(tf.variables_initializer(uninitialized_variables))
def __init__(self):
self.model_path = 'model_data/yolo.h5'
self.anchors_path = 'model_data/yolo_anchors.txt'
self.classes_path = 'model_data/coco_classes.txt'
self.score = 0.5
self.iou = 0.5
self.class_names = self._get_class()
self.anchors = self._get_anchors()
self.sess = K.get_session()
self.model_image_size = (416, 416) # fixed size or (None, None)
self.is_fixed_size = self.model_image_size != (None, None)
self.boxes, self.scores, self.classes = self.generate()
def main(args):
# If output_model path is relative and in cwd, make it absolute from root
output_model = FLAGS.output_model
if str(Path(output_model).parent) == '.':
output_model = str((Path.cwd() / output_model))
output_fld = Path(output_model).parent
output_model_name = Path(output_model).name
output_model_stem = Path(output_model).stem
output_model_pbtxt_name = output_model_stem + '.pbtxt'
# Create output directory if it does not exist
# Path(output_model).parent.mkdir(parents=True)
if FLAGS.channels_first:
K.set_image_data_format('channels_first')
else:
K.set_image_data_format('channels_last')
model = load_model(FLAGS.input_model, FLAGS.input_model_json)
# TODO(amirabdi): Support networks with multiple inputs
orig_output_node_names = [node.op.name for node in model.outputs]
if FLAGS.output_nodes_prefix:
num_output = len(orig_output_node_names)
pred = [None] * num_output
converted_output_node_names = [None] * num_output
# Create dummy tf nodes to rename output
for i in range(num_output):
converted_output_node_names[i] = '{}{}'.format(
FLAGS.output_nodes_prefix, i)
pred[i] = tf.identity(model.outputs[i],
name=converted_output_node_names[i])
else:
converted_output_node_names = orig_output_node_names
logging.info('Converted output node names are: %s',
str(converted_output_node_names))
sess = K.get_session()
if FLAGS.output_meta_ckpt:
saver = tf.train.Saver()
saver.save(sess, str(output_fld / output_model_stem))
if FLAGS.save_graph_def:
tf.train.write_graph(sess.graph.as_graph_def(),
str(output_fld),
output_model_pbtxt_name,
as_text=True)
logging.info('Saved the graph definition in ascii format at %s',
str(Path(output_fld) / output_model_pbtxt_name))
if FLAGS.quantize:
from tensorflow.tools.graph_transforms import TransformGraph
transforms = ["quantize_weights", "quantize_nodes"]
transformed_graph_def = TransformGraph(sess.graph.as_graph_def(), [],
converted_output_node_names,
transforms)
constant_graph = graph_util.convert_variables_to_constants(
sess, transformed_graph_def, converted_output_node_names)
else:
constant_graph = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), converted_output_node_names)
graph_io.write_graph(constant_graph,
str(output_fld),
output_model_name,
as_text=False)
logging.info('Saved the freezed graph at %s',
str(Path(output_fld) / output_model_name))
def build_model(self, predict, custom_batch_size=None):
conf = self.conf
model_conf = conf['model']
rnn_size = model_conf['rnn_size']
rnn_type = model_conf['rnn_type']
regularization = model_conf['regularization']
dense_regularization = model_conf['dense_regularization']
use_batch_norm = False
if 'use_batch_norm' in model_conf:
use_batch_norm = model_conf['use_batch_norm']
dropout_prob = model_conf['dropout_prob']
length = model_conf['length']
pred_length = model_conf['pred_length']
# skip = model_conf['skip']
stateful = model_conf['stateful']
return_sequences = model_conf['return_sequences']
# model_conf['output_activation']
output_activation = conf['data']['target'].activation
use_signals = conf['paths']['use_signals']
num_signals = sum([sig.num_channels for sig in use_signals])
num_conv_filters = model_conf['num_conv_filters']
# num_conv_layers = model_conf['num_conv_layers']
size_conv_filters = model_conf['size_conv_filters']
pool_size = model_conf['pool_size']
dense_size = model_conf['dense_size']
batch_size = self.conf['training']['batch_size']
if predict:
batch_size = self.conf['model']['pred_batch_size']
# so we can predict with one time point at a time!
if return_sequences:
length = pred_length
else:
length = 1
if custom_batch_size is not None:
batch_size = custom_batch_size
if rnn_type == 'LSTM':
rnn_model = LSTM
elif rnn_type == 'SimpleRNN':
rnn_model = SimpleRNN
else:
print('Unkown Model Type, exiting.')
exit(1)
batch_input_shape = (batch_size, length, num_signals)
# batch_shape_non_temporal = (batch_size, num_signals)
indices_0d, indices_1d, num_0D, num_1D = self.get_0D_1D_indices()
def slicer(x, indices):
return x[:, indices]
def slicer_output_shape(input_shape, indices):
shape_curr = list(input_shape)
assert len(shape_curr) == 2 # only valid for 3D tensors
shape_curr[-1] = len(indices)
return tuple(shape_curr)
pre_rnn_input = Input(shape=(num_signals, ))
if num_1D > 0:
pre_rnn_1D = Lambda(
lambda x: x[:, len(indices_0d):],
output_shape=(len(indices_1d), ))(pre_rnn_input)
pre_rnn_0D = Lambda(
lambda x: x[:, :len(indices_0d)],
output_shape=(len(indices_0d), ))(pre_rnn_input)
# slicer(x,indices_0d),lambda s:
# slicer_output_shape(s,indices_0d))(pre_rnn_input)
pre_rnn_1D = Reshape(
(num_1D, len(indices_1d) // num_1D))(pre_rnn_1D)
pre_rnn_1D = Permute((2, 1))(pre_rnn_1D)
for i in range(model_conf['num_conv_layers']):
div_fac = 2**i
'''The first conv layer learns `num_conv_filters//div_fac`
filters (aka kernels), each of size
`(size_conv_filters, num1D)`. Its output will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac), i.e., for
each position in the input spatial series (direction along
radius), the activation of each filter at that position.
'''
'''For i=1 first conv layer would get:
(None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size-size_conv_filters + 1,num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(num_conv_filters // div_fac,
size_conv_filters,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''The output of the second conv layer will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac),
i.e., for each position in the input spatial series
(direction along radius), the activation of each filter
at that position.
For i=1, the second layer would output
(None, (len(indices_1d)//num_1D - size_conv_filters + 1)/
pool_size-size_conv_filters + 1,num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(num_conv_filters // div_fac,
1,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size, num_conv_filters//div_fac)
For i=1, the pooling layer would output:
(None,((len(indices_1d)//num_1D- size_conv_filters
+ 1)/pool_size-size_conv_filters+1)/pool_size,
num_conv_filters//div_fac)
'''
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
pre_rnn_1D = Flatten()(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size // 4,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn = Concatenate()([pre_rnn_0D, pre_rnn_1D])
else:
pre_rnn = pre_rnn_input
if model_conf['rnn_layers'] == 0 or (
'extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input']):
pre_rnn = Dense(
dense_size,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size // 2,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size // 4,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn_model = Model(inputs=pre_rnn_input, outputs=pre_rnn)
# pre_rnn_model.summary()
x_input = Input(batch_shape=batch_input_shape)
x_in = TimeDistributed(pre_rnn_model)(x_input)
for _ in range(model_conf['rnn_layers']):
x_in = rnn_model(
rnn_size,
return_sequences=return_sequences,
# batch_input_shape=batch_input_shape,
stateful=stateful,
kernel_regularizer=l2(regularization),
recurrent_regularizer=l2(regularization),
bias_regularizer=l2(regularization),
dropout=dropout_prob,
recurrent_dropout=dropout_prob)(x_in)
x_in = Dropout(dropout_prob)(x_in)
if return_sequences:
# x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in)
x_out = TimeDistributed(Dense(1,
activation=output_activation))(x_in)
else:
x_out = Dense(1, activation=output_activation)(x_in)
model = Model(inputs=x_input, outputs=x_out)
# bug with tensorflow/Keras
# TODO(KGF): what is this bug? this is the only direct "tensorflow"
# import outside of mpi_runner.py and runner.py
if (conf['model']['backend'] == 'tf'
or conf['model']['backend'] == 'tensorflow'):
first_time = "tensorflow" not in sys.modules
import tensorflow as tf
if first_time:
K.get_session().run(tf.global_variables_initializer())
model.reset_states()
return model
def auc(y_true, y_pred):
""" Tensor-based ROC-AUC metric for use as loss function """
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
return auc
def write_graph(self, path):
graph = K.get_session().graph
writer = tf.compat.v1.summary.FileWriter(logdir=path, graph=graph)
def limit_mem():
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
if args.game == "Grid": env = GridEnv() else: env = gym_env(args.game) #print(env.observation_space.n) modelOps = DqnOps(env.action_count) modelOps.dueling_network = args.dueling_dqn modelOps.INPUT_SIZE = env.observation_space.n modelOps.LEARNING_RATE = 0.2 q_model = TabularQModel(modelOps) summary_writer = tf.summary.FileWriter(args.logdir, K.get_session().graph) if not args.logdir is None else None agentOps = DqnAgentOps() agentOps.double_dqn = args.double_dqn replay_buffer = NStepBuffer(1, args.nstep) agent = DqnAgent(env.action_space, q_model, replay_buffer, None, agentOps, summary_writer) egreedyOps = EGreedyOps() egreedyOps.REPLAY_START_SIZE = 1 egreedyOps.FINAL_EXPLORATION_FRAME = 10000 egreedyAgent = EGreedyAgent(env.action_space, egreedyOps, agent) runner = Runner(env, egreedyAgent, None, 1) runner.listen(replay_buffer, None) runner.listen(agent, None)
TuneResult["LearnRate"] = []
TuneResult["TrainAUC"] = []
TuneResult["TrainAccuracy"] = []
TuneResult["ValidAUC"] = []
TuneResult["ValidAccuracy"] = []
##
## The better model
TheBetterModel = "Empty"
print("Prepare!")
################################################################################
##
## Tune loop
for p in Parameter:
##
## Reproducible session
backend.get_session()
numpy.random.seed(2018)
tensorflow.set_random_seed(2018)
##
## Create model
from BaseOnModel.UseImageVariable.UseKeras.UseHoldOut.Model import Model
model = Model.CustomResNet(ImageSize = Resize + (3,), VariableSize = VariableSize)
##
## Optimizer
if(p["Optimizer"]=="Adadelta"):
TheOptimizer = keras.optimizers.Adadelta
if(p["Optimizer"]=="Adam"):
TheOptimizer = keras.optimizers.Adam
if(p["Optimizer"]=="SGD"):
TheOptimizer = keras.optimizers.SGD
##
def limit_mem():
""" Limits memory use on GPUs """
K.get_session().close()
config = K.tf.ConfigProto()
config.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=config))
def main():
dataset = sys.argv[1]
if len(sys.argv) > 2:
start_from = int(sys.argv[2])
else:
start_from = 0
black_box = 'DNN'
neigh_type = 'hrgp'
random_state = 0
ae_name = 'aae'
num_classes = 10
nbr_experiments = 200
if dataset not in ['mnist', 'cifar10', 'fashion']:
print('unknown dataset %s' % dataset)
return -1
if black_box not in ['RF', 'AB', 'DNN']:
print('unknown black box %s' % black_box)
return -1
if neigh_type not in ['rnd', 'gntp', 'hrgp']:
print('unknown neigh type %s' % neigh_type)
return -1
path = './'
path_models = path + 'models/'
path_results = path + 'results/rel/'
path_aemodels = path + 'aemodels/%s/%s/' % (dataset, ae_name)
path_expl = './expl/'
black_box_filename = path_models + '%s_%s' % (dataset, black_box)
results_filename = path_results + 'rel_%s_%s_%s.json' % (
dataset, black_box, neigh_type)
expl_filename = path_expl + 'alore_%s_%s_%s.json.gz' % (dataset, black_box,
neigh_type)
_, _, X_test, Y_test, use_rgb = get_dataset(dataset)
bb, transform = get_black_box(black_box,
black_box_filename,
use_rgb,
return_model=True)
bb_predict, bb_predict_proba = get_black_box(black_box, black_box_filename,
use_rgb)
ae = get_autoencoder(X_test, ae_name, dataset, path_aemodels)
ae.load_model()
class_name = 'class'
class_values = ['%s' % i for i in range(len(np.unique(Y_test)))]
explainer = ILOREM(bb_predict,
class_name,
class_values,
neigh_type=neigh_type,
use_prob=True,
size=1000,
ocr=0.1,
kernel_width=None,
kernel=None,
autoencoder=ae,
use_rgb=use_rgb,
valid_thr=0.5,
filter_crules=True,
random_state=random_state,
verbose=False,
alpha1=0.5,
alpha2=0.5,
metric=neuclidean,
ngen=10,
mutpb=0.2,
cxpb=0.5,
tournsize=3,
halloffame_ratio=0.1,
bb_predict_proba=bb_predict_proba)
input_tensor = bb.layers[0].input
bb_model = Model(inputs=input_tensor, outputs=bb.layers[-1].output)
target_tensor = bb_model(input_tensor)
de_list = ['grad*input', 'saliency', 'intgrad', 'elrp', 'occlusion']
errors = open(
path_results + 'errors_relevancy_%s_%s.csv' % (dataset, black_box),
'w')
with DeepExplain(session=K.get_session()) as de:
for i2e in range(nbr_experiments):
if i2e < start_from:
continue
jrow_rel_o = {
'i2e': i2e,
'dataset': dataset,
'black_box': black_box
}
img = X_test[i2e]
bbo = bb_predict(np.array([img]))
jrow_list = list()
try:
# Alore
exp = explainer.explain_instance(img,
num_samples=1000,
use_weights=True,
metric=neuclidean)
_, diff = exp.get_image_rule(features=None, samples=100)
relevancy = 1.0 - np.abs(diff - 127.5) / 127.5
for color in [0, 127, 255]:
jrow_rel = copy.deepcopy(jrow_rel_o)
jrow_rel['method'] = 'alore'
jrow_rel['color'] = color
jrow_rel = apply_relevancy(jrow_rel, img, bb_predict,
bbo[0], relevancy, color)
jrow_list.append(jrow_rel)
print(
datetime.datetime.now(),
'[%s/%s] %s %s %s %s - 25: %d, 50: %d, 75: %d' %
(i2e, nbr_experiments, dataset, black_box, 'alore',
color, jrow_rel['color%s_c25' % color],
jrow_rel['color%s_c50' % color],
jrow_rel['color%s_c75' % color]))
jrow_neigh = {
'i2e': i2e,
'dataset': dataset,
'black_box': black_box,
'expl': diff,
'rule': str(exp.rule)
}
json_str = (
'%s\n' %
json.dumps(jrow_neigh, cls=NumpyEncoder)).encode('utf-8')
with gzip.GzipFile(expl_filename, 'a') as fout:
fout.write(json_str)
# Deep Explain
xs = transform(np.array([img]))
ys = to_categorical(bbo, num_classes)
for det in de_list:
if det == 'shapley_sampling':
relevancy = de.explain(det,
target_tensor,
input_tensor,
xs,
ys=ys,
samples=10)[0]
else:
relevancy = de.explain(det,
target_tensor,
input_tensor,
xs,
ys=ys)[0]
relevancy = np.mean(relevancy, axis=2)
for color in [0, 127, 255]:
jrow_rel = copy.deepcopy(jrow_rel_o)
jrow_rel['method'] = det
jrow_rel['color'] = color
jrow_rel = apply_relevancy(jrow_rel, img, bb_predict,
bbo[0], relevancy, color)
jrow_list.append(jrow_rel)
print(
datetime.datetime.now(),
'[%s/%s] %s %s %s %s - 25: %d, 50: %d, 75: %d' %
(i2e, nbr_experiments, dataset, black_box, det,
color, jrow_rel['color%s_c25' % color],
jrow_rel['color%s_c50' % color],
jrow_rel['color%s_c75' % color]))
except Exception:
print('error instance to explain: %d' % i2e)
errors.write('%d\n' % i2e)
continue
results = open(results_filename, 'a')
for jrow in jrow_list:
results.write('%s\n' % json.dumps(jrow))
results.close()
errors.close()
#l Lad the model using Keras, standard way for saving netwrok's weights is HDF5 format.
model = load_model('model.model')
# Produce np array from image, using PIL (one of the thousand ways for loading an image)
img = Image.open('img2.png')
img.load()
data = np.asarray(img, dtype="int32")
print(np.argmax(model.predict(np.array([data]))[0]),' should be 4 BUT NOT NOW!')
# We need te function that incapsulate the gradient of the loss wrt the input.
# ! MOST IMPORTANT: the loss function is the main actor here. It defines what we want to search.
# In this case, we want the distance between the prediciton and the target label 4.
# Hence, we produce the loss written there.
session = K.get_session()
d_model_d_x = K.gradients( keras.losses.mean_squared_error(target, model.output), model.input)
x0 = data
conf = model.predict(np.array([x0]))[0]
# The attack may last forever?
# YES! But I tried with a black image and it converges.
# You should put here a fixed number of iterations...
while np.argmax(conf) != TARGET:
# Thank you Keras + Tensorflow!
# That [0][0] is just ugly, but it is needed to obtain the value as an array.
eval_grad = session.run(d_model_d_x, feed_dict={model.input:np.array([x0])} )[0][0]
# Compute the perturbation!
def reinitialize_weights():
session = K.get_session()
for layer in vae.layers:
if hasattr(layer, 'kernel_initializer'):
layer.kernel.initializer.run(session=session)
def estimate_fisher_diagonal(model,
X,
Y=None,
fisher_n=0,
len_weights=None,
return_gradients=False,
true_estimate=True,
nonzero_gradient_mean=True,
xs=None):
if xs == None:
xs = model.trainable_weights
if len_weights is None:
len_weights = len(K.batch_get_value(model.trainable_weights))
fisher_estimates = [
np.zeros_like(w)
for w in K.batch_get_value(model.trainable_weights)[0:len_weights]
]
if fisher_n is 0 or fisher_n > X.shape[0]:
fisher_n = X.shape[0]
true_length = len(X)
X = X[0:fisher_n]
gradients = []
sess = K.get_session()
if Y is None:
X = np.random.permutation(X)
#note for future: turns a model output vector (e.g. [0.3, 0.4, 0.2, 0.1]) into a one-hot (e.g. [0,1,0,0]) by keeping making a max-only copy, making a boolean vector by checking element equality with the elements of the original, and drawing from 0s where false and 1s when true.
label_tensor = tf.where(
tf.equal(tf.reduce_max(model.output, 1, keepdims=True),
model.output),
tf.constant(1, shape=(1, model.output.shape[1])),
tf.constant(0, shape=(1, model.output.shape[1])))
grads_tensor = K.gradients(
categorical_crossentropy(label_tensor, model.output), xs)
else:
Y = Y[0:fisher_n]
X, Y = shuffle(X, Y)
label = Y
y_placeholder = K.placeholder(dtype='float32', shape=label[0].shape)
grads_tensor = K.gradients(
categorical_crossentropy(y_placeholder, model.output), xs)
for i in tqdm(range(fisher_n), desc='Calculating gradients'):
feed_dict = {model.input: np.array([X[i]])}
if Y is not None:
feed_dict[y_placeholder] = Y[i]
gradient = sess.run(grads_tensor, feed_dict=feed_dict)
gradients.append(gradient)
if nonzero_gradient_mean:
gradient_means = []
for j in tqdm(range(len_weights), desc='Calculating gradient means'):
grad_sum = 0
for i in range(fisher_n):
grad_sum += gradients[i][j]
grad_sum /= fisher_n
gradient_means.append(grad_sum)
else:
gradient_means = []
for j in range(len_weights):
gradient_means.append(0)
for i in tqdm(range(fisher_n), desc='Estimating FIM diagonal'):
for j in range(len_weights):
fisher_estimates[j] += (gradients[i][j] -
gradient_means[j])**2 / fisher_n
fisher_estimates = [(true_length if true_estimate else 1) * x
for x in fisher_estimates]
if return_gradients:
return fisher_estimates, gradients
return fisher_estimates
for i in range(6 - len(v_splited[-1])):
str_v += '0'
print(str_v)
a.append(str_v + 'f')
print(len(a))
print(a)
with open('./input.txt', "w") as f:
f.write(','.join(a))
input_tensor = tf.reshape(input_rounded, [1, 20, 24, 24])
pooling_tenosr = keras.layers.MaxPool2D(pool_size=2,
strides=2,
padding='valid',
data_format='channels_first')
with backend.get_session() as sess:
print("image:")
image = sess.run(pooling_tenosr(input_tensor))
print(image.shape)
image = np.reshape(image, 1 * 20 * 12 * 12)
print(image.shape)
for v in image:
str_v = str(v)
v_splited = str(v).split('.')
if len(v_splited[-1]) < 6:
for i in range(6 - len(v_splited[-1])):
str_v += '0'
print(str_v)
b.append(str_v + 'f')
print(len(b))
def run_nn(hWnd):
global loaded
model_path = os.path.expanduser('YAD2K/model_data/yolo.h5')
anchors_path = os.path.expanduser('YAD2K/model_data/yolo_anchors.txt')
classes_path = os.path.expanduser('YAD2K/model_data/league_classes.txt')
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
yolo_model, _ = create_model(anchors, class_names)
yolo_model.load_weights('trained_stage_3_best.h5')
# Verify model, anchors, and classes are compatible
num_classes = len(class_names)
num_anchors = len(anchors)
# TODO: Assumes dim ordering is channel last
model_output_channels = yolo_model.layers[-1].output_shape[-1]
assert model_output_channels == num_anchors * (
num_classes +
5), 'Mismatch between model and given anchor and class sizes. '
print('{} model, anchors, and classes loaded.'.format(model_path))
# Check if model is fully convolutional, assuming channel last order.
model_image_size = yolo_model.layers[0].input_shape[1:3]
is_fixed_size = model_image_size != (None, None)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(class_names), 1., 1.)
for x in range(len(class_names))]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
colors))
random.seed(10101) # Fixed seed for consistent colors across runs.
random.shuffle(colors) # Shuffle colors to decorrelate adjacent classes.
random.seed(None) # Reset seed to default.
# Generate output tensor targets for filtered bounding boxes.
# TODO: Wrap these backend operations with Keras layers.
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold)
# Save the output into a compact JSON file.
outfile = open('output/game_data.json', 'w')
# This will be appended with an object for every frame.
data_to_write = []
loaded = True
win32gui.RedrawWindow(hWnd, None, None,
win32con.RDW_INVALIDATE | win32con.RDW_ERASE)
with mss.mss() as sct:
monitor = {'top': 0, 'left': 0, 'width': 1920, 'height': 1080}
while 'Screen capturing':
last_time = time.time()
# Get raw pixels from the screen, save it to a Numpy array
img = np.array(sct.grab(monitor))
img = Image.fromarray(img)
img.load()
background = Image.new("RGB", img.size, (255, 255, 255))
background.paste(img, mask=img.split()[3])
test_yolo(background, is_fixed_size, model_image_size, sess, boxes,
scores, classes, yolo_model, input_image_shape,
class_names, colors)
win32gui.RedrawWindow(hWnd, None, None,
win32con.RDW_INVALIDATE | win32con.RDW_ERASE)
# img = test_yolo(background)
# basewidth = 700
# wpercent = (basewidth/float(img.size[0]))
# hsize = int((float(img.size[1])*float(wpercent)))
# img = img.resize((basewidth,hsize), Image.ANTIALIAS)
# img = np.array(img)
# # Display the picture
# cv2.imshow('OpenCV/Numpy normal', img)
# Press "q" to quit
if cv2.waitKey(25) & 0xFF == ord(';'):
cv2.destroyAllWindows()
break
sess.close()
return
def tf_auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
return auc
def __init__(self, **kwargs):
self.__dict__.update(self._defaults)
self.class_names = self._get_class()
self.sess = K.get_session()
self.generate()
self.bbox_util = BBoxUtility(self.num_classes, nms_thresh=self.nms_iou)
from apphelper.redisbase import redisDataBase
from config import *
from crnn.keys import alphabetChinese, alphabetEnglish
if ocrFlag == 'keras':
if GPU:
os.environ["CUDA_VISIBLE_DEVICES"] = str(GPUID)
import tensorflow as tf
from keras import backend as K
config = tf.ConfigProto()
config.gpu_options.allocator_type = 'BFC'
config.gpu_options.per_process_gpu_memory_fraction = 0.1 ## GPU最大占用量
config.gpu_options.allow_growth = True ##GPU是否可动态增加
K.set_session(tf.Session(config=config))
K.get_session().run(tf.global_variables_initializer())
else:
##CPU启动
os.environ["CUDA_VISIBLE_DEVICES"] = ''
if ocrFlag == 'keras':
from crnn.network_keras import CRNN
if chineseModel:
alphabet = alphabetChinese
if LSTMFLAG:
ocrModel = ocrModelKerasLstm
else:
ocrModel = ocrModelKerasDense
else:
ocrModel = ocrModelKerasEng
test_npz_path = os.path.expanduser(args.test_npz_path)
if args.subcommand == 'mp4':
test_mp4_vod_path = os.path.expanduser(args.test_mp4_vod_path)
if args.subcommand == 'youtube':
test_youtube_link = os.path.expanduser(args.test_youtube_link)
youtube_download_path = os.path.expanduser(args.youtube_download_path)
start_time = os.path.expanduser(args.start_time)
end_time = os.path.expanduser(args.end_time)
if not os.path.exists(output_path):
print('Creating output path {}'.format(output_path))
os.mkdir(output_path)
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
# yolo_model = load_model(model_path)
yolo_model, _ = create_model(anchors, class_names)
yolo_model.load_weights('trained_stage_3_best.h5')
# Verify model, anchors, and classes are compatible
def load_yolo(self):
model_path = os.path.expanduser(self.model_path)
assert model_path.endswith(
'.h5'), 'Keras model or weights must be a .h5 file.'
self.class_names = self.get_class()
self.anchors = self.get_anchors()
num_anchors = len(self.anchors)
num_classes = len(self.class_names)
# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / len(self.class_names), 1., 1.)
for x in range(len(self.class_names))]
self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
self.colors = list(
map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
self.colors))
self.sess = K.get_session()
# Load model, or construct model and load weights.
self.yolo4_model = yolo4_body(Input(shape=(608, 608, 3)),
num_anchors // 3, num_classes)
# Read and convert darknet weight
print('Loading weights.')
weights_file = open(self.weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3, ),
dtype='int32',
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype='int64',
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype='int32',
buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
convs_to_load = []
bns_to_load = []
for i in range(len(self.yolo4_model.layers)):
layer_name = self.yolo4_model.layers[i].name
if layer_name.startswith('conv2d_'):
convs_to_load.append((int(layer_name[7:]), i))
if layer_name.startswith('batch_normalization_'):
bns_to_load.append((int(layer_name[20:]), i))
convs_sorted = sorted(convs_to_load, key=itemgetter(0))
bns_sorted = sorted(bns_to_load, key=itemgetter(0))
bn_index = 0
for i in range(len(convs_sorted)):
print('Converting ', i)
if i == 93 or i == 101 or i == 109:
#no bn, with bias
weights_shape = self.yolo4_model.layers[
convs_sorted[i][1]].get_weights()[0].shape
bias_shape = self.yolo4_model.layers[
convs_sorted[i][1]].get_weights()[0].shape[3]
filters = bias_shape
size = weights_shape[0]
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(
weights_size * 4))
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
self.yolo4_model.layers[convs_sorted[i][1]].set_weights(
[conv_weights, conv_bias])
else:
#with bn, no bias
weights_shape = self.yolo4_model.layers[
convs_sorted[i][1]].get_weights()[0].shape
size = weights_shape[0]
bn_shape = self.yolo4_model.layers[bns_sorted[bn_index]
[1]].get_weights()[0].shape
filters = bn_shape[0]
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
bn_weights = np.ndarray(shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
self.yolo4_model.layers[bns_sorted[bn_index][1]].set_weights(
bn_weight_list)
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(
weights_size * 4))
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
self.yolo4_model.layers[convs_sorted[i][1]].set_weights(
[conv_weights])
bn_index += 1
weights_file.close()
self.yolo4_model.save(self.model_path)
if self.gpu_num >= 2:
self.yolo4_model = multi_gpu_model(self.yolo4_model,
gpus=self.gpu_num)
self.input_image_shape = K.placeholder(shape=(2, ))
self.boxes, self.scores, self.classes = yolo_eval(
self.yolo4_model.output,
self.anchors,
len(self.class_names),
self.input_image_shape,
score_threshold=self.score)
def train(self, datapath, feature_type, batch_size=32, epoch=20):
assert (batch_size % CLASS_NUM == 0)
data = DataSpeech(datapath, feature_type, 'train')
num_data = sum(data.DataNum) # 获取数据的数�?
os.system('pkill tensorboard')
os.system('rm -rf ./checkpoints/files_summary/* ')
train_writter = tf.summary.FileWriter(
os.path.join(os.getcwd(), 'checkpoints', 'files_summary'))
os.system(
'tensorboard --logdir=/home/zhaok14/example/PycharmProjects/setsail/individual_spp/checkpoints/files_summary/ --port=6006 &'
)
print('\n')
print(90 * '*')
print(40 * ' ', self.modelname)
print(90 * '*')
iterations_per_epoch = min(
data.DataNum) // (batch_size // CLASS_NUM) + 1
# iterations_per_epoch = 30
print('trainer info:')
print('training data size: %d' % num_data)
print('increased epoches: ', epoch)
print('minibatch size: %d' % batch_size)
print('iterations per epoch: %d' % iterations_per_epoch)
sess = k.get_session()
train_writter.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
best_score = 0
# epoch = 2
duration = 0
for i in range(0, epoch):
iteration = 0
yielddatas = data.data_genetator(batch_size, epoch)
pbar = tqdm(yielddatas)
for input, labels in pbar:
stime = time.time()
loss = self.model.train_on_batch(input[0], labels)
dtime = time.time() - stime
duration = duration + dtime
train_summary = tf.Summary()
train_summary.value.add(tag='loss', simple_value=loss)
train_writter.add_summary(train_summary,
iteration + i * iterations_per_epoch)
pr = 'epoch:%d/%d,iteration: %d/%d ,loss: %s' % (
epoch, i, iterations_per_epoch, iteration, loss)
pbar.set_description(pr)
if iteration == iterations_per_epoch:
break
else:
iteration += 1
pbar.close()
self.TestModel(sess=sess,
feature_type=feature_type,
datapath=datapath,
str_dataset='train',
data_count=1000,
writer=train_writter,
step=i)
metrics = self.TestModel(sess=sess,
feature_type=feature_type,
datapath=datapath,
str_dataset='eval',
data_count=-1,
writer=train_writter,
step=i)
if i > 0:
if metrics['score'] >= best_score:
self.metrics = metrics
self.metrics['epoch'] = i
best_score = metrics['score']
clrdir(self.clearPath)
self.savpath = []
self.savpath.append(
(self.baseSavPath[0] + '_epoch' + str(i) + '.h5'))
self.savpath.append(
(self.baseSavPath[1] + '_epoch' + str(i) + '.h5'))
self.model.save(self.savpath[0])
self.model.save_weights(self.savpath[1])
if 'epoch' in self.metrics.keys():
print(
'The best metric (without restriction) took place in the epoch: ',
self.metrics['epoch'])
print('Sensitivity: {}; Specificity: {}; Score: {}; Accuracy: {}'.
format(self.metrics['sensitivity'],
self.metrics['specificity'], self.metrics['score'],
self.metrics['accuracy']))
# self.TestGenerability(feature_type = feature_type, weightspath=self.savpath[1])
else:
print('The best metric (without restriction) is not found. Done!')
print('Training duration: {}s'.format(round(duration, 2)))
return self.metrics['accuracy']
from fne import full_network_embedding
from sklearn.metrics import classification_report, confusion_matrix
import numpy as np
if __name__ == '__main__':
# This shows an example of calling the full_network_embedding method using
# the VGG16 architecture pretrained on ILSVRC2012 (aka ImageNet), as
# provided by the keras package. Using any other pretrained CNN
# model is straightforward.
# Load model
img_width, img_height = 224, 224
applications.VGG16(weights="imagenet",
include_top=True,
input_shape=(img_width, img_height, 3))
graph_d = K.get_session().graph_def
# Define input and target tensors, that is where we want
#to enter data, and which activations we wish to extract
input_tensor = 'input_1:0'
target_tensors = [
'block1_conv1/Relu:0', 'block1_conv2/Relu:0', 'block2_conv1/Relu:0',
'block2_conv2/Relu:0', 'block3_conv1/Relu:0', 'block3_conv2/Relu:0',
'block3_conv3/Relu:0', 'block4_conv1/Relu:0', 'block4_conv2/Relu:0',
'block4_conv3/Relu:0', 'block5_conv1/Relu:0', 'block5_conv2/Relu:0',
'block5_conv3/Relu:0', 'fc1/Relu:0', 'fc2/Relu:0'
]
# Define data splits
#Train set
train_path = '/gpfs/projects/nct00/nct00001/mit67/train/'
#Load MNIST data and normalize to [0,1]
(data_train, _), (data_test, _) = mnist.load_data()
data_train = data_train/255.0
data_test = data_test/255.0
#Flatten dataset (New shape for training and testing set is (60000,784) and (10000, 784))
data_train = data_train.reshape((len(data_train), np.prod(data_train.shape[1:])))
data_test = data_test.reshape((len(data_test), np.prod(data_test.shape[1:])))
#Load classifier model whose gradients will be used to create adversarial examples
keras_model = load_model('fc-100-100-10.h5')
backend.set_learning_phase(False)
#Create adversarial examples on testing data
sess = backend.get_session()
epsilon = 0.50
wrap = KerasModelWrapper(keras_model)
fgsm = FastGradientMethod(wrap, sess=sess)
adv_train_x = fgsm.generate_np(data_train, eps=epsilon, clip_min=0., clip_max=1.)
adv_test_x = fgsm.generate_np(data_test, eps=epsilon, clip_min=0., clip_max=1.)
#Total datasets
data_total_train = np.vstack([data_train, adv_train_x])
data_total_test = np.vstack([data_test, adv_test_x])
#Create labels that correspond to clean reconstructions
labels_total_train = np.vstack([data_train, data_train])
labels_total_test = np.vstack([data_test, data_test])
#Create the model
labels,
all_doc_topics_embedding)
loss_topics = loss_lda(preds,
labels,
all_doc_topics_embedding)
loss_words = loss_word2vec(preds,
labels)
# define gradient descent and initialize variables
train_step = tf.train.GradientDescentOptimizer(0.05).minimize(loss)
init_op = tf.global_variables_initializer()
sess = K.get_session() # get session from keras
sess.run(init_op)
# Run training loop
with sess.as_default():
for i in range(200000):
idx = np.random.randint(0,
len(train_labels)-1,
batch_size).tolist()
# training happens here
losses = sess.run([train_step, loss, loss_topics, loss_words, preds, labels],
feed_dict = {input_target:np.reshape(train_data_target[idx],
(-1,1)),
def export_model_keras(keras_model='model.h5',
export_dir="graph",
model_name="temp_model_name",
sequential=True,
custom_objects=None):
K.clear_session() # Clears existing graph.
if os.path.isfile(keras_model):
if custom_objects is None:
model = load_model(keras_model)
else:
model = load_model(keras_model, custom_objects=custom_objects)
else:
return
# All new operations will be in test mode from now on.
K.set_learning_phase(0)
# Serialize the model and get its weights, for quick re-building.
config = model.get_config()
weights = model.get_weights()
# Re-build a model where the learning phase is now hard-coded to 0.
if sequential:
new_model = Sequential.from_config(config,
custom_objects=custom_objects)
else:
new_model = Model.from_config(config, custom_objects=custom_objects)
new_model.set_weights(weights)
temp_dir = "graph"
checkpoint_prefix = os.path.join(temp_dir, "saved_checkpoint")
checkpoint_state_name = "checkpoint_state"
input_graph_name = "untrained_input_graph.pb"
# Temporary save graph to disk without weights included.
saver = tf.train.Saver()
checkpoint_path = saver.save(K.get_session(),
checkpoint_prefix,
global_step=0,
latest_filename=checkpoint_state_name)
tf.train.write_graph(K.get_session().graph, temp_dir, input_graph_name)
input_graph_path = os.path.join(temp_dir, input_graph_name)
# input_saver_def_path = ""
input_saver_def_path = None
input_binary = False
input_node_names = [node.op.name for node in model.inputs]
output_node_names = [node.op.name for node in model.outputs]
print("Input layer name: ", input_node_names)
print("Output layer name: ", output_node_names)
restore_op_name = "save/restore_all"
filename_tensor_name = "save/Const:0"
output_graph_path = export_dir + '/frozen_' + model_name + '.pb'
clear_devices = True # Remove all the explicit device specifications for this node. This helps to
# make the graph more portable.
# Embed weights inside the graph and save to disk.
freeze_graph.freeze_graph(input_graph_path, input_saver_def_path,
input_binary, checkpoint_path,
*output_node_names, restore_op_name,
filename_tensor_name, output_graph_path,
clear_devices, "")
input_graph_def = tf.GraphDef()
with tf.gfile.Open(export_dir + '/frozen_' + model_name + '.pb',
"rb") as f:
input_graph_def.ParseFromString(f.read())
output_graph_def = optimize_for_inference_lib.optimize_for_inference(
input_graph_def, input_node_names, output_node_names,
tf.float32.as_datatype_enum)
with tf.gfile.FastGFile(export_dir + '/opt_' + model_name + '.pb',
"wb") as f:
f.write(output_graph_def.SerializeToString())
print("Graph Saved - Output Directories: ")
print("1 - Standard Frozen Model:",
export_dir + '/frozen_' + model_name + '.pb')
print("2 - Android Optimized Model:",
export_dir + '/opt_' + model_name + '.pb')
print_graph_nodes(export_dir + '/frozen_' + model_name + '.pb')
return model
def reset_weights(model):
session = K.get_session()
for layer in model.layers:
if hasattr(layer, 'kernel_initializer'):
layer.kernel.initializer.run(session=session)
# Model paths
model_dir = os.path.join(os.path.dirname(settings.BASE_DIR), 'models', 'keras')
model_arch_path = os.path.join(model_dir, indian_model_name + '.json')
model_weight_path = os.path.join(model_dir, indian_model_name + '.h5')
# load json and create model
json_file = open('C:/Users/raksh/Downloads/FIC-In-C7-B32-E11.json')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(model_weight_path)
print('\n Loading model from disk ------------------\n')
graph = K.get_session().graph
def upload_img(request):
form = ClassifierForm(request.POST or None, request.FILES or None)
if form.is_valid():
m = Classifier()
m.image = form.cleaned_data['image']
print(type(form.cleaned_data['image']))
m.save()
# result = feedfowrd()
return HttpResponseRedirect('/predict')
context = {
input_size=config['model']['input_size'],
labels=config['model']['labels'],
max_box_per_image=config['model']['max_box_per_image'],
anchors=config['model']['anchors'])
yolo.model.load_weights(config['train']['pretrained_weights'])
model = yolo.model
model = stupidmodel()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
# model.load_weights(config['train']['pretrained_weights'])
# yolo.model.load_weights('tiny_yolo_cloth.h5')
frozen_graph = freeze_session(
K.get_session(),
output_names=[out.op.name for out in model.outputs])
tf.train.write_graph(frozen_graph,
"t",
"sTFMNet_model.pbtxt",
as_text=True)
tf.train.write_graph(frozen_graph,
"t",
"sTFMNet_model.pb",
as_text=False)
# ms = [vgg16.VGG16, resnet50.ResNet50, mobilenet.MobileNet, densenet.DenseNet121]
# mname = ['vgg16', 'resnet50', 'mobilenet', 'densenet121']
# ms = [stupidmodel()]
# mname = ['bn']
# for i in range(len(ms)):
def init_attack(model, attack):
wrap = KerasModelWrapper(model)
sess = K.get_session()
attack = getattr(cleverhans.attacks, attack)
return attack(wrap, sess)
boxes = scale_boxes(boxes, image_shape)
# Use one of the functions you've implemented to perform Non-max suppression with a threshold of iou_threshold (≈1 line)
scores, boxes, classes = yolo_non_max_suppression(
scores,
boxes,
classes,
max_boxes=max_boxes,
iou_threshold=iou_threshold)
### END CODE HERE ###
return scores, boxes, classes
sess = K.get_session()
class_names = read_classes("model_data/coco_classes.txt")
anchors = read_anchors("model_data/yolo_anchors.txt")
image_shape = (720., 1280.)
yolo_model = load_model("model_data/yolo.h5")
yolo_model.summary()
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
scores, boxes, classes = yolo_eval(yolo_outputs, image_shape)
def predict(sess, image_file):
def mean_iou_2(y_true, y_pred, NUM_CLASSES=29):
score, up_opt = tf.metrics.mean_iou(y_true, y_pred, NUM_CLASSES)
K.get_session().run(tf.local_variables_initializer())
with tf.control_dependencies([up_opt]):
score = tf.identity(score)
return score
