def test_keras_consistency(self):
""" Exported model in Keras should get same result as original """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
def keras_load_and_preproc(fpath):
img = load_img(fpath, target_size=(299, 299))
img_arr = img_to_array(img)
img_iv3_input = iv3.preprocess_input(img_arr)
return np.expand_dims(img_iv3_input, axis=0)
imgs_iv3_input = np.vstack([keras_load_and_preproc(fp) for fp in img_fpaths])
model_ref = InceptionV3(weights="imagenet")
preds_ref = model_ref.predict(imgs_iv3_input)
with IsolatedSession(using_keras=True) as issn:
K.set_learning_phase(0)
model = InceptionV3(weights="imagenet")
gfn = issn.asGraphFunction(model.inputs, model.outputs)
with IsolatedSession(using_keras=True) as issn:
K.set_learning_phase(0)
feeds, fetches = issn.importGraphFunction(gfn, prefix="InceptionV3")
preds_tgt = issn.run(fetches[0], {feeds[0]: imgs_iv3_input})
self.assertTrue(np.all(preds_tgt == preds_ref))
def test_prediction_vs_tensorflow_inceptionV3(self):
output_col = "prediction"
image_df = image_utils.getSampleImageDF()
# An example of how a pre-trained keras model can be used with TFImageTransformer
with KSessionWrap() as (sess, g):
with g.as_default():
K.set_learning_phase(0) # this is important but it's on the user to call it.
# nChannels needed for input_tensor in the InceptionV3 call below
image_string = utils.imageInputPlaceholder(nChannels = 3)
resized_images = tf.image.resize_images(image_string,
InceptionV3Constants.INPUT_SHAPE)
preprocessed = preprocess_input(resized_images)
model = InceptionV3(input_tensor=preprocessed, weights="imagenet")
graph = tfx.strip_and_freeze_until([model.output], g, sess, return_graph=True)
transformer = TFImageTransformer(inputCol="image", outputCol=output_col, graph=graph,
inputTensor=image_string, outputTensor=model.output,
outputMode="vector")
transformed_df = transformer.transform(image_df.limit(10))
self.assertDfHasCols(transformed_df, [output_col])
collected = transformed_df.collect()
transformer_values, transformer_topK = self.transformOutputToComparables(collected,
"filePath",
output_col)
tf_values, tf_topK = self._executeTensorflow(graph, image_string.name, model.output.name,
image_df)
self.compareClassSets(tf_topK, transformer_topK)
self.compareClassOrderings(tf_topK, transformer_topK)
self.compareArrays(tf_values, transformer_values)
def test_bare_keras_module(self):
""" Keras GraphFunctions should give the same result as standard Keras models """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
for model_gen, preproc_fn in [(InceptionV3, iv3.preprocess_input),
(Xception, xcpt.preprocess_input),
(ResNet50, rsnt.preprocess_input)]:
keras_model = model_gen(weights="imagenet")
target_size = tuple(keras_model.input.shape.as_list()[1:-1])
_preproc_img_list = []
for fpath in img_fpaths:
img = load_img(fpath, target_size=target_size)
# WARNING: must apply expand dimensions first, or ResNet50 preprocessor fails
img_arr = np.expand_dims(img_to_array(img), axis=0)
_preproc_img_list.append(preproc_fn(img_arr))
imgs_input = np.vstack(_preproc_img_list)
preds_ref = keras_model.predict(imgs_input)
gfn_bare_keras = GraphFunction.fromKeras(keras_model)
with IsolatedSession(using_keras=True) as issn:
K.set_learning_phase(0)
feeds, fetches = issn.importGraphFunction(gfn_bare_keras)
preds_tgt = issn.run(fetches[0], {feeds[0]: imgs_input})
self.assertTrue(np.all(preds_tgt == preds_ref))
def run(Data, Model, path):
sess = K.get_session()
K.set_learning_phase(False)
data, model = Data(), Model(path)
if Data == MNIST:
attack = CarliniL2(sess, model, batch_size=100, max_iterations=2000,
binary_search_steps=5, initial_const=1., learning_rate=1e-1,
targeted=False)
else:
attack = CarliniL2(sess, model, batch_size=100, max_iterations=200,
binary_search_steps=3, initial_const=.01, learning_rate=1e-2,
targeted=True, confidence=2)
now = time.time()
for name,X,y in [["test",data.test_data, data.test_labels]]:
print("OKAY",name)
for k in range(0,len(y),5000):
#if os.path.exists("tmp/"+path.split("/")[1]+"."+name+".adv.X."+str(k)+".npy"):
# print('skip',k)
# continue
now = time.time()
adv = attack.attack(X[k:k+100], y[k:k+100])
#print('time',time.time()-now)
#print('accuracy',np.mean(np.argmax(model.model.predict(adv),axis=1)==np.argmax(y[k:k+5000],axis=1)))
#print('mean distortion',np.mean(np.sum((adv-X[k:k+5000])**2,axis=(1,2,3))**.5))
np.save("/tmp/"+path.split("/")[1]+"."+name+".adv.X."+str(k),adv)
def model_fn(input_dim,
labels_dim,
hidden_units=[100, 70, 50, 20],
learning_rate=0.1):
"""Create a Keras Sequential model with layers.
Args:
input_dim: (int) Input dimensions for input layer.
labels_dim: (int) Label dimensions for input layer.
hidden_units: [int] the layer sizes of the DNN (input layer first)
learning_rate: (float) the learning rate for the optimizer.
Returns:
A Keras model.
"""
# "set_learning_phase" to False to avoid:
# AbortionError(code=StatusCode.INVALID_ARGUMENT during online prediction.
K.set_learning_phase(False)
model = models.Sequential()
for units in hidden_units:
model.add(layers.Dense(units=units, input_dim=input_dim, activation=relu))
input_dim = units
# Add a dense final layer with sigmoid function.
model.add(layers.Dense(labels_dim, activation='sigmoid'))
compile_model(model, learning_rate)
return model
def main(_):
if FLAGS.dataset == 'cifar10':
(X_train, y_train), (_, _) = cifar10.load_data()
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=0)
else:
with open('data/train.p', mode='rb') as f:
train = pickle.load(f)
X_train, X_val, y_train, y_val = train_test_split(train['features'], train['labels'], test_size=0.33, random_state=0)
train_output_file = "{}_{}_{}.p".format(FLAGS.network, FLAGS.dataset, 'bottleneck_features_train')
validation_output_file = "{}_{}_{}.p".format(FLAGS.network, FLAGS.dataset, 'bottleneck_features_validation')
print("Resizing to", (w, h, ch))
print("Saving to ...")
print(train_output_file)
print(validation_output_file)
with tf.Session() as sess:
K.set_session(sess)
K.set_learning_phase(1)
model = create_model()
print('Bottleneck training')
train_gen = gen(sess, X_train, y_train, batch_size)
bottleneck_features_train = model.predict_generator(train_gen(), X_train.shape[0])
data = {'features': bottleneck_features_train, 'labels': y_train}
pickle.dump(data, open(train_output_file, 'wb'))
print('Bottleneck validation')
val_gen = gen(sess, X_val, y_val, batch_size)
bottleneck_features_validation = model.predict_generator(val_gen(), X_val.shape[0])
data = {'features': bottleneck_features_validation, 'labels': y_val}
pickle.dump(data, open(validation_output_file, 'wb'))
def run_test(Data, Model, path):
sess = K.get_session()
K.set_learning_phase(False)
data = Data()
model = Model(path)
N = 1000
X = data.train_data[np.random.choice(np.arange(len(data.train_data)), N, replace=False)].reshape((N,-1))
#Y = data.train_data[np.random.choice(np.arange(len(data.train_data)), N, replace=False)].reshape((N,-1))
Y = data.test_data[np.random.choice(np.arange(len(data.test_data)), N, replace=False)].reshape((N,-1))
#attack = FGS(sess, model, N, .275)
attack = CarliniL2(sess, model, batch_size=100, binary_search_steps=2, initial_const=1, targeted=False, max_iterations=500)
idx = np.random.choice(np.arange(len(data.test_data)), N, replace=False)
Y = attack.attack(data.test_data[idx], data.test_labels[idx]).reshape((N,-1))
iterations = 1000
sigma2 = 100
mmd2u, mmd2u_null, p_value = kernel_two_sample_test(X, Y, iterations=iterations,
kernel_function='rbf',
gamma=1.0/sigma2,
verbose=True)
def cnn_model_fn(features, labels, mode):
"""Model function for CNN."""
# Input Layer
# Reshape X to 4-D tensor: [batch_size, width, height, channels]
# MNIST images are 28x28 pixels, and have one color channel
input_layer = tf.reshape(features["x"], [-1, 28, 28, 1])
if mode == tf.estimator.ModeKeys.TRAIN:
K.set_learning_phase(1)
else:
K.set_learning_phase(0)
conv1 = Convolution2D(32, (5, 5), activation='relu', input_shape=(28,28,1))(input_layer)
pool1 = MaxPooling2D(pool_size=(2,2))(conv1)
conv2 = Convolution2D(64, (5, 5), activation='relu')(pool1)
pool2 = MaxPooling2D(pool_size=(2,2))(conv2)
pool2_flat = Flatten()(pool2)
dense = Dense(1024, activation='relu')(pool2_flat)
dropout = Dropout(0.4)(dense)
logits = Dense(10, activation='linear')(dropout)
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
# Add `softmax_tensor` to the graph. It is used for PREDICT and by the
# `logging_hook`.
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
prediction_output = tf.estimator.export.PredictOutput({"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")})
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions,
export_outputs={tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_output})
# Calculate Loss (for both TRAIN and EVAL modes)
onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10)
loss = tf.losses.softmax_cross_entropy(
onehot_labels=onehot_labels, logits=logits)
tf.summary.scalar('loss', loss)
tf.summary.histogram('conv1', conv1)
tf.summary.histogram('dense', dense)
# Configure the Training Op (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
# Add evaluation metrics (for EVAL mode)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=labels, predictions=predictions["classes"])}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def cnn_model(features, labels, mode):
input_layer = tf.reshape(features['x'], shape=[-1, 28, 28, 1])
if mode == tf.estimator.ModeKeys.TRAIN:
K.set_learning_phase(1)
else:
K.set_learning_phase(0)
conv1 = layers.Conv2D(filters=32, kernel_size=(3, 3), padding='same', input_shape=(28, 28, 1),
activation='relu')(input_layer)
conv2 = layers.Conv2D(filters=64, kernel_size=(3, 3), padding='same', activation='relu')(conv1)
pool1 = layers.MaxPool2D(pool_size=(2, 2))(conv2)
dropout = layers.Dropout(0.5)(pool1)
conv3 = layers.Conv2D(filters=128, kernel_size=(3, 3), padding='same', activation='relu')(dropout)
conv4 = layers.Conv2D(filters=256, kernel_size=(3, 3), padding='same', activation='relu')(conv3)
pool2 = layers.MaxPool2D(pool_size=(3, 3))(conv4)
dropout2 = layers.Dropout(0.5)(pool2)
flatten = layers.Flatten()(dropout2)
dense1 = layers.Dense(256)(flatten)
lrelu = layers.LeakyReLU()(dense1)
dropout3 = layers.Dropout(0.5)(lrelu)
dense2 = layers.Dense(256)(dropout3)
lrelu2 = layers.LeakyReLU()(dense2)
logits = layers.Dense(10, activation='linear')(lrelu2)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits=logits, name='softmax_tensor')
}
prediction_output = tf.estimator.export.PredictOutput({"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")})
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, export_outputs={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_output
})
one_hot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10)
loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels, logits=logits)
tf.summary.scalar('loss', loss)
tf.summary.histogram('conv1', conv1)
tf.summary.histogram('dense', dense1)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metrics_ops = {
'accuracy': tf.metrics.accuracy(labels=labels, predictions=predictions['classes'])
}
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metrics_ops)
def ds2_gru_model(input_dim=161, fc_size=1024, rnn_size=512, output_dim=29, initialization='glorot_uniform',
conv_layers=1, gru_layers=1, use_conv=True):
""" DeepSpeech 2 implementation
Architecture:
Input Spectrogram TIMEx161
1 Batch Normalisation layer on input
1-3 Convolutional Layers
1 Batch Normalisation layer
1-7 BiDirectional GRU Layers
1 Batch Normalisation layer
1 Fully connected Dense
1 Softmax output
Details:
- Uses Spectrogram as input rather than MFCC
- Did not use BN on the first input
- Network does not dynamically adapt to maximum audio size in the first convolutional layer. Max conv
length padded at 2048 chars, otherwise use_conv=False
Reference:
https://arxiv.org/abs/1512.02595
https://github.com/robmsmt/KerasDeepSpeech
"""
K.set_learning_phase(1)
# Main acoustic input
input_data = Input(name='the_input', shape=(None, input_dim))
m = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)(input_data)
if use_conv:
conv = ZeroPadding1D(padding=(0, 2048))(m)
for l in range(conv_layers):
m = Conv1D(filters=fc_size, name='conv_{}'.format(l+1), kernel_size=11, padding='valid',
activation='relu', strides=2)(conv)
else:
for l in range(conv_layers):
m = TimeDistributed(Dense(fc_size, name='fc_{}'.format(l + 1), activation='relu'))(m)
m = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)(m)
for l in range(gru_layers):
m = Bidirectional(GRU(rnn_size, name='fc_{}'.format(l + 1), return_sequences=True, activation='relu',
kernel_initializer=initialization), merge_mode='sum')(m)
m = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)(m)
# Last Layer 5+6 Time Dist Dense Layer & Softmax
m = TimeDistributed(Dense(fc_size, activation=clipped_relu))(m)
y_pred = TimeDistributed(Dense(output_dim, name="y_pred", activation="softmax"))(m)
model = Model(inputs=input_data, outputs=y_pred)
if use_conv:
model.output_length = lambda x: cnn_output_length(x, 11, 'valid', 2)
else:
model.output_length = lambda x: x
print(model.summary())
return model
def policy_batch_update(model, update, r_holder, X, R, batch_size, iterations):
K.set_learning_phase(1) #set learning phase
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for iteration in range(iterations):
indices = np.random.choice(len(X), size=batch_size,replace = False )
X_batch, R_batch = X[indices], np.reshape(R[indices],(batch_size, 1))
sess.run([update], feed_dict={model.input:X_batch, r_holder:R_batch})
print('Iteration: {}'.format(iteration))
K.set_learning_phase(0) #set learning phase
def run_pca(Data, num_components=10, invert=False):
data = Data()
sess = K.get_session()
K.set_learning_phase(False)
shape = (-1, 784)
pca = sklearn.decomposition.PCA(n_components=num_components)
pca.fit(data.train_data.reshape(shape)) # [:10000]
if invert:
model = MNISTModel("models/mnist-pca-cnn-top-"+str(num_components))
else:
model = make_model(num_components)
model.load_weights("models/mnist-pca-top-"+str(num_components))
model = Wrap(model,pca)
tf_mean = tf.constant(pca.mean_,dtype=tf.float32)
tf_components = tf.constant(pca.components_.T,dtype=tf.float32)
def new_predict(xs):
# map to PCA space
xs = tf.reshape(xs,(-1,784))
xs -= tf_mean
xs = tf.matmul(xs, tf_components)
# map back
xs = tf.matmul(xs, tf.transpose(tf_components))
xs += tf_mean
xs = tf.reshape(xs, (-1, 28, 28, 1))
return model.model(xs)
if invert:
model.predict = new_predict
attack = CarliniL2(sess, model, batch_size=100, max_iterations=3000,
binary_search_steps=6, targeted=False,
initial_const=1)
N = 100
test_adv = attack.attack(data.test_data[:N], data.test_labels[:N])
print('accuracy',np.mean(np.argmax(sess.run(model.predict(tf.constant(data.test_data,dtype=np.float32))),axis=1)==np.argmax(data.test_labels,axis=1)))
print(list(test_adv[0].flatten()))
print('dist',np.mean(np.sum((test_adv-data.test_data[:N])**2,axis=(1,2,3))**.5))
it = np.argmax(sess.run(model.predict(tf.constant(test_adv))),axis=1)
print('success',np.mean(it==np.argmax(data.test_labels,axis=1)[:N]))
def visualize_features(model, problem_name):
out_path = 'output/figures/{}/features/'
im_size = (52, 52)
K.set_learning_phase(0)
if model is None:
model = ModelLipnet4(verbose=True, compile_on_build=False, include_top=True)
model.build_model((1, im_size[0], im_size[1]), 3)
model.restore('/home/sergii/Documents/Thesis/lipnet/output/models/{}_model_lipnet6.h5'.format(problem_name))
vis = VisualizerFeatures()
vis.visualize_model(model.model, out_path.format(problem_name), im_size)
def getModelData(self, featurize):
sess = tf.Session()
with sess.as_default():
K.set_learning_phase(0)
inputImage = imageInputPlaceholder(nChannels=3)
preprocessed = self.preprocess(inputImage)
model = self.model(preprocessed, featurize)
return dict(inputTensorName=inputImage.name,
outputTensorName=model.output.name,
session=sess,
inputTensorSize=self.inputShape(),
outputMode="vector")
def _loadTFGraph(self):
with KSessionWrap() as (sess, g):
assert K.backend() == "tensorflow", \
"Keras backend is not tensorflow but KerasImageTransformer only supports " + \
"tensorflow-backed Keras models."
with g.as_default():
K.set_learning_phase(0) # Testing phase
model = load_model(self.getModelFile())
out_op_name = tfx.op_name(g, model.output)
self._inputTensor = model.input.name
self._outputTensor = model.output.name
return tfx.strip_and_freeze_until([out_op_name], g, sess, return_graph=True)
def _buildInceptionV3Session(featurize):
sess = tf.Session()
with sess.as_default():
K.set_learning_phase(0)
inputImage = imageInputPlaceholder(nChannels=3)
preprocessed = inception_v3.preprocess_input(inputImage)
model = InceptionV3(input_tensor=preprocessed, weights="imagenet",
include_top=(not featurize))
return dict(inputTensorName=inputImage.name,
outputTensorName=model.output.name,
session=sess,
inputTensorSize=InceptionV3Constants.INPUT_SHAPE,
outputMode="vector")
def _fromKerasModelFile(cls, file_path):
"""
Load a Keras model from a file path into a `GraphFunction`.
:param file_path: the (HDF5) file path
"""
assert file_path.endswith('.h5'), \
'Keras model must be specified as HDF5 file'
with IsolatedSession(using_keras=True) as issn:
K.set_learning_phase(0) # Testing phase
model = load_model(file_path)
gfn = issn.asGraphFunction(model.inputs, model.outputs)
return gfn
def test_batchnorm_trainable():
bn_mean = 0.5
bn_std = 10.
def get_model(bn_mean, bn_std):
input = Input(shape=(1,))
x = normalization.BatchNormalization()(input)
model = Model(input, x)
model.set_weights([np.array([1.]), np.array([0.]),
np.array([bn_mean]), np.array([bn_std ** 2])])
return model
# Simulates training-mode with trainable layer. Should use mini-batch statistics.
K.set_learning_phase(1)
model = get_model(bn_mean, bn_std)
model.compile(loss='mse', optimizer='rmsprop')
out = model.predict(input_4)
assert_allclose((input_4 - np.mean(input_4)) / np.std(input_4), out, atol=1e-3)
def run_hidden_pca(Data, Model, path=None):
sess = K.get_session()
K.set_learning_phase(False)
data = Data()
model = Model(path)
model2 = Model(path)
hidden_layer = pop(model2.model) # once to remove dense(10)
hidden_layer = pop(hidden_layer) # once to remove ReLU
train_hidden = hidden_layer.predict(data.test_data)
#val_hidden = hidden_layer.predict(data.validation_data)
test_hidden = hidden_layer.predict(data.test_data)
pca = sklearn.decomposition.PCA(n_components=test_hidden.shape[1])
pca.fit(train_hidden)
#r_val = pca.transform(hidden_layer.predict(data.validation_data))
r_test = pca.transform(hidden_layer.predict(data.test_data))
attack = FGS(sess, model, eps=.2)
#attack = CarliniL2(sess, model, batch_size=100, max_iterations=1000,
# binary_search_steps=2, targeted=False)
N = 10000
test_adv = attack.attack(data.test_data[:N], data.test_labels[:N])
r_test_adv = pca.transform(hidden_layer.predict(test_adv[:N]))
print(r_test_adv[0])
show(test_adv[0])
#compute_thresholds(r_val, r_val_adv)
plt.figure(figsize=(4,3))
plt.xlabel('Component Number')
plt.ylabel('Mean Absolute Value (log scale)')
plt.semilogy(range(r_test.shape[1]),np.mean(np.abs(r_test),axis=0))
plt.semilogy(range(r_test_adv.shape[1]),np.mean(np.abs(r_test_adv),axis=0))
plt.show()
def load(self, data_dir):
"""Load graph and weight data.
Args:
data_dir (:obj:`str`): location of Keras checkpoint (`.hdf5`) files
and model (in `.json`) structure. The default behavior
is to take the latest of each, by OS timestamp.
"""
# for tensorflow compatibility
K.set_learning_phase(0)
# find newest ckpt and graph files
try:
latest_ckpt = max(glob.iglob(
os.path.join(data_dir, '*.h*5')), key=os.path.getctime)
latest_ckpt_name = os.path.basename(latest_ckpt)
latest_ckpt_time = str(
datetime.fromtimestamp(os.path.getmtime(latest_ckpt)))
except ValueError:
raise FileNotFoundError('No checkpoint (.hdf5 or .h5) files '
'available at {}'.format(data_dir))
try:
latest_json = max(glob.iglob(os.path.join(data_dir, '*.json')),
key=os.path.getctime)
with open(latest_json, 'r') as f:
model_json = json.loads(f.read())
self._model = model_from_json(model_json)
self._model.load_weights(latest_ckpt)
except ValueError:
try:
self._model = load_model(latest_ckpt)
except ValueError:
raise FileNotFoundError('The (.hdf5 or .h5) files available at'
'{} don\'t have the model'
' architecture.'
.format(latest_ckpt))
self._sess = K.get_session()
self._tf_predict_var = self._model.outputs[0]
self._tf_input_var = self._model.inputs[0]
self._model_name = type(self).__name__
self._latest_ckpt_name = latest_ckpt_name
self._latest_ckpt_time = latest_ckpt_time
def run_pca(Data, Model, path=None):
sess = K.get_session()
K.set_learning_phase(False)
data = Data()
model = Model(path)
shape = (-1, model.num_channels*model.image_size**2)
pca = sklearn.decomposition.PCA(n_components=shape[1])
pca.fit(data.train_data.reshape(shape))
print(pca.explained_variance_ratio_)
r_test = pca.transform(data.test_data.reshape(shape))
#attack = FGS(sess, model, eps=.3)
attack = CarliniL2(sess, model, batch_size=100, max_iterations=1000,
binary_search_steps=2, targeted=False,
initial_const=10)
N = 10000
#test_adv = attack.attack(data.test_data[:N], data.test_labels[:N])
test_adv = np.load("tmp/outlieradvtest.npy")
r_test_adv = pca.transform(test_adv[:N].reshape(shape))
fig = plt.figure(figsize=(4,3))
fig.subplots_adjust(bottom=0.17,left=.19)
plt.xlabel('Component Number')
plt.ylabel('Mean Absolute Value (log scale)')
plt.semilogy(range(r_test.shape[1]),np.mean(np.abs(r_test),axis=0),label='Valid')
plt.semilogy(range(r_test_adv.shape[1]),np.mean(np.abs(r_test_adv),axis=0), label='Adversarial')
plt.legend()
pp = PdfPages('/tmp/a.pdf')
plt.savefig(pp, format='pdf')
pp.close()
plt.show()
def main():
"""Convert any Keras model to the frugally-deep model format."""
usage = 'usage: [Keras model in HDF5 format] [image output directory]'
if len(sys.argv) != 3:
print(usage)
sys.exit(1)
else:
assert K.backend() == "tensorflow"
assert K.floatx() == "float32"
assert K.image_data_format() == 'channels_last'
in_path = sys.argv[1]
out_dir = sys.argv[2]
print('loading {}'.format(in_path))
K.set_learning_phase(1)
model = load_model(in_path)
model = convert_sequential_to_model(model)
process_layers(model, out_dir)
def _loadTFGraph(self, sess, graph):
"""
Loads the Keras model into memory, then uses the passed-in session to load the
model's inference-related ops into the passed-in Tensorflow graph.
:return: A tuple (graph, input_name, output_name) where graph is the TF graph
corresponding to the Keras model's inference subgraph, input_name is the name of the
Keras model's input tensor, and output_name is the name of the Keras model's output tensor.
"""
keras_backend = K.backend()
assert keras_backend == "tensorflow", \
"Only tensorflow-backed Keras models are supported, tried to load Keras model " \
"with backend %s." % (keras_backend)
with graph.as_default():
K.set_learning_phase(0) # Inference phase
model = load_model(self.getModelFile())
out_op_name = tfx.op_name(model.output, graph)
stripped_graph = tfx.strip_and_freeze_until([out_op_name], graph, sess,
return_graph=True)
return stripped_graph, model.input.name, model.output.name
def executeKerasInceptionV3(image_df, uri_col="filePath"):
"""
Apply Keras InceptionV3 Model on input DataFrame.
:param image_df: Dataset. contains a column (uri_col) for where the image file lives.
:param uri_col: str. name of the column indicating where each row's image file lives.
:return: ({str => np.array[float]}, {str => (str, str, float)}).
image file uri to prediction probability array,
image file uri to top K predictions (class id, class description, probability).
"""
K.set_learning_phase(0)
model = InceptionV3(weights="imagenet")
values = {}
topK = {}
for row in image_df.select(uri_col).collect():
raw_uri = row[uri_col]
image = loadAndPreprocessKerasInceptionV3(raw_uri)
values[raw_uri] = model.predict(image)
topK[raw_uri] = decode_predictions(values[raw_uri], top=5)[0]
return values, topK
def test_deep_interpreter_cnn(self):
K.set_learning_phase(0)
with DeepInterpreter(session=K.get_session()) as di:
# 1. Load the persisted model
# 2. Retrieve the input tensor from the loaded model
retrieved_model = self._load('skater/tests/pre_trained_models/mnist_cnn/model_mnist_cnn_epoch_3')
input_tensor = retrieved_model.layers[0].input
output_tensor = retrieved_model.layers[-2].output
# 3. We will using the last dense layer(pre-softmax) as the output layer
# 4. Instantiate a model with the new input and output tensor
new_model = Model(inputs=input_tensor, outputs=output_tensor)
target_tensor = new_model(input_tensor)
xs = self.x_test[0:2]
ys = self.y_test[0:2]
relevance_scores_elrp = di.explain('elrp', target_tensor * ys, input_tensor, xs, use_case='image')
relevance_scores_ig = di.explain('ig', target_tensor * ys, input_tensor, xs, use_case='image')
self.assertEquals(relevance_scores_elrp.shape, (2, 28, 28, 1))
self.assertEquals(relevance_scores_ig.shape, (2, 28, 28, 1))
def VGG16_predict(test_data_dir, weigths_dir, N_classes):
K.set_learning_phase(False)
# build model
model = VGG16_Model(img_rows=128, img_cols=128, train=False)
# load weights
model.load_weights(weigths_dir)
# load test data
test, test_labels = load_image(test_data_dir, N_classes, one_hot=False)
print("Test images loaded.")
# predcit labels of test images with new weights
probs = model.predict(test)
print("Labels predicted.")
# calculate accuracy
predicted_labels = []
for i in range(np.shape(test_labels)[0]):
cls_prob = probs[i]
predicted_labels.append(np.argmax(cls_prob))
acc = np.mean(np.cast['float32'](np.equal(test_labels, predicted_labels)))
print("accuracy on test: " + str(acc))
end = timer()
print("Total time: ", end - start)
def __init__(self, model, fn_A, fn_B, batch_size, perceptual_loss):
K.set_learning_phase(1)
assert batch_size % 2 == 0, "batch_size must be an even number"
self.batch_size = batch_size
self.model = model
self.use_lsgan = True
self.use_mixup = True
self.mixup_alpha = 0.2
self.use_perceptual_loss = perceptual_loss
self.use_mask_refinement = False #OPTIONAL After 15k iteration**
self.lrD = 1e-4 # Discriminator learning rate
self.lrG = 1e-4 # Generator learning rate
generator = GANTrainingDataGenerator(self.random_transform_args, 220, 6, 2)
self.train_batchA = generator.minibatchAB(fn_A, batch_size)
self.train_batchB = generator.minibatchAB(fn_B, batch_size)
self.avg_counter = self.errDA_sum = self.errDB_sum = self.errGA_sum = self.errGB_sum = 0
self.setup()
def train(name, resume):
# paths
log_path = "logs/{}.json".format(name)
out_path = "snapshots/" + name + ".{epoch:06d}.h5"
echo('log path', log_path)
echo('out path', out_path)
lib.log.info(log_path, {'_commandline': {'name': name, 'resume': resume}})
# init
echo('train', (name, resume))
session = tf.Session('')
K.set_session(session)
K.set_learning_phase(1)
# dataset
echo('dataset loading...')
(x_train, y_train), (x_test, y_test) = dataset.load()
# model building
echo('model building...')
model = lib.model.build()
model.summary()
if resume:
echo('Resume Learning from {}'.format(resume))
model.load_weights(resume, by_name=True)
# training
echo('start learning...')
callbacks = [
lib.log.JsonLog(log_path),
keras.callbacks.ModelCheckpoint(out_path, monitor='val_loss', save_weights_only=True)
]
model.fit(x_train, y_train, batch_size=30, epochs=10,
callbacks=callbacks,
validation_data=(x_test, y_test))
def gen_model(name, license, model, model_file, version=VERSION, featurize=True):
g = tf.Graph()
with tf.Session(graph=g) as session:
K.set_learning_phase(0)
inTensor = tf.placeholder(dtype=tf.string, shape=[], name="%s_input" % name)
decoded = tf.decode_raw(inTensor, tf.uint8)
imageTensor = tf.to_float(
tf.reshape(
decoded,
shape=[
1,
model.inputShape()[0],
model.inputShape()[1],
3]))
m = model.model(preprocessed=model.preprocess(imageTensor), featurize=featurize)
outTensor = tf.to_double(tf.reshape(m.output, [-1]), name="%s_sparkdl_output__" % name)
gdef = tfx.strip_and_freeze_until([outTensor], session.graph, session, False)
g2 = tf.Graph()
with tf.Session(graph=g2) as session:
tf.import_graph_def(gdef, name='')
filename = "sparkdl-%s_%s.pb" % (name, version)
print('writing out ', filename)
tf.train.write_graph(g2.as_graph_def(), logdir="./", name=filename, as_text=False)
with open("./" + filename, "r") as f:
h = sha256(f.read()).digest()
base64_hash = b64encode(h)
print('h', base64_hash)
model_file.write(indent(
scala_template % {
"license": license,
"name": name,
"height": model.inputShape()[0],
"width": model.inputShape()[1],
"filename": filename,
"base64": base64_hash},2))
return g2
def main(video_path, output_path):
try:
os.mkdir('./img')
except OSError:
pass
K.set_learning_phase(0) # make sure its testing mode
# face_cascade = cv2.CascadeClassifier('lbpcascade_frontalface_improved.xml')
detector = MTCNN()
# load model and weights
img_size = 64
stage_num = [3, 3, 3]
lambda_local = 1
lambda_d = 1
img_idx = 0
detected = '' #make this not local variable
time_detection = 0
time_network = 0
time_plot = 0
skip_frame = 5 # every 5 frame do 1 detection and network forward propagation
ad = 0.6
#Parameters
num_capsule = 3
dim_capsule = 16
routings = 2
stage_num = [3, 3, 3]
lambda_d = 1
num_classes = 3
image_size = 64
num_primcaps = 7 * 3
m_dim = 5
S_set = [num_capsule, dim_capsule, routings, num_primcaps, m_dim]
model1 = FSA_net_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
model2 = FSA_net_Var_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
num_primcaps = 8 * 8 * 3
S_set = [num_capsule, dim_capsule, routings, num_primcaps, m_dim]
model3 = FSA_net_noS_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
print('Loading models ...')
weight_file1 = '../pre-trained/300W_LP_models/fsanet_capsule_3_16_2_21_5/fsanet_capsule_3_16_2_21_5.h5'
model1.load_weights(weight_file1)
print('Finished loading model 1.')
weight_file2 = '../pre-trained/300W_LP_models/fsanet_var_capsule_3_16_2_21_5/fsanet_var_capsule_3_16_2_21_5.h5'
model2.load_weights(weight_file2)
print('Finished loading model 2.')
weight_file3 = '../pre-trained/300W_LP_models/fsanet_noS_capsule_3_16_2_192_5/fsanet_noS_capsule_3_16_2_192_5.h5'
model3.load_weights(weight_file3)
print('Finished loading model 3.')
inputs = Input(shape=(64, 64, 3))
x1 = model1(inputs) #1x1
x2 = model2(inputs) #var
x3 = model3(inputs) #w/o
avg_model = Average()([x1, x2, x3])
model = Model(inputs=inputs, outputs=avg_model)
# capture video
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1024 * 1)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 768 * 1)
print('Start detecting pose ...')
detected_pre = []
while True:
# get video frame
ret, input_img = cap.read()
img_idx = img_idx + 1
img_h, img_w, _ = np.shape(input_img)
if img_idx == 1 or img_idx % skip_frame == 0:
time_detection = 0
time_network = 0
time_plot = 0
# detect faces using LBP detector
gray_img = cv2.cvtColor(input_img, cv2.COLOR_BGR2GRAY)
# detected = face_cascade.detectMultiScale(gray_img, 1.1)
detected = detector.detect_faces(input_img)
if len(detected_pre) > 0 and len(detected) == 0:
detected = detected_pre
faces = np.empty((len(detected), img_size, img_size, 3))
input_img = draw_results_mtcnn(detected, input_img, faces, ad,
img_size, img_w, img_h, model,
time_detection, time_network,
time_plot)
cv2.imwrite('img/' + str(img_idx) + '.png', input_img)
else:
input_img = draw_results_mtcnn(detected, input_img, faces, ad,
img_size, img_w, img_h, model,
time_detection, time_network,
time_plot)
if len(detected) > len(detected_pre) or img_idx % (skip_frame *
3) == 0:
detected_pre = detected
key = cv2.waitKey(1)
def fam(train_i, train_o, test_i, test_o):
sess = tf.Session()
K.set_session(sess)
K.set_learning_phase(1)
batch_size = 60
nb_classes = len(MOD)
nb_epoch = 20
img_rows, img_cols = 2 * P * L, 2 * Np
nb_filters = 96
nb_pool = 2
X_train, Y_train = shuffle_in_unison_inplace(np.array(train_i),
np.array(train_o))
model = Sequential()
model.add(
Convolution2D(64,
11,
11,
subsample=(2, 2),
input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(128, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes, init='normal'))
model.add(Activation('softmax', name="out"))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
"""
datagen = ImageDataGenerator(
#featurewise_center=True,
#featurewise_std_normalization=True,
rotation_range=20,
#width_shift_range=0.3,
#height_shift_range=0.3,
#zoom_range=[0,1.3],
horizontal_flip=True,
vertical_flip=True)
datagen.fit(X_train)
model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size,shuffle=True),
samples_per_epoch=len(X_train), nb_epoch=5,verbose=1,validation_data=(test_i[0], test_o[0]))
"""
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=1,
shuffle=True,
validation_data=(test_i[0], test_o[0]))
for s in range(len(test_i)):
if len(test_i[s]) == 0:
continue
X_test = test_i[s]
Y_test = test_o[s]
score = model.evaluate(X_test, Y_test, verbose=0)
print("SNR", SNR[s], "Test accuracy:", score[1])
K.set_learning_phase(0)
config = model.get_config()
weights = model.get_weights()
new_model = Sequential.from_config(config)
new_model.set_weights(weights)
export_path = "/tmp/fam"
export_version = 1
labels_tensor = tf.constant(MOD)
saver = tf.train.Saver(sharded=True)
model_exporter = exporter.Exporter(saver)
signature = exporter.classification_signature(
input_tensor=new_model.input,
classes_tensor=labels_tensor,
scores_tensor=new_model.output)
model_exporter.init(sess.graph.as_graph_def(),
default_graph_signature=signature)
model_exporter.export(export_path, tf.constant(export_version), sess)
import tensorflow as tf
from tensorflow.python.framework.graph_util import convert_variables_to_constants
from keras.models import load_model
import keras.backend as K
def freeze(session, keep_var_names = None, output_names = None, clear_devices = True):
graph = session.graph() #sessoin graph is like the model
with graph.as_default(): #get default value
freeze_var_names = list(set(v.op.name for v in tf.global_variables()).difference(keep_var_names or []))
#list of all variable names in graph
output_names = output_names or [] #operations for final frozen graph
output_names +=[v.op.name for v in tf.global_variables()]
input_graph_def = graph.as_graph_def() #definition is like shape of the graph
if clear_devices:
for node in input_graph_def.node:
node.device = '' #set to empty line
frozen_graph = convert_variables_to_constants(sess = session, input_graph = input_graph_def , output_node_names = output_names,
variable_names_whitelist = freeze_var_names) #converts variable nodes in to constants
return frozen_graph
model = load_model(filepath= 'Image-classifier.h5')
K.set_learning_phase(1) #helps eliminate problem
frozen_graph = freeze_graph(K.get_session(),output_names = [model.output.op.name])
tf.train.write(frozen_graph, '-', 'Image-classifier.pb',False) #set it to binary file with False
#write the file
import argparse
import os
import shutil
import keras
import tensorflow
from keras import backend as K
from keras.models import Sequential
from tensorflow.contrib.session_bundle import exporter
from tensorflow.core.protobuf import saver_pb2
from tensorflow.python.framework import graph_io
from tensorflow.python.tools import freeze_graph
K.set_learning_phase(0) # all new operations will be in test mode from now on
def export_model_to_tensorflow(path_to_trained_keras_model: str):
print("Loading model for exporting to Protocol Buffer format...")
model = keras.models.load_model(path_to_trained_keras_model)
sess = K.get_session()
# serialize the model and get its weights, for quick re-building
config = model.get_config()
weights = model.get_weights()
# re-build a model where the learning phase is now hard-coded to 0
new_model = Sequential.from_config(config)
new_model.set_weights(weights)
export_path = os.path.abspath(os.path.join(
def _main():
# Set testing mode (dropout/batchnormalization)
K.set_learning_phase(TEST_PHASE)
# Generate testing data
test_datagen = utils.DroneDataGenerator(rescale=1. / 255)
test_generator = test_datagen.flow_from_directory(
FLAGS.test_dir,
shuffle=False,
color_mode=FLAGS.img_mode,
target_size=(FLAGS.img_width, FLAGS.img_height),
crop_size=(FLAGS.crop_img_height, FLAGS.crop_img_width),
batch_size=FLAGS.batch_size)
# Load json and create model
json_model_path = os.path.join(FLAGS.experiment_rootdir,
FLAGS.json_model_fname)
model = utils.jsonToModel(json_model_path)
# Load weights
weights_load_path = os.path.join(FLAGS.experiment_rootdir,
FLAGS.weights_fname)
try:
model.load_weights(weights_load_path)
print("Loaded model from {}".format(weights_load_path))
except:
print("Impossible to find weight path. Returning untrained model")
# Compile model
model.compile(loss='mse', optimizer='adam')
# Get predictions and ground truth
n_samples = test_generator.samples
nb_batches = int(np.ceil(n_samples / FLAGS.batch_size))
predictions, ground_truth, t = utils.compute_predictions_and_gt(
model, test_generator, nb_batches, verbose=1)
# Param t. t=1 steering, t=0 collision
t_mask = t == 1
# ************************* Steering evaluation ***************************
# Predicted and real steerings
pred_steerings = predictions[t_mask, 0]
real_steerings = ground_truth[t_mask, 0]
# Compute random and constant baselines for steerings
random_steerings = random_regression_baseline(real_steerings)
constant_steerings = constant_baseline(real_steerings)
# Create dictionary with filenames
dict_fname = {
'test_regression.json': pred_steerings,
'random_regression.json': random_steerings,
'constant_regression.json': constant_steerings
}
steering_dict = []
# Evaluate predictions: EVA, residuals, and highest errors
for fname, pred in dict_fname.items():
abs_fname = os.path.join(FLAGS.experiment_rootdir, fname)
if fname == 'test_regression.json':
steering_dict.append(
evaluate_regression(pred, real_steerings, abs_fname))
else:
evaluate_regression(pred, real_steerings, abs_fname)
# Write predicted and real steerings
dict_test = {
'pred_steerings': pred_steerings.tolist(),
'real_steerings': real_steerings.tolist()
}
#utils.write_to_file(dict_test,os.path.join(FLAGS.experiment_rootdir,
#` 'predicted_and_real_steerings.json'))
# *********************** Collision evaluation ****************************
# Predicted probabilities and real labels
pred_prob = predictions[~t_mask, 1]
pred_labels = np.zeros_like(pred_prob)
pred_labels[pred_prob >= 0.5] = 1
real_labels = ground_truth[~t_mask, 1]
# Compute random, weighted and majorirty-class baselines for collision
random_labels = random_classification_baseline(real_labels)
# Create dictionary with filenames
dict_fname = {
'test_classification.json': pred_labels,
'random_classification.json': random_labels
}
collision_dict = []
# Evaluate predictions: accuracy, precision, recall, F1-score, and highest errors
for fname, pred in dict_fname.items():
abs_fname = os.path.join(FLAGS.experiment_rootdir, fname)
if fname == 'test_classification.json':
collision_dict.append(
evaluate_classification(pred_prob, pred, real_labels,
abs_fname))
else:
evaluate_classification(pred_prob, pred, real_labels, abs_fname)
# Write predicted probabilities and real labels
dict_test = {
'pred_probabilities': pred_prob.tolist(),
'real_labels': real_labels.tolist()
}
#utils.write_to_file(dict_test,os.path.join(FLAGS.experiment_rootdir,
# 'predicted_and_real_labels.json'))
return steering_dict, collision_dict
def main(self, name, opts):
logging.basicConfig(filename=opts.log_file,
format='%(levelname)s (%(asctime)s): %(message)s')
log = logging.getLogger(name)
if opts.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
log.debug(opts)
if opts.seed is not None:
np.random.seed(opts.seed)
if not opts.model_files:
raise ValueError('No model files provided!')
log.info('Loading model ...')
K.set_learning_phase(0)
model = mod.load_model(opts.model_files, log=log.info)
weight_layer, act_layer = mod.get_first_conv_layer(model.layers, True)
log.info('Using activation layer "%s"' % act_layer.name)
log.info('Using weight layer "%s"' % weight_layer.name)
try:
dna_idx = model.input_names.index('dna')
except BaseException:
raise IOError('Model is not a valid DNA model!')
fun_outputs = to_list(act_layer.output)
if opts.store_preds:
fun_outputs += to_list(model.output)
fun = K.function([to_list(model.input)[dna_idx]], fun_outputs)
log.info('Reading data ...')
if opts.store_outputs or opts.store_preds:
output_names = model.output_names
else:
output_names = None
data_reader = mod.DataReader(
output_names=output_names,
use_dna=True,
dna_wlen=to_list(model.input_shape)[dna_idx][1]
)
nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample)
data_reader = data_reader(opts.data_files,
nb_sample=nb_sample,
batch_size=opts.batch_size,
loop=False,
shuffle=False)
meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'],
nb_sample=nb_sample,
batch_size=opts.batch_size,
loop=False,
shuffle=False)
out_file = h5.File(opts.out_file, 'w')
out_group = out_file
weights = weight_layer.get_weights()
out_group['weights/weights'] = weights[0]
out_group['weights/bias'] = weights[1]
def h5_dump(path, data, idx, dtype=None, compression='gzip'):
if path not in out_group:
if dtype is None:
dtype = data.dtype
out_group.create_dataset(
name=path,
shape=[nb_sample] + list(data.shape[1:]),
dtype=dtype,
compression=compression
)
out_group[path][idx:idx+len(data)] = data
log.info('Computing activations')
progbar = ProgressBar(nb_sample, log.info)
idx = 0
for data in data_reader:
if isinstance(data, tuple):
inputs, outputs, weights = data
else:
inputs = data
if isinstance(inputs, dict):
inputs = list(inputs.values())
batch_size = len(inputs[0])
progbar.update(batch_size)
if opts.store_inputs:
for i, name in enumerate(model.input_names):
h5_dump('inputs/%s' % name,
dna.onehot_to_int(inputs[i]), idx)
if opts.store_outputs:
for name, output in outputs.items():
h5_dump('outputs/%s' % name, output, idx)
fun_eval = fun(inputs)
act = fun_eval[0]
if opts.act_wlen:
delta = opts.act_wlen // 2
ctr = act.shape[1] // 2
act = act[:, (ctr-delta):(ctr+delta+1)]
if opts.act_fun:
if opts.act_fun == 'mean':
act = act.mean(axis=1)
elif opts.act_fun == 'wmean':
weights = linear_weights(act.shape[1])
act = np.average(act, axis=1, weights=weights)
elif opts.act_fun == 'max':
act = act.max(axis=1)
else:
raise ValueError('Invalid function "%s"!' % (opts.act_fun))
h5_dump('act', act, idx)
if opts.store_preds:
preds = fun_eval[1:]
for i, name in enumerate(model.output_names):
h5_dump('preds/%s' % name, preds[i].squeeze(), idx)
for name, value in next(meta_reader).items():
h5_dump(name, value, idx)
idx += batch_size
progbar.close()
out_file.close()
log.info('Done!')
return 0
from keras.utils import layer_utils
from keras.utils.data_utils import get_file
from keras.applications.imagenet_utils import preprocess_input
#from IPython.display import SVG
#from keras.utils.vis_utils import model_to_dot
#from keras.utils import plot_model
#from resnets_utils import *
#from keras.initializers import glorot_uniform
#import scipy.misc
from matplotlib.pyplot import imshow
from keras.preprocessing.image import ImageDataGenerator
from keras.preprocessing import image
import keras.backend as K
K.set_image_data_format('channels_last')
K.set_learning_phase(1)
#import os
#from scipy.misc import imread, imsave, imresize
#from sklearn.utils import shuffle
#from sklearn.cross_validation import train_test_split
#from sklearn import preprocessing
#import glob
#import scipy.misc
import time
#from tkinter import filedialog
#from tkinter import *
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
tic = time.time()
size = 299
model = load_model('DR_model.h5')
def mal_agent(X_shard, Y_shard, mal_data_X, mal_data_Y, t, gpu_id, return_dict,
mal_visible, X_test, Y_test):
args = gv.args
shared_weights = np.load(gv.dir_name + 'global_weights_t%s.npy' % t)
holdoff_flag = 0
if 'holdoff' in args.mal_strat:
print('Checking holdoff')
if 'single' in args.mal_obj:
target, target_conf, actual, actual_conf = mal_eval_single(
mal_data_X, mal_data_Y, shared_weights)
if target_conf > 0.8:
print('Holding off')
holdoff_flag = 1
# tf.reset_default_graph()
K.set_learning_phase(1)
print('Malicious Agent on GPU %s' % gpu_id)
# set enviornment
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
if args.dataset == 'census':
x = tf.placeholder(shape=(None, gv.DATA_DIM), dtype=tf.float32)
y = tf.placeholder(dtype=tf.int64)
else:
x = tf.placeholder(shape=(None, gv.IMAGE_ROWS, gv.IMAGE_COLS,
gv.NUM_CHANNELS),
dtype=tf.float32)
y = tf.placeholder(dtype=tf.int64)
if 'MNIST' in args.dataset:
agent_model = model_mnist(type=args.model_num)
elif args.dataset == 'CIFAR-10':
agent_model = cifar_10_model()
elif args.dataset == 'census':
agent_model = census_model_1()
logits = agent_model(x)
prediction = tf.nn.softmax(logits)
eval_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
logits=logits))
config = tf.ConfigProto(gpu_options=gv.gpu_options)
# config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
if t >= args.mal_delay and holdoff_flag == 0:
if args.mal_obj == 'all':
final_delta = mal_all_algs(x, y, logits, agent_model,
shared_weights, sess, mal_data_X,
mal_data_Y, t)
elif args.mal_obj == 'single' or 'multiple' in args.mal_obj:
final_delta, penul_delta = mal_single_algs(
x, y, logits, agent_model, shared_weights, sess, mal_data_X,
mal_data_Y, t, mal_visible, X_shard, Y_shard)
elif t < args.mal_delay or holdoff_flag == 1:
print('Delay/Hold-off')
final_delta, _ = benign_train(x, y, agent_model, logits, X_shard,
Y_shard, sess, shared_weights)
final_weights = shared_weights + final_delta
agent_model.set_weights(final_weights)
print('---Eval at mal agent---')
if 'single' in args.mal_obj:
target, target_conf, actual, actual_conf = mal_eval_single(
mal_data_X, mal_data_Y, final_weights)
print(
'Target:%s with conf. %s, Curr_pred on malicious model for iter %s:%s with conf. %s'
% (target, target_conf, t, actual, actual_conf))
elif 'multiple' in args.mal_obj:
suc_count_local = mal_eval_multiple(mal_data_X, mal_data_Y,
final_weights)
print('%s of %s targets achieved' % (suc_count_local, args.mal_num))
eval_success, eval_loss = eval_minimal(X_test, Y_test, final_weights)
return_dict['mal_success'] = eval_success
print('Malicious Agent: success {}, loss {}'.format(
eval_success, eval_loss))
write_dict = {}
# just to maintain ordering
write_dict['t'] = t + 1
write_dict['eval_success'] = eval_success
write_dict['eval_loss'] = eval_loss
file_write(write_dict, purpose='mal_eval_loss')
return_dict[str(gv.mal_agent_index)] = np.array(final_delta)
np.save(gv.dir_name + 'mal_delta_t%s.npy' % t, final_delta)
if 'auto' in args.mal_strat or 'multiple' in args.mal_obj:
penul_weights = shared_weights + penul_delta
if 'single' in args.mal_obj:
target, target_conf, actual, actual_conf = mal_eval_single(
mal_data_X, mal_data_Y, penul_weights)
print(
'Penul weights ---- Target:%s with conf. %s, Curr_pred on malicious model for iter %s:%s with conf. %s'
% (target, target_conf, t, actual, actual_conf))
elif 'multiple' in args.mal_obj:
suc_count_local = mal_eval_multiple(mal_data_X, mal_data_Y,
penul_weights)
print('%s of %s targets achieved' %
(suc_count_local, args.mal_num))
eval_success, eval_loss = eval_minimal(X_test, Y_test, penul_weights)
print('Penul weights ---- Malicious Agent: success {}, loss {}'.format(
eval_success, eval_loss))
return
def setUpClass(cls):
k.set_learning_phase(1)
# Get MNIST
(x_train, y_train), (x_test, y_test), _, _ = load_mnist()
x_train, y_train, x_test, y_test = x_train[:
NB_TRAIN], y_train[:
NB_TRAIN], x_test[:
NB_TEST], y_test[:
NB_TEST]
cls.mnist = (x_train, y_train), (x_test, y_test)
# Keras classifier
cls.classifier_k = cls._cnn_mnist_k([28, 28, 1])
cls.classifier_k.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
nb_epochs=2)
scores = cls.classifier_k._model.evaluate(x_train, y_train)
logger.info('[Keras, MNIST] Accuracy on training set: %.2f%%',
(scores[1] * 100))
scores = cls.classifier_k._model.evaluate(x_test, y_test)
logger.info('[Keras, MNIST] Accuracy on test set: %.2f%%',
(scores[1] * 100))
# Create basic CNN on MNIST using TensorFlow
cls.classifier_tf = cls._cnn_mnist_tf([28, 28, 1])
cls.classifier_tf.fit(x_train,
y_train,
nb_epochs=2,
batch_size=BATCH_SIZE)
scores = get_labels_np_array(cls.classifier_tf.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(
y_train, axis=1)) / y_train.shape[0]
logger.info('[TF, MNIST] Accuracy on training set: %.2f%%',
(acc * 100))
scores = get_labels_np_array(cls.classifier_tf.predict(x_test))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(
y_test, axis=1)) / y_test.shape[0]
logger.info('[TF, MNIST] Accuracy on test set: %.2f%%', (acc * 100))
# Create basic PyTorch model
cls.classifier_py = cls._cnn_mnist_py()
x_train, x_test = np.swapaxes(x_train, 1, 3), np.swapaxes(x_test, 1, 3)
cls.classifier_py.fit(x_train,
y_train,
nb_epochs=2,
batch_size=BATCH_SIZE)
scores = get_labels_np_array(cls.classifier_py.predict(x_train))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(
y_train, axis=1)) / y_train.shape[0]
logger.info('[PyTorch, MNIST] Accuracy on training set: %.2f%%',
(acc * 100))
scores = get_labels_np_array(cls.classifier_py.predict(x_test))
acc = np.sum(np.argmax(scores, axis=1) == np.argmax(
y_test, axis=1)) / y_test.shape[0]
logger.info('\n[PyTorch, MNIST] Accuracy on test set: %.2f%%',
(acc * 100))
def detect_image(self, image, img):
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
K.set_learning_phase(0)
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
# K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
# create blank txt file for the image
filename, file_extension = os.path.splitext(img)
# open(self.predictions_directory+filename+'.txt', 'a').close()
classnm = []
topnm = []
leftnm = []
bottomnm = []
rightnm = []
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
classnm.append("\n")
classnm.append(predicted_class)
classnm.append("\t")
box = out_boxes[i]
score = out_scores[i]
# label = '{} {:.2f}'.format(predicted_class, score)
label = predicted_class
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top - 10).astype('int32'))
left = max(0, np.floor(left - 10).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 10).astype('int32'))
right = min(image.size[0], np.floor(right + 10).astype('int32'))
print(label, (left, top), (right, bottom))
classnm.append(top)
classnm.append(",")
classnm.append(left)
classnm.append(",")
classnm.append(bottom)
classnm.append(",")
classnm.append(right)
# write bounding boxes to text file
# with open(self.predictions_directory+filename+'.txt', 'a') as f:
# f.write(label+' '+str(left)+' '+str(top)+' '+str(right)+' '+str(bottom)+'\n')
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
print(end - start)
# save images
image.save(self.predictions_directory + img)
return classnm
from numpy.random import seed
seed(1)
from sklearn.model_selection import train_test_split as tts
import logging
import plotly.plotly as py
import plotly.graph_objs as go
import matplotlib.pyplot as plt
import pandas as pd
import pydot
import keras
from keras import backend as k
k.set_learning_phase(1)
from keras.preprocessing.text import Tokenizer
from keras import initializers
from keras.optimizers import RMSprop
from keras.models import Sequential, Model
from keras.layers import Dense, LSTM, Dropout, Input, Activation, Add, Concatenate
from keras.layers.advanced_activations import LeakyReLU, PReLU
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from keras.optimizers import Adam
#keras.utils.vis_utils import plot_model
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\
level=logging.INFO)
#######################model params###########################
def run_kde(Data, Model, path):
global DECONST
sess = K.get_session()
K.set_learning_phase(False)
data, model = Data(), Model(path)
model2 = Model(path)
hidden_layer = pop(model2.model) # once to remove dense(10)
hidden_layer = pop(hidden_layer) # once to remove ReLU
#compute_optimal_sigma(sess, model, hidden_layer, data)
#MNIST SIGMA: 20
de = [
DensityEstimate(
sess,
hidden_layer,
data.train_data[np.argmax(data.train_labels, axis=1) == i],
model.image_size,
model.num_channels,
sigma=20) for i in range(10)
]
de2 = [
DensityEstimate(
sess,
hidden_layer,
data.train_data[np.argmax(data.train_labels, axis=1) == i][:100],
model.image_size,
model.num_channels,
sigma=20) for i in range(10)
]
p = tf.placeholder(
tf.float32,
(None, model.image_size, model.image_size, model.num_channels))
#print(np.log(de[0].predict(data.test_data[:10])))
#print(sess.run(rmodel.predict(p)[1], {p: data.test_data[:10]}))
#exit(0)
N = 1
print(model.model.predict(data.train_data[:N]))
print(hidden_layer.predict(data.train_data[:N]))
for i in range(10):
print(de[i].predict(data.train_data[:N]))
start_density = estimate_density_full(model, de,
data.test_data[M:M + N]) + 1e-30
print("starting density", np.log(start_density))
DECONST = -np.log(start_density)
l = np.zeros((N, 10))
#l[np.arange(N),np.random.random_integers(0,9,N)] = 1
for i in range(N):
r = np.random.random_integers(0, 9)
while r == np.argmax(data.test_labels[i]):
r = np.random.random_integers(0, 9)
l[i, r] = 1
attack1 = CarliniL2(sess,
model,
batch_size=1,
max_iterations=3000,
binary_search_steps=3,
initial_const=1.0,
learning_rate=1e-1,
targeted=True)
attack2 = CarliniL2New(sess,
model,
batch_size=1,
max_iterations=10000,
binary_search_steps=5,
initial_const=1.0,
learning_rate=1e-2,
targeted=True,
extra_loss=extra_loss(de2, np.argmax(l)))
#l = data.test_labels[:N]
#l = np.zeros((N,10))
#l[np.arange(N),1] = 1
print("RUN PHASE 1")
adv = attack1.attack(data.test_data[M:M + N], l)
print(
'mean distortion',
np.mean(
np.sum((adv - data.test_data[M:M + N])**2, axis=(1, 2, 3))**.5))
print("RUN PHASE 2")
adv = attack2.attack(data.test_data[M:M + N], adv, l)
np.save("/tmp/q" + str(M), adv)
#adv = np.load("/tmp/qq.npy")
print(
'labels',
np.mean(np.argmax(sess.run(model.predict(p), {p: adv}), axis=1) == l))
print(
'mean distortion',
np.mean(
np.sum((adv - data.test_data[M:M + N])**2, axis=(1, 2, 3))**.5))
a = estimate_density_full(model, de, data.test_data[M:M + N]) + 1e-30
b = estimate_density_full(model, de, adv) + 1e-30
show(adv)
print('de of test', np.mean(np.log(a)))
print('de of adv', np.mean(np.log(b)))
print('better ratio', np.mean(np.array(a) > np.array(b)))
exit(0)
#density = gaussian_kde(np.array(np.log(a))-np.array(np.log(b)))
#density_a = gaussian_kde(np.log(a))
#density_b = gaussian_kde(np.log(b))
xs = np.linspace(-25, 25, 200)
fig = plt.figure(figsize=(4, 3))
fig.subplots_adjust(bottom=0.17, left=.15, right=.85)
plt.xlabel('log(KDE(valid))-log(KDE(adversarial))')
plt.ylabel('Occurrances')
#plt.hist(np.log(a),100)
#plt.hist(np.log(b),100)
plt.hist(np.log(a) - np.log(b), 100)
#plt.hist(np.array(np.log(a))-np.array(np.log(b)),100)
#a = plt.plot(xs,density_a(xs), 'r--',color='blue', label='Valid')
#b = plt.plot(xs,density_b(xs), color='red', label='Adversarial')
#plt.plot(xs,density(xs))
#plt.legend(handles=[a[0], b[0]])
pp = PdfPages('/tmp/a.pdf')
plt.savefig(pp, format='pdf')
pp.close()
plt.show()
import numpy as np import matplotlib.pyplot as plt from keras.layers import Input from keras.models import Model from keras.utils import multi_gpu_model from chinese_ocr.train.train import random_uniform_num, get_session, get_model from chinese_ocr.densenet_common import densenet from chinese_ocr.densenet_common.densenet_model import data_generator from chinese_ocr.train.synthtext_config import SynthtextConfig from chinese_ocr.densenet_common.dataset_format import DataSetSynthtext from predict_tf_tool import DensenetOcr reload(densenet) K.set_learning_phase(0) dataset_path = "/media/chenhao/study/code/other/out" class_id_file = "char_7476.txt" train_label_name = "label_train.txt" val_label_name = "label_val.txt" test_label_name = "label_test.txt" dataset_format = 0 sub_img_folder = "default" data_test = DataSetSynthtext() data_test.load_data(class_id_file, dataset_path, test_label_name, subset=sub_img_folder) # label_list = data.load_instance(33) # print(label_list) data_test.prepare()
def train_GAN(batch_size, epochs, img_classifier, dataset, eeg_embs, d_optim,
g_optim):
"""
Function to train the GAN and show the performance as it trains.
Function first instantiates and compiles the generator, discriminator, GAN
and parameters such as the noise dimension. It then uses nested for-loops
to train in batches. For each batch we see fake and real combinations of
labels, embeddings, and images being created. These are used to train all
models. At the end of each epoch the performance metrics are printed and
we see what the generator is currently producing, given the EEG embeddings.
input: batch_size: mini-batch size.
input: epochs: number of epochs.
input: img_classifier: pretrained classifier to use in the auxiliary feature
of the discriminator.
input: dataset: dataset used, in the form [labels, EEG embeddings, images].
input: eeg_embs: a list of the 20 eeg embeddings, one for each class.
input: d_optim: the optimizer used to train the discriminator.
input: g_optim: the optimizer used to train the generator.
"""
K.set_learning_phase(False)
all_labels, all_eeg_embs, all_real_images = dataset
input_noise_dim = 126
feature_encoding_dim = 126
tot_num_images = all_real_images.shape[0]
print('tot images: ', tot_num_images)
num_classes = 20
d = build_discriminator((64, 64), img_classifier)
d.trainable = True
d.compile(loss=['binary_crossentropy', 'categorical_crossentropy'],
optimizer=d_optim)
g = build_generator(input_noise_dim, feature_encoding_dim)
g.compile(loss='categorical_crossentropy', optimizer=g_optim)
d_on_g = build_gan(input_noise_dim, feature_encoding_dim, g, d)
d_on_g.compile(loss=['binary_crossentropy', 'categorical_crossentropy'],
optimizer=g_optim)
num_batches = int(tot_num_images / batch_size)
print("Number of batches:", num_batches)
for epoch in range(1, epochs + 1):
for index in range(num_batches):
# generate noise from a uniform distribution
noise = np.random.uniform(-1, 1, (batch_size, input_noise_dim))
# get some random embedding and label pairings from the dataset (these are
# used these to generate the fake images)
rand_indices = np.random.randint(0, tot_num_images, batch_size)
fake_labels = all_labels[rand_indices]
fake_eeg_embs = all_eeg_embs[rand_indices]
# get real images and corresponding labels and eeg embeddings
real_labels_batch = all_labels[index * batch_size:(index + 1) *
batch_size]
real_images_batch = all_real_images[index *
batch_size:(index + 1) *
batch_size]
# generate fake images using the generator
generated_images = g.predict([noise, fake_eeg_embs], verbose=0)
# train discriminator on real images (target is 1 - meaning real images)
d_loss_real = d.train_on_batch(
real_images_batch,
[np.array([1] * batch_size),
np.array(real_labels_batch)])
# train discriminator on fake images (target is 0 - meaning fake images)
d_loss_fake = d.train_on_batch(generated_images, [
np.array([0] * batch_size),
np.array(fake_labels).reshape(batch_size, num_classes)
])
d_loss = (d_loss_fake[0] + d_loss_real[0]) * 0.5
d.trainable = False
d.compile(loss=['binary_crossentropy', 'categorical_crossentropy'],
optimizer=d_optim)
# train the GAN. The generator will generate fake images (target is 1 - meaning real images)
g_loss = d_on_g.train_on_batch([noise, fake_eeg_embs], [
np.array([1] * batch_size),
np.array(fake_labels).reshape(batch_size, num_classes)
])
d.trainable = True
d.compile(loss=['binary_crossentropy', 'categorical_crossentropy'],
optimizer=d_optim)
print(
"Epoch {0}: discriminator_loss : {1:.4f}, generator_loss : {2:.4f}."
.format(epoch + 1, d_loss, g_loss[0]))
print_results(g, eeg_embs)
def train_generation():
datagen = image.ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=0.3,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.,
zoom_range=0.2,
channel_shift_range=0.,
fill_mode='nearest',
cval=0.0,
horizontal_flip=False,
vertical_flip=False,
rescale=1. / 255,
preprocessing_function=None,
# data_format=K.image_data_format(),
)
train_generator = datagen.flow_from_directory(
# '/Users/imperatore/tmp/num_ocr', # this is the target directory
r'F:\number_ok1', # this is the target directory
target_size=(48, 48), # all images will be resized to 48*40
batch_size=256,
class_mode='categorical',
color_mode='grayscale')
validation_generator = datagen.flow_from_directory(
# '/Users/imperatore/tmp/nums_classed',
r'F:\number_ok1',
target_size=(48, 48),
batch_size=128,
class_mode='categorical',
color_mode='grayscale')
num_class = 10
input_tensor = Input((48, 48, 1))
def resnet(input_tensor, units=32, kernel_size=(3, 3)):
x = input_tensor
for i in range(3):
x = res_block(x, units, kernel_size=kernel_size)
x = Dropout(drop)(x)
x = MaxPool2D(pool_size=(2, 2))(x)
return x
def conv2d_bn(x,
units,
kernel_size,
strides=(1, 1),
padding='same',
name=None):
if name is not None:
bn_name = name + '_bn'
conv_name = name + '_conv'
else:
bn_name = None
conv_name = None
x = Conv2D(units,
kernel_size,
padding=padding,
strides=strides,
activation='relu',
name=conv_name)(x)
x = BatchNormalization(axis=3, name=bn_name)(x)
return x
def res_block(inpt,
units,
kernel_size,
strides=(1, 1),
with_conv_shortcut=False):
x = conv2d_bn(inpt,
units=units,
kernel_size=kernel_size,
strides=strides,
padding='same')
x = conv2d_bn(x, units=units, kernel_size=kernel_size, padding='same')
if with_conv_shortcut:
shortcut = conv2d_bn(inpt,
units=units,
strides=strides,
kernel_size=kernel_size)
x = add([x, shortcut])
return x
else:
x = add([x, inpt])
return x
drop = 0.1
input_tensor = Input((48, 48, 1))
x = input_tensor
x = Conv2D(32,
kernel_size=(3, 3),
padding='same',
strides=(1, 1),
name=None)(x)
x = Activation(activation='relu')(x)
# x = BatchNormalization(axis=3, name=None)(x)
x = MaxPool2D(pool_size=(2, 2))(x)
x = Dropout(drop)(x)
x = Conv2D(64,
kernel_size=(3, 3),
padding='same',
strides=(1, 1),
name=None)(x)
x = Activation(activation='relu')(x)
# x = BatchNormalization(axis=3, name=None)(x)
x = MaxPool2D(pool_size=(2, 2))(x)
x = Dropout(drop)(x)
# x = Conv2D(256, kernel_size=(3, 3), padding='same', strides=(1, 1), name=None)(x)
# x = Activation(activation='relu')(x)
# # x = BatchNormalization(axis=3, name=None)(x)
# x = MaxPool2D(pool_size=(2, 2))(x)
# x = Dropout(drop)(x)
x = Conv2D(512,
kernel_size=(3, 3),
padding='same',
strides=(1, 1),
name=None)(x)
x = Activation(activation='relu')(x)
# x = BatchNormalization(axis=3, name=None)(x)
x = MaxPool2D(pool_size=(2, 2))(x)
x = Dropout(drop)(x)
# x = resnet(x)
x = Flatten()(x)
x = Dense(1000, kernel_initializer='he_normal')(x)
# x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(drop)(x)
x = Dense(10, kernel_initializer='he_normal')(x)
model = Model(inputs=input_tensor, outputs=x)
print(model.layers)
kb.set_learning_phase(0)
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['accuracy'])
# model.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])
print('\n'.join([str(tmp) for tmp in model.layers]))
print('model length: %s' % len(model.layers))
early_stopping = EarlyStopping(monitor='val_loss', patience=30)
model.fit_generator(train_generator,
steps_per_epoch=512,
epochs=20,
validation_data=validation_generator,
nb_val_samples=100,
verbose=True,
callbacks=[early_stopping])
model.save('cnn3_gen_1.4.h5'
) # always save your weights after training or during training
import numpy as np
from keras.datasets import mnist
import keras
import matplotlib.pyplot as plt
from keras import backend
from keras.models import load_model
import tensorflow as tf
import sys
sys.path.append('/home/labadmin/src/cleverhans')
import cleverhans
from cleverhans.attacks import FastGradientMethod
from cleverhans.utils_keras import KerasModelWrapper
from keras.optimizers import Adam
###################################### Trained Model ############################################
backend.set_learning_phase(False)
keras_model = load_model(
'/home/labadmin/Desktop/iccad_2018/fgsm_retrain/simple_nn_iccad.h5')
#################################### Importing the dataset ######################################
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
########################### Set TF random seed to improve reproducibility #######################
tf.set_random_seed(1234)
def setKerasOptions():
K._LEARNING_PHASE = tf.constant(0)
K.set_learning_phase(False)
K.set_learning_phase(0)
K.set_image_data_format('channels_last')
adam = Adam(lr=0.0001, clipvalue=0.5)
print("Loaded model from disk")
y_pred = model.get_layer('softmax').output
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=adam)
return model
mode = 'train'
load_previous = raw_input('Type yes/no if you want to load previous model: ')
print mode
K.set_learning_phase(1) # all new operations will be in train mode from now on
data_path = train_path
lab_file = train_lab_file
data_gen = DataGenerator(minibatch_size=minibatch_size,
img_dim=img_dim,
maxlen=maxlen,
val_split=val_split,
nb_classes=nb_classes,
data_path=data_path,
lab_file=lab_file)
# Resume training.
if load_previous == 'yes':
model = load_model()
def deprocess_image(x):
# Util function to convert a tensor into a valid image.
if K.image_data_format() == 'channels_first':
x = x.reshape((3, x.shape[2], x.shape[3]))
x = x.transpose((1, 2, 0))
else:
x = x.reshape((x.shape[1], x.shape[2], 3))
x /= 2.
x += 0.5
x *= 255.
x = np.clip(x, 0, 255).astype('uint8')
return x
K.set_learning_phase(0)
# Build the InceptionV3 network with our placeholder.
# The model will be loaded with pre-trained ImageNet weights.
model = inception_v3.InceptionV3(weights='imagenet',
include_top=False)
dream = model.input
print('Model loaded.')
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])
# Define the loss.
loss = K.variable(0.)
for layer_name in settings['features']:
# Add the L2 norm of the features of a layer to the loss.
from PIL import Image
import os
import re
##########
#target_model_time = "1527999111"
#target_layer_name = "conv2d_60"
##########
target_model_time = "1528320726"
target_layer_name = "conv2d_116"
##########
backend.set_learning_phase(1)
output_dir = "output2"
output_map_dir = "output_map"
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if not os.path.exists(output_dir+"/"+target_model_time):
os.mkdir(output_dir+"/"+target_model_time)
if not os.path.exists(output_map_dir):
os.mkdir(output_map_dir)
model = load_model("model/"+target_model_time+".h5")
count_total_1 = 0
count_correct_1 = 0
accuracy_1 = 0.0
def render_dream(avatar):
K.clear_session()
def eval_loss_and_grads(x):
outs = fetch_loss_and_grads([x])
loss_value = outs[0]
grad_values = outs[1]
return loss_value, grad_values
def gradient_ascent(x, iterations, step, max_loss=None):
for i in range(iterations):
loss_value, grad_values = eval_loss_and_grads(x)
if max_loss is not None and loss_value > max_loss:
break
x += step * grad_values
return x
def preprocess_image(image_path):
# Util function to open, resize and format pictures
# into appropriate tensors.
img = img_to_array(image_path)
img = np.expand_dims(img, axis=0)
img = inception_v3.preprocess_input(img)
return img
def deprocess_image(x):
# Util function to convert a tensor into a valid image.
if K.image_data_format() == 'channels_first':
x = x.reshape((3, x.shape[2], x.shape[3]))
x = x.transpose((1, 2, 0))
else:
x = x.reshape((x.shape[1], x.shape[2], 3))
x /= 2.
x += 0.5
x *= 255.
x = np.clip(x, 0, 255).astype('uint8')
return x
def resize_img(img, size):
img = np.copy(img)
if K.image_data_format() == 'channels_first':
factors = (1, 1, float(size[0]) / img.shape[2],
float(size[1]) / img.shape[3])
else:
factors = (1, float(size[0]) / img.shape[1],
float(size[1]) / img.shape[2], 1)
return scipy.ndimage.zoom(img, factors, order=1)
settings = {
'features': {
'mixed2': randint(1, 5) / 10,
'mixed3': randint(1, 10) / 10,
'mixed4': randint(1, 20) / 10,
'mixed5': randint(1, 50) / 10,
},
}
K.set_learning_phase(0)
# Build the InceptionV3 network with our placeholder.
# The model will be loaded with pre-trained ImageNet weights.
# model = inception_v3.InceptionV3(weights='imagenet',
# include_top=False)
model = load_model('model.hdf5', compile=False)
dream = model.input
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])
# Define the loss.
loss = K.variable(0.)
for layer_name in settings['features']:
# Add the L2 norm of the features of a layer to the loss.
if layer_name not in layer_dict:
raise ValueError('Layer ' + layer_name + ' not found in model.')
coeff = settings['features'][layer_name]
x = layer_dict[layer_name].output
# We avoid border artifacts by only involving non-border pixels in the loss.
scaling = K.prod(K.cast(K.shape(x), 'float32'))
if K.image_data_format() == 'channels_first':
loss += coeff * K.sum(K.square(x[:, :, 2:-2, 2:-2])) / scaling
else:
loss += coeff * K.sum(K.square(x[:, 2:-2, 2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
fetch_loss_and_grads = K.function([dream], outputs)
step = 0.01 # Gradient ascent step size
num_octave = 3 # Number of scales at which to run gradient ascent
octave_scale = 1.4 # Size ratio between scales
iterations = 20 # Number of ascent steps per scale
max_loss = 10.
img = preprocess_image(avatar)
if K.image_data_format() == 'channels_first':
original_shape = img.shape[2:]
else:
original_shape = img.shape[1:3]
successive_shapes = [original_shape]
for i in range(1, num_octave):
shape = tuple([int(dim / (octave_scale**i)) for dim in original_shape])
successive_shapes.append(shape)
successive_shapes = successive_shapes[::-1]
original_img = np.copy(img)
shrunk_original_img = resize_img(img, successive_shapes[0])
for shape in successive_shapes:
img = resize_img(img, shape)
img = gradient_ascent(img,
iterations=iterations,
step=step,
max_loss=max_loss)
upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)
same_size_original = resize_img(original_img, shape)
lost_detail = same_size_original - upscaled_shrunk_original_img
img += lost_detail
return deprocess_image(np.copy(img))
def define_and_train(iteration, model_name, load_for_inference, training_data, training_labels, val_data, val_labels, batch_size, classes, input_shape, alpha, strides, train_datagen, epochs, progress_verbose, checkpoint_verbose, train_steps_per_epoch, val_steps_per_epoch, num_gpus):
K.set_learning_phase(1)
model, parallel_model, model_file = define_model(iteration, model_name, "cifar", input_shape, classes, alpha, strides, num_gpus, weights=None)
get_model_weights_CNN_cifar(model, parallel_model, model_name, load_for_inference, model_file, training_data, training_labels, val_data, val_labels, train_datagen, batch_size, epochs, progress_verbose, checkpoint_verbose, train_steps_per_epoch, val_steps_per_epoch, num_gpus)
return model
import hdf5storage from tqdm import tqdm # GPU allocation K.clear_session() tf.reset_default_graph() os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "0" # Tensorflow memory allocation config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 1. session = tf.Session(config=config) K.tensorflow_backend.set_session(session) K.set_learning_phase(False) # Directories of pretrained models/data model_dir = 'trained_models/lord/model/' data_loc = 'trained_models/lord/data/celeba_test.npz' cbk_loc = 'trained_codebooks/one_sample_fixed.mat' # Load data data = np.load(data_loc) x_d_test = np.copy(data['imgs'] / 255.) y_d_test = np.copy(data['classes']) # Rearrange y_test as ordinal classes (since absolute value of class doesn't matter) _, y_d_test_ordinal = np.unique(y_d_test, return_inverse=True) # Instantiate and load VGGFace with VGG16 core latent_dim = 128
def learnModelMulti(MODEL_FOLDER, MODEL_NAME, LOGS_FOLDER, train_set,
valid_set):
if (MODEL_NAME == 'RESNET'):
K.set_learning_phase(
0
) # set model to inference / test mode manually (required for BN layers)
base_model = applications.ResNet50(include_top=False,
weights='imagenet',
input_shape=(IMG_SIZE, IMG_SIZE, 3))
x = base_model.output
x = layers.GlobalAveragePooling2D()(x)
#x = layers.Dropout(0.5)(x)
#base_model.summary()
elif (MODEL_NAME == 'Xception'):
K.set_learning_phase(
0
) # set model to inference / test mode manually (required for BN layers)
base_model = applications.Xception(weights='imagenet',
include_top=False,
input_shape=(IMG_SIZE, IMG_SIZE, 3))
x = base_model.output
x = layers.GlobalAveragePooling2D()(x)
#ox = layers.Dropout(0.7)(x)
else:
base_model = applications.VGG16(weights='imagenet',
include_top=False,
input_shape=(IMG_SIZE, IMG_SIZE, 3))
x = base_model.output
x = layers.Flatten()(x) # flatten 3D output to 1D
print('***** Base model {} loaded:'.format(MODEL_NAME))
#n_layers_base = len(base_model.layers)
if (MODEL_NAME == 'Xception' or MODEL_NAME == 'RESNET'):
K.set_learning_phase(
1) # set model to training mode manually (required for BN layers)
MLT_shared_repr = layers.Dense(NUM_HIDDEN_UNITS, name='shared_repr')(x)
MLT_shared_repr = layers.BatchNormalization()(MLT_shared_repr)
MLT_shared_repr = layers.Activation('relu')(MLT_shared_repr)
# MLT_shared_repr = layers.Dense(NUM_HIDDEN_UNITS, activation = 'relu', name='shared_repr')(x)
# MLT_shared_repr = layers.Activation('relu')(MLT_shared_repr)
drop_out_layer = layers.Dropout(DROPOUT)(MLT_shared_repr)
artist_prediction = layers.Dense(
nClassesArtist,
activation='softmax',
name='artist',
kernel_regularizer=regularizers.l2(0.001))(drop_out_layer)
year_prediction = layers.Dense(1, name='year')(
drop_out_layer) # regression hence no activation function
type_prediction = layers.Dense(nClassesType,
activation='sigmoid',
name='type')(drop_out_layer)
mat_prediction = layers.Dense(nClassesMat,
activation='sigmoid',
name='mat')(drop_out_layer)
global custom_model
custom_model = Model(
base_model.input,
[artist_prediction, year_prediction, type_prediction, mat_prediction])
#print("***** Full model")
#custom_model.summary()
print('# trainable weights '
'before freezing the conv base:',
len(custom_model.trainable_weights))
for layer in base_model.layers:
layer.trainable = False
print('# trainable weights '
'after freezing the conv base:', len(custom_model.trainable_weights))
artist_loss_weight = 1
year_loss_weight = 0 #0.1 #0.05
type_loss_weight = 1
mat_loss_weight = 1
f = open(LOGS_FOLDER + 'Run_info.txt', 'a')
msg = '\nartist_loss_weight = {:.2f}\n'.format(artist_loss_weight)
msg += 'year_loss_weight = {:.2f}\n'.format(year_loss_weight)
msg += 'type_loss_weight = {:.2f}\n'.format(type_loss_weight)
msg += 'mat_loss_weight = {:.2f}\n'.format(mat_loss_weight)
print(msg)
f.write(msg)
f.close()
#optimizer = tf.train.RMSPropOptimizer(learning_rate=2e-3, decay=0.9)
#custom_model.compile(optimizer='rmsprop',
#sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True)
mae_tol = cm.mae_tol_param(0.15)
#loss_artits_w = cm.categorical_crossentropy_w_wrap(artist_weights_matrix)
if (FOCAL and WEIGHTING):
artistLoss = cm.w_categorical_focal_loss(alpha=1, gamma=2)
elif (FOCAL and not WEIGHTING):
artistLoss = cm.categorical_focal_loss(alpha=1, gamma=2)
elif (not FOCAL and WEIGHTING):
artistLoss = lambda y_true, y_pred: cm.w_categorical_crossentropy(
y_true, y_pred, weights=artist_weights_matrix)
elif (not FOCAL and not WEIGHTING):
artistLoss = cm.categorical_crossentropy_abs
custom_model.compile(optimizer='adam',
loss={
'artist': artistLoss,
'year': 'mae',
'type': 'binary_crossentropy',
'mat': 'binary_crossentropy'
},
loss_weights={
'artist': artist_loss_weight,
'year': year_loss_weight,
'type': type_loss_weight,
'mat': mat_loss_weight
},
metrics={
'artist': [cm.accuracy_abs, cm.accuracy_w],
'year': ['mae', mae_tol],
'type': cm.precision,
'mat': cm.precision
})
train_gen = data_generator(MODEL_NAME, train_set, dataAugm=True)
valid_gen = data_generator(MODEL_NAME, valid_set, dataAugm=False)
#callbacks = [keras.callbacks.TensorBoard(log_dir='../logs',histogram_freq=1]
#csv_logger = callbacks.CSVLogger(LOGS_FOLDER + MODEL_NAME +'.log')
callbacks_list = [
callbacks.CSVLogger(LOGS_FOLDER + MODEL_NAME + '.log'),
#callbacks.EarlyStopping(monitor='val_artist_accuracy_w',patience=4),
callbacks.ModelCheckpoint(filepath=MODEL_FOLDER + MODEL_NAME + '.h5',
monitor='val_artist_accuracy_w',
save_best_only=True)
]
history = custom_model.fit_generator(
train_gen,
validation_data=valid_gen,
validation_steps=ceil(valid_set.shape[0] / BATCH_SIZE),
steps_per_epoch=ceil(2 * train_set.shape[0] / BATCH_SIZE),
epochs=NUM_EPOCHS,
verbose=1,
callbacks=callbacks_list)
print('***** Training logs saved as ' + LOGS_FOLDER + MODEL_NAME + '.log')
#custom_model.save(LOGS_FOLDER + 'artistsRD.h5')
print('***** Model saved as artistsRD.h5')
def train(self, model):
'''
This function will carry out the training of the gan, including the discriminator step
'''
batch_size = 16
self.output_path = os.path.join(
self.output_path,
datetime.datetime.now().strftime("%Y-%m-%d--%Hh%Mm"))
os.mkdir(self.output_path)
os.mkdir(os.path.join(self.output_path, "images"))
self.writer = tf.summary.FileWriter(self.output_path)
train_data_path = os.path.join(self.image_path, "Train_1")
validation_data_path = os.path.join(self.image_path, "Validation_1")
# partition = {"train": [], "validation": []}
# for image in os.listdir(train_data_path):
# partition["train"].append(os.path.join(train_data_path, image))
# for image in os.listdir(validation_data_path):
# partition["validation"].append(os.path.join(validation_data_path, image))
# train_dataset = tf.data.Dataset.from_tensor_slices(partition["train"])
# train_dataset = train_dataset.apply(tf.data.experimental.shuffle_and_repeat(len(partition["train"])))
# train_dataset = train_dataset.map(mse_parse_function_gan, num_parallel_calls=8)
# train_dataset = train_dataset.batch(batch_size)
# train_dataset = train_dataset.prefetch(1)
# train_iterator = train_dataset.make_one_shot_iterator()
# validation_dataset = tf.data.Dataset.from_tensor_slices(partition["validation"])
# validation_dataset = validation_dataset.shuffle(len(partition["validation"]))
# validation_dataset = validation_dataset.map(mse_parse_function_gan, num_parallel_calls=4).repeat()
# validation_dataset = validation_dataset.batch(batch_size)
# validation_dataset = validation_dataset.prefetch(1)
# validation_iterator = train_dataset.make_one_shot_iterator()
datagen = image.ImageDataGenerator(rescale=(1. / 255))
val_datagen = image.ImageDataGenerator(rescale=(1. / 255))
epochs = 2000
# TODO: Try with 2, 3
disc_training_steps = 1
num_train_batches = 258500 // batch_size
num_validation_batches = 10000 // batch_size
# num_train_batches = 32//batch_size
# num_validation_batches = 32//batch_size
#one sided smoothing (https://arxiv.org/pdf/1606.03498.pdf)
real_images_labels = np.full((batch_size, 1), 0.9)
generated_images_labels = np.full((batch_size, 1), 0)
start_time = time.time()
def val_batch_generator(batch_size):
for val_batch in val_datagen.flow_from_directory(
validation_data_path,
target_size=(image_size, image_size),
class_mode="input",
batch_size=batch_size):
val_lab = color.rgb2lab(val_batch[0]).astype(np.float32)
X = preprocess_and_return_X_batch(val_lab)
Y = val_lab[:, :, :, 1:] / 128
generated_ab_val = self.generator.predict(X, steps=1)
disc_x_val = np.concatenate((Y, generated_ab_val), axis=0)
disc_y_val = np.concatenate((np.ones(
(batch_size, 1)), np.zeros((batch_size, 1))),
axis=0)
yield ([disc_x_val, disc_y_val])
gen_loss = 0
for e in range(epochs):
curr_batch = 0
for batch in datagen.flow_from_directory(train_data_path,
target_size=(image_size,
image_size),
class_mode="input",
batch_size=batch_size):
lab = color.rgb2lab(batch[0]).astype(np.float32)
X_train = preprocess_and_return_X_batch(lab)
Y_train = lab[:, :, :, 1:] / 128
generated_ab = self.generator.predict(X_train, steps=1)
# disc_x_train = np.concatenate((Y_train, generated_ab), axis=0)
# disc_y_train = np.concatenate((real_images_labels, generated_images_labels), axis=0)
# shuffle_indices = np.arange(disc_x_train.shape[0])
# np.random.shuffle(shuffle_indices)
# disc_x_train_shuffled = np.squeeze(disc_x_train[shuffle_indices])
# disc_y_train_shuffled = np.squeeze(disc_y_train[shuffle_indices])
#train on real batch then on fake batch (https://github.com/soumith/ganhacks/blob/master/README.md point 4)
#Set learning phase manually due to Dropout and BatchNormalization layers
K.set_learning_phase(1)
#TODO: maybe train whenever acc falls below a certain threshold (90?)
disc_loss_r, disc_acc_r = self.discriminator.train_on_batch(
Y_train, real_images_labels)
disc_loss_f, disc_acc_f = self.discriminator.train_on_batch(
generated_ab, generated_images_labels)
disc_loss_summary = tf.Summary(value=[
tf.Summary.Value(tag="disc_loss",
simple_value=(disc_loss_r + disc_loss_f) /
2)
])
# disc_acc_summary = tf.Summary(value=[tf.Summary.Value(tag="disc_acc", simple_value=(disc_acc_r + disc_acc_f) / 2)])
if curr_batch % 3 == 2:
gen_loss = model.train_on_batch(
X_train, [np.ones((batch_size, 1)), Y_train])
gen_loss_summary = tf.Summary(value=[
tf.Summary.Value(tag="gen_loss",
simple_value=gen_loss[0])
])
K.set_learning_phase(0)
current_step = (e) * num_train_batches + curr_batch
self.writer.add_summary(disc_loss_summary,
current_step * disc_training_steps)
# self.writer.add_summary(disc_acc_summary, current_step*disc_training_steps)
if curr_batch % 3 == 2:
self.writer.add_summary(gen_loss_summary, current_step)
curr_batch = curr_batch + 1
sys.stdout.write(
"\r[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss_bin: %f loss_mse: %f loss: %f] Estimated time left: %s"
% (e, epochs, curr_batch, num_train_batches,
(disc_loss_r + disc_loss_f) / 2, gen_loss[1],
gen_loss[2], gen_loss[0],
str(
datetime.timedelta(
seconds=(
(time.time() - start_time) / curr_batch) *
(num_train_batches - curr_batch)))))
sys.stdout.flush()
if curr_batch >= num_train_batches:
print(
str(self.discriminator.metrics_names) + " : " + str(
self.discriminator.evaluate_generator(
val_batch_generator(batch_size),
steps=num_validation_batches)))
model.save(os.path.join(self.output_path, "model.h5"))
self.discriminator.save(
os.path.join(self.output_path, "discriminator.h5"))
self.generator.save(
os.path.join(self.output_path, "generator.h5"))
images = np.concatenate(
(X_train * 50 + 50, generated_ab * 128), axis=3)
os.mkdir(
os.path.join(self.output_path, "images",
"epoch_{}".format(e)))
for i in range(batch_size):
curr_image = color.lab2rgb(images[i])
io.imsave(
os.path.join(self.output_path, "images",
"epoch_{}".format(e),
"image_{}.jpg".format(i)), curr_image)
break
import time
import math
from utility import plot_util
from utility.data_loader import load_data_multi_timestamp
from utility.test_util import convert_rel_to_44matrix, iround
from utility.networks import build_model_cross_att
# keras
from keras import backend as K
K.set_image_dim_ordering('tf')
K.set_session(
K.tf.Session(config=K.tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False))) #
K.set_learning_phase(0) # Run testing mode
SCALER = 1.0 # scale label: 1, 100, 10000
RADIUS_2_DEGREE = 180.0 / math.pi
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--seqs',
type=str,
required=True,
help='h5 file sequences, e.g, 1, 6, 13')
parser.add_argument('--model',
type=str,
required=True,
help='model architecture')
from keras import callbacks
from keras.utils import plot_model
import time
from futils.util import dice
import argparse
import os
### set important paths
dir_path = os.path.dirname(os.path.realpath(__file__))
path_model = os.path.join(dir_path,'models')
test_dir = os.path.join(dir_path,'data/train')
val_dir = os.path.join(dir_path,'data/val')
K.set_learning_phase(1) # try with 1
parser = argparse.ArgumentParser(
description='End2End Supervised Lobe Segmentation')
parser.add_argument(
'-path',
'--path',
help='Model Path',
type=str,
default='/models/')
parser.add_argument(
'-train',
'--train_dir',
help='Train Data',
type=str,
from optparse import OptionParser
import pickle
from keras.callbacks import ReduceLROnPlateau
from keras import backend as K
from keras.optimizers import Adam, SGD, RMSprop
from keras.layers import Input
from keras.models import Model
from keras_frcnn import config, data_generators
from keras_frcnn import losses as losses
import keras_frcnn.roi_helpers as roi_helpers
from keras.utils import generic_utils
sys.setrecursionlimit(40000)
K.set_learning_phase(0) #set learning phase -- 0 for test ; 1 for Train
parser = OptionParser()
parser.add_option("-p",
"--path",
dest="train_path",
help="Path to training data.")
parser.add_option("-o",
"--parser",
dest="parser",
help="Parser to use. One of simple or pascal_voc",
default="pascal_voc")
parser.add_option("-n",
"--num_rois",
type="int",
dest="num_rois",
from __future__ import print_function
import os
import time
import h5py #conda install -c anaconda h5py=2.6.0y
import numpy as np
import tensorflow as tf
from scipy.misc import imsave
from keras import backend as K
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers.convolutional import Convolution3D
from keras.layers.pooling import MaxPooling3D
from keras.layers.core import Dense, Dropout, Activation, Flatten
K.set_learning_phase(
0) # solves Issues in Keras model loading in Tensorflow Serving
#**************************************************************
#misc functions
#**************************************************************
def normalize(x):
# utility function to normalize a tensor by its L2 norm
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
# util function to convert a tensor into a valid image
def deprocess_image(x):
# normalize tensor: center on 0., ensure std is 0.1
x -= x.mean()
x /= (x.std() + 1e-5)
print('the true=',
' '.join([self.idx2w[num] for num in row_true]))
print('the acc = ', np.mean(acc_list))
return y_pred, y_true
if __name__ == '__main__':
path1 = 'data_aishell'
path2 = 'data_thchs30'
path3 = 'data_pinyin'
if not os.path.exists(path3):
os.mkdir(path3)
K.set_learning_phase(1) #set learning phase
os.environ["CUDA_VISIBLE_DEVICES"] = "2" # 选择使用的GPU
wav_files = get_thchs_wav_paths(path2)
wav_len = len(wav_files)
train2_wavs = wav_files[:int(0.8 * wav_len)]
val2_wavs = wav_files[int(0.8 * wav_len):int(0.9 * wav_len)]
test2_wavs = wav_files[int(0.9 * wav_len):]
train1_wavs, val1_wavs, test1_wavs = get_wav_files(path1)
# name_to_seq, tok = get_thchs_corpus(path2,wav_files, maxlen=48)
trans2_lines = get_thchs_trans_text(path2, wav_files)
trans1_lines = get_trans_text(path1)
def dream(cli_params):
base_image_path = cli_params.base_image_path
result_prefix = cli_params.result_prefix
iterations = cli_params.iterations
step = cli_params.step
num_octave = cli_params.num_octave
octave_scale = cli_params.octave_scale
if not os.path.isdir(base_image_path):
raise Exception('base must be a directory')
# Find the first image in the given directory.
file_types = ('*.jpg', '*.png')
image_files = []
for file_type in file_types:
image_files.extend(
glob.glob('{}/*{}'.format(base_image_path, file_type)))
if not image_files:
types_as_str = ', '.join(file_types)
raise Exception('no image files ({}) under {}'.format(
types_as_str, base_image_path))
image_file = image_files[0]
# These are the names of the layers
# for which we try to maximize activation,
# as well as their weight in the final loss
# we try to maximize.
# You can tweak these setting to obtain new visual effects.
settings = {
'features': {
'mixed2': 0.2,
'mixed3': 0.5,
'mixed4': 2.,
'mixed5': 1.5,
},
}
def preprocess_image(image_path):
# Util function to open, resize and format pictures
# into appropriate tensors.
img = load_img(image_path)
img = img_to_array(img)
img = np.expand_dims(img, axis=0)
img = inception_v3.preprocess_input(img)
return img
def postprocess_image(x):
# Util function to convert a tensor into a valid image.
if K.image_data_format() == 'channels_first':
x = x.reshape((3, x.shape[2], x.shape[3]))
x = x.transpose((1, 2, 0))
else:
x = x.reshape((x.shape[1], x.shape[2], 3))
x /= 2.
x += 0.5
x *= 255.
x = np.clip(x, 0, 255).astype('uint8')
return x
K.set_learning_phase(0)
# Build the InceptionV3 network with our placeholder.
# The model will be loaded with pre-trained ImageNet weights.
model = inception_v3.InceptionV3(weights='imagenet', include_top=False)
dream = model.input
print('Model loaded.')
# Get the symbolic outputs of each "key" layer (we gave them unique names).
layer_dict = dict([(layer.name, layer) for layer in model.layers])
# Define the loss.
loss = K.variable(0.)
for layer_name in settings['features']:
# Add the L2 norm of the features of a layer to the loss.
if layer_name not in layer_dict:
raise ValueError('Layer ' + layer_name + ' not found in model.')
coeff = settings['features'][layer_name]
x = layer_dict[layer_name].output
# We avoid border artifacts by only involving non-border pixels in the loss.
scaling = K.prod(K.cast(K.shape(x), 'float32'))
if K.image_data_format() == 'channels_first':
loss = loss + coeff * K.sum(K.square(x[:, :, 2:-2,
2:-2])) / scaling
else:
loss = loss + coeff * K.sum(K.square(x[:, 2:-2,
2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
fetch_loss_and_grads = K.function([dream], outputs)
def eval_loss_and_grads(x):
outs = fetch_loss_and_grads([x])
loss_value = outs[0]
grad_values = outs[1]
return loss_value, grad_values
def resize_img(img, size):
img = np.copy(img)
if K.image_data_format() == 'channels_first':
factors = (1, 1, float(size[0]) / img.shape[2],
float(size[1]) / img.shape[3])
else:
factors = (1, float(size[0]) / img.shape[1],
float(size[1]) / img.shape[2], 1)
return scipy.ndimage.zoom(img, factors, order=1)
def gradient_ascent(x, shape_number, iterations, step, max_loss=None):
for i in range(iterations):
loss_value, grad_values = eval_loss_and_grads(x)
if max_loss is not None and loss_value > max_loss:
break
print(
json.dumps({
'shape_number': shape_number,
'iteration': i,
'loss_value': loss_value.item(),
}))
x += step * grad_values
return x
"""Process:
- Load the original image.
- Define a number of processing scales (i.e. image shapes),
from smallest to largest.
- Resize the original image to the smallest scale.
- For every scale, starting with the smallest (i.e. current one):
- Run gradient ascent
- Upscale image to the next scale
- Reinject the detail that was lost at upscaling time
- Stop when we are back to the original size.
To obtain the detail lost during upscaling, we simply
take the original image, shrink it down, upscale it,
and compare the result to the (resized) original image.
"""
# Playing with these hyperparameters will also allow you to achieve new effects
max_loss = 10.
img = preprocess_image(image_file)
if K.image_data_format() == 'channels_first':
original_shape = img.shape[2:]
else:
original_shape = img.shape[1:3]
successive_shapes = [original_shape]
for i in range(1, num_octave):
shape = tuple([int(dim / (octave_scale**i)) for dim in original_shape])
successive_shapes.append(shape)
successive_shapes = successive_shapes[::-1]
original_img = np.copy(img)
shrunk_original_img = resize_img(img, successive_shapes[0])
shape_number = 0
for shape in successive_shapes:
print('Processing image shape', shape)
shape_number += 1
img = resize_img(img, shape)
img = gradient_ascent(img,
shape_number=shape_number,
iterations=iterations,
step=step,
max_loss=max_loss)
upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)
same_size_original = resize_img(original_img, shape)
lost_detail = same_size_original - upscaled_shrunk_original_img
img += lost_detail
shrunk_original_img = resize_img(original_img, shape)
save_img(result_prefix + '.png', postprocess_image(np.copy(img)))
