def main(saved_weights_path):
prepare_log_dir()
train_data, train_labels = load_svhn_data("train", "full")
valid_data, valid_labels = load_svhn_data("valid", "full")
test_data, test_labels = load_svhn_data("test", "full")
print("TrainData", train_data.shape)
print("Valid Data", valid_data.shape)
print("Test Data", test_data.shape)
train_size = len(train_labels)
train_regressor(train_data, train_labels, valid_data, valid_labels,
test_data, test_labels, train_size, saved_weights_path)
def main(saved_weights_path):
prepare_log_dir()
train_data, train_labels = load_svhn_data("train", "full")
valid_data, valid_labels = load_svhn_data("valid", "full")
test_data, test_labels = load_svhn_data("test", "full")
print("TrainData", train_data.shape)
print("Valid Data", valid_data.shape)
print("Test Data", test_data.shape)
train_size = len(train_labels)
train_regressor(train_data, train_labels, valid_data, valid_labels,
test_data, test_labels, train_size, saved_weights_path)
def main(saved_weights_path):
prepare_log_dir()
train_data, train_labels = load_svhn_data("train", "cropped")
valid_data, valid_labels = load_svhn_data("valid", "cropped")
test_data, test_labels = load_svhn_data("test", "cropped")
print("Training", train_data.shape)
print("Valid", valid_data.shape)
print("Test", test_data.shape)
train_size = train_labels.shape[0]
#saved_weights_path = None
train_classification(train_data, train_labels, valid_data, valid_labels,
test_data, test_labels, train_size,
saved_weights_path)
def main(saved_weights_path):
prepare_log_dir()
train_data, train_labels = load_svhn_data("train", "cropped")
valid_data, valid_labels = load_svhn_data("valid", "cropped")
test_data, test_labels = load_svhn_data("test", "cropped")
print("Training", train_data.shape)
print("Valid", valid_data.shape)
print("Test", test_data.shape)
train_size = train_labels.shape[0]
saved_weights_path = None
train_classification(train_data, train_labels,
valid_data, valid_labels,
test_data, test_labels, train_size,
saved_weights_path)
import sys
import os
import numpy as np
import tensorflow as tf
import PIL.Image as Image
import matplotlib.pyplot as plt
import cv2 as cv
from svhn_model import regression_head
from svhn_data import load_svhn_data
import time
test_dataset, test_labels = load_svhn_data("test", "full")
WEIGHTS_FILE = "regression.ckpt"
# tf.reset_default_graph()
def optimistic_restore(session, save_file):
reader = tf.train.NewCheckpointReader(save_file)
saved_shapes = reader.get_variable_to_shape_map()
var_names = sorted([(var.name, var.name.split(':')[0])
for var in tf.global_variables()
if var.name.split(':')[0] in saved_shapes])
restore_vars = []
name2var = dict(
zip(map(lambda x: x.name.split(':')[0], tf.global_variables()),
tf.global_variables()))
with tf.variable_scope('', reuse=True):
for var_name, saved_var_name in var_names:
def main(saved_weights_path):
prepare_log_dir()
train_data, train_labels = load_svhn_data("train", "full")
valid_data, valid_labels = load_svhn_data("valid", "full")
test_data, test_labels = load_svhn_data("test", "full")
print("\nTrain data", train_data.shape)
print("Valid data", valid_data.shape)
print("Test data", test_data.shape)
print("Train labels", train_labels.shape)
print("Valid labels", valid_labels.shape)
print("Test labels", test_labels.shape)
train_size = len(train_labels); print('train_size',train_size)
valid_size = len(valid_labels); print('valid_size',valid_size)
test_size = len(test_labels); print('test_size',test_size,'\n')
data_size = [train_size , valid_size , test_size]
print('1. starting the grouping data into subsets process.')
# Grouping the data into subsets, each subset has a certain num of digits in every image # data_groups (k) are as follows: k=0 -> train k=1 -> valid k=2 -> test
# For each data/label group, subsets have a set number of digits (a number: 1 to 5)
# create list templates
data_list_of_lists_tvt = [ [ [],[],[],[],[],[] ], [ [],[],[],[],[],[] ] , [ [],[],[],[],[],[] ] ]
num_digits_list = [[0,0,0,0,0,0] , [0,0,0,0,0,0] , [0,0,0,0,0,0]]
print('2. created list templates')
data_set = [train_data, valid_data, test_data]
labels_set = [train_labels, valid_labels, test_labels]
# Forming the list of lists & counting the number of images with x digits, for data group (train / valid / test) # data_groups: k=0 -> train k=1 -> valid k=2 -> test
for data_group in range(3):
labels = labels_set[data_group]
for i in range(data_size[data_group]):
for num_digits_in_data_subset in range(6):
data_num_digits = labels[i, 0]
if data_num_digits == (num_digits_in_data_subset):
data_list_of_lists_tvt[data_group][num_digits_in_data_subset].append(i)
num_digits_list[data_group][num_digits_in_data_subset] += 1
labels = None
if TEST_FLAG:
print('np.array(num_digits_list).shape',np.array(num_digits_list).shape)
print('np.array(num_digits_list[0]).shape',np.array(num_digits_list[0]).shape)
print('np.array(num_digits_list[0][1]).shape',np.array(num_digits_list[0][1]).shape,'If this shape is (), this reflects that this object is a scalar value.')
print('\n3. created populated data/labels lists')
# print the count of number of images in each group / subset. each subset reflects num_digits
print('\nThe following are counts of number of images for each group and (num_digits) subset. Groups are as follows: k=0 -> train k=1 -> valid k=2 -> test')
for data_group in range(3):
for num_digits_in_data_subset in range(6):
print('Data group:',data_group,'\tNum of digits_',num_digits_in_data_subset,' = \t',num_digits_list[data_group][num_digits_in_data_subset])
# create lists of zero arrays, for each group and each subset (for both data and labels)
data_group_with_digits = [[0,0,0,0,0,0] , [0,0,0,0,0,0] , [0,0,0,0,0,0]]
label_group_with_digits = [[0,0,0,0,0,0] , [0,0,0,0,0,0] , [0,0,0,0,0,0]]
for data_group in range(3):
for num_digits_in_data_subset in range(6):
data_group_with_digits[data_group][num_digits_in_data_subset] = np.zeros(shape=(num_digits_list[data_group][num_digits_in_data_subset] , IMG_HEIGHT , IMG_WIDTH , NUM_CHANNELS), dtype=np.float32) #shape_data = [train_data.shape , valid_data.shape , test_data.shape]
label_group_with_digits[data_group][num_digits_in_data_subset] =np.zeros(shape=(num_digits_list[data_group][num_digits_in_data_subset],6) , dtype=np.int32)
if data_group ==0: print('num_digits_list[data_group][num_digits_in_data_subset]',num_digits_list[data_group][num_digits_in_data_subset])
#if data_group == 0:
# if num_digits_in_data_subset == 4:
# print('num_digits_list[data_group][num_digits_in_data_subset]',num_digits_list[data_group][num_digits_in_data_subset])
# print('label_group_with_digits[data_group][num_digits_in_data_subset]',label_group_with_digits[data_group][num_digits_in_data_subset])
print('\n4. created lists of zero arrays\n')
if TEST_FLAG:
print('data_group_with_digits.shape',np.array(data_group_with_digits).shape)
print('data_group_with_digits[0].shape',np.array(data_group_with_digits[0]).shape)
print('data_group_with_digits[0][1].shape',np.array(data_group_with_digits[0][1]).shape)
print('np.array(num_digits_list).shape',np.array(num_digits_list).shape)
print('np.array(num_digits_list[0]).shape',np.array(num_digits_list[0]).shape)
print('np.array(num_digits_list[0][1]).shape',np.array(num_digits_list[0][1]).shape)
print('np.array(label_group_with_digits).shape',np.array(label_group_with_digits).shape)
print('np.array(label_group_with_digits[0])',np.array(label_group_with_digits[0]).shape)
print('np.array(label_group_with_digits[0][1])',np.array(label_group_with_digits[0][1] ).shape)
#populate the arrays with data/labels
for data_group in range(3):
labels = labels_set[data_group]
data = data_set[data_group]
for num_digits_in_data_subset in range(6):
if TEST_FLAG: print('data_group',data_group,'num_digits_in_data_subset',num_digits_in_data_subset)
data_group_with_digits[data_group][num_digits_in_data_subset] = [data[i,:,:,:] for i in data_list_of_lists_tvt[data_group][num_digits_in_data_subset] if num_digits_list[data_group][num_digits_in_data_subset] != 0 ]
label_group_with_digits[data_group][num_digits_in_data_subset] = [ labels[i,:] for i in data_list_of_lists_tvt[data_group][num_digits_in_data_subset] if num_digits_list[data_group][num_digits_in_data_subset] != 0 ]
print('\n5. populated the arrays with data/labels\n')
if TEST_FLAG:
print('data_group_with_digits.shape',np.array(data_group_with_digits).shape)
print('data_group_with_digits[0].shape',np.array(data_group_with_digits[0]).shape)
print('data_group_with_digits[0][0].shape',np.array(data_group_with_digits[0][0]).shape)
print('data_group_with_digits[0][1].shape',np.array(data_group_with_digits[0][1]).shape)
#print('data_group_with_digits[0][1].shape',np.array(data_group_with_digits[0][1]).shape)
print('np.array(label_group_with_digits).shape',np.array(label_group_with_digits).shape)
print('np.array(label_group_with_digits[0]).shape',np.array(label_group_with_digits[0]).shape)
print('np.array(label_group_with_digits[0][0]).shape',np.array(label_group_with_digits[0][0]).shape)
print('np.array(label_group_with_digits[0][1]).shape',np.array(label_group_with_digits[0][1]).shape)
""" TRAIN THE REGRESSOR"""
# define num_digits_in_data_subset , such that we can train the regressor , for a given num_digits_in_data_subset
# num_digits_in_data_subset is input into train_regressor function
# training => data_group == 0
for num_digits_in_data_subset in range(6):
data_train = np.array( data_group_with_digits[0][num_digits_in_data_subset] )
labels_train = np.array( label_group_with_digits[0][num_digits_in_data_subset] )
data_valid = np.array( data_group_with_digits[1][num_digits_in_data_subset] )
labels_valid = np.array( label_group_with_digits[1][num_digits_in_data_subset] )
data_test = np.array( data_group_with_digits[2][num_digits_in_data_subset] )
labels_test = np.array( label_group_with_digits[2][num_digits_in_data_subset] )
data_shape = np.array(data_train).shape # data_shape[0] is the training data length
if data_shape[0] > 0: # we check there is some data, for a given num_of_digits, so that we input some data (vs. NULL) into regression model during training
if num_digits_in_data_subset > 0: # we don't need to train the model to predict that there are no digits in an image (in this use-case)
print('num_digits_in_data_subset',num_digits_in_data_subset,'data_shape[0]',data_shape[0])
train_regressor(data_train, labels_train, data_valid, labels_valid, data_test, labels_test, train_size=data_shape[0], saved_weights_path=saved_weights_path, num_digits_in_data_subset=num_digits_in_data_subset)
import sys
import os
import numpy as np
import tensorflow as tf
import PIL.Image as Image
import matplotlib.pyplot as plt
from svhn_model import regression_head
from svhn_data import load_svhn_data
import time
test_dataset, test_labels = load_svhn_data("test", "full")
WEIGHTS_FILE = "regression.ckpt"
def prediction_to_string(pred_array):
pred_str = ""
for i in range(len(pred_array)):
if pred_array[i] != 10:
pred_str += str(pred_array[i])
else:
return pred_str
return pred_str
def detect(img_path, saved_model_weights):
sample_img = Image.open(img_path)
plt.imshow(sample_img)
plt.show()
pix = np.array(sample_img)
# start = cv2.getTickCount()
score = anogan.compute_anomaly_score(model,
test_imgs[i].reshape(1, 32, 32, 3),
iterations=500,
d=None)
# time = (cv2.getTickCount() - start) / cv2.getTickFrequency() * 1000
print('%d label, %d : done' % (1, i), '%.2f' % score)
score_for_2.append(score)
score_for_2 = np.sort(score_for_2)[::-1]
print(score_for_2)
import pickle
pickle.dump(score_for_2, open("cifar100_score.pickle", "wb"))
### compute outlier scores on SVHN
import svhn_data
(X_train, y_train), (X_test, y_test) = svhn_data.load_svhn_data()
X_train = (X_train.astype(np.float32) - 127.5) / 127.5
X_test = (X_test.astype(np.float32) - 127.5) / 127.5
# X_train = X_train[:,:,:,None]
# X_test = X_test[:,:,:,None]
perm_test = np.random.permutation(len(X_train))
X_train = X_train[perm_test]
test_imgs = X_train[0:200]
score_for_2 = []
# test_img = np.random.uniform(-1,1, (28,28,1))
model = anogan.anomaly_detector(g=None, d=None)
for i in range(np.shape(test_imgs)[0]):
# start = cv2.getTickCount()
score = anogan.compute_anomaly_score(model,
test_imgs[i].reshape(1, 32, 32, 3),