args = parser.parse_args()
print('input args:\n', json.dumps(vars(args), indent=4, separators=(',', ':')))
assert args.nr_gpu == len(''.join(
filter(str.isdigit, os.environ["CUDA_VISIBLE_DEVICES"])))
# -----------------------------------------------------------------------------
np.random.seed(seed=42)
tf.reset_default_graph()
tf.set_random_seed(args.tf_seed)
# config
configs_dir = __file__.split('/')[-2]
config = importlib.import_module('%s.%s' % (configs_dir, args.config_name))
if not args.resume:
experiment_id = '%s-%s' % (args.config_name.split('.')[-1],
time.strftime("%Y_%m_%d", time.localtime()))
utils.autodir('metadata')
save_dir = 'metadata/' + experiment_id
utils.autodir(save_dir)
else:
save_dir = utils.find_model_metadata('metadata/', args.config_name)
experiment_id = os.path.dirname(save_dir).split('/')[-1]
with open(save_dir + '/meta.pkl', 'rb') as f:
resumed_metadata = pickle.load(f)
last_lr = resumed_metadata['lr']
last_iteration = resumed_metadata['iteration']
print('Last iteration', last_iteration)
print('Last learning rate', last_lr)
# logs
utils.autodir('logs')
sys.stdout = logger.Logger('logs/%s.log' % experiment_id)
if args.mask_dims == 0:
assert args.eps_corr == 0.
# -----------------------------------------------------------------------------
rng = np.random.RandomState(42)
tf.set_random_seed(42)
# config
configs_dir = __file__.split('/')[-2]
config = importlib.import_module('%s.%s' % (configs_dir, args.config_name))
save_dir = utils.find_model_metadata('metadata/', args.config_name)
experiment_id = os.path.dirname(save_dir)
print('exp_id', experiment_id)
target_path = save_dir + '/misc_plots'
utils.autodir(target_path)
# create the model
model = tf.make_template('model', config.build_model)
all_params = tf.trainable_variables()
x_in = tf.placeholder(tf.float32, shape=(config.seq_len, ) + config.obs_shape)
model_output = model(x_in)
log_probs, latent_log_probs, latent_log_probs_prior = model_output[
0], model_output[1], model_output[2]
saver = tf.train.Saver()
data_iter = config.test_data_iter if args.set == 'test' else config.train_data_iter
with tf.Session() as sess:
def plot_anomaly(config_name, n_sequences, n_ignore=8):
configs_dir = __file__.split('/')[-2]
config = importlib.import_module('%s.%s' % (configs_dir, config_name))
# metadata
save_dir = utils.find_model_metadata('metadata/', args.config_name)
expid = os.path.dirname(save_dir).split('/')[-1]
# samples
target_path = save_dir + "/anomaly/"
utils.autodir(target_path)
print('Building the model', expid)
model = tf.make_template('model', config.build_model)
data_iter = config.test_data_iter
data_iter.batch_size = 1
x_in = tf.placeholder(tf.float32,
shape=(data_iter.batch_size, ) + config.obs_shape)
model_output = model(x_in)
latent_log_probs, latent_log_probs_prior = model_output[1], model_output[2]
saver = tf.train.Saver()
with tf.Session() as sess:
ckpt_file = save_dir + 'params.ckpt'
print('restoring parameters from', ckpt_file)
saver.restore(sess, tf.train.latest_checkpoint(save_dir))
for iteration, (x_batch, y_batch) in zip(range(n_sequences),
data_iter.generate_anomaly()):
assert x_batch.shape[0] == data_iter.batch_size
assert data_iter.batch_size == 1
lp, lpp = sess.run([latent_log_probs, latent_log_probs_prior],
feed_dict={x_in: x_batch})
lp = np.squeeze(lp)
lpp = np.squeeze(lpp)
scores = []
for i in range(y_batch.shape[-1]):
print(i, lp[i], lpp[i], lp[i] - lpp[i], y_batch[0, i])
scores.append(lp[i] - lpp[i])
print('-------------')
quartile_1, quartile_3 = np.percentile(scores[n_ignore:], [25, 75])
iqr = quartile_3 - quartile_1
lower_bound = quartile_1 - (iqr * 1.5)
anomaly_idxs = np.where(np.asarray(scores) < lower_bound)[0]
# don't count the first image as an outlier
for i in range(n_ignore):
anomaly_idxs = np.delete(anomaly_idxs,
np.argwhere(anomaly_idxs == i))
print(anomaly_idxs)
x_batch = np.squeeze(x_batch)
x_batch = x_batch[anomaly_idxs]
if len(anomaly_idxs) != 0:
plt.figure(figsize=(4, 1.5))
gs = gridspec.GridSpec(nrows=len(anomaly_idxs),
ncols=6,
hspace=0.1,
wspace=0.1)
ax0 = plt.subplot(gs[:, :-1])
for i in anomaly_idxs:
plt.plot((i + 1, i + 1),
(min(scores) - 2, max(scores) + 2),
'r--',
linewidth=0.5,
dashes=(1, 0.5))
plt.plot(range(1, args.seq_len + 1),
scores,
'black',
linewidth=1.)
plt.gca().axes.set_ylim([min(scores) - 2, max(scores) + 2])
plt.gca().axes.set_xlim([-0.2, args.seq_len + 0.2])
plt.scatter(range(1, args.seq_len + 1),
scores,
c='black',
s=1.5)
plt.xticks(fontsize=6)
plt.yticks(fontsize=6)
plt.tick_params(axis='both', which='major', labelsize=6)
plt.xlabel('n', fontsize=8, labelpad=0)
plt.ylabel('score', fontsize=8, labelpad=0)
for i in range(len(anomaly_idxs)):
ax1 = plt.subplot(gs[i, -1])
plt.imshow(x_batch[i], cmap='gray', interpolation='None')
plt.xticks([])
plt.yticks([])
plt.axis('off')
plt.savefig(target_path + '/anomaly_%s_%s.png' %
(iteration, args.mask_dims),
bbox_inches='tight',
dpi=600,
pad_inches=0)
def tchfiles(datapath):
return [os.path.join( autodir(datapath), "%s.tch" % genkey(datapath)),]
def tchfiles(datapath):
return [
os.path.join(autodir(datapath), "%s.tch" % genkey(datapath)),
]
def classify(config_name, seq_len, n_trials, batch_size):
configs_dir = __file__.split('/')[-2]
config = importlib.import_module('%s.%s' % (configs_dir, config_name))
# metadata
save_dir = utils.find_model_metadata('metadata/', args.config_name)
expid = os.path.dirname(save_dir).split('/')[-1]
assert seq_len == config.seq_len
utils.autodir('logs')
sys.stdout = logger.Logger(
'logs/%s_test_class_%s_%s_%s.log' % (expid, n_trials, config.seq_len, batch_size))
sys.stderr = sys.stdout
print('Building the model', expid)
model = tf.make_template('model', config.build_model)
data_iter = config.test_data_iter2
data_iter.batch_size = batch_size
x_in = tf.placeholder(tf.float32, shape=(data_iter.batch_size,) + config.obs_shape)
log_probs = model(x_in)[0]
saver = tf.train.Saver()
with tf.Session() as sess:
ckpt_file = save_dir + 'params.ckpt'
print('restoring parameters from', ckpt_file)
saver.restore(sess, tf.train.latest_checkpoint(save_dir))
trial_accuracies = []
for trial in range(n_trials):
generator = data_iter.generate(trial=trial)
n_correct = 0
n_total = 0
x_number2scores = defaultdict(list)
x_number2true_y = {}
x_number2ys = {}
for iteration, (x_batch, y_batch, x_number) in enumerate(generator):
y_true = int(y_batch[0, -1])
log_p = sess.run(log_probs, feed_dict={x_in: x_batch})
log_p = log_p.reshape((data_iter.batch_size, config.seq_len))[:, -1]
x_number2scores[x_number].append(log_p)
x_number2true_y[x_number] = y_true
x_number2ys[x_number] = y_batch[:, 0]
if (1. * iteration + 1) % 1000 == 0 or n_trials == 1:
print(x_number + 1)
# average scores
for k, v in x_number2scores.items():
y_true = x_number2true_y[k]
avg_score = np.mean(np.asarray(v), axis=0)
max_idx = np.argmax(avg_score)
if x_number2ys[k][max_idx] == y_true:
n_correct += 1
n_total += 1
acc = n_correct / n_total
print(trial, 'accuracy', acc)
print('n test examples', n_total)
trial_accuracies.append(acc)
print(trial_accuracies)
print('---------------------------------------------')
print(n_trials, config.seq_len)
print(trial_accuracies)
print('average accuracy over trials', np.mean(trial_accuracies))
print('std accuracy over trials', np.std(trial_accuracies))