def main_test(dataset):
"""
Predictor Test
"""
tf.reset_default_graph()
lr = logistic_regressor.LogisticRegressor(feature_num=conf.FEATURE_NUM,
learning_rate=conf.LEARNING_RATE,
random_seed=None)
saver, logits, loss, train_op, stat_merged = lr.build_graph()
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
saver.restore(sess, '{}/model'.format(conf.CHKPT_DIR))
in_feature_vecs = dataset[0][:100]
in_labels = dataset[1][:100]
in_labels = np.expand_dims(in_labels, 1)
feed_dict = {
lr.input_feature_vectors: in_feature_vecs,
lr.input_labels: in_labels
}
out_logits, out_weights, out_biases = sess.run(
[logits, lr.weights, lr.biases], feed_dict=feed_dict)
print("accuracy: {}, f1: {}, auc: {}".format(
metrics.calc_accuracy(out_logits, in_labels),
metrics.calc_f1(out_logits, in_labels, log_confusion_matrix=True),
metrics.calc_auc(out_logits, in_labels)))
print("weights: ", out_weights)
print("biases: ", out_biases)
def incremental_evaluate(sess, model, minibatch_iter, size, test=False):
t_test = time.time()
val_losses = []
val_preds = []
labels = []
iter_num = 0
finished = False
while not finished:
feed_dict_val, batch_labels, finished, _ = \
minibatch_iter.incremental_node_val_feed_dict(
size, iter_num, test=test)
node_outs_val = sess.run([model.preds, model.loss],
feed_dict=feed_dict_val)
val_preds.append(node_outs_val[0])
labels.append(batch_labels)
val_losses.append(node_outs_val[1])
iter_num += 1
# TODO 放进model
val_preds = np.vstack(val_preds)
labels = np.vstack(labels)
f1_scores = calc_f1(labels, val_preds)
report = classification_report(labels, val_preds)
# precision, recall, thresholds = precision_recall_curve(
# labels[:, 1], val_preds[:, 1])
# area = auc(recall, precision)
return np.mean(val_losses), f1_scores[0], f1_scores[1], report, (
time.time() - t_test) #, area
def evaluate(sess, model, minibatch_iter, size=None):
t_test = time.time()
feed_dict_val, labels = minibatch_iter.node_val_feed_dict(size)
node_outs_val = sess.run([model.preds, model.loss],
feed_dict=feed_dict_val)
mic, mac = calc_f1(labels, node_outs_val[0])
return node_outs_val[1], mic, mac, (time.time() - t_test)
def temporal_score(self, iou_list, video_clips, bg_class=0):
output_scores = self.__init_out_score_dict(iou_list)
for iou in iou_list:
confusion_mat = Dict(fp=0, tp=0, fn=0)
class_confusion_mat = Dict()
for c in range(self.num_classes):
class_confusion_mat[c] = Dict(fp=0, tp=0, fn=0)
for video_name, clip_list in video_clips.items():
clips = video_clips[video_name]
for c in range(self.num_classes):
targets = (np.array(clips.targets) == c)
predictions = (np.array(clips.predictions) == c)
tp1, fp1, fn1 = f_score(predictions,
targets,
iou,
bg_class=0)
class_confusion_mat[c].fp += fp1
class_confusion_mat[c].tp += tp1
class_confusion_mat[c].fn += fn1
tp1, fp1, fn1 = f_score(clips.predictions,
clips.targets,
iou,
bg_class=bg_class)
confusion_mat.tp += tp1
confusion_mat.fp += fp1
confusion_mat.fn += fn1
for c in range(self.num_classes):
output_scores["class_{}".format(c)]["iou_{:.2f}".format(
iou)] = calc_f1(class_confusion_mat[c].fn,
class_confusion_mat[c].fp,
class_confusion_mat[c].tp)
output_scores.overall["iou_{:.2f}".format(iou)] = calc_f1(
confusion_mat.fn, confusion_mat.fp, confusion_mat.tp)
return output_scores
def get_loss_score(model, loader, device):
trainig = model.training
model.eval()
loss_sum = 0
acc_sum = 0
num_pts = 0
for _, inputs, labels in loader:
inputs = inputs.to(device)
labels = labels.to(device)
with torch.set_grad_enabled(False):
outputs = model(inputs)
loss = dice_loss(outputs, labels)
acc = calc_f1(outputs, labels)
loss_sum += loss * inputs.size(0)
acc_sum += acc * inputs.size(0)
num_pts += inputs.size(0)
model.train() if model.training else model.eval()
return loss_sum / num_pts, acc_sum / num_pts
def main(dataset):
"""
Tainer
"""
# build graph
tf.reset_default_graph()
lr = logistic_regressor.LogisticRegressor(feature_num=conf.FEATURE_NUM,
learning_rate=conf.LEARNING_RATE,
random_seed=None)
saver, logits, loss, train_op, stat_merged = lr.build_graph()
# training
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
# log dir
log_dir = conf.LOG_DIR
if os.path.exists(log_dir):
shutil.rmtree(log_dir)
summary_writer = tf.summary.FileWriter(log_dir,
graph=tf.get_default_graph())
# checkpoint dir
chkpt_dir = conf.CHKPT_DIR
if os.path.exists(chkpt_dir):
shutil.rmtree(chkpt_dir)
if not os.path.isdir(chkpt_dir):
os.makedirs(chkpt_dir)
for epoch in range(conf.EPOCHES):
batch_cnt = 0
batches_per_epoch = math.floor(
(len(dataset[0]) - 1) * 1.0 / conf.BATCH_SIZE) + 1
best_loss = np.inf
cur_loss = np.inf
cur_accuracy = 0
training_data = list(zip(dataset[0], dataset[1]))
random.shuffle(training_data)
for tu in utils.batch(training_data, n=conf.BATCH_SIZE):
X, y = zip(*tu)
y = np.expand_dims(y, 1)
feed_dict = {lr.input_feature_vectors: X, lr.input_labels: y}
sess.run(train_op, feed_dict=feed_dict)
batch_cnt += 1
global_step = epoch * batches_per_epoch + batch_cnt
if global_step % conf.DISPLAY_STEP == 0:
in_f = dataset[0]
in_l = np.expand_dims(dataset[1], 1)
feed_dict = {
lr.input_feature_vectors: in_f,
lr.input_labels: in_l
}
cur_loss, cur_logits = sess.run([loss, logits],
feed_dict=feed_dict)
summary_train = sess.run(stat_merged, feed_dict=feed_dict)
summary_writer.add_summary(summary_train,
global_step=global_step)
print("epoch: {}, global_step: {}, loss: {}, "
"accuracy: {}, f1: {}, auc: {}".format(
epoch, global_step, cur_loss,
metrics.calc_accuracy(cur_logits, in_l),
metrics.calc_f1(cur_logits, in_l),
metrics.calc_auc(cur_logits, in_l)))
if cur_loss < best_loss:
best_loss = cur_loss
saver.save(sess, '{}/model'.format(chkpt_dir))
if cnt%100==0:
print(cnt)
q,a = line.strip('\n').split('\t')
a1 = crowded.answer_beams(q)
crowded_pairs.append([list(a1), list(a)])
a2 = fake.answer_beams(q)
fake_pairs.append([list(a2), list(a)])
a3 = fake_kd.answer_beams(q)
fake_kd_pairs.append([list(a3), list(a)])
a4 = crowded_fake.answer_beams(q)
crowded_fake_pairs.append([list(a4), list(a)])
a5 = crowded_fake_kd.answer_beams(q)
crowded_fake_kd_pairs.append([list(a5), list(a)])
res = [q, a, a1, a2, a3, a4, a5]
fw.write('\t'.join(res) + '\n')
for res in [crowded_pairs, fake_pairs, fake_kd_pairs, crowded_fake_pairs, crowded_fake_kd_pairs]:
f1 = calc_f1(res)
bleu = calc_bleu(res)
distinct = calc_distinct(res)
avg_len = calc_avg_len(res)
print('f1: ', f1)
print('bleu: ', bleu)
print('distinct: ', distinct)
print('avg_len: ', avg_len)
'''
while True:
message = input('>')
print('crowded', crowded.answer_beams(message))
print('crowded_fake', crowded_fake.answer_beams(message))
print('fake', fake.answer_beams(message))
'''
def classValidation(self, epoch):
self.model.eval()
predictions = {}
targets = {}
tp, fp, fn = 0, 0, 0
with torch.no_grad():
progress_bar = tqdm(self.valLoader)
for step, (data, masks, anomaly, category, _,
clipNames) in enumerate(progress_bar):
if self.modelType == "mstcn":
anomaly = anomaly.view(-1)
clipNames = np.array(clipNames).reshape(-1).tolist()
if torch.cuda.is_available():
data = data.cuda().float()
anomaly = anomaly.cuda().float()
masks = masks.cuda().float()
outputs = self.model(data, masks)
loss = 0
for output in outputs:
loss += self.ceLoss(
output.transpose(2, 1).reshape(-1, 2), anomaly.long())
outputs = outputs[-1].transpose(2, 1).reshape(-1,
2).cpu().numpy()
anomaly = anomaly.cpu().numpy().flatten().tolist()
if step % 10 == 0:
progress_bar.set_description(
"Val [{}]:[{}/{}] Loss: {:.2f}".format(
epoch, step, self.valLoader.__len__(),
loss.item()))
for clipName, prediction, target in zip(
clipNames, outputs, anomaly):
predictions[clipName] = prediction
targets[clipName] = target
videoClips = self.valLoader.dataset.__getVideoClips__()
for videoName, clipList in tqdm(videoClips.items()):
clipPredictions = []
clipTargets = []
for clipName in clipList:
clipPredictions.append(predictions[clipName])
clipTargets.append(targets[clipName])
clipPredictions = np.argmax(np.array(clipPredictions), axis=1)
utils.visualizeTemporalPredictions(clipPredictions,
clipTargets, self.expFolder,
videoName)
tp1, fp1, fn1 = f_score(clipPredictions,
clipTargets,
0.1,
bg_class=-1)
tp += tp1
fp += fp1
fn += fn1
f1, precision, recall = calc_f1(fn, fp, tp)
print('F1@%0.2f-%0.2f : %.4f, TP: %.4f, FP: %.4f, FN: %.4f' %
(0.1, 0.5, f1, tp, fp, fn))
self.writer.add_scalar("Eval/F1_0.10-%0.2f" % 0.5, f1,
self.stepCounter)
self.writer.add_scalar("Confusion/%0.2f/TP_0.10" % 0.5, tp,
self.stepCounter)
self.writer.add_scalar("Confusion/%0.2f/FP_0.10" % 0.5, fp,
self.stepCounter)
self.writer.add_scalar("Confusion/%0.2f/TP_0.10" % 0.5, fn,
self.stepCounter)
return f1
def binaryValidation(self, epoch):
self.model.eval()
anomalyPredictions = []
anomalyTargets = []
predictions = {}
targets = {}
thresholds = [0.5, 0.75, 0.9]
IOUs = [0.1, 0.25, 0.5]
score = 0
with torch.no_grad():
progress_bar = tqdm(self.valLoader)
for step, (data, masks, anomaly, category, _,
clipNames) in enumerate(progress_bar):
if self.modelType == "tcn" or self.modelType == "mstcn" or self.modelType == "mcbtcn":
anomaly = anomaly.view(-1)
clipNames = np.array(clipNames).reshape(-1).tolist()
if torch.cuda.is_available():
data = data.float().cuda()
anomaly = anomaly.float().cuda()
masks = masks.float().cuda()
if self.modelType == "mstcn":
outputs = self.model(data, masks)
outputs = outputs[-1].view(-1)
elif self.modelType == "mcbtcn":
classOutputs, binaryOutputs = self.model(data, masks)
outputs = binaryOutputs[-1].view(-1)
else:
outputs = self.model(data)
mask = (anomaly != self.maskValue).nonzero().squeeze().cpu()
outputs = outputs[mask]
clipNames = np.array(clipNames)[mask].tolist()
anomaly = anomaly[mask]
loss = self.mseLoss(outputs.squeeze(), anomaly)
outputs = outputs.reshape(-1).cpu().numpy().tolist()
anomaly = anomaly.cpu().numpy().flatten().tolist()
anomalyTargets += anomaly
anomalyPredictions += outputs
if step % 10 == 0:
progress_bar.set_description(
"Val [{}]:[{}/{}] Loss: {:.2f}".format(
epoch, step, self.valLoader.__len__(),
loss.item()))
for clipName, prediction, target in zip(
clipNames, outputs, anomaly):
if clipName not in predictions:
predictions[clipName] = []
if clipName not in targets:
targets[clipName] = []
predictions[clipName].append(prediction)
targets[clipName].append(target)
videoClips = self.valLoader.dataset.__getVideoClips__()
for iou in IOUs:
for s, threshold in enumerate(thresholds):
tp, fp, fn = 0, 0, 0
normal = {"tp": 0, "fp": 0, "fn": 0}
abnormal = {"tp": 0, "fp": 0, "fn": 0}
for videoName, clipList in tqdm(videoClips.items()):
clipPredictions = []
clipTargets = []
for clipName in clipList:
clipPredictions.append(
np.mean(np.array(predictions[clipName])))
clipTargets.append(
np.mean(np.array(targets[clipName])))
# if "Assault010_x264" in videoName:
# auc_score = sklrn.roc_auc_score(clipTargets, clipPredictions)
# utils.visualizeHeatMapPredictions(clipPredictions, clipTargets, self.expFolder, videoName)
# print("AUC Score of selected video: {}".format(auc_score))
clipPredictions = (
np.array(clipPredictions) >
threshold).astype("float32").tolist()
if iou == 0.25 and threshold == 0.5:
utils.visualizeTemporalPredictions(
clipPredictions, clipTargets, self.expFolder,
videoName)
tp1, fp1, fn1 = f_score(clipPredictions,
clipTargets,
iou,
bg_class=0)
abnormal["tp"] += tp1
abnormal["fp"] += fp1
abnormal["fn"] += fn1
tp1, fp1, fn1 = f_score(clipPredictions,
clipTargets,
iou,
bg_class=1)
normal["tp"] += tp1
normal["fp"] += fp1
normal["fn"] += fn1
if self.noNormalSegmentation:
tp1, fp1, fn1 = f_score(clipPredictions,
clipTargets,
iou,
bg_class=0)
else:
tp1, fp1, fn1 = f_score(clipPredictions,
clipTargets,
iou,
bg_class=-1)
# if "Assault010_x264" in videoName:
# precision = tp1 / float(tp1 + fp1 + 1e-10)
# recall = tp1 / float(tp1 + fn1 + 1e-10)
# f1 = 2.0 * (precision * recall) / (precision + recall + 1e-10)
# print("F1 Score of selected video: {}".format(f1))
tp += tp1
fp += fp1
fn += fn1
a_f1, a_precision, a_recall = calc_f1(
abnormal["fn"], abnormal["fp"], abnormal["tp"])
print(
'Abnormal F1@%0.2f-%0.2f : %.4f, Precision: %.4f, Recall: %.4f'
% (iou, threshold, a_f1, a_precision * 100,
a_recall * 100))
n_f1, n_precision, n_recall = calc_f1(
normal["fn"], normal["fp"], normal["tp"])
print(
'Normal F1@%0.2f-%0.2f : %.4f, Precision: %.4f, Recall: %.4f'
% (iou, threshold, n_f1, n_precision * 100,
n_recall * 100))
f1, precision, recall = calc_f1(fn, fp, tp)
if iou == 0.25 and threshold == 0.5:
score = f1
print(
'F1@%0.2f-%0.2f : %.2f, TP: %.2f, FP: %.2f, FN: %.2f' %
(iou, threshold, f1, tp, fp, fn))
# print('Precision@%0.2f-%0.2f : %.2f, Recall@%0.2f-%0.2f: %.2f' % (iou, threshold, precision * 100,
# iou, threshold, recall * 100))
if self.writer is not None:
self.writer.add_scalar(
"Eval/F1_%0.2f-%0.2f" % (iou, threshold), f1,
self.stepCounter)
self.writer.add_scalar(
"Confusion/TP_%0.2f-%0.2f" % (iou, threshold), tp,
self.stepCounter)
self.writer.add_scalar(
"Confusion/FP_%0.2f-%0.2f" % (iou, threshold), fp,
self.stepCounter)
self.writer.add_scalar(
"Confusion/FN_%0.2f-%0.2f" % (iou, threshold), fn,
self.stepCounter)
fpr, tpr, _ = sklrn.roc_curve(anomalyTargets, anomalyPredictions)
rocAUC = sklrn.auc(fpr, tpr)
if self.writer is not None:
self.writer.add_scalar("Eval/AUC", rocAUC, self.stepCounter)
print('AUC Score %0.2f' % (rocAUC * 100))
return score
def train_model(model,
dataloaders,
policy_learner,
optimizer,
scheduler,
num_epochs,
device,
writer,
n_images=None):
loader = {'val': dataloaders['val']}
# best_model_wts = copy.deepcopy(model.state_dict())
best_loss = 1e10
if n_images is None:
n_images = {'train': 0, 'val': 0}
for epoch in range(num_epochs):
loader['train'] = policy_learner()
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
since = time.time()
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
# print('+++++++++ len loader', len(loader[phase]))
if phase == 'train':
if scheduler:
scheduler.step()
for param_group in optimizer.param_groups:
print("LR", param_group['lr'])
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
metrics = defaultdict(float)
epoch_samples = 0
for enum_id, (idxs, inputs,
labels) in tqdm(enumerate(loader[phase]),
total=len(loader[phase])):
inputs = inputs.to(device)
labels = labels.to(device)
# if phase == 'train' and enum_id < 3:
# for idx in idxs:
# torch.save(torch.tensor(1),
# f'tmp/trash/{policy_learner.__class__.__name__}_{epoch}_{enum_id}__{idx}'
# )
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
# loss, loss_sum, loss_bce, loss_dice = calc_loss(outputs, labels, 0)
loss = dice_loss(outputs, labels)
acc_f1 = calc_f1(outputs, labels)
# acc_iou = calc_IOU(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
plot_grad_flow(epoch, enum_id,
model.named_parameters())
optimizer.step()
# statistics
epoch_samples += inputs.size(0)
n_images[phase] += inputs.size(0)
writer.add_scalar(f'{phase}/loss',
loss.data.cpu().numpy(), n_images[phase])
# writer.add_scalar(f'{phase}/bce', loss_bce, n_images[phase])
# writer.add_scalar(f'{phase}/dice', loss_dice, n_images[phase])
metrics['loss'] += loss * inputs.size(0)
metrics['f1'] += acc_f1 * inputs.size(0)
# metrics['iou'] += acc_iou * inputs.size(0)
print_metrics(writer, metrics, epoch_samples, phase)
epoch_loss = metrics['loss'] / epoch_samples
writer.add_scalar(f'{phase}/epoch_loss', epoch_loss, epoch)
epoch_f1 = metrics['f1'] / epoch_samples
writer.add_scalar(f'{phase}/epoch_F1', epoch_f1, epoch)
# epoch_iou = metrics['iou'] / epoch_samples
# writer.add_scalar(f'{phase}/epoch_IOU', epoch_iou, epoch)
# # deep copy the model
# if phase == 'val' and epoch_loss < best_loss:
# print("saving best model")
# best_loss = epoch_loss
# best_model_wts = copy.deepcopy(model.state_dict())
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
# model.load_state_dict(best_model_wts)
return model, n_images