def main():
"""Run inference."""
from model import model
parser = argparse.ArgumentParser(
description='PyTorch RNN Anomaly Detection Model')
parser.add_argument('--prediction_window_size',
type=int,
default=10,
help='prediction_window_size')
parser.add_argument(
'--data',
type=str,
default='ecg',
help=
'type of the dataset (ecg, gesture, power_demand, space_shuttle, respiration, nyc_taxi'
)
parser.add_argument('--filename',
type=str,
default='chfdb_chf13_45590.pkl',
help='filename of the dataset')
parser.add_argument('--save_fig',
action='store_true',
help='save results as figures')
parser.add_argument(
'--compensate',
action='store_true',
help='compensate anomaly score using anomaly score esimation')
parser.add_argument('--beta',
type=float,
default=1.0,
help='beta value for f-beta score')
parser.add_argument('--device',
type=str,
default='cuda',
help='cuda or cpu')
args_ = parser.parse_args()
print('-' * 89)
print("=> loading checkpoint ")
if args_.device == 'cpu':
checkpoint = torch.load(str(
Path('save', args_.data, 'checkpoint',
args_.filename).with_suffix('.pth')),
map_location=torch.device('cpu'))
else:
checkpoint = torch.load(
str(
Path('save', args_.data, 'checkpoint',
args_.filename).with_suffix('.pth')))
args = checkpoint['args']
args.prediction_window_size = args_.prediction_window_size
args.beta = args_.beta
args.save_fig = args_.save_fig
args.compensate = args_.compensate
args.device = args_.device
print("=> loaded checkpoint")
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
###############################################################################
# Load data
###############################################################################
TimeseriesData = preprocess_data.PickleDataLoad(data_type=args.data,
filename=args.filename,
augment_test_data=False)
train_dataset = TimeseriesData.batchify(
args, TimeseriesData.trainData[:TimeseriesData.length], bsz=1)
test_dataset = TimeseriesData.batchify(args,
TimeseriesData.testData,
bsz=1)
###############################################################################
# Build the model
###############################################################################
nfeatures = TimeseriesData.trainData.size(-1)
model = model.RNNPredictor(rnn_type=args.model,
enc_inp_size=nfeatures,
rnn_inp_size=args.emsize,
rnn_hid_size=args.nhid,
dec_out_size=nfeatures,
nlayers=args.nlayers,
res_connection=args.res_connection).to(
args.device)
model.load_state_dict(checkpoint['state_dict'])
scores, predicted_scores, precisions, recalls, f_betas = list(), list(
), list(), list(), list()
targets, mean_predictions, oneStep_predictions, Nstep_predictions = list(
), list(), list(), list()
try:
# For each channel in the dataset
for channel_idx in range(nfeatures):
''' 1. Load mean and covariance if they are pre-calculated, if not calculate them. '''
# Mean and covariance are calculated on train dataset.
if 'means' in checkpoint.keys() and 'covs' in checkpoint.keys():
print('=> loading pre-calculated mean and covariance')
mean, cov = checkpoint['means'][channel_idx], checkpoint[
'covs'][channel_idx]
else:
print('=> calculating mean and covariance')
mean, cov = fit_norm_distribution_param(
args, model, train_dataset, channel_idx=channel_idx)
''' 2. Train anomaly score predictor using support vector regression (SVR). (Optional) '''
# An anomaly score predictor is trained
# given hidden layer output and the corresponding anomaly score on train dataset.
# Predicted anomaly scores on test dataset can be used for the baseline of the adaptive threshold.
if args.compensate:
print('=> training an SVR as anomaly score predictor')
train_score, _, _, hiddens, _ = anomalyScore(
args,
model,
train_dataset,
mean,
cov,
channel_idx=channel_idx)
score_predictor = GridSearchCV(SVR(),
cv=5,
param_grid={
"C": [1e0, 1e1, 1e2],
"gamma":
np.logspace(-1, 1, 3)
})
score_predictor.fit(
torch.cat(hiddens, dim=0).numpy(),
train_score.cpu().numpy())
else:
score_predictor = None
''' 3. Calculate anomaly scores'''
# Anomaly scores are calculated on the test dataset
# given the mean and the covariance calculated on the train dataset
print('=> calculating anomaly scores')
score, sorted_prediction, sorted_error, _, predicted_score = anomalyScore(
args,
model,
test_dataset,
mean,
cov,
score_predictor=score_predictor,
channel_idx=channel_idx)
''' 4. Evaluate the result '''
# The obtained anomaly scores are evaluated by measuring precision, recall, and f_beta scores
# The precision, recall, f_beta scores are are calculated repeatedly,
# sampling the threshold from 1 to the maximum anomaly score value, either equidistantly or logarithmically.
print('=> calculating precision, recall, and f_beta')
precision, recall, f_beta = get_precision_recall(
args,
score,
num_samples=1000,
beta=args.beta,
label=TimeseriesData.testLabel.to(args.device))
print('data: ', args.data, ' filename: ', args.filename,
' f-beta (no compensation): ',
f_beta.max().item(), ' beta: ', args.beta)
if args.compensate:
precision, recall, f_beta = get_precision_recall(
args,
score,
num_samples=1000,
beta=args.beta,
label=TimeseriesData.testLabel.to(args.device),
predicted_score=predicted_score)
print('data: ', args.data, ' filename: ', args.filename,
' f-beta (compensation): ',
f_beta.max().item(), ' beta: ', args.beta)
target = preprocess_data.reconstruct(
test_dataset.cpu()[:, 0, channel_idx],
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
mean_prediction = preprocess_data.reconstruct(
sorted_prediction.mean(dim=1).cpu(),
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
oneStep_prediction = preprocess_data.reconstruct(
sorted_prediction[:,
-1].cpu(), TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
Nstep_prediction = preprocess_data.reconstruct(
sorted_prediction[:,
0].cpu(), TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
sorted_errors_mean = sorted_error.abs().mean(dim=1).cpu()
sorted_errors_mean *= TimeseriesData.std[channel_idx]
sorted_errors_mean = sorted_errors_mean.numpy()
score = score.cpu()
scores.append(score), targets.append(
target), predicted_scores.append(predicted_score)
mean_predictions.append(
mean_prediction), oneStep_predictions.append(
oneStep_prediction)
Nstep_predictions.append(Nstep_prediction)
precisions.append(precision), recalls.append(
recall), f_betas.append(f_beta)
if args.save_fig:
save_dir = Path(
'result', args.data,
args.filename).with_suffix('').joinpath('fig_detection')
save_dir.mkdir(parents=True, exist_ok=True)
plt.plot(precision.cpu().numpy(), label='precision')
plt.plot(recall.cpu().numpy(), label='recall')
plt.plot(f_beta.cpu().numpy(), label='f1')
plt.legend()
plt.xlabel('Threshold (log scale)')
plt.ylabel('Value')
plt.title('Anomaly Detection on ' + args.data + ' Dataset',
fontsize=18,
fontweight='bold')
plt.savefig(
str(
save_dir.joinpath('fig_f_beta_channel' +
str(channel_idx)).with_suffix(
'.png')))
plt.close()
fig, ax1 = plt.subplots(figsize=(15, 5))
ax1.plot(target,
label='Target',
color='black',
marker='.',
linestyle='--',
markersize=1,
linewidth=0.5)
ax1.plot(mean_prediction,
label='Mean predictions',
color='purple',
marker='.',
linestyle='--',
markersize=1,
linewidth=0.5)
ax1.plot(oneStep_prediction,
label='1-step predictions',
color='green',
marker='.',
linestyle='--',
markersize=1,
linewidth=0.5)
ax1.plot(Nstep_prediction,
label=str(args.prediction_window_size) +
'-step predictions',
color='blue',
marker='.',
linestyle='--',
markersize=1,
linewidth=0.5)
ax1.plot(sorted_errors_mean,
label='Absolute mean prediction errors',
color='orange',
marker='.',
linestyle='--',
markersize=1,
linewidth=1.0)
ax1.legend(loc='upper left')
ax1.set_ylabel('Value', fontsize=15)
ax1.set_xlabel('Index', fontsize=15)
ax2 = ax1.twinx()
ax2.plot(
score.numpy().reshape(-1, 1),
label=
'Anomaly scores from \nmultivariate normal distribution',
color='red',
marker='.',
linestyle='--',
markersize=1,
linewidth=1)
if args.compensate:
ax2.plot(predicted_score,
label='Predicted anomaly scores from SVR',
color='cyan',
marker='.',
linestyle='--',
markersize=1,
linewidth=1)
ax2.legend(loc='upper right')
ax2.set_ylabel('anomaly score', fontsize=15)
plt.title('Anomaly Detection on ' + args.data + ' Dataset',
fontsize=18,
fontweight='bold')
plt.tight_layout()
plt.xlim([0, len(test_dataset)])
plt.savefig(
str(
save_dir.joinpath('fig_scores_channel' +
str(channel_idx)).with_suffix(
'.png')))
plt.close()
except KeyboardInterrupt:
print('-' * 89)
print('Exiting from training early')
print('=> saving the results as pickle extensions')
save_dir = Path('result', args.data, args.filename).with_suffix('')
save_dir.mkdir(parents=True, exist_ok=True)
pickle.dump(targets, open(str(save_dir.joinpath('target.pkl')), 'wb'))
pickle.dump(mean_predictions,
open(str(save_dir.joinpath('mean_predictions.pkl')), 'wb'))
pickle.dump(oneStep_predictions,
open(str(save_dir.joinpath('oneStep_predictions.pkl')), 'wb'))
pickle.dump(Nstep_predictions,
open(str(save_dir.joinpath('Nstep_predictions.pkl')), 'wb'))
pickle.dump(scores, open(str(save_dir.joinpath('score.pkl')), 'wb'))
pickle.dump(predicted_scores,
open(str(save_dir.joinpath('predicted_scores.pkl')), 'wb'))
pickle.dump(precisions, open(str(save_dir.joinpath('precision.pkl')),
'wb'))
pickle.dump(recalls, open(str(save_dir.joinpath('recall.pkl')), 'wb'))
pickle.dump(f_betas, open(str(save_dir.joinpath('f_beta.pkl')), 'wb'))
print('-' * 89)
' f-beta (no compensation): ',
f_beta.max().item(), ' beta: ', args.beta)
if args.compensate:
precision, recall, f_beta = get_precision_recall(
args,
score,
num_samples=1000,
beta=args.beta,
label=TimeseriesData.testLabel.to(args.device),
predicted_score=predicted_score)
print('data: ', args.data, ' filename: ', args.filename,
' f-beta (compensation): ',
f_beta.max().item(), ' beta: ', args.beta)
target = preprocess_data.reconstruct(
test_dataset.cpu()[:, 0,
channel_idx], TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
mean_prediction = preprocess_data.reconstruct(
sorted_prediction.mean(dim=1).cpu(),
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
oneStep_prediction = preprocess_data.reconstruct(
sorted_prediction[:, -1].cpu(), TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
Nstep_prediction = preprocess_data.reconstruct(
sorted_prediction[:, 0].cpu(), TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
sorted_errors_mean = sorted_error.abs().mean(dim=1).cpu()
sorted_errors_mean *= TimeseriesData.std[channel_idx]
sorted_errors_mean = sorted_errors_mean.numpy()
score = score.cpu()
def generate_output(args,
epoch,
model,
gen_dataset,
disp_uncertainty=True,
startPoint=500,
endPoint=3500):
if args.save_fig:
# Turn on evaluation mode which disables dropout.
model.eval()
hidden = model.init_hidden(1)
outSeq = []
upperlim95 = []
lowerlim95 = []
with torch.no_grad():
for i in range(endPoint):
if i >= startPoint:
# if disp_uncertainty and epoch > 40:
# outs = []
# model.train()
# for i in range(20):
# out_, hidden_ = model.forward(out+0.01*Variable(torch.randn(out.size())).cuda(),hidden,noise=True)
# outs.append(out_)
# model.eval()
# outs = torch.cat(outs,dim=0)
# out_mean = torch.mean(outs,dim=0) # [bsz * feature_dim]
# out_std = torch.std(outs,dim=0) # [bsz * feature_dim]
# upperlim95.append(out_mean + 2.58*out_std/np.sqrt(20))
# lowerlim95.append(out_mean - 2.58*out_std/np.sqrt(20))
out, hidden = model.forward(out, hidden)
#print(out_mean,out)
else:
out, hidden = model.forward(gen_dataset[i].unsqueeze(0),
hidden)
outSeq.append(out.data.cpu()[0][0].unsqueeze(0))
outSeq = torch.cat(outSeq, dim=0) # [seqLength * feature_dim]
target = preprocess_data.reconstruct(gen_dataset.cpu(),
TimeseriesData.mean,
TimeseriesData.std)
outSeq = preprocess_data.reconstruct(outSeq, TimeseriesData.mean,
TimeseriesData.std)
# if epoch>40:
# upperlim95 = torch.cat(upperlim95, dim=0)
# lowerlim95 = torch.cat(lowerlim95, dim=0)
# upperlim95 = preprocess_data.reconstruct(upperlim95.data.cpu().numpy(),TimeseriesData.mean,TimeseriesData.std)
# lowerlim95 = preprocess_data.reconstruct(lowerlim95.data.cpu().numpy(),TimeseriesData.mean,TimeseriesData.std)
plt.figure(figsize=(15, 5))
for i in range(target.size(-1)):
plt.plot(target[:, :, i].numpy(),
label='Target' + str(i),
color='black',
marker='.',
linestyle='--',
markersize=1,
linewidth=0.5)
plt.plot(range(startPoint),
outSeq[:startPoint, i].numpy(),
label='1-step predictions for target' + str(i),
color='green',
marker='.',
linestyle='--',
markersize=1.5,
linewidth=1)
# if epoch>40:
# plt.plot(range(startPoint, endPoint), upperlim95[:,i].numpy(), label='upperlim'+str(i),
# color='skyblue', marker='.', linestyle='--', markersize=1.5, linewidth=1)
# plt.plot(range(startPoint, endPoint), lowerlim95[:,i].numpy(), label='lowerlim'+str(i),
# color='skyblue', marker='.', linestyle='--', markersize=1.5, linewidth=1)
plt.plot(range(startPoint, endPoint),
outSeq[startPoint:, i].numpy(),
label='Recursive predictions for target' + str(i),
color='blue',
marker='.',
linestyle='--',
markersize=1.5,
linewidth=1)
plt.xlim([startPoint - 500, endPoint])
plt.xlabel('Index', fontsize=15)
plt.ylabel('Value', fontsize=15)
plt.title('Time-series Prediction on ' + args.data + ' Dataset',
fontsize=18,
fontweight='bold')
plt.legend()
plt.tight_layout()
plt.text(startPoint - 500 + 10,
target.min(),
'Epoch: ' + str(epoch),
fontsize=15)
save_dir = Path(
args.path_save + '/result', args.data,
args.filename).with_suffix('').joinpath('fig_prediction')
save_dir.mkdir(parents=True, exist_ok=True)
plt.savefig(
save_dir.joinpath('fig_epoch' + str(epoch)).with_suffix('.png'))
#plt.show()
plt.close()
return outSeq
else:
pass
def generate_output(args,epoch, model, gen_dataset, disp_uncertainty=True,startPoint=500, endPoint=3500):
if args.save_fig:
# Turn on evaluation mode which disables dropout.
model.eval()
hidden = model.init_hidden(1)
outSeq = []
upperlim95 = []
lowerlim95 = []
with torch.no_grad():
for i in range(endPoint):
if i>=startPoint:
# if disp_uncertainty and epoch > 40:
# outs = []
# model.train()
# for i in range(20):
# out_, hidden_ = model.forward(out+0.01*Variable(torch.randn(out.size())).cuda(),hidden,noise=True)
# outs.append(out_)
# model.eval()
# outs = torch.cat(outs,dim=0)
# out_mean = torch.mean(outs,dim=0) # [bsz * feature_dim]
# out_std = torch.std(outs,dim=0) # [bsz * feature_dim]
# upperlim95.append(out_mean + 2.58*out_std/np.sqrt(20))
# lowerlim95.append(out_mean - 2.58*out_std/np.sqrt(20))
out, hidden = model.forward(out, hidden)
#print(out_mean,out)
else:
out, hidden = model.forward(gen_dataset[i].unsqueeze(0), hidden)
outSeq.append(out.data.cpu()[0][0].unsqueeze(0))
outSeq = torch.cat(outSeq,dim=0) # [seqLength * feature_dim]
target= preprocess_data.reconstruct(gen_dataset.cpu().numpy(), TimeseriesData.mean, TimeseriesData.std)
outSeq = preprocess_data.reconstruct(outSeq.numpy(), TimeseriesData.mean, TimeseriesData.std)
# if epoch>40:
# upperlim95 = torch.cat(upperlim95, dim=0)
# lowerlim95 = torch.cat(lowerlim95, dim=0)
# upperlim95 = preprocess_data.reconstruct(upperlim95.data.cpu().numpy(),TimeseriesData.mean,TimeseriesData.std)
# lowerlim95 = preprocess_data.reconstruct(lowerlim95.data.cpu().numpy(),TimeseriesData.mean,TimeseriesData.std)
plt.figure(figsize=(15,5))
for i in range(target.size(-1)):
plt.plot(target[:,:,i].numpy(), label='Target'+str(i),
color='black', marker='.', linestyle='--', markersize=1, linewidth=0.5)
plt.plot(range(startPoint), outSeq[:startPoint,i].numpy(), label='1-step predictions for target'+str(i),
color='green', marker='.', linestyle='--', markersize=1.5, linewidth=1)
# if epoch>40:
# plt.plot(range(startPoint, endPoint), upperlim95[:,i].numpy(), label='upperlim'+str(i),
# color='skyblue', marker='.', linestyle='--', markersize=1.5, linewidth=1)
# plt.plot(range(startPoint, endPoint), lowerlim95[:,i].numpy(), label='lowerlim'+str(i),
# color='skyblue', marker='.', linestyle='--', markersize=1.5, linewidth=1)
plt.plot(range(startPoint, endPoint), outSeq[startPoint:,i].numpy(), label='Recursive predictions for target'+str(i),
color='blue', marker='.', linestyle='--', markersize=1.5, linewidth=1)
plt.xlim([startPoint-500, endPoint])
plt.xlabel('Index',fontsize=15)
plt.ylabel('Value',fontsize=15)
plt.title('Time-series Prediction on ' + args.data + ' Dataset', fontsize=18, fontweight='bold')
plt.legend()
plt.tight_layout()
plt.text(startPoint-500+10, target.min(), 'Epoch: '+str(epoch),fontsize=15)
save_dir = Path('result',args.data,args.filename).with_suffix('').joinpath('fig_prediction')
save_dir.mkdir(parents=True,exist_ok=True)
plt.savefig(save_dir.joinpath('fig_epoch'+str(epoch)).with_suffix('.png'))
#plt.show()
plt.close()
return outSeq
else:
pass
# At any point you can hit Ctrl + C t
endPoint=3500
try:
mean, cov = fit_norm_distribution_param(args, model, train_dataset, endPoint)
scores, sorted_predictions,sorted_errors = anomalyScore(args, model, test_dataset, mean, cov, endPoint)
sorted_predictions = torch.cat(sorted_predictions, dim=0)
sorted_errors = torch.cat(sorted_errors,dim=0)
scores = np.array(scores)
except KeyboardInterrupt:
print('-' * 89)
print('Exiting from training early')
target= preprocess_data.reconstruct(test_dataset.cpu()[:, 0, 0].numpy(),
TimeseriesData.trainData['seqData_mean'],
TimeseriesData.trainData['seqData_std'])
sorted_predictions_mean = preprocess_data.reconstruct(sorted_predictions.mean(dim=1).numpy(),
TimeseriesData.trainData['seqData_mean'],
TimeseriesData.trainData['seqData_std'])
sorted_predictions_1step = preprocess_data.reconstruct(sorted_predictions[:,-1].numpy(),
TimeseriesData.trainData['seqData_mean'],
TimeseriesData.trainData['seqData_std'])
sorted_predictions_Nstep = preprocess_data.reconstruct(sorted_predictions[:,0].numpy(),
TimeseriesData.trainData['seqData_mean'],
TimeseriesData.trainData['seqData_std'])
#sorted_errors_mean = sorted_errors.mean(dim=1).abs().cpu().numpy()
sorted_errors_mean = sorted_errors.abs().mean(dim=1).cpu().numpy()
def generate_output(args,
epoch,
model,
gen_dataset,
startPoint=500,
endPoint=3500):
# Turn on evaluation mode which disables dropout.
model.eval()
hidden = model.init_hidden(1)
outSeq = []
for i in range(endPoint):
if i > startPoint:
out, hidden = model.forward(out, hidden)
else:
out, hidden = model.forward(
Variable(gen_dataset[i].unsqueeze(0), volatile=True), hidden)
outValue = out.data.cpu()[0][0][0]
outSeq.append(outValue)
target = preprocess_data.reconstruct(
gen_dataset.cpu()[:, 0,
0].numpy(), TimeseriesData.trainData['seqData_mean'],
TimeseriesData.trainData['seqData_std'])
outSeq = preprocess_data.reconstruct(
np.array(outSeq), TimeseriesData.trainData['seqData_mean'],
TimeseriesData.trainData['seqData_std'])
plt.figure(figsize=(15, 5))
plot1 = plt.plot(target,
label='Target',
color='black',
marker='.',
linestyle='--',
markersize=1,
linewidth=0.5)
plot2 = plt.plot(range(startPoint),
outSeq[:startPoint],
label='1-step predictions',
color='green',
marker='.',
linestyle='--',
markersize=1.5,
linewidth=1)
plot3 = plt.plot(range(startPoint, endPoint, 1),
outSeq[startPoint:],
label='Multi-step predictions',
color='blue',
marker='.',
linestyle='--',
markersize=1.5,
linewidth=1)
plt.xlim([1500, endPoint])
plt.xlabel('Index', fontsize=15)
plt.ylabel('Value', fontsize=15)
plt.title('Time-series Prediction on ' + args.data + ' Dataset',
fontsize=18,
fontweight='bold')
plt.legend()
plt.tight_layout()
plt.text(1520, 32000, 'Epoch: ' + str(epoch), fontsize=15)
plt.savefig('result/nyc_taxi/fig_epoch' + str(epoch) + '.png')
#plt.show()
plt.close()
return outSeq
# sampling the threshold from 1 to the maximum anomaly score value, either equidistantly or logarithmically.
print('=> calculating precision, recall, and f_beta')
precision, recall, f_beta = get_precision_recall(args, score, num_samples=1000, beta=args.beta,
label=TimeseriesData.testLabel.to(args.device))
print('data: ',args.data,' filename: ',args.filename,
' f-beta (no compensation): ', f_beta.max().item(),' beta: ',args.beta)
if args.compensate:
precision, recall, f_beta = get_precision_recall(args, score, num_samples=1000, beta=args.beta,
label=TimeseriesData.testLabel.to(args.device),
predicted_score=predicted_score)
print('data: ',args.data,' filename: ',args.filename,
' f-beta (compensation): ', f_beta.max().item(),' beta: ',args.beta)
target = preprocess_data.reconstruct(test_dataset.cpu()[:, 0, channel_idx],
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
mean_prediction = preprocess_data.reconstruct(sorted_prediction.mean(dim=1).cpu(),
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
oneStep_prediction = preprocess_data.reconstruct(sorted_prediction[:, -1].cpu(),
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
Nstep_prediction = preprocess_data.reconstruct(sorted_prediction[:, 0].cpu(),
TimeseriesData.mean[channel_idx],
TimeseriesData.std[channel_idx]).numpy()
sorted_errors_mean = sorted_error.abs().mean(dim=1).cpu()
sorted_errors_mean *= TimeseriesData.std[channel_idx]
sorted_errors_mean = sorted_errors_mean.numpy()
score = score.cpu()
scores.append(score), targets.append(targets), predicted_scores.append(predicted_score)
