def main():
K = 50
trainData = np.load('train.npy')
testData = np.load('test.npy')
# UNCOMMENT BELOW AND REMOVE FURTHER BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS
print("[INFO] Building Correlation Matrix")
try:
corrMatrix = np.load('correlation_matrix.npy')
except FileNotFoundError:
corrMatrix = build_corrMatrix(trainData, 'correlation_matrix.npy')
t0 = time.process_time()
# REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE
# corrMatrix = build_corrMatrix(trainData, 'correlation_matrix.npy')
print("[INFO] Running")
reconstructedTrainBasic = collaborative_basic(trainData, testData, corrMatrix, K)
RMSEbasic, SRCbasic, precisionTopKbasic = get_metrics(reconstructedTrainBasic, testData)
t1 = time.process_time()
print('basic: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSEbasic, SRCbasic, precisionTopKbasic, t1-t0))
# REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE
# corrMatrix = build_corrMatrix(trainData, 'correlation_matrix.npy')
reconstructedTrainBaseline = collaborative_baseline(trainData, testData, corrMatrix, K)
RMSEbaseline, SRCbaseline, precisionTopKbaseline = get_metrics(reconstructedTrainBaseline, testData)
t2 = time.process_time()
print('baseline: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSEbaseline, SRCbaseline, precisionTopKbaseline, t2-t1))
def eval_single_model(self, indexes):
model = self.model[indexes]['model']
# loss_fn = self.model[indexes]['loss_fn']
# optimizer = self.model[indexes]['optimizer']
Xtrain = self.data['Xtrain']
Xvalid = self.data['Xvalid']
Xtest = self.data['Xtest']
ytrain = self.data['ytrain']
yvalid = self.data['yvalid']
ytest = self.data['ytest']
ztrain = self.data['ztrain']
zvalid = self.data['zvalid']
ztest = self.data['ztest']
model.eval()
ypred_valid = model(Xvalid)
zpred_valid = None
if self.adversarial:
adv_model = self.model[indexes]['adv_model']
adv_model.eval()
if self.method == 'parity':
adv_input_valid = ypred_valid
zpred_valid = adv_model(adv_input_valid)
elif self.method == 'odds':
adv_input_valid = torch.cat((ypred_valid, yvalid), 1)
zpred_valid = adv_model(adv_input_valid)
elif self.method == 'opportunity':
zpred_valid = None
if zpred_valid is not None:
metrics_valid = pd.DataFrame(get_metrics(ypred_valid.data.numpy(), yvalid.data.numpy(), zvalid.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='valid_set', zpred=zpred_valid.data.numpy()), index=[0])
else:
metrics_valid = pd.DataFrame(get_metrics(ypred_valid.data.numpy(), yvalid.data.numpy(), zvalid.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='valid_set'), index=[0])
print
print('Final test metrics for model with ' + self.hyperparams_to_string(indexes) + ' on validation:')
pprint.pprint(metrics_valid)
ypred_test = model(Xtest)
zpred_test = None
if self.adversarial:
if self.method == 'parity':
adv_input_test = ypred_test
zpred_test = adv_model(adv_input_test)
elif self.method == 'odds':
adv_input_test = torch.cat((ypred_test, ytest), 1)
zpred_test = adv_model(adv_input_test)
elif self.method == 'opportunity':
zpred_test = None
if zpred_test is not None:
metrics_test = pd.DataFrame(get_metrics(ypred_test.data.numpy(), ytest.data.numpy(), ztest.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='test_set', zpred=zpred_test.data.numpy()), index=[0])
else:
metrics_test = pd.DataFrame(get_metrics(ypred_test.data.numpy(), ytest.data.numpy(), ztest.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='test_set'), index=[0])
print
print('Final test metrics for model with ' + self.hyperparams_to_string(indexes) + ' on test:')
pprint.pprint(metrics_test)
return pd.concat([metrics_valid, metrics_test])
def run(**kwargs):
# load graph
graph_nx, dump_folder = loader.load_dataset(kwargs['dataset'])
# initialize tNodeEmbed
task = kwargs['task']
test_size = kwargs['test_size']
tnodeembed = models.tNodeEmbed(graph_nx,
task=task,
dump_folder=dump_folder,
test_size=test_size,
**kwargs['n2vargs'])
# load dataset
X, y = tnodeembed.get_dataset(train_skip=kwargs['train_skip'])
# fit
keras_args = kwargs['keras_args']
batch_size = keras_args.pop('batch_size', 32)
steps_per_epoch = ceil(len(X['train']) / batch_size)
# tNodeEmbed
generator = loader.dataset_generator(X['train'],
y['train'],
tnodeembed.graph_nx,
tnodeembed.train_time_steps,
batch_size=batch_size)
tnodeembed.fit_generator(generator, steps_per_epoch, **keras_args)
# node2vec
static_model = models.StaticModel(task=task)
generator = loader.dataset_generator(X['train'],
y['train'],
tnodeembed.graph_nx,
[max(tnodeembed.train_time_steps)],
batch_size=batch_size)
static_model.fit_generator(generator, steps_per_epoch, **keras_args)
# predict
steps = ceil(len(X['test']) / batch_size)
generator = loader.dataset_generator(X['test'],
y['test'],
tnodeembed.graph_nx,
tnodeembed.train_time_steps,
batch_size=batch_size,
shuffle=False)
tnodeembed_metrics = get_metrics(
y['test'], tnodeembed.predict_generator(generator, steps))
generator = loader.dataset_generator(X['test'],
y['test'],
tnodeembed.graph_nx,
[max(tnodeembed.train_time_steps)],
batch_size=batch_size,
shuffle=False)
node2vec_metrics = get_metrics(
y['test'], static_model.predict_generator(generator, steps))
print(f'tnodeembed: {tnodeembed_metrics}')
print(f'node2vec: {node2vec_metrics}')
def run_metrics_from_dataloader(validation_dataloader, run_path, qrels_path):
query_docids_map = get_query_docids_map(run_path=run_path)
query_rel_doc_map = get_query_rel_doc_map(qrels_path=qrels_path)
label_list = []
for step, batch in enumerate(validation_dataloader):
label_list += flatten_list(batch[3].cpu().numpy().tolist())
labels_groups, scores_groups, queries_groups, doc_ids_groups, rel_docs_groups = group_bert_outputs_by_query(
score_list=label_list,
label_list=label_list,
query_docids_map=query_docids_map,
query_rel_doc_map=query_rel_doc_map)
metrics_strings, label_metrics, _, oracle_metrics = get_metrics(
labels_groups=labels_groups,
scores_groups=scores_groups,
rel_docs_groups=rel_docs_groups)
label_string = get_metrics_string(metrics_strings=metrics_strings,
metrics=label_metrics,
name='RANKING')
oracle_string = get_metrics_string(metrics_strings=metrics_strings,
metrics=oracle_metrics,
name='ORACLE')
print(label_string)
print(oracle_string)
return label_string
def evaluate(model, dataloader, criterion, ctx_weight, device, save_dir=None):
model = model.to(device).eval()
stats_meter = metrics.MetricsAverageMeter()
cls_preds = []
with torch.no_grad():
for batch_index, (features, cls_labels,
ctx_labels) in enumerate(dataloader):
cls_pred, ctx_pred = model(features)
stats = metrics.get_metrics(cls_pred, ctx_pred, cls_labels,
ctx_labels, criterion, criterion,
ctx_weight)
stats_meter.update(stats)
cls_preds += cls_pred.argmax(dim=-1).tolist()
if save_dir is not None:
# save class prediction only
os.makedirs(save_dir, exist_ok=True)
with open(os.path.join(save_dir, 'submit.txt'), 'w') as f:
f.write('\n'.join([str(x) for x in cls_preds]))
stats = stats_meter.mean()
return stats
def bert_evaluate(model, eval_dataloader, device):
"""Evaluation of trained checkpoint."""
model.to(device)
model.eval()
predictions = []
true_labels = []
data_iterator = tqdm(eval_dataloader, desc="Iteration")
for step, batch in enumerate(data_iterator):
input_ids, input_mask, labels = batch
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
with torch.no_grad():
outputs = model(input_ids,
token_type_ids=None,
attention_mask=input_mask)
#loss is only output when labels are provided as input to the model
logits = outputs[0]
print(type(logits))
logits = logits.to('cpu').numpy()
label_ids = labels.to('cpu').numpy()
for label, logit in zip(label_ids, logits):
true_labels.append(label)
predictions.append(np.argmax(logit))
#print(predictions)
#print(true_labels)
metrics = get_metrics(true_labels, predictions)
return metrics
def test(args):
# parse config
config = parse_config(args.config)
test_config = merge_configs(config, 'test', vars(args))
print_configs(test_config, 'Test')
place = fluid.CUDAPlace(0)
with fluid.dygraph.guard(place):
video_model = AttentionCluster("AttentionCluster",
test_config,
mode="test")
model_dict, _ = fluid.load_dygraph(args.weights)
video_model.set_dict(model_dict)
test_reader = FeatureReader(name="ATTENTIONCLUSTER",
mode='test',
cfg=test_config)
test_reader = test_reader.create_reader()
video_model.eval()
total_loss = 0.0
total_acc1 = 0.0
total_sample = 0
for batch_id, data in enumerate(test_reader()):
rgb = np.array([item[0] for item in data
]).reshape([-1, 100, 1024]).astype('float32')
audio = np.array([item[1] for item in data
]).reshape([-1, 100, 128]).astype('float32')
y_data = np.array([item[2] for item in data]).astype('float32')
rgb = to_variable(rgb)
audio = to_variable(audio)
labels = to_variable(y_data)
labels.stop_gradient = True
output, logit = video_model([rgb, audio])
loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logit,
label=labels)
loss = fluid.layers.reduce_sum(loss, dim=-1)
avg_loss = fluid.layers.mean(loss)
# get metrics
valid_metrics = get_metrics(args.model_name.upper(), 'valid',
test_config)
hit_at_one, perr, gap = valid_metrics.calculate_and_log_out(
loss,
logit,
labels,
info='[TEST] test_iter {} '.format(batch_id))
total_loss += avg_loss.numpy()[0]
total_acc1 += hit_at_one
total_sample += 1
print('TEST iter {}, loss = {}, acc1 {}'.format(
batch_id,
avg_loss.numpy()[0], hit_at_one))
print('Finish loss {} , acc1 {}'.format(total_loss / total_sample,
total_acc1 / total_sample))
def monitor_processes(server_process, metrics, interval, socket):
""" Monitor the metrics of server_process and its child processes
"""
while True:
message = []
collected_metrics = get_metrics(server_process,
get_child_processes(server_process),
logger)
metrics_msg = []
for metric in metrics:
message.append(str(collected_metrics.get(metric, 0)))
if collected_metrics.get(metric) is not None:
metrics_msg.append("{0} : {1}".format(
metric, collected_metrics.get(metric, 0)))
message = "\t".join(message) + "\t\n"
logger.info("%s", " -- ".join(metrics_msg))
if socket:
try:
socket.send(message.encode("latin-1"))
except BrokenPipeError:
logger.info(
"Stopping monitoring as socket connection is closed.")
break
# TODO - log metrics to a file METRICS_LOG_FILE if METRICS_LOG_FILE is provided
gevent.sleep(interval)
def eval(path, sheet, epoch):
dset = dataset.get_dataset(1, 'MPEG', False)
cr = CNNCRluma().cuda()
sr = CNNSRluma().cuda()
print(torch.load(path)['epoch'])
#loads net weigths
cr.load_state_dict(torch.load(path)['cr'])
sr.load_state_dict(torch.load(path)['sr'])
cr.eval()
sr.eval()
total, rate = 0.0, 0.0
for iteration, data in enumerate(dset, 1):
input, name = data[0].cuda(), data[1]
with torch.no_grad():
#FORWARD PASS========
#down-sample input
ds, _ = cr(input)
ds = ds.clamp(0,1) #Because of intperolation
#code down-sampled input
coded, bpp = encode(ds, 25)
#up-sampled decoded image
us = sr(coded).clamp(0,1)
#====================
out = transforms.ToPILImage(mode='L')(us[0][0].cpu())
gt = transforms.ToPILImage(mode='L')(input[0][0].cpu())
psnr = get_metrics(gt, out, False)[0]
total += psnr
rate += bpp
print('Bpp: {} --- PSNR: {}'.format(bpp, psnr))
torch.cuda.empty_cache()
print(rate/len(dset))
print(total/len(dset))
def evaluate(self, eval_dataloader):
"""Evaluation of trained checkpoint."""
self.model.to(self.device)
self.model.eval()
predictions = []
true_labels = []
data_iterator = tqdm(eval_dataloader, desc="Iteration")
for step, batch in enumerate(data_iterator):
input_ids, labels = batch
input_ids = input_ids.to(self.device)
with torch.no_grad():
outputs = self.model(input_ids)
outputs = outputs.to('cpu').numpy()
print(type(outputs))
print("Shape:")
print(outputs.shape)
label_ids = labels.to('cpu').numpy()
for label, output in zip(label_ids, outputs):
true_labels.append(label)
predictions.append(np.argmax(output))
# print(predictions)
# print(true_labels)
metrics = get_metrics(true_labels, predictions)
return metrics
def main():
K = 50
trainData = np.load('train.npy')
testData = np.load('test.npy')
# UNCOMMENT BELOW AND REMOVE FURTHER BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS
'''
try:
svd_corrMatrix = np.load('svd_correlation_matrix.npy')
svd_90_corrMatrix = np.load('svd_90_correlation_matrix.npy')
except FileNotFoundError:
u, vt, sigma = build_svd_matrices(trainData)
new_u, new_vt, new_sigma = top_90_energy(u, vt, sigma)
users_in_svd_space = np.matmul(u, sigma)
users_in_svd_90_space = np.matmul(new_u, new_sigma)
svd_corrMatrix = build_corrMatrix(users_in_svd_space, 'svd_correlation_matrix.npy')
svd_90_corrMatrix = build_corrMatrix(users_in_svd_90_space, 'svd_90_correlation_matrix.npy')
'''
t0 = time.clock()
# REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE
u, vt, sigma = build_svd_matrices(trainData)
users_in_svd_space = np.matmul(u, sigma)
svd_corrMatrix = build_corrMatrix(users_in_svd_space, 'svd_correlation_matrix.npy')
result_svd = collaborative_basic(trainData, testData, svd_corrMatrix, K)
RMSE_svd, SRC_svd, precisionTopK_svd = get_metrics(result_svd, testData)
del result_svd
del svd_corrMatrix
t1 = time.clock()
print('SVD: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSE_svd, SRC_svd, precisionTopK_svd, t1-t0))
t2 = time.clock()
# REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE
new_u, new_vt, new_sigma = top_90_energy(u, vt, sigma)
users_in_svd_90_space = np.matmul(new_u, new_sigma)
svd_90_corrMatrix = build_corrMatrix(users_in_svd_90_space, 'svd_90_correlation_matrix.npy')
result_svd_90 = collaborative_basic(trainData, testData, svd_90_corrMatrix, K)
RMSE_svd_90, SRC_svd_90, precisionTopK_svd_90 = get_metrics(result_svd_90, testData)
del result_svd_90
del svd_90_corrMatrix
t3 = time.clock()
print('SVD 90%: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSE_svd_90, SRC_svd_90, precisionTopK_svd_90, t3-t2))
def get_val_metrics(cnn, val_annot_dir, dataset_dir, in_w, out_w, bs):
"""
Return the TP, FP, TN, FN, defined_sum, duration
for the {cnn} on the validation set
TODO - This is too similar to the train loop. Merge both and use flags.
"""
start = time.time()
fnames = ls(val_annot_dir)
fnames = [a for a in fnames if im_utils.is_photo(a)]
cnn.half()
# TODO: In order to speed things up, be a bit smarter here
# by only segmenting the parts of the image where we have
# some annotation defined.
# implement a 'partial segment' which exlcudes tiles with no
# annotation defined.
tps = 0
fps = 0
tns = 0
fns = 0
defined_sum = 0
for fname in fnames:
annot_path = os.path.join(val_annot_dir,
os.path.splitext(fname)[0] + '.png')
# reading the image may throw an exception.
# I suspect this is due to it being only partially written to disk
# simply retry if this happens.
try:
annot = imread(annot_path)
except Exception as ex:
print('Exception reading annotation inside validation method.'
'Will retry in 0.1 seconsds')
print(fname, ex)
time.sleep(0.1)
annot = imread(annot_path)
annot = np.array(annot)
foreground = annot[:, :, 0].astype(bool).astype(int)
background = annot[:, :, 1].astype(bool).astype(int)
image_path_part = os.path.join(dataset_dir, os.path.splitext(fname)[0])
image_path = glob.glob(image_path_part + '.*')[0]
image = im_utils.load_image(image_path)
predicted = unet_segment(cnn, image, bs, in_w, out_w, threshold=0.5)
# mask defines which pixels are defined in the annotation.
mask = foreground + background
mask = mask.astype(bool).astype(int)
predicted *= mask
predicted = predicted.astype(bool).astype(int)
y_defined = mask.reshape(-1)
y_pred = predicted.reshape(-1)[y_defined > 0]
y_true = foreground.reshape(-1)[y_defined > 0]
tps += np.sum(np.logical_and(y_pred == 1, y_true == 1))
tns += np.sum(np.logical_and(y_pred == 0, y_true == 0))
fps += np.sum(np.logical_and(y_pred == 1, y_true == 0))
fns += np.sum(np.logical_and(y_pred == 0, y_true == 1))
defined_sum += np.sum(y_defined > 0)
duration = round(time.time() - start, 3)
metrics = get_metrics(tps, fps, tns, fns, defined_sum, duration)
return metrics
def infer(args):
# parse config
config = parse_config(args.config)
infer_config = merge_configs(config, 'infer', vars(args))
print_configs(infer_config, "Infer")
infer_model = models.get_model(args.model_name,
infer_config,
mode='infer',
is_videotag=True)
infer_model.build_input(use_dataloader=False)
infer_model.build_model()
infer_feeds = infer_model.feeds()
infer_outputs = infer_model.outputs()
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
filelist = args.filelist or infer_config.INFER.filelist
filepath = args.video_path or infer_config.INFER.get('filepath', '')
if filepath != '':
assert os.path.exists(filepath), "{} not exist.".format(filepath)
else:
assert os.path.exists(filelist), "{} not exist.".format(filelist)
# get infer reader
infer_reader = get_reader(args.model_name.upper(), 'infer', infer_config)
if args.weights:
assert os.path.exists(
args.weights), "Given weight dir {} not exist.".format(
args.weights)
# if no weight files specified, download weights from paddle
weights = args.weights or infer_model.get_weights()
infer_model.load_test_weights(exe, weights, fluid.default_main_program())
infer_feeder = fluid.DataFeeder(place=place, feed_list=infer_feeds)
fetch_list = infer_model.fetches()
infer_metrics = get_metrics(args.model_name.upper(), 'infer', infer_config)
infer_metrics.reset()
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
for infer_iter, data in enumerate(infer_reader()):
data_feed_in = [items[:-1] for items in data]
video_id = [items[-1] for items in data]
bs = len(video_id)
feature_outs = exe.run(fetch_list=fetch_list,
feed=infer_feeder.feed(data_feed_in))
for i in range(bs):
filename = video_id[i].split('/')[-1][:-4]
np.save(os.path.join(args.save_dir, filename + '.npy'),
feature_outs[0][i]) #shape: seg_num*feature_dim
logger.info("Feature extraction End~")
def test(args):
# parse config
config = parse_config(args.config)
test_config = merge_configs(config, 'test', vars(args))
print_configs(test_config, "Test")
# build model
test_model = models.get_model(args.model_name, test_config, mode='test')
test_model.build_input(use_pyreader=False)
test_model.build_model()
test_feeds = test_model.feeds()
test_outputs = test_model.outputs()
test_loss = test_model.loss()
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
if args.weights:
assert os.path.exists(
args.weights), "Given weight dir {} not exist.".format(
args.weights)
weights = args.weights or test_model.get_weights()
test_model.load_test_weights(exe, weights, fluid.default_main_program(),
place)
# get reader and metrics
test_reader = get_reader(args.model_name.upper(), 'test', test_config)
test_metrics = get_metrics(args.model_name.upper(), 'test', test_config)
test_feeder = fluid.DataFeeder(place=place, feed_list=test_feeds)
if test_loss is None:
fetch_list = [x.name for x in test_outputs] + [test_feeds[-1].name]
else:
fetch_list = [test_loss.name] + [x.name for x in test_outputs
] + [test_feeds[-1].name]
epoch_period = []
for test_iter, data in enumerate(test_reader()):
cur_time = time.time()
test_outs = exe.run(fetch_list=fetch_list, feed=test_feeder.feed(data))
period = time.time() - cur_time
epoch_period.append(period)
if test_loss is None:
loss = np.zeros(1, ).astype('float32')
pred = np.array(test_outs[0])
label = np.array(test_outs[-1])
else:
loss = np.array(test_outs[0])
pred = np.array(test_outs[1])
label = np.array(test_outs[-1])
test_metrics.accumulate(loss, pred, label)
# metric here
if args.log_interval > 0 and test_iter % args.log_interval == 0:
info_str = '[EVAL] Batch {}'.format(test_iter)
test_metrics.calculate_and_log_out(loss, pred, label, info_str)
test_metrics.finalize_and_log_out("[EVAL] eval finished. ")
def evaluate(prediction, labels, out, beta):
# create folder if not exists
os.makedirs(out, exist_ok=True)
# sort files by Images names
pred_df = pd.read_csv(prediction)
pred_df = pred_df.sort_values(by=['Images'])
label_df = pd.read_csv(labels)
label_df = label_df.sort_values(by=['Images'])
assert len(pred_df) == len(
label_df), "Mismatch between predictions and labels"
header = list(pred_df.keys())[1:]
## calculate scores
resp = dict()
# disease-wise
ovr_fscore = 0
for t in range(len(header)):
pred_t = np.array(pred_df[header[t]])
disease = header[t]
label_t = np.array(label_df[disease])
precision, recall, f_score = get_metrics(pred_t,
label_t,
beta=beta,
average_type='binary')
fpr, tpr, thresholds = roc_curve(label_t, pred_t, pos_label=1)
_auc = auc(fpr, tpr)
resp['precision_{}'.format(disease)] = precision
resp['recall_{}'.format(disease)] = recall
resp['f_score_{}'.format(disease)] = f_score
resp['auc_{}'.format(disease)] = _auc
ovr_fscore += f_score
print(ovr_fscore / len(header))
# overall
preds_array = pred_df[header].to_numpy()
threshold = 0.5
preds_array = np.vectorize(lambda x: 1
if x >= threshold else 0)(preds_array)
targets_array = label_df[[c for c in header]].to_numpy()
resp['overall_f_score'] = fbeta_score(y_pred=preds_array,
y_true=targets_array,
beta=beta,
average='weighted')
resp['overall_precision'] = precision_score(y_pred=preds_array,
y_true=targets_array,
average='weighted')
resp['overall_recall'] = recall_score(y_pred=preds_array,
y_true=targets_array,
average='weighted')
print(resp)
with open(os.path.join(out, 'metrics.json'), 'w') as f:
json.dump(resp, f)
def evaluate_model(self, input_data_x, input_data_y):
pred = self.decode_sequences(input_data_x,
self.config['outputs_validation'])
assert len(pred) == len(input_data_y)
y_gt_input = input_data_y
_, ades, fdes = get_metrics(pred, y_gt_input)
del pred
return {"ADE": np.mean(ades), "FDE": np.mean(fdes)}
def single_imagenet_experiment(regularizer, coeff, train_dir=CLUSTER_LOCAL_IMAGENET_TRAIN_DIR,
val_dir=CLUSTER_LOCAL_IMAGENET_VAL_DIR, verbose=2):
print("Started experiment at {}".format(datetime.datetime.now()))
train_dataset = ImageNetTFRecordDataset(train_dir, BATCH_SIZE)
val_dataset = ImageNetTFRecordDataset(val_dir, BATCH_SIZE)
train_iter = train_dataset.create_dataset()
val_iter = val_dataset.create_dataset()
print("Finished creating datasets at {}".format(datetime.datetime.now()))
start_time = time.time()
# get model
# model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 6,
# [128, 128, 128, 128, 128, 128],
# [11, 5, 5, 5, 5, 5], [4, 2, 1, 1, 2, 1], 2,
# 400, regularizer, coeff, CONV_ACTIVATION, FC_ACTIVATION,
# LAST_ACTIVATION, OPTIMIZER, LOSS, get_metrics(),
# padding='valid', batch_norm=False, tensors=tensors)
# model = model_creation.alexnet(tensors, metrics=get_metrics())
# model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 5, [16, 32, 64, 128, 256],
# 3, 2, 2, 400, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION,
# OPTIMIZER, LOSS, get_metrics())
# model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 6, [32, 64, 128, 256, 512, 1028],
# [5, 3, 3, 3, 3, 3], [3, 2, 2, 2, 2, 2], 4, 1500, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION,
# OPTIMIZER, LOSS, get_metrics())
# model = model_creation.alexnet(None, get_metrics())
# model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 6, [32, 64, 128, 256, 512, 1028],
# [5, 3, 3, 3, 3, 3], [3, 2, 2, 2, 2, 2], 3, 2000, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION,
# OPTIMIZER, LOSS, get_metrics())
# model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 5, [32, 64, 128, 256, 512],
# [11, 3, 3, 3, 3], [5, 2, 2, 2, 2], 4, 2000, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION,
# OPTIMIZER, LOSS, get_metrics())
model = model_creation.alexnet(dropout=True, metrics=get_metrics())
model.summary()
regularizer_name = "None" if regularizer is None else regularizer.__name__
model_name = "nadavnet_{}_coeff_{}_23_June".format(regularizer_name, coeff)
print("Model name is: {}".format(model_name))
tracker = TrackerForTFRecords(model_name, train_dataset.get_x(), train_dataset.get_y(), True, EXAMPLES_FOR_APPROX, True, True)
callbacks = get_non_tracker_callbacks(model_name) + [TrackerCallback(tracker)]
# callbacks = None
print("Finished preparing everything at {}. Ready to start fitting.".format(datetime.datetime.now()))
# fit
model.fit(x=train_iter, epochs=EPOCHS, verbose=verbose, validation_data=val_iter,
steps_per_epoch=IMAGENET_EPOCH_SIZE // BATCH_SIZE, callbacks=callbacks,
validation_steps=IMAGENET_VALIDATION_SIZE // BATCH_SIZE)
print("Experiment of model {} finished. "
"This experiment took {:.2f} minutes.".format(model_name, int(time.time() - start_time) / 60))
def test(args):
# parse config
config = parse_config(args.config)
test_config = merge_configs(config, 'test', vars(args))
print_configs(test_config, "Test")
use_dali = test_config['TEST'].get('use_dali', False)
# build model
test_model = models.get_model(args.model_name, test_config, mode='test')
test_model.build_input(use_dataloader=False)
test_model.build_model()
test_feeds = test_model.feeds()
test_fetch_list = test_model.fetches()
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if args.weights:
assert os.path.exists(
args.weights), "Given weight dir {} not exist.".format(
args.weights)
weights = args.weights or test_model.get_weights()
logger.info('load test weights from {}'.format(weights))
test_model.load_test_weights(exe, weights, fluid.default_main_program())
# get reader and metrics
test_reader = get_reader(args.model_name.upper(), 'test', test_config)
test_metrics = get_metrics(args.model_name.upper(), 'test', test_config)
test_feeder = fluid.DataFeeder(place=place, feed_list=test_feeds)
epoch_period = []
for test_iter, data in enumerate(test_reader()):
cur_time = time.time()
test_outs = exe.run(fetch_list=test_fetch_list,
feed=test_feeder.feed(data))
period = time.time() - cur_time
epoch_period.append(period)
test_metrics.accumulate(test_outs)
# metric here
if args.log_interval > 0 and test_iter % args.log_interval == 0:
info_str = '[EVAL] Batch {}'.format(test_iter)
test_metrics.calculate_and_log_out(test_outs, info_str)
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
test_metrics.finalize_and_log_out("[EVAL] eval finished. ", args.save_dir)
def select_params(train_type, param_type, candidates):
param = []
mean_errors = []
if train_type == 'purchase':
for candicate in candidates:
purchase_params[param_type] = candicate
offline_predict_purchase = offline_predict('purchase', offline_train_X_purchase, offline_train_y_purchase, offline_test_X_purchase)
[sum_error,mean_error,std_error] = get_metrics(offline_test_y_purchase, offline_predict_purchase)
param.append(candicate)
mean_errors.append(mean_error)
plt.scatter(param, mean_errors)
plt.show()
elif train_type == 'redeem':
for candicate in candidates:
redeem_params[param_type] = candicate
offline_predict_redeem = offline_predict('redeem', offline_train_X_redeem, offline_train_y_redeem, offline_test_X_redeem)
[sum_error,mean_error,std_error] = get_metrics(offline_test_y_redeem, offline_predict_redeem)
param.append(candicate)
mean_errors.append(mean_error)
plt.scatter(param, mean_errors)
plt.show()
print param[mean_errors.index(min(mean_errors))], min(mean_errors)
def _calc_resource_stats(self, interval):
result = super()._calc_resource_stats(interval)
server_pid = get_process_pid_from_file(get_server_pidfile(PID_FILE))
server_process = get_server_processes(server_pid)
result.update(get_metrics(server_process, get_child_processes(server_process), self.log))
metrics_msg = []
updated_result = {}
for key in self.metrics:
if result.get(key) is not None:
metrics_msg.append("{0} : {1}".format(key, result[key]))
updated_result[key] = result.get(key)
self.log.info("{0}".format(" -- ".join(metrics_msg)))
return updated_result
def main():
init_env('1')
loaders = make_data_loaders(cfg)
model = build_model(cfg)
model = model.cuda()
task_name = 'base_unet'
log_dir = os.path.join(cfg.LOG_DIR, task_name)
cfg.TASK_NAME = task_name
mkdir(log_dir)
logger = setup_logger('train', log_dir, filename='train.log')
logger.info(cfg)
logger = setup_logger('eval', log_dir, filename='eval.log')
optimizer, scheduler = make_optimizer(cfg, model)
metrics = get_metrics(cfg)
losses = get_losses(cfg)
train_val(model, loaders, optimizer, scheduler, losses, metrics)
def test_single(network, data_loader, device):
# PSNR, SSIM, RMSE
psnr_all, ssim_all, rmse_all = [], [], []
with torch.no_grad():
for i, (x, y) in enumerate(data_loader):
x = x.float().to(device)
y = y.float().to(device)
pred = network(x)
predict_result = metrics.get_metrics(y, pred)
psnr_all.append(predict_result[0])
ssim_all.append(predict_result[1])
rmse_all.append(predict_result[2])
n = len(data_loader)
return sum(psnr_all) / n, sum(ssim_all) / n, sum(rmse_all) / n
def test(self):
"""Summary
After the test end, calculate the metrics
Args:
Returns:
TYPE: Description
"""
# inference dataset
if self.hparams.infer == 'valid':
output_, target_ = self.test_step()
else:
output_ = self.test_step()
resp = dict()
to_csv = {'Images': self.names}
for t in range(len(self.num_tasks)):
if self.hparams.multi_cls:
y_pred = np.reshape(output_, output_.shape[0])
else:
y_pred = np.transpose(output_)[t]
to_csv[self.labels[self.num_tasks[t]]] = y_pred
# Only save scores to json file when in valid mode
if self.hparams.infer == 'valid':
overall_pre, overall_rec, overall_fscore = get_metrics(
copy.deepcopy(output_), copy.deepcopy(target_), self.cfg.beta,
self.cfg.threshold, self.cfg.metric_type)
resp['overall_pre'] = overall_pre
resp['overall_rec'] = overall_rec
resp['overall_f_score'] = overall_fscore
with open(
os.path.join(os.path.dirname(self.hparams.load),
'scores_{}.csv'.format(uuid.uuid4())),
'w') as f:
json.dump(resp, f)
# Save predictions to csv file for computing metrics in off-line mode
path_df = DataFrame(to_csv, columns=to_csv.keys())
path_df.to_csv(os.path.join(os.path.dirname(self.hparams.load),
'predictions_{}.csv'.format(uuid.uuid4())),
index=False)
return resp
def compile_model(config: ml_collections.ConfigDict) -> tf.keras.Model:
"""Builds a graph and compiles a tf.keras.Model based on the configuration."""
model = build_model_graph(config, config.outcomes)
losses = {}
loss_weights = {}
for outcome in config.outcomes:
losses[outcome.name] = metrics.get_loss(outcome)
loss_weights[outcome.name] = outcome.get('loss_weight', 1.0)
model.compile(
optimizer=get_optimizer(config),
loss=losses,
loss_weights=loss_weights,
weighted_metrics=metrics.get_metrics(config.outcomes))
return model
def benchmark(args, dataset_folder, dataset):
data = prepare_dataset(dataset_folder, dataset, args.nrows)
results = {}
# "all" runs all algorithms
if args.algorithm == "all":
args.algorithm = "xgb-gpu,xgb-cpu,lgbm-cpu,lgbm-gpu,cat-cpu,cat-gpu"
for alg in args.algorithm.split(","):
print("Running '%s' ..." % alg)
runner = algorithms.Algorithm.create(alg)
with runner:
train_time = runner.fit(data, args)
pred = runner.test(data)
results[alg] = {
"train_time": train_time,
"accuracy": get_metrics(data, pred),
}
return results
def do_GET(self):
query_re = re.compile("query=(.*)")
logs_table_re = re.compile(b"<!--\\W*logs\\W*-->\\W*<table>\\W*</table>")
metrics_table_re = re.compile(b"<!--\\W*metrics\\W*-->\\W*<table>\\W*</table>")
load_test_size_re = re.compile(b"<!--\\W*LOAD_TEST_SIZE\\W*-->")
log_rows = []
metrics_rows = []
if self.path == "/" or self.path == "/index.html":
self.path = "/index.html"
elif self.path.startswith("/index.html?"):
query = query_re.search(self.path).group(1)
log = list(get_single_request(query))
log_rows = [log]
logger.log(logger.FILE_NAME, log_rows)
self.path = "/index.html"
elif self.path == "/metrics" or self.path == "/metrics/load_test":
if self.path == "/metrics/load_test":
queries = open("random_queries.txt", 'r').read().split(' ')
get_load_request(queries, LOAD_TEST_SIZE)
log_rows = logger.read(logger.FILE_NAME)
metrics_rows = metrics.get_metrics(LOAD_TEST_SIZE, log_rows)
self.path = "/metrics.html"
try:
# Check the file extension required and
# set the right mime type
mimetype = 'text/html'
# Open the static file requested and send it
f = open(os.curdir + os.sep + self.path, 'rb')
self.send_response(200)
self.send_header('Content-type', mimetype)
self.end_headers()
html = f.read()
html = logs_table_re.sub(logger.draw_table(logger.LOG_TABLE_HEADER, log_rows), html)
html = metrics_table_re.sub(logger.draw_table(logger.METRICS_TABLE_HEADER, metrics_rows), html)
html = load_test_size_re.sub(str(LOAD_TEST_SIZE).encode('utf-8'), html)
self.wfile.write(html)
f.close()
return
except IOError:
self.send_error(404, 'File Not Found: %s' % self.path)
def build_ensemble(models: List[tf.keras.Model],
config: ml_collections.ConfigDict) -> tf.keras.Model:
"""Compiles and returns an averaged ensemble model."""
model = _build_ensemble_graph(models, config)
losses = {}
loss_weights = {}
for outcome in config.outcomes:
losses[outcome.name] = metrics.get_loss(outcome)
loss_weights[outcome.name] = outcome.get('loss_weight', 1.0)
model.compile(
optimizer=model_utils.get_optimizer(config),
loss=losses,
loss_weights=loss_weights,
metrics=metrics.get_metrics(config.outcomes))
return model
def run_eval_alg(algorithm, train, test, dataset, processed_data, all_sensitive_attributes,
single_sensitive, tag):
"""
Train and evaluate the algorithm; and gets the resulting metric evaluations.
:return: params
"""
privileged_vals = dataset.get_privileged_class_names_with_joint(tag)
positive_val = dataset.get_positive_class_val(tag)
##############TRAIN CLASSIFICATION MODEL ON TRAIN AND PREDICT ON TEST
# get the actual classifications and sensitive attributes
actual = test[dataset.get_class_attribute()].values.tolist()
alg_output = run_alg(algorithm, train, test, dataset, all_sensitive_attributes, single_sensitive,
privileged_vals, positive_val)
##############EVAL AGGREGATED METRICS ON TEST SET
# make dictionary mapping sensitive names to sensitive attr test data lists
dict_sensitive_lists = {}
for sens in all_sensitive_attributes:
dict_sensitive_lists[sens] = test[sens].values.tolist()
sensitive_dict = processed_data.get_sensitive_values(tag)
metrics_eval = []
for metric in get_metrics(dataset, sensitive_dict, tag):
result = metric.calc(actual, alg_output['predictions'], dict_sensitive_lists, single_sensitive,
privileged_vals, positive_val)
metrics_eval.append(result)
for metric in deepcopy(get_bayesian_metrics(dataset, sensitive_dict, tag)):
result = metric.calc(actual, alg_output['predictions'], dict_sensitive_lists, single_sensitive,
privileged_vals, positive_val)
metrics_eval.append(result)
results_lol = []
return {
'params': alg_output['params'],
'metrics_eval': metrics_eval,
'results_lol': results_lol,
'test_prediction_probs': alg_output['predicted_probs']
}
def find_best_fixed_threshold(self, output_, target_):
score = list()
thrs = np.arange(0, 1.0, 0.01)
pre_rec = list()
for thr in tqdm.tqdm(thrs):
pre, rec, fscore = get_metrics(copy.deepcopy(output_),
copy.deepcopy(target_),
self.cfg.beta, thr,
self.cfg.metric_type)
score.append(fscore)
pre_rec.append([pre, rec])
score = np.array(score)
pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
best_pre, best_rec = pre_rec[pm]
print('thr={} F2={} prec{} rec{}'.format(best_thr, best_score,
best_pre, best_rec))
return best_pre, best_rec, best_score
def val(epoch, model, cfg, args, valid_config):
reader = FeatureReader(name="ATTENTIONCLUSTER", mode="valid", cfg=cfg)
reader = reader.create_reader()
total_loss = 0.0
total_acc1 = 0.0
total_sample = 0
for batch_id, data in enumerate(reader()):
rgb = np.array([item[0]
for item in data]).reshape([-1, 100,
1024]).astype('float32')
audio = np.array([item[1]
for item in data]).reshape([-1, 100,
128]).astype('float32')
y_data = np.array([item[2] for item in data]).astype('float32')
rgb = to_variable(rgb)
audio = to_variable(audio)
labels = to_variable(y_data)
labels.stop_gradient = True
output, logit = model([rgb, audio])
loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logit,
label=labels)
loss = fluid.layers.reduce_sum(loss, dim=-1)
avg_loss = fluid.layers.mean(loss)
# get metrics
valid_metrics = get_metrics(args.model_name.upper(), 'valid',
valid_config)
hit_at_one, perr, gap = valid_metrics.calculate_and_log_out(
loss, logit, labels, info='[TEST] test_iter {} '.format(batch_id))
total_loss += avg_loss.numpy()[0]
total_acc1 += hit_at_one
total_sample += 1
print('TEST Epoch {}, iter {}, loss = {}, acc1 {}'.format(
epoch, batch_id,
avg_loss.numpy()[0], hit_at_one))
print('Finish loss {} , acc1 {}'.format(total_loss / total_sample,
total_acc1 / total_sample))
def train(self, data):
n_temp = len(self.args.T_list)
for (iT, T) in enumerate(self.args.T_list):
self.model = get_model(data.train_in.shape,
feat_ext=self.args.FEAT,
hid_act=self.args.ACT,
hid_filters=self.args.HF,
kernels=self.args.K,
pbc=self.args.PBC)
self.model.compile(optimizer=self.args.OPT,
loss=self.loss,
metrics=self.metrics_list)
hist = self.model.fit(
x=data.train_in[iT * self.args.nTR:(iT + 1) *
self.args.nTR][:self.args.TRS],
y=data.train_out[iT * self.args.nTR:(iT + 1) *
self.args.nTR][:self.args.TRS],
batch_size=self.args.BS,
epochs=self.args.EP,
verbose=self.args.VB,
callbacks=self.callbacks,
validation_data=(
data.test_in[iT * self.args.nTE:(iT + 1) *
self.args.nTE][:self.args.VALS],
data.test_out[iT * self.args.nTE:(iT + 1) *
self.args.nTE][:self.args.VALS]))
self.metrics = get_metrics(hist, reg=self.reg_flag)
### Save files ###
npsave('%s/%s/Met%.4f.npy' % (self.args.metrics_dir, self.name, T),
self.metrics)
self.model.save('%s/%s/Net%.4f.h5' %
(self.args.model_dir, self.name, T))
print('Temperature %d / %d done!' % (iT + 1, n_temp))
def processLine(line):
print("Execute metrics_by_protein to [" + line + "]")
metrics_by_protein = metrics.get_metrics(line.rstrip())
output_file.write(metrics_by_protein + os.linesep)
