def _predict(args):
data_iter = data_loader(args)
preds = model.predict(data_iter).asnumpy()
truths = data_iter.get_truths()
observers = data_iter.get_observers()
print(preds.shape, truths.shape, observers.shape)
pred_output = {}
for i, label in enumerate(data_iter._data_format.class_labels):
pred_output['class_%s' % label] = truths[:, i]
pred_output['score_%s' % label] = preds[:, i]
for i, obs in enumerate(data_iter._data_format.obs_vars):
pred_output[obs] = observers[:, i]
import pandas as pd
df = pd.DataFrame(pred_output)
if args.predict_output:
logging.info('Write prediction file to %s' % args.predict_output)
outdir = os.path.dirname(args.predict_output)
if not os.path.exists(outdir):
os.makedirs(outdir)
df.to_hdf(args.predict_output, 'Events', format='table')
from common.util import plotROC
plotROC(preds, truths, output=os.path.join(outdir, 'roc.pdf'))
from root_numpy import array2root
array2root(df.to_records(index=False),
filename=args.predict_output.rsplit('.', 1)[0] +
'.root',
treename='Events',
mode='RECREATE')
def _predict(data_iter, outpath):
# prediction loop
preds = []
for eval_batch in data_iter:
# prepare data
data = [
eval_batch.data[idx].as_in_context(devs)
for idx, meta in enumerate(data_iter.provide_data)
]
# forward
with mx.autograd.predict_mode():
predD = netD(*data)[1]
probs = mx.nd.exp(predD)
preds.append(probs.asnumpy())
import numpy as np
preds = np.concatenate(preds)
truths = data_iter.get_truths()
observers = data_iter.get_observers()
print(preds.shape, truths.shape, observers.shape)
pred_output = {}
for i, label in enumerate(data_iter._data_format.class_labels):
pred_output['class_%s' % label] = truths[:, i]
pred_output['score_%s' % label] = preds[:, i]
for i, obs in enumerate(data_iter._data_format.obs_vars):
pred_output[obs] = observers[:, i]
import pandas as pd
df = pd.DataFrame(pred_output)
if outpath:
logging.info('Write prediction file to %s' % outpath)
outdir = os.path.dirname(outpath)
if not os.path.exists(outdir):
os.makedirs(outdir)
df.to_hdf(outpath, 'Events', format='table')
from common.util import plotROC
plotROC(preds, truths, output=os.path.join(outdir, 'roc.pdf'))
from root_numpy import array2root
array2root(df.to_records(index=False),
filename=outpath.rsplit('.', 1)[0] + '.root',
treename='Events',
mode='RECREATE')
def _predict(args):
# data iterators
data_iter = data_loader(args)
_softmaxD, _symAdv, _param_file, _adv_param_file = _load_model(args)
if _softmaxD is not None:
if symSoftmax.tojson() != _softmaxD.tojson():
print(symSoftmax.tojson())
print('-' * 50)
print(_softmaxD.tojson())
logging.warning('Inconsistent json!')
raise RuntimeError
try:
netD.load_parameters(
_param_file,
ctx=devs) # works with block.save_parameters()
except AssertionError:
netD.collect_params().load(
_param_file, ctx=devs) # work with block.export()
# prediction loop
preds = []
for eval_batch in data_iter:
# prepare data
data = [
eval_batch.data[idx].as_in_context(devs)
for idx, meta in enumerate(data_iter.provide_data)
]
# forward
with mx.autograd.predict_mode():
predD = netD(*data)[1]
probs = mx.nd.exp(predD)
preds.append(probs.asnumpy())
import numpy as np
preds = np.concatenate(preds)
truths = data_iter.get_truths()
observers = data_iter.get_observers()
print(preds.shape, truths.shape, observers.shape)
pred_output = {}
for i, label in enumerate(data_iter._data_format.class_labels):
pred_output['class_%s' % label] = truths[:, i]
pred_output['score_%s' % label] = preds[:, i]
for i, obs in enumerate(data_iter._data_format.obs_vars):
pred_output[obs] = observers[:, i]
import pandas as pd
df = pd.DataFrame(pred_output)
if args.predict_output:
logging.info('Write prediction file to %s' % args.predict_output)
outdir = os.path.dirname(args.predict_output)
if not os.path.exists(outdir):
os.makedirs(outdir)
# df.to_hdf(args.predict_output, 'Events', format='table')
from common.util import plotROC
plotROC(preds, truths, output=os.path.join(outdir, 'roc.pdf'))
from root_numpy import array2root
array2root(df.to_records(index=False),
filename=args.predict_output.rsplit('.', 1)[0] +
'.root',
treename='Events',
mode='RECREATE')
def _predict(data_iter, outpath):
# prediction loop
mass_label_name = 'label_%s' % data_iter._data_format.extra_label_vars[
0]
pt_label_name = 'label_%s' % data_iter._data_format.extra_label_vars[1]
preds = []
for eval_batch in data_iter:
# prepare data
data = [
eval_batch.data[idx].as_in_context(devs)
for idx, meta in enumerate(data_iter.provide_data)
]
with mx.autograd.predict_mode():
base_preds = base_classifier_net(*data)
_labels = {
meta[0]: eval_batch.label[idx]
for idx, meta in enumerate(eval_batch.provide_label)
}
_mass = _labels[mass_label_name].as_in_context(devs)
_pt = _labels[pt_label_name].as_in_context(devs)
_rho = 2. * mx.nd.log(
mx.nd.clip(_mass, 1, 1000) / _pt) # rho = ln(m^2/p^2)
_extra = mx.nd.stack(_mass / 1000.,
mx.nd.log(_pt) / 10.,
_rho / 10.,
axis=1)
preds_plus_extra = mx.nd.concat(base_preds, _extra, dim=1)
# forward
with mx.autograd.predict_mode():
predD = netD(preds_plus_extra)
probs = mx.nd.exp(predD)
preds.append(probs.asnumpy())
import numpy as np
preds = np.concatenate(preds)
truths = data_iter.get_truths()
observers = data_iter.get_observers()
print(preds.shape, truths.shape, observers.shape)
pred_output = {}
for i, label in enumerate(data_iter._data_format.class_labels):
pred_output['class_%s' % label] = truths[:, i]
pred_output['score_%s' % label] = preds[:, i]
for i, obs in enumerate(data_iter._data_format.obs_vars):
pred_output[obs] = observers[:, i]
import pandas as pd
df = pd.DataFrame(pred_output)
if outpath:
logging.info('Write prediction file to %s' % outpath)
outdir = os.path.dirname(outpath)
if not os.path.exists(outdir):
os.makedirs(outdir)
df.to_hdf(outpath, 'Events', format='table')
from common.util import plotROC
plotROC(preds, truths, output=os.path.join(outdir, 'roc.pdf'))
from root_numpy import array2root
array2root(df.to_records(index=False),
filename=outpath.rsplit('.', 1)[0] + '.root',
treename='Events',
mode='RECREATE')
def predict(args, symbol, data_loader, **kwargs):
"""
predict with a trained a model
args : argparse returns
data_loader : function that returns the train and val data iterators
"""
# logging
head = '%(asctime)-15s Node[0] %(message)s'
logging.basicConfig(level=logging.DEBUG, format=head)
logging.info('start with arguments %s', args)
# data iterators
data_iter = data_loader(args)
# devices for training
devs = mx.cpu() if args.gpus is None or args.gpus is '' else [
mx.gpu(int(i)) for i in args.gpus.split(',')
]
if len(devs) == 1:
devs = devs[0]
# load model
netD, netAdv, symD, symAdv, symSoftmax = symbol.get_net(
data_iter._data_format.num_classes, use_softmax=True, **vars(args))
_softmaxD, _symAdv, _param_file, _adv_param_file = _load_model(args)
if _softmaxD is not None:
assert symSoftmax.tojson() == _softmaxD.tojson()
netD.load_params(_param_file, ctx=devs)
# prediction loop
preds = []
for eval_batch in data_iter:
# prepare data
data = [
eval_batch.data[idx].as_in_context(devs)
for idx, meta in enumerate(data_iter.provide_data)
]
# forward
with mx.autograd.predict_mode():
predD = netD(*data)
probs = mx.nd.exp(predD)
preds.append(probs.asnumpy())
import numpy as np
preds = np.concatenate(preds)
truths = data_iter.get_truths()
observers = data_iter.get_observers()
print(preds.shape, truths.shape, observers.shape)
pred_output = {}
for i, label in enumerate(data_iter._data_format.class_labels):
pred_output['class_%s' % label] = truths[:, i]
pred_output['score_%s' % label] = preds[:, i]
for i, obs in enumerate(data_iter._data_format.obs_vars):
pred_output[obs] = observers[:, i]
import pandas as pd
df = pd.DataFrame(pred_output)
if args.predict_output:
logging.info('Write prediction file to %s' % args.predict_output)
outdir = os.path.dirname(args.predict_output)
if not os.path.exists(outdir):
os.makedirs(outdir)
df.to_hdf(args.predict_output, 'Events', format='table')
from common.util import plotROC
plotROC(preds, truths, output=os.path.join(outdir, 'roc.pdf'))
from root_numpy import array2root
array2root(df.to_records(index=False),
filename=args.predict_output.rsplit('.', 1)[0] + '.root',
treename='Events',
mode='RECREATE')