def _test_roc_auc_evaluator_with_labels(data1):
"""test `pos_labels` and `ignore_labels` behavior"""
predictor = DummyPredictor()
dataset = NumpyTupleDataset(*data1)
iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
evaluator = ROCAUCEvaluator(
iterator, predictor, name='val',
pos_labels=[1, 2], ignore_labels=-1,
)
# --- test evaluate ---
repo = chainer.Reporter()
repo.add_observer('target', predictor)
with repo:
observation = evaluator.evaluate()
expected_roc_auc = 0.75
# print('observation ', observation)
assert observation['target/roc_auc'] == expected_roc_auc
# --- test __call__ ---
result = evaluator()
# print('result ', result)
assert result['val/main/roc_auc'] == expected_roc_auc
def _test_roc_auc_evaluator_raise_error(data, raise_value_error=True):
predictor = DummyPredictor()
dataset = NumpyTupleDataset(*data)
iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
evaluator = ROCAUCEvaluator(iterator,
predictor,
name='train',
pos_labels=1,
ignore_labels=None,
raise_value_error=raise_value_error)
repo = chainer.Reporter()
repo.add_observer('target', predictor)
with repo:
observation = evaluator.evaluate()
return observation['target/roc_auc']
def _test_roc_auc_evaluator_default_args(data0):
predictor = DummyPredictor()
dataset = NumpyTupleDataset(*data0)
iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
evaluator = ROCAUCEvaluator(
iterator, predictor, name='train',
pos_labels=1, ignore_labels=None
)
repo = chainer.Reporter()
repo.add_observer('target', predictor)
with repo:
observation = evaluator.evaluate()
expected_roc_auc = 0.75
# print('observation ', observation)
assert observation['target/roc_auc'] == expected_roc_auc
# --- test __call__ ---
result = evaluator()
# print('result ', result)
assert result['train/main/roc_auc'] == expected_roc_auc
def main():
args = parse_arguments()
# Set up some useful variables that will be used later on.
dataset_name = args.dataset
method = args.method
num_data = args.num_data
if args.label:
labels = args.label
cache_dir = os.path.join(
'input', '{}_{}_{}'.format(dataset_name, method, labels))
else:
labels = None
cache_dir = os.path.join('input',
'{}_{}_all'.format(dataset_name, method))
# Load the cached dataset.
filename = dataset_part_filename('test', num_data)
path = os.path.join(cache_dir, filename)
if os.path.exists(path):
print('Loading cached dataset from {}.'.format(path))
test = NumpyTupleDataset.load(path)
else:
_, _, test = download_entire_dataset(dataset_name, num_data, labels,
method, cache_dir)
# Model-related data is stored this directory.
model_dir = os.path.join(args.in_dir, os.path.basename(cache_dir))
model_filename = {
'classification': 'classifier.pkl',
'regression': 'regressor.pkl'
}
task_type = molnet_default_config[dataset_name]['task_type']
model_path = os.path.join(model_dir, model_filename[task_type])
print("model_path=" + model_path)
print('Loading model weights from {}...'.format(model_path))
if task_type == 'classification':
model = Classifier.load_pickle(model_path, device=args.gpu)
elif task_type == 'regression':
model = Regressor.load_pickle(model_path, device=args.gpu)
else:
raise ValueError('Invalid task type ({}) encountered when processing '
'dataset ({}).'.format(task_type, dataset_name))
# Re-load the best-validation score snapshot
# serializers.load_npz(os.path.join(
# model_dir, "best_val_" + model_filename[task_type]), model)
# Run an evaluator on the test dataset.
print('Evaluating...')
test_iterator = SerialIterator(test, 16, repeat=False, shuffle=False)
eval_result = Evaluator(test_iterator,
model,
converter=concat_mols,
device=args.gpu)()
print('Evaluation result: ', eval_result)
# Add more stats
if task_type == 'regression':
# loss = cuda.to_cpu(numpy.array(eval_result['main/loss']))
# eval_result['main/loss'] = loss
# convert to native values..
for k, v in eval_result.items():
eval_result[k] = float(v)
elif task_type == "classification":
# For Classifier, we do not equip the model with ROC-AUC evalation function
# use a seperate ROC-AUC Evaluator here
rocauc_result = ROCAUCEvaluator(test_iterator,
model,
converter=concat_mols,
device=args.gpu,
eval_func=model.predictor,
name='test',
ignore_labels=-1)()
print('ROCAUC Evaluation result: ', rocauc_result)
save_json(os.path.join(model_dir, 'rocauc_result.json'), rocauc_result)
else:
print('[WARNING] unknown task_type {}.'.format(task_type))
# Save the evaluation results.
save_json(os.path.join(model_dir, 'eval_result.json'), eval_result)
def main():
parser = argparse.ArgumentParser(
description='Predict with a trained model.')
parser.add_argument('--in-dir',
'-i',
type=str,
default='result',
help='Path to the result directory of the training '
'script.')
parser.add_argument('--batchsize',
'-b',
type=int,
default=128,
help='batch size')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID to use. Negative value indicates '
'not to use GPU and to run the code in CPU.')
parser.add_argument('--model-filename',
type=str,
default='classifier.pkl',
help='file name for pickled model')
parser.add_argument('--num-data',
type=int,
default=-1,
help='Number of data to be parsed from parser.'
'-1 indicates to parse all data.')
args = parser.parse_args()
with open(os.path.join(args.in_dir, 'config.json'), 'r') as i:
config = json.loads(i.read())
method = config['method']
labels = config['labels']
_, test, _ = data.load_dataset(method, labels, num_data=args.num_data)
y_test = test.get_datasets()[-1]
# Load pretrained model
clf = Classifier.load_pickle(os.path.join(args.in_dir,
args.model_filename),
device=args.gpu) # type: Classifier
# ---- predict ---
print('Predicting...')
# We need to feed only input features `x` to `predict`/`predict_proba`.
# This converter extracts only inputs (x1, x2, ...) from the features which
# consist of input `x` and label `t` (x1, x2, ..., t).
def extract_inputs(batch, device=None):
return concat_mols(batch, device=device)[:-1]
def postprocess_pred(x):
x_array = cuda.to_cpu(x.data)
return numpy.where(x_array > 0, 1, 0)
y_pred = clf.predict(test,
converter=extract_inputs,
postprocess_fn=postprocess_pred)
y_proba = clf.predict_proba(test,
converter=extract_inputs,
postprocess_fn=F.sigmoid)
# `predict` method returns the prediction label (0: non-toxic, 1:toxic)
print('y_pread.shape = {}, y_pred[:5, 0] = {}'.format(
y_pred.shape, y_pred[:5, 0]))
# `predict_proba` method returns the probability to be toxic
print('y_proba.shape = {}, y_proba[:5, 0] = {}'.format(
y_proba.shape, y_proba[:5, 0]))
# --- predict end ---
if y_pred.ndim == 1:
y_pred = y_pred[:, None]
if y_pred.shape != y_test.shape:
raise RuntimeError('The shape of the prediction result array and '
'that of the ground truth array do not match. '
'Contents of the input directory may be corrupted '
'or modified.')
statistics = []
for t, p in six.moves.zip(y_test.T, y_pred.T):
idx = t != -1
n_correct = (t[idx] == p[idx]).sum()
n_total = len(t[idx])
accuracy = float(n_correct) / n_total
statistics.append([n_correct, n_total, accuracy])
print('{:>6} {:>8} {:>8} {:>8}'.format('TaskID', 'Correct', 'Total',
'Accuracy'))
for idx, (n_correct, n_total, accuracy) in enumerate(statistics):
print('task{:>2} {:>8} {:>8} {:>8.4f}'.format(idx, n_correct, n_total,
accuracy))
prediction_result_file = 'prediction.npz'
print('Save prediction result to {}'.format(prediction_result_file))
numpy.savez_compressed(prediction_result_file, y_pred)
# --- evaluate ---
# To calc loss/accuracy, we can use `Evaluator`, `ROCAUCEvaluator`
print('Evaluating...')
test_iterator = SerialIterator(test, 16, repeat=False, shuffle=False)
eval_result = Evaluator(test_iterator,
clf,
converter=concat_mols,
device=args.gpu)()
print('Evaluation result: ', eval_result)
rocauc_result = ROCAUCEvaluator(test_iterator,
clf,
converter=concat_mols,
device=args.gpu,
eval_func=clf.predictor,
name='test',
ignore_labels=-1)()
print('ROCAUC Evaluation result: ', rocauc_result)
with open(os.path.join(args.in_dir, 'eval_result.json'), 'w') as f:
json.dump(rocauc_result, f)
def main():
args = parse_arguments()
# Set up some useful variables that will be used later on.
dataset_name = args.dataset
method = args.method
num_data = args.num_data
if args.label:
labels = args.label
cache_dir = os.path.join(
'input', '{}_{}_{}'.format(dataset_name, method, labels))
else:
labels = None
cache_dir = os.path.join('input',
'{}_{}_all'.format(dataset_name, method))
# Load the cached dataset.
filename = dataset_part_filename('test', num_data)
path = os.path.join(cache_dir, filename)
if os.path.exists(path):
print('Loading cached dataset from {}.'.format(path))
test = NumpyTupleDataset.load(path)
else:
_, _, test = download_entire_dataset(dataset_name, num_data, labels,
method, cache_dir)
# # Load the standard scaler parameters, if necessary.
# if args.scale == 'standardize':
# scaler_path = os.path.join(args.in_dir, 'scaler.pkl')
# print('Loading scaler parameters from {}.'.format(scaler_path))
# with open(scaler_path, mode='rb') as f:
# scaler = pickle.load(f)
# else:
# print('No standard scaling was selected.')
# scaler = None
# Model-related data is stored this directory.
model_dir = os.path.join(args.in_dir, os.path.basename(cache_dir))
model_filename = {
'classification': 'classifier.pkl',
'regression': 'regressor.pkl'
}
task_type = molnet_default_config[dataset_name]['task_type']
model_path = os.path.join(model_dir, model_filename[task_type])
print("model_path=" + model_path)
print('Loading model weights from {}...'.format(model_path))
if task_type == 'classification':
model = Classifier.load_pickle(model_path, device=args.gpu)
elif task_type == 'regression':
model = Regressor.load_pickle(model_path, device=args.gpu)
else:
raise ValueError('Invalid task type ({}) encountered when processing '
'dataset ({}).'.format(task_type, dataset_name))
# Proposed by Ishiguro
# ToDo: consider go/no-go with following modification
# Re-load the best-validation score snapshot
serializers.load_npz(
os.path.join(model_dir, "best_val_" + model_filename[task_type]),
model)
# # Replace the default predictor with one that scales the output labels.
# scaled_predictor = ScaledGraphConvPredictor(model.predictor)
# scaled_predictor.scaler = scaler
# model.predictor = scaled_predictor
# Run an evaluator on the test dataset.
print('Evaluating...')
test_iterator = SerialIterator(test, 16, repeat=False, shuffle=False)
eval_result = Evaluator(test_iterator,
model,
converter=concat_mols,
device=args.gpu)()
print('Evaluation result: ', eval_result)
# Proposed by Ishiguro: add more stats
# ToDo: considre go/no-go with the following modification
if task_type == 'regression':
# loss = cuda.to_cpu(numpy.array(eval_result['main/loss']))
# eval_result['main/loss'] = loss
# convert to native values..
for k, v in eval_result.items():
eval_result[k] = float(v)
save_json(os.path.join(args.in_dir, 'eval_result.json'), eval_result)
elif task_type == "classification":
# For Classifier, we do not equip the model with ROC-AUC evalation function
# use a seperate ROC-AUC Evaluator here
rocauc_result = ROCAUCEvaluator(test_iterator,
model,
converter=concat_mols,
device=args.gpu,
eval_func=model.predictor,
name='test',
ignore_labels=-1)()
print('ROCAUC Evaluation result: ', rocauc_result)
save_json(os.path.join(args.in_dir, 'eval_result.json'), rocauc_result)
else:
pass
# Save the evaluation results.
save_json(os.path.join(model_dir, 'eval_result.json'), eval_result)
def main():
# Supported preprocessing/network list
method_list = ['nfp', 'ggnn', 'schnet', 'weavenet', 'nfpdrop', 'ggnndrop']
label_names = D.get_tox21_label_names() + ['pyridine']
iterator_type = ['serial', 'balanced']
parser = argparse.ArgumentParser(
description='Multitask Learning with Tox21.')
parser.add_argument('--method',
'-m',
type=str,
choices=method_list,
default='nfp',
help='graph convolution model to use '
'as a predictor.')
parser.add_argument('--label',
'-l',
type=str,
choices=label_names,
default='',
help='target label for logistic '
'regression. Use all labels if this option '
'is not specified.')
parser.add_argument('--iterator-type',
type=str,
choices=iterator_type,
default='serial',
help='iterator type. If `balanced` '
'is specified, data is sampled to take same number of'
'positive/negative labels during training.')
parser.add_argument('--conv-layers',
'-c',
type=int,
default=4,
help='number of convolution layers')
parser.add_argument('--n-layers',
type=int,
default=1,
help='number of mlp layers')
parser.add_argument('--batchsize',
'-b',
type=int,
default=32,
help='batch size')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID to use. Negative value indicates '
'not to use GPU and to run the code in CPU.')
parser.add_argument('--out',
'-o',
type=str,
default='result',
help='path to output directory')
parser.add_argument('--epoch',
'-e',
type=int,
default=10,
help='number of epochs')
parser.add_argument('--unit-num',
'-u',
type=int,
default=16,
help='number of units in one layer of the model')
parser.add_argument('--resume',
'-r',
type=str,
default='',
help='path to a trainer snapshot')
parser.add_argument('--frequency',
'-f',
type=int,
default=-1,
help='Frequency of taking a snapshot')
parser.add_argument('--dropout-ratio',
'-d',
type=float,
default=0.25,
help='dropout_ratio')
parser.add_argument('--seed', type=int, default=0, help='random seed')
parser.add_argument('--num-train',
type=int,
default=-1,
help='number of training data to be used, '
'negative value indicates use all train data')
args = parser.parse_args()
method = args.method
if args.label:
labels = args.label
class_num = len(labels) if isinstance(labels, list) else 1
else:
labels = None
class_num = len(label_names)
# Dataset preparation
train, val, test, train_smiles, val_smiles, test_smiles = data.load_dataset(
method, labels)
num_train = args.num_train # 100
if num_train > 0:
# reduce size of train data
seed = args.seed # 0
np.random.seed(seed)
train_selected_label = np.random.permutation(np.arange(
len(train)))[:num_train]
print('num_train', num_train, len(train_selected_label),
train_selected_label)
train = NumpyTupleDataset(*train.features[train_selected_label, :])
# Network
predictor_ = predictor.build_predictor(method, args.unit_num,
args.conv_layers, class_num,
args.dropout_ratio, args.n_layers)
iterator_type = args.iterator_type
if iterator_type == 'serial':
train_iter = I.SerialIterator(train, args.batchsize)
elif iterator_type == 'balanced':
if class_num > 1:
raise ValueError('BalancedSerialIterator can be used with only one'
'label classification, please specify label to'
'be predicted by --label option.')
train_iter = BalancedSerialIterator(train,
args.batchsize,
train.features[:, -1],
ignore_labels=-1)
train_iter.show_label_stats()
else:
raise ValueError('Invalid iterator type {}'.format(iterator_type))
val_iter = I.SerialIterator(val,
args.batchsize,
repeat=False,
shuffle=False)
classifier = L.Classifier(predictor_,
lossfun=F.sigmoid_cross_entropy,
accfun=F.binary_accuracy)
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
classifier.to_gpu()
optimizer = O.Adam()
optimizer.setup(classifier)
updater = training.StandardUpdater(train_iter,
optimizer,
device=args.gpu,
converter=concat_mols)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(
E.Evaluator(val_iter,
classifier,
device=args.gpu,
converter=concat_mols))
trainer.extend(E.LogReport())
# --- ROCAUC Evaluator ---
train_eval_iter = I.SerialIterator(train,
args.batchsize,
repeat=False,
shuffle=False)
trainer.extend(
ROCAUCEvaluator(train_eval_iter,
classifier,
eval_func=predictor_,
device=args.gpu,
converter=concat_mols,
name='train'))
trainer.extend(
ROCAUCEvaluator(val_iter,
classifier,
eval_func=predictor_,
device=args.gpu,
converter=concat_mols,
name='val'))
trainer.extend(
E.PrintReport([
'epoch', 'main/loss', 'main/accuracy', 'train/main/roc_auc',
'validation/main/loss', 'validation/main/accuracy',
'val/main/roc_auc', 'elapsed_time'
]))
trainer.extend(E.ProgressBar(update_interval=10))
if args.resume:
chainer.serializers.load_npz(args.resume, trainer)
trainer.run()
with open(os.path.join(args.out, 'args.json'), 'w') as f:
json.dump(vars(args), f, indent=4)
chainer.serializers.save_npz(os.path.join(args.out, 'predictor.npz'),
predictor_)
def main():
label_names = D.get_tox21_label_names()
parser = argparse.ArgumentParser(
description='Predict with a trained model.')
parser.add_argument('--in-dir',
'-i',
type=str,
default='result',
help='Path to the result directory of the training '
'script.')
parser.add_argument('--trainer-snapshot',
'-s',
type=str,
default='',
help='Path to the snapshot file of the Chainer '
'trainer from which serialized model parameters '
'are extracted. If it is not specified, this '
'script searches the training result directory '
'for the latest snapshot, assuming that '
'the naming convension of snapshot files is '
'`snapshot_iter_N` where N is the number of '
'iterations, which is the default configuration '
'of Chainer.')
parser.add_argument('--batchsize',
'-b',
type=int,
default=128,
help='batch size')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID to use. Negative value indicates '
'not to use GPU and to run the code in CPU.')
args = parser.parse_args()
with open(os.path.join(args.in_dir, 'config.json'), 'r') as i:
config = json.loads(i.read())
method = config['method']
labels = config['labels']
if labels:
class_num = len(labels) if isinstance(labels, list) else 1
else:
class_num = len(label_names)
_, test, _ = data.load_dataset(method, labels)
y_test = test.get_datasets()[-1]
# Load pretrained model
predictor_ = predictor.build_predictor(method, config['unit_num'],
config['conv_layers'], class_num)
snapshot_file = args.trainer_snapshot
if not snapshot_file:
snapshot_file = _find_latest_snapshot(args.in_dir)
print('Loading pretrained model parameters from {}'.format(snapshot_file))
chainer.serializers.load_npz(snapshot_file, predictor_,
'updater/model:main/predictor/')
clf = Classifier(predictor=predictor_,
device=args.gpu,
lossfun=F.sigmoid_cross_entropy,
metrics_fun=F.binary_accuracy)
# ---- predict ---
print('Predicting...')
# We need to feed only input features `x` to `predict`/`predict_proba`.
# This converter extracts only inputs (x1, x2, ...) from the features which
# consist of input `x` and label `t` (x1, x2, ..., t).
def extract_inputs(batch, device=None):
return concat_mols(batch, device=device)[:-1]
def postprocess_pred(x):
x_array = cuda.to_cpu(x.data)
return numpy.where(x_array > 0, 1, 0)
y_pred = clf.predict(test,
converter=extract_inputs,
postprocess_fn=postprocess_pred)
y_proba = clf.predict_proba(test,
converter=extract_inputs,
postprocess_fn=F.sigmoid)
# `predict` method returns the prediction label (0: non-toxic, 1:toxic)
print('y_pread.shape = {}, y_pred[:5, 0] = {}'.format(
y_pred.shape, y_pred[:5, 0]))
# `predict_proba` method returns the probability to be toxic
print('y_proba.shape = {}, y_proba[:5, 0] = {}'.format(
y_proba.shape, y_proba[:5, 0]))
# --- predict end ---
if y_pred.ndim == 1:
y_pred = y_pred[:, None]
if y_pred.shape != y_test.shape:
raise RuntimeError('The shape of the prediction result array and '
'that of the ground truth array do not match. '
'Contents of the input directory may be corrupted '
'or modified.')
statistics = []
for t, p in six.moves.zip(y_test.T, y_pred.T):
idx = t != -1
n_correct = (t[idx] == p[idx]).sum()
n_total = len(t[idx])
accuracy = float(n_correct) / n_total
statistics.append([n_correct, n_total, accuracy])
print('{:>6} {:>8} {:>8} {:>8}'.format('TaskID', 'Correct', 'Total',
'Accuracy'))
for idx, (n_correct, n_total, accuracy) in enumerate(statistics):
print('task{:>2} {:>8} {:>8} {:>8.4f}'.format(idx, n_correct, n_total,
accuracy))
prediction_result_file = 'prediction.npz'
print('Save prediction result to {}'.format(prediction_result_file))
numpy.savez_compressed(prediction_result_file, y_pred)
# --- evaluate ---
# To calc loss/accuracy, we can use `Evaluator`, `ROCAUCEvaluator`
print('Evaluating...')
test_iterator = SerialIterator(test, 16, repeat=False, shuffle=False)
eval_result = Evaluator(test_iterator,
clf,
converter=concat_mols,
device=args.gpu)()
print('Evaluation result: ', eval_result)
rocauc_result = ROCAUCEvaluator(test_iterator,
clf,
converter=concat_mols,
device=args.gpu,
eval_func=predictor_,
name='test',
ignore_labels=-1)()
print('ROCAUC Evaluation result: ', rocauc_result)