def test_set_optimizer(self):
n_data_points = 20
n_features = 2
X = np.random.rand(n_data_points, n_features)
y = [[0, 1] for x in range(n_data_points)]
dataset = NumpyDataset(X, y)
features = Feature(shape=(None, n_features))
dense = Dense(out_channels=2, in_layers=[features])
output = SoftMax(in_layers=[dense])
label = Label(shape=(None, 2))
smce = SoftMaxCrossEntropy(in_layers=[label, dense])
loss = ReduceMean(in_layers=[smce])
tg = dc.models.TensorGraph(learning_rate=0.01, use_queue=False)
tg.add_output(output)
tg.set_loss(loss)
global_step = tg.get_global_step()
learning_rate = ExponentialDecay(initial_rate=0.1,
decay_rate=0.96,
decay_steps=100000)
tg.set_optimizer(GradientDescent(learning_rate=learning_rate))
tg.fit(dataset, nb_epoch=1000)
prediction = np.squeeze(tg.predict_on_batch(X))
tg.save()
tg1 = TensorGraph.load_from_dir(tg.model_dir)
prediction2 = np.squeeze(tg1.predict_on_batch(X))
assert_true(np.all(np.isclose(prediction, prediction2, atol=0.01)))
def test_ANI_multitask_regression_overfit(self):
"""Test ANI-1 regression overfits tiny data."""
input_file = os.path.join(self.current_dir, "example_DTNN.mat")
np.random.seed(123)
tf.set_random_seed(123)
dataset = scipy.io.loadmat(input_file)
X = np.concatenate([np.expand_dims(dataset['Z'], 2), dataset['R']],
axis=2)
X = X[:, :13, :]
y = dataset['T']
w = np.ones_like(y)
dataset = dc.data.DiskDataset.from_numpy(X, y, w, ids=None)
regression_metric = dc.metrics.Metric(dc.metrics.pearson_r2_score,
mode="regression")
n_tasks = y.shape[1]
batch_size = 10
transformers = [
dc.trans.NormalizationTransformer(transform_y=True,
dataset=dataset),
]
for transformer in transformers:
dataset = transformer.transform(dataset)
model = dc.models.ANIRegression(n_tasks,
13,
atom_number_cases=[1, 6, 7, 8],
batch_size=batch_size,
learning_rate=ExponentialDecay(
0.01, 0.7, 100),
mode="regression")
# Fit trained model
model.fit(dataset, nb_epoch=500)
# Eval model on train
scores = model.evaluate(dataset, [regression_metric],
transformers[0:1])
assert scores[regression_metric.name] > .7
train_smiles = train_dataset.ids
tokens = set()
for s in train_smiles:
tokens = tokens.union(set(s))
tokens = sorted(list(tokens))
max_length = max(len(s) for s in train_smiles)
#training
from deepchem.models.tensorgraph.optimizers import Adam, ExponentialDecay
from deepchem.models.tensorgraph.models.seqtoseq import AspuruGuzikAutoEncoder
#the encoder is a CNN and the decoder is a GRU
model = AspuruGuzikAutoEncoder(tokens, max_length, model_dir='vae')
batches_per_epoch = len(train_smiles)/model.batch_size
learning_rate = ExponentialDecay(0.001, 0.95, batches_per_epoch)
model.set_optimizer(Adam(learning_rate=learning_rate))
def generate_sequences(epochs):
for i in range(epochs):
for s in train_smiles:
yield (s, s)
model.summary()
model.fit_sequences(generate_sequences(1))
#check that the molecules are valid
import numpy as np
from rdkit import Chem
predictions = model.predict_from_embeddings(np.random.normal(size=(1000,196)))
molecules = []
for p in predictions:
tokens = sorted(list(tokens))
print(tokens[0:5])
max_length = max(len(s) for s in train_smiles)
model = dc.models.SeqToSeq(tokens,
tokens,
max_length,
encoder_layers=2,
decoder_layers=2,
embedding_dimension=256,
model_dir='fingerprint')
batches_per_epoch = len(train_smiles) / model.batch_size
model.set_optimizer(
Adam(learning_rate=ExponentialDecay(0.004, 0.9, batches_per_epoch)))
def generate_sequences(epochs):
for i in range(epochs):
for s in train_smiles:
yield (s, s)
model.fit_sequences(generate_sequences(40))
predicted = model.predict_from_sequences(valid_smiles[:500])
count = 0
for s, p in zip(valid_smiles[:500], predicted):
if ''.join(p) == s:
count += 1
from deepchem import metrics
from deepchem.metrics import Metric
from deepchem.metrics import to_one_hot
from deepchem.utils.evaluate import Evaluator
from deepchem.models import MultiTaskClassifier
from deepchem.models.tensorgraph.optimizers import ExponentialDecay
np.random.seed(123)
pcba_tasks, pcba_datasets, transformers = load_pcba()
(train_dataset, valid_dataset, test_dataset) = pcba_datasets
metric = Metric(metrics.roc_auc_score, np.mean, mode="classification")
n_features = train_dataset.get_data_shape()[0]
rate = ExponentialDecay(0.001, 0.8, 1000)
model = MultiTaskClassifier(
len(pcba_tasks),
n_features,
dropouts=[.25],
learning_rate=rate,
weight_init_stddevs=[.1],
batch_size=64)
# Fit trained model
model.fit(train_dataset)
model.save()
train_evaluator = Evaluator(model, train_dataset, transformers)
train_scores = train_evaluator.compute_model_performance([metric])
# Fit models
metric = [
dc.metrics.Metric(dc.metrics.mean_absolute_error, mode="regression"),
dc.metrics.Metric(dc.metrics.pearson_r2_score, mode="regression")
]
# Batch size of models
batch_size = 50
n_embedding = 30
n_distance = 51
distance_min = -1.
distance_max = 9.2
n_hidden = 15
rate = ExponentialDecay(0.0001, 0.97, 5000)
model = dc.models.DTNNModel(len(tasks),
n_embedding=n_embedding,
n_hidden=n_hidden,
n_distance=n_distance,
distance_min=distance_min,
distance_max=distance_max,
output_activation=False,
batch_size=batch_size,
learning_rate=rate,
use_queue=False,
mode="regression")
#model.restore()
# Fit trained model
model.fit(train_dataset, nb_epoch=3000)
tokens = tokens.union(set(c for c in s))
tokens = sorted(list(tokens))
print(tokens[0:5])
max_length = max(len(s) for s in train_smiles)
model = dc.models.SeqToSeq(tokens,
tokens,
max_length,
encoder_layers=2,
decoder_layers=2,
embedding_dimension=256,
model_dir='fingerprint')
batches_per_epoch = len(train_smiles)/model.batch_size
model.set_optimizer(Adam(learning_rate=ExponentialDecay(0.004, 0.9, batches_per_epoch)))
def generate_sequences(epochs):
for i in range(epochs):
for s in train_smiles:
yield (s, s)
model.fit_sequences(generate_sequences(40))
predicted = model.predict_from_sequences(valid_smiles[:500])
count = 0
for s,p in zip(valid_smiles[:500], predicted):
if ''.join(p) == s:
count += 1
print('reproduced', count, 'of 500 validation SMILES strings')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--img_spec", default="std", help="Image specification to load")
parser.add_argument("--early_stopping_epoch", default=10, type=int, help="Number of epochs to check early stopping after for")
parser.add_argument("--use_augment", action='store_true', help="Whether to perform real-time augmentation")
parser.add_argument("--base_filters", default=16, type=int, help="Number of base filters to use")
parser.add_argument("--inception_per_block", default=3, type=int, help="Number of inception layers per block")
parser.add_argument("--batch_size", default=32, type=int, help="Batch size used for training")
parser.add_argument("--learning_rate", default=1e-4, type=float, help="Learning rate used.")
parser.add_argument("--dataset", default="tox21", help="Dataset to train on.")
args = parser.parse_args()
layers_per_block = args.inception_per_block
inception_blocks = {"A": layers_per_block, "B": layers_per_block, "C": layers_per_block}
mode = get_task_mode(args.dataset)
DIRNAME = os.path.join(os.environ.get("SCRATCH", "./"), "deepchem-data")
load_fn = loaders[args.dataset]
tasks, dataset, transformers = load_fn(featurizer="smiles2img", data_dir=DIRNAME, save_dir=DIRNAME, img_spec=args.img_spec, split="stratified")
metric_type = metric_types[args.dataset]
task_averager = np.mean
if len(tasks) == 1:
task_averager = None
metric = dc.metrics.Metric(metric_type, task_averager=task_averager, mode=mode, verbose=False)
train, valid, test = dataset
# Setup directory for experiment
exp_name = dt.now().strftime("%d-%m-%Y--%H-%M-%S")
hparams_dir = "filters_{}_blocklayers_{}_imgspec_{}".format(args.base_filters, layers_per_block, args.img_spec)
model_dir_1 = os.path.join(DIRNAME, args.dataset, "chemception", hparams_dir, exp_name)
# Optimizer and logging
optimizer = RMSProp(learning_rate=args.learning_rate)
logger.info("Dataset used: {}".format(args.dataset))
logger.info("Args used: {}".format(args))
logger.info("Num_tasks: {}".format(len(tasks)))
###### TRAINING FIRST PART WITH CONSTANT LEARNING RATE ###############
model = ChemCeption(n_tasks=len(tasks), img_spec=args.img_spec,
inception_blocks=inception_blocks,
base_filters=args.base_filters, augment=args.use_augment,
model_dir=model_dir_1, mode=mode,
n_classes=2, batch_size=args.batch_size,
optimizer=optimizer, tensorboard=True,
tensorboard_log_frequency=100)
model._ensure_built()
train, valid, test = dataset
logger.info("Created model dir at {}".format(model_dir_1))
best_models_dir_1 = os.path.join(DIRNAME, args.dataset, "chemception", hparams_dir, "best-models", exp_name)
logger.info("Saving best model so far")
model.save_checkpoint(model_dir=best_models_dir_1)
loss_old = compute_loss_on_valid(valid, model, tasks, mode=mode)
train_scores = model.evaluate(train, [metric], [])
valid_scores = model.evaluate(valid, [metric], [])
test_scores = model.evaluate(test, [metric], [])
logger.info("Train-{}: {}".format(metric.name, train_scores[metric.name]))
logger.info("Valid-{}: {}".format(metric.name, valid_scores[metric.name]))
logger.info("Test-{}: {}".format(metric.name, test_scores[metric.name]))
for rep_num in range(2):
logger.info("Training model for {} epochs.".format(args.early_stopping_epoch))
model.fit(train, nb_epoch=args.early_stopping_epoch, checkpoint_interval=0)
loss_new = compute_loss_on_valid(valid, model, tasks, mode=mode, verbose=False)
train_scores = model.evaluate(train, [metric], [])
valid_scores = model.evaluate(valid, [metric], [])
test_scores = model.evaluate(test, [metric], [])
logger.info("Train-{}: {}".format(metric.name, train_scores[metric.name]))
logger.info("Valid-{}: {}".format(metric.name, valid_scores[metric.name]))
logger.info("Test-{}: {}".format(metric.name, test_scores[metric.name]))
logger.info("Computed loss on validation set after {} epochs: {}".format(args.early_stopping_epoch, loss_new))
if loss_new > loss_old:
logger.info("No improvement in validation loss. Enforcing early stopping.")
break
logger.info("Saving best model so far")
model.save_checkpoint(model_dir=best_models_dir_1)
loss_old = loss_new
######### TRAINING SECOND PART WITH DECAYING LEARNING RATE
# Optimizer and logging
decay_steps = args.early_stopping_epoch * train.y.shape[0] // args.batch_size
logger.info("Decay steps: {}".format(decay_steps))
lr = ExponentialDecay(initial_rate=args.learning_rate, decay_rate=0.92, decay_steps=decay_steps, staircase=True)
optimizer = RMSProp(learning_rate=lr)
# Setup directory for experiment
exp_name = dt.now().strftime("%d-%m-%Y--%H-%M-%S")
hparams_dir = "filters_{}_blocklayers_{}_imgspec_{}".format(args.base_filters, layers_per_block, args.img_spec)
new_model = ChemCeption(n_tasks=len(tasks), img_spec=args.img_spec,
inception_blocks=inception_blocks,
base_filters=args.base_filters, augment=args.use_augment,
model_dir=model_dir_1, mode=mode,
n_classes=2, batch_size=args.batch_size,
optimizer=optimizer, tensorboard=True,
tensorboard_log_frequency=100)
new_model.restore(model_dir=best_models_dir_1)
best_models_dir_2 = os.path.join(DIRNAME, args.dataset, "chemception", hparams_dir, "best-models", exp_name)
logger.info("Created best model dir for second stage at {}".format(best_models_dir_2))
loss_old = compute_loss_on_valid(valid, new_model, tasks, mode=mode)
for rep_num in range(2):
logger.info("Training model for {} epochs.".format(args.early_stopping_epoch))
new_model.fit(train, nb_epoch=args.early_stopping_epoch, checkpoint_interval=0)
loss_new = compute_loss_on_valid(valid, new_model, tasks, mode=mode, verbose=False)
train_scores = new_model.evaluate(train, [metric], [])
valid_scores = new_model.evaluate(valid, [metric], [])
test_scores = new_model.evaluate(test, [metric], [])
logger.info("Train-{}: {}".format(metric.name, train_scores[metric.name]))
logger.info("Valid-{}: {}".format(metric.name, valid_scores[metric.name]))
logger.info("Test-{}: {}".format(metric.name, test_scores[metric.name]))
logger.info("Computed loss on validation set after {} epochs: {}".format(args.early_stopping_epoch, loss_new))
if loss_new > loss_old:
logger.info("No improvement in validation loss. Enforcing early stopping.")
break
logger.info("Saving best model so far")
new_model.save_checkpoint(model_dir=best_models_dir_2)
loss_old = loss_new