def test_tf_multitask_regression_overfit(self): """Test tf multitask overfits tiny data.""" n_tasks = 10 n_samples = 10 n_features = 3 n_classes = 2 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.zeros((n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = Metric(metrics.mean_squared_error, verbosity=verbosity, task_averager=np.mean, mode="regression") tensorflow_model = TensorflowMultiTaskRegressor( n_tasks, n_features, self.model_dir, dropouts=[0.], learning_rate=0.0003, weight_init_stddevs=[.1], batch_size=n_samples, verbosity=verbosity) model = TensorflowModel(tensorflow_model, self.model_dir) # Fit trained model model.fit(dataset, nb_epoch=50) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] < .1
def test_sklearn_multitask_regression_overfit(self): """Test SKLearn singletask-to-multitask overfits tiny regression data.""" n_tasks = 2 tasks = ["task%d" % task for task in range(n_tasks)] n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.rand(n_samples, n_tasks) w = np.ones((n_samples, n_tasks)) dataset = DiskDataset.from_numpy(self.train_dir, X, y, w, ids) verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity, task_averager=np.mean) def model_builder(model_dir): sklearn_model = RandomForestRegressor() return SklearnModel(sklearn_model, model_dir) model = SingletaskToMultitask(tasks, model_builder, self.model_dir) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .7
def test_tf_skewed_classification_overfit(self): """Test tensorflow models can overfit 0/1 datasets with few actives.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} #n_samples = 100 n_samples = 100 n_features = 3 n_tasks = len(tasks) n_classes = 2 # Generate dummy dataset np.random.seed(123) p = .05 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "layer_sizes": [1500], "dropouts": [.0], "learning_rate": 0.003, "momentum": .9, "batch_size": n_samples, "num_classification_tasks": 1, "num_classes": n_classes, "num_features": n_features, "weight_init_stddevs": [1.], "bias_init_consts": [1.], "nb_epoch": 200, "penalty": 0.0, "optimizer": "adam", "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = TensorflowModel(tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .8
def test_tf_classification_overfit(self): """Test that tensorflow models can overfit simple classification datasets.""" n_samples = 10 n_features = 3 n_tasks = 1 n_classes = 2 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.zeros((n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity) tensorflow_model = TensorflowMultiTaskClassifier( n_tasks, n_features, self.model_dir, dropouts=[0.], learning_rate=0.0003, weight_init_stddevs=[.1], batch_size=n_samples, verbosity=verbosity) model = TensorflowModel(tensorflow_model, self.model_dir) # Fit trained model model.fit(dataset, nb_epoch=100) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_sklearn_classification_overfit(self): """Test that sklearn models can overfit simple classification datasets.""" n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) sklearn_model = RandomForestClassifier() model = SklearnModel(sklearn_model, self.model_dir) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_tf_regression_overfit(self): """Test that TensorFlow models can overfit simple regression datasets.""" n_samples = 10 n_features = 3 n_tasks = 1 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.zeros((n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = Metric(metrics.mean_squared_error, verbosity=verbosity) # TODO(rbharath): This breaks with optimizer="momentum". Why? tensorflow_model = TensorflowMultiTaskRegressor( n_tasks, n_features, self.model_dir, dropouts=[0.], learning_rate=0.003, weight_init_stddevs=[np.sqrt(6)/np.sqrt(1000)], batch_size=n_samples, verbosity=verbosity) model = TensorflowModel(tensorflow_model, self.model_dir) # Fit trained model model.fit(dataset, nb_epoch=100) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] < .1
def evaluate(self, dataset, metrics, transformers=[], per_task_metrics=False): """ Evaluates the performance of this model on specified dataset. Parameters ---------- dataset: dc.data.Dataset Dataset object. metric: deepchem.metrics.Metric Evaluation metric transformers: list List of deepchem.transformers.Transformer per_task_metrics: bool If True, return per-task scores. Returns ------- dict Maps tasks to scores under metric. """ evaluator = Evaluator(self, dataset, transformers) if not per_task_metrics: scores = evaluator.compute_model_performance(metrics) return scores else: scores, per_task_scores = evaluator.compute_model_performance( metrics, per_task_metrics=per_task_metrics) return scores, per_task_scores
def evaluate(self, dataset, metrics, transformers=[], per_task_metrics=False): """ Evaluates the performance of this model on specified dataset. Parameters ---------- dataset: dc.data.Dataset Dataset object. metric: deepchem.metrics.Metric Evaluation metric transformers: list List of deepchem.transformers.Transformer per_task_metrics: bool If True, return per-task scores. Returns ------- dict Maps tasks to scores under metric. """ evaluator = Evaluator(self, dataset, transformers) if not per_task_metrics: scores = evaluator.compute_model_performance(metrics) return scores else: scores, per_task_scores = evaluator.compute_model_performance( metrics, per_task_metrics=per_task_metrics) return scores, per_task_scores
def test_keras_multitask_regression_overfit(self): """Test keras multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 10 n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity, task_averager=np.mean, mode="regression") keras_model = MultiTaskDNN(n_tasks, n_features, "regression", dropout=0., learning_rate=.1, decay=1e-4) model = KerasModel(keras_model, self.model_dir, verbosity=verbosity) # Fit trained model model.fit(dataset, nb_epoch=100) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .75
def test_graph_conv_singletask_classification_overfit(self): """Test graph-conv multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 1 n_samples = 10 n_features = 3 n_classes = 2 # Load mini log-solubility dataset. splittype = "scaffold" featurizer = ConvMolFeaturizer() tasks = ["outcome"] task_type = "classification" task_types = {task: task_type for task in tasks} input_file = os.path.join(self.current_dir, "example_classification.csv") loader = DataLoader(tasks=tasks, smiles_field=self.smiles_field, featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file, self.data_dir) verbosity = "high" classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity) #n_atoms = 50 n_feat = 71 batch_size = 10 graph_model = SequentialGraphModel(n_feat) graph_model.add(GraphConv(64, activation='relu')) graph_model.add(BatchNormalization(epsilon=1e-5, mode=1)) graph_model.add(GraphPool()) # Gather Projection graph_model.add(Dense(128, activation='relu')) graph_model.add(BatchNormalization(epsilon=1e-5, mode=1)) graph_model.add(GraphGather(batch_size, activation="tanh")) with self.test_session() as sess: model = MultitaskGraphClassifier( sess, graph_model, n_tasks, self.model_dir, batch_size=batch_size, learning_rate=1e-3, learning_rate_decay_time=1000, optimizer_type="adam", beta1=.9, beta2=.999, verbosity="high") # Fit trained model model.fit(dataset, nb_epoch=20) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) ######################################################### DEBUG print("scores") print(scores) ######################################################### DEBUG assert scores[classification_metric.name] > .85
def test_tf_multitask_classification_overfit(self): """Test tf multitask overfits tiny data.""" n_tasks = 10 tasks = ["task%d" % task for task in range(n_tasks)] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 n_classes = 2 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) #y = np.random.randint(n_classes, size=(n_samples, n_tasks)) y = np.zeros((n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "layer_sizes": [1000], "dropouts": [.0], "learning_rate": 0.0003, "momentum": .9, "batch_size": n_samples, "num_classification_tasks": n_tasks, "num_classes": n_classes, "num_features": n_features, "weight_init_stddevs": [.1], "bias_init_consts": [1.], "nb_epoch": 100, "penalty": 0.0, "optimizer": "adam", "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity) model = TensorflowModel(tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_sklearn_multitask_classification_overfit(self): """Test SKLearn singletask-to-multitask overfits tiny data.""" n_tasks = 10 tasks = ["task%d" % task for task in range(n_tasks)] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "batch_size": None, "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) def model_builder(tasks, task_types, model_params, model_dir, verbosity=None): return SklearnModel(tasks, task_types, model_params, model_dir, mode="classification", model_instance=RandomForestClassifier(), verbosity=verbosity) model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir, model_builder, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_tf_reload(self): """Test that tensorflow models can overfit simple classification datasets.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) n_classes = 2 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(n_classes, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "layer_sizes": [1000], "dropouts": [0.0], "learning_rate": 0.003, "momentum": 0.9, "batch_size": n_samples, "num_classification_tasks": 1, "num_classes": n_classes, "num_features": n_features, "weight_init_stddevs": [1.0], "bias_init_consts": [1.0], "nb_epoch": 100, "penalty": 0.0, "optimizer": "adam", "data_shape": dataset.get_data_shape(), } verbosity = "high" classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity) model = TensorflowModel( tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity ) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_model = TensorflowModel( tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity ) reloaded_model.reload() assert reloaded_model.eval_model._restored_model # Eval model on train transformers = [] evaluator = Evaluator(reloaded_model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > 0.9
def test_singletask_to_multitask_classification(self): splittype = "scaffold" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] output_transformers = [] tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: "classification" for task in tasks} input_file = "multitask_example.csv" n_features = 10 n_tasks = len(tasks) # Define train dataset n_train = 100 X_train = np.random.rand(n_train, n_features) y_train = np.random.randint(2, size=(n_train, n_tasks)) w_train = np.ones_like(y_train) ids_train = ["C"] * n_train train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train, w_train, ids_train, tasks) # Define test dataset n_test = 10 X_test = np.random.rand(n_test, n_features) y_test = np.random.randint(2, size=(n_test, n_tasks)) w_test = np.ones_like(y_test) ids_test = ["C"] * n_test test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test, w_test, ids_test, tasks) params_dict = { "batch_size": 32, "data_shape": train_dataset.get_data_shape() } classification_metrics = [Metric(metrics.roc_auc_score)] def model_builder(tasks, task_types, model_params, model_builder, verbosity=None): return SklearnModel(tasks, task_types, model_params, model_builder, model_instance=LogisticRegression()) multitask_model = SingletaskToMultitask(tasks, task_types, params_dict, self.model_dir, model_builder) # Fit trained model multitask_model.fit(train_dataset) multitask_model.save() # Eval multitask_model on train evaluator = Evaluator(multitask_model, train_dataset, output_transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics) # Eval multitask_model on test evaluator = Evaluator(multitask_model, test_dataset, output_transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics)
def _evaluate(self,model,metrics,output_transformers,*datasets): scores=[] for dataset in datasets: evaluator= Evaluator( model, dataset, output_transformers) score = evaluator.compute_model_performance(metrics) scores.append(score) # print(scores) return scores
def eval_trained_model(model_type, model_dir, data_dir, csv_out, stats_out): """Evaluates a trained model on specified data.""" model = Model.load(model_type, model_dir) data = Dataset(data_dir) evaluator = Evaluator(model, data, verbose=True) _, perf_df = evaluator.compute_model_performance(csv_out, stats_out) print("Model Performance.") print(perf_df)
def test_sklearn_multitask_classification(self): """Test that sklearn models can learn on simple multitask classification.""" np.random.seed(123) n_tasks = 4 dataset = sklearn.datasets.load_digits(n_class=2) X, y = dataset.data, dataset.target y = np.reshape(y, (len(y), 1)) y = np.hstack([y] * n_tasks) frac_train = .7 n_samples = len(X) X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples] X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:] train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train) test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test) tasks = train_dataset.get_task_names() task_types = {task: "classification" for task in tasks} model_params = { "batch_size": None, "data_shape": train_dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) def model_builder(tasks, task_types, model_params, model_dir, verbosity=None): return SklearnModel(tasks, task_types, model_params, model_dir, mode="classification", model_instance=LogisticRegression(), verbosity=verbosity) model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir, model_builder, verbosity=verbosity) # Fit trained model model.fit(train_dataset) model.save() # Eval model on train transformers = [] train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity) train_scores = train_evaluator.compute_model_performance([classification_metric]) print("train_scores") print(train_scores) # Eval model on test transformers = [] evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) print("scores") print(scores) for score in scores[classification_metric.name]: assert score > .5
def test_tf_regression_overfit(self): """Test that TensorFlow models can overfit simple regression datasets.""" tasks = ["task0"] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.zeros((n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "layer_sizes": [1000], "dropouts": [.0], "learning_rate": 0.003, "momentum": .9, "batch_size": n_samples, "num_regression_tasks": 1, "num_features": n_features, "weight_init_stddevs": [np.sqrt(6) / np.sqrt(1000)], "bias_init_consts": [1.], "nb_epoch": 100, "penalty": 0.0, "optimizer": "momentum", "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.mean_squared_error, verbosity=verbosity) model = TensorflowModel(tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskRegressor, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] < .1
def test_keras_skewed_classification_overfit(self): """Test keras models can overfit 0/1 datasets with few actives.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 100 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) p = .05 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "nb_hidden": 1000, "activation": "relu", "dropout": .0, "learning_rate": .15, "momentum": .9, "nesterov": False, "decay": 1e-4, "batch_size": n_samples, "nb_epoch": 200, "init": "glorot_uniform", "nb_layers": 1, "batchnorm": False, "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_keras_reload(self): """Test that trained keras models can be reloaded correctly.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "nb_hidden": 1000, "activation": "relu", "dropout": 0.0, "learning_rate": 0.15, "momentum": 0.9, "nesterov": False, "decay": 1e-4, "batch_size": n_samples, "nb_epoch": 200, "init": "glorot_uniform", "nb_layers": 1, "batchnorm": False, "data_shape": dataset.get_data_shape(), } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir, verbosity=verbosity) reloaded_model.reload() # Eval model on train transformers = [] evaluator = Evaluator(reloaded_model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > 0.9
def test_tf_skewed_classification_overfit(self): """Test tensorflow models can overfit 0/1 datasets with few actives.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} #n_samples = 100 n_samples = 100 n_features = 3 n_tasks = len(tasks) n_classes = 2 # Generate dummy dataset np.random.seed(123) p = .05 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "layer_sizes": [1500], "dropouts": [.0], "learning_rate": 0.003, "momentum": .9, "batch_size": n_samples, "num_classification_tasks": 1, "num_classes": n_classes, "num_features": n_features, "weight_init_stddevs": [1.], "bias_init_consts": [1.], "nb_epoch": 200, "penalty": 0.0, "optimizer": "adam", "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = TensorflowModel( tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .8
def test_multitask_keras_mlp_ECFP_classification_API(self): """Straightforward test of Keras multitask deepchem classification API.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): task_type = "classification" # TODO(rbharath): There should be some automatic check to ensure that all # required model_params are specified. # TODO(rbharath): Turning off dropout to make tests behave. model_params = {"nb_hidden": 10, "activation": "relu", "dropout": .0, "learning_rate": .01, "momentum": .9, "nesterov": False, "decay": 1e-4, "batch_size": 5, "nb_epoch": 2, "init": "glorot_uniform", "nb_layers": 1, "batchnorm": False} input_file = os.path.join(self.current_dir, "multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: task_type for task in tasks} featurizer = CircularFingerprint(size=1024) loader = DataLoader(tasks=tasks, smiles_field=self.smiles_field, featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file, self.data_dir) splitter = ScaffoldSplitter() train_dataset, test_dataset = splitter.train_test_split( dataset, self.train_dir, self.test_dir) transformers = [] model_params["data_shape"] = train_dataset.get_data_shape() classification_metrics = [Metric(metrics.roc_auc_score), Metric(metrics.matthews_corrcoef), Metric(metrics.recall_score), Metric(metrics.accuracy_score)] model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir) # Fit trained model model.fit(train_dataset) model.save() # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics)
def test_multiclass_classification_singletask(): """Test multiclass classification evaluation.""" X = np.random.rand(100, 5) y = np.random.randint(5, size=(100,)) dataset = dc.data.NumpyDataset(X, y) model = dc.models.MultitaskClassifier(1, 5, n_classes=5) evaluator = Evaluator(model, dataset, []) multitask_scores = evaluator.compute_model_performance( dc.metrics.roc_auc_score, n_classes=5) assert len(multitask_scores) == 1 assert multitask_scores["metric-1"] >= 0
def test_keras_multitask_regression_overfit(self): """Test keras multitask overfits tiny data.""" n_tasks = 10 tasks = ["task%d" % task for task in range(n_tasks)] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "nb_hidden": 1000, "activation": "relu", "dropout": .0, "learning_rate": .15, "momentum": .9, "nesterov": False, "decay": 1e-4, "batch_size": n_samples, "nb_epoch": 200, "init": "glorot_uniform", "nb_layers": 1, "batchnorm": False, "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .9
def test_keras_reload(self): """Test that trained keras models can be reloaded correctly.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) keras_model = MultiTaskDNN(n_tasks, n_features, "classification", dropout=0.) model = KerasModel(keras_model, self.model_dir) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_keras_model = MultiTaskDNN(n_tasks, n_features, "classification", dropout=0.) reloaded_model = KerasModel(reloaded_keras_model, self.model_dir) reloaded_model.reload( custom_objects={"MultiTaskDNN": MultiTaskDNN}) # Eval model on train transformers = [] evaluator = Evaluator(reloaded_model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance( [classification_metric]) assert scores[classification_metric.name] > .6
def test_sklearn_reload(self): """Test that trained model can be reloaded correctly.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "batch_size": None, "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="classification", model_instance=RandomForestClassifier()) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="classification") reloaded_model.reload() # Eval model on train transformers = [] evaluator = Evaluator(reloaded_model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_tf_multitask_regression_overfit(self): """Test tf multitask overfits tiny data.""" n_tasks = 10 tasks = ["task%d" % task for task in range(n_tasks)] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 n_classes = 2 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) #y = np.random.randint(n_classes, size=(n_samples, n_tasks)) y = np.zeros((n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "layer_sizes": [1000], "dropouts": [.0], "learning_rate": 0.0003, "momentum": .9, "batch_size": n_samples, "num_regression_tasks": n_tasks, "num_classes": n_classes, "num_features": n_features, "weight_init_stddevs": [.1], "bias_init_consts": [1.], "nb_epoch": 100, "penalty": 0.0, "optimizer": "adam", "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) model = TensorflowModel( tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskRegressor, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .9
def test_evaluator_dc_metric(): """Test an evaluator on a dataset.""" X = np.random.rand(10, 5) y = np.random.rand(10, 1) dataset = dc.data.NumpyDataset(X, y) model = dc.models.MultitaskRegressor(1, 5) evaluator = Evaluator(model, dataset, []) metric = dc.metrics.Metric(dc.metrics.mae_score) multitask_scores = evaluator.compute_model_performance(metric) assert isinstance(multitask_scores, dict) assert len(multitask_scores) == 1 assert multitask_scores['mae_score'] > 0
def test_keras_multitask_regression_overfit(self): """Test keras multitask overfits tiny data.""" g = tf.Graph() sess = tf.Session(graph=g) K.set_session(sess) with g.as_default(): n_tasks = 10 tasks = ["task%d" % task for task in range(n_tasks)] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "nb_hidden": 1000, "activation": "relu", "dropout": .0, "learning_rate": .15, "momentum": .9, "nesterov": False, "decay": 1e-4, "batch_size": n_samples, "nb_epoch": 200, "init": "glorot_uniform", "nb_layers": 1, "batchnorm": False, "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .9
def test_sklearn_multiclass_classification_singletask(): """Test multiclass classification evaluation.""" X = np.random.rand(100, 5) y = np.random.randint(5, size=(100,)) dataset = dc.data.NumpyDataset(X, y) rf = sklearn.ensemble.RandomForestClassifier(50) model = dc.models.SklearnModel(rf) model.fit(dataset) evaluator = Evaluator(model, dataset, []) multitask_scores = evaluator.compute_model_performance( dc.metrics.roc_auc_score, n_classes=5) assert len(multitask_scores) == 1 assert multitask_scores["metric-1"] >= 0
def test_tf_reload(self): """Test that tensorflow models can overfit simple classification datasets.""" n_samples = 10 n_features = 3 n_tasks = 1 n_classes = 2 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(n_classes, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity) tensorflow_model = TensorflowMultiTaskClassifier(n_tasks, n_features, self.model_dir, dropouts=[0.], verbosity=verbosity) model = TensorflowModel(tensorflow_model, self.model_dir) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_tensorflow_model = TensorflowMultiTaskClassifier( n_tasks, n_features, self.model_dir, dropouts=[0.], verbosity=verbosity) reloaded_model = TensorflowModel(reloaded_tensorflow_model, self.model_dir) reloaded_model.reload() # Eval model on train transformers = [] evaluator = Evaluator(reloaded_model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .6
def test_sklearn_classification(self): """Test that sklearn models can learn on simple classification datasets.""" np.random.seed(123) dataset = sklearn.datasets.load_digits(n_class=2) X, y = dataset.data, dataset.target frac_train = .7 n_samples = len(X) X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples] X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:] print("X_train.shape, y_train.shape, X_test.shape, y_test.shape") print(X_train.shape, y_train.shape, X_test.shape, y_test.shape) train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train) test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test) tasks = train_dataset.get_task_names() task_types = {task: "classification" for task in tasks} model_params = { "batch_size": None, "data_shape": train_dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="classification", model_instance=LogisticRegression()) # Fit trained model model.fit(train_dataset) model.save() # Eval model on train transformers = [] train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity) train_scores = train_evaluator.compute_model_performance([classification_metric]) print("train_scores") print(train_scores) # Eval model on test transformers = [] evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) print("scores") print(scores) assert scores[classification_metric.name] > .5
def test_singletask_sklearn_rf_RDKIT_descriptor_regression_API(self): """Test of singletask RF RDKIT-descriptor regression API.""" splittype = "scaffold" featurizer = RDKitDescriptors() tasks = ["log-solubility"] task_type = "regression" task_types = {task: task_type for task in tasks} model_params = {} input_file = os.path.join(self.current_dir, "example.csv") loader = DataLoader(tasks=tasks, smiles_field=self.smiles_field, featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file, self.data_dir) splitter = ScaffoldSplitter() train_dataset, test_dataset = splitter.train_test_split( dataset, self.train_dir, self.test_dir) input_transformers = [ NormalizationTransformer(transform_X=True, dataset=train_dataset), ClippingTransformer(transform_X=True, dataset=train_dataset)] output_transformers = [ NormalizationTransformer(transform_y=True, dataset=train_dataset)] transformers = input_transformers + output_transformers for dataset in [train_dataset, test_dataset]: for transformer in transformers: transformer.transform(dataset) model_params["data_shape"] = train_dataset.get_data_shape() regression_metrics = [Metric(metrics.r2_score), Metric(metrics.mean_squared_error), Metric(metrics.mean_absolute_error)] model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="regression", model_instance=RandomForestRegressor()) # Fit trained model model.fit(train_dataset) model.save() # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(regression_metrics) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(regression_metrics)
def test_gc_multiclass_classification(): """Test multiclass classification evaluation.""" np.random.seed(1234) smiles = ["C", "CC"] featurizer = dc.feat.ConvMolFeaturizer() X = featurizer.featurize(smiles) y = np.random.randint(5, size=(len(smiles),)) dataset = dc.data.NumpyDataset(X, y) model = dc.models.GraphConvModel(1, mode="classification", n_classes=5) evaluator = Evaluator(model, dataset, []) multitask_scores = evaluator.compute_model_performance( dc.metrics.accuracy_score, n_classes=5) assert len(multitask_scores) == 1 assert multitask_scores["metric-1"] >= 0
def test_evaluator_sklearn_metric(): """Test an evaluator on a dataset.""" X = np.random.rand(10, 5) y = np.random.rand(10, 1) dataset = dc.data.NumpyDataset(X, y) model = dc.models.MultitaskRegressor(1, 5) evaluator = Evaluator(model, dataset, []) multitask_scores = evaluator.compute_model_performance( dc.metrics.mean_absolute_error) assert isinstance(multitask_scores, dict) assert len(multitask_scores) == 1 # Note that since no name as provided, metrics are index by order # given. assert multitask_scores['metric-1'] > 0
def test_multitask_model_evaluate_sklearn(): """Test evaluation of a multitask metric.""" X = np.random.rand(10, 5) y = np.random.rand(10, 2) dataset = dc.data.NumpyDataset(X, y) model = dc.models.MultitaskRegressor(2, 5) evaluator = Evaluator(model, dataset, []) multitask_scores, all_task_scores = evaluator.compute_model_performance( dc.metrics.mean_absolute_error, per_task_metrics=True) assert isinstance(multitask_scores, dict) assert len(multitask_scores) == 1 assert multitask_scores['metric-1'] > 0 assert isinstance(all_task_scores, dict) assert len(multitask_scores) == 1
def test_gc_binary_classification(): """Test multiclass classification evaluation.""" smiles = ["C", "CC"] featurizer = dc.feat.ConvMolFeaturizer() X = featurizer.featurize(smiles) y = np.random.randint(2, size=(len(smiles),)) dataset = dc.data.NumpyDataset(X, y) model = dc.models.GraphConvModel(1, mode="classification") # TODO: Fix this case with correct thresholding evaluator = Evaluator(model, dataset, []) multitask_scores = evaluator.compute_model_performance( dc.metrics.accuracy_score, n_classes=2) assert len(multitask_scores) == 1 assert multitask_scores["metric-1"] >= 0
def test_singletask_sklearn_rf_ECFP_regression_sharded_API(self): """Test of singletask RF ECFP regression API: sharded edition.""" splittype = "scaffold" featurizer = CircularFingerprint(size=1024) model_params = {} tasks = ["label"] task_type = "regression" task_types = {task: task_type for task in tasks} input_file = os.path.join( self.current_dir, "../../../datasets/pdbbind_core_df.pkl.gz") loader = DataLoader(tasks=tasks, smiles_field=self.smiles_field, featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file, self.data_dir) splitter = ScaffoldSplitter() train_dataset, test_dataset = splitter.train_test_split( dataset, self.train_dir, self.test_dir) input_transformers = [] output_transformers = [ NormalizationTransformer(transform_y=True, dataset=train_dataset)] transformers = input_transformers + output_transformers for dataset in [train_dataset, test_dataset]: for transformer in transformers: transformer.transform(dataset) # We set shard size above to force the creation of multiple shards of the data. # pdbbind_core has ~200 examples. model_params["data_shape"] = train_dataset.get_data_shape() regression_metrics = [Metric(metrics.r2_score), Metric(metrics.mean_squared_error), Metric(metrics.mean_absolute_error)] model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="regression", model_instance=RandomForestRegressor()) # Fit trained model model.fit(train_dataset) model.save() # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(regression_metrics) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(regression_metrics)
def test_API(self): """Straightforward test of multitask deepchem classification API.""" splittype = "scaffold" feature_types = ["ECFP"] output_transforms = [] input_transforms = [] task_type = "classification" # TODO(rbharath): There should be some automatic check to ensure that all # required model_params are specified. model_params = {"nb_hidden": 10, "activation": "relu", "dropout": .5, "learning_rate": .01, "momentum": .9, "nesterov": False, "decay": 1e-4, "batch_size": 5, "nb_epoch": 2} model_name = "multitask_deep_classifier" # Featurize input featurizer = DataFeaturizer(tasks=self.tasks, smiles_field=self.smiles_field, verbose=True) feature_files = featurizer.featurize(self.input_file, feature_types, self.feature_dir) # Transform data into arrays for ML samples = FeaturizedSamples(self.samplesdir, feature_files, reload_data=False) # Split into train/test train_samples, test_samples = samples.train_test_split( splittype, self.train_dir, self.test_dir) train_dataset = Dataset(self.train_dir, train_samples, feature_types) test_dataset = Dataset(self.test_dir, test_samples, feature_types) # Transforming train/test data train_dataset.transform(input_transforms, output_transforms) test_dataset.transform(input_transforms, output_transforms) # Fit model task_types = {task: task_type for task in self.tasks} model_params["data_shape"] = train_dataset.get_data_shape() model = Model.model_builder(model_name, task_types, model_params) model.fit(train_dataset) model.save(self.model_dir) # Eval model on train evaluator = Evaluator(model, test_dataset, verbose=True) with tempfile.NamedTemporaryFile() as test_csv_out: with tempfile.NamedTemporaryFile() as test_stats_out: evaluator.compute_model_performance(test_csv_out, test_stats_out)
def test_multitask_evaluator(): """Test evaluation of a multitask metric.""" n_tasks = 2 X = np.random.rand(10, 5) y = np.random.rand(10, 2, 1) dataset = dc.data.NumpyDataset(X, y) model = dc.models.MultitaskRegressor(2, 5) evaluator = Evaluator(model, dataset, []) metric = dc.metrics.Metric(dc.metrics.mae_score) multitask_scores, all_task_scores = evaluator.compute_model_performance( metric, per_task_metrics=True) assert isinstance(multitask_scores, dict) assert len(multitask_scores) == 1 assert multitask_scores['mae_score'] > 0 assert isinstance(all_task_scores, dict) assert len(multitask_scores) == 1
def test_tf_skewed_missing_classification_overfit(self): """TF, skewed data, few actives Test tensorflow models overfit 0/1 datasets with missing data and few actives. This is intended to be as close to singletask MUV datasets as possible. """ n_samples = 5120 n_features = 6 n_tasks = 1 n_classes = 2 # Generate dummy dataset np.random.seed(123) p = .002 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) y_flat, w_flat = np.squeeze(y), np.squeeze(w) y_nonzero = y_flat[w_flat != 0] num_nonzero = np.count_nonzero(y_nonzero) weight_nonzero = len(y_nonzero)/num_nonzero w_flat[y_flat != 0] = weight_nonzero w = np.reshape(w_flat, (n_samples, n_tasks)) dataset = NumpyDataset(X, y, w, ids) verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) tensorflow_model = TensorflowMultiTaskClassifier( n_tasks, n_features, self.model_dir, dropouts=[0.], learning_rate=0.003, weight_init_stddevs=[1.], batch_size=n_samples, verbosity=verbosity) model = TensorflowModel(tensorflow_model, self.model_dir) # Fit trained model model.fit(dataset, nb_epoch=50) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .8
def test_sklearn_regression_overfit(self): """Test that sklearn models can overfit simple regression datasets.""" tasks = ["task0"] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.rand(n_samples, n_tasks) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "batch_size": None, "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="regression", model_instance=RandomForestRegressor()) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .7
def test_sklearn_reload(self): """Test that trained model can be reloaded correctly.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.randint(2, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = {"batch_size": None, "data_shape": dataset.get_data_shape()} verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = SklearnModel( tasks, task_types, model_params, self.model_dir, mode="classification", model_instance=RandomForestClassifier(), ) # Fit trained model model.fit(dataset) model.save() # Load trained model reloaded_model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="classification") reloaded_model.reload() # Eval model on train transformers = [] evaluator = Evaluator(reloaded_model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > 0.9
def test_sklearn_multitask_regression_overfit(self): """Test SKLearn singletask-to-multitask overfits tiny regression data.""" n_tasks = 2 tasks = ["task%d" % task for task in range(n_tasks)] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.rand(n_samples, n_tasks) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "batch_size": None, "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) def model_builder(tasks, task_types, model_params, model_dir, verbosity=None): return SklearnModel(tasks, task_types, model_params, model_dir, mode="regression", model_instance=RandomForestRegressor(), verbosity=verbosity) model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir, model_builder, verbosity=verbosity) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .7
def evaluate(self, dataset, metrics, transformers=[]): """ Evaluates the performance of this model on specified dataset. Parameters ---------- dataset: dc.data.Dataset Dataset object. metric: deepchem.metrics.Metric Evaluation metric transformers: list List of deepchem.transformers.Transformer Returns ------- dict Maps tasks to scores under metric. """ evaluator = Evaluator(self, dataset, transformers) scores = evaluator.compute_model_performance(metrics) return scores
def evaluate(self, dataset, metrics, transformers=[]): """ Evaluates the performance of this model on specified dataset. Parameters ---------- dataset: dc.data.Dataset Dataset object. metric: deepchem.metrics.Metric Evaluation metric transformers: list List of deepchem.transformers.Transformer Returns ------- dict Maps tasks to scores under metric. """ evaluator = Evaluator(self, dataset, transformers) scores = evaluator.compute_model_performance(metrics) return scores
def _create_model(self, train_dataset, test_dataset, model, transformers, metrics): """Helper method to create model for test.""" # Fit trained model model.fit(train_dataset) model.save(self.model_dir) # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbose=True) with tempfile.NamedTemporaryFile() as train_csv_out: with tempfile.NamedTemporaryFile() as train_stats_out: _, _, _ = evaluator.compute_model_performance( metrics, train_csv_out, train_stats_out) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbose=True) with tempfile.NamedTemporaryFile() as test_csv_out: with tempfile.NamedTemporaryFile() as test_stats_out: _, _, _ = evaluator.compute_model_performance( metrics, test_csv_out, test_stats_out)
def _create_model(self, splittype, feature_types, input_transforms, output_transforms, task_type, model_params, model_name, input_file, tasks, protein_pdb_field=None, ligand_pdb_field=None): """Helper method to create model for test.""" # Featurize input input_file = os.path.join(self.current_dir, input_file) featurizer = DataFeaturizer(tasks=tasks, smiles_field=self.smiles_field, protein_pdb_field=protein_pdb_field, ligand_pdb_field=ligand_pdb_field, verbose=True) feature_files = featurizer.featurize(input_file, feature_types, self.feature_dir) # Transform data into arrays for ML samples = FeaturizedSamples(self.samplesdir, feature_files, reload_data=False) # Split into train/test train_samples, test_samples = samples.train_test_split( splittype, self.train_dir, self.test_dir) train_dataset = Dataset(self.train_dir, train_samples, feature_types) test_dataset = Dataset(self.test_dir, test_samples, feature_types) # Transforming train/test data train_dataset.transform(input_transforms, output_transforms) test_dataset.transform(input_transforms, output_transforms) # Fit model task_types = {task: task_type for task in tasks} model_params["data_shape"] = train_dataset.get_data_shape() model = Model.model_builder(model_name, task_types, model_params) model.fit(train_dataset) model.save(self.model_dir) # Eval model on train evaluator = Evaluator(model, test_dataset, verbose=True) with tempfile.NamedTemporaryFile() as test_csv_out: with tempfile.NamedTemporaryFile() as test_stats_out: _, _ = evaluator.compute_model_performance( test_csv_out, test_stats_out)
def create_and_eval_model(train_dataset, test_dataset, task_type, model_params, model_name, model_dir, tasks): """Helper method to create model for test.""" # Fit model task_types = {task: task_type for task in tasks} model_params["data_shape"] = train_dataset.get_data_shape() print("Creating Model object.") import deepchem.models.deep model = Model.model_builder(model_name, task_types, model_params) print("About to fit model") model.fit(train_dataset) print("Done fitting, about to save...") model.save(model_dir) # Eval model on train evaluator = Evaluator(model, train_dataset, verbose=True) with tempfile.NamedTemporaryFile() as train_csv_out: with tempfile.NamedTemporaryFile() as train_stats_out: _, performance_df = evaluator.compute_model_performance( train_csv_out, train_stats_out) print("train_performance_df") print(performance_df) evaluator = Evaluator(model, test_dataset, verbose=True) with tempfile.NamedTemporaryFile() as test_csv_out: with tempfile.NamedTemporaryFile() as test_stats_out: _, performance_df = evaluator.compute_model_performance( test_csv_out, test_stats_out) print("test_performance_df") print(performance_df) return performance_df.iterrows().next()[1]["r2_score"]
def test_sklearn_skewed_classification_overfit(self): """Test sklearn models can overfit 0/1 datasets with few actives.""" tasks = ["task0"] task_types = {task: "classification" for task in tasks} n_samples = 100 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) p = .05 ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.binomial(1, p, size=(n_samples, n_tasks)) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "batch_size": None, "data_shape": dataset.get_data_shape() } verbosity = "high" classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="classification", model_instance=RandomForestClassifier()) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([classification_metric]) assert scores[classification_metric.name] > .9
def test_sklearn_regression_overfit(self): """Test that sklearn models can overfit simple regression datasets.""" tasks = ["task0"] task_types = {task: "regression" for task in tasks} n_samples = 10 n_features = 3 n_tasks = len(tasks) # Generate dummy dataset np.random.seed(123) ids = np.arange(n_samples) X = np.random.rand(n_samples, n_features) y = np.random.rand(n_samples, n_tasks) w = np.ones((n_samples, n_tasks)) dataset = Dataset.from_numpy(self.train_dir, X, y, w, ids, tasks) model_params = { "batch_size": None, "data_shape": dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="regression", model_instance=RandomForestRegressor()) # Fit trained model model.fit(dataset) model.save() # Eval model on train transformers = [] evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) assert scores[regression_metric.name] > .7
def f(l00=0, l01=0, l02=0, l03=0, l04=0, l05=0, l06=0, l07=0, l08=0, l09=0, l10=0, l11=0, l12=0, l13=0, l14=0, l15=0, l16=0, l17=0, l18=0, l19=0): """ Optimizing function Take in hyper parameter values and return valid set performances Parameters ---------- l00~l19: int or float placeholders for hyperparameters being optimized, hyper_parameters dict is rebuilt based on input values of placeholders Returns: -------- valid_scores: float valid set performances """ args = locals() # Input hyper parameters i = 0 for hp in hp_list_single: hyper_parameters[hp] = float(args[param_name[i]]) if param_range[i][0] == 'int': hyper_parameters[hp] = int(hyper_parameters[hp]) i = i + 1 for hp in hp_list_multiple: hyper_parameters[hp[0]] = [ float(args[param_name[j]]) for j in range(i, i + hp[1]) ] if param_range[i][0] == 'int': hyper_parameters[hp[0]] = map(int, hyper_parameters[hp[0]]) i = i + hp[1] logger.info(hyper_parameters) # Run benchmark with open(log_file, 'a') as f: # Record hyperparameters f.write(str(hyper_parameters)) f.write('\n') if isinstance(self.model_class, str) or isinstance( self.model_class, unicode): try: train_scores, valid_scores, _ = benchmark_classification( train_dataset, valid_dataset, valid_dataset, ['task_placeholder'] * n_tasks, output_transformers, n_features, metric, self.model_class, hyper_parameters=hyper_parameters) except AssertionError: train_scores, valid_scores, _ = benchmark_regression( train_dataset, valid_dataset, valid_dataset, ['task_placeholder'] * n_tasks, output_transformers, n_features, metric, self.model_class, hyper_parameters=hyper_parameters) score = valid_scores[self.model_class][metric[0].name] else: model_dir = tempfile.mkdtemp() model = self.model_class(hyper_parameters, model_dir) model.fit(train_dataset, **hyper_parameters) model.save() evaluator = Evaluator(model, valid_dataset, output_transformers) multitask_scores = evaluator.compute_model_performance(metric) score = multitask_scores[metric[0].name] with open(log_file, 'a') as f: # Record performances f.write(str(score)) f.write('\n') # GPGO maximize performance by default, set performance to its negative value for minimization if direction: return score else: return -score
def hyperparam_search(self, params_dict, train_dataset, valid_dataset, output_transformers, metric, use_max=True, logdir=None): """Perform hyperparams search according to params_dict. Each key to hyperparams_dict is a model_param. The values should be a list of potential values for that hyperparam. TODO(rbharath): This shouldn't be stored in a temporary directory. """ hyperparams = params_dict.keys() hyperparam_vals = params_dict.values() for hyperparam_list in params_dict.values(): assert isinstance(hyperparam_list, collections.Iterable) number_combinations = reduce(mul, [len(vals) for vals in hyperparam_vals]) valid_csv_out = tempfile.NamedTemporaryFile() valid_stats_out = tempfile.NamedTemporaryFile() if use_max: best_validation_score = -np.inf else: best_validation_score = np.inf best_hyperparams = None best_model, best_model_dir = None, None all_scores = {} for ind, hyperparameter_tuple in enumerate(itertools.product(*hyperparam_vals)): model_params = {} log("Fitting model %d/%d" % (ind+1, number_combinations), self.verbose) for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple): model_params[hyperparam] = hyperparam_val log("hyperparameters: %s" % str(model_params), self.verbose) if logdir is not None: model_dir = os.path.join(logdir, str(ind)) log("model_dir is %s" % model_dir, self.verbose) try: os.makedirs(model_dir) except OSError: if not os.path.isdir(model_dir): log("Error creating model_dir, using tempfile directory", self.verbose) model_dir = tempfile.mkdtemp() else: model_dir = tempfile.mkdtemp() model = self.model_class(model_params, model_dir) model.fit(train_dataset, **model_params) model.save() evaluator = Evaluator(model, valid_dataset, output_transformers) multitask_scores = evaluator.compute_model_performance( [metric], valid_csv_out.name, valid_stats_out.name) valid_score = multitask_scores[metric.name] all_scores[str(hyperparameter_tuple)] = valid_score if (use_max and valid_score >= best_validation_score) or ( not use_max and valid_score <= best_validation_score): best_validation_score = valid_score best_hyperparams = hyperparameter_tuple if best_model_dir is not None: shutil.rmtree(best_model_dir) best_model_dir = model_dir best_model = model else: shutil.rmtree(model_dir) log("Model %d/%d, Metric %s, Validation set %s: %f" % (ind+1, number_combinations, metric.name, ind, valid_score), self.verbose) log("\tbest_validation_score so far: %f" % best_validation_score, self.verbose) if best_model is None: log("No models trained correctly.", self.verbose) # arbitrarily return last model best_model, best_hyperparams = model, hyperparameter_tuple return best_model, best_hyperparams, all_scores train_csv_out = tempfile.NamedTemporaryFile() train_stats_out = tempfile.NamedTemporaryFile() train_evaluator = Evaluator(best_model, train_dataset, output_transformers) multitask_scores = train_evaluator.compute_model_performance( [metric], train_csv_out.name, train_stats_out.name) train_score = multitask_scores[metric.name] log("Best hyperparameters: %s" % str(best_hyperparams), self.verbose) log("train_score: %f" % train_score, self.verbose) log("validation_score: %f" % best_validation_score, self.verbose) return best_model, best_hyperparams, all_scores
def hyperparam_search(self, params_dict, train_dataset, valid_dataset, output_transformers, metric, use_max=True, logdir=None): """Perform hyperparams search according to params_dict. Each key to hyperparams_dict is a model_param. The values should be a list of potential values for that hyperparam. """ hyperparams = params_dict.keys() hyperparam_vals = params_dict.values() for hyperparam_list in params_dict.itervalues(): assert isinstance(hyperparam_list, collections.Iterable) number_combinations = reduce(mul, [len(vals) for vals in hyperparam_vals]) valid_csv_out = tempfile.NamedTemporaryFile() valid_stats_out = tempfile.NamedTemporaryFile() if use_max: best_validation_score = -np.inf else: best_validation_score = np.inf best_hyperparams = None best_model, best_model_dir = None, None all_scores = {} for ind, hyperparameter_tuple in enumerate(itertools.product(*hyperparam_vals)): model_params = {} for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple): model_params[hyperparam] = hyperparam_val if logdir is not None: model_dir = logdir else: model_dir = tempfile.mkdtemp() if logdir is not None: #TODO(JG) Fit transformers for TF models model = self.model_class(self.task_types, model_params, model_dir, verbosity=self.verbosity) else: if self.fit_transformers: model = self.model_class(self.task_types, model_params, fit_transformers=self.fit_transformers, verbosity=self.verbosity) else: model = self.model_class(self.task_types, model_params, verbosity=self.verbosity) model.fit(train_dataset) model.save(model_dir) evaluator = Evaluator(model, valid_dataset, output_transformers) df, scores_df, multitask_scores = evaluator.compute_model_performance( [metric], valid_csv_out, valid_stats_out) if not metric.is_multitask: valid_score = scores_df.iloc[0][metric.name] else: valid_score = multitask_scores[metric.name] all_scores[hyperparameter_tuple] = valid_score if (use_max and valid_score >= best_validation_score) or ( not use_max and valid_score <= best_validation_score): best_validation_score = valid_score best_hyperparams = hyperparameter_tuple if best_model_dir is not None: shutil.rmtree(best_model_dir) best_model_dir = model_dir best_model = model else: shutil.rmtree(model_dir) log("Model %d/%d, Metric %s, Validation set %s: %f" % (ind, number_combinations, metric.name, ind, valid_score), self.verbosity) log("\tbest_validation_score so far: %f" % best_validation_score, self.verbosity) if best_model is None: log("No models trained correctly.", self.verbosity) return best_model, best_hyperparams, all_scores train_csv_out = tempfile.NamedTemporaryFile() train_stats_out = tempfile.NamedTemporaryFile() train_evaluator = Evaluator(best_model, train_dataset, output_transformers) train_df, train_score, multitask_scores = train_evaluator.compute_model_performance( [metric], train_csv_out, train_stats_out) if not metric.is_multitask: train_score = train_score.iloc[0][metric.name] else: train_score = multitask_scores[metric.name] log("Best hyperparameters: %s" % str(zip(hyperparams, best_hyperparams)), self.verbosity) log("train_score: %f" % train_score, self.verbosity) log("validation_score: %f" % best_validation_score, self.verbosity) return best_model, best_hyperparams, all_scores
np.random.seed(123) pcba_tasks, pcba_datasets, transformers = load_pcba() (train_dataset, valid_dataset) = pcba_datasets metric = Metric(metrics.roc_auc_score, np.mean, mode="classification") model = TensorflowMultiTaskClassifier( len(pcba_tasks), n_features, model_dir, dropouts=[.25], learning_rate=0.001, weight_init_stddevs=[.1], batch_size=64, verbosity="high") # Fit trained model model.fit(train_dataset) model.save() train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity) train_scores = train_evaluator.compute_model_performance([metric]) print("Train scores") print(train_scores) valid_evaluator = Evaluator(model, valid_dataset, transformers, verbosity=verbosity) valid_scores = valid_evaluator.compute_model_performance([metric]) print("Validation scores") print(valid_scores)
def test_singletask_tf_mlp_ECFP_classification_API(self): """Straightforward test of Tensorflow singletask deepchem classification API.""" splittype = "scaffold" output_transformers = [] input_transformers = [] task_type = "classification" featurizer = CircularFingerprint(size=1024) tasks = ["outcome"] task_type = "classification" task_types = {task: task_type for task in tasks} input_file = os.path.join(self.current_dir, "example_classification.csv") loader = DataLoader(tasks=tasks, smiles_field=self.smiles_field, featurizer=featurizer, verbosity="low") dataset = loader.featurize(input_file, self.data_dir) splitter = ScaffoldSplitter() train_dataset, test_dataset = splitter.train_test_split( dataset, self.train_dir, self.test_dir) input_transformers = [] output_transformers = [ NormalizationTransformer(transform_y=True, dataset=train_dataset)] transformers = input_transformers + output_transformers for dataset in [train_dataset, test_dataset]: for transformer in transformers: transformer.transform(dataset) model_params = { "batch_size": 2, "num_classification_tasks": 1, "num_features": 1024, "layer_sizes": [1024], "weight_init_stddevs": [1.], "bias_init_consts": [0.], "dropouts": [.5], "num_classes": 2, "nb_epoch": 1, "penalty": 0.0, "optimizer": "adam", "learning_rate": .001, "data_shape": train_dataset.get_data_shape() } classification_metrics = [Metric(metrics.roc_auc_score), Metric(metrics.matthews_corrcoef), Metric(metrics.recall_score), Metric(metrics.accuracy_score)] model = TensorflowModel( tasks, task_types, model_params, self.model_dir, tf_class=TensorflowMultiTaskClassifier) # Fit trained model model.fit(train_dataset) model.save() # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics)
def test_sklearn_transformed_regression(self): """Test that sklearn models can learn on simple transformed regression datasets.""" np.random.seed(123) dataset = sklearn.datasets.load_diabetes() X, y = dataset.data, dataset.target frac_train = .7 n_samples = len(X) X_train, y_train = X[:frac_train*n_samples], y[:frac_train*n_samples] X_test, y_test = X[frac_train*n_samples:], y[frac_train*n_samples:] train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train) test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test) # Eval model on train input_transformers = [ NormalizationTransformer(transform_X=True, dataset=train_dataset), ClippingTransformer(transform_X=True, dataset=train_dataset)] output_transformers = [ NormalizationTransformer(transform_y=True, dataset=train_dataset)] transformers = input_transformers + output_transformers for transformer in transformers: transformer.transform(train_dataset) for transformer in transformers: transformer.transform(test_dataset) tasks = train_dataset.get_task_names() task_types = {task: "regression" for task in tasks} model_params = { "batch_size": None, "data_shape": train_dataset.get_data_shape() } verbosity = "high" regression_metric = Metric(metrics.r2_score, verbosity=verbosity) model = SklearnModel(tasks, task_types, model_params, self.model_dir, mode="regression", model_instance=LinearRegression()) # Fit trained model model.fit(train_dataset) model.save() train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity) train_scores = train_evaluator.compute_model_performance([regression_metric]) print("train_scores") print(train_scores) assert train_scores[regression_metric.name] > .5 # Eval model on test transformers = [] evaluator = Evaluator(model, test_dataset, transformers, verbosity=verbosity) scores = evaluator.compute_model_performance([regression_metric]) print("scores") print(scores) assert scores[regression_metric.name] > .5
print("About to perform train/valid/test split.") splitter = RandomSplitter(verbosity=verbosity) print("Performing new split.") train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split( nci_dataset, train_dir, valid_dir, test_dir) classification_metric = Metric(metrics.roc_auc_score, np.mean, verbosity=verbosity, mode="classification") def model_builder(model_dir): sklearn_model = RandomForestRegressor(n_estimators=500) return SklearnModel(sklearn_model, model_dir) model = SingletaskToMultitask(nci_tasks, model_builder, model_dir) # Fit trained model model.fit(train_dataset) model.save() train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity) train_scores = train_evaluator.compute_model_performance([classification_metric]) print("Train scores") print(train_scores) valid_evaluator = Evaluator(model, valid_dataset, transformers, verbosity=verbosity) valid_scores = valid_evaluator.compute_model_performance([classification_metric]) print("Validation scores") print(valid_scores)
def bace_rf_model(mode="classification", verbosity="high", split="20-80"): """Train random forests on BACE dataset.""" (bace_tasks, train_dataset, valid_dataset, test_dataset, crystal_dataset, transformers) = load_bace(mode=mode, transform=False, split=split) if mode == "regression": r2_metric = Metric(metrics.r2_score, verbosity=verbosity) rms_metric = Metric(metrics.rms_score, verbosity=verbosity) mae_metric = Metric(metrics.mae_score, verbosity=verbosity) all_metrics = [r2_metric, rms_metric, mae_metric] metric = r2_metric model_class = RandomForestRegressor def rf_model_builder(model_params, model_dir): sklearn_model = RandomForestRegressor(**model_params) return SklearnModel(sklearn_model, model_dir) elif mode == "classification": roc_auc_metric = Metric(metrics.roc_auc_score, verbosity=verbosity) accuracy_metric = Metric(metrics.accuracy_score, verbosity=verbosity) mcc_metric = Metric(metrics.matthews_corrcoef, verbosity=verbosity) # Note sensitivity = recall recall_metric = Metric(metrics.recall_score, verbosity=verbosity) model_class = RandomForestClassifier all_metrics = [accuracy_metric, mcc_metric, recall_metric, roc_auc_metric] metric = roc_auc_metric def rf_model_builder(model_params, model_dir): sklearn_model = RandomForestClassifier(**model_params) return SklearnModel(sklearn_model, model_dir) else: raise ValueError("Invalid mode %s" % mode) params_dict = { "n_estimators": [10, 100], "max_features": ["auto", "sqrt", "log2", None], } optimizer = HyperparamOpt(rf_model_builder, verbosity="low") best_rf, best_rf_hyperparams, all_rf_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, transformers, metric=metric) if len(train_dataset) > 0: rf_train_evaluator = Evaluator(best_rf, train_dataset, transformers, verbosity=verbosity) csv_out = "rf_%s_%s_train.csv" % (mode, split) stats_out = "rf_%s_%s_train_stats.txt" % (mode, split) rf_train_score = rf_train_evaluator.compute_model_performance( all_metrics, csv_out=csv_out, stats_out=stats_out) print("RF Train set scores: %s" % (str(rf_train_score))) if len(valid_dataset) > 0: rf_valid_evaluator = Evaluator(best_rf, valid_dataset, transformers, verbosity=verbosity) csv_out = "rf_%s_%s_valid.csv" % (mode, split) stats_out = "rf_%s_%s_valid_stats.txt" % (mode, split) rf_valid_score = rf_valid_evaluator.compute_model_performance( all_metrics, csv_out=csv_out, stats_out=stats_out) print("RF Valid set scores: %s" % (str(rf_valid_score))) if len(test_dataset) > 0: rf_test_evaluator = Evaluator(best_rf, test_dataset, transformers, verbosity=verbosity) csv_out = "rf_%s_%s_test.csv" % (mode, split) stats_out = "rf_%s_%s_test_stats.txt" % (mode, split) rf_test_score = rf_test_evaluator.compute_model_performance( all_metrics, csv_out=csv_out, stats_out=stats_out) print("RF Test set: %s" % (str(rf_test_score))) if len(crystal_dataset) > 0: rf_crystal_evaluator = Evaluator(best_rf, crystal_dataset, transformers, verbosity) csv_out = "rf_%s_%s_crystal.csv" % (mode, split) stats_out = "rf_%s_%s_crystal_stats.txt" % (mode, split) rf_crystal_score = rf_crystal_evaluator.compute_model_performance( all_metrics, csv_out=csv_out, stats_out=stats_out) print("RF Crystal set: %s" % (str(rf_crystal_score)))