def get_model_training_data_by_uuid(uuid): """Retrieve data used to train, validate, and test a model given the uuid Args: uuid (str): model uuid Returns: a tuple of datafraes containint training data, validation data, and test data including the compound ID, RDKIT SMILES, and response value """ model_meta = get_metadata_by_uuid(uuid) response_col = model_meta['training_dataset']['response_cols'] smiles_col = model_meta['training_dataset']['smiles_col'] full_data = dsf.retrieve_dataset_by_dataset_oid(model_meta['training_dataset']['dataset_oid'], verbose=False) # Pull split data and merge into initial dataset split_meta = dsf.search_datasets_by_key_value('split_dataset_uuid', model_meta['splitting_parameters']['Splitting']['split_uuid']) split_oid = split_meta['dataset_oid'].values[0] split_data = dsf.retrieve_dataset_by_dataset_oid(split_oid, verbose=False) split_data['compound_id'] = split_data['cmpd_id'] split_data = split_data.drop(columns=['cmpd_id']) full_data = pd.merge(full_data, split_data, how='inner', on=['compound_id']) train_data = full_data[full_data['subset'] == 'train'][['compound_id',smiles_col,*response_col]].reset_index(drop=True) valid_data = full_data[full_data['subset'] == 'valid'][['compound_id',smiles_col,*response_col]].reset_index(drop=True) test_data = full_data[full_data['subset'] == 'test'][['compound_id',smiles_col,*response_col]].reset_index(drop=True) return train_data, valid_data, test_data
def get_model_training_data_by_uuid(uuid): """Retrieve data used to train, validate, and test a model given the uuid Args: uuid (str): model uuid Returns: a tuple of datafraes containint training data, validation data, and test data including the compound ID, RDKIT SMILES, and response value """ if not mlmt_supported: print( "Model tracker not supported in your environment; can load models from filesystem only." ) return None model_meta = get_metadata_by_uuid(uuid) response_col = model_meta['training_dataset']['response_cols'] smiles_col = model_meta['training_dataset']['smiles_col'] id_col = model_meta['training_dataset']['id_col'] full_data = dsf.retrieve_dataset_by_dataset_oid( model_meta['training_dataset']['dataset_oid']) # Pull split data and merge into initial dataset split_meta = dsf.search_datasets_by_key_value( 'split_dataset_uuid', model_meta['splitting_parameters']['split_uuid']) split_oid = split_meta['dataset_oid'].values[0] split_data = dsf.retrieve_dataset_by_dataset_oid(split_oid) split_data['compound_id'] = split_data['cmpd_id'] split_data = split_data.drop(columns=['cmpd_id']) full_data = pd.merge(full_data, split_data, how='inner', left_on=[id_col], right_on=['compound_id']) train_data = full_data[full_data['subset'] == 'train'][[ 'compound_id', smiles_col, id_col, *response_col ]].reset_index(drop=True) valid_data = full_data[full_data['subset'] == 'valid'][[ 'compound_id', smiles_col, id_col, *response_col ]].reset_index(drop=True) test_data = full_data[full_data['subset'] == 'test'][[ 'compound_id', smiles_col, id_col, *response_col ]].reset_index(drop=True) return train_data, valid_data, test_data
def export_model(model_uuid, collection, model_dir): """ Export the metadata (parameters) and other files needed to recreate a model from the model tracker database to a gzipped tar archive. Args: model_uuid (str): Model unique identifier collection (str): Name of the collection holding the model in the database. model_dir (str): Path to directory where the model metadata and parameter files will be written. The directory will be created if it doesn't already exist. Subsequently, the directory contents will be packed into a gzipped tar archive named model_dir.tar.gz. Returns: none """ if not mlmt_supported: print( "Model tracker not supported in your environment; can load models from filesystem only." ) return ds_client = dsf.config_client() metadata_dict = get_metadata_by_uuid(model_uuid, collection_name=collection) # Get the tarball containing the saved model from the datastore, and extract it into model_dir. if 'ModelMetadata' in metadata_dict: # Convert old style metadata metadata_dict = convert_metadata(metadata_dict) if 'model_parameters' in metadata_dict: model_parameters = metadata_dict['model_parameters'] model_dataset_oid = model_parameters['model_dataset_oid'] else: raise Exception("Bad metadata for model UUID %s" % model_uuid) os.makedirs(model_dir, exist_ok=True) # Unpack the model state tarball into a subdirectory of the new archive extract_dir = dsf.retrieve_dataset_by_dataset_oid(model_dataset_oid, client=ds_client, return_metadata=False, nrows=None, print_metadata=False, sep=False, tarpath='%s/best_model' % model_dir) # Download the transformers pickle file if there is one try: transformer_oid = model_parameters["transformer_oid"] trans_fp = ds_client.open_dataset(transformer_oid, mode='b') trans_data = trans_fp.read() trans_fp.close() trans_path = "%s/transformers.pkl" % model_dir trans_out = open(trans_path, mode='wb') trans_out.write(trans_data) trans_out.close() del model_parameters['transformer_oid'] model_parameters['transformer_key'] = 'transformers.pkl' except KeyError: # OK if there are no transformers pass # Save the metadata params meta_path = "%s/model_metadata.json" % model_dir with open(meta_path, 'w') as meta_out: json.dump(metadata_dict, meta_out, indent=4) # Create a new tarball containing both the metadata and the parameters from the retrieved model tarball new_tarpath = "%s.tar.gz" % model_dir tarball = tarfile.open(new_tarpath, mode='w:gz') tarball.add(model_dir, arcname='.') tarball.close() print("Wrote model files to %s" % new_tarpath)
def analyze_split(params, id_col='compound_id', smiles_col='rdkit_smiles', active_col='active'): """ Evaluate the AVE bias for the training/validation and training/test set splits of the given dataset. Also show the active frequencies in each subset and for the dataset as a whole. id_col, smiles_col and active_col are defaults to be used in case they aren't found in the dataset metadata; if found the metadata values are used instead. Args: params (argparse.Namespace): Pipeline parameters. id_col (str): Dataset column containing compound IDs. smiles_col (str): Dataset column containing SMILES strings. active_col (str): Dataset column containing binary classifications. Returns: :obj:`pandas.DataFrame`: Table of split subsets showing sizes, numbers and fractions of active compounds """ dset_key = params.dataset_key bucket = params.bucket split_uuid = params.split_uuid ds_client = dsf.config_client() try: split_metadata = dsf.search_datasets_by_key_value('split_dataset_uuid', split_uuid, ds_client, operator='in', bucket=bucket) split_oid = split_metadata['dataset_oid'].values[0] split_df = dsf.retrieve_dataset_by_dataset_oid(split_oid, client=ds_client) except Exception as e: print("Error when loading split file:\n%s" % str(e)) raise try: dataset_df = dsf.retrieve_dataset_by_datasetkey(dset_key, bucket, client=ds_client) dataset_meta = dsf.retrieve_dataset_by_datasetkey(dset_key, bucket, client=ds_client, return_metadata=True) except Exception as e: print("Error when loading dataset:\n%s" % str(e)) raise kv_dict = dsf.get_key_val(dataset_meta['metadata']) id_col = kv_dict.get('id_col', id_col) smiles_col = kv_dict.get('smiles_col', smiles_col) active_col = kv_dict.get('response_col', active_col) try: print('Dataset has %d unique compound IDs' % len(set(dataset_df[id_col].values))) print('Split table has %d unique compound IDs' % len(set(split_df.cmpd_id.values))) dset_df = dataset_df.merge(split_df, how='inner', left_on=id_col, right_on='cmpd_id').drop('cmpd_id', axis=1) except Exception as e: print("Error when joining dataset with split dataset:\n%s" % str(e)) raise featurization = feat.create_featurization(params) data = md.create_model_dataset(params, featurization, ds_client) data.get_featurized_data() feat_arr = data.dataset.X # TODO: impute missing values if necessary y = data.dataset.y.flatten() if len(set(y) - set([0, 1])) > 0: raise ValueError( 'AVEMinSplitter only works on binary classification datasets') ids = data.dataset.ids active_ind = np.where(y == 1)[0] inactive_ind = np.where(y == 0)[0] active_feat = feat_arr[active_ind, :] inactive_feat = feat_arr[inactive_ind, :] num_active = len(active_ind) num_inactive = len(inactive_ind) active_ids = ids[active_ind] inactive_ids = ids[inactive_ind] active_id_ind = dict(zip(active_ids, range(len(active_ids)))) inactive_id_ind = dict(zip(inactive_ids, range(len(inactive_ids)))) if params.featurizer == 'ecfp': metric = 'jaccard' elif params.featurizer == 'graphconv': raise ValueError( "ave_min splitter dopesn't support graphconv features") else: metric = 'euclidean' # Calculate distance thresholds where nearest neighborfunction should be evaluated if metric == 'jaccard': max_nn_dist = 1.0 else: nan_mat = np.isnan(feat_arr) nnan = np.sum(nan_mat) if nnan > 0: log.info('Input feature matrix has %d NaN elements' % nnan) not_nan = ~nan_mat for i in range(feat_arr.shape[1]): feat_arr[nan_mat[:, i], i] = np.mean(feat_arr[not_nan[:, i], i]) nn_dist = np.sort(squareform(pdist(feat_arr, metric)))[:, 1] med_nn_dist = np.median(nn_dist) max_nn_dist = 3.0 * med_nn_dist ndist = 100 dist_thresh = np.linspace(0.0, max_nn_dist, ndist) # Compute distance matrices between subsets num_workers = 1 aa_dist = _calc_dist_mat(active_feat, active_feat, metric, None, num_workers) ii_dist = _calc_dist_mat(inactive_feat, inactive_feat, metric, None, num_workers) ai_dist = _calc_dist_mat(active_feat, inactive_feat, metric, None, num_workers) ia_dist = ai_dist.transpose() subsets = sorted(set(dset_df.subset.values)) subset_active_ind = {} subset_inactive_ind = {} if 'train' in subsets: # this is a TVT split subsets = ['train', 'valid', 'test'] for subset in subsets: subset_df = dset_df[dset_df.subset == subset] active_df = subset_df[subset_df[active_col] == 1] inactive_df = subset_df[subset_df[active_col] == 0] subset_active_ids = active_df[id_col].values subset_inactive_ids = inactive_df[id_col].values subset_active_ind[subset] = [ active_id_ind[id] for id in subset_active_ids ] subset_inactive_ind[subset] = [ inactive_id_ind[id] for id in subset_inactive_ids ] taI = subset_active_ind['train'] tiI = subset_inactive_ind['train'] print("Results for %s split with %s %s features:" % (params.splitter, params.descriptor_type, params.featurizer)) for valid_set in ['valid', 'test']: vaI = subset_active_ind[valid_set] viI = subset_inactive_ind[valid_set] split_params = ((vaI, viI, taI, tiI), aa_dist, ii_dist, ai_dist, ia_dist, dist_thresh) _plot_nn_dist_distr(split_params) bias = _plot_bias(split_params, niter=0) print("For train/%s split: AVE bias = %.5f" % (valid_set, bias)) else: # TODO: deal with k-fold splits later print('k-fold CV splits not supported yet') return # Tabulate the fractions of actives in the full dataset and each subset subset_list = [] size_list = [] frac_list = [] active_frac_list = [] dset_size = data.dataset.X.shape[0] dset_active = sum(data.dataset.y) subset_list.append('full dataset') size_list.append(dset_size) frac_list.append(1.0) active_frac_list.append(dset_active / dset_size) for subset in subsets: active_size = len(subset_active_ind[subset]) inactive_size = len(subset_inactive_ind[subset]) subset_size = active_size + inactive_size active_frac = active_size / subset_size subset_list.append(subset) size_list.append(subset_size) frac_list.append(subset_size / dset_size) active_frac_list.append(active_frac) frac_df = pd.DataFrame( dict(subset=subset_list, size=size_list, fraction=frac_list, active_frac=active_frac_list)) print('\nSplit subsets:') print(frac_df) return frac_df