def gen_training_data(self, pdbbind_dir, pdbbind_versions=(2007, 2012, 2013, 2014, 2015, 2016), home_dir=None): pdbbind_versions = sorted(pdbbind_versions) # generate metadata df = [] for pdbbind_version in pdbbind_versions: p = pdbbind('%s/v%i/' % (pdbbind_dir, pdbbind_version), version=pdbbind_version, opt={'b': None}) # Core set tmp_df = pd.DataFrame({'pdbid': list(p.sets['core'].keys()), '%i_core' % pdbbind_version: list(p.sets['core'].values())}) df = pd.merge(tmp_df, df, how='outer', on='pdbid') if len(df) else tmp_df # Refined Set tmp_df = pd.DataFrame({'pdbid': list(p.sets['refined'].keys()), '%i_refined' % pdbbind_version: list(p.sets['refined'].values())}) df = pd.merge(tmp_df, df, how='outer', on='pdbid') # General Set general_name = 'general_PL' if pdbbind_version > 2007 else 'general' tmp_df = pd.DataFrame({'pdbid': list(p.sets[general_name].keys()), '%i_general' % pdbbind_version: list(p.sets[general_name].values())}) df = pd.merge(tmp_df, df, how='outer', on='pdbid') df.sort_values('pdbid', inplace=True) tmp_act = df['%i_general' % pdbbind_versions[-1]].values df = df.set_index('pdbid').notnull() df['act'] = tmp_act # take non-empty and core + refined set df = df[df['act'].notnull() & df.filter(regex='.*_[refined,core]').any(axis=1)] # build descriptos pdbbind_db = pdbbind('%s/v%i/' % (pdbbind_dir, pdbbind_versions[-1]), version=pdbbind_versions[-1]) if not home_dir: home_dir = dirname(__file__) + '/RFScore' result = Parallel(n_jobs=self.n_jobs, verbose=1)(delayed(_parallel_helper)(self.descriptor_generator, 'build', [pdbbind_db[pid].ligand], protein=pdbbind_db[pid].pocket) for pid in df.index.values if pdbbind_db[pid].pocket is not None) descs = np.vstack(result) for i in range(len(self.descriptor_generator)): df[str(i)] = descs[:, i] df.to_csv(home_dir + '/rfscore_descs_v%i.csv' % self.version, float_format='%.5g')
def gen_training_data(self, pdbbind_dir, pdbbind_version='2007', home_dir=None, sf_pickle=''): # build train and test cpus = self.n_jobs if self.n_jobs > 0 else -1 #pool = Pool(processes=cpus) pdbbind_db = pdbbind(pdbbind_dir, int(pdbbind_version), opt={'b': None}) if not home_dir: home_dir = dirname(__file__) + '/RFScore' pdbbind_db.default_set = 'core' core_set = pdbbind_db.ids core_act = np.array(pdbbind_db.activities) # core_desc = np.vstack([self.descriptor_generator.build([pid.ligand], protein=pid.pocket) for pid in pdbbind_db]) result = Parallel(n_jobs=cpus)( delayed(_parallel_helper)(self.descriptor_generator, 'build', [pid.ligand], protein=pid.pocket) for pid in pdbbind_db if pid.pocket) core_desc = np.vstack(result) pdbbind_db.default_set = 'general' refined_set = [pid for pid in pdbbind_db.ids if not pid in core_set] refined_act = np.array([ pdbbind_db.sets[pdbbind_db.default_set][pid] for pid in refined_set ]) # refined_desc = np.vstack([self.descriptor_generator.build([pid.ligand], protein=pid.pocket) for pid in pdbbind_db]) result = Parallel(n_jobs=cpus)( delayed(_parallel_helper)(self.descriptor_generator, 'build', [pid.ligand], protein=pid.pocket) for pid in pdbbind_db if pid.pocket and not pid.id in core_set) refined_desc = np.vstack(result) self.train_descs = refined_desc self.train_target = refined_act self.test_descs = core_desc self.test_target = core_act # save numpy arrays np.savetxt(home_dir + '/train_descs_v%i.csv' % (self.version), self.train_descs, fmt='%g', delimiter=',') np.savetxt(home_dir + '/train_target.csv', self.train_target, fmt='%.2f', delimiter=',') np.savetxt(home_dir + '/test_descs_v%i.csv' % (self.version), self.test_descs, fmt='%g', delimiter=',') np.savetxt(home_dir + '/test_target.csv', self.test_target, fmt='%.2f', delimiter=',')
def gen_training_data(self, pdbbind_dir, pdbbind_version = '2007', home_dir = None, sf_pickle = ''): # build train and test cpus = self.n_jobs if self.n_jobs > 0 else -1 #pool = Pool(processes=cpus) pdbbind_db = pdbbind(pdbbind_dir, int(pdbbind_version), opt={'b':None}) if not home_dir: home_dir = dirname(__file__) + '/RFScore' pdbbind_db.default_set = 'core' core_set = pdbbind_db.ids core_act = np.array(pdbbind_db.activities) # core_desc = np.vstack([self.descriptor_generator.build([pid.ligand], protein=pid.pocket) for pid in pdbbind_db]) result = Parallel(n_jobs=cpus)(delayed(_parallel_helper)(self.descriptor_generator, 'build', [pid.ligand], protein=pid.pocket) for pid in pdbbind_db if pid.pocket) core_desc = np.vstack(result) pdbbind_db.default_set = 'general' refined_set = [pid for pid in pdbbind_db.ids if not pid in core_set] refined_act = np.array([pdbbind_db.sets[pdbbind_db.default_set][pid] for pid in refined_set]) # refined_desc = np.vstack([self.descriptor_generator.build([pid.ligand], protein=pid.pocket) for pid in pdbbind_db]) result = Parallel(n_jobs=cpus)(delayed(_parallel_helper)(self.descriptor_generator, 'build', [pid.ligand], protein=pid.pocket) for pid in pdbbind_db if pid.pocket and not pid.id in core_set) refined_desc = np.vstack(result) self.train_descs = refined_desc self.train_target = refined_act self.test_descs = core_desc self.test_target = core_act # save numpy arrays np.savetxt(home_dir + '/train_descs_v%i.csv' % (self.version), self.train_descs, fmt='%g', delimiter=',') np.savetxt(home_dir + '/train_target.csv', self.train_target, fmt='%.2f', delimiter=',') np.savetxt(home_dir + '/test_descs_v%i.csv' % (self.version), self.test_descs, fmt='%g', delimiter=',') np.savetxt(home_dir + '/test_target.csv', self.test_target, fmt='%.2f', delimiter=',')
def _gen_pdbbind_desc(self, pdbbind_dir, pdbbind_versions=(2007, 2012, 2013, 2014, 2015, 2016), desc_path=None, **kwargs): pdbbind_versions = sorted(pdbbind_versions) if 'opt' in kwargs: opt = kwargs.pop('opt') else: opt = {} # generate metadata df = None for pdbbind_version in pdbbind_versions: p = pdbbind('%s/v%i/' % (pdbbind_dir, pdbbind_version), version=pdbbind_version, opt=opt) # Core set for set_name in p.pdbind_sets: if set_name == 'general_PL': dataset_key = '%i_general' % pdbbind_version else: dataset_key = '%i_%s' % (pdbbind_version, set_name) tmp_df = pd.DataFrame({ 'pdbid': list(p.sets[set_name].keys()), dataset_key: list(p.sets[set_name].values()) }) if df is not None: df = pd.merge(tmp_df, df, how='outer', on='pdbid') else: df = tmp_df df.sort_values('pdbid', inplace=True) tmp_act = df['%i_general' % pdbbind_versions[-1]].values df = df.set_index('pdbid').notnull() df['act'] = tmp_act # take non-empty and core + refined set df = df[df['act'].notnull() & df.filter(regex='.*_[refined,core]').any(axis=1)] # build descriptos pdbbind_db = pdbbind('%s/v%i/' % (pdbbind_dir, pdbbind_versions[-1]), version=pdbbind_versions[-1]) if not desc_path: desc_path = path_join(dirname(__file__) + 'descs.csv') if self.n_jobs is None: n_jobs = -1 else: n_jobs = self.n_jobs result = Parallel(n_jobs=n_jobs, verbose=1)( delayed(method_caller)(self.descriptor_generator, 'build', [pdbbind_db[pid].ligand], protein=pdbbind_db[pid].pocket) for pid in df.index.values if pdbbind_db[pid].pocket is not None) descs = np.vstack(result) for i in range(len(self.descriptor_generator)): df[str(i)] = descs[:, i] df.to_csv(desc_path, float_format='%.5g')
def gen_training_data(self, pdbbind_dir, pdbbind_version=2007, home_dir=None, sf_pickle=''): # build train and test pdbbind_db = pdbbind(pdbbind_dir, pdbbind_version, opt={'b': None}) if not home_dir: home_dir = dirname(__file__) + '/RFScore' pdbbind_db.default_set = 'core' core_set = pdbbind_db.ids core_act = np.array(pdbbind_db.activities) # core_desc = np.vstack([self.descriptor_generator.build([pid.ligand], protein=pid.protein) for pid in pdbbind_db]) result = Parallel(n_jobs=self.n_jobs)( delayed(_parallel_helper)(self.descriptor_generator, 'build', [pid.ligand], protein=pid.pocket) for pid in pdbbind_db if pid.pocket is not None) core_desc = np.vstack(result) pdbbind_db.default_set = 'refined' refined_set = [pid for pid in pdbbind_db.ids if pid not in core_set] refined_act = np.array([ pdbbind_db.sets[pdbbind_db.default_set][pid] for pid in refined_set ]) # refined_desc = np.vstack([self.descriptor_generator.build([pid.ligand], protein=pid.protein) for pid in pdbbind_db]) result = Parallel(n_jobs=self.n_jobs)( delayed(_parallel_helper)(self.descriptor_generator, 'build', [pid.ligand], protein=pid.pocket) for pid in pdbbind_db if pid.pocket is not None and pid.id not in core_set) refined_desc = np.vstack(result) self.train_descs = refined_desc self.train_target = refined_act self.test_descs = core_desc self.test_target = core_act # save numpy arrays header = 'RFScore data generated using PDBBind v%i' % pdbbind_version np.savetxt(home_dir + '/train_descs_v%i_pdbbind%i.csv' % (self.version, pdbbind_version), self.train_descs, fmt='%g', delimiter=',', header=header) np.savetxt(home_dir + '/train_target_pdbbind%i.csv' % pdbbind_version, self.train_target, fmt='%.2f', delimiter=',', header=header) np.savetxt(home_dir + '/test_descs_v%i_pdbbind%i.csv' % (self.version, pdbbind_version), self.test_descs, fmt='%g', delimiter=',', header=header) np.savetxt(home_dir + '/test_target_pdbbind%i.csv' % pdbbind_version, self.test_target, fmt='%.2f', delimiter=',', header=header)
def test_pdbbind(): results = { 'core': (['4yef', '10gs'], [5.35, 6.4]), 'refined': (['1nlp', '1imx', '4yef', '10gs'], [4.96, 3.52, 5.35, 6.4]), 'general_PL': (['1k9q', '1nlo', '1nlp', '1imx', '4yef', '10gs'], [3.15, 5.47, 4.96, 3.52, 5.35, 6.4]), } assert_raises(ValueError, pdbbind, home=os.path.join(test_data_dir, 'data', 'pdbbind')) for year in [2007, 2013, 2016]: pdbbind_db = pdbbind(home=os.path.join(test_data_dir, 'data', 'pdbbind'), version=year, default_set='core') for set_name, (ids, activities) in results.items(): if set_name == 'general_PL' and year == 2007: set_name = 'general' pdbbind_db.default_set = set_name assert_equal(pdbbind_db.ids, ids) assert_equal(pdbbind_db.activities, activities) for pid in pdbbind_db: assert_is_instance(pid.pocket, oddt.toolkit.Molecule) assert_greater(len(pid.pocket.atoms), 0) assert_is_instance(pid.ligand, oddt.toolkit.Molecule) assert_greater(len(pid.ligand.atoms), 0) if pid.id == '10gs': assert_equal(pid.protein, None) else: assert_is_instance(pid.protein, oddt.toolkit.Molecule) assert_greater(len(pid.protein.atoms), 0) # reset the pdbbind set pdbbind_db.default_set = 'refined' # getting by name assert_equal(pdbbind_db['1imx'].id, '1imx') # getting by id assert_equal(pdbbind_db[-3].id, '1imx') assert_equal(pdbbind_db[1].id, '1imx') assert_raises(KeyError, pdbbind_db.__getitem__, 'xxxx') assert_raises(KeyError, pdbbind_db.__getitem__, 123456) assert_raises(KeyError, pdbbind_db.__getitem__, -123456) pid = pdbbind_db['1imx'] # get ligand ligand = pid.ligand ligand.removeh() assert_equal(len(ligand.atoms), 60) # get pocket pocket = pid.pocket pocket.removeh() assert_equal(len(pocket.atoms), 234) # protein do exist protein = pid.protein protein.removeh() assert_equal(len(protein.atoms), 478)
def test_pdbbind(): results = { 'core': (['4yef', '10gs'], [5.35, 6.4]), 'refined': (['1nlp', '1imx', '4yef', '10gs'], [4.96, 3.52, 5.35, 6.4]), 'general_PL': (['1k9q', '1nlo', '1nlp', '1imx', '4yef', '10gs'], [3.15, 5.47, 4.96, 3.52, 5.35, 6.4]), } with pytest.raises(ValueError): pdbbind(home=os.path.join(test_data_dir, 'data', 'pdbbind')) for year in [2007, 2013, 2016]: pdbbind_db = pdbbind(home=os.path.join(test_data_dir, 'data', 'pdbbind'), version=year, default_set='core') for set_name, (ids, activities) in results.items(): if set_name == 'general_PL' and year == 2007: set_name = 'general' pdbbind_db.default_set = set_name assert pdbbind_db.ids == ids assert pdbbind_db.activities == activities for pid in pdbbind_db: assert isinstance(pid.pocket, oddt.toolkit.Molecule) assert len(pid.pocket.atoms) > 0 assert isinstance(pid.ligand, oddt.toolkit.Molecule) assert len(pid.ligand.atoms) > 0 if pid.id == '10gs': assert pid.protein is None else: assert isinstance(pid.protein, oddt.toolkit.Molecule) assert len(pid.protein.atoms) > 0 # reset the pdbbind set pdbbind_db.default_set = 'refined' # getting by name assert pdbbind_db['1imx'].id == '1imx' # getting by id assert pdbbind_db[-3].id == '1imx' assert pdbbind_db[1].id == '1imx' with pytest.raises(KeyError): pdbbind_db['xxxx'] with pytest.raises(KeyError): pdbbind_db[123456] with pytest.raises(KeyError): pdbbind_db[-123456] pid = pdbbind_db['1imx'] # get ligand ligand = pid.ligand ligand.removeh() assert len(ligand.atoms) == 60 # get pocket pocket = pid.pocket pocket.removeh() assert len(pocket.atoms) == 234 # protein do exist protein = pid.protein protein.removeh() assert len(protein.atoms) == 478
import htmd.smallmol.smallmol as sm import csv from tqdm import * import os import pickle import numpy as np import multiprocessing as mp from sklearn.model_selection import train_test_split import h5py from oddt import toolkit from oddt import datasets # Directory paths data_dir = "../dataset" pdbbind_dir = os.path.join(data_dir, "refined-set-2016/") pdbbind_dataset = datasets.pdbbind(home=pdbbind_dir, default_set='refined', version=2016) def get_pdb_complex_feature(protein_file, ligand_file): """ Returns voxel features for a pdb complex """ def get_prop(mol, left_most_point): """ Returns atom occupancies """ n = [24, 24, 24] # Voxel size # Get the channels channels = vd._getAtomtypePropertiesPDBQT(mol) sigmas = vd._getRadii(mol) channels = sigmas[:, np.newaxis] * channels.astype(float) # Choose the grid centers
def _gen_pdbbind_desc(self, pdbbind_dir, pdbbind_versions=(2007, 2012, 2013, 2014, 2015, 2016), desc_path=None, include_general_set=False, use_proteins=False, **kwargs): pdbbind_versions = sorted(pdbbind_versions) opt = kwargs.get('opt', {}) # generate metadata df = None for pdbbind_version in pdbbind_versions: p = pdbbind('%s/v%i/' % (pdbbind_dir, pdbbind_version), version=pdbbind_version, opt=opt) # Core set for set_name in p.pdbind_sets: if set_name == 'general_PL': dataset_key = '%i_general' % pdbbind_version else: dataset_key = '%i_%s' % (pdbbind_version, set_name) tmp_df = pd.DataFrame({ 'pdbid': list(p.sets[set_name].keys()), dataset_key: list(p.sets[set_name].values()) }) if df is not None: df = pd.merge(tmp_df, df, how='outer', on='pdbid') else: df = tmp_df df.sort_values('pdbid', inplace=True) tmp_act = df['%i_general' % pdbbind_versions[-1]].values df = df.set_index('pdbid').notnull() df['act'] = tmp_act # take non-empty and core + refined set df = df[df['act'].notnull() & (df.filter(regex='.*_[refined,core]').any(axis=1) | include_general_set)] # build descriptos pdbbind_db = pdbbind('%s/v%i/' % (pdbbind_dir, pdbbind_versions[-1]), version=pdbbind_versions[-1]) if not desc_path: desc_path = path_join(dirname(__file__) + 'descs.csv') if self.n_jobs is None: n_jobs = -1 else: n_jobs = self.n_jobs blacklist = [] if use_proteins: # list of protein files that segfault OB 2.4.1 blacklist = pdbbind_db.protein_blacklist[oddt.toolkit.backend] # check if PDBID exists or is blacklisted desc_idx = [pid for pid in df.index.values if (pid not in blacklist and getattr(pdbbind_db[pid], 'protein' if use_proteins else 'pocket') is not None)] result = Parallel(n_jobs=n_jobs, verbose=1)( delayed(method_caller)( self.descriptor_generator, 'build', [pdbbind_db[pid].ligand], protein=getattr(pdbbind_db[pid], 'protein' if use_proteins else 'pocket')) for pid in desc_idx) # sparse descs may have different shapes, dense are stored np.array sparse = (hasattr(self.descriptor_generator, 'sparse') and self.descriptor_generator.sparse) if not sparse: result = np.vstack(result) # create dataframe with descriptors with pdbids as index df_desc = pd.DataFrame(result, index=desc_idx) df_desc.index.rename('pdbid', inplace=True) # for sparse features leave one column and cast explicitly to list if sparse: if len(df_desc.columns) > 1: raise Exception('There are more than one column in the ' 'sparse descriptor table.') df_desc.columns = ['sparse'] df_desc['sparse'] = df_desc['sparse'].map( lambda x: csr_matrix_to_sparse(x).tolist()) compression = None if desc_path[-3:] == '.gz': compression = 'gzip' # DF are joined by index (pdbid) since some might be missing df.join(df_desc, how='inner').to_csv(desc_path, float_format='%.5g', compression=compression)
def test_pdbbind(): results = { 'core': (['4yef', '10gs'], [5.35, 6.4]), 'refined': (['1nlp', '1imx', '4yef', '10gs'], [4.96, 3.52, 5.35, 6.4]), 'general_PL': (['1k9q', '1nlo', '1nlp', '1imx', '4yef', '10gs'], [3.15, 5.47, 4.96, 3.52, 5.35, 6.4]), } with pytest.raises(ValueError): pdbbind(home=os.path.join(test_data_dir, 'data', 'pdbbind')) for year in [2007, 2013, 2016]: pdbbind_db = pdbbind(home=os.path.join(test_data_dir, 'data', 'pdbbind'), version=year, default_set='core') for set_name, (ids, activities) in results.items(): if set_name == 'general_PL' and year == 2007: set_name = 'general' pdbbind_db.default_set = set_name assert pdbbind_db.ids == ids assert pdbbind_db.activities == activities for pid in pdbbind_db: assert isinstance(pid.pocket, oddt.toolkit.Molecule) assert len(pid.pocket.atoms) > 0 assert isinstance(pid.ligand, oddt.toolkit.Molecule) assert len(pid.ligand.atoms) > 0 if pid.id == '10gs': assert pid.protein is None else: assert isinstance(pid.protein, oddt.toolkit.Molecule) assert len(pid.protein.atoms) > 0 # reset the pdbbind set pdbbind_db.default_set = 'refined' # getting by name assert pdbbind_db['1imx'].id == '1imx' # getting by id assert pdbbind_db[-3].id == '1imx' assert pdbbind_db[1].id == '1imx' with pytest.raises(KeyError): pdbbind_db['xxxx'] with pytest.raises(KeyError): pdbbind_db[123456] with pytest.raises(KeyError): pdbbind_db[-123456] pid = pdbbind_db['1imx'] # get ligand ligand = pid.ligand ligand.removeh() assert len(ligand.atoms) == 60 # get pocket pocket = pid.pocket pocket.removeh() assert len(pocket.atoms) == 234 # protein do exist protein = pid.protein protein.removeh() assert len(protein.atoms) == 478