from pyspark.sql import SparkSession from pyspark.sql.types import IntegerType, StringType import pyspark.sql.functions as F from db_config import url, properties from mhfp.encoder import MHFPEncoder from pyspark.sql.types import IntegerType, StringType sudo apt-get install libboost-all-dev mhfp_encoder = MHFPEncoder() spark = SparkSession.builder.getOrCreate() # Defined multiple UDF via PySpark Sql appfunctions.py module def filename(path): return path countCarbons = F.udf(lambda x : str(x).lower().count('c'), IntegerType()) sourceFile = F.udf(filename, StringType()) mhfp_smiles = F.udf(lambda x : mhfp_encoder.encode(x, radius=3, rings=True, kekulize=True, sanitize=True), StringType()) # Created DataFrames here with the new columns that I required and dropped the duplicates df = spark.read.format('csv').option('delimiter','\t').option('header', 'false')\ .load('s3a://zincdata/zinc/AA/AAAA.txt') df = df.withColumn('mhfp', mhfp_smiles('smiles')) df = df.dropDuplicates(['smiles']) df.show() # Performed my dataframe write with the help of jdbc #df.write.jdbc(url='jdbc:%s' % url, table="zincmap", mode='append', properties=properties)
if not os.path.isfile('fps.dat'): with open('drugbank.smi', 'r') as f: i = 0 for line in f: smiles = line.split()[0].strip() mol = AllChem.MolFromSmiles(smiles) if mol: fps.append(enc.encode_mol(mol)) i += 1 if i > 2000: break pickle.dump(fps, open('fps.dat', 'wb')) else: fps = pickle.load(open('fps.dat', 'rb')) m = enc.encode( "N=C(N)NCCC[C@H](NC(=O)[C@@H]1CCCN1C(=O)[C@H](N)Cc1ccccc1)C(=O)N1CCC[C@H]1C(=O)NCC(=O)NCC(=O)NCC(=O)NCC(=O)N[C@@H](CC(=O)N)C(=O)NCC(=O)N[C@@H](CC(=O)O)C(=O)N[C@@H](Cc1ccccc1)C(=O)N[C@@H](CCC(=O)O)C(=O)N[C@@H](CCC(=O)O)C(=O)N[C@@H]([C@@H](C)CC)C(=O)N1CCC[C@H]1C(=O)N[C@@H](CCC(=O)O)C(=O)N[C@@H](CCC(=O)O)C(=O)N[C@@H](Cc1ccc(O)cc1)C(=O)N[C@@H](CC(C)C)C(=O)O" ) n = enc.encode( "O=C(N1[C@@H](CCC1)C(=O)NNC(=O)N)[C@@H](NC(=O)[C@@H](NC(=O)[C@H](NC(=O)[C@@H](NC(=O)[C@@H](NC(=O)[C@@H](NC(=O)[C@@H](NC(=O)[C@H]1NC(=O)CC1)Cc1[nH]cnc1)Cc1c2c([nH]c1)cccc2)CO)Cc1ccc(O)cc1)COC(C)(C)C)CC(C)C)CCCN=C(N)N" ) q = enc.encode( "[C@@H](C(=O)NCC(=O)N[C@@H](C(=O)N[C@@H](C)C(=O)N[C@@H](C(=O)N[C@H](C(=O)N[C@@H](C(=O)N[C@H](C(=O)N[C@@H](C(=O)N[C@H](C(=O)N[C@@H](C(=O)N[C@H](C(=O)N[C@@H](C(=O)N[C@H](C(=O)NCCO)Cc1c2c(cccc2)[nH]c1)CC(C)C)Cc1c[nH]c2c1cccc2)CC(C)C)Cc1c[nH]c2c1cccc2)CC(C)C)Cc1c[nH]c2c1cccc2)C(C)C)C(C)C)C(C)C)CC(C)C)(C(C)C)NC=O" ) r = enc.encode('CNCNCNCNC') lf_classic = LSHForest(512, 64) for i, e in enumerate(fps): lf_classic.add(i, e) lf_classic.index() start = timer()