from collections import Counter from synonymes.SynfileMap import SynfileMap from textmining.SyngrepHitFile import SyngrepHitFile from utils.idutils import dataDir, loadExludeWords resultBase = dataDir + "/miRExplore/textmine/results/" indexFoundSyns = Counter() excludedSyns = loadExludeWords() checkResultsFor = 'disease' analyseFiles = 100 maxFiles = 892 checkSynsMap = SynfileMap(resultBase + "/" + checkResultsFor + "/synfile.map") checkSynsMap.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) for splitFileID in range(maxFiles, maxFiles - analyseFiles - 1, -1): fileID = "{:>4}".format(splitFileID).replace(" ", "0") print(fileID) indexFile = resultBase + "/" + checkResultsFor + "/medline17n" + fileID + ".index" foundHits = SyngrepHitFile(indexFile, checkSynsMap) for doc in foundHits: docHits = foundHits.getHitsForDocument(doc) for hit in docHits:
import glob import os from mjoppich.geneontology import GeneOntology from porestat.utils.Parallel import MapReduce from database.Neo4JInterface import neo4jInterface from synonymes.SynfileMap import SynfileMap from textmining.SyngrepHitFile import SyngrepHitFile from utils.idutils import dataDir, eprint resultBase = dataDir + "/miRExplore/textmine/results/" diseaseMap = SynfileMap(resultBase + "/disease/synfile.map") diseaseMap.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) diseaseObo = GeneOntology(dataDir + "miRExplore/doid.obo") db = neo4jInterface(simulate=False) db.deleteRelationship('n', ['DISEASE'], None, 'm', ['PUBMED'], None, ['DISEASE_MENTION'], None) allfiles = glob.glob(resultBase + "/hgnc/medline17n*.index") allfileIDs = [ int(os.path.basename(x).replace('medline17n', '').replace('.index', '')) for x in allfiles ] allfileIDs = sorted(allfileIDs, reverse=True) addUnknownPubmeds = False retVal = db.matchNodes(['PUBMED'], None, nodename='n') relevantPMIDs = set()
from database.ORGMIRs import ORGMIRDB from synonymes.SynfileMap import SynfileMap from synonymes.SynonymFile import Synfile from synonymes.mirnaID import miRNA, miRNAPART from textmining.SentenceDB import SentenceDB, RegPos from textmining.SyngrepHitFile import SyngrepHitFile from utils.idutils import ltype2label, makeDBGeneID, mirtarbase_exp_type, mirtarbase_function_label, speciesName2TaxID, \ dataDir from database.Neo4JInterface import neo4jInterface from utils.parallel import MapReduce from enum import Enum resultBase = dataDir + "/miRExplore/textmine/results/" mirnaSyns = SynfileMap(resultBase + "/mirna/synfile.map") mirnaSyns.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) hgncSyns = SynfileMap(resultBase + "/hgnc/synfile.map") hgncSyns.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) db = None if False: db = neo4jInterface(simulate=False) db.deleteRelationship('n', ['GENE'], None, 'm', ['PUBMED'], None, ['ST_MENTION'], None, 'r') db.deleteRelationship('n', ['PUBMED_AUTHOR'], None, 'm', ['PUBMED'], None, ['IS_AUTHOR'], None, 'r') db.deleteRelationship('n', ['DISEASE'], None, 'm', ['PUBMED'], None,
type=str, help='where are all the index-files?', required=True) parser.add_argument('-d', '--datadir', type=str, help='where is te miRExplore bsae?', required=True) args = parser.parse_args() #resultBase = dataDir + "/miRExplore/textmine/results_pmc/" resultBase = args.resultdir diseaseSyns = SynfileMap(resultBase + "/model_anatomy/synfile.map") diseaseSyns.loadSynFiles( ('/home/users/joppich/ownCloud/data/', args.datadir)) allfiles = glob.glob(resultBase + "/model_anatomy/*.index") allfileIDs = [os.path.basename(x).replace(".index", "") for x in allfiles] allfileIDs = sorted(allfileIDs, reverse=True) #allfileIDs = [894] fmaObo = GeneOntology(args.datadir + "miRExplore/foundational_model_anatomy/fma_obo.obo") def getTerm(synid, obo): if synid in obo.dTerms: return obo.getID(synid)
import glob import os from mjoppich.geneontology import GeneOntology from porestat.utils.Parallel import MapReduce from database.Neo4JInterface import neo4jInterface from synonymes.SynfileMap import SynfileMap from textmining.SyngrepHitFile import SyngrepHitFile from utils.idutils import dataDir, speciesName2TaxID, eprint resultBase = dataDir + "/miRExplore/textmine/results/" celllinesMap = SynfileMap(resultBase + "/cellline/synfile.map") celllinesMap.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) knownTaxIDs = set() knownTaxIDs.add('all') for org in speciesName2TaxID: knownTaxIDs.add(str(speciesName2TaxID[org])) synfileID2tax = {} for synfileID in celllinesMap.synfiles: synfileName = celllinesMap.synfiles[synfileID] hitOrgs = [] for org in knownTaxIDs: if "." + org + "." in synfileName: hitOrgs.append(org) if len(hitOrgs) != 1: print("No or multiple files for org: " + str(synfileName) + " " +
print("NLPs loaded", file=sys.stderr) print("Creating relChecker", file=sys.stderr) relChecker = SentenceRelationChecker(nlp, nlp_ent) print("Creating relClassifier", file=sys.stderr) relClassifier = SentenceRelationClassifier(args.datadir + '/obodir/allrels.csv') print("miRExplore relation extraction models loaded", file=sys.stderr) #resultBase = dataDir + "/miRExplore/textmine/results_pmc/" resultBase = args.resultdir dataDir = args.datadir print("Getting Folder1 synfile.map", file=sys.stderr) ent1Syns = SynfileMap(resultBase + "/" + args.folder1 + "/synfile.map") ent1Syns.loadSynFiles((args.mine_path, dataDir)) print("Getting Folder2 synfile.map", file=sys.stderr) ent2Syns = SynfileMap(resultBase + "/" + args.folder2 + "/synfile.map") ent2Syns.loadSynFiles((args.mine_path, dataDir)) print("Getting relations synfile.map", file=sys.stderr) relSyns = SynfileMap(resultBase + "/relations/synfile.map") relSyns.loadSynFiles((args.mine_path, dataDir)) print("Getting obodir/allrels.csv", file=sys.stderr) relationSyns = AssocSynfile(args.datadir + '/obodir/allrels.csv') print("All maps loaded", file=sys.stderr) accept_pmids = None
parser.add_argument('-f1', '--folder1', type=str, help='entity 1: hgnc, mirna', default="hgnc", required=False) parser.add_argument('-f2', '--folder2', type=str, help='entity 2: mgi, mirna', default="mirna", required=False) parser.add_argument('-ft1', '--folderType1', type=str, help='entity type 1: entity: mirna, gene, lncrna, ...', default="gene", required=False) parser.add_argument('-ft2', '--folderType2', type=str, help='entity type 2: entity: mirna', default="mirna", required=False) args = parser.parse_args() #resultBase = dataDir + "/miRExplore/textmine/results_pmc/" resultBase = args.resultdir dataDir = args.datadir ent1Syns = SynfileMap(resultBase + "/"+args.folder1+"/synfile.map") ent1Syns.loadSynFiles(('/mnt/c/ownCloud/data', dataDir)) ent2Syns = SynfileMap(resultBase + "/"+args.folder2+"/synfile.map") ent2Syns.loadSynFiles(('/mnt/c/ownCloud/data', dataDir)) relSyns = SynfileMap(resultBase + "/relations/synfile.map") relSyns.loadSynFiles(('/mnt/c/ownCloud/data', dataDir)) relationSyns = AssocSynfile(args.datadir + '/miRExplore/relations/allrels.csv') idTuple2Pubmed = defaultdict(set) orgmirDB = ORGMIRDB(dataDir + "/miRExplore/orgmir.tsv") allfiles = glob.glob(resultBase + "/"+args.folder1+"/*.index") allfileIDs = [os.path.basename(x).replace(".index", "") for x in allfiles]
doidSynIDs = getSynIDs(dataDir + "miRExplore/doid.obo", ['DOID:104']) goSynIDs = getSynIDs(dataDir + "miRExplore/textmine/neutrophils.obo", ['NP:001']) for x in neutrophilSynIDs: if x in tissueIDs: tissueIDs.remove(x) print(neutrophilSynIDs) print(tissueIDs) print(doidSynIDs) print(goSynIDs) resultBase = dataDir + "/miRExplore/textmine/results/" fmaSyns = SynfileMap(resultBase + "/model_anatomy/synfile.map") fmaSyns.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) doidSyns = SynfileMap(resultBase + "/disease/synfile.map") doidSyns.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) goSyns = SynfileMap(resultBase + "/neutrophils/synfile.map") goSyns.loadSynFiles(('/home/users/joppich/ownCloud/data/', dataDir)) allfiles = glob.glob(resultBase + "/hgnc/pubmed18n*.index") allfileIDs = [ int(os.path.basename(x).replace('pubmed18n', '').replace('.index', '')) for x in allfiles ] allfileIDs = sorted(allfileIDs, reverse=True) #allfileIDs = allfileIDs[0:10]
if args.accept_pmids != None: accept_pmids = set() for line in args.accept_pmids: line = line.strip() if len(line) > 0: accept_pmids.add(line) #resultBase = dataDir + "/miRExplore/textmine/results_pmc/" resultBase = args.resultdir oboSyns = SynfileMap(resultBase + "/synfile.map") oboSyns.loadSynFiles((args.mine_path, args.datadir)) allfiles = glob.glob(resultBase + "/*.index") allfileIDs = [os.path.basename(x).replace(".index", "") for x in allfiles] allfileIDs = sorted(allfileIDs, reverse=True) #allfileIDs = [894] celloObo = GeneOntology(args.obo.name) def getTerm(synid, obo): if synid in obo.dTerms: return obo.getID(synid) synid = synid.replace('_', ':', 1)