def addRnaQuantification(self): """ Adds an rnaQuantification into this repo """ self._openRepo() dataset = self._repo.getDatasetByName(self._args.datasetName) biosampleId = "" if self._args.biosampleName: biosample = dataset.getBiosampleByName(self._args.biosampleName) biosampleId = biosample.getId() if self._args.name is None: name = getNameFromPath(self._args.quantificationFilePath) else: name = self._args.name # TODO: programs not fully supported by GA4GH yet programs = "" featureType = "gene" if self._args.transcript: featureType = "transcript" rnaseq2ga.rnaseq2ga( self._args.quantificationFilePath, self._args.filePath, name, self._args.format, dataset=dataset, featureType=featureType, description=self._args.description, programs=programs, featureSetNames=self._args.featureSetNames, readGroupSetNames=self._args.readGroupSetName, biosampleId=biosampleId)
def testLoadRsemData(self): """ Test ingest of rsem data. """ tempDir = tempfile.mkdtemp(prefix="ga4gh_rna_quant", dir=tempfile.gettempdir()) dbName = os.path.join(tempDir, "rnaQuantDB") storeDb = rnaseq2ga.RnaSqliteStore(dbName) storeDb.createTables() testTsvFile = os.path.join( paths.testDataDir, "datasets/dataset1/rnaQuant/rsem_test_data.tsv") rnaQuantId = "rqsId" rnaseq2ga.rnaseq2ga(testTsvFile, dbName, rnaQuantId, 'rsem', featureType="gene") shutil.rmtree(tempDir)
def testLoadRsemData(self): """ Test ingest of rsem data. """ tempDir = tempfile.mkdtemp(prefix="ga4gh_rna_quant", dir=tempfile.gettempdir()) dbName = os.path.join(tempDir, "rnaQuantDB") storeDb = rnaseq2ga.RnaSqliteStore(dbName) storeDb.createTables() testTsvFile = os.path.join( paths.testDataDir, "datasets/dataset1/rnaQuant/rsem_test_data.tsv") rnaQuantId = "rqsId" rnaseq2ga.rnaseq2ga(testTsvFile, dbName, rnaQuantId, 'rsem', featureType="gene") shutil.rmtree(tempDir)
def run(self): if not os.path.exists(self.outputDirectory): os.makedirs(self.outputDirectory) self.repo.open("w") self.repo.initialise() referenceFileName = "ref_brca1.fa" inputRef = os.path.join(self.inputDirectory, referenceFileName) outputRef = os.path.join(self.outputDirectory, referenceFileName) shutil.copy(inputRef, outputRef) fastaFilePath = os.path.join(self.outputDirectory, referenceFileName + '.gz') pysam.tabix_compress(outputRef, fastaFilePath) with open(os.path.join(self.inputDirectory, "ref_brca1.json")) as refMetadataFile: refMetadata = json.load(refMetadataFile) with open(os.path.join(self.inputDirectory, "referenceset_hg37.json")) as refMetadataFile: refSetMetadata = json.load(refMetadataFile) referenceSet = references.HtslibReferenceSet( refSetMetadata['assemblyId']) referenceSet.populateFromFile(os.path.abspath(fastaFilePath)) referenceSet.setAssemblyId(refSetMetadata['assemblyId']) referenceSet.setDescription(refSetMetadata['description']) if refSetMetadata['species']: speciesJson = json.dumps(refSetMetadata['species']) referenceSet.setSpeciesFromJson(speciesJson) # needs a string referenceSet.setIsDerived(refSetMetadata['isDerived']) referenceSet.setSourceUri(refSetMetadata['sourceUri']) referenceSet.setSourceAccessions(refSetMetadata['sourceAccessions']) for reference in referenceSet.getReferences(): if refSetMetadata['species']: speciesJsonStr = json.dumps(refMetadata['species']) reference.setSpeciesFromJson(speciesJsonStr) reference.setSourceAccessions(refMetadata['sourceAccessions']) self.repo.insertReferenceSet(referenceSet) dataset = datasets.Dataset("brca1") # Some info is set, it isn't important what dataset.setAttributes({"version": ga4gh.server.__version__}) self.repo.insertDataset(dataset) hg00096Individual = biodata.Individual(dataset, "HG00096") with open(os.path.join(self.inputDirectory, "individual_HG00096.json")) as jsonString: hg00096Individual.populateFromJson(jsonString.read()) self.repo.insertIndividual(hg00096Individual) hg00096Biosample = biodata.Biosample(dataset, "HG00096") with open(os.path.join(self.inputDirectory, "biosample_HG00096.json")) as jsonString: hg00096Biosample.populateFromJson(jsonString.read()) hg00096Biosample.setIndividualId(hg00096Individual.getId()) self.repo.insertBiosample(hg00096Biosample) hg00099Individual = biodata.Individual(dataset, "HG00099") with open(os.path.join(self.inputDirectory, "individual_HG00099.json")) as jsonString: hg00099Individual.populateFromJson(jsonString.read()) self.repo.insertIndividual(hg00099Individual) hg00099Biosample = biodata.Biosample(dataset, "HG00099") with open(os.path.join(self.inputDirectory, "biosample_HG00099.json")) as jsonString: hg00099Biosample.populateFromJson(jsonString.read()) hg00099Biosample.setIndividualId(hg00099Individual.getId()) self.repo.insertBiosample(hg00099Biosample) hg00101Individual = biodata.Individual(dataset, "HG00101") with open(os.path.join(self.inputDirectory, "individual_HG00101.json")) as jsonString: hg00101Individual.populateFromJson(jsonString.read()) self.repo.insertIndividual(hg00101Individual) hg00101Biosample = biodata.Biosample(dataset, "HG00101") with open(os.path.join(self.inputDirectory, "biosample_HG00101.json")) as jsonString: hg00101Biosample.populateFromJson(jsonString.read()) hg00101Biosample.setIndividualId(hg00101Individual.getId()) self.repo.insertBiosample(hg00101Biosample) readFiles = [ "brca1_HG00096.sam", "brca1_HG00099.sam", "brca1_HG00101.sam" ] for readFile in readFiles: name = readFile.split('_')[1].split('.')[0] readSrc = pysam.AlignmentFile( os.path.join(self.inputDirectory, readFile), "r") readDest = pysam.AlignmentFile(os.path.join( self.outputDirectory, name + ".bam"), "wb", header=readSrc.header) destFilePath = readDest.filename for readData in readSrc: readDest.write(readData) readDest.close() readSrc.close() pysam.index(destFilePath) readGroupSet = reads.HtslibReadGroupSet(dataset, name) readGroupSet.populateFromFile( os.path.abspath(destFilePath), os.path.abspath(destFilePath + ".bai")) readGroupSet.setReferenceSet(referenceSet) dataset.addReadGroupSet(readGroupSet) biosamples = [hg00096Biosample, hg00099Biosample, hg00101Biosample] for readGroup in readGroupSet.getReadGroups(): for biosample in biosamples: if biosample.getLocalId() == readGroup.getSampleName(): readGroup.setBiosampleId(biosample.getId()) self.repo.insertReadGroupSet(readGroupSet) ontologyMapFileName = "so-xp-simple.obo" inputOntologyMap = os.path.join(self.inputDirectory, ontologyMapFileName) outputOntologyMap = os.path.join(self.outputDirectory, ontologyMapFileName) shutil.copy(inputOntologyMap, outputOntologyMap) sequenceOntology = ontologies.Ontology("so-xp-simple") sequenceOntology.populateFromFile(os.path.abspath(outputOntologyMap)) sequenceOntology._id = "so-xp-simple" self.repo.insertOntology(sequenceOntology) self.repo.addOntology(sequenceOntology) vcfFiles = [ "brca1_1kgPhase3_variants.vcf", "brca1_WASH7P_annotation.vcf", "brca1_OR4F_annotation.vcf" ] for vcfFile in vcfFiles: self.addVariantSet(vcfFile, dataset, referenceSet, sequenceOntology, biosamples) # Sequence annotations seqAnnFile = "brca1_gencodev19.gff3" seqAnnSrc = os.path.join(self.inputDirectory, seqAnnFile) seqAnnDest = os.path.join(self.outputDirectory, "gencodev19.db") dbgen = generate_gff3_db.Gff32Db(seqAnnSrc, seqAnnDest) dbgen.run() gencode = sequence_annotations.Gff3DbFeatureSet(dataset, "gencodev19") gencode.setOntology(sequenceOntology) gencode.populateFromFile(os.path.abspath(seqAnnDest)) gencode.setReferenceSet(referenceSet) self.repo.insertFeatureSet(gencode) # add g2p featureSet g2pPath = os.path.join(self.inputDirectory, "cgd") # copy all files input directory to output path outputG2PPath = os.path.join(self.outputDirectory, "cgd") os.makedirs(outputG2PPath) for filename in glob.glob(os.path.join(g2pPath, '*.*')): shutil.copy(filename, outputG2PPath) featuresetG2P = g2p_featureset.PhenotypeAssociationFeatureSet( dataset, os.path.abspath(outputG2PPath)) featuresetG2P.setOntology(sequenceOntology) featuresetG2P.setReferenceSet(referenceSet) featuresetG2P.populateFromFile(os.path.abspath(outputG2PPath)) self.repo.insertFeatureSet(featuresetG2P) # add g2p phenotypeAssociationSet phenotypeAssociationSet = \ g2p_associationset.RdfPhenotypeAssociationSet( dataset, "cgd", os.path.abspath(outputG2PPath)) self.repo.insertPhenotypeAssociationSet(phenotypeAssociationSet) dataset.addFeatureSet(gencode) # RNA Quantification rnaDbName = os.path.join(self.outputDirectory, "rnaseq.db") store = rnaseq2ga.RnaSqliteStore(rnaDbName) store.createTables() rnaseq2ga.rnaseq2ga(self.inputDirectory + "/rna_brca1.tsv", rnaDbName, "rna_brca1.tsv", "rsem", featureType="transcript", readGroupSetNames="HG00096", dataset=dataset, featureSetNames="gencodev19", biosampleId=hg00096Biosample.getId()) rnaQuantificationSet = rna_quantification.SqliteRnaQuantificationSet( dataset, "rnaseq") rnaQuantificationSet.setReferenceSet(referenceSet) rnaQuantificationSet.populateFromFile(os.path.abspath(rnaDbName)) self.repo.insertRnaQuantificationSet(rnaQuantificationSet)
def run(self): if not os.path.exists(self.outputDirectory): os.makedirs(self.outputDirectory) self.repo.open("w") self.repo.initialise() referenceFileName = "ref_brca1.fa" inputRef = os.path.join( self.inputDirectory, referenceFileName) outputRef = os.path.join( self.outputDirectory, referenceFileName) shutil.copy(inputRef, outputRef) fastaFilePath = os.path.join( self.outputDirectory, referenceFileName + '.gz') pysam.tabix_compress( outputRef, fastaFilePath) with open( os.path.join( self.inputDirectory, "ref_brca1.json")) as refMetadataFile: refMetadata = json.load(refMetadataFile) with open( os.path.join( self.inputDirectory, "referenceset_hg37.json")) as refMetadataFile: refSetMetadata = json.load(refMetadataFile) referenceSet = references.HtslibReferenceSet( refSetMetadata['assemblyId']) referenceSet.populateFromFile(os.path.abspath(fastaFilePath)) referenceSet.setAssemblyId(refSetMetadata['assemblyId']) referenceSet.setDescription(refSetMetadata['description']) if refSetMetadata['species']: speciesJson = json.dumps(refSetMetadata['species']) referenceSet.setSpeciesFromJson(speciesJson) # needs a string referenceSet.setIsDerived(refSetMetadata['isDerived']) referenceSet.setSourceUri(refSetMetadata['sourceUri']) referenceSet.setSourceAccessions(refSetMetadata['sourceAccessions']) for reference in referenceSet.getReferences(): if refSetMetadata['species']: speciesJsonStr = json.dumps(refMetadata['species']) reference.setSpeciesFromJson(speciesJsonStr) reference.setSourceAccessions( refMetadata['sourceAccessions']) self.repo.insertReferenceSet(referenceSet) dataset = datasets.Dataset("brca1") # Some info is set, it isn't important what dataset.setAttributes({"version": ga4gh.server.__version__}) self.repo.insertDataset(dataset) hg00096Individual = biodata.Individual(dataset, "HG00096") with open( os.path.join( self.inputDirectory, "individual_HG00096.json")) as jsonString: hg00096Individual.populateFromJson(jsonString.read()) self.repo.insertIndividual(hg00096Individual) hg00096Biosample = biodata.Biosample(dataset, "HG00096") with open( os.path.join( self.inputDirectory, "biosample_HG00096.json")) as jsonString: hg00096Biosample.populateFromJson(jsonString.read()) hg00096Biosample.setIndividualId(hg00096Individual.getId()) self.repo.insertBiosample(hg00096Biosample) hg00099Individual = biodata.Individual(dataset, "HG00099") with open( os.path.join( self.inputDirectory, "individual_HG00099.json")) as jsonString: hg00099Individual.populateFromJson(jsonString.read()) self.repo.insertIndividual(hg00099Individual) hg00099Biosample = biodata.Biosample(dataset, "HG00099") with open( os.path.join( self.inputDirectory, "biosample_HG00099.json")) as jsonString: hg00099Biosample.populateFromJson(jsonString.read()) hg00099Biosample.setIndividualId(hg00099Individual.getId()) self.repo.insertBiosample(hg00099Biosample) hg00101Individual = biodata.Individual(dataset, "HG00101") with open( os.path.join( self.inputDirectory, "individual_HG00101.json")) as jsonString: hg00101Individual.populateFromJson(jsonString.read()) self.repo.insertIndividual(hg00101Individual) hg00101Biosample = biodata.Biosample(dataset, "HG00101") with open( os.path.join( self.inputDirectory, "biosample_HG00101.json")) as jsonString: hg00101Biosample.populateFromJson(jsonString.read()) hg00101Biosample.setIndividualId(hg00101Individual.getId()) self.repo.insertBiosample(hg00101Biosample) readFiles = [ "brca1_HG00096.sam", "brca1_HG00099.sam", "brca1_HG00101.sam"] for readFile in readFiles: name = readFile.split('_')[1].split('.')[0] readSrc = pysam.AlignmentFile( os.path.join(self.inputDirectory, readFile), "r") readDest = pysam.AlignmentFile( os.path.join( self.outputDirectory, name + ".bam"), "wb", header=readSrc.header) destFilePath = readDest.filename for readData in readSrc: readDest.write(readData) readDest.close() readSrc.close() pysam.index(destFilePath) readGroupSet = reads.HtslibReadGroupSet(dataset, name) readGroupSet.populateFromFile(os.path.abspath( destFilePath), os.path.abspath(destFilePath + ".bai")) readGroupSet.setReferenceSet(referenceSet) dataset.addReadGroupSet(readGroupSet) biosamples = [hg00096Biosample, hg00099Biosample, hg00101Biosample] for readGroup in readGroupSet.getReadGroups(): for biosample in biosamples: if biosample.getLocalId() == readGroup.getSampleName(): readGroup.setBiosampleId(biosample.getId()) self.repo.insertReadGroupSet(readGroupSet) ontologyMapFileName = "so-xp-simple.obo" inputOntologyMap = os.path.join( self.inputDirectory, ontologyMapFileName) outputOntologyMap = os.path.join( self.outputDirectory, ontologyMapFileName) shutil.copy(inputOntologyMap, outputOntologyMap) sequenceOntology = ontologies.Ontology("so-xp-simple") sequenceOntology.populateFromFile(os.path.abspath(outputOntologyMap)) sequenceOntology._id = "so-xp-simple" self.repo.insertOntology(sequenceOntology) self.repo.addOntology(sequenceOntology) vcfFiles = [ "brca1_1kgPhase3_variants.vcf", "brca1_WASH7P_annotation.vcf", "brca1_OR4F_annotation.vcf"] for vcfFile in vcfFiles: self.addVariantSet( vcfFile, dataset, referenceSet, sequenceOntology, biosamples) # Sequence annotations seqAnnFile = "brca1_gencodev19.gff3" seqAnnSrc = os.path.join(self.inputDirectory, seqAnnFile) seqAnnDest = os.path.join(self.outputDirectory, "gencodev19.db") dbgen = generate_gff3_db.Gff32Db(seqAnnSrc, seqAnnDest) dbgen.run() gencode = sequence_annotations.Gff3DbFeatureSet(dataset, "gencodev19") gencode.setOntology(sequenceOntology) gencode.populateFromFile(os.path.abspath(seqAnnDest)) gencode.setReferenceSet(referenceSet) self.repo.insertFeatureSet(gencode) # Continuous data continuousFile = ("wgEncodeCaltechRnaSeqNhekR1x75dTh1014Ilna" "MinusSignalRep1.bigWig") continuousFileSrc = os.path.join( self.inputDirectory, continuousFile) continuousFileDest = os.path.join( self.outputDirectory, continuousFile) shutil.copy(continuousFileSrc, continuousFileDest) signalData = continuous.FileContinuousSet(dataset, "signalData") signalData.populateFromFile(os.path.abspath(continuousFileDest)) signalData.setReferenceSet(referenceSet) self.repo.insertContinuousSet(signalData) # add g2p featureSet g2pPath = os.path.join(self.inputDirectory, "cgd") # copy all files input directory to output path outputG2PPath = os.path.join( self.outputDirectory, "cgd") os.makedirs(outputG2PPath) for filename in glob.glob(os.path.join(g2pPath, '*.*')): shutil.copy(filename, outputG2PPath) featuresetG2P = g2p_featureset.PhenotypeAssociationFeatureSet( dataset, os.path.abspath(outputG2PPath)) featuresetG2P.setOntology(sequenceOntology) featuresetG2P.setReferenceSet(referenceSet) featuresetG2P.populateFromFile(os.path.abspath(outputG2PPath)) self.repo.insertFeatureSet(featuresetG2P) # add g2p phenotypeAssociationSet phenotypeAssociationSet = \ g2p_associationset.RdfPhenotypeAssociationSet( dataset, "cgd", os.path.abspath(outputG2PPath)) self.repo.insertPhenotypeAssociationSet(phenotypeAssociationSet) dataset.addFeatureSet(gencode) # RNA Quantification rnaDbName = os.path.join(self.outputDirectory, "rnaseq.db") store = rnaseq2ga.RnaSqliteStore(rnaDbName) store.createTables() rnaseq2ga.rnaseq2ga( self.inputDirectory + "/rna_brca1.tsv", rnaDbName, "rna_brca1.tsv", "rsem", featureType="transcript", readGroupSetNames="HG00096", dataset=dataset, featureSetNames="gencodev19", biosampleId=hg00096Biosample.getId()) rnaQuantificationSet = rna_quantification.SqliteRnaQuantificationSet( dataset, "rnaseq") rnaQuantificationSet.setReferenceSet(referenceSet) rnaQuantificationSet.populateFromFile(os.path.abspath(rnaDbName)) self.repo.insertRnaQuantificationSet(rnaQuantificationSet)