def addRnaQuantification(self):
"""
Adds an rnaQuantification into this repo
"""
self._openRepo()
dataset = self._repo.getDatasetByName(self._args.datasetName)
biosampleId = ""
if self._args.biosampleName:
biosample = dataset.getBiosampleByName(self._args.biosampleName)
biosampleId = biosample.getId()
if self._args.name is None:
name = getNameFromPath(self._args.quantificationFilePath)
else:
name = self._args.name
# TODO: programs not fully supported by GA4GH yet
programs = ""
featureType = "gene"
if self._args.transcript:
featureType = "transcript"
rnaseq2ga.rnaseq2ga(
self._args.quantificationFilePath, self._args.filePath, name,
self._args.format, dataset=dataset, featureType=featureType,
description=self._args.description, programs=programs,
featureSetNames=self._args.featureSetNames,
readGroupSetNames=self._args.readGroupSetName,
biosampleId=biosampleId)
def testLoadRsemData(self):
"""
Test ingest of rsem data.
"""
tempDir = tempfile.mkdtemp(prefix="ga4gh_rna_quant",
dir=tempfile.gettempdir())
dbName = os.path.join(tempDir, "rnaQuantDB")
storeDb = rnaseq2ga.RnaSqliteStore(dbName)
storeDb.createTables()
testTsvFile = os.path.join(
paths.testDataDir,
"datasets/dataset1/rnaQuant/rsem_test_data.tsv")
rnaQuantId = "rqsId"
rnaseq2ga.rnaseq2ga(testTsvFile, dbName, rnaQuantId,
'rsem', featureType="gene")
shutil.rmtree(tempDir)
def testLoadRsemData(self):
"""
Test ingest of rsem data.
"""
tempDir = tempfile.mkdtemp(prefix="ga4gh_rna_quant",
dir=tempfile.gettempdir())
dbName = os.path.join(tempDir, "rnaQuantDB")
storeDb = rnaseq2ga.RnaSqliteStore(dbName)
storeDb.createTables()
testTsvFile = os.path.join(
paths.testDataDir, "datasets/dataset1/rnaQuant/rsem_test_data.tsv")
rnaQuantId = "rqsId"
rnaseq2ga.rnaseq2ga(testTsvFile,
dbName,
rnaQuantId,
'rsem',
featureType="gene")
shutil.rmtree(tempDir)
def run(self):
if not os.path.exists(self.outputDirectory):
os.makedirs(self.outputDirectory)
self.repo.open("w")
self.repo.initialise()
referenceFileName = "ref_brca1.fa"
inputRef = os.path.join(self.inputDirectory, referenceFileName)
outputRef = os.path.join(self.outputDirectory, referenceFileName)
shutil.copy(inputRef, outputRef)
fastaFilePath = os.path.join(self.outputDirectory,
referenceFileName + '.gz')
pysam.tabix_compress(outputRef, fastaFilePath)
with open(os.path.join(self.inputDirectory,
"ref_brca1.json")) as refMetadataFile:
refMetadata = json.load(refMetadataFile)
with open(os.path.join(self.inputDirectory,
"referenceset_hg37.json")) as refMetadataFile:
refSetMetadata = json.load(refMetadataFile)
referenceSet = references.HtslibReferenceSet(
refSetMetadata['assemblyId'])
referenceSet.populateFromFile(os.path.abspath(fastaFilePath))
referenceSet.setAssemblyId(refSetMetadata['assemblyId'])
referenceSet.setDescription(refSetMetadata['description'])
if refSetMetadata['species']:
speciesJson = json.dumps(refSetMetadata['species'])
referenceSet.setSpeciesFromJson(speciesJson) # needs a string
referenceSet.setIsDerived(refSetMetadata['isDerived'])
referenceSet.setSourceUri(refSetMetadata['sourceUri'])
referenceSet.setSourceAccessions(refSetMetadata['sourceAccessions'])
for reference in referenceSet.getReferences():
if refSetMetadata['species']:
speciesJsonStr = json.dumps(refMetadata['species'])
reference.setSpeciesFromJson(speciesJsonStr)
reference.setSourceAccessions(refMetadata['sourceAccessions'])
self.repo.insertReferenceSet(referenceSet)
dataset = datasets.Dataset("brca1")
# Some info is set, it isn't important what
dataset.setAttributes({"version": ga4gh.server.__version__})
self.repo.insertDataset(dataset)
hg00096Individual = biodata.Individual(dataset, "HG00096")
with open(os.path.join(self.inputDirectory,
"individual_HG00096.json")) as jsonString:
hg00096Individual.populateFromJson(jsonString.read())
self.repo.insertIndividual(hg00096Individual)
hg00096Biosample = biodata.Biosample(dataset, "HG00096")
with open(os.path.join(self.inputDirectory,
"biosample_HG00096.json")) as jsonString:
hg00096Biosample.populateFromJson(jsonString.read())
hg00096Biosample.setIndividualId(hg00096Individual.getId())
self.repo.insertBiosample(hg00096Biosample)
hg00099Individual = biodata.Individual(dataset, "HG00099")
with open(os.path.join(self.inputDirectory,
"individual_HG00099.json")) as jsonString:
hg00099Individual.populateFromJson(jsonString.read())
self.repo.insertIndividual(hg00099Individual)
hg00099Biosample = biodata.Biosample(dataset, "HG00099")
with open(os.path.join(self.inputDirectory,
"biosample_HG00099.json")) as jsonString:
hg00099Biosample.populateFromJson(jsonString.read())
hg00099Biosample.setIndividualId(hg00099Individual.getId())
self.repo.insertBiosample(hg00099Biosample)
hg00101Individual = biodata.Individual(dataset, "HG00101")
with open(os.path.join(self.inputDirectory,
"individual_HG00101.json")) as jsonString:
hg00101Individual.populateFromJson(jsonString.read())
self.repo.insertIndividual(hg00101Individual)
hg00101Biosample = biodata.Biosample(dataset, "HG00101")
with open(os.path.join(self.inputDirectory,
"biosample_HG00101.json")) as jsonString:
hg00101Biosample.populateFromJson(jsonString.read())
hg00101Biosample.setIndividualId(hg00101Individual.getId())
self.repo.insertBiosample(hg00101Biosample)
readFiles = [
"brca1_HG00096.sam", "brca1_HG00099.sam", "brca1_HG00101.sam"
]
for readFile in readFiles:
name = readFile.split('_')[1].split('.')[0]
readSrc = pysam.AlignmentFile(
os.path.join(self.inputDirectory, readFile), "r")
readDest = pysam.AlignmentFile(os.path.join(
self.outputDirectory, name + ".bam"),
"wb",
header=readSrc.header)
destFilePath = readDest.filename
for readData in readSrc:
readDest.write(readData)
readDest.close()
readSrc.close()
pysam.index(destFilePath)
readGroupSet = reads.HtslibReadGroupSet(dataset, name)
readGroupSet.populateFromFile(
os.path.abspath(destFilePath),
os.path.abspath(destFilePath + ".bai"))
readGroupSet.setReferenceSet(referenceSet)
dataset.addReadGroupSet(readGroupSet)
biosamples = [hg00096Biosample, hg00099Biosample, hg00101Biosample]
for readGroup in readGroupSet.getReadGroups():
for biosample in biosamples:
if biosample.getLocalId() == readGroup.getSampleName():
readGroup.setBiosampleId(biosample.getId())
self.repo.insertReadGroupSet(readGroupSet)
ontologyMapFileName = "so-xp-simple.obo"
inputOntologyMap = os.path.join(self.inputDirectory,
ontologyMapFileName)
outputOntologyMap = os.path.join(self.outputDirectory,
ontologyMapFileName)
shutil.copy(inputOntologyMap, outputOntologyMap)
sequenceOntology = ontologies.Ontology("so-xp-simple")
sequenceOntology.populateFromFile(os.path.abspath(outputOntologyMap))
sequenceOntology._id = "so-xp-simple"
self.repo.insertOntology(sequenceOntology)
self.repo.addOntology(sequenceOntology)
vcfFiles = [
"brca1_1kgPhase3_variants.vcf", "brca1_WASH7P_annotation.vcf",
"brca1_OR4F_annotation.vcf"
]
for vcfFile in vcfFiles:
self.addVariantSet(vcfFile, dataset, referenceSet,
sequenceOntology, biosamples)
# Sequence annotations
seqAnnFile = "brca1_gencodev19.gff3"
seqAnnSrc = os.path.join(self.inputDirectory, seqAnnFile)
seqAnnDest = os.path.join(self.outputDirectory, "gencodev19.db")
dbgen = generate_gff3_db.Gff32Db(seqAnnSrc, seqAnnDest)
dbgen.run()
gencode = sequence_annotations.Gff3DbFeatureSet(dataset, "gencodev19")
gencode.setOntology(sequenceOntology)
gencode.populateFromFile(os.path.abspath(seqAnnDest))
gencode.setReferenceSet(referenceSet)
self.repo.insertFeatureSet(gencode)
# add g2p featureSet
g2pPath = os.path.join(self.inputDirectory, "cgd")
# copy all files input directory to output path
outputG2PPath = os.path.join(self.outputDirectory, "cgd")
os.makedirs(outputG2PPath)
for filename in glob.glob(os.path.join(g2pPath, '*.*')):
shutil.copy(filename, outputG2PPath)
featuresetG2P = g2p_featureset.PhenotypeAssociationFeatureSet(
dataset, os.path.abspath(outputG2PPath))
featuresetG2P.setOntology(sequenceOntology)
featuresetG2P.setReferenceSet(referenceSet)
featuresetG2P.populateFromFile(os.path.abspath(outputG2PPath))
self.repo.insertFeatureSet(featuresetG2P)
# add g2p phenotypeAssociationSet
phenotypeAssociationSet = \
g2p_associationset.RdfPhenotypeAssociationSet(
dataset, "cgd", os.path.abspath(outputG2PPath))
self.repo.insertPhenotypeAssociationSet(phenotypeAssociationSet)
dataset.addFeatureSet(gencode)
# RNA Quantification
rnaDbName = os.path.join(self.outputDirectory, "rnaseq.db")
store = rnaseq2ga.RnaSqliteStore(rnaDbName)
store.createTables()
rnaseq2ga.rnaseq2ga(self.inputDirectory + "/rna_brca1.tsv",
rnaDbName,
"rna_brca1.tsv",
"rsem",
featureType="transcript",
readGroupSetNames="HG00096",
dataset=dataset,
featureSetNames="gencodev19",
biosampleId=hg00096Biosample.getId())
rnaQuantificationSet = rna_quantification.SqliteRnaQuantificationSet(
dataset, "rnaseq")
rnaQuantificationSet.setReferenceSet(referenceSet)
rnaQuantificationSet.populateFromFile(os.path.abspath(rnaDbName))
self.repo.insertRnaQuantificationSet(rnaQuantificationSet)
def run(self):
if not os.path.exists(self.outputDirectory):
os.makedirs(self.outputDirectory)
self.repo.open("w")
self.repo.initialise()
referenceFileName = "ref_brca1.fa"
inputRef = os.path.join(
self.inputDirectory, referenceFileName)
outputRef = os.path.join(
self.outputDirectory, referenceFileName)
shutil.copy(inputRef, outputRef)
fastaFilePath = os.path.join(
self.outputDirectory,
referenceFileName + '.gz')
pysam.tabix_compress(
outputRef, fastaFilePath)
with open(
os.path.join(
self.inputDirectory, "ref_brca1.json")) as refMetadataFile:
refMetadata = json.load(refMetadataFile)
with open(
os.path.join(
self.inputDirectory,
"referenceset_hg37.json")) as refMetadataFile:
refSetMetadata = json.load(refMetadataFile)
referenceSet = references.HtslibReferenceSet(
refSetMetadata['assemblyId'])
referenceSet.populateFromFile(os.path.abspath(fastaFilePath))
referenceSet.setAssemblyId(refSetMetadata['assemblyId'])
referenceSet.setDescription(refSetMetadata['description'])
if refSetMetadata['species']:
speciesJson = json.dumps(refSetMetadata['species'])
referenceSet.setSpeciesFromJson(speciesJson) # needs a string
referenceSet.setIsDerived(refSetMetadata['isDerived'])
referenceSet.setSourceUri(refSetMetadata['sourceUri'])
referenceSet.setSourceAccessions(refSetMetadata['sourceAccessions'])
for reference in referenceSet.getReferences():
if refSetMetadata['species']:
speciesJsonStr = json.dumps(refMetadata['species'])
reference.setSpeciesFromJson(speciesJsonStr)
reference.setSourceAccessions(
refMetadata['sourceAccessions'])
self.repo.insertReferenceSet(referenceSet)
dataset = datasets.Dataset("brca1")
# Some info is set, it isn't important what
dataset.setAttributes({"version": ga4gh.server.__version__})
self.repo.insertDataset(dataset)
hg00096Individual = biodata.Individual(dataset, "HG00096")
with open(
os.path.join(
self.inputDirectory,
"individual_HG00096.json")) as jsonString:
hg00096Individual.populateFromJson(jsonString.read())
self.repo.insertIndividual(hg00096Individual)
hg00096Biosample = biodata.Biosample(dataset, "HG00096")
with open(
os.path.join(
self.inputDirectory,
"biosample_HG00096.json")) as jsonString:
hg00096Biosample.populateFromJson(jsonString.read())
hg00096Biosample.setIndividualId(hg00096Individual.getId())
self.repo.insertBiosample(hg00096Biosample)
hg00099Individual = biodata.Individual(dataset, "HG00099")
with open(
os.path.join(
self.inputDirectory,
"individual_HG00099.json")) as jsonString:
hg00099Individual.populateFromJson(jsonString.read())
self.repo.insertIndividual(hg00099Individual)
hg00099Biosample = biodata.Biosample(dataset, "HG00099")
with open(
os.path.join(
self.inputDirectory,
"biosample_HG00099.json")) as jsonString:
hg00099Biosample.populateFromJson(jsonString.read())
hg00099Biosample.setIndividualId(hg00099Individual.getId())
self.repo.insertBiosample(hg00099Biosample)
hg00101Individual = biodata.Individual(dataset, "HG00101")
with open(
os.path.join(
self.inputDirectory,
"individual_HG00101.json")) as jsonString:
hg00101Individual.populateFromJson(jsonString.read())
self.repo.insertIndividual(hg00101Individual)
hg00101Biosample = biodata.Biosample(dataset, "HG00101")
with open(
os.path.join(
self.inputDirectory,
"biosample_HG00101.json")) as jsonString:
hg00101Biosample.populateFromJson(jsonString.read())
hg00101Biosample.setIndividualId(hg00101Individual.getId())
self.repo.insertBiosample(hg00101Biosample)
readFiles = [
"brca1_HG00096.sam",
"brca1_HG00099.sam",
"brca1_HG00101.sam"]
for readFile in readFiles:
name = readFile.split('_')[1].split('.')[0]
readSrc = pysam.AlignmentFile(
os.path.join(self.inputDirectory, readFile), "r")
readDest = pysam.AlignmentFile(
os.path.join(
self.outputDirectory,
name + ".bam"),
"wb", header=readSrc.header)
destFilePath = readDest.filename
for readData in readSrc:
readDest.write(readData)
readDest.close()
readSrc.close()
pysam.index(destFilePath)
readGroupSet = reads.HtslibReadGroupSet(dataset, name)
readGroupSet.populateFromFile(os.path.abspath(
destFilePath), os.path.abspath(destFilePath + ".bai"))
readGroupSet.setReferenceSet(referenceSet)
dataset.addReadGroupSet(readGroupSet)
biosamples = [hg00096Biosample, hg00099Biosample, hg00101Biosample]
for readGroup in readGroupSet.getReadGroups():
for biosample in biosamples:
if biosample.getLocalId() == readGroup.getSampleName():
readGroup.setBiosampleId(biosample.getId())
self.repo.insertReadGroupSet(readGroupSet)
ontologyMapFileName = "so-xp-simple.obo"
inputOntologyMap = os.path.join(
self.inputDirectory, ontologyMapFileName)
outputOntologyMap = os.path.join(
self.outputDirectory, ontologyMapFileName)
shutil.copy(inputOntologyMap, outputOntologyMap)
sequenceOntology = ontologies.Ontology("so-xp-simple")
sequenceOntology.populateFromFile(os.path.abspath(outputOntologyMap))
sequenceOntology._id = "so-xp-simple"
self.repo.insertOntology(sequenceOntology)
self.repo.addOntology(sequenceOntology)
vcfFiles = [
"brca1_1kgPhase3_variants.vcf",
"brca1_WASH7P_annotation.vcf",
"brca1_OR4F_annotation.vcf"]
for vcfFile in vcfFiles:
self.addVariantSet(
vcfFile,
dataset,
referenceSet,
sequenceOntology,
biosamples)
# Sequence annotations
seqAnnFile = "brca1_gencodev19.gff3"
seqAnnSrc = os.path.join(self.inputDirectory, seqAnnFile)
seqAnnDest = os.path.join(self.outputDirectory, "gencodev19.db")
dbgen = generate_gff3_db.Gff32Db(seqAnnSrc, seqAnnDest)
dbgen.run()
gencode = sequence_annotations.Gff3DbFeatureSet(dataset, "gencodev19")
gencode.setOntology(sequenceOntology)
gencode.populateFromFile(os.path.abspath(seqAnnDest))
gencode.setReferenceSet(referenceSet)
self.repo.insertFeatureSet(gencode)
# Continuous data
continuousFile = ("wgEncodeCaltechRnaSeqNhekR1x75dTh1014Ilna"
"MinusSignalRep1.bigWig")
continuousFileSrc = os.path.join(
self.inputDirectory, continuousFile)
continuousFileDest = os.path.join(
self.outputDirectory, continuousFile)
shutil.copy(continuousFileSrc, continuousFileDest)
signalData = continuous.FileContinuousSet(dataset, "signalData")
signalData.populateFromFile(os.path.abspath(continuousFileDest))
signalData.setReferenceSet(referenceSet)
self.repo.insertContinuousSet(signalData)
# add g2p featureSet
g2pPath = os.path.join(self.inputDirectory, "cgd")
# copy all files input directory to output path
outputG2PPath = os.path.join(
self.outputDirectory, "cgd")
os.makedirs(outputG2PPath)
for filename in glob.glob(os.path.join(g2pPath, '*.*')):
shutil.copy(filename, outputG2PPath)
featuresetG2P = g2p_featureset.PhenotypeAssociationFeatureSet(
dataset, os.path.abspath(outputG2PPath))
featuresetG2P.setOntology(sequenceOntology)
featuresetG2P.setReferenceSet(referenceSet)
featuresetG2P.populateFromFile(os.path.abspath(outputG2PPath))
self.repo.insertFeatureSet(featuresetG2P)
# add g2p phenotypeAssociationSet
phenotypeAssociationSet = \
g2p_associationset.RdfPhenotypeAssociationSet(
dataset, "cgd", os.path.abspath(outputG2PPath))
self.repo.insertPhenotypeAssociationSet(phenotypeAssociationSet)
dataset.addFeatureSet(gencode)
# RNA Quantification
rnaDbName = os.path.join(self.outputDirectory, "rnaseq.db")
store = rnaseq2ga.RnaSqliteStore(rnaDbName)
store.createTables()
rnaseq2ga.rnaseq2ga(
self.inputDirectory + "/rna_brca1.tsv",
rnaDbName, "rna_brca1.tsv", "rsem",
featureType="transcript",
readGroupSetNames="HG00096",
dataset=dataset,
featureSetNames="gencodev19",
biosampleId=hg00096Biosample.getId())
rnaQuantificationSet = rna_quantification.SqliteRnaQuantificationSet(
dataset, "rnaseq")
rnaQuantificationSet.setReferenceSet(referenceSet)
rnaQuantificationSet.populateFromFile(os.path.abspath(rnaDbName))
self.repo.insertRnaQuantificationSet(rnaQuantificationSet)