def test_full_seg_file_annotations(self):
        """Test that we can read in a seg file, do a proper full annotation, and output as SIMPLE_TSV"""
        inputFilename = "testdata/seg/Patient0.seg.txt"
        output_filename = "out/test_full_seg_file_annotations.tsv"
        db_dir = self.config.get('DEFAULT',"dbDir")
        if os.path.exists(output_filename):
            os.remove(output_filename)

        annotator = Annotator()
        run_spec = RunSpecificationFactory.create_run_spec("SEG_FILE", "SIMPLE_TSV", inputFilename, output_filename,
                                                           datasourceDir=db_dir, annotating_type=RunSpecification.ANNOTATE_SEGMENTS)
        annotator.initialize(run_spec)
        annotator.annotate()

        # Now check the output
        output_reader = GenericTsvReader(output_filename)

        required_cols = ["Sample", "Num_Probes", "Segment_Mean"]
        headers = output_reader.getFieldNames()
        for rcol in required_cols:
            self.assertTrue(rcol in headers)

        for line_dict in output_reader:
            self.assertTrue(line_dict['start'] is not None)
            self.assertTrue(line_dict['start'].strip() != "")
            self.assertTrue(line_dict['end'] is not None)
            self.assertTrue(line_dict['end'].strip() != "")
            self.assertTrue("genes" in line_dict.keys())
            self.assertTrue(len(line_dict["genes"].split(",")) > 0)
    def test_basic_rendering(self):
        """Test that we can render a basic seg file as a gene list"""
        inputFilename = "testdata/seg/Patient0.seg.txt"
        output_filename = "out/test_basic_rendering.gene_list.tsv"
        db_dir = self.config.get('DEFAULT',"dbDir")
        if os.path.exists(output_filename):
            os.remove(output_filename)

        annotator = Annotator()
        run_spec = RunSpecificationFactory.create_run_spec("SEG_FILE", "GENE_LIST", inputFilename, output_filename,
                                                           datasourceDir=db_dir, annotating_type=RunSpecification.ANNOTATE_SEGMENTS)
        annotator.initialize(run_spec)
        annotator.annotate()

        # Now check the output
        output_reader = GenericTsvReader(output_filename)

        headers = output_reader.getFieldNames()

        for line_dict in output_reader:
            self.assertTrue(line_dict['segment_start'] is not None)
            self.assertTrue(line_dict['segment_start'].strip() != "")
            self.assertTrue(line_dict['segment_end'] is not None)
            self.assertTrue(line_dict['segment_end'].strip() != "")
            self.assertTrue("gene" in line_dict.keys())
            self.assertTrue(len(line_dict["gene"]) > 0)
            self.assertTrue(float(line_dict["segment_num_probes"]))
            self.assertTrue(line_dict['sample'] == "Patient0")
Example #3
0
    def test_no_overwriting_muts(self):
        """Ensure that (given configuration that disallows) we cannot annotate from a datasource when a value was specified in the input."""
        # We will have an input with a "Who" annotation that this datasource will try to write.
        gene_ds = DatasourceFactory.createDatasource(
            "testdata/thaga_janakari_gene_ds/hg19/tj_data.config", "testdata/thaga_janakari_gene_ds/hg19/"
        )
        input_filename = "testdata/maflite/who_alt1_vs_alt2.maflite"
        output_filename = "out/who_alt1_vs_alt2.maf.annotated"
        input_format = "MAFLITE"
        output_format = "TCGAMAF"

        other_opts = {OptionConstants.ALLOW_ANNOTATION_OVERWRITING: False, OptionConstants.NO_PREPEND: True}

        run_spec = RunSpecificationFactory.create_run_spec_given_datasources(
            input_format,
            output_format,
            input_filename,
            output_filename,
            datasource_list=[gene_ds],
            other_opts=other_opts,
        )
        annotator = Annotator()
        annotator.initialize(run_spec)

        self.assertRaises(DuplicateAnnotationException, annotator.annotate)
Example #4
0
 def _simple_annotate(self, is_skip_no_alts):
     runSpec = RunSpecification()
     runSpec.initialize(None, None, datasources=[], is_skip_no_alts=is_skip_no_alts)
     # Initialize the annotator with the runspec
     annotator = Annotator()
     annotator.initialize(runSpec)
     m = MutationData()
     m.chr = "1"
     m.start = "12941796"
     m.end = "12941796"
     m.alt_allele = "G"
     m.ref_allele = "T"
     m.createAnnotation("alt_allele_seen", "False")
     m2 = MutationData()
     m2.chr = "1"
     m2.start = "12941796"
     m2.end = "12941796"
     m2.alt_allele = "G"
     m2.ref_allele = "T"
     muts = [m, m2]
     muts = annotator.annotate_mutations(muts)
     ctr = 0
     for m in muts:
         ctr += 1
     return ctr
Example #5
0
    def testAnnotateListOfMutations(self):
        """Test that we can initialize an Annotator, without an input or output and then feed mutations,
        one at a time... using a runspec"""

        # Locate the datasource directory and create a runspec
        dbDir = self.config.get("DEFAULT", "dbDir")
        ds = DatasourceFactory.createDatasources(dbDir)
        runSpec = RunSpecification()
        runSpec.initialize(None, None, datasources=ds)

        # Initialize the annotator with the runspec
        annotator = Annotator()
        annotator.initialize(runSpec)

        m = MutationData()
        m.chr = "1"
        m.start = "12941796"
        m.end = "12941796"
        m.alt_allele = "G"
        m.ref_allele = "T"

        muts = [m]

        muts = annotator.annotate_mutations(muts)
        m2 = muts.next()
        self.assertTrue(m2.get("gene", None) is not None)
Example #6
0
    def test_overwriting_muts(self):
        """Ensure that (given correct configuration) we can annotate from a datasource, even if the datasource will overwrite an existing mutation."""
        # We will have an input with a "Who" annotation that this datasource will try to write.
        gene_ds = DatasourceFactory.createDatasource(
            "testdata/thaga_janakari_gene_ds/hg19/tj_data.config", "testdata/thaga_janakari_gene_ds/hg19/"
        )
        input_filename = "testdata/maflite/who_alt1_vs_alt2.maflite"
        output_filename = "out/who_alt1_vs_alt2.maf.annotated"
        input_format = "MAFLITE"
        output_format = "TCGAMAF"

        other_opts = {OptionConstants.ALLOW_ANNOTATION_OVERWRITING: True, OptionConstants.NO_PREPEND: True}

        run_spec = RunSpecificationFactory.create_run_spec_given_datasources(
            input_format,
            output_format,
            input_filename,
            output_filename,
            datasource_list=[gene_ds],
            other_opts=other_opts,
        )
        annotator = Annotator()
        annotator.initialize(run_spec)

        annotator.annotate()

        tsv_reader = GenericTsvReader(output_filename)

        for i, line_dict in enumerate(tsv_reader):
            self.assertTrue(line_dict.get("TJ_Data_Who", "") != "Tromokratis")
Example #7
0
 def test_rendering_combined_to_tsv(self):
     """Test that we produce a merged ONP simple tsv file without crashing """
     input_filename = os.path.join(*["testdata", "maflite", "onp_combination.maf.txt"])
     output_filename = os.path.join("out", "onp_combination.tsv")
     spec = RunSpecificationFactory.create_run_spec("MAFLITE","SIMPLE_TSV",input_filename, output_filename,
                                             other_opts={OptionConstants.INFER_ONPS: True})
     annotator = Annotator()
     annotator.initialize(spec)
     annotator.annotate()
Example #8
0
 def testVersionHeader(self):
     """ This method simply tests that the version string returned by the annotator does not cause an exception.
         Minimal checking that the returned sting is actually correct.
         Does not attempt to initialize input or output.  Only a gaf datasource.
      """
     annotator = Annotator()
     annotator.addDatasource(TestUtils.createTranscriptProviderDatasource(self.config))
     tmp = annotator.createHeaderString()
     self.assertTrue(tmp.find("Gaf ") != -1 or tmp.find("GENCODE") != -1, "Could not find Gaf or GENCODE version in header string.")
     self.assertTrue(tmp.find("Oncotator") != -1, "Could not find the word Oncotator in header string.")
    def testTCGAMAFAsInputAndQuickAnnotate(self):
        """ Test that we can take in a TCGA MAF (using MAFLITE), do annotating, and still render it properly """
        inputFilename = "testdata/maf/Patient0.maf.annotated"
        tmp = MafliteInputMutationCreator(inputFilename, 'configs/maflite_input.config')
        outputFilename = "out/testTCGAMAFAsInputAndQuickAnnotate.tsv"
        outputRenderer = TcgaMafOutputRenderer(outputFilename, 'configs/tcgaMAF2.4_output.config')
        annotator = Annotator()
        
        annotator.setInputCreator(tmp)
        annotator.setOutputRenderer(outputRenderer)
        ds = DatasourceFactory.createDatasource("testdata/thaga_janakari_gene_ds/hg19/tj_data.config", "testdata/thaga_janakari_gene_ds/hg19/")
        annotator.addDatasource(ds)
        annotator.annotate()
        
        statinfo = os.stat(outputFilename)
        self.assertTrue(statinfo.st_size > 0, "Generated MAF file (" + outputFilename + ") is empty.")
        tsvReaderIn = GenericTsvReader(inputFilename)
        tsvReader = GenericTsvReader(outputFilename)
        
        self.assertTrue(tsvReader.getComments().find('#version') != -1, "First line did not specify a version number")
        self.assertTrue("i_TJ_Data_Why" in tsvReader.getFieldNames(), "New field missing (i_TJ_Data_Why) from header")
        self.assertTrue("i_TJ_Data_Who" in tsvReader.getFieldNames(), "New field missing (i_TJ_Data_Who) from header")
        
        ctrOut = 0
        for lineDict in tsvReader:
            ctrOut += 1
        ctrIn = 0
        for lineDict in tsvReaderIn:
            ctrIn += 1
        ctrIn += len(tsvReaderIn.getCommentsAsList())
        ctrOut += len(tsvReader.getCommentsAsList())

        self.assertTrue(ctrOut == (ctrIn + 2), "Output file should have same number of lines plus two (for maf version and Oncotator version comments) as input file.  (In,Out): " + str(ctrIn) + ", " + str(ctrOut))
Example #10
0
    def test_simple_transcript_annotation(self):
        """Test web api backend call /transcript/ """
        # http://www.broadinstitute.org/oncotator/transcript/ENST00000215832.6/
        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)

        tx = annotator.retrieve_transcript_by_id("ENST00000215832.6")
        self.assertTrue(tx is not None)
        self.assertTrue(tx.get_gene() == "MAPK1")
Example #11
0
    def test_querying_transcripts_by_genes(self):
        """Test that we can get all of the transcripts for a given set of genes. """

        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)

        # Step 1 get all of the relevant transcripts
        txs = annotator.retrieve_transcripts_by_genes(["MAPK1", "PIK3CA"])
        self.assertTrue(len(txs) > 3)
    def _annotateTest(self, inputFilename, outputFilename, datasource_dir, inputFormat="MAFLITE", outputFormat="TCGAMAF", default_annotations=TCGA_MAF_DEFAULTS, override_annotations=None, is_skip_no_alts=False):
        self.logger.info("Initializing Annotator...")

        if override_annotations is None:
            override_annotations = dict()

        annotator = Annotator()
        runSpec = RunSpecificationFactory.create_run_spec(inputFormat, outputFormat, inputFilename, outputFilename, defaultAnnotations=default_annotations, datasourceDir=datasource_dir, globalAnnotations=override_annotations, is_skip_no_alts=is_skip_no_alts)
        annotator.initialize(runSpec)
        self.logger.info("Annotation starting...")
        return annotator.annotate()
Example #13
0
 def test_single_sample_onp_combiner(self):
     """test that we can create an onp combined TCGA maf without crashing"""
     input_filename = 'testdata/maflite/onp.singlesample.maf.txt'
     output_filename = 'out/testSingleSampleOnpCombiner.maf'
     config = TestUtils.createUnitTestConfig()
     defaultdb = config.get('DEFAULT',"dbDir")
     spec = RunSpecificationFactory.create_run_spec("MAFLITE","TCGAMAF", input_filename, output_filename,
                                                    datasource_dir=defaultdb,
                                             other_opts={OptionConstants.INFER_ONPS: True})
     annotator = Annotator()
     annotator.initialize(spec)
     annotator.annotate()
    def testAnnotationWithMafliteWithTrailingSpaces(self):
        """
        Tests the ability to annotate a maflite file that contains trailing spaces in ref and alt alleles.
        """
        db_dir = self.config.get('DEFAULT',"dbDir")
        inputFilename = os.path.join(*["testdata", "maflite", "example.trailing_whitespace_in_alleles.maflite"])
        outputFilename = os.path.join("out", "example.trailing_whitespace_in_alleles.maf.txt")

        annotator = Annotator()
        run_spec = RunSpecificationFactory.create_run_spec("MAFLITE", "TCGAMAF", inputFilename, outputFilename,
                                                           datasource_dir=db_dir, annotating_type=RunSpecification.ANNOTATE_MUTATIONS)
        annotator.initialize(run_spec)
        annotator.annotate()
Example #15
0
 def _annotate_m2_vcf(self, input_vcf_file, output_tcgamaf_file):
     # For this conversion, you must specify the barcodes manually
     override_annotations = dict()
     override_annotations.update({'tumor_barcode': 'Patient0-Tumor', 'normal_barcode': 'Patient0-Normal'})
     other_opts = {OptionConstants.COLLAPSE_FILTER_COLS: True, OptionConstants.NO_PREPEND: True,
                   OptionConstants.SPLIT_ALLELIC_DEPTH: False, OptionConstants.INFER_ONPS: True}
     # Use an empty datasource dir in order to speed this up.
     annotator = Annotator()
     runSpec = RunSpecificationFactory.create_run_spec("VCF", "TCGAMAF", input_vcf_file, output_tcgamaf_file,
                                                       datasource_dir=".", global_annotations=override_annotations,
                                                       is_skip_no_alts=True, other_opts=other_opts)
     annotator.initialize(runSpec)
     annotator.annotate()
Example #16
0
    def test_simple_genes_by_gene_annotation(self):
        """Test web api backend call /gene/ """
        # http://www.broadinstitute.org/oncotator/gene/MAPK1/
        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)

        txs = annotator.retrieve_transcripts_by_genes(["MAPK1"])
        self.assertTranscriptsFound(txs)

        mut_dict = annotator.annotate_genes_given_txs(txs)
        self.assertTrue(len(mut_dict.keys()) == 1)
 def testBasicAnnotation(self):
     ''' Annotate from a basic tsv of Genomic positions.  This tests both single- and multiple-nucleotide variants.  The tsv is already installed (i.e. proper config file created).
     '''
     outputFilename = 'out/genericGenomePositionTest.out.tsv'
     
     gpDS = DatasourceFactory.createDatasource("testdata/small_genome_position_tsv_ds/oreganno_trim.config", "testdata/small_genome_position_tsv_ds/")
     
     annotator = Annotator()
     annotator.setInputCreator(MafliteInputMutationCreator('testdata/maflite/tiny_maflite.maf.txt'))
     annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
     annotator.addDatasource(gpDS)
     testFilename = annotator.annotate()
     
     # Make sure that some values were populated
     self.assertTrue(os.path.exists(testFilename))
     tsvReader = GenericTsvReader(testFilename) 
     
     ctr = 1
     # Two overlap, one does not.  Repeat...
     for lineDict in tsvReader:
         if (ctr % 3 == 0):
             self.assertTrue(lineDict["ORegAnno_hg19.oreganno.id"] == '', "Line " + str(ctr) + " should have had blank value, but did not: " + lineDict["ORegAnno_hg19.oreganno.id"])
         else:
             self.assertFalse(lineDict["ORegAnno_hg19.oreganno.id"] == '', "Line " + str(ctr) + " should not have had blank value, but did.")
             self.assertTrue(lineDict["ORegAnno_hg19.oreganno.id"] == 'OREG0013034', "Line " + str(ctr) + " did not have correct value: " + lineDict["ORegAnno_hg19.oreganno.id"])
         ctr = ctr + 1
    def testCreationAndAnnotation(self):
        """ Test the datasource creation and then do a simple annotation
        """
        outputFilename = 'out/genericGeneProteinPositionTest.out.tsv'

        gafDS = TestUtils.createTranscriptProviderDatasource(self.config)
        gppDS = DatasourceFactory.createDatasource("testdata/simple_uniprot_natvar/simple_uniprot_natvar.config", "testdata/simple_uniprot_natvar/")

        annotator = Annotator()
        annotator.setInputCreator(MafliteInputMutationCreator('testdata/maflite/tiny_maflite_natvar.maf.tsv'))
        annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
        annotator.addDatasource(gafDS)
        annotator.addDatasource(gppDS)
        testFilename = annotator.annotate()

        # Make sure that some values were populated
        self.assertTrue(os.path.exists(testFilename))
        tsvReader = GenericTsvReader(testFilename)

        ctr = 0
        for lineDict in tsvReader:
            colName = "UniProt_NatVar_natural_variations"
            self.assertTrue(sorted(lineDict[colName].split("|")) == sorted("R -> RR (in EDMD2).|R -> Q (in EDMD2).".split("|")), "Annotation value did not match: " + lineDict[colName])
            ctr += 1

        self.assertTrue(ctr == 1, "Number of mutations incorrect (1): " + str(ctr) )
Example #19
0
    def testManualAnnotations(self):
        """ Test that the manual annotation facility in the Annotator is working properly. """
        annotator = Annotator()
        overrides = {'source': 'Capture', 'status': 'Somatic', 'phase': 'Phase_I', 'sequencer': 'Illumina GAIIx'}
        annotator.setManualAnnotations(overrides)
        inputCreator = MafliteInputMutationCreator('testdata/maflite/Patient0.snp.maf.txt')
        outputRenderer = SimpleOutputRenderer("out/testManualAnnotationsFile.tsv")
        annotator.setInputCreator(inputCreator)
        annotator.setOutputRenderer(outputRenderer)

        testOutputFilename = annotator.annotate()

        keysOfInterest = overrides.keys()

        statinfo = os.stat(testOutputFilename)
        self.assertTrue(statinfo.st_size > 0, "Generated TSV file (" + testOutputFilename + ") is empty.")

        tsvReader = GenericTsvReader(testOutputFilename)

        ctr = 1
        for lineDict in tsvReader:
            for k in keysOfInterest:
                self.assertTrue(lineDict[k] != "__UNKNOWN__",
                                "__UNKNOWN__ value seen on line " + str(ctr) + ", when it should be populated: " + k)
                self.assertTrue(lineDict[k] != "",
                                "Blank value seen on line " + str(ctr) + ", when it should be populated: " + k)
                self.assertTrue(lineDict[k] == overrides[k],
                                "Value for " + k + " on line " + str(ctr) + " did not match override: " + str(
                                    lineDict[k]) + " <> " + str(overrides[k]))
            ctr += 1
Example #20
0
    def test_simple_genes_by_region_annotation(self):
        """Test web api backend call /genes/ """
        # http://www.broadinstitute.org/oncotator/genes/chr22_22112223_22312558/
        # Two genes: chr22:22,112,223-22,312,558
        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)

        # Here is what the API would call....
        txs = annotator.retrieve_transcripts_by_region("22", 22112223, 22312558)
        self.assertTranscriptsFound(txs)

        mut_dict = annotator.annotate_genes_given_txs(txs)

        # Each mut will be for a separate gene
        for gene in mut_dict.keys():
            mut = mut_dict[gene]
            alt_accessions = mut["UniProt_alt_uniprot_accessions"].split("|")
            tcgascape_amp_peaks = mut["TCGAScape_Amplification_Peaks"].split("|")
            tcgascape_del_peaks = mut["TCGAScape_Deletion_Peaks"].split("|")
            tumorscape_amp_peaks = mut["TUMORScape_Amplification_Peaks"].split("|")
            tumorscape_del_peaks = mut["TUMORScape_Deletion_Peaks"].split("|")
            full_name = mut["HGNC_Approved Name"]
            cosmic = {
                "tissue_types_affected": mut["COSMIC_Tissue_tissue_types_affected"],
                "total_alterations_in_gene": mut["COSMIC_Tissue_tissue_types_affected"],
            }
            alt_aliases = list(
                itertools.chain([mut["HGNC_Previous Symbols"].split(", "), mut["HGNC_Synonyms"].split(", ")])
            )
            location = mut["HGNC_Chromosome"]
            uniprot_accession = mut["UniProt_uniprot_accession"]
            transcripts = mut["transcripts"]
            self.assertTrue(transcripts is not None)
            self.assertTrue(len(transcripts) > 0)
            self.assertTrue(transcripts.startswith("ENST"))
            strand = mut["strand"]
            klass = mut["class"]
            uniprot_experimentals = mut["UniProt_AA_experimental_info"].split("|")
            self.assertTrue(uniprot_experimentals is not None)
            uniprot_natural_variations = mut["UniProt_AA_natural_variation"].split("|")
            uniprot_regions = mut["UniProt_AA_region"].split("|")
            uniprot_sites = mut["UniProt_AA_site"].split("|")
            uniprot_go_biological_processes = mut["UniProt_GO_Biological_Process"].split("|")
            uniprot_go_cellular_components = mut["UniProt_GO_Cellular_Component"].split("|")
            self.assertTrue(uniprot_go_cellular_components is not None)
            uniprot_go_molecular_functions = mut["UniProt_GO_Molecular_Function"].split("|")
            pass
    def test_annotating_uniprot_test_file(self):
        """Test variants with known issues with older version of UniProt datasource. This test will fail if using older version of uniprot datasource (pre-2014) """
        db_dir = TestUtils.createUnitTestConfig().get('DEFAULT',"dbDir")
        annotator = Annotator()
        out_file_name = "out/uniprot_recovery.maf.annotated"
        runSpec = RunSpecificationFactory.create_run_spec("MAFLITE", "TCGAMAF", "testdata/maflite/uniprot_recovery.maflite",
                                                          out_file_name, datasource_dir=db_dir, tx_mode=TranscriptProvider.TX_MODE_BEST_EFFECT)
        annotator.initialize(runSpec)
        annotator.annotate()

        out_file_reader = GenericTsvReader(out_file_name)
        for i,line_dict in enumerate(out_file_reader):
            self.assertTrue(line_dict['UniProt_AApos'] != "0")

            #TODO: The fourth entry is currently not picking up the uniprot entry for this.  Remove the "if" statement once issue #253 is addressed
            if i != 4:
                self.assertTrue(line_dict['SwissProt_entry_Id'].endswith("HUMAN"))
    def test_proper_conversion_vcf_to_maf_with_collapse_filter_cols(self):
        """Test FILTER col is properly rendered when using the collapse-filter-cols option."""

        input_fname = 'testdata/vcf/example.vcf'
        output_fname = 'out/example.one_filter_col.maf.txt'
        annotator = Annotator()
        other_opts = {'collapse_filter_cols': True}

        from oncotator.utils.RunSpecification import RunSpecification
        run_spec = RunSpecificationFactory.create_run_spec('VCF', 'TCGAMAF', input_fname, output_fname, other_opts=other_opts)
        annotator.initialize(run_spec)
        annotator.annotate()

        tsv_reader = GenericTsvReader(output_fname)
        for line_dict in tsv_reader:
            self.assertIn('i_filter', line_dict)
            self.assertTrue(line_dict['i_filter'] in ['PASS', 'q10'])
    def testDuplicateAnnotation(self):
        """
        Tests that the duplicate annotations are parsed correctly.
        """
        inputFilename = os.path.join(*["testdata", "vcf", "example.duplicate_annotation.vcf"])
        outputFilename = os.path.join("out", "example.duplicate_annotation.out.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename)
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()

        tsvReader = GenericTsvReader(outputFilename)
        fieldnames = tsvReader.getFieldNames()
        self.assertTrue("variant_status" in fieldnames, "variant_status field is missing in the header.")
        self.assertTrue("sample_variant_status" in fieldnames, "sample_variant_status is missing in the header.")

        row = tsvReader.next()
        self.assertTrue("variant_status" in row, "variant_status field is missing in the row.")
        self.assertTrue("sample_variant_status" in row, "sample_variant_status is missing in the row.")

        self.assertEqual("2", row["variant_status"], "Incorrect value of variant_status.")
        self.assertEqual("0", row["sample_variant_status"], "Incorrect value of sample_variant_status")
    def testSNPsAndIndelStartAndEndPos(self):
        """
        Tests that the start and end positions of SNPs and Indels are parsed as defined by the NCI's MAF specification
        (https://wiki.nci.nih.gov/display/TCGA/Mutation+Annotation+Format+(MAF)+Specification).
        """
        inputFilename = os.path.join(*["testdata", "vcf", "example.snps.indels.vcf"])
        outputFilename = os.path.join("out", "example.snps.indels.out.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename)
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()

        tsvReader = GenericTsvReader(outputFilename)
        for row in tsvReader:
            if row['start'] == "16890445":
                self.assertEqual(row["end"], "16890445", "The value should be %s but it was %s." % ("16890445",
                                                                                                    row["end"]))
            elif row["start"] == "154524458":
                self.assertEqual(row["end"], "154524459", "The value should be %s but it was %s." % ("154524459",
                                                                                                     row["end"]))
            elif row["start"] == "114189432":
                self.assertEqual(row["end"], "114189433", "The value should be %s but it was %s." % ("114189433",
                                                                                                     row["end"]))
    def testSimpleAnnotationWithExampleVcf(self):
        """
        Tests the ability to do a simple Gaf 3.0 annotation.
        """
        inputFilename = os.path.join(*["testdata", "vcf", "example.vcf"])
        outputFilename = os.path.join("out", "simpleVCF.Gaf.annotated.out.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename, [])
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.addDatasource(TestUtils.createTranscriptProviderDatasource(self.config))
        annotator.annotate()
    def testMafInput(self):
        """Make sure that we can render a TCGA VCF from a TCGA MAF -- using no datasources"""
        inputFile = "testdata/maf/Patient1.snp.maf.annotated"
        outputFilename = "out/maf2tcgavcf.vcf"
        mafIn = MafliteInputMutationCreator(inputFile)
        vcfOR = TcgaVcfOutputRenderer(outputFilename)

        annotator = Annotator()
        annotator.setInputCreator(mafIn)
        annotator.setOutputRenderer(vcfOR)
        annotator.setManualAnnotations(self._createManualAnnotations())
        annotator.annotate()
        self.assertTrue(os.path.exists(outputFilename))
        statinfo = os.stat(outputFilename)
        self.assertTrue(statinfo.st_size > 0, "Generated VCF file (" + outputFilename + ") is empty.")
Example #27
0
    def test_querying_transcripts_by_region(self):
        """Test web api backend call /transcripts/.... """
        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)
        txs = annotator.retrieve_transcripts_by_region("4", 50164411, 60164411)
        self.assertTranscriptsFound(txs)

        ## Here is an example of getting enough data to populate the json in doc/transcript_json_commented.json.txt
        # None of these values are validated.
        for tx in txs:
            transcript_id = tx.get_transcript_id()
            tx_start = tx.determine_transcript_start()
            tx_end = tx.determine_transcript_stop()
            gene = tx.get_gene()
            chr = tx.get_contig()
            n_exons = len(tx.get_exons())
            strand = tx.get_strand()
            footprint_start, footprint_end = tx.determine_cds_footprint()
            klass = tx.get_gene_type()
            cds_start = tx.determine_cds_start()
            cds_end = tx.determine_cds_stop()
            id = tx.get_gene_id()
            genomic_coords = [[exon[0], exon[1]] for exon in tx.get_exons()]
            transcript_coords = [
                [TranscriptProviderUtils.convert_genomic_space_to_exon_space(exon[0] + 1, exon[1], tx)]
                for exon in tx.get_exons()
            ]
            code_len = int(cds_end) - int(cds_start) + 1

            # If refseq datasources are not available, this will fail.
            # Step 2 annotate the transcript, which produces a dummy mutation with the refseq annotations.
            dummy_mut = annotator.annotate_transcript(tx)
            refseq_mRNA_id = dummy_mut["gencode_xref_refseq_mRNA_id"]
            refseq_prot_id = dummy_mut["gencode_xref_refseq_prot_acc"]

            # Description is unavailable right now
            description = ""

            self.assertTrue(refseq_mRNA_id is not None)
            self.assertTrue(refseq_prot_id is not None)
            self.assertTrue(len(transcript_coords) == n_exons)
    def testBasicAnnotation(self):
        """ Test annotation from a generic TSV based on a transcript annotation.  Only confirms the proper headers of the output. """
        # We need a gaf data source to annotate gene

        gafDatasource = TestUtils.createTranscriptProviderDatasource(config=self.config)
        transcriptDS = DatasourceFactory.createDatasource(
            "testdata/small_transcript_tsv_ds/small_transcript_tsv_ds.config", "testdata/small_transcript_tsv_ds/"
        )
        outputFilename = "out/genericTranscriptTest.out.tsv"

        annotator = Annotator()
        annotator.setInputCreator(MafliteInputMutationCreator("testdata/maflite/Patient0.snp.maf.txt"))
        annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
        annotator.addDatasource(gafDatasource)
        annotator.addDatasource(transcriptDS)
        outputFilename = annotator.annotate()

        tsvReader = GenericTsvReader(outputFilename)
        headers = tsvReader.getFieldNames()
        self.assertTrue("refseq_test_mRNA_Id" in headers, "refseq_test_mRNA_Id not found in headers: " + str(headers))
        self.assertTrue("refseq_test_prot_Id" in headers, "refseq_test_prot_Id not found in headers: " + str(headers))
Example #29
0
    def testSimpleAnnotationWithAComplexVcf(self):
        """
        Tests the ability to parse a rather complex VCF file without any errors.
        """
        inputFilename = os.path.join(*["testdata", "vcf", "random.vcf"])
        outputFilename = os.path.join("out", "random.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename, [])
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()
Example #30
0
    def test_full_seg_file_annotations(self):
        """Test that we can read in a seg file, do a proper full annotation, and output as SIMPLE_TSV"""
        inputFilename = "testdata/seg/Patient0.seg.txt"
        output_filename = "out/test_full_seg_file_annotations.tsv"
        db_dir = self.config.get('DEFAULT', "dbDir")
        if os.path.exists(output_filename):
            os.remove(output_filename)

        annotator = Annotator()
        run_spec = RunSpecificationFactory.create_run_spec(
            "SEG_FILE",
            "SIMPLE_TSV",
            inputFilename,
            output_filename,
            datasourceDir=db_dir,
            annotating_type=RunSpecification.ANNOTATE_SEGMENTS)
        annotator.initialize(run_spec)
        annotator.annotate()

        # Now check the output
        output_reader = GenericTsvReader(output_filename)

        required_cols = ["Sample", "Num_Probes", "Segment_Mean"]
        headers = output_reader.getFieldNames()
        for rcol in required_cols:
            self.assertTrue(rcol in headers)

        for line_dict in output_reader:
            self.assertTrue(line_dict['start'] is not None)
            self.assertTrue(line_dict['start'].strip() != "")
            self.assertTrue(line_dict['end'] is not None)
            self.assertTrue(line_dict['end'].strip() != "")
            self.assertTrue("genes" in line_dict.keys())
            self.assertTrue(len(line_dict["genes"].split(",")) > 0)
Example #31
0
 def _annotate_m2_vcf(self, input_vcf_file, output_tcgamaf_file):
     # For this conversion, you must specify the barcodes manually
     override_annotations = dict()
     override_annotations.update({
         'tumor_barcode': 'Patient0-Tumor',
         'normal_barcode': 'Patient0-Normal'
     })
     other_opts = {
         OptionConstants.COLLAPSE_FILTER_COLS: True,
         OptionConstants.NO_PREPEND: True,
         OptionConstants.SPLIT_ALLELIC_DEPTH: False,
         OptionConstants.INFER_ONPS: True
     }
     # Use an empty datasource dir in order to speed this up.
     annotator = Annotator()
     runSpec = RunSpecificationFactory.create_run_spec(
         "VCF",
         "TCGAMAF",
         input_vcf_file,
         output_tcgamaf_file,
         datasource_dir=".",
         global_annotations=override_annotations,
         is_skip_no_alts=True,
         other_opts=other_opts)
     annotator.initialize(runSpec)
     annotator.annotate()
    def testDoubleAnnotationError(self):
        ''' Given a maf file that used to cause a duplicate annotation exception, do not throw that (or any) exception. '''
        outputFilename = 'out/genericGenomePositionDoubleAnnotationTest.out.tsv'

        gpDS = DatasourceFactory.createDatasource(
            "testdata/small_genome_position_tsv_ds/oreganno_trim.config",
            "testdata/small_genome_position_tsv_ds/")

        annotator = Annotator()
        annotator.setInputCreator(
            MafliteInputMutationCreator(
                'testdata/maflite/testDoubleAnnotate.maf.tsv'))
        annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
        annotator.addDatasource(gpDS)
        testFilename = annotator.annotate()

        # Make sure that some values were populated
        self.assertTrue(os.path.exists(testFilename))
Example #33
0
    def testSkippingAltsForSingleMut(self):
        """Test a simple case where a single mutation with alt_allele_seen of False is not produced."""

        runSpec = RunSpecification()
        runSpec.initialize(None, None, datasources=[], is_skip_no_alts=True)

        # Initialize the annotator with the runspec
        annotator = Annotator()
        annotator.initialize(runSpec)

        m = MutationData()
        m.chr = "1"
        m.start = "12941796"
        m.end = "12941796"
        m.alt_allele = "G"
        m.ref_allele = "T"
        m.createAnnotation("alt_allele_seen", "False")

        muts = [m]

        muts = annotator.annotate_mutations(muts)
        self.assertRaises(StopIteration, muts.next)
Example #34
0
    def testDefaultAnnotations(self):
        """Test that the default annotation values populate properly. """
        annotator = Annotator()
        default_annotations = {"test2": "foo2", "test3": "Should not be seen"}
        overrides = {'test3': 'foo3'}

        m1 = MutationData()
        m1.createAnnotation("test1", "foo1")
        m1.createAnnotation("test2", "")

        m2 = MutationData()
        m2.createAnnotation("test1", "")

        m3 = MutationData()
        m3.createAnnotation("test1", "")
        m3.createAnnotation("test2", "foo2-original")

        muts = [m1, m2, m3]

        muts2 = annotator._applyManualAnnotations(muts, overrides)
        muts_final_gen = annotator._applyDefaultAnnotations(
            muts2, default_annotations)

        muts_final = []
        for m in muts_final_gen:
            self.assertTrue(m['test3'] == "foo3", "Override did not work")
            muts_final.append(m)

        self.assertTrue(muts_final[0]['test1'] == "foo1")
        self.assertTrue(muts_final[0]['test2'] == "foo2")
        self.assertTrue(muts_final[0]['test3'] == "foo3")

        self.assertTrue(muts_final[1]['test1'] == "")
        self.assertTrue(muts_final[1]['test2'] == "foo2")
        self.assertTrue(muts_final[1]['test3'] == "foo3")

        self.assertTrue(muts_final[2]['test1'] == "")
        self.assertTrue(muts_final[2]['test2'] == "foo2-original")
        self.assertTrue(muts_final[2]['test3'] == "foo3")
    def testAnnotationWithDuplicateValuesInVcf(self):
        """
        Tests the ability to parse a VCF that contains an INFO, FILTER, and INFO field with the same name.
        """
        inputFilename = os.path.join(
            *["testdata", "vcf", "example.duplicate_fields.vcf"])
        outputFilename = os.path.join("out", "example.duplicate_fields2.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename, [])
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()
    def testAnnotationWithNoSampleNameExampleVcf(self):
        """
        Tests whether parsed annotations match the actual annotations when the input is a VCF file that has no samples.
        """
        inputFilename = os.path.join(
            *["testdata", "vcf", "example.sampleName.removed.vcf"])
        outputFilename = os.path.join("out",
                                      "example.sampleName.removed.out.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        renderer = SimpleOutputRenderer(outputFilename)
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()
Example #37
0
    def testManualAnnotations(self):
        """ Test that the manual annotation facility in the Annotator is working properly. """
        annotator = Annotator()
        overrides = {
            'source': 'Capture',
            'status': 'Somatic',
            'phase': 'Phase_I',
            'sequencer': 'Illumina GAIIx'
        }
        annotator.setManualAnnotations(overrides)
        inputCreator = MafliteInputMutationCreator(
            'testdata/maflite/Patient0.snp.maf.txt')
        outputRenderer = SimpleOutputRenderer(
            "out/testManualAnnotationsFile.tsv")
        annotator.setInputCreator(inputCreator)
        annotator.setOutputRenderer(outputRenderer)

        testOutputFilename = annotator.annotate()

        keysOfInterest = overrides.keys()

        statinfo = os.stat(testOutputFilename)
        self.assertTrue(
            statinfo.st_size > 0,
            "Generated TSV file (" + testOutputFilename + ") is empty.")

        tsvReader = GenericTsvReader(testOutputFilename)

        ctr = 1
        for lineDict in tsvReader:
            for k in keysOfInterest:
                self.assertTrue(
                    lineDict[k] != "__UNKNOWN__",
                    "__UNKNOWN__ value seen on line " + str(ctr) +
                    ", when it should be populated: " + k)
                self.assertTrue(
                    lineDict[k] != "", "Blank value seen on line " + str(ctr) +
                    ", when it should be populated: " + k)
                self.assertTrue(
                    lineDict[k] == overrides[k], "Value for " + k +
                    " on line " + str(ctr) + " did not match override: " +
                    str(lineDict[k]) + " <> " + str(overrides[k]))
            ctr += 1
    def testAnnotationWithExampleVcf(self):
        """
        Tests whether parsed annotations match the actual annotations in a simple TSV.  Missing format fields yield -->""  ".,." --> ","
        """
        inputFilename = os.path.join(*["testdata", "vcf", "example.vcf"])
        outputFilename = os.path.join("out", "example.out.tsv")
        expectedOutputFilename = os.path.join(
            *["testdata", "vcf", "example.expected.out.tsv"])

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename)
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()

        tsvReader = GenericTsvReader(outputFilename)

        current = pandas.read_csv(outputFilename,
                                  sep='\t',
                                  header=len(tsvReader.getCommentsAsList()))
        expected = pandas.read_csv(expectedOutputFilename, sep='\t')

        currentColNames = set()
        for i in range(len(current.columns)):
            currentColNames.add(current.columns[i])

        expectedColNames = set()
        for i in range(len(expected.columns)):
            expectedColNames.add(expected.columns[i])

        self.assertTrue(
            len(currentColNames.symmetric_difference(expectedColNames)) is 0,
            "Should have the same columns")
        self.assertTrue(
            len(current.index) == len(expected.index),
            "Should have the same number of rows")

        for colName in currentColNames:
            self.assertTrue(
                sum((current[colName] == expected[colName])
                    | (pandas.isnull(current[colName])
                       & pandas.isnull(expected[colName]))) == len(
                           current.index),
                "Should have the same values in column " + colName + ": \n" +
                str(current[colName]) + "\nvs\n" + str(expected[colName]))
    def testMissingFilter(self):
        """
        Tests that the missing FILTER fields are parsed correctly.
        """
        inputFilename = os.path.join(
            *["testdata", "vcf", "example.missing_filters.vcf"])
        outputFilename = os.path.join("out", "example.missing_filters.out.tsv")
        expectedOutputFilename = os.path.join(
            *["testdata", "vcf", "example.expected.missing_filters.out.tsv"])

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename)
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.annotate()

        tsvReader = GenericTsvReader(outputFilename)

        current = pandas.read_csv(outputFilename,
                                  sep='\t',
                                  header=len(tsvReader.getCommentsAsList()))
        expected = pandas.read_csv(expectedOutputFilename, sep='\t')

        currentColNames = set()
        for i in range(len(current.columns)):
            currentColNames.add(current.columns[i])

        expectedColNames = set()
        for i in range(len(expected.columns)):
            expectedColNames.add(expected.columns[i])

        self.assertTrue(
            len(currentColNames.symmetric_difference(expectedColNames)) is 0,
            "Should have the same columns")
        self.assertTrue(
            len(current.index) == len(expected.index),
            "Should have the same number of rows")

        for colName in currentColNames:
            self.assertTrue(
                sum((current[colName] == expected[colName])
                    | (pandas.isnull(current[colName])
                       & pandas.isnull(expected[colName]))) == len(
                           current.index),
                "Should have the same values in column " + colName)
Example #40
0
 def test_rendering_combined_to_tsv(self):
     """Test that we produce a merged ONP simple tsv file without crashing """
     input_filename = os.path.join(*["testdata", "maflite", "onp_combination.maf.txt"])
     output_filename = os.path.join("out", "onp_combination.tsv")
     spec = RunSpecificationFactory.create_run_spec("MAFLITE","SIMPLE_TSV",input_filename, output_filename,
                                             other_opts={OptionConstants.INFER_ONPS: True})
     annotator = Annotator()
     annotator.initialize(spec)
     annotator.annotate()
    def test_simple_seg_file_annotations(self):
        """Test that we can read in a seg file, do GENCODE annotation, and output as SIMPLE_TSV"""
        inputFilename = "testdata/seg/Patient0.seg.txt"
        output_filename = "out/test_simple_seg_file_annotations.tsv"
        if os.path.exists(output_filename):
            os.remove(output_filename)
        ic = MafliteInputMutationCreator(inputFilename, None,
                                         'configs/seg_file_input.config')
        segs = ic.createMutations()

        i = 1
        for i, seg in enumerate(segs):
            pass

        self.assertTrue(
            (i + 1) == 27,
            "Found %d segments when there should have been 27." % (i + 1))

        ic = MafliteInputMutationCreator(inputFilename, None,
                                         'configs/seg_file_input.config')
        segs = ic.createMutations()

        gencode_ds = TestUtils._create_test_gencode_v19_ds(
            "out/seg_file_gencode_ds")
        annotator = Annotator()

        segs_annotated = []
        for seg in segs:
            segs_annotated.append(gencode_ds.annotate_segment(seg))

        outputRenderer = SimpleOutputRenderer(output_filename, '')
        outputRenderer.renderMutations(segs_annotated.__iter__())

        # Now check the output
        output_reader = GenericTsvReader(output_filename)

        required_cols = ["Sample", "Num_Probes", "Segment_Mean"]
        headers = output_reader.getFieldNames()
        for rcol in required_cols:
            self.assertTrue(rcol in headers)

        for line_dict in output_reader:
            self.assertTrue(line_dict['start'] is not None)
            self.assertTrue(line_dict['start'].strip() != "")
            self.assertTrue(line_dict['end'] is not None)
            self.assertTrue(line_dict['end'].strip() != "")
            self.assertTrue("genes" in line_dict.keys())
Example #42
0
 def test_single_sample_onp_combiner(self):
     """test that we can create an onp combined TCGA maf without crashing"""
     input_filename = 'testdata/maflite/onp.singlesample.maf.txt'
     output_filename = 'out/testSingleSampleOnpCombiner.maf'
     config = TestUtils.createUnitTestConfig()
     defaultdb = config.get('DEFAULT',"dbDir")
     spec = RunSpecificationFactory.create_run_spec("MAFLITE","TCGAMAF", input_filename, output_filename,datasourceDir=defaultdb,
                                             other_opts={OptionConstants.INFER_ONPS: True})
     annotator = Annotator()
     annotator.initialize(spec)
     annotator.annotate()
    def testBasicAnnotation(self):
        ''' Annotate from a basic tsv of Genomic positions.  This tests both single- and multiple-nucleotide variants.  The tsv is already installed (i.e. proper config file created).
        '''
        outputFilename = 'out/genericGenomePositionTest.out.tsv'

        gpDS = DatasourceFactory.createDatasource(
            "testdata/small_genome_position_tsv_ds/oreganno_trim.config",
            "testdata/small_genome_position_tsv_ds/")

        annotator = Annotator()
        annotator.setInputCreator(
            MafliteInputMutationCreator(
                'testdata/maflite/tiny_maflite.maf.txt'))
        annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
        annotator.addDatasource(gpDS)
        testFilename = annotator.annotate()

        # Make sure that some values were populated
        self.assertTrue(os.path.exists(testFilename))
        tsvReader = GenericTsvReader(testFilename)

        ctr = 1
        # Two overlap, one does not.  Repeat...
        for lineDict in tsvReader:
            if (ctr % 3 == 0):
                self.assertTrue(
                    lineDict["ORegAnno_hg19.oreganno.id"] == '', "Line " +
                    str(ctr) + " should have had blank value, but did not: " +
                    lineDict["ORegAnno_hg19.oreganno.id"])
            else:
                self.assertFalse(
                    lineDict["ORegAnno_hg19.oreganno.id"] == '', "Line " +
                    str(ctr) + " should not have had blank value, but did.")
                self.assertTrue(
                    lineDict["ORegAnno_hg19.oreganno.id"] == 'OREG0013034',
                    "Line " + str(ctr) + " did not have correct value: " +
                    lineDict["ORegAnno_hg19.oreganno.id"])
            ctr = ctr + 1
Example #44
0
    def testSwitchedFieldsWithExampleVcf(self):
        """
        Tests whether the switched tags are ignored.
        """
        inputFilename = os.path.join(*["testdata", "vcf", "example.bad.switched.fields.vcf"])
        outputFilename = os.path.join("out", "example.switched.out.tsv")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = SimpleOutputRenderer(outputFilename, [])
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
Example #45
0
    def test_simple_genes_by_gene_annotation(self):
        """Test web api backend call /gene/ """
        # http://www.broadinstitute.org/oncotator/gene/MAPK1/
        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)

        txs = annotator.retrieve_transcripts_by_genes(["MAPK1"])
        self.assertTranscriptsFound(txs)

        mut_dict = annotator.annotate_genes_given_txs(txs)
        self.assertTrue(len(mut_dict.keys()) == 1)
    def test_proper_conversion_vcf_to_maf_with_collapse_filter_cols(self):
        """Test FILTER col is properly rendered when using the collapse-filter-cols option."""

        input_fname = 'testdata/vcf/example.vcf'
        output_fname = 'out/example.one_filter_col.maf.txt'
        annotator = Annotator()
        other_opts = {'collapse_filter_cols': True}

        run_spec = RunSpecificationFactory.create_run_spec(
            'VCF', 'TCGAMAF', input_fname, output_fname, other_opts=other_opts)
        annotator.initialize(run_spec)
        annotator.annotate()

        tsv_reader = GenericTsvReader(output_fname)
        for line_dict in tsv_reader:
            self.assertIn('i_filter', line_dict)
            self.assertTrue(line_dict['i_filter'] in ['PASS', 'q10'])
Example #47
0
    def test_simple_genes_by_region_annotation(self):
        """Test web api backend call /genes/ """
        # http://www.broadinstitute.org/oncotator/genes/chr22_22112223_22312558/
        # Two genes: chr22:22,112,223-22,312,558
        datasource_list = DatasourceFactory.createDatasources(self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)

        # Here is what the API would call....
        txs = annotator.retrieve_transcripts_by_region("22", 22112223, 22312558)
        self.assertTranscriptsFound(txs)

        mut_dict = annotator.annotate_genes_given_txs(txs)

        # Each mut will be for a separate gene
        for gene in mut_dict.keys():
            mut = mut_dict[gene]
            alt_accessions = mut['UniProt_alt_uniprot_accessions'].split("|")
            tcgascape_amp_peaks = mut['TCGAScape_Amplification_Peaks'].split("|")
            tcgascape_del_peaks = mut['TCGAScape_Deletion_Peaks'].split("|")
            tumorscape_amp_peaks = mut['TUMORScape_Amplification_Peaks'].split("|")
            tumorscape_del_peaks = mut['TUMORScape_Deletion_Peaks'].split("|")
            full_name = mut['HGNC_Approved Name']
            cosmic = {"tissue_types_affected": mut['COSMIC_Tissue_tissue_types_affected'], "total_alterations_in_gene": mut["COSMIC_Tissue_tissue_types_affected"]}
            alt_aliases = list(itertools.chain([mut["HGNC_Previous Symbols"].split(", "), mut["HGNC_Synonyms"].split(", ")]))
            location = mut["HGNC_Chromosome"]
            uniprot_accession = mut["UniProt_uniprot_accession"]
            transcripts = mut['transcripts']
            self.assertTrue(transcripts is not None)
            self.assertTrue(len(transcripts) > 0)
            self.assertTrue(transcripts.startswith('ENST'))
            strand = mut['strand']
            klass = mut['class']
            uniprot_experimentals = mut['UniProt_AA_experimental_info'].split("|")
            self.assertTrue(uniprot_experimentals is not None)
            uniprot_natural_variations = mut['UniProt_AA_natural_variation'].split("|")
            uniprot_regions = mut['UniProt_AA_region'].split("|")
            uniprot_sites = mut['UniProt_AA_site'].split("|")
            uniprot_go_biological_processes = mut["UniProt_GO_Biological_Process"].split("|")
            uniprot_go_cellular_components = mut["UniProt_GO_Cellular_Component"].split("|")
            self.assertTrue(uniprot_go_cellular_components is not None)
            uniprot_go_molecular_functions = mut["UniProt_GO_Molecular_Function"].split("|")
            pass
Example #48
0
    def test_querying_transcripts_by_region(self):
        """Test web api backend call /transcripts/.... """
        datasource_list = DatasourceFactory.createDatasources(
            self._determine_db_dir(), "hg19", isMulticore=False)
        annotator = Annotator()
        for ds in datasource_list:
            annotator.addDatasource(ds)
        txs = annotator.retrieve_transcripts_by_region("4", 50164411, 60164411)
        self.assertTranscriptsFound(txs)

        ## Here is an example of getting enough data to populate the json in doc/transcript_json_commented.json.txt
        # None of these values are validated.
        for tx in txs:
            transcript_id = tx.get_transcript_id()
            tx_start = tx.determine_transcript_start()
            tx_end = tx.determine_transcript_stop()
            gene = tx.get_gene()
            chr = tx.get_contig()
            n_exons = len(tx.get_exons())
            strand = tx.get_strand()
            footprint_start, footprint_end = tx.determine_cds_footprint()
            klass = tx.get_gene_type()
            cds_start = tx.determine_cds_start()
            cds_end = tx.determine_cds_stop()
            id = tx.get_gene_id()
            genomic_coords = [[exon[0], exon[1]] for exon in tx.get_exons()]
            transcript_coords = [[
                TranscriptProviderUtils.convert_genomic_space_to_exon_space(
                    exon[0] + 1, exon[1], tx)
            ] for exon in tx.get_exons()]
            code_len = int(cds_end) - int(cds_start) + 1

            # If refseq datasources are not available, this will fail.
            # Step 2 annotate the transcript, which produces a dummy mutation with the refseq annotations.
            dummy_mut = annotator.annotate_transcript(tx)
            refseq_mRNA_id = dummy_mut["gencode_xref_refseq_mRNA_id"]
            refseq_prot_id = dummy_mut["gencode_xref_refseq_prot_acc"]

            # Description is unavailable right now
            description = ""

            self.assertTrue(refseq_mRNA_id is not None)
            self.assertTrue(refseq_prot_id is not None)
            self.assertTrue(len(transcript_coords) == n_exons)
Example #49
0
 def test_onp_combiner_snp_then_multiallelic(self):
     """test that we can handle reading a SNP then multiallelic from a VCF without crashing"""
     input_filename = 'testdata/vcf/infer_onp_fail_snp_then_multiallelic.vcf'
     output_filename = 'out/testSNPThenMultiallelic.maf.annotated'
     config = TestUtils.createUnitTestConfig()
     default_db = config.get('DEFAULT', "dbDir")
     spec = RunSpecificationFactory.create_run_spec(
         "VCF",
         "TCGAMAF",
         input_filename,
         output_filename,
         datasource_dir=default_db,
         is_skip_no_alts=True,
         other_opts={
             OptionConstants.INFER_ONPS: True,
             OptionConstants.COLLAPSE_NUMBER_ANNOTATIONS: True
         })
     annotator = Annotator()
     annotator.initialize(spec)
     annotator.annotate()
    def testAnnotationWithMafliteWithTrailingSpaces(self):
        """
        Tests the ability to annotate a maflite file that contains trailing spaces in ref and alt alleles.
        """
        db_dir = self.config.get('DEFAULT', "dbDir")
        inputFilename = os.path.join(*[
            "testdata", "maflite",
            "example.trailing_whitespace_in_alleles.maflite"
        ])
        outputFilename = os.path.join(
            "out", "example.trailing_whitespace_in_alleles.maf.txt")

        annotator = Annotator()
        run_spec = RunSpecificationFactory.create_run_spec(
            "MAFLITE",
            "TCGAMAF",
            inputFilename,
            outputFilename,
            datasource_dir=db_dir,
            annotating_type=RunSpecification.ANNOTATE_MUTATIONS)
        annotator.initialize(run_spec)
        annotator.annotate()
Example #51
0
    def test_overwriting_muts(self):
        """Ensure that (given correct configuration) we can annotate from a datasource, even if the datasource will overwrite an existing mutation."""
        # We will have an input with a "Who" annotation that this datasource will try to write.
        gene_ds = DatasourceFactory.createDatasource("testdata/thaga_janakari_gene_ds/hg19/tj_data.config", "testdata/thaga_janakari_gene_ds/hg19/")
        input_filename = "testdata/maflite/who_alt1_vs_alt2.maflite"
        output_filename = "out/who_alt1_vs_alt2.maf.annotated"
        input_format = "MAFLITE"
        output_format = "TCGAMAF"

        other_opts = {OptionConstants.ALLOW_ANNOTATION_OVERWRITING: True, OptionConstants.NO_PREPEND: True}

        run_spec = RunSpecificationFactory.create_run_spec_given_datasources(input_format, output_format, input_filename, output_filename,
                        datasource_list=[gene_ds], other_opts=other_opts)
        annotator = Annotator()
        annotator.initialize(run_spec)

        annotator.annotate()

        tsv_reader = GenericTsvReader(output_filename)

        for i, line_dict in enumerate(tsv_reader):
            self.assertTrue(line_dict.get('TJ_Data_Who', "") != "Tromokratis")
Example #52
0
    def testAnnotationWithMafliteWithTrailingSpaces(self):
        """
        Tests the ability to annotate a VCF file that contains trailing spaces in ref and alt alleles.
        """
        db_dir = self.config.get('DEFAULT',"dbDir")
        inputFilename = os.path.join(*["testdata", "vcf", "example.trailing_whitespace_in_alleles.vcf"])
        outputFilename = os.path.join("out", "example.trailing_whitespace_in_alleles.vcf")

        annotator = Annotator()
        from oncotator.utils.RunSpecification import RunSpecification
        run_spec = RunSpecificationFactory.create_run_spec("VCF", "VCF", inputFilename, outputFilename,
                                                           datasource_dir=db_dir, annotating_type=RunSpecification.ANNOTATE_MUTATIONS,
                                                           other_opts={'vcf_out_infer_genotypes': False})
        annotator.initialize(run_spec)
        annotator.annotate()

        #check output
        vcf_data = open(outputFilename).read()
        self.assertIn('\n1\t14907\t.\tA\tG\t', vcf_data)
        self.assertIn('\n1\t14930\trs150145850\tA\tG\t', vcf_data)
        self.assertIn('\n1\t14933\trs138566748\tG\tA\t', vcf_data)
        self.assertIn('\n1\t14948\trs148911281\tG\tA\t', vcf_data)
    def test_annotating_uniprot_test_file(self):
        """Test variants with known issues with older version of UniProt datasource. This test will fail if using older version of uniprot datasource (pre-2014) """
        db_dir = TestUtils.createUnitTestConfig().get('DEFAULT', "dbDir")
        annotator = Annotator()
        out_file_name = "out/uniprot_recovery.maf.annotated"
        runSpec = RunSpecificationFactory.create_run_spec(
            "MAFLITE",
            "TCGAMAF",
            "testdata/maflite/uniprot_recovery.maflite",
            out_file_name,
            datasourceDir=db_dir,
            tx_mode=TranscriptProvider.TX_MODE_BEST_EFFECT)
        annotator.initialize(runSpec)
        annotator.annotate()

        out_file_reader = GenericTsvReader(out_file_name)
        for i, line_dict in enumerate(out_file_reader):
            self.assertTrue(line_dict['UniProt_AApos'] != "0")

            #TODO: The fourth entry is currently not picking up the uniprot entry for this.  Remove the "if" statement once issue #253 is addressed
            if i != 4:
                self.assertTrue(
                    line_dict['SwissProt_entry_Id'].endswith("HUMAN"))
    def test_rendering_with_exons(self):
        """Test that we can render a seg file that includes exons at end points"""
        inputFilename = "testdata/seg/Middle_of_exon.seg.txt"
        output_filename = "out/test_exon_seg2.gene_list.tsv"
        db_dir = self.config.get('DEFAULT',"dbDir")
        if os.path.exists(output_filename):
            os.remove(output_filename)

        annotator = Annotator()
        run_spec = RunSpecificationFactory.create_run_spec("SEG_FILE", "GENE_LIST", inputFilename, output_filename,
                                                           datasource_dir=db_dir, annotating_type=RunSpecification.ANNOTATE_SEGMENTS)
        annotator.initialize(run_spec)
        annotator.annotate()

        # Now check the output
        output_reader = GenericTsvReader(output_filename)

        headers = output_reader.getFieldNames()

        for line_dict in output_reader:
            self.assertTrue(line_dict['segment_start'] is not None)
            self.assertTrue(line_dict['segment_start'].strip() != "")
            if line_dict['segment_end_gene'] == "MAPK1":
                self.assertTrue(line_dict['segment_end_exon'].strip() == "8+", "Should have been 8+, but saw: %s" % line_dict['segment_end_exon'].strip())
Example #55
0
    def testBasicAnnotation(self):
        ''' Test annotation from a generic TSV based on a transcript annotation.  Only confirms the proper headers of the output. '''
        # We need a gaf data source to annotate gene

        gafDatasource = TestUtils.createTranscriptProviderDatasource(
            config=self.config)
        transcriptDS = DatasourceFactory.createDatasource(
            "testdata/small_transcript_tsv_ds/small_transcript_tsv_ds.config",
            "testdata/small_transcript_tsv_ds/")
        outputFilename = 'out/genericTranscriptTest.out.tsv'

        annotator = Annotator()
        annotator.setInputCreator(
            MafliteInputMutationCreator(
                'testdata/maflite/Patient0.snp.maf.txt'))
        annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
        annotator.addDatasource(gafDatasource)
        annotator.addDatasource(transcriptDS)
        outputFilename = annotator.annotate()

        tsvReader = GenericTsvReader(outputFilename)
        headers = tsvReader.getFieldNames()
        self.assertTrue(
            "refseq_test_mRNA_Id" in headers,
            "refseq_test_mRNA_Id not found in headers: " + str(headers))
        self.assertTrue(
            "refseq_test_prot_Id" in headers,
            "refseq_test_prot_Id not found in headers: " + str(headers))
    def testBasicAnnotation(self):
        ''' Annotate from a basic tsv gene file.  Use the Gaf to annotate before trying the tsv -- required since the gene annotation must be populated.
        Using trimmed CancerGeneCensus as basis for this test.
        '''

        # cut -f 1 oncotator/test/testdata/small_tsv_ds/CancerGeneCensus_Table_1_full_2012-03-15_trim.txt | egrep -v Symbol | sed -r "s/^/'/g" | sed ':a;N;$!ba;s/\n/,/g' | sed -r "s/,'/','/g"
        genesAvailable = [
            'ABL1', 'ABL2', 'ACSL3', 'AF15Q14', 'AF1Q', 'AF3p21', 'AF5q31',
            'AKAP9', 'AKT1', 'AKT2', 'ALDH2', 'ALK', 'ALO17', 'APC',
            'ARHGEF12', 'ARHH', 'ARID1A', 'ARID2', 'ARNT', 'ASPSCR1', 'ASXL1',
            'ATF1', 'ATIC', 'ATM', 'ATRX', 'BAP1', 'BCL10', 'BCL11A', 'BCL11B'
        ]

        # We need a gaf data source to annotate gene

        gafDatasource = TestUtils.createTranscriptProviderDatasource(
            config=self.config)
        geneDS = DatasourceFactory.createDatasource(
            "testdata/small_tsv_ds/small_tsv_ds.config",
            "testdata/small_tsv_ds/")
        outputFilename = 'out/genericGeneTest.out.tsv'

        annotator = Annotator()
        annotator.setInputCreator(
            MafliteInputMutationCreator(
                'testdata/maflite/Patient0.snp.maf.txt'))
        annotator.setOutputRenderer(SimpleOutputRenderer(outputFilename))
        annotator.addDatasource(gafDatasource)
        annotator.addDatasource(geneDS)
        annotator.annotate()

        # Check that there were actual annotations performed.
        tsvReader = GenericTsvReader(outputFilename)

        fields = tsvReader.getFieldNames()
        self.assertTrue(
            'CGC_Abridged_Other Syndrome/Disease' in fields,
            "'CGC_Other Syndrome/Disease' was not present in the header")
        self.assertTrue(
            'CGC_Abridged_Mutation Type' in fields,
            "'CGC_Abridged_Mutation Type' was not present in the header")

        ctr = 1
        linesThatShouldBeAnnotated = 0
        for lineDict in tsvReader:
            self.assertTrue('gene' in lineDict.keys())
            if lineDict['gene'] in genesAvailable:
                self.assertTrue(
                    lineDict['CGC_Abridged_GeneID'] != '',
                    "'CGC_Abridged_GeneID' was missing on a row that should have been populated.  Line: "
                    + str(ctr))
                linesThatShouldBeAnnotated += 1
            ctr += 1
        self.assertTrue((linesThatShouldBeAnnotated) > 0,
                        "Bad data -- cannot test missed detects.")
Example #57
0
def main(argv=None):  # IGNORE:C0111
    """Command line options."""
    from oncotator.utils.OncotatorCLIUtils import OncotatorCLIUtils
    from oncotator.Annotator import Annotator

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_version = "%s" % __version__
    program_version_message = '%%(prog)s %s' % program_version

    try:
        args = parseOptions(program_version_message)
        verbose = args.verbose
        if verbose > 0:
            print("Verbose mode on")

        logFilename = args.log_name  # 'oncotator.log'

        # Create a basic logger to a file
        loggingFormat = '%(asctime)s %(levelname)s [%(name)s:%(lineno)d] %(message)s'
        logging.basicConfig(filename=logFilename,
                            level=logging.INFO,
                            format=loggingFormat)

        # Add a console logger to the root logger, which means that all loggers generated will have the console dump.
        #    Output on the console will be the same as what is in the log file.
        ch = logging.StreamHandler()
        ch.setLevel(logging.WARN)
        formatter = logging.Formatter(loggingFormat)
        ch.setFormatter(formatter)

        if verbose:
            ch.setLevel(logging.INFO)
            print("Path:")
            print(sys.path)
            print(" ")

        logging.getLogger('').addHandler(ch)

        logger = logging.getLogger(__name__)
        logger.info("Oncotator " + program_version)
        logger.info("Args: " + str(args))
        logger.info('Log file: ' + os.path.abspath(logFilename))

        if DEBUG:
            logger.setLevel(logging.DEBUG)

        if not NGSLIB_INSTALLED:
            logger.warn(
                "ngslib module not installed.  Will be unable to annotate with BigWig datasources."
            )

        # Initiate an Oncotator session.
        inputFilename = os.path.expanduser(args.input_file)
        outputFilename = os.path.expanduser(args.output_file)
        inputFormat = args.input_format.upper()
        outputFormat = args.output_format.upper()

        datasourceDir = os.path.expanduser(args.dbDir)
        cache_url = args.cache_url
        read_only_cache = args.read_only_cache
        tx_mode = args.tx_mode
        is_skip_no_alts = args.skip_no_alt
        genome_build = args.genome_build
        is_no_prepend = not args.prepend

        # Parse annotation overrides
        commandLineManualOverrides = args.override_cli
        overrideConfigFile = args.override_config
        if overrideConfigFile is not None and not os.path.exists(
                overrideConfigFile):
            logger.warn("Could not find " + overrideConfigFile +
                        "   ... proceeding anyway.")
            overrideConfigFile = None
        manualOverrides = OncotatorCLIUtils.determineAllAnnotationValues(
            commandLineManualOverrides, overrideConfigFile)

        # Parse default overrides
        commandLineDefaultValues = args.default_cli
        defaultConfigFile = args.default_config
        if defaultConfigFile is not None and not os.path.exists(
                defaultConfigFile):
            if defaultConfigFile != DEFAULT_DEFAULT_ANNOTATIONS:
                logger.warn("Could not find " + defaultConfigFile +
                            "   ... proceeding anyway.")
            else:
                logger.info(
                    "Could not find Broad-specific " + defaultConfigFile +
                    "   ... proceeding without any default annotations.  __UNKNOWN__ may appear in TCGA MAF outputs."
                )
            defaultConfigFile = None
        defaultValues = OncotatorCLIUtils.determineAllAnnotationValues(
            commandLineDefaultValues, defaultConfigFile)

        # Create a run configuration to pass to the Annotator class.
        annotating_type = None
        if inputFormat == "SEG_FILE":
            annotating_type = RunSpecification.ANNOTATE_SEGMENTS
        runConfig = RunSpecificationFactory.create_run_spec(
            inputFormat,
            outputFormat,
            inputFilename,
            outputFilename,
            globalAnnotations=manualOverrides,
            datasourceDir=datasourceDir,
            isMulticore=(not args.noMulticore),
            defaultAnnotations=defaultValues,
            cacheUrl=cache_url,
            read_only_cache=read_only_cache,
            tx_mode=tx_mode,
            is_skip_no_alts=is_skip_no_alts,
            genomeBuild=genome_build,
            other_opts=determineOtherOptions(args),
            annotating_type=annotating_type)

        annotator = Annotator()
        annotator.initialize(runConfig)
        annotator.annotate()

        return 0
    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
Example #58
0
    def testTCGAMAFRendering(self):
        """
        Tests the ability to render a germline VCF file as a TCGA MAF file.
        """
        inputFilename = os.path.join(*["testdata", "vcf", "example.vcf"])
        outputFilename = os.path.join("out", "example.vcf.maf.annotated")

        creator = VcfInputMutationCreator(inputFilename)
        creator.createMutations()
        renderer = TcgaMafOutputRenderer(outputFilename)
        annotator = Annotator()
        annotator.setInputCreator(creator)
        annotator.setOutputRenderer(renderer)
        annotator.setManualAnnotations(self._createTCGAMAFOverridesForVCF())
        datasources = self._createDatasourceCorpus()
        for ds in datasources:
            annotator.addDatasource(ds)
        filename = annotator.annotate()

        self._validateTcgaMafContents(filename)