Пример #1
0
 def testSampleNameSelectorWithMaf(self):
     input = MafliteInputMutationCreator("testdata/maflite/tiny_maflite.maf.txt")
     first_mut = next(input.createMutations())
     s = SampleNameSelector(first_mut)
     for mut in input.createMutations():
         self.assertEqual("Patient0-Normal-Patient0-Tumor", s.getSampleName(mut))
     self.assertEqual(s.getAnnotationSource(),"OUTPUT")
     self.assertEqual(s.getOutputAnnotationName(), MutUtils.SAMPLE_NAME_ANNOTATION_NAME)
Пример #2
0
 def testSampleNameSelectorWithVCF(self):
     input = VcfInputMutationCreator("testdata/vcf/example.1row.vcf")
     first_mut = next(input.createMutations())
     s = SampleNameSelector(first_mut)
     expected = ["NA 00001", "NA 00002", "NA 00003"]
     for mut in input.createMutations():
         self.assertIn(s.getSampleName(mut), expected)
     self.assertEqual(s.getAnnotationSource(), "INPUT")
     self.assertEquals(s.getOutputAnnotationName(), "sample_name")
Пример #3
0
 def testSampleNameSelectorWithVCF(self):
     input = VcfInputMutationCreator("testdata/vcf/example.1row.vcf")
     first_mut = next(input.createMutations())
     s = SampleNameSelector(first_mut)
     expected = ["NA 00001", "NA 00002", "NA 00003"]
     for mut in input.createMutations():
         self.assertIn(s.getSampleName(mut), expected)
     self.assertEqual(s.getAnnotationSource(), "INPUT")
     self.assertEquals(s.getOutputAnnotationName(), "sample_name")
Пример #4
0
 def testSampleNameSelectorWithMaf(self):
     input = MafliteInputMutationCreator(
         "testdata/maflite/tiny_maflite.maf.txt")
     first_mut = next(input.createMutations())
     s = SampleNameSelector(first_mut)
     for mut in input.createMutations():
         self.assertEqual("Patient0-Normal-Patient0-Tumor",
                          s.getSampleName(mut))
     self.assertEqual(s.getAnnotationSource(), "OUTPUT")
     self.assertEqual(s.getOutputAnnotationName(),
                      MutUtils.SAMPLE_NAME_ANNOTATION_NAME)
Пример #5
0
 def __init__(self, mutations):
     """
     Initialize an new queue with a MutationData iterator
     :param mutations: any MutationData producing Iterator
     """
     self.mutations = more_itertools.peekable(mutations)
     self.sns = SampleNameSelector(self.mutations.peek())
     self.queue = collections.defaultdict(list)
     self.indel_queue = []
     self.last = 0
     self.logger = logging.getLogger(__name__)
     self.warned_about_order = False
Пример #6
0
    def __init__(self, mutations, mutation_data_factory):
        """
        Initialize an new queue with a MutationData iterator

        :param mutations: any MutationData producing Iterator
        :param mutation_data_factory: a MutationDataFactory to be used to produce new mutations for the ONPs
        """
        self.mutations = more_itertools.peekable(mutations)
        self.sns = SampleNameSelector(self.mutations.peek())
        self.queue = collections.defaultdict(list)
        self.indel_queue = []
        self.last = 0
        self.logger = logging.getLogger(__name__)
        self.warned_about_order = False
        self._mutation_data_factory = mutation_data_factory
Пример #7
0
    def _writeMuts2Tsv(self, muts, path):
        """
        Given a mutation generator, this methods writes a tab separated file for all mutations in the mutation
        generator. In addition, this method computes the appropriate sample name in scenarios where the mutation is
        missing sample name annotation. It also computes a list of all chromosomes and sample names contained within
        the generator.

        :param path: temporary filename
        :param muts: generator object with mutations
        """

        sampleNames = set()
        chroms = set()

        writer = None

        # create a temporary file to write tab-separated file
        tempTsvFile = tempfile.NamedTemporaryFile(dir=path, delete=False)
        self.logger.debug("Creating intermediate tsv file at %s" % tempTsvFile.name)

        mutAttributeNames = []
        sampleNameSelector = SampleNameSelector(self.mutation,
                                                configFile=self.configTable.getConfigFilename(),
                                                section="OTHER")

        with open(tempTsvFile.name, 'w') as fptr:
            ctr = 0
            sampleNameAnnotationName = sampleNameSelector.getOutputAnnotationName()
            sampleNameSource = sampleNameSelector.getAnnotationSource()

            for mut in muts:
                if len(mutAttributeNames) == 0:
                    mutAttributeNames = mut.getAttributeNames()

                sampleName = sampleNameSelector.getSampleName(mut)
                if sampleName is not None:
                    if mut.get(sampleNameAnnotationName, None) is None:
                        mut.createAnnotation(sampleNameAnnotationName, sampleName, sampleNameSource)
                    sampleNames.add(sampleName)

                # Parse chromosome
                chroms.add(mut.chr)

                updated_start, updated_ref_allele, updated_alt_allele = MutUtils.retrieveMutCoordinatesForRendering(mut)
                mut.ref_allele = updated_ref_allele
                mut.alt_allele = updated_alt_allele
                mut.start = updated_start

                if ctr == 0:
                    fieldnames2Render = MutUtils.getAllAttributeNames(mut)
                    if sampleNameAnnotationName is not None:
                        fieldnames2Render += [sampleNameAnnotationName]
                    for fieldname in fieldnames2Render:  # fieldnames that start "_" aren't rendered
                        if fieldname.startswith("_"):
                            fieldnames2Render.remove(fieldname)

                    writer = csv.DictWriter(fptr, fieldnames2Render, extrasaction='ignore', delimiter=self.delimiter,
                                            lineterminator=self.lineterminator)
                    writer.writeheader()

                writer.writerow(mut)

                ctr += 1
                if (ctr % 1000) == 0:
                    self.logger.info("Wrote " + str(ctr) + " mutations to tsv.")

        sampleNames = list(sampleNames)
        sampleNames.sort()
        chroms = list(chroms)

        return chroms, sampleNames, tempTsvFile.name
Пример #8
0
class OnpQueue(object):
    """
    Bookkeeping class to maintain the mutations waiting to be combined
    """
    def __init__(self, mutations):
        """
        Initialize an new queue with a MutationData iterator
        :param mutations: any MutationData producing Iterator
        """
        self.mutations = more_itertools.peekable(mutations)
        self.sns = SampleNameSelector(self.mutations.peek())
        self.queue = collections.defaultdict(list)
        self.indel_queue = []
        self.last = 0
        self.logger = logging.getLogger(__name__)
        self.warned_about_order = False

    @staticmethod
    def _create_start_position_dict(mutations):
        """
        Create a start_position -> mutation dict
        :param mutations: a collection of MutationData
        :return: a dictionary containing all the input MutationData grouped by start postion
        """
        assert (mutations is not None)
        starts = collections.defaultdict(list)
        for mut in mutations:
            starts[int(mut.start)] += [mut]
        return starts

    @staticmethod
    def _paths(finished_paths, path_so_far, start, muts):
        """Return all paths from the start position through the mutation graph
        :param finished_paths: completed paths
        :param path_so_far: the accumulated mutation->mutation path so far
        :param start: the start position to travers the muts from
        :param muts: a dictionary in the form {start_position: [Mutation]}
        :return: All paths through adjacent mutations starting with mutations at chromosome position start
        """
        if muts == [] or start not in muts:
            finished_paths.append(path_so_far)
        else:
            # return reduce(operator.concat, lambda mut: OnpCombiner._paths(path + [mut], mut.end+1, muts), [])
            # path =  map(lambda mut: OnpQueue._paths(path + [mut], int(mut.end)+1, muts), muts[start])
            for mut in muts[start]:
                OnpQueue._paths(finished_paths, path_so_far + [mut],
                                int(mut.end) + 1, muts)
            return finished_paths
            #return reduce(operator.concat, path)

    def _add(self, mutation):
        variant_type = TranscriptProviderUtils.infer_variant_type(
            mutation.ref_allele, mutation.alt_allele)
        # only combine ONPs, not indels
        if not TranscriptProviderUtils.is_xnp(variant_type):
            self.indel_queue.append(mutation)
        else:
            self.queue[self.sns.getSampleName(mutation)].append(mutation)

    def _walk_mutation_paths(self, muts):
        """
        Find all paths through adjacent mutations and return those as combined mutations

        Find the first mutations by chromosome position and compute all paths through adjacent mutations reachable from them.
        If there are any nodes that were not reached, choose the first position with unreached nodes and repeat
        :param muts: a list of mutations to walk throught
        :return: a list of new mutations combined from
        """
        unreached = muts
        paths = []
        starts = self._create_start_position_dict(muts)
        while unreached:
            paths += self._paths([], [],
                                 min([int(mut.start) for mut in unreached]),
                                 starts)
            reached = [mut for path in paths for mut in path]
            unreached = [mut for mut in muts if mut not in reached]

        paths = [OnpQueue._combine_mutations(path) for path in paths]
        return paths

    def _dump_all(self):
        results = []
        for (sample, muts) in self.queue.iteritems():
            results += self._walk_mutation_paths(muts)

        #add all stored up indels
        results += self.indel_queue or []
        self.indel_queue = []
        results.sort(key=lambda x: (int(x.start), int(x.end)))

        self.queue.clear()
        return results

    def _get_all_values(self):
        return [j for i in self.queue.values() for j in i]

    def _is_adjacent_to_any_xnp(self, new_mutation):
        return self._is_adjacent(new_mutation, self._get_all_values())

    def _is_adjacent(self, new_mutation, mutations):
        if mutations:
            ends = [int(x.end) for x in mutations]
            return int(new_mutation.start) <= 1 + max(ends)
        else:
            return False

    @staticmethod
    def _combine_mutations(mutations):
        """
        Merge multiple adjacent mutations into a single new mutation.

        :param mutations: an ordered list of MutationData
        :returns a new MutationData

        :warning: _combine_mutations does not make any attempt to sanity check input mutations
        it will happily combine overlapping and non-adjacent mutations on disparate chromosomes
        """
        if len(mutations) == 0:
            return None
        if len(mutations) == 1:
            return mutations[0]

        # special logic for the attributes
        start = min([mut.start for mut in mutations])
        end = max([mut.end for mut in mutations])
        chr = mutations[0].chr
        ref = "".join([mut.ref_allele for mut in mutations])
        alt = "".join([mut.alt_allele for mut in mutations])
        build = "|".join(set([x.build for x in mutations]))

        #create the new mutation
        newmut = MutationData(chr=chr,
                              start=start,
                              end=end,
                              ref_allele=ref,
                              alt_allele=alt,
                              build=build)

        #add annotations to the mutation
        allAnnotations = set(flatmap(lambda x: x.keys(), mutations))
        annotationNames = allAnnotations - set(
            mutations[0].getAttributeNames())
        for annotName in annotationNames:
            annotations = []
            for mut in mutations:
                try:
                    annotations.append(mut.getAnnotation(annotName))
                except KeyError:
                    pass

            values = sorted(
                (set([x.getValue() for x in annotations if x.getValue()])))
            value = "|".join(values)
            tags = sorted(set(flatmap(lambda x: x.getTags(), annotations)))
            source = annotations[0].getDatasource()
            datatype = annotations[0].getDataType()
            number = annotations[0].getNumber()
            description = annotations[0].getDescription()
            newmut.createAnnotation(annotationName=annotName,
                                    annotationValue=value,
                                    annotationSource=source,
                                    annotationDataType=datatype,
                                    annotationDescription=description,
                                    tags=tags,
                                    number=number)
        return newmut

    def _combine_with_indels(self, output):
        """add the indels to output and sort by start position"""
        output += self.indel_queue or []
        self.indel_queue = []
        output.sort(key=lambda x: int(x.start))
        return

    def get_combined_mutations(self):
        """
        :return: a generator yielding mutations, adjacent SNPs,DNPs, and ONPs will be merged together.
        """
        # assumes mutations are sorted by start position and then sample
        #if they're not, it won't find DNPs
        last_chr = -1
        last_start = -1
        for mut in self.mutations:
            output = []
            #if we're on a new chromosome, dump all mutations, then add the new one to the queue
            if mut.chr != last_chr:
                output = self._dump_all()
                self._add(mut)
            #if we're at the same start position, add the new mutation to the queue
            elif mut.start == last_start:
                self._add(mut)
            #if we are at a new position on the same chromosome
            elif self._is_adjacent_to_any_xnp(mut):
                #  if we are adjacent/overlapping to one of our existing positions
                #   add the mutation
                if not self.warned_about_order and int(mut.start) < last_start:
                    self.logger.warn(
                        "Mutations are not sorted by start position, this may cause unexpected behavior or "
                        "increased memory requirements.  It is recommended that your sort any files that you"
                        "using with --infer-onps by position and sample name.")
                    self.warned_about_order = True
                self._add(mut)
            #  if we are not adjacent to any existing queue position,
            #   dump mutations, then add the mutation
            else:
                output = self._dump_all()
                self._add(mut)
            last_chr = mut.chr
            last_start = mut.start

            for mut in output:
                yield mut

        #when we're finished, be sure to dump any last mutations
        output = self._dump_all()
        for mut in output:
            yield mut
Пример #9
0
class OnpQueue(object):
    """
    Bookkeeping class to maintain the mutations waiting to be combined
    """

    def __init__(self, mutations, mutation_data_factory):
        """
        Initialize an new queue with a MutationData iterator

        :param mutations: any MutationData producing Iterator
        :param mutation_data_factory: a MutationDataFactory to be used to produce new mutations for the ONPs
        """
        self.mutations = more_itertools.peekable(mutations)
        self.sns = SampleNameSelector(self.mutations.peek())
        self.queue = collections.defaultdict(list)
        self.indel_queue = []
        self.last = 0
        self.logger = logging.getLogger(__name__)
        self.warned_about_order = False
        self._mutation_data_factory = mutation_data_factory

    @staticmethod
    def _create_start_position_dict(mutations):
        """
        Create a start_position -> mutation dict
        :param mutations: a collection of MutationData
        :return: a dictionary containing all the input MutationData grouped by start postion
        """
        assert mutations is not None
        starts = collections.defaultdict(list)
        for mut in mutations:
            starts[int(mut.start)] += [mut]
        return starts

    @staticmethod
    def _paths(finished_paths, path_so_far, start, muts):
        """Return all paths from the start position through the mutation graph
        :param finished_paths: completed paths
        :param path_so_far: the accumulated mutation->mutation path so far
        :param start: the start position to travers the muts from
        :param muts: a dictionary in the form {start_position: [Mutation]}
        :return: All paths through adjacent mutations starting with mutations at chromosome position start
        """
        if muts == [] or start not in muts:

            # No mutations available to continue this chain
            finished_paths.append(path_so_far)
        else:
            # return reduce(operator.concat, lambda mut: OnpCombiner._paths(path + [mut], mut.end+1, muts), [])
            # path =  map(lambda mut: OnpQueue._paths(path + [mut], int(mut.end)+1, muts), muts[start])
            for mut in muts[start]:
                if len(path_so_far) > 0 and not PhasingUtils.is_in_phase(path_so_far[-1], mut):

                    # Next mutation not in phase, so stop this path here.
                    finished_paths.append(path_so_far)
                else:
                    OnpQueue._paths(finished_paths, path_so_far + [mut], int(mut.end) + 1, muts)
            return finished_paths
            # return reduce(operator.concat, path)

    def _add(self, mutation):
        variant_type = TranscriptProviderUtils.infer_variant_type(mutation.ref_allele, mutation.alt_allele)
        # only combine ONPs, not indels
        if not TranscriptProviderUtils.is_xnp(variant_type):
            self.indel_queue.append(mutation)
        else:
            self.queue[self.sns.getSampleName(mutation)].append(mutation)

    def _walk_mutation_paths(self, muts):
        """
        Find all paths through adjacent mutations and return those as combined mutations

        Find the first mutations by chromosome position and compute all paths through adjacent mutations reachable from them.
        If there are any nodes that were not reached, choose the first position with unreached nodes and repeat
        :param muts: a list of mutations to walk throught
        :return: a list of new mutations combined from
        """
        unreached = muts
        paths = []
        starts = self._create_start_position_dict(muts)
        while unreached:
            paths += self._paths([], [], min([int(mut.start) for mut in unreached]), starts)
            reached = [mut for path in paths for mut in path]
            unreached = [mut for mut in muts if mut not in reached]

        paths = [OnpQueue._combine_mutations(path, self._mutation_data_factory) for path in paths]
        return paths

    def _dump_all(self):
        results = []
        for (sample, muts) in self.queue.iteritems():
            results += self._walk_mutation_paths(muts)

        # add all stored up indels
        results += self.indel_queue or []
        self.indel_queue = []
        results.sort(key=lambda x: (int(x.start), int(x.end)))

        self.queue.clear()
        return results

    def _get_all_values(self):
        return [j for i in self.queue.values() for j in i]

    def _is_adjacent_to_any_xnp(self, new_mutation):
        return self._is_adjacent(new_mutation, self._get_all_values())

    def _is_adjacent(self, new_mutation, mutations):
        if mutations:
            ends = [int(x.end) for x in mutations]
            return int(new_mutation.start) <= 1 + max(ends)
        else:
            return False

    @staticmethod
    def _combine_mutations(mutations, mutation_data_factory):
        """
        Merge multiple adjacent mutations into a single new mutation.

        :param mutations: an ordered list of MutationData
        :returns a new MutationData

        :warning: _combine_mutations does not make any attempt to sanity check input mutations
        it will happily combine overlapping and non-adjacent mutations on disparate chromosomes
        """
        if len(mutations) == 0:
            return None
        if len(mutations) == 1:
            return mutations[0]

        # special logic for the attributes
        start = min([mut.start for mut in mutations])
        end = max([mut.end for mut in mutations])
        chr = mutations[0].chr
        ref = "".join([mut.ref_allele for mut in mutations])
        alt = "".join([mut.alt_allele for mut in mutations])
        build = "|".join(set([x.build for x in mutations]))

        # create the new mutation
        newmut = mutation_data_factory.create(
            chr=chr, start=start, end=end, ref_allele=ref, alt_allele=alt, build=build
        )

        # add annotations to the mutation
        allAnnotations = set(flatmap(lambda x: x.keys(), mutations))
        annotationNames = allAnnotations - set(mutations[0].getAttributeNames())
        for annotName in annotationNames:
            annotations = []
            for mut in mutations:
                try:
                    annotations.append(mut.getAnnotation(annotName))
                except KeyError:
                    pass

            values = [x.getValue() for x in annotations]
            if len(set(values)) == 1:
                value = values[0]  # if all annotations are identical then don't pipe separate them
            else:
                value = "|".join(values)

            tags = sorted(set(flatmap(lambda x: x.getTags(), annotations)))
            source = annotations[0].getDatasource()
            datatype = annotations[0].getDataType()
            number = annotations[0].getNumber()
            description = annotations[0].getDescription()
            newmut.createAnnotation(
                annotationName=annotName,
                annotationValue=value,
                annotationSource=source,
                annotationDataType=datatype,
                annotationDescription=description,
                tags=tags,
                number=number,
            )
        return newmut

    def _combine_with_indels(self, output):
        """add the indels to output and sort by start position"""
        output += self.indel_queue or []
        self.indel_queue = []
        output.sort(key=lambda x: int(x.start))
        return

    def get_combined_mutations(self):
        """
        :return: a generator yielding mutations, adjacent SNPs,DNPs, and ONPs will be merged together.
        """
        # assumes mutations are sorted by start position and then sample
        # if they're not, it won't find DNPs
        last_chr = -1
        last_start = -1
        for mut in self.mutations:
            output = []
            # if we're on a new chromosome, dump all mutations, then add the new one to the queue
            if mut.chr != last_chr:
                output = self._dump_all()
                self._add(mut)
            # if we're at the same start position, add the new mutation to the queue
            elif mut.start == last_start:
                self._add(mut)
            # if we are at a new position on the same chromosome
            elif self._is_adjacent_to_any_xnp(mut):
                #  if we are adjacent/overlapping to one of our existing positions
                #   add the mutation
                if not self.warned_about_order and int(mut.start) < last_start:
                    self.logger.warn(
                        "Mutations are not sorted by start position, this may cause unexpected behavior or "
                        "increased memory requirements.  It is recommended that your sort any files that you"
                        "using with --infer-onps by position and sample name."
                    )
                    self.warned_about_order = True
                self._add(mut)
            #  if we are not adjacent to any existing queue position,
            #   dump mutations, then add the mutation
            else:
                output = self._dump_all()
                self._add(mut)
            last_chr = mut.chr
            last_start = mut.start

            for mut in output:
                yield mut

        # when we're finished, be sure to dump any last mutations
        output = self._dump_all()
        for mut in output:
            yield mut