예제 #1
0
    def _loadIpdTable(self, nullModelGroup):
        """
        Read the null kinetic model into a shared numpy array dataset
        """
        nullModelDataset = nullModelGroup["KineticValues"]

        # assert that the dataset is a uint8
        assert(nullModelDataset.dtype == uint8)

        # Construct a 'shared array' (a numpy wrapper around some shared memory
        # Read the LUT into this table
        self.sharedArray = SharedArray('B', nullModelDataset.shape[0])
        lutArray = self.sharedArray.getNumpyWrapper()
        nullModelDataset.read_direct(lutArray)

        # Load the second-level LUT
        self.floatLut = nullModelGroup["Lut"][:]
예제 #2
0
class IpdModel:

    """
    Predicts the IPD of an any context, possibly containing multiple modifications.
    We use a 4^12 entry LUT to get the predictions for contexts without modifications,
    then we use the GbmModel to get predictions in the presence of arbitrary mods.
        Note on the coding scheme.  For each contig we store a byte-array that has size = contig.length + 2*self.pad
        The upper 4 bits contain a lookup into seqReverseMap, which can contains N's. This is used for giving
        template snippets that may contains N's if the reference sequence does, or if the snippet
        The lowe 4 bits contain a lookup into lutReverseMap, which
    """

    def __init__(self, fastaRecords, modelFile=None, modelIterations=-1):
        """
        Load the reference sequences and the ipd lut into shared arrays that can be
        used as numpy arrays in worker processes.
        fastaRecords is a list of FastaRecords, in the cmp.h5 file order
        """

        self.pre = 10
        self.post = 4

        self.pad = 30
        self.base4 = 4 ** np.array(range(self.pre + self.post + 1))

        self.refDict = {}
        self.refLengthDict = {}

        for contig in fastaRecords:
            if contig.id is None:
                # This contig has no mapped reads -- skip it
                continue

            rawSeq = contig.sequence
            refSeq = np.fromstring(rawSeq, dtype=byte)

            # Store the reference length
            self.refLengthDict[contig.id] = len(rawSeq)

            # Make a shared array
            sa = SharedArray(dtype='B', shape=len(rawSeq) + self.pad * 2)
            saWrap = sa.getNumpyWrapper()

            # Lut Codes convert Ns to As so that we don't put Ns into the Gbm Model
            # Seq Codes leaves Ns as Ns for getting reference snippets out
            innerLutCodes = lutCodeMap[refSeq]
            innerSeqCodes = seqCodeMap[refSeq]
            innerCodes = np.bitwise_or(innerLutCodes, np.left_shift(innerSeqCodes, 4))

            saWrap[self.pad:(len(rawSeq) + self.pad)] = innerCodes

            # Padding codes -- the lut array is padded with 0s the sequence array is padded with N's (4)
            outerCodes = np.left_shift(np.ones(self.pad, dtype=uint8) * 4, 4)
            saWrap[0:self.pad] = outerCodes
            saWrap[(len(rawSeq) + self.pad):(len(rawSeq) + 2 * self.pad)] = outerCodes

            self.refDict[contig.id] = sa

        # No correction factor for IPDs everything is normalized to 1
        self.meanIpd = 1

        # Find and open the ipd model file

        if modelFile:
            self.lutPath = modelFile
        else:
            self.lutPath = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + "kineticLut.h5"

        if os.path.exists(self.lutPath):
            h5File = h5py.File(self.lutPath, mode='r')

            gbmModelGroup = h5File["/AllMods_GbmModel"]
            self.gbmModel = GbmContextModel(gbmModelGroup, modelIterations)

            # We always use the model -- no more LUTS
            self.predictIpdFunc = self.predictIpdFuncModel
            self.predictManyIpdFunc = self.predictManyIpdFuncModel
        else:
            logging.info("Couldn't find model file: %s" % self.lutPath)

    def _loadIpdTable(self, nullModelGroup):
        """
        Read the null kinetic model into a shared numpy array dataset
        """
        nullModelDataset = nullModelGroup["KineticValues"]

        # assert that the dataset is a uint8
        assert(nullModelDataset.dtype == uint8)

        # Construct a 'shared array' (a numpy wrapper around some shared memory
        # Read the LUT into this table
        self.sharedArray = SharedArray('B', nullModelDataset.shape[0])
        lutArray = self.sharedArray.getNumpyWrapper()
        nullModelDataset.read_direct(lutArray)

        # Load the second-level LUT
        self.floatLut = nullModelGroup["Lut"][:]

    def refLength(self, refId):
        return self.refLengthDict[refId]

    def cognateBaseFunc(self, refId):
        """
        Return a function that returns a snippet of the reference sequence around a given position
        """

        # FIXME -- what is the correct strand to return?!
        # FIXME -- what to do about padding when the snippet runs off the end of the reference
        # how do we account for / indicate what is happening
        refArray = self.refDict[refId].getNumpyWrapper()

        def f(tplPos, tplStrand):

            # skip over the padding
            tplPos += self.pad

            # Forward strand
            if tplStrand == 0:
                slc = refArray[tplPos]
                slc = np.right_shift(slc, 4)
                return seqMap[slc]

            # Reverse strand
            else:
                slc = refArray[tplPos]
                slc = np.right_shift(slc, 4)
                return seqMapComplement[slc]

        return f

    def snippetFunc(self, refId, pre, post):
        """
        Return a function that returns a snippet of the reference sequence around a given position
        """

        refArray = self.refDict[refId].getNumpyWrapper()

        def f(tplPos, tplStrand):
            """Closure for returning a reference snippet. The reference is padded with N's for bases falling outside the extents of the reference"""
            # skip over the padding
            tplPos += self.pad

            # Forward strand
            if tplStrand == 0:
                slc = refArray[(tplPos - pre):(tplPos + 1 + post)]
                slc = np.right_shift(slc, 4)
                return seqMapNp[slc].tostring()

            # Reverse strand
            else:
                slc = refArray[(tplPos + pre):(tplPos - post - 1):-1]
                slc = np.right_shift(slc, 4)
                return seqMapComplementNp[slc].tostring()

        return f

    def getReferenceWindow(self, refId, tplStrand, start, end):
        """
        Return  a snippet of the reference sequence
        """

        refArray = self.refDict[refId].getNumpyWrapper()

        # adjust position for reference padding
        start += self.pad
        end += self.pad

        # Forward strand
        if tplStrand == 0:
            slc = refArray[start:end]
            slc = np.right_shift(slc, 4)
            return "".join(seqMap[x] for x in slc)

        # Reverse strand
        else:
            slc = refArray[end:start:-1]
            slc = np.right_shift(slc, 4)
            return "".join(seqMapComplement[x] for x in slc)

    def predictIpdFuncLut(self, refId):
        """
        Each (pre+post+1) base context gets mapped to an integer
        by converting each nucleotide to a base-4 number A=0, C=1, etc,
        and treating the 'pre' end of the context of the least significant
        digit.  This code is used to lookup the expected IPD in a
        pre-computed table.  Contexts near the ends of the reference
        are coded by padding the context with 0
        """

        # Materialized the numpy wrapper around the shared data
        refArray = self.refDict[refId].getNumpyWrapper()
        lutArray = self.sharedArray.getNumpyWrapper()
        floatLut = self.floatLut

        def f(tplPos, tplStrand):

            # skip over the padding
            tplPos += self.pad

            # Forward strand
            if tplStrand == 0:
                slc = np.bitwise_and(refArray[(tplPos + self.pre):(tplPos - self.post - 1):-1], 0xf)

            # Reverse strand
            else:
                slc = 3 - np.bitwise_and(refArray[(tplPos - self.pre):(tplPos + 1 + self.post)], 0xf)

            code = (self.base4 * slc).sum()
            return floatLut[max(1, lutArray[code])]

        return f

    def predictIpdFuncModel(self, refId):
        """
        Each (pre+post+1) base context gets mapped to an integer
        by converting each nucleotide to a base-4 number A=0, C=1, etc,
        and treating the 'pre' end of the context of the least significant
        digit.  This code is used to lookup the expected IPD in a
        pre-computed table.  Contexts near the ends of the reference
        are coded by padding the context with 0
        """

        # Materialized the numpy wrapper around the shared data
        snipFunction = self.snippetFunc(refId, self.post, self.pre)

        def f(tplPos, tplStrand):
            # Get context string
            context = snipFunction(tplPos, tplStrand)

            # Get prediction
            return self.gbmModel.getPredictions([context])[0]

        return f

    def predictManyIpdFuncModel(self, refId):
        """
        Each (pre+post+1) base context gets mapped to an integer
        by converting each nucleotide to a base-4 number A=0, C=1, etc,
        and treating the 'pre' end of the context of the least significant
        digit.  This code is used to lookup the expected IPD in a
        pre-computed table.  Contexts near the ends of the reference
        are coded by padding the context with 0
        """

        # Materialized the numpy wrapper around the shared data
        snipFunction = self.snippetFunc(refId, self.post, self.pre)

        def fMany(sites):
            contexts = [snipFunction(x[0], x[1]) for x in sites]
            return self.gbmModel.getPredictions(contexts)

        return fMany

    def modPredictIpdFunc(self, refId, mod):
        """
        Each (pre+post+1) base context gets mapped to an integer
        by converting each nucleotide to a base-4 number A=0, C=1, etc,
        and treating the 'pre' end of the context of the least significant
        digit.  This code is used to lookup the expected IPD in a
        pre-computed table.  Contexts near the ends of the reference
        are coded by padding the context with 0
        """

        refArray = self.refDict[refId].getNumpyWrapper()

        def f(tplPos, relativeModPos, readStrand):

            # skip over the padding
            tplPos += self.pad

            # Read sequence matches forward strand
            if readStrand == 0:
                slc = 3 - np.bitwise_and(refArray[(tplPos - self.pre):(tplPos + 1 + self.post)], 0xf)

            # Reverse strand
            else:
                slc = np.bitwise_and(refArray[(tplPos + self.pre):(tplPos - self.post - 1):-1], 0xf)

            # Modify the indicated position
            slc[relativeModPos + self.pre] = baseToCode[mod]

            slcString = "".join([codeToBase[x] for x in slc])

            # Get the prediction for this context
            # return self.gbmModel.getPredictions([slcString])[0]
            return 0.0

        return f
예제 #3
0
    def __init__(self, fastaRecords, modelFile=None, modelIterations=-1):
        """
        Load the reference sequences and the ipd lut into shared arrays that can be
        used as numpy arrays in worker processes.
        fastaRecords is a list of FastaRecords, in the cmp.h5 file order
        """

        self.pre = 10
        self.post = 4

        self.pad = 30
        self.base4 = 4 ** np.array(range(self.pre + self.post + 1))

        self.refDict = {}
        self.refLengthDict = {}

        for contig in fastaRecords:
            if contig.id is None:
                # This contig has no mapped reads -- skip it
                continue

            rawSeq = contig.sequence
            refSeq = np.fromstring(rawSeq, dtype=byte)

            # Store the reference length
            self.refLengthDict[contig.id] = len(rawSeq)

            # Make a shared array
            sa = SharedArray(dtype='B', shape=len(rawSeq) + self.pad * 2)
            saWrap = sa.getNumpyWrapper()

            # Lut Codes convert Ns to As so that we don't put Ns into the Gbm Model
            # Seq Codes leaves Ns as Ns for getting reference snippets out
            innerLutCodes = lutCodeMap[refSeq]
            innerSeqCodes = seqCodeMap[refSeq]
            innerCodes = np.bitwise_or(innerLutCodes, np.left_shift(innerSeqCodes, 4))

            saWrap[self.pad:(len(rawSeq) + self.pad)] = innerCodes

            # Padding codes -- the lut array is padded with 0s the sequence array is padded with N's (4)
            outerCodes = np.left_shift(np.ones(self.pad, dtype=uint8) * 4, 4)
            saWrap[0:self.pad] = outerCodes
            saWrap[(len(rawSeq) + self.pad):(len(rawSeq) + 2 * self.pad)] = outerCodes

            self.refDict[contig.id] = sa

        # No correction factor for IPDs everything is normalized to 1
        self.meanIpd = 1

        # Find and open the ipd model file

        if modelFile:
            self.lutPath = modelFile
        else:
            self.lutPath = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + "kineticLut.h5"

        if os.path.exists(self.lutPath):
            h5File = h5py.File(self.lutPath, mode='r')

            gbmModelGroup = h5File["/AllMods_GbmModel"]
            self.gbmModel = GbmContextModel(gbmModelGroup, modelIterations)

            # We always use the model -- no more LUTS
            self.predictIpdFunc = self.predictIpdFuncModel
            self.predictManyIpdFunc = self.predictManyIpdFuncModel
        else:
            logging.info("Couldn't find model file: %s" % self.lutPath)
예제 #4
0
    def __init__(self, fastaRecords, modelFile, modelIterations=-1):
        """
        Load the reference sequences and the ipd lut into shared arrays that can be
        used as numpy arrays in worker processes.
        fastaRecords is a list of FastaRecords, in the alignments file order
        """

        self.pre = 10
        self.post = 4

        self.pad = 30
        self.base4 = 4**np.array(range(self.pre + self.post + 1))

        self.refDict = {}
        self.refLengthDict = {}

        for contig in fastaRecords:
            if contig.alignmentID is None:
                # This contig has no mapped reads -- skip it
                continue

            rawSeq = contig.sequence[:]
            refSeq = np.frombuffer(rawSeq.encode("utf-8"), dtype=byte)

            # Store the reference length
            self.refLengthDict[contig.alignmentID] = len(rawSeq)

            # Make a shared array
            sa = SharedArray(dtype='B', shape=len(rawSeq) + self.pad * 2)
            saWrap = sa.getNumpyWrapper()

            # Lut Codes convert Ns to As so that we don't put Ns into the Gbm Model
            # Seq Codes leaves Ns as Ns for getting reference snippets out
            innerLutCodes = lutCodeMap[refSeq]
            innerSeqCodes = seqCodeMap[refSeq]
            innerCodes = np.bitwise_or(innerLutCodes,
                                       np.left_shift(innerSeqCodes, 4))

            saWrap[self.pad:(len(rawSeq) + self.pad)] = innerCodes

            # Padding codes -- the lut array is padded with 0s the sequence
            # array is padded with N's (4)
            outerCodes = np.left_shift(np.ones(self.pad, dtype=uint8) * 4, 4)
            saWrap[0:self.pad] = outerCodes
            saWrap[(len(rawSeq) + self.pad):(len(rawSeq) +
                                             2 * self.pad)] = outerCodes

            self.refDict[contig.alignmentID] = sa

        # No correction factor for IPDs everything is normalized to 1
        self.meanIpd = 1

        # Find and open the ipd model file
        self.lutPath = modelFile
        if os.path.exists(self.lutPath):
            with gzip.open(self.lutPath, "rb") as npz_in:
                gbmModelData = np.load(npz_in, allow_pickle=True)
                self.gbmModel = GbmContextModel(gbmModelData, modelIterations)

            # We always use the model -- no more LUTS
            self.predictIpdFunc = self.predictIpdFuncModel
            self.predictManyIpdFunc = self.predictManyIpdFuncModel
        else:
            logging.info("Couldn't find model file: %s" % self.lutPath)