Beispiel #1
0
def histogram(direction, var, bins, nsector, normed=False, blowto=False):
    if len(var) != len(direction):
        raise ValueError("var and direction must have same length")

    angle = 360.0 / nsector

    dir_bins = np.arange(-angle / 2, 360.0 + angle, angle, dtype=np.float)
    dir_edges = dir_bins.tolist()
    dir_edges.pop(-1)
    dir_edges[0] = dir_edges.pop(-1)
    dir_bins[0] = 0.0

    var_bins = bins.tolist()
    var_bins.append(np.inf)

    if blowto:
        direction = direction + 180.0
        direction[direction >= 360.0] = direction[direction >= 360.0] - 360

    table = histogram2d(x=var,
                        y=direction,
                        bins=[var_bins, dir_bins],
                        normed=False)[0]
    # add the last value to the first to have the table of North winds
    table[:, 0] = table[:, 0] + table[:, -1]
    # and remove the last col
    table = table[:, :-1]
    if normed:
        table = table * 100 / table.sum()

    return dir_edges, var_bins, table
Beispiel #2
0
def histogram(direction, var, bins, nsector, normed=False, blowto=False):
    """
    Returns an array where, for each sector of wind
    (centred on the north), we have the number of time the wind comes with a
    particular var (speed, polluant concentration, ...).

    Parameters
    ----------
    direction : 1D array
        directions the wind blows from, North centred
    var : 1D array
        values of the variable to compute. Typically the wind speeds
    bins : list
        list of var category against we're going to compute the table
    nsector : integer
        number of sectors

    Other Parameters
    ----------------
    normed : boolean, default False
        The resulting table is normed in percent or not.
    blowto : boolean, default False
        Normaly a windrose is computed with directions as wind blows from. If
        true, the table will be reversed (usefull for pollutantrose)
    """

    if len(var) != len(direction):
        raise (ValueError("var and direction must have same length"))

    angle = 360. / nsector

    dir_bins = np.arange(-angle / 2, 360. + angle, angle, dtype=np.float)
    dir_edges = dir_bins.tolist()
    dir_edges.pop(-1)
    dir_edges[0] = dir_edges.pop(-1)
    dir_bins[0] = 0.

    var_bins = bins.tolist()
    var_bins.append(np.inf)

    if blowto:
        direction = direction + 180.
        direction[direction >= 360.] = direction[direction >= 360.] - 360

    table = histogram2d(x=var,
                        y=direction,
                        bins=[var_bins, dir_bins],
                        normed=False)[0]
    # add the last value to the first to have the table of North winds
    table[:, 0] = table[:, 0] + table[:, -1]
    # and remove the last col
    table = table[:, :-1]
    if normed:
        table = table * 100 / table.sum()

    return dir_edges, var_bins, table
Beispiel #3
0
def histogram(dir, var, bins, nsector, normed=False, blowto=False):
    """
    Returns an array where, for each sector of wind
    (centred on the north), we have the number of time the wind comes with a
    particular var (speed, polluant concentration, ...).

      * dir : 1D array - directions the wind blows from, North centred
      * var : 1D array - values of the variable to compute. Typically the wind
        speeds
      * bins : list - list of var category against we're going to compute the table
      * nsector : integer - number of sectors
      * normed : boolean - The resulting table is normed in percent or not.
      * blowto : boolean - Normaly a windrose is computed with directions
        as wind blows from. If true, the table will be reversed (usefull for
        pollutantrose)

    """

    if len(var) != len(dir):
        raise ValueError("var and dir must have same length")

    angle = old_div(360., nsector)

    dir_bins = np.arange(old_div(-angle, 2),
                         360. + angle,
                         angle,
                         dtype=np.float)
    dir_edges = dir_bins.tolist()
    dir_edges.pop(-1)
    dir_edges[0] = dir_edges.pop(-1)
    dir_bins[0] = 0.

    var_bins = bins.tolist()
    var_bins.append(np.inf)

    if blowto:
        dir = dir + 180.
        dir[dir >= 360.] = dir[dir >= 360.] - 360

    table = histogram2d(x=var, y=dir, bins=[var_bins, dir_bins],
                        normed=False)[0]
    # add the last value to the first to have the table of North winds
    table[:, 0] = table[:, 0] + table[:, -1]
    # and remove the last col
    table = table[:, :-1]
    # count the number of calms

    numcalms = np.average(table[0, :])
    table[0, :] = numcalms
    if normed:
        table = table * 100 / table.sum()

    return dir_edges, var_bins, table
Beispiel #4
0
def histogram(direction, var, bins, nsector, normed=False,
              blowto=False):
    """
    Returns an array where, for each sector of wind
    (centred on the north), we have the number of time the wind comes with a
    particular var (speed, polluant concentration, ...).
    * direction : 1D array - directions the wind blows from, North centred
    * var : 1D array - values of the variable to compute. Typically the wind
        speeds
    * bins : list - list of var category against we're going to compute the table
    * nsector : integer - number of sectors
    * normed : boolean - The resulting table is normed in percent or not.
    * blowto : boolean - Normaly a windrose is computed with directions
    as wind blows from. If true, the table will be reversed (usefull for
    pollutantrose)

    """

    if len(var) != len(direction):
        raise (ValueError("var and direction must have same length"))

    angle = 360. / nsector

    dir_bins = np.arange(-angle / 2, 360. + angle, angle,
                         dtype=np.float)
    dir_edges = dir_bins.tolist()
    dir_edges.pop(-1)
    dir_edges[0] = dir_edges.pop(-1)
    dir_bins[0] = 0.

    var_bins = bins.tolist()
    # print var_bins
    # var_bins.append(np.inf)

    if blowto:
        direction = direction + 180.
        direction[direction >= 360.] = direction[
                                           direction >= 360.] - 360

    table = histogram2d(x=var, y=direction, bins=[var_bins, dir_bins],
                        normed=False)[0]
    # add the last value to the first to have the table of North winds
    table[:, 0] = table[:, 0] + table[:, -1]
    # and remove the last col
    table = table[:, :-1]
    if normed:
        table = table * 100 / table.sum()

    return dir_edges, var_bins, table
Beispiel #5
0
def main():
    cvs_file_abs_name_gz = os.path.join(cingDirData, 'PluginCode', 'Whatif',
                                        cvs_file_abs_name + '.gz')
    gunzip(cvs_file_abs_name_gz)
    reader = csv.reader(open(cvs_file_abs_name, "rb"), quoting=csv.QUOTE_NONE)
    valuesBySsAndResType = {}
    histJaninBySsAndResType = {}
    histJaninBySsAndCombinedResType = {}
    #    histByCombinedSsAndResType = {}
    histJaninCtupleBySsAndResType = {}
    valuesByEntrySsAndResType = {}
    hrange = (xRange, yRange)

    #    rowCount = 0
    for row in reader:
        #        rowCount += 1
        #        7a3h,A,VAL ,   5,H, -62.8, -52.8
        #        7a3h,A,VAL ,   6,H, -71.2, -33.6
        #        7a3h,A,GLU ,   7,H, -63.5, -41.6
        (entryId, _chainId, resType, _resNum, ssType, chi1, chi2,
         _max_bfactor) = row
        ssType = to3StateDssp(ssType)[0]
        resType = resType.strip()
        chi1 = chi1.strip()
        chi2 = chi2.strip()
        chi1 = floatParse(chi1)
        chi2 = floatParse(chi2)
        if isNaN(chi1) or isNaN(chi2):
            continue
        if not inRange(chi1):
            nTerror("chi1 not in range for row: %s" % repr(row))
            return
        if not inRange(chi2):
            nTerror("chi2 not in range for row: %s" % repr(row))
            return
        if not common20AADict.has_key(resType):
            nTdebug("Residue not in common 20 for row: %s" % repr(row))
            #            rowCount -= 1
            continue

        appendDeepByKeys(valuesBySsAndResType, chi1, ssType, resType, 'chi1')
        appendDeepByKeys(valuesByEntrySsAndResType, chi1, entryId, ssType,
                         resType, 'chi1')
        appendDeepByKeys(valuesBySsAndResType, chi2, ssType, resType, 'chi2')
        appendDeepByKeys(valuesByEntrySsAndResType, chi2, entryId, ssType,
                         resType, 'chi2')
#        nTdebug('resType,ssType,chi1: %4s %1s %s' % (resType,ssType,floatFormat(chi1, "%6.1f")))
#        nTdebug('resType,ssType,chi2: %4s %1s %s' % (resType,ssType,floatFormat(chi2, "%6.1f")))
    del (reader)  # closes the file handles
    os.unlink(cvs_file_abs_name)

    for ssType in valuesBySsAndResType.keys():
        for resType in valuesBySsAndResType[ssType].keys():
            chi1 = valuesBySsAndResType[ssType][resType]['chi1']
            chi2 = valuesBySsAndResType[ssType][resType]['chi2']
            if chi1 and chi2:
                hist2d, _xedges, _yedges = histogram2d(chi2,
                                                       chi1,
                                                       bins=binCount,
                                                       range=hrange)
                setDeepByKeys(histJaninBySsAndResType, hist2d, ssType, resType)
                cTuple = getEnsembleAverageAndSigmaHis(hist2d)
                (c_av, c_sd, hisMin, hisMax) = cTuple
                cTuple += tuple([str([ssType, resType])
                                 ])  # append the hash keys as a way of id.
                nTdebug(
                    "For ssType %s residue type %s found (av/sd/min/max) %8.0f %8.0f %8.0f %8.0f"
                    % (ssType, resType, c_av, c_sd, hisMin, hisMax))
                if c_sd == None:
                    nTdebug(
                        'Failed to get c_sd when testing not all residues are present in smaller sets.'
                    )
                    continue
                if c_sd == 0.:
                    nTdebug(
                        'Got zero c_sd, ignoring histogram. This should only occur in smaller sets. Not setting values.'
                    )
                    continue
                setDeepByKeys(histJaninCtupleBySsAndResType, cTuple, ssType,
                              resType)

    for ssType in valuesBySsAndResType.keys():
        chi1 = []
        chi2 = []
        for resType in valuesBySsAndResType[ssType].keys():
            if resType == 'PRO' or resType == 'GLY':
                continue
            chi1 += valuesBySsAndResType[ssType][resType]['chi1']
            chi2 += valuesBySsAndResType[ssType][resType]['chi2']
        if chi1 and chi2:
            hist2d, _xedges, _yedges = histogram2d(
                chi2,  # Note that the x is the chi2 for some stupid reason,
                chi1,  # otherwise the imagery but also the [row][column] notation is screwed.
                bins=binCount,
                range=hrange)
            #        hist2d = zscaleHist( hist2d, Cav, Csd )
            setDeepByKeys(histJaninBySsAndCombinedResType, hist2d, ssType)

    # Throws a verbose error message on python 2.6.3 as per issue http://code.google.com/p/cing/issues/detail?id=211
    # Using Pickle instead


#    dbase = shelve.open( dbase_file_abs_name )
#    dbase.close()

    if os.path.exists(dbase_file_abs_name):
        os.unlink(dbase_file_abs_name)
    output = open(dbase_file_abs_name, 'wb')
    dbase = {}
    dbase['histJaninBySsAndCombinedResType'] = histJaninBySsAndCombinedResType
    dbase['histJaninBySsAndResType'] = histJaninBySsAndResType
    dbase['histJaninCtupleBySsAndResType'] = histJaninCtupleBySsAndResType
    #    histJaninCtupleBySsAndResType
    cPickle.dump(dbase, output, 2)
    output.close()
Beispiel #6
0
def main():
    cvs_file_abs_name_gz = os.path.join(cingDirData, 'PluginCode', 'Whatif', cvs_file_abs_name + '.gz')
    gunzip(cvs_file_abs_name_gz)
    reader = csv.reader(open(cvs_file_abs_name, "rb"), quoting=csv.QUOTE_NONE)
    valuesBySsAndResType = {}
    histJaninBySsAndResType = {}
    histJaninBySsAndCombinedResType = {}
#    histByCombinedSsAndResType = {}
    histJaninCtupleBySsAndResType = {}
    valuesByEntrySsAndResType = {}
    hrange = (xRange, yRange)

#    rowCount = 0
    for row in reader:
#        rowCount += 1
#        7a3h,A,VAL ,   5,H, -62.8, -52.8
#        7a3h,A,VAL ,   6,H, -71.2, -33.6
#        7a3h,A,GLU ,   7,H, -63.5, -41.6
        (entryId, _chainId, resType, _resNum, ssType, chi1, chi2, _max_bfactor) = row
        ssType = to3StateDssp(ssType)[0]
        resType = resType.strip()
        chi1 = chi1.strip()
        chi2 = chi2.strip()
        chi1 = floatParse(chi1)
        chi2 = floatParse(chi2)
        if isNaN(chi1) or isNaN(chi2):
            continue
        if not inRange(chi1):
            nTerror("chi1 not in range for row: %s" % repr(row))
            return
        if not inRange(chi2):
            nTerror("chi2 not in range for row: %s" % repr(row))
            return
        if not common20AADict.has_key(resType):
            nTdebug("Residue not in common 20 for row: %s" % repr(row))
#            rowCount -= 1
            continue

        appendDeepByKeys(valuesBySsAndResType, chi1, ssType, resType, 'chi1')
        appendDeepByKeys(valuesByEntrySsAndResType, chi1, entryId, ssType, resType, 'chi1')
        appendDeepByKeys(valuesBySsAndResType, chi2, ssType, resType, 'chi2')
        appendDeepByKeys(valuesByEntrySsAndResType, chi2, entryId, ssType, resType, 'chi2')
#        nTdebug('resType,ssType,chi1: %4s %1s %s' % (resType,ssType,floatFormat(chi1, "%6.1f")))
#        nTdebug('resType,ssType,chi2: %4s %1s %s' % (resType,ssType,floatFormat(chi2, "%6.1f")))
    del(reader) # closes the file handles
    os.unlink(cvs_file_abs_name)

    for ssType in valuesBySsAndResType.keys():
        for resType in valuesBySsAndResType[ssType].keys():
            chi1 = valuesBySsAndResType[ssType][resType]['chi1']
            chi2 = valuesBySsAndResType[ssType][resType]['chi2']
            if chi1 and chi2:
                hist2d, _xedges, _yedges = histogram2d(
                    chi2, chi1,
                    bins=binCount,
                    range=hrange)
                setDeepByKeys(histJaninBySsAndResType, hist2d, ssType, resType)
                cTuple = getEnsembleAverageAndSigmaHis(hist2d)
                (c_av, c_sd, hisMin, hisMax) = cTuple
                cTuple += tuple([str([ssType, resType])]) # append the hash keys as a way of id.
                nTdebug("For ssType %s residue type %s found (av/sd/min/max) %8.0f %8.0f %8.0f %8.0f" % (
                    ssType, resType, c_av, c_sd, hisMin, hisMax))
                if c_sd == None:
                    nTdebug('Failed to get c_sd when testing not all residues are present in smaller sets.')
                    continue
                if c_sd == 0.:
                    nTdebug('Got zero c_sd, ignoring histogram. This should only occur in smaller sets. Not setting values.')
                    continue
                setDeepByKeys(histJaninCtupleBySsAndResType, cTuple, ssType, resType)

    for ssType in valuesBySsAndResType.keys():
        chi1 = []
        chi2 = []
        for resType in valuesBySsAndResType[ssType].keys():
            if resType == 'PRO' or resType == 'GLY':
                continue
            chi1 += valuesBySsAndResType[ssType][resType]['chi1']
            chi2 += valuesBySsAndResType[ssType][resType]['chi2']
        if chi1 and chi2:
            hist2d, _xedges, _yedges = histogram2d(
                chi2, # Note that the x is the chi2 for some stupid reason,
                chi1, # otherwise the imagery but also the [row][column] notation is screwed.
                bins=binCount,
                range=hrange)
    #        hist2d = zscaleHist( hist2d, Cav, Csd )
            setDeepByKeys(histJaninBySsAndCombinedResType, hist2d, ssType)

    # Throws a verbose error message on python 2.6.3 as per issue https://github.com/VuisterLab/cing/issues/211
    # Using Pickle instead
#    dbase = shelve.open( dbase_file_abs_name )
#    dbase.close()

    if os.path.exists(dbase_file_abs_name):
        os.unlink(dbase_file_abs_name)
    output = open(dbase_file_abs_name, 'wb')
    dbase = {}
    dbase[ 'histJaninBySsAndCombinedResType' ] = histJaninBySsAndCombinedResType
    dbase[ 'histJaninBySsAndResType' ] = histJaninBySsAndResType
    dbase[ 'histJaninCtupleBySsAndResType' ] = histJaninCtupleBySsAndResType
#    histJaninCtupleBySsAndResType
    cPickle.dump(dbase, output, 2)
    output.close()
Beispiel #7
0
def main():
    cvs_file_abs_name_gz = cvs_file_abs_name + '.gz'
    gunzip(cvs_file_abs_name_gz)
    reader = csv.reader(open(cvs_file_abs_name, "rb"), quoting=csv.QUOTE_NONE)
    valuesBySsAndResType = {}
    histRamaBySsAndResType = {}
    histRamaBySsAndCombinedResType = {}
    #    histByCombinedSsAndResType = {}
    histRamaCtupleBySsAndResType = {}
    valuesByEntrySsAndResType = {}
    hrange = (xRange, yRange)

    rowCount = 0
    for row in reader:
        rowCount += 1
        #        7a3h,A,VAL ,   5,H, -62.8, -52.8
        #        7a3h,A,VAL ,   6,H, -71.2, -33.6
        #        7a3h,A,GLU ,   7,H, -63.5, -41.6
        (entryId, _chainId, resType, _resNum, ssType, phi, psi,
         _max_bfactor) = row
        ssType = to3StateDssp(ssType)[0]
        resType = resType.strip()
        phi = float(phi)
        psi = float(psi)
        if not (inRange(phi, isRange360=isRange360)
                and inRange(psi, isRange360=isRange360)):
            nTerror("phi and/or psi not in range for row: %s" % repr(row))
            return
        if not common20AADict.has_key(resType):
            nTdebug("Residue not in common 20 for row: %s" % repr(row))
            rowCount -= 1
            continue

        appendDeepByKeys(valuesBySsAndResType, phi, ssType, resType, 'phi')
        appendDeepByKeys(valuesBySsAndResType, psi, ssType, resType, 'psi')
        #        nTdebug('resType,ssType,phi,psi: %4s %1s %8.3f %8.3f' % (resType,ssType,phi,psi))
        appendDeepByKeys(valuesByEntrySsAndResType, phi, entryId, ssType,
                         resType, 'phi')
        appendDeepByKeys(valuesByEntrySsAndResType, psi, entryId, ssType,
                         resType, 'psi')
    del (reader)  # closes the file handles
    os.unlink(cvs_file_abs_name)
    nTdebug('Total number of included residues including PRO/GLY: %d' %
            rowCount)
    #    nTdebug('valuesByEntrySsAndResType:\n%s'%valuesByEntrySsAndResType)
    #    (cAv, cSd, _Cn) = getRescaling(valuesByEntrySsAndResType)
    (cAv, cSd) = (1.0, 1.0)
    nTdebug("Overall found av,sd: %r %r" % (cAv, cSd))

    for ssType in valuesBySsAndResType.keys():
        for resType in valuesBySsAndResType[ssType].keys():
            hist2d, _xedges, _yedges = histogram2d(
                valuesBySsAndResType[ssType][resType]['psi'],
                valuesBySsAndResType[ssType][resType]['phi'],
                bins=binCount,
                range=hrange)
            #            hist2d = zscaleHist( hist2d, cAv, cSd )
            setDeepByKeys(histRamaBySsAndResType, hist2d, ssType, resType)
            #            nTdebug('hist2d ssType, resType: %s %s\n%s' % (ssType, resType, hist2d))
            cTuple = getEnsembleAverageAndSigmaHis(hist2d)
            (c_av, c_sd, hisMin, hisMax) = cTuple
            cTuple += tuple([str([ssType, resType])
                             ])  # append the hash keys as a way of id.
            nTdebug(
                "For ssType %s residue type %s found (av/sd/min/max) %8.0f %8.0f %8.0f %8.0f"
                % (ssType, resType, c_av, c_sd, hisMin, hisMax))
            #            nTdebug("xedges %s" % repr(xedges))
            #            sys.exit(1)
            if c_sd == None:
                nTdebug(
                    'Failed to get c_sd when testing not all residues are present in smaller sets.'
                )
                continue
            if c_sd == 0.:
                nTdebug(
                    'Got zero c_sd, ignoring histogram. This should only occur in smaller sets. Not setting values.'
                )
                continue
            setDeepByKeys(histRamaCtupleBySsAndResType, cTuple, ssType,
                          resType)

    for ssType in valuesBySsAndResType.keys():
        phi = []
        psi = []
        for resType in valuesBySsAndResType[ssType].keys():
            if resType == 'PRO' or resType == 'GLY':
                continue
            phi += valuesBySsAndResType[ssType][resType]['phi']
            psi += valuesBySsAndResType[ssType][resType]['psi']
        hist2d, _xedges, _yedges = histogram2d(
            psi,  # Note that the x is the psi for some stupid reason,
            phi,  # otherwise the imagery but also the [row][column] notation is screwed.
            bins=binCount,
            range=hrange)
        #        hist2d = zscaleHist( hist2d, cAv, cSd )
        setDeepByKeys(histRamaBySsAndCombinedResType, hist2d, ssType)

    phi = []
    psi = []
    for ssType in valuesBySsAndResType.keys():
        for resType in valuesBySsAndResType[ssType].keys():
            if resType == 'PRO' or resType == 'GLY':
                continue
            phi += valuesBySsAndResType[ssType][resType]['phi']
            psi += valuesBySsAndResType[ssType][resType]['psi']

    nTdebug('Total number of residues without PRO/GLY: %d' % len(psi))
    hist2d, _xedges, _yedges = histogram2d(
        psi,  # Note that the x is the psi for some stupid reason,
        phi,  # otherwise the imagery but also the [row][column] notation is screwed.
        bins=binCount,
        range=hrange)
    #    sumHistCombined = sum( hist2d )
    #    sumsumHistCombined = sum( sumHistCombined )
    nTdebug('hist2d         : \n%s' % hist2d)
    #    nTdebug('sumHistCombined   : %s' % repr(sumHistCombined))
    #    nTdebug('sumsumHistCombined: %.0f' % sumsumHistCombined)
    #    hist2d = zscaleHist( hist2d, cAv, cSd )
    #    nTdebug('hist2d scaled  : \n%s' % hist2d)

    if os.path.exists(dbase_file_abs_name):
        os.unlink(dbase_file_abs_name)
#    dbase = shelve.open( dbase_file_abs_name )
    output = open(dbase_file_abs_name, 'wb')
    #    dbase = {'bar':'milky'}
    dbase = {}
    # Pickle the list using the highest protocol available.
    dbase['histRamaCombined'] = hist2d
    dbase['histRamaBySsAndCombinedResType'] = histRamaBySsAndCombinedResType
    dbase['histRamaBySsAndResType'] = histRamaBySsAndResType
    dbase['histRamaCtupleBySsAndResType'] = histRamaCtupleBySsAndResType
    #    pickle.dump(dbase, output, -1)
    #    pickle.dump(dbase, output)
    cPickle.dump(
        dbase, output,
        2)  # Was -1 for the most recent version but this caused an issue 239
    # NB 2 is the highest listed protocol too but behind the scenes cPickle will probably write something higher still.
    # If the protocol parameter is omitted, protocol 0 is used.
    # If protocol is specified as a negative value or HIGHEST_PROTOCOL, the highest protocol version will be used.

    output.close()
Beispiel #8
0
def main():
    cvs_file_abs_name_gz = cvs_file_abs_name + '.gz'
    gunzip(cvs_file_abs_name_gz)
    reader = csv.reader(open(cvs_file_abs_name, "rb"), quoting=csv.QUOTE_NONE)
    valuesBySsAndResType = {}
    histRamaBySsAndResType = {}
    histRamaBySsAndCombinedResType = {}
#    histByCombinedSsAndResType = {}
    histRamaCtupleBySsAndResType = {}
    valuesByEntrySsAndResType = {}
    hrange = (xRange, yRange)

    rowCount = 0
    for row in reader:
        rowCount += 1
#        7a3h,A,VAL ,   5,H, -62.8, -52.8
#        7a3h,A,VAL ,   6,H, -71.2, -33.6
#        7a3h,A,GLU ,   7,H, -63.5, -41.6
        (entryId, _chainId, resType, _resNum, ssType, phi, psi, _max_bfactor) = row
        ssType = to3StateDssp(ssType)[0]
        resType = resType.strip()
        phi = float(phi)
        psi = float(psi)
        if not (inRange(phi, isRange360=isRange360) and inRange(psi, isRange360=isRange360)):
            nTerror("phi and/or psi not in range for row: %s" % repr(row))
            return
        if not common20AADict.has_key(resType):
            nTdebug("Residue not in common 20 for row: %s" % repr(row))
            rowCount -= 1
            continue

        appendDeepByKeys(valuesBySsAndResType, phi, ssType, resType, 'phi')
        appendDeepByKeys(valuesBySsAndResType, psi, ssType, resType, 'psi')
#        nTdebug('resType,ssType,phi,psi: %4s %1s %8.3f %8.3f' % (resType,ssType,phi,psi))
        appendDeepByKeys(valuesByEntrySsAndResType, phi, entryId, ssType, resType, 'phi')
        appendDeepByKeys(valuesByEntrySsAndResType, psi, entryId, ssType, resType, 'psi')
    del(reader) # closes the file handles
    os.unlink(cvs_file_abs_name)
    nTdebug('Total number of included residues including PRO/GLY: %d' % rowCount)
#    nTdebug('valuesByEntrySsAndResType:\n%s'%valuesByEntrySsAndResType)
#    (cAv, cSd, _Cn) = getRescaling(valuesByEntrySsAndResType)
    (cAv, cSd) = (1.0, 1.0)
    nTdebug("Overall found av,sd: %r %r" % (cAv, cSd))

    for ssType in valuesBySsAndResType.keys():
        for resType in valuesBySsAndResType[ssType].keys():
            hist2d, _xedges, _yedges = histogram2d(
                valuesBySsAndResType[ssType][resType]['psi'],
                valuesBySsAndResType[ssType][resType]['phi'],
                bins=binCount,
                range=hrange)
#            hist2d = zscaleHist( hist2d, cAv, cSd )
            setDeepByKeys(histRamaBySsAndResType, hist2d, ssType, resType)
#            nTdebug('hist2d ssType, resType: %s %s\n%s' % (ssType, resType, hist2d))
            cTuple = getEnsembleAverageAndSigmaHis(hist2d)
            (c_av, c_sd, hisMin, hisMax) = cTuple
            cTuple += tuple([str([ssType, resType])]) # append the hash keys as a way of id.
            nTdebug("For ssType %s residue type %s found (av/sd/min/max) %8.0f %8.0f %8.0f %8.0f" % (
                ssType, resType, c_av, c_sd, hisMin, hisMax))
#            nTdebug("xedges %s" % repr(xedges))
#            sys.exit(1)
            if c_sd == None:
                nTdebug('Failed to get c_sd when testing not all residues are present in smaller sets.')
                continue
            if c_sd == 0.:
                nTdebug('Got zero c_sd, ignoring histogram. This should only occur in smaller sets. Not setting values.')
                continue
            setDeepByKeys(histRamaCtupleBySsAndResType, cTuple, ssType, resType)

    for ssType in valuesBySsAndResType.keys():
        phi = []
        psi = []
        for resType in valuesBySsAndResType[ssType].keys():
            if resType == 'PRO' or resType == 'GLY':
                continue
            phi += valuesBySsAndResType[ssType][resType]['phi']
            psi += valuesBySsAndResType[ssType][resType]['psi']
        hist2d, _xedges, _yedges = histogram2d(
            psi, # Note that the x is the psi for some stupid reason,
            phi, # otherwise the imagery but also the [row][column] notation is screwed.
            bins=binCount,
            range=hrange)
#        hist2d = zscaleHist( hist2d, cAv, cSd )
        setDeepByKeys(histRamaBySsAndCombinedResType, hist2d, ssType)

    phi = []
    psi = []
    for ssType in valuesBySsAndResType.keys():
        for resType in valuesBySsAndResType[ssType].keys():
            if resType == 'PRO' or resType == 'GLY':
                continue
            phi += valuesBySsAndResType[ssType][resType]['phi']
            psi += valuesBySsAndResType[ssType][resType]['psi']

    nTdebug('Total number of residues without PRO/GLY: %d' % len(psi))
    hist2d, _xedges, _yedges = histogram2d(
        psi, # Note that the x is the psi for some stupid reason,
        phi, # otherwise the imagery but also the [row][column] notation is screwed.
        bins=binCount,
        range=hrange)
#    sumHistCombined = sum( hist2d )
#    sumsumHistCombined = sum( sumHistCombined )
    nTdebug('hist2d         : \n%s' % hist2d)
#    nTdebug('sumHistCombined   : %s' % repr(sumHistCombined))
#    nTdebug('sumsumHistCombined: %.0f' % sumsumHistCombined)
#    hist2d = zscaleHist( hist2d, cAv, cSd )
#    nTdebug('hist2d scaled  : \n%s' % hist2d)

    if os.path.exists(dbase_file_abs_name):
        os.unlink(dbase_file_abs_name)
#    dbase = shelve.open( dbase_file_abs_name )
    output = open(dbase_file_abs_name, 'wb')
#    dbase = {'bar':'milky'}
    dbase = {}
    # Pickle the list using the highest protocol available.
    dbase[ 'histRamaCombined' ] = hist2d
    dbase[ 'histRamaBySsAndCombinedResType' ] = histRamaBySsAndCombinedResType
    dbase[ 'histRamaBySsAndResType' ] = histRamaBySsAndResType
    dbase[ 'histRamaCtupleBySsAndResType' ] = histRamaCtupleBySsAndResType
#    pickle.dump(dbase, output, -1)
#    pickle.dump(dbase, output)
    cPickle.dump(dbase, output, 2) # Was -1 for the most recent version but this caused an issue 239
    # NB 2 is the highest listed protocol too but behind the scenes cPickle will probably write something higher still.
    # If the protocol parameter is omitted, protocol 0 is used.
    # If protocol is specified as a negative value or HIGHEST_PROTOCOL, the highest protocol version will be used.

    output.close()