Пример #1
0
def synergyCP(X, y, n_source=3):
    listIndSrcErrRate = []  # empty list
    listSCPErrRate = []  # empty list
    listIndSrcEff = []  # empty list
    listSCPEff = []  # empty list
    listIndSrcVal = []  # empty list
    listSCPVal = []  # empty list
    listIndSrcOF = []  # empty list
    listSCPOF = []  # empty list

    for i in range(1):
        X_train, X_test, y_train, y_test \
            = train_test_split(X, y, test_size=0.2)

        X_train, X_calib, y_train, y_calib \
            = train_test_split(X_train, y_train, test_size=0.3)

        nrTrainCases = len(y_train)
        n_labels = len(np.unique(y_train))
        randIndex = random.sample(list(range(0, nrTrainCases)), nrTrainCases)
        splitLen = int(nrTrainCases / n_source)
        # split training data into equal parts
        trainIndex = randIndex[0:splitLen]

        meanCalibPredProb = np.zeros((len(y_calib), n_labels))
        meanTestPredProb = np.zeros((len(y_test), n_labels))
        for indexSrc in range(0, n_source):
            sourceData = X_train[trainIndex, :]
            sourceTarget = y_train[trainIndex]
            calibPredProb, testPredProb = icp.ICPClassification(sourceData, sourceTarget, X_calib,\
                                                                X_test, method = "rf", nrTrees = 10)
            if indexSrc == 0:
                srcMCListConfScores = icp.computeConformityScores(
                    calibPredProb, y_calib)
                pValues = icp.computePValues(srcMCListConfScores, testPredProb)
                errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
                    pValues, y_test)
                listIndSrcVal.append(val)
                listIndSrcErrRate.append(errRate)
                listIndSrcEff.append(eff)
                listIndSrcOF.append(obsFuzz)
                pm.CalibrationPlot(pValues, y_test, color='orange')

            meanCalibPredProb = np.add(meanCalibPredProb, calibPredProb)
            meanTestPredProb = np.add(meanTestPredProb, testPredProb)
            trainIndex = randIndex[splitLen * (indexSrc + 1):splitLen *
                                   (indexSrc + 2)]

        meanCalibPredProb = meanCalibPredProb / n_source
        meanTestPredProb = meanTestPredProb / n_source
        srcMCListConfScores = icp.computeConformityScores(
            meanCalibPredProb, y_calib)
        pValues = icp.computePValues(srcMCListConfScores, meanTestPredProb)
        errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
        pm.CalibrationPlot(pValues, y_test, color='b')
        print(val)
Пример #2
0
def CCP(X, y, n_source=3, method='linear_svm', path='filename'):
    listCCPOF = []  # empty list
    x = PrettyTable()
    x.field_names = [
        "Validity", "Efficiency", "ErrorRate", "Observed Fuzziness"
    ]
    n_labels = len(np.unique(y))

    for i in range(10):
        XX, X_test, yy, y_test \
            = train_test_split(X, y, test_size=0.2, random_state=i)

        scaler = MinMaxScaler()
        scaler.fit(XX)
        XX = scaler.transform(XX)
        X_test = scaler.transform(X_test)

        meanPValues = np.zeros((len(y_test), n_labels))

        #sss = StratifiedShuffleSplit(n_splits=n_source, test_size=1/n_source)
        sss = ShuffleSplit(n_splits=n_source,
                           test_size=1 / n_source,
                           random_state=i)
        for train_index, test_index in sss.split(XX, yy):
            # print("TRAIN:", train_index, "TEST:", test_index)
            X_train, X_calib = XX[train_index], XX[test_index]
            y_train, y_calib = yy[train_index], yy[test_index]
            calibPredProb, testPredProb = icp.ICPClassification(X_train,
                                                                y_train,
                                                                X_calib,
                                                                X_test,
                                                                method=method)
            srcMCListConfScores = icp.computeConformityScores(
                calibPredProb, y_calib)
            pValues = icp.computePValues(srcMCListConfScores, testPredProb)
            meanPValues = np.add(meanPValues, pValues)

        meanPValues = meanPValues / n_source
        errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
            meanPValues, y_test)
        #pm.CalibrationPlot(meanPValues, y_test, color='b')
        listCCPOF.append(obsFuzz)

    if not os.path.exists('json_ccp'):
        os.makedirs('json_ccp')

    with open(path, 'w') as fh:
        fh.write(json.dumps(listCCPOF))

    return listCCPOF
Пример #3
0
def ACP_addCal(X,
               y,
               n_source=20,
               method="linear_svm",
               X_addCal=None,
               y_addCal=None):
    listACPOF = []  # empty list
    x = PrettyTable()
    x.field_names = [
        "Validity", "Efficiency", "ErrorRate", "Observed Fuzziness"
    ]
    n_labels = len(np.unique(y))

    for i in range(10):
        XX, X_test, yy, y_test \
            = train_test_split(X, y, test_size=0.2)

        meanPValues = np.zeros((len(y_test), n_labels))

        for j in range(n_source):
            X_train, X_calib, y_train, y_calib = train_test_split(XX,
                                                                  yy,
                                                                  test_size=1 /
                                                                  3,
                                                                  stratify=yy)
            if np.any(X_addCal) == True:
                X_calib = np.append(X_calib, X_addCal, axis=0)
                y_calib = np.append(y_calib, y_addCal, axis=0)

            calibPredProb, testPredProb = icp.ICPClassification(X_train,
                                                                y_train,
                                                                X_calib,
                                                                X_test,
                                                                method=method)
            srcMCListConfScores = icp.computeConformityScores(
                calibPredProb, y_calib)
            pValues = icp.computePValues(srcMCListConfScores, testPredProb)
            meanPValues = np.add(meanPValues, pValues)

        meanPValues = meanPValues / n_source
        errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
            meanPValues, y_test)
        # pm.CalibrationPlot(meanPValues, y_test, color='b')
        listACPOF.append(obsFuzz)

    print(val)
    return listACPOF
Пример #4
0
def ACP(X, y, n_source=3):
    listACPOF = []  # empty list
    x = PrettyTable()
    x.field_names = [
        "Validity", "Efficiency", "ErrorRate", "Observed Fuzziness"
    ]
    n_labels = len(np.unique(y))

    for i in range(1):
        XX, X_test, yy, y_test \
            = train_test_split(X, y, test_size=0.2)

        meanPValues = np.zeros((len(y_test), n_labels))

        #sss = StratifiedShuffleSplit(n_splits=n_source, test_size=1/n_source)
        sss = ShuffleSplit(n_splits=n_source, test_size=1 / n_source)
        for train_index, test_index in sss.split(XX, yy):
            # print("TRAIN:", train_index, "TEST:", test_index)
            X_train, X_calib = XX[train_index], XX[test_index]
            y_train, y_calib = yy[train_index], yy[test_index]
            calibPredProb, testPredProb = icp.ICPClassification(X_train,
                                                                y_train,
                                                                X_calib,
                                                                X_test,
                                                                method="rf",
                                                                nrTrees=10)
            srcMCListConfScores = icp.computeConformityScores(
                calibPredProb, y_calib)
            pValues = icp.computePValues(srcMCListConfScores, testPredProb)
            meanPValues = np.add(meanPValues, pValues)

        meanPValues = meanPValues / n_source
        errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
            meanPValues, y_test)
        pm.CalibrationPlot(meanPValues, y_test, color='b')
        listACPOF.append(obsFuzz)

    print(val)
    return listACPOF
Пример #5
0
def synergyCP(X, y, n_source = 3, methods = None, path = None):
    # initialize lists
    listIndSrcVal = []  # empty list
    listSCPVal = []  # empty list
    listIndSrcOF = []  # empty list
    listSCPOF = []  # empty list
    for i in range(n_source):
        listIndSrcOF.append([])

    if methods is None:
        methods = ['linear_svm'] * n_source

    for i in range(10):
        X_train, X_test, y_train, y_test \
            = train_test_split(X, y, test_size=0.2, random_state=i)

        X_train, X_calib, y_train, y_calib \
            = train_test_split(X_train, y_train, test_size=.3, random_state=i)

        # scale data to be in the same range
        scaler = MinMaxScaler()
        #scaler = StandardScaler()
        scaler.fit(X_train)
        X_train = scaler.transform(X_train)
        X_test = scaler.transform(X_test)

        nrTrainCases = len(y_train)
        n_labels = len(np.unique(y_train))
        randIndex = random.sample(list(range(0, nrTrainCases)), nrTrainCases)
        splitLen = int(nrTrainCases / n_source)
        # split training data into equal parts
        trainIndex = randIndex[0:splitLen]

        meanCalibPredProb = np.zeros((len(y_calib), n_labels))
        meanTestPredProb = np.zeros((len(y_test), n_labels))
        for indexSrc in range(0, n_source):
            sourceData = X_train[trainIndex, :]
            sourceTarget = y_train[trainIndex]
            calibPredProb, testPredProb = icp.ICPClassification(sourceData, sourceTarget, X_calib,\
                                                                X_test, method=methods[indexSrc])

            srcMCListConfScores = icp.computeConformityScores(calibPredProb, y_calib)
            pValues = icp.computePValues(srcMCListConfScores, testPredProb)
            errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
            listIndSrcVal.append(val)
            listIndSrcOF[indexSrc].append(obsFuzz)

            meanCalibPredProb = np.add(meanCalibPredProb, calibPredProb)
            meanTestPredProb = np.add(meanTestPredProb, testPredProb)
            trainIndex = randIndex[splitLen * (indexSrc + 1):splitLen * (indexSrc + 2)]

        meanCalibPredProb = meanCalibPredProb / n_source
        meanTestPredProb = meanTestPredProb / n_source
        srcMCListConfScores = icp.computeConformityScores(meanCalibPredProb, y_calib)
        pValues = icp.computePValues(srcMCListConfScores, meanTestPredProb)
        errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
        listSCPVal.append(val)
        listSCPOF.append(obsFuzz)

    results = OrderedDict()
    results["source1"] = listIndSrcOF[0]
    results["source2"] = listIndSrcOF[1]
    results["source3"] = listIndSrcOF[2]
    results["SCP_OF"] = listSCPOF
    results["SCP_val"] = listSCPVal

    import json
    with open(path, 'w') as fh:
        fh.write(json.dumps(results))

    print(path, np.round(np.median(listIndSrcOF[0]), 3),
          np.round(np.median(listIndSrcOF[1]), 3),
          np.round(np.median(listIndSrcOF[2]), 3),
          np.round(np.median(listSCPOF), 3))
Пример #6
0
def synergyCP(X, y, methods = None, path = None):
    n_source = len(methods)
    listIndSrcVal = []  # empty list
    listSCPVal = []  # empty list
    listIndSrcOF = []  # empty list
    listSCPOF = []  # empty list
    for i in range(n_source):
        listIndSrcOF.append([])

    for i in range(iterate):
        X_train, X_test, y_train, y_test \
            = train_test_split(X, y, test_size=0.2, random_state=i)

        #scaler = StandardScaler()
        scaler = MinMaxScaler()
        scaler.fit(X_train)
        X_train = scaler.transform(X_train)
        X_test = scaler.transform(X_test)

        X_train, X_calib, y_train, y_calib \
            = train_test_split(X_train, y_train, test_size=0.3, random_state=i)

        n_labels = len(np.unique(y_train))
        meanCalibPredProb = np.zeros((len(y_calib), n_labels))
        meanTestPredProb = np.zeros((len(y_test), n_labels))

        for indexSrc in range(n_source):
            sourceData = X_train
            sourceTarget = y_train
            calibPredProb, testPredProb = icp.ICPClassification(sourceData, sourceTarget, X_calib,\
                                                        X_test, method=methods[indexSrc])

            srcMCListConfScores = icp.computeConformityScores(calibPredProb, y_calib)
            pValues = icp.computePValues(srcMCListConfScores, testPredProb)
            errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
            listIndSrcVal.append(val)
            listIndSrcOF[indexSrc].append(obsFuzz)

            meanCalibPredProb = np.add(meanCalibPredProb, calibPredProb)
            meanTestPredProb = np.add(meanTestPredProb, testPredProb)

        meanCalibPredProb = meanCalibPredProb / n_source
        meanTestPredProb = meanTestPredProb / n_source
        srcMCListConfScores = icp.computeConformityScores(meanCalibPredProb, y_calib)
        pValues = icp.computePValues(srcMCListConfScores, meanTestPredProb)
        errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
        listSCPVal.append(val)
        listSCPOF.append(obsFuzz)

    results = OrderedDict()
    results["source1"] = listIndSrcOF[0]
    results["source2"] = listIndSrcOF[1]
    results["source3"] = listIndSrcOF[2]
    results["SCP_OF"] = listSCPOF
    results["SCP_val"] = listSCPVal

    import json
    with open(path, 'w') as fh:
        fh.write(json.dumps(results))

    print(path, np.round(np.median(listIndSrcOF[0]), 3),
          np.round(np.median(listIndSrcOF[1]), 3),
          np.round(np.median(listIndSrcOF[2]), 3),
          np.round(np.median(listSCPOF), 3))