def synergyCP(X, y, n_source=3):
listIndSrcErrRate = [] # empty list
listSCPErrRate = [] # empty list
listIndSrcEff = [] # empty list
listSCPEff = [] # empty list
listIndSrcVal = [] # empty list
listSCPVal = [] # empty list
listIndSrcOF = [] # empty list
listSCPOF = [] # empty list
for i in range(1):
X_train, X_test, y_train, y_test \
= train_test_split(X, y, test_size=0.2)
X_train, X_calib, y_train, y_calib \
= train_test_split(X_train, y_train, test_size=0.3)
nrTrainCases = len(y_train)
n_labels = len(np.unique(y_train))
randIndex = random.sample(list(range(0, nrTrainCases)), nrTrainCases)
splitLen = int(nrTrainCases / n_source)
# split training data into equal parts
trainIndex = randIndex[0:splitLen]
meanCalibPredProb = np.zeros((len(y_calib), n_labels))
meanTestPredProb = np.zeros((len(y_test), n_labels))
for indexSrc in range(0, n_source):
sourceData = X_train[trainIndex, :]
sourceTarget = y_train[trainIndex]
calibPredProb, testPredProb = icp.ICPClassification(sourceData, sourceTarget, X_calib,\
X_test, method = "rf", nrTrees = 10)
if indexSrc == 0:
srcMCListConfScores = icp.computeConformityScores(
calibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, testPredProb)
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
pValues, y_test)
listIndSrcVal.append(val)
listIndSrcErrRate.append(errRate)
listIndSrcEff.append(eff)
listIndSrcOF.append(obsFuzz)
pm.CalibrationPlot(pValues, y_test, color='orange')
meanCalibPredProb = np.add(meanCalibPredProb, calibPredProb)
meanTestPredProb = np.add(meanTestPredProb, testPredProb)
trainIndex = randIndex[splitLen * (indexSrc + 1):splitLen *
(indexSrc + 2)]
meanCalibPredProb = meanCalibPredProb / n_source
meanTestPredProb = meanTestPredProb / n_source
srcMCListConfScores = icp.computeConformityScores(
meanCalibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, meanTestPredProb)
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
pm.CalibrationPlot(pValues, y_test, color='b')
print(val)
def CCP(X, y, n_source=3, method='linear_svm', path='filename'):
listCCPOF = [] # empty list
x = PrettyTable()
x.field_names = [
"Validity", "Efficiency", "ErrorRate", "Observed Fuzziness"
]
n_labels = len(np.unique(y))
for i in range(10):
XX, X_test, yy, y_test \
= train_test_split(X, y, test_size=0.2, random_state=i)
scaler = MinMaxScaler()
scaler.fit(XX)
XX = scaler.transform(XX)
X_test = scaler.transform(X_test)
meanPValues = np.zeros((len(y_test), n_labels))
#sss = StratifiedShuffleSplit(n_splits=n_source, test_size=1/n_source)
sss = ShuffleSplit(n_splits=n_source,
test_size=1 / n_source,
random_state=i)
for train_index, test_index in sss.split(XX, yy):
# print("TRAIN:", train_index, "TEST:", test_index)
X_train, X_calib = XX[train_index], XX[test_index]
y_train, y_calib = yy[train_index], yy[test_index]
calibPredProb, testPredProb = icp.ICPClassification(X_train,
y_train,
X_calib,
X_test,
method=method)
srcMCListConfScores = icp.computeConformityScores(
calibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, testPredProb)
meanPValues = np.add(meanPValues, pValues)
meanPValues = meanPValues / n_source
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
meanPValues, y_test)
#pm.CalibrationPlot(meanPValues, y_test, color='b')
listCCPOF.append(obsFuzz)
if not os.path.exists('json_ccp'):
os.makedirs('json_ccp')
with open(path, 'w') as fh:
fh.write(json.dumps(listCCPOF))
return listCCPOF
def ACP_addCal(X,
y,
n_source=20,
method="linear_svm",
X_addCal=None,
y_addCal=None):
listACPOF = [] # empty list
x = PrettyTable()
x.field_names = [
"Validity", "Efficiency", "ErrorRate", "Observed Fuzziness"
]
n_labels = len(np.unique(y))
for i in range(10):
XX, X_test, yy, y_test \
= train_test_split(X, y, test_size=0.2)
meanPValues = np.zeros((len(y_test), n_labels))
for j in range(n_source):
X_train, X_calib, y_train, y_calib = train_test_split(XX,
yy,
test_size=1 /
3,
stratify=yy)
if np.any(X_addCal) == True:
X_calib = np.append(X_calib, X_addCal, axis=0)
y_calib = np.append(y_calib, y_addCal, axis=0)
calibPredProb, testPredProb = icp.ICPClassification(X_train,
y_train,
X_calib,
X_test,
method=method)
srcMCListConfScores = icp.computeConformityScores(
calibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, testPredProb)
meanPValues = np.add(meanPValues, pValues)
meanPValues = meanPValues / n_source
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
meanPValues, y_test)
# pm.CalibrationPlot(meanPValues, y_test, color='b')
listACPOF.append(obsFuzz)
print(val)
return listACPOF
def ACP(X, y, n_source=3):
listACPOF = [] # empty list
x = PrettyTable()
x.field_names = [
"Validity", "Efficiency", "ErrorRate", "Observed Fuzziness"
]
n_labels = len(np.unique(y))
for i in range(1):
XX, X_test, yy, y_test \
= train_test_split(X, y, test_size=0.2)
meanPValues = np.zeros((len(y_test), n_labels))
#sss = StratifiedShuffleSplit(n_splits=n_source, test_size=1/n_source)
sss = ShuffleSplit(n_splits=n_source, test_size=1 / n_source)
for train_index, test_index in sss.split(XX, yy):
# print("TRAIN:", train_index, "TEST:", test_index)
X_train, X_calib = XX[train_index], XX[test_index]
y_train, y_calib = yy[train_index], yy[test_index]
calibPredProb, testPredProb = icp.ICPClassification(X_train,
y_train,
X_calib,
X_test,
method="rf",
nrTrees=10)
srcMCListConfScores = icp.computeConformityScores(
calibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, testPredProb)
meanPValues = np.add(meanPValues, pValues)
meanPValues = meanPValues / n_source
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(
meanPValues, y_test)
pm.CalibrationPlot(meanPValues, y_test, color='b')
listACPOF.append(obsFuzz)
print(val)
return listACPOF
def synergyCP(X, y, n_source = 3, methods = None, path = None):
# initialize lists
listIndSrcVal = [] # empty list
listSCPVal = [] # empty list
listIndSrcOF = [] # empty list
listSCPOF = [] # empty list
for i in range(n_source):
listIndSrcOF.append([])
if methods is None:
methods = ['linear_svm'] * n_source
for i in range(10):
X_train, X_test, y_train, y_test \
= train_test_split(X, y, test_size=0.2, random_state=i)
X_train, X_calib, y_train, y_calib \
= train_test_split(X_train, y_train, test_size=.3, random_state=i)
# scale data to be in the same range
scaler = MinMaxScaler()
#scaler = StandardScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
nrTrainCases = len(y_train)
n_labels = len(np.unique(y_train))
randIndex = random.sample(list(range(0, nrTrainCases)), nrTrainCases)
splitLen = int(nrTrainCases / n_source)
# split training data into equal parts
trainIndex = randIndex[0:splitLen]
meanCalibPredProb = np.zeros((len(y_calib), n_labels))
meanTestPredProb = np.zeros((len(y_test), n_labels))
for indexSrc in range(0, n_source):
sourceData = X_train[trainIndex, :]
sourceTarget = y_train[trainIndex]
calibPredProb, testPredProb = icp.ICPClassification(sourceData, sourceTarget, X_calib,\
X_test, method=methods[indexSrc])
srcMCListConfScores = icp.computeConformityScores(calibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, testPredProb)
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
listIndSrcVal.append(val)
listIndSrcOF[indexSrc].append(obsFuzz)
meanCalibPredProb = np.add(meanCalibPredProb, calibPredProb)
meanTestPredProb = np.add(meanTestPredProb, testPredProb)
trainIndex = randIndex[splitLen * (indexSrc + 1):splitLen * (indexSrc + 2)]
meanCalibPredProb = meanCalibPredProb / n_source
meanTestPredProb = meanTestPredProb / n_source
srcMCListConfScores = icp.computeConformityScores(meanCalibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, meanTestPredProb)
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
listSCPVal.append(val)
listSCPOF.append(obsFuzz)
results = OrderedDict()
results["source1"] = listIndSrcOF[0]
results["source2"] = listIndSrcOF[1]
results["source3"] = listIndSrcOF[2]
results["SCP_OF"] = listSCPOF
results["SCP_val"] = listSCPVal
import json
with open(path, 'w') as fh:
fh.write(json.dumps(results))
print(path, np.round(np.median(listIndSrcOF[0]), 3),
np.round(np.median(listIndSrcOF[1]), 3),
np.round(np.median(listIndSrcOF[2]), 3),
np.round(np.median(listSCPOF), 3))
def synergyCP(X, y, methods = None, path = None):
n_source = len(methods)
listIndSrcVal = [] # empty list
listSCPVal = [] # empty list
listIndSrcOF = [] # empty list
listSCPOF = [] # empty list
for i in range(n_source):
listIndSrcOF.append([])
for i in range(iterate):
X_train, X_test, y_train, y_test \
= train_test_split(X, y, test_size=0.2, random_state=i)
#scaler = StandardScaler()
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
X_train, X_calib, y_train, y_calib \
= train_test_split(X_train, y_train, test_size=0.3, random_state=i)
n_labels = len(np.unique(y_train))
meanCalibPredProb = np.zeros((len(y_calib), n_labels))
meanTestPredProb = np.zeros((len(y_test), n_labels))
for indexSrc in range(n_source):
sourceData = X_train
sourceTarget = y_train
calibPredProb, testPredProb = icp.ICPClassification(sourceData, sourceTarget, X_calib,\
X_test, method=methods[indexSrc])
srcMCListConfScores = icp.computeConformityScores(calibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, testPredProb)
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
listIndSrcVal.append(val)
listIndSrcOF[indexSrc].append(obsFuzz)
meanCalibPredProb = np.add(meanCalibPredProb, calibPredProb)
meanTestPredProb = np.add(meanTestPredProb, testPredProb)
meanCalibPredProb = meanCalibPredProb / n_source
meanTestPredProb = meanTestPredProb / n_source
srcMCListConfScores = icp.computeConformityScores(meanCalibPredProb, y_calib)
pValues = icp.computePValues(srcMCListConfScores, meanTestPredProb)
errRate, eff, val, obsFuzz = pm.pValues2PerfMetrics(pValues, y_test)
listSCPVal.append(val)
listSCPOF.append(obsFuzz)
results = OrderedDict()
results["source1"] = listIndSrcOF[0]
results["source2"] = listIndSrcOF[1]
results["source3"] = listIndSrcOF[2]
results["SCP_OF"] = listSCPOF
results["SCP_val"] = listSCPVal
import json
with open(path, 'w') as fh:
fh.write(json.dumps(results))
print(path, np.round(np.median(listIndSrcOF[0]), 3),
np.round(np.median(listIndSrcOF[1]), 3),
np.round(np.median(listIndSrcOF[2]), 3),
np.round(np.median(listSCPOF), 3))