def permutation(arr: list, D0: list, D1: list): """Permutation test for MSEs""" indecies: np.ndarray = np.random.randint(2, size=len(arr[0])) scales: np.ndarray = arr[indecies, range(len(arr[0]))] afcs: np.ndarray = arr[1 - indecies, range(len(arr[0]))] x0 = [scales[i] + 1 for i in range(128)] x1 = [scales[i] + 1 for i in range(128, 256)] scaleROC: dict = ROC1.rocxy(x1, x0) x0 = [afcs[i] + 1 for i in range(128)] x1 = [afcs[i] + 1 for i in range(128, 256)] afcROC: dict = ROC1.rocxy(x1, x0) return MSE(None, None, scaleROC, afcROC)[1]
def simulation_ELO_targetAUC(args: list, rounds: int = 14, retRoc=False):
"""
Args is of the form (dist, auc, n0, n1).
Rounds is how many rounds of (n0 + n1)/2 comparisons it will so.
retRoc determines if the function will return its ROC curve or its statistics.
@Author: Francesc Massanes ([email protected])
@Version: 0.1 (really beta)
"""
dist, auc, n0, n1 = args
if n0 != n1:
raise NotImplementedError("n0 must equal n1")
N = n0
##
# DATA GENERATION
#
K1 = 400
K2 = 32
sep = genSep(dist, float(auc))
if dist == 'exponential':
neg = np.random.exponential(size=(N, 1))
plus = np.random.exponential(scale=sep, size=(N, 1))
elif dist == 'normal':
neg = np.random.normal(0, 1, (N, 1))
plus = np.random.normal(sep, 1, (N, 1))
else:
print("invalid argument", dist)
return None
x0 = np.array(neg)[:, 0]
x1 = np.array(plus)[:, 0]
empiricROC = rocxy(x1, x0)
scores = np.append(neg, plus)
truth = np.append(mb.zeros((N, 1)), mb.ones((N, 1)), axis=0)
rating = np.append(mb.zeros((N, 1)), mb.zeros((N, 1)), axis=0)
pc = list()
cnt = 0
ncmp = 0
results = list()
for eloRound in range(1, rounds + 1):
toCompare = mb.zeros((2 * N, 1))
if eloRound == 1:
# option A: only compare + vs -
arr = list(range(N))
#np.random.shuffle(arr)
toCompare[0::2] = np.array(arr, ndmin=2).transpose()
arr = list(range(N, 2 * N))
#np.random.shuffle(arr)
toCompare[1::2] = np.array(arr, ndmin=2).transpose()
else:
# option B: everything is valid
arr = list(range(2 * N))
np.random.shuffle(arr)
toCompare = np.array(arr, ndmin=2).transpose()
for i in range(1, 2 * N, 2):
a = int(toCompare[i - 1])
b = int(toCompare[i])
QA = 10**(int(rating[a]) / K1)
QB = 10**(int(rating[b]) / K1)
EA = QA / (QA + QB)
EB = QB / (QA + QB)
if scores[a] < scores[b]:
SA = 0
SB = 1
else:
SA = 1
SB = 0
if bool(truth[a]) ^ bool(truth[b]):
if (SA == 1 and truth[a] == 1):
cnt = cnt + 1
if (SB == 1 and truth[b] == 1):
cnt = cnt + 1
pc.append(cnt / (len(pc) + 1))
ncmp = ncmp + 1
rating[a] = rating[a] + K2 * (SA - EA)
rating[b] = rating[b] + K2 * (SB - EB)
x0 = np.array(rating[0:N])[:, 0]
x1 = np.array(rating[N:])[:, 0]
roc = rocxy(x1, x0)
if retRoc:
results.append((roc, ncmp))
else:
sm = successmatrix(x1, np.transpose(x0))
auc, var = np.mean(sm), unbiasedMeanMatrixVar(sm)
mseTruth, mseEmpiric, auc = MSE(sep, dist, roc, empiricROC)
results.append(
(N, cnt, ncmp, var, auc, mseTruth, mseEmpiric, pc[-1]))
return results
ax1.plot(empiricROC['x'],
empiricROC['y'],
linestyle=':',
lw=2,
label='empirical')
ax1.plot(roc['x'], roc['y'], label='predicted')
ax1.legend(loc=4)
ax1.set_title(
f"ELO\n{i+1}, {mseTheo[0]*1000:02.3f}E(-3), {mseEmp[0]*1000:02.3f}E(-3)"
)
groups = next(merge)
rocs = []
for group in groups:
rocs.append(genROC(group, D0, D1))
roc = avROC(rocs)
mseTheo, mseEmp, auc = MSE(7.72, 'exponential',
zip(*roc)), MSE(
7.72, 'exponential', zip(*roc),
zip(empiricROC['x'],
empiricROC['y']))
ax2.plot(x, y, linestyle='--', label='true', lw=3)
ax2.plot(empiricROC['x'],
empiricROC['y'],
linestyle=':',
lw=2,
label='empirical')
ax2.plot(*roc, label='predicted')
ax2.legend()
ax2.set_title(
f"merge\n{i+1}, {mseTheo[0]*1000:02.3f}E(-3), {mseEmp[0]*1000:02.3f}E(-3)"
)
plt.savefig(f"both")
ranks[0, x] = scoresScale[name]
ranks[1, x] = afcRanks[x]
ranks[0] = stats.rankdata(ranks[0])
ranks[1] = stats.rankdata(ranks[1])
scaleROC: dict = ROC1.rocxy(x1, x0)
x0 = ranks[0][0:n0]
x1 = ranks[0][n0:n0 + n1]
scaleSM = ROC1.successmatrix(x1, x0)
scaleAUC = ROC1.auc(x1, x0)
afcSM = ROC1.successmatrix(afcX1, afcX0)
afcAUC = ROC1.auc(afcX1, afcX0)
afcROC = ROC1.rocxy(x1, x0)
mse: float = MSE(None, None, scaleROC, afcROC)[1]
scatterAxes[i].plot([0, n0 + n1], [0, n0 + n1], 'r:')
for x in range(n0):
scatterAxes[i].scatter(ranks[0][x], ranks[1][x], c="b", marker="^", s=2)
for x in range(n0, n0 + n1):
scatterAxes[i].scatter(ranks[0][x], ranks[1][x], c="g", marker="o", s=2)
#scatterAxes[i].text(20, (n0 + n1)*0.9, reader[-1])
scatterAxes[i].set_aspect('equal', 'box')
tau: float = stats.kendalltau(ranks[0], ranks[1])[0]
scatterAxes[i].set_xticks([1, (n0 + n1) // 2, n0 + n1])
scatterAxes[i].set_yticks([1, (n0 + n1) // 2, n0 + n1])
scatterAxes[i].set_xticklabels([str(x) for x in [1, (n0 + n1) // 2, n0 + n1]], fontsize=fontSize)
scatterAxes[i].set_yticklabels([str(x) for x in [1, (n0 + n1) // 2, n0 + n1]], fontsize=fontSize)
scatterAxes[i].set_title(reader)
scatterAxes[i].set_xlabel("Image Ranks from Rating Data", fontsize=fontSize)