'RID', 'Forecast Month', 'Forecast Date', 'CN relative probability',
'MCI relative probability', 'AD relative probability', 'ADAS13',
'ADAS13 50% CI lower', 'ADAS13 50% CI upper', 'Ventricles_ICV',
'Ventricles_ICV 50% CI lower', 'Ventricles_ICV 50% CI upper'
])
output['Forecast Month'] = output['Forecast Month'].astype(int)
output['Forecast Date'] = ym1
str_out_final = os.path.join(
str_exp, 'IntermediateData',
'TADPOLE_Submission_Leaderboard_BenchmarkSVM.csv')
output.to_csv(str_out_final, header=True, index=False)
print('Evaluate predictions')
R = pd.read_csv('./Data/TADPOLE_LB4.csv')
import evalOneSubmission as eos
mAUC, bca, adasMAE, ventsMAE, adasWES, ventsWES, adasCPA, ventsCPA, adasEstim, trueADASFilt = eos.evalOneSub(
R, output)
print 'Diagnosis:'
print 'mAUC = ' + "%0.3f" % mAUC,
print 'BAC = ' + "%0.3f" % bca
print 'ADAS:'
print 'MAE = ' + "%0.3f" % adasMAE,
print 'WES = ' + "%0.3f" % adasWES,
print 'CPA = ' + "%0.3f" % adasCPA
print 'VENTS:'
print 'MAE = ' + "%0.3e" % ventsMAE,
print 'WES = ' + "%0.3e" % ventsWES,
print 'CPA = ' + "%0.3f" % ventsCPA
def downloadLeaderboardSubmissions():
htmlFile = '%sTable.html' % tag
dropboxRemoteFolder = '/ProAD/public_html'
uploadsFldRemote = '/ProAD/uploads'
ldbSubmissionsFld = 'leaderboardSubmissions'
ldbDropbox = DropboxObj()
fileListAll = ldbDropbox.list_folder(uploadsFldRemote, '/')
fileListLdb = [
x for x in fileListAll.keys()
if x.startswith('TADPOLE_Submission_%s' % tag)
]
fileListLdb.sort()
print('fileListLdb ', fileListLdb)
os.system('mkdir -p %s' % ldbSubmissionsFld)
nrEntries = len(fileListLdb)
teamNames = [
f.split('.')[0][len('TADPOLE_Submission_%s_' % tag):]
for f in fileListLdb
]
evalResFile = '%s/evalResAll.npz' % ldbSubmissionsFld
# entriesList = [0,1,2]
tableColumns = ('TEAMNAME', 'RANK', 'MAUC', 'BCA', 'adasMAE', 'ventsMAE',
'adasWES', 'ventsWES', 'adasCPA', 'ventsCPA')
if args.runPart[0] == 'R':
if args.fast:
# load submissions already evaluated and only evaluate the new ones
dataStruct = pickle.load(open(evalResFile, 'rb'))
evalResults = dataStruct['evalResults']
fileDatesRemote = dataStruct['fileDatesRemote']
entriesList = [
e for e, f in enumerate(teamNames)
if (evalResults['TEAMNAME'].str.contains(f).sum() == 0)
]
nanSeries = pd.DataFrame(np.nan,
index=range(len(entriesList)),
columns=tableColumns)
nrEntriesSoFar = evalResults.shape[0]
evalResults = evalResults.append(nanSeries, ignore_index=True)
print('teamNames', teamNames)
print('entriesList', entriesList)
print('evalResults', evalResults)
# print(adsa)
else:
evalResults = pd.DataFrame(np.nan,
index=range(nrEntries),
columns=tableColumns)
fileDatesRemote = []
entriesList = range(nrEntries)
nrEntriesSoFar = 0
lb4Df = pd.read_csv('../data/TADPOLE_LB4.csv')
lb4Df = lb4Df[lb4Df['LB4'] == 1] # only keep the LB4 entries
lb4Df.reset_index(drop=True, inplace=True)
indexInTable = 0
entryToAddIndex = nrEntriesSoFar
for f in entriesList:
fileName = fileListLdb[f]
teamName = teamNames[f]
# print('teamname ', teamName)
remotePath = '%s/%s' % (uploadsFldRemote, fileName)
localPath = '%s/%s' % (ldbSubmissionsFld, fileName)
ldbDropbox.download(localPath, remotePath)
metadataFileRemote = ldbDropbox.dbx.files_get_metadata(remotePath)
fileDatesRemote += [metadataFileRemote.server_modified]
print('Evaluating %s' % fileName)
forecastDf = pd.read_csv(localPath)
try:
evalResults.loc[entryToAddIndex, ['MAUC', 'BCA',
'adasMAE', 'ventsMAE', 'adasWES', 'ventsWES', 'adasCPA', 'ventsCPA']] = \
evalOneSubmission.evalOneSub(lb4Df, forecastDf)
evalResults.loc[entryToAddIndex, 'TEAMNAME'] = teamName
except:
print('Error while processing submission %s' % fileName)
pass
# if not np.isnan(evalResults['MAUC'].iloc[f]):
entryToAddIndex += 1
nanMask = np.isnan(evalResults['MAUC'])
evalResults = evalResults[np.logical_not(nanMask)]
evalResults.reset_index(drop=True, inplace=True)
# # compute the ranks using MAUC
# rankOrder = np.argsort(evalResults.as_matrix(columns = ['MAUC']).reshape(-1))[::-1] # sort them by MAUC
# rankOrder += 1 # make them start from 1
# print('ranks', evalResults['MAUC'], rankOrder, evalResults.as_matrix(columns = ['MAUC']).reshape(-1))
# for f in range(evalResults.shape[0]):
# evalResults.loc[f, 'RANK'] = rankOrder[f]
dataStruct = dict(evalResults=evalResults,
fileDatesRemote=fileDatesRemote)
pickle.dump(dataStruct,
open(evalResFile, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL)
else:
dataStruct = pickle.load(open(evalResFile, 'rb'))
fileDatesRemote = dataStruct['fileDatesRemote']
evalResults = dataStruct['evalResults']
rankMAUC = rankdata(
rankdata(-evalResults.as_matrix(columns=['MAUC']).reshape(-1),
method='average'),
method='average')
rankADAS = rankdata(rankdata(
evalResults.as_matrix(columns=['adasMAE']).reshape(-1),
method='average'),
method='average')
rankVENTS = rankdata(rankdata(
evalResults.as_matrix(columns=['ventsMAE']).reshape(-1),
method='average'),
method='average')
print('rankMAUC', rankMAUC)
print('rankADAS', rankADAS)
print('rankVENTS', rankVENTS)
rankSum = rankMAUC + rankADAS + rankVENTS
rankOrder = rankdata(rankSum, method='average') # make them start from 1
for f in range(evalResults.shape[0]):
evalResults.loc[f, 'RANK'] = rankOrder[f]
# print('evalResults before\n', evalResults)
evalResults = evalResults.sort_values(by=['MAUC', 'BCA'], ascending=False)
evalResults = evalResults.reset_index(drop=True)
print('evalResults after\n', evalResults)
htmlFileFullPathRemote = '%s/%s' % (dropboxRemoteFolder, htmlFile)
htmlFileFullPathLocal = '%s/%s' % (ldbSubmissionsFld, htmlFile)
writeHTMLtable(evalResults, htmlFileFullPathLocal, fileListLdb,
fileDatesRemote)
ldbDropbox.upload(htmlFileFullPathLocal, htmlFileFullPathRemote)
columns=[
'RID', 'Forecast Month', 'Forecast Date', 'CN relative probability',
'MCI relative probability', 'AD relative probability', 'ADAS13',
'ADAS13 50% CI lower', 'ADAS13 50% CI upper', 'Ventricles_ICV',
'Ventricles_ICV 50% CI lower', 'Ventricles_ICV 50% CI upper'
])
output['Forecast Month'] = output['Forecast Month'].astype(int)
output['Forecast Date'] = ym1
str_out_final = os.path.join(str_exp, 'IntermediateData',
'TADPOLE_Submission_BenchmarkSVM.csv')
output.to_csv(str_out_final, header=True, index=False)
print('Evaluate predictions')
R = pd.read_csv('./IntermediateData/D4_dummy.csv')
import evalOneSubmission as eos
mAUC, bca, adasMAE, ventsMAE, adasWES, ventsWES, adasCPA, ventsCPA = eos.evalOneSub(
R, output)
print('Diagnosis:')
print('mAUC = ' + "%0.3f" % mAUC)
print('BAC = ' + "%0.3f" % bca)
print('ADAS:')
print('MAE = ' + "%0.3f" % adasMAE)
print('WES = ' + "%0.3f" % adasWES)
print('CPA = ' + "%0.3f" % adasCPA)
print('VENTS:')
print('MAE = ' + "%0.3e" % ventsMAE)
print('WES = ' + "%0.3e" % ventsWES)
print('CPA = ' + "%0.3f" % ventsCPA)
for i in range(len(o)):
o1[count:count+nr_pred]=o[i]
o1[count:count+nr_pred,1]=range(1,nr_pred+1)
count=count+nr_pred
output=pd.DataFrame(o1, columns=['RID','Forecast Month','Forecast Date','CN relative probability','MCI relative probability','AD relative probability'])
output['Forecast Month'] = output['Forecast Month'].astype(int)
output['Forecast Date'] = ym1
str_out_final = os.path.join(str_exp, 'IntermediateData','TADPOLE_Submission_Leaderboard_BenchmarkSVM.csv')
output.to_csv(str_out_final,header=True,index=False)
print('Evaluate predictions')
R=pd.read_csv('./TADPOLE_LB4.csv')
import evalOneSubmission as eos
mAUC, bca = eos.evalOneSub(R,output)
print('Diagnosis:')
print('mAUC = ' + "%0.3f" % mAUC,)
print ('BAC = ' + "%0.3f" % bca)
# print('ADAS:')
# print('MAE = ' + "%0.3f" % adasMAE, )
# print('WES = ' + "%0.3f" % adasWES,)
# print('CPA = ' + "%0.3f" % adasCPA )
# print('VENTS:')
# print('MAE = ' + "%0.3e" % ventsMAE,)
# print('WES = ' + "%0.3e" % ventsWES,)
# print('CPA = ' + "%0.3f" % ventsCPA )
def downloadLeaderboardSubmissions():
htmlFile = 'leaderboardTable.html'
dropboxRemoteFolder = '/ProAD/public_html'
uploadsFldRemote = '/ProAD/uploads'
ldbSubmissionsFld = 'leaderboardSubmissions'
ldbDropbox = DropboxObj()
fileListAll = ldbDropbox.list_folder(uploadsFldRemote, '/')
fileListLdb = [x for x in fileListAll.keys() if x.startswith('TADPOLE_Submission_Leaderboard')]
fileListLdb.sort()
print('fileListLdb ', fileListLdb)
os.system('mkdir -p %s' % ldbSubmissionsFld)
nrEntries = len(fileListLdb)
evalResFile = '%s/evalResAll.npz' % ldbSubmissionsFld
entriesList = range(nrEntries)
# entriesList = [0,1,2]
if args.runPart[0] == 'R':
evalResults = pd.DataFrame(np.nan, index=range(nrEntries), columns=('TEAMNAME', 'RANK' , 'MAUC', 'BCA',
'adasMAE', 'ventsMAE', 'adasWES', 'ventsWES', 'adasCPA', 'ventsCPA'))
lb4Df = pd.read_csv('TADPOLE_LB4.csv')
lb4Df = lb4Df[lb4Df['LB4'] == 1] # only keep the LB4 entries
lb4Df.reset_index(drop=True, inplace=True)
fileDatesRemote = []
indexInTable = 0
for f in entriesList:
fileName = fileListLdb[f]
remotePath = '%s/%s' % (uploadsFldRemote, fileName)
localPath = '%s/%s' % (ldbSubmissionsFld, fileName)
ldbDropbox.download(localPath, remotePath)
metadataFileRemote = ldbDropbox.dbx.files_get_metadata(remotePath)
fileDatesRemote += [metadataFileRemote.server_modified]
print('Evaluating %s' % fileName)
forecastDf = pd.read_csv(localPath)
try:
evalResults.loc[f, ['MAUC', 'BCA',
'adasMAE', 'ventsMAE', 'adasWES', 'ventsWES', 'adasCPA', 'ventsCPA']] = \
evalOneSubmission.evalOneSub(lb4Df, forecastDf)
except :
print('Error while processing submission %s' % fileName)
pass
if not np.isnan(evalResults['MAUC'].iloc[f]):
teamName = fileName.split('.')[0][len('TADPOLE_Submission_Leaderboard_'):]
print('teamname ', teamName)
evalResults.loc[f, 'TEAMNAME'] = teamName
nanMask = np.isnan(evalResults['MAUC'])
evalResults = evalResults[np.logical_not(nanMask)]
evalResults.reset_index(drop = True, inplace = True)
# # compute the ranks using MAUC
# rankOrder = np.argsort(evalResults.as_matrix(columns = ['MAUC']).reshape(-1))[::-1] # sort them by MAUC
# rankOrder += 1 # make them start from 1
# print('ranks', evalResults['MAUC'], rankOrder, evalResults.as_matrix(columns = ['MAUC']).reshape(-1))
# for f in range(evalResults.shape[0]):
# evalResults.loc[f, 'RANK'] = rankOrder[f]
dataStruct = dict(evalResults=evalResults, fileDatesRemote=fileDatesRemote)
pickle.dump(dataStruct, open(evalResFile, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
else:
dataStruct = pickle.load(open(evalResFile, 'rb'))
fileDatesRemote = dataStruct['fileDatesRemote']
evalResults = dataStruct['evalResults']
# compute the ranks using MAUC
rankOrder = np.argsort(evalResults.as_matrix(columns=['MAUC']).reshape(-1))[::-1] # sort them by MAUC
rankOrder = np.argsort(rankOrder) + 1 # make them start from 1
print('ranks', evalResults['MAUC'], rankOrder, evalResults.as_matrix(columns=['MAUC']).reshape(-1), np.argsort(rankOrder))
for f in range(evalResults.shape[0]):
evalResults.loc[f, 'RANK'] = rankOrder[f]
print('evalResults before\n', evalResults)
evalResults = evalResults.sort_values(by=['MAUC', 'BCA'],ascending=False)
evalResults = evalResults.reset_index(drop=True)
print('evalResults after\n', evalResults)
htmlFileFullPathRemote = '%s/%s' % (dropboxRemoteFolder, htmlFile)
htmlFileFullPathLocal = '%s/%s' % (ldbSubmissionsFld, htmlFile)
writeHTMLtable(evalResults, htmlFileFullPathLocal, fileListLdb, fileDatesRemote)
ldbDropbox.upload(htmlFileFullPathLocal, htmlFileFullPathRemote)
submission_table.to_csv(outputFile, index=False)
print('Evaluate predictions')
from datetime import datetime
d4Df = pd.read_csv('./TADPOLE_D4_corr.csv')
d4Df['CognitiveAssessmentDate'] = [
datetime.strptime(x, '%Y-%m-%d') for x in d4Df['CognitiveAssessmentDate']
]
d4Df['ScanDate'] = [datetime.strptime(x, '%Y-%m-%d') for x in d4Df['ScanDate']]
mapping = {'CN': 0, 'MCI': 1, 'AD': 2}
d4Df.replace({'Diagnosis': mapping}, inplace=True)
import evalOneSubmission as eos
mAUC, bca, adasMAE, ventsMAE, adasWES, ventsWES, adasCPA, ventsCPA = eos.evalOneSub(
d4Df, submission_table)
print('Diagnosis:')
print('mAUC = ' + "%0.3f" % mAUC)
print('BAC = ' + "%0.3f" % bca)
print('ADAS:')
print('MAE = ' + "%0.3f" % adasMAE)
print('WES = ' + "%0.3f" % adasWES)
print('CPA = ' + "%0.3f" % adasCPA)
print('VENTS:')
print('MAE = ' + "%0.3e" % ventsMAE)
print('WES = ' + "%0.3e" % ventsWES)
print('CPA = ' + "%0.3f" % ventsCPA)
output = pd.DataFrame(o1,
columns=[
'RID', 'Forecast Month', 'Forecast Date',
'CN relative probability',
'MCI relative probability', 'AD relative probability'
])
output['Forecast Month'] = output['Forecast Month'].astype(int)
output['Forecast Date'] = ym1
str_out_final = os.path.join(
str_exp, 'IntermediateData',
'TADPOLE_Submission_Leaderboard_BenchmarkSVM.csv')
output.to_csv(str_out_final, header=True, index=False)
print('Evaluate predictions')
R = pd.read_csv('./TADPOLE_LB4.csv')
import evalOneSubmission as eos
mAUC, bca = eos.evalOneSub(R, output)
print('Diagnosis:')
print('mAUC = ' + "%0.3f" % mAUC, )
print('BAC = ' + "%0.3f" % bca)
# print('ADAS:')
# print('MAE = ' + "%0.3f" % adasMAE, )
# print('WES = ' + "%0.3f" % adasWES,)
# print('CPA = ' + "%0.3f" % adasCPA )
# print('VENTS:')
# print('MAE = ' + "%0.3e" % ventsMAE,)
# print('WES = ' + "%0.3e" % ventsWES,)
# print('CPA = ' + "%0.3f" % ventsCPA )