class SC2BSubmissions(SubmissionTools, ST2): def __init__(self, client=None, version=2): super(SC2BSubmissions, self).__init__(client=client, name="SC2B") self.version = version from hpn import HPNAdmin self.hpn = HPNAdmin(client=self.client) # download missing file automatically if needed. from dreamtools import Challenge c = Challenge('D8C1') c._download_data('experimental.zip', 'syn1920412') def load_submissions(self, startweek=0, endweek=9, keep_latest=True): """Loads all SCORED submissions from SC2A Attaches the week, status, ranking and zscores """ # load all scored submissions self.submissions = self.hpn.get_submissions_prediction_insilico(status="SCORED") print("Got %s SCORED submissions" % len(self.submissions)) # attach week and filter the submissions self.submissions = self.attach_week_to_submissions(self.submissions,"sc2b") self.submissions = [sub for sub in self.submissions if sub['week']<=endweek] self.submissions = [sub for sub in self.submissions if sub['week']>=startweek] print("Keeping %s submissions in the week range requested" % len(self.submissions)) if keep_latest: self.submissions = self._keep_latest_only() print("Keep %s latest scored submissions " % len(self.submissions)) # attach the zscore print("attaching submissions") self.submissions = self.attach_status_to_submissions(self.submissions) print("remove some users") self.remove_users() print("attaching scores and compute final ranking") self._attach_zscores() # attach zscores print("all submissions available in the **submissions** attribute") for sub in self.submissions: rmse = json.loads(sub['substatus']['report']) sub['old_rmses'] = copy.deepcopy(rmse) #def remove_users(self, userIds=["375805", "1991105", "1971259"]): def remove_users(self, userIds=["375805", "1991105"]): """ * 375805 alphabeta is a test from TC * 1991105 sakve from week 5 has different id from sakev week 6. renmove week5 that has a lower score anyway * HD systems see SC1A function docstring 1971259 """ submissions = [x for x in self.submissions if x['userId'] not in userIds] self.submissions = submissions def _get_ranking(self): ranking = scoring.HPNScoringPredictionInsilico_ranking() for i,sub in enumerate(self.submissions): rmse = json.loads(sub['substatus']['report']) filename = self.client.getSubmission(sub, downloadFile=True, ifcollision="keep.local")['filePath'] s = scoring.HPNScoringPredictionInsilico(filename, version=self.version) s.compute_all_rmse() rmse = copy.deepcopy(s.rmse) ranking.add_rmse(rmse, sub['submitterAlias'] +"_"+ str(i)) return ranking def _attach_zscores(self): """attach mean zscore, 32 individual zscores """ ranking = self._get_ranking() zscores = ranking.get_mean_zscores() ranks = ranking.get_mean_ranks() for i,participant in enumerate(ranking.participants): # mean zscore self.submissions[i]['zscore'] = zscores[participant] # final rank self.submissions[i]['ranking'] = ranks[participant] rmse = ranking.rmse[i] self.submissions[i]['rmses'] = rmse all_rmse = [rmse[k1][k2] for k1 in rmse.keys() for k2 in rmse[k1].keys()] all_rmse = [x for x in all_rmse if np.isnan(x)==False] # exclude mean_rmse = np.mean(all_rmse) self.submissions[i]['mean_rmse'] = mean_rmse def summary_final(self, show=True): # an alias subs = self.submissions # sorted indices of the mean ranks ranks = np.argsort([sub['ranking'] for sub in subs]) teams = [subs[rank]['submitterAlias'] for rank in ranks] df = pd.DataFrame(index=teams, data=ranks, columns=['mean Rank']) # finally the ranks df['Team Name'] = [subs[rank]['submitterAlias'] for rank in ranks] df['Team Id'] = [subs[rank]['userId'] for rank in ranks] df['Submission Id'] = [subs[rank]['substatus']['id'] for rank in ranks] df['Entity Id'] = [subs[rank]['substatus']['entityId'] for rank in ranks] df['Mean Rank'] = [subs[rank]['ranking'] for rank in ranks] df['Mean RMSE'] = [subs[rank]['mean_rmse'] for rank in ranks] ranks = df['Mean Rank'].rank() df['Final Rank'] = ranks.values df = df.set_index('Final Rank') df = df[['Team Name', u'Team Id', u'Submission Id', u'Entity Id', 'Mean Rank', u'Mean RMSE']] ranks = np.argsort([sub['ranking'] for sub in subs]) if show is False: return df print(dataframe_towiki(df)) return df def save_rmse_to_json(self): super(SC2BSubmissions, self).save_rmse_to_json(N1=231, N2=254)
class SC2BSubmissions(SubmissionTools): def __init__(self, client=None, version='official'): super(SC2BSubmissions, self).__init__(client=client, name="SC2B") from hpn import HPNAdmin self.hpn = HPNAdmin(client=self.client) self.version = version # download missing file automatically if needed. from dreamtools import Challenge c = Challenge('D8C1') c._download_data('experimental.zip', 'syn1920412') def load_submissions(self, startweek=0, endweek=9, keep_latest=True): """Loads all SCORED submissions from SC2A Attaches the week, status, ranking and zscores """ # load all scored submissions self.submissions = self.hpn.get_submissions_prediction_insilico(status="SCORED") print("Got %s SCORED submissions" % len(self.submissions)) # attach week and filter the submissions self.submissions = self.attach_week_to_submissions(self.submissions,"sc2b") self.submissions = [sub for sub in self.submissions if sub['week']<=endweek] self.submissions = [sub for sub in self.submissions if sub['week']>=startweek] print("Keeping %s submissions in the week range requested" % len(self.submissions)) if keep_latest: self.submissions = self._keep_latest_only() print("Keep %s latest scored submissions " % len(self.submissions)) # attach the zscore print("attaching submissions") self.submissions = self.attach_status_to_submissions(self.submissions) print("remove some users") self.remove_users() print("attaching scores and compute final ranking") self._attach_zscores() # attach zscores print("all submissions available in the **submissions** attribute") for sub in self.submissions: rmse = json.loads(sub['substatus']['report']) sub['old_rmses'] = copy.deepcopy(rmse) #def remove_users(self, userIds=["375805", "1991105", "1971259"]): def remove_users(self, userIds=["375805", "1991105"]): """ * 375805 alphabeta is a test from TC * 1991105 sakve from week 5 has different id from sakev week 6. renmove week5 that has a lower score anyway * HD systems see SC1A function docstring 1971259 """ submissions = [x for x in self.submissions if x['userId'] not in userIds] self.submissions = submissions def _get_ranking(self): ranking = scoring.HPNScoringPredictionInsilico_ranking() for i,sub in enumerate(self.submissions): rmse = json.loads(sub['substatus']['report']) filename = self.client.getSubmission(sub, downloadFile=True, ifcollision="keep.local")['filePath'] s = scoring.HPNScoringPredictionInsilico(filename, version=self.version) s.compute_all_rmse() rmse = copy.deepcopy(s.rmse) ranking.add_rmse(rmse, sub['submitterAlias'] +"_"+ str(i)) return ranking def _attach_zscores(self): """attach mean zscore, 32 individual zscores """ ranking = self._get_ranking() zscores = ranking.get_mean_zscores() ranks = ranking.get_mean_ranks() for i,participant in enumerate(ranking.participants): # mean zscore self.submissions[i]['zscore'] = zscores[participant] # final rank self.submissions[i]['ranking'] = ranks[participant] rmse = ranking.rmse[i] self.submissions[i]['rmses'] = rmse all_rmse = [rmse[k1][k2] for k1 in rmse.keys() for k2 in rmse[k1].keys()] all_rmse = [x for x in all_rmse if np.isnan(x)==False] # exclude mean_rmse = np.mean(all_rmse) self.submissions[i]['mean_rmse'] = mean_rmse def summary(self, show="all"): ranks = np.argsort([sub['ranking'] for sub in self.submissions]) print("| Rank | User Id | Submitted Alias | Week | created on | status | RMSE | zscore | mean Rank |") for i, rank in enumerate(ranks): sub = self.submissions[rank] print("|%s | %10s | %5s |%5s |%s|%s| %10.6s | %10.6s |%10.6s |" % (i+1, sub['userId'], sub['submitterAlias'],sub['week'], sub['substatus']['status'], sub['createdOn'], sub['mean_rmse'],sub['ranking'],sub['zscore'])) def summary_final(self): ranks = np.argsort([sub['ranking'] for sub in self.submissions]) print("| Rank | Team name | SynapseId Id | mean RMSE | mean Rank | mean zscore |") print("|----|-----|-----|----|----|------|") results = {} for i, rank in enumerate(ranks): sub = self.submissions[rank] print("|%s | %20s | %20s | %20s | %20s | %10.6s | %10.6s |%10.6s |" % (i+1, sub['submitterAlias'], sub['userId'], sub['substatus']['id'], sub['substatus']['entityId'], sub['mean_rmse'],sub['ranking'],sub['zscore'])) results[sub['submitterAlias']] = i+1 return results