def test_get_leaderboard_only_new_submissions(session_toy_db, leaderboard_type,
expected_html):
# only check that the submission should be shown as new when the
# dispatcher was not started.
if expected_html is not None:
assert get_leaderboard(session_toy_db, leaderboard_type, 'iris_test')
else:
assert (get_leaderboard(
session_toy_db, leaderboard_type, 'iris_test') is expected_html)
def test_update_leaderboard_functions(session_toy_function):
event_name = 'iris_test'
user_name = 'test_user'
for leaderboard_type in ['public', 'private', 'failed',
'public competition', 'private competition']:
leaderboard = get_leaderboard(session_toy_function, leaderboard_type,
event_name)
assert leaderboard is None
leaderboard = get_leaderboard(session_toy_function, 'new', event_name)
assert leaderboard
event = get_event(session_toy_function, event_name)
assert event.private_leaderboard_html is None
assert event.public_leaderboard_html_with_links is None
assert event.public_leaderboard_html_no_links is None
assert event.failed_leaderboard_html is None
assert event.public_competition_leaderboard_html is None
assert event.private_competition_leaderboard_html is None
assert event.new_leaderboard_html
event_team = get_event_team_by_name(session_toy_function, event_name,
user_name)
assert event_team.leaderboard_html is None
assert event_team.failed_leaderboard_html is None
assert event_team.new_leaderboard_html
event_teams = (session_toy_function.query(EventTeam)
.filter_by(event=event)
.all())
for et in event_teams:
assert et.leaderboard_html is None
assert et.failed_leaderboard_html is None
assert et.new_leaderboard_html
# run the dispatcher to process the different submissions
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
dispatcher = Dispatcher(
config, event_config, n_workers=-1, hunger_policy='exit'
)
dispatcher.launch()
session_toy_function.commit()
update_leaderboards(session_toy_function, event_name)
event = get_event(session_toy_function, event_name)
assert event.private_leaderboard_html
assert event.public_leaderboard_html_with_links
assert event.public_leaderboard_html_no_links
assert event.failed_leaderboard_html
assert event.public_competition_leaderboard_html
assert event.private_competition_leaderboard_html
assert event.new_leaderboard_html is None
update_user_leaderboards(session_toy_function, event_name, user_name)
event_team = get_event_team_by_name(session_toy_function, event_name,
user_name)
assert event_team.leaderboard_html
assert event_team.failed_leaderboard_html
assert event_team.new_leaderboard_html is None
update_all_user_leaderboards(session_toy_function, event_name)
event_teams = (session_toy_function.query(EventTeam)
.filter_by(event=event)
.all())
for et in event_teams:
assert et.leaderboard_html
assert et.failed_leaderboard_html
assert et.new_leaderboard_html is None
def test_get_leaderboard(session_toy_db):
leaderboard_new = get_leaderboard(session_toy_db, 'new', 'iris_test')
assert leaderboard_new.count('<tr>') == 6
leaderboard_new = get_leaderboard(session_toy_db, 'new', 'iris_test',
'test_user')
assert leaderboard_new.count('<tr>') == 3
# run the dispatcher to process the different submissions
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
dispatcher = Dispatcher(
config, event_config, n_workers=-1, hunger_policy='exit'
)
dispatcher.launch()
session_toy_db.commit()
assert get_leaderboard(session_toy_db, 'new', 'iris_test') is None
# the iris dataset has a single submission which is failing
leaderboard_failed = get_leaderboard(session_toy_db, 'failed', 'iris_test')
assert leaderboard_failed.count('<tr>') == 2
leaderboard_failed = get_leaderboard(session_toy_db, 'failed', 'iris_test',
'test_user')
assert leaderboard_failed.count('<tr>') == 1
# the remaining submission should be successful
leaderboard_public = get_leaderboard(session_toy_db, 'public', 'iris_test')
assert leaderboard_public.count('<tr>') == 4
leaderboard_public = get_leaderboard(session_toy_db, 'public', 'iris_test',
'test_user')
assert leaderboard_public.count('<tr>') == 2
leaderboard_private = get_leaderboard(session_toy_db, 'private',
'iris_test')
assert leaderboard_private.count('<tr>') == 4
leaderboard_private = get_leaderboard(session_toy_db, 'private',
'iris_test', 'test_user')
assert leaderboard_private.count('<tr>') == 2
# the competition leaderboard will have the best solution for each user
competition_public = get_leaderboard(session_toy_db, 'public competition',
'iris_test')
assert competition_public.count('<tr>') == 2
competition_private = get_leaderboard(session_toy_db,
'private competition', 'iris_test')
assert competition_private.count('<tr>') == 2
# check the difference between the public and private leaderboard
assert leaderboard_private.count('<td>') > leaderboard_public.count('<td>')
for private_term in ['bag', 'mean', 'std', 'private']:
assert private_term not in leaderboard_public
assert private_term in leaderboard_private
# check the column name in each leaderboard
assert """<th>team</th>
<th>submission</th>
<th>bag public acc</th>
<th>bag public error</th>
<th>bag public nll</th>
<th>bag public f1_70</th>
<th>bag private acc</th>
<th>bag private error</th>
<th>bag private nll</th>
<th>bag private f1_70</th>
<th>mean public acc</th>
<th>mean public error</th>
<th>mean public nll</th>
<th>mean public f1_70</th>
<th>mean private acc</th>
<th>mean private error</th>
<th>mean private nll</th>
<th>mean private f1_70</th>
<th>std public acc</th>
<th>std public error</th>
<th>std public nll</th>
<th>std public f1_70</th>
<th>std private acc</th>
<th>std private error</th>
<th>std private nll</th>
<th>std private f1_70</th>
<th>contributivity</th>
<th>historical contributivity</th>
<th>train time [s]</th>
<th>test time [s]</th>
<th>max RAM [MB]</th>
<th>submitted at (UTC)</th>""" in leaderboard_private
assert """<th>team</th>
<th>submission</th>
<th>acc</th>
<th>error</th>
<th>nll</th>
<th>f1_70</th>
<th>contributivity</th>
<th>historical contributivity</th>
<th>train time [s]</th>
<th>test time [s]</th>
<th>max RAM [MB]</th>
<th>submitted at (UTC)</th>""" in leaderboard_public
assert """<th>team</th>
<th>submission</th>
<th>submitted at (UTC)</th>
<th>error</th>""" in leaderboard_failed
# check the same for the competition leaderboard
assert """<th>rank</th>
<th>team</th>
<th>submission</th>
<th>acc</th>
<th>train time [s]</th>
<th>test time [s]</th>
<th>submitted at (UTC)</th>""" in competition_public
assert """<th>rank</th>
<th>move</th>
<th>team</th>
<th>submission</th>
<th>acc</th>
<th>train time [s]</th>
<th>test time [s]</th>
<th>submitted at (UTC)</th>""" in competition_private