def _create_demo_challenge(self):
demo = Challenge.objects.create(
short_name="demo",
description="demo project",
creator=self.users["demo"],
use_evaluation=True,
hidden=False,
display_forum_link=True,
)
demo.add_participant(self.users["demop"])
Page.objects.create(challenge=demo,
title="all",
permission_level="ALL")
Page.objects.create(challenge=demo,
title="reg",
permission_level="REG")
Page.objects.create(challenge=demo,
title="adm",
permission_level="ADM")
Phase.objects.create(challenge=demo, title="Phase 2")
for phase_num, phase in enumerate(demo.phase_set.all()):
phase.score_title = "Accuracy ± std"
phase.score_jsonpath = "acc.mean"
phase.score_error_jsonpath = "acc.std"
phase.extra_results_columns = [{
"title": "Dice ± std",
"path": "dice.mean",
"error_path": "dice.std",
"order": "desc",
}]
phase.submission_kind = phase.SubmissionKind.ALGORITHM
phase.save()
method = Method(phase=phase, creator=self.users["demo"])
container = ContentFile(base64.b64decode(b""))
method.image.save("test.tar", container)
method.save()
submission = Submission(phase=phase, creator=self.users["demop"])
content = ContentFile(base64.b64decode(b""))
submission.predictions_file.save("test.csv", content)
submission.save()
e = Evaluation.objects.create(submission=submission,
method=method,
status=Evaluation.SUCCESS)
e.create_result(
result={
"acc": {
"mean": 0.1 * phase_num,
"std": 0.1
},
"dice": {
"mean": 0.71,
"std": 0.05
},
})
def _create_demo_challenge(self):
demo = Challenge.objects.create(
short_name="demo",
description="demo project",
creator=self.users["demo"],
use_evaluation=True,
hidden=False,
)
demo.add_participant(self.users["demop"])
Page.objects.create(challenge=demo,
title="all",
permission_level="ALL")
Page.objects.create(challenge=demo,
title="reg",
permission_level="REG")
Page.objects.create(challenge=demo,
title="adm",
permission_level="ADM")
method = Method(challenge=demo, creator=self.users["demo"])
container = ContentFile(base64.b64decode(b""))
method.image.save("test.tar", container)
method.save()
submission = Submission(challenge=demo, creator=self.users["demop"])
content = ContentFile(base64.b64decode(b""))
submission.file.save("test.csv", content)
submission.save()
e = Evaluation.objects.create(submission=submission,
method=method,
status=Evaluation.SUCCESS)
e.create_result(
result={
"acc": {
"mean": 0.5,
"std": 0.1
},
"dice": {
"mean": 0.71,
"std": 0.05
},
})
demo.evaluation_config.score_title = "Accuracy ± std"
demo.evaluation_config.score_jsonpath = "acc.mean"
demo.evaluation_config.score_error_jsonpath = "acc.std"
demo.evaluation_config.extra_results_columns = [{
"title": "Dice ± std",
"path": "dice.mean",
"error_path": "dice.std",
"order": "desc",
}]
demo.evaluation_config.save()
def _create_challenge(
*, creator, participant, archive, suffix, inputs, outputs
):
c = Challenge.objects.create(
short_name=f"algorithm-evaluation-{suffix}",
creator=creator,
hidden=False,
logo=create_uploaded_image(),
)
c.add_participant(participant)
p = c.phase_set.first()
p.algorithm_inputs.set(inputs)
p.algorithm_outputs.set(outputs)
p.title = "Algorithm Evaluation"
p.submission_kind = p.SubmissionKind.ALGORITHM
p.archive = archive
p.score_jsonpath = "score"
p.submission_limit = 10
p.save()
m = Method(creator=creator, phase=p)
with _uploaded_container_image() as container:
m.image.save("algorithm_io.tar", container)
def handle(self, *args, **options):
"""
Creates the main project, demo user and demo challenge
"""
# Set the default domain that is used in RequestFactory
site = Site.objects.get(pk=settings.SITE_ID)
site.domain = "gc.localhost"
site.save()
challenge, created = Challenge.objects.get_or_create(
short_name=settings.MAIN_PROJECT_NAME,
description="main project",
use_registration_page=False,
disclaimer=
"You <b>must</b> delete the admin, demo, and demop users before deploying to production!",
)
if created:
Page.objects.create(
title="home",
challenge=challenge,
hidden=True,
html=
'<h1>\r\n\tDefault Page</h1>\r\n<p>\r\n\tYou almost have COMIC framework up and running. Congratulations.<br />\r\n\tFor up to date info, please visit <a href="https://github.com/comic/grand-challenge.org">https://github.com/comic/grand-challenge.org</a></p>\r\n<p>\r\n\tOn COMIC framework anyone can sign up and create a project. A project is a collection of pages, data and software under a single name.<br />\r\n\tThe page you are reading now is the page "home" in the project "comic". The project "comic" is special in two ways: First, it is shown by default if you navigate to the root url and second, it\'s pages appear as a menu below each and every page of the framework. To make a project other than \'comic\' the main project, change the MAIN_PROJECT_NAME setting in comic/settings/00_default.conf.</p>\r\n<h2>\r\n\tUseful Code</h2>\r\n<p>\r\n\tAt the base level a project in COMIC framework is a collection of html pages under a single header. Most if the interesting functionality comes from using django <em>template tags</em> which you can use as functions inside you html code. They look like { % render_graph results.csv % } and are rendered by django when encountered. A list of template tags can be found on page "<a href="../template_tags">template tags</a>"</p>\r\n<p>\r\n\t\tcode for a sign in/ create new project button on a page:</p>\r\n<p>\r\n\t </p>\r\n<div style="text-align: center;background-color:#DFDFDF;float:none;margin-left:10px;margin-right:16px;margin-top:1px;padding:12px;width:164px;">\r\n\t\r\n\t<p>\r\n\t\t <a class="createNewComicSiteLink" href="{% url \'challenges:create\' %}">Create a new project</a></div>\r\n<p>There is a <a href=\'/site/testsite/\'>test site here</a>.</p><p> <h1>Site Stats</h1><div class=\'row no-gutters\'><div class=\'col-sm-4\'></div><div class=\'col-sm-4\'>{% allusers_statistics False %}</div><div class=\'col-sm-4\'></div></div> </p>\r\n',
)
Page.objects.create(
title="template_tags",
display_title="template tags",
challenge=challenge,
html=
'<p>\r\n\tBelow is a list of all template tags which can be used on any page of the framework. For example, the table below is rendered by inserting { % taglist % } into the html of the page. Some template tags use additional arguments. These are described in the right column.</p>\r\n<p>\r\n\t </p>\r\n<p>\r\n\t{% taglist %}</p>\r\n<p>\r\n\t </p>\r\n<p>\r\n\tIn addition, any <a href="https://docs.djangoproject.com/en/dev/ref/templates/builtins/">built-in django template tag</a> is also available.</p>\r\n',
)
demoadmin = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
demoparticipant = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
adminuser = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
adminuser.is_staff = True
adminuser.save()
demo = Challenge.objects.create(
short_name="demo",
description="demo project",
creator=demoadmin,
use_evaluation=True,
hidden=False,
)
demo.add_participant(demoparticipant)
Page.objects.create(challenge=demo,
title="all",
permission_lvl="ALL")
Page.objects.create(challenge=demo,
title="reg",
permission_lvl="REG")
Page.objects.create(challenge=demo,
title="adm",
permission_lvl="ADM")
method = Method(challenge=demo, creator=demoadmin)
container = ContentFile(base64.b64decode(b""))
method.image.save("test.tar", container)
method.save()
submission = Submission(challenge=demo, creator=demoparticipant)
content = ContentFile(base64.b64decode(b""))
submission.file.save("test.csv", content)
submission.save()
job = Job.objects.create(submission=submission, method=method)
Result.objects.create(
challenge=demo,
metrics={
"acc": {
"mean": 0.5,
"std": 0.1
},
"dice": {
"mean": 0.71,
"std": 0.05
},
},
job=job,
)
demo.evaluation_config.score_title = "Accuracy ± std"
demo.evaluation_config.score_jsonpath = "acc.mean"
demo.evaluation_config.score_error_jsonpath = "acc.std"
demo.evaluation_config.extra_results_columns = [{
"title": "Dice ± std",
"path": "dice.mean",
"error_path": "dice.std",
"order": "desc",
}]
demo.evaluation_config.save()
ex_challenge = ExternalChallenge.objects.create(
creator=demoadmin,
homepage="https://www.example.com",
short_name="EXAMPLE2018",
title="Example External Challenge 2018",
description="An example of an external challenge",
event_name="Example Event",
event_url="https://www.example.com/2018",
publication_journal_name="Nature",
publication_url="https://doi.org/10.1038/s41586-018-0367-9",
hidden=False,
)
TaskType.objects.create(type="Segmentation")
TaskType.objects.create(type="Classification")
regions_structures = {
"Head and Neck": ["Brain", "Teeth"],
"Thorax": ["Lung"],
"Cardiac": ["Heart"],
"Abdomen": ["Liver", "Pancreas", "Kidney", "Spleen"],
"Pelvis": ["Prostate", "Cervix"],
"Spine": ["Spinal Cord"],
"Upper Limb": ["Hand"],
"Lower Limb": ["Knee"],
}
for region, structures in regions_structures.items():
r = BodyRegion.objects.create(region=region)
for structure in structures:
BodyStructure.objects.create(structure=structure, region=r)
modalities = (
"CT",
"MR",
"XR",
"PET",
"PET-CT",
"PET-MR",
"Mammography",
"CT-MR",
"US",
"TEM",
"Histology",
)
for modality in modalities:
ImagingModality.objects.create(modality=modality)
mr_modality = ImagingModality.objects.get(modality="MR")
ex_challenge.modalities.add(mr_modality)
ex_challenge.save()
def _create_demo_challenge(self):
demo = Challenge.objects.create(
short_name="demo",
description="demo project",
creator=self.users["demo"],
use_evaluation=True,
hidden=False,
display_forum_link=True,
)
demo.add_participant(self.users["demop"])
Page.objects.create(challenge=demo,
title="all",
permission_level="ALL")
Page.objects.create(challenge=demo,
title="reg",
permission_level="REG")
Page.objects.create(challenge=demo,
title="adm",
permission_level="ADM")
Phase.objects.create(challenge=demo, title="Phase 2")
for phase_num, phase in enumerate(demo.phase_set.all()):
phase.score_title = "Accuracy ± std"
phase.score_jsonpath = "acc.mean"
phase.score_error_jsonpath = "acc.std"
phase.extra_results_columns = [{
"title": "Dice ± std",
"path": "dice.mean",
"error_path": "dice.std",
"order": "desc",
}]
phase.submission_kind = phase.SubmissionKind.ALGORITHM
phase.evaluation_detail_observable_url = (
phase.evaluation_comparison_observable_url
) = "https://observablehq.com/embed/@grand-challenge/data-fetch-example?cell=*"
phase.save()
method = Method(phase=phase, creator=self.users["demo"])
container = ContentFile(base64.b64decode(b""))
method.image.save("test.tar", container)
method.save()
submission = Submission(phase=phase, creator=self.users["demop"])
content = ContentFile(base64.b64decode(b""))
submission.predictions_file.save("test.csv", content)
submission.save()
e = Evaluation.objects.create(
submission=submission,
method=method,
status=Evaluation.SUCCESS,
)
def create_result(evaluation, result: dict):
interface = ComponentInterface.objects.get(
slug="metrics-json-file")
try:
output_civ = evaluation.outputs.get(interface=interface)
output_civ.value = result
output_civ.save()
except ObjectDoesNotExist:
output_civ = ComponentInterfaceValue.objects.create(
interface=interface, value=result)
evaluation.outputs.add(output_civ)
create_result(
e,
{
"acc": {
"mean": 0.1 * phase_num,
"std": 0.1
},
"dice": {
"mean": 0.71,
"std": 0.05
},
},
)
def handle(self, *args, **options):
"""
Creates the main project, demo user and demo challenge
"""
# Set the default domain that is used in RequestFactory
site = Site.objects.get(pk=settings.SITE_ID)
site.domain = "gc.localhost"
site.save()
challenge, created = Challenge.objects.get_or_create(
short_name=settings.MAIN_PROJECT_NAME,
description="main project",
use_registration_page=False,
disclaimer="You <b>must</b> delete the admin, demo, demop, and retina_demo users before deploying to production!",
)
if created:
Page.objects.create(
title="home",
challenge=challenge,
hidden=True,
html='<h1>\r\n\tDefault Page</h1>\r\n<p>\r\n\tYou almost have COMIC framework up and running. Congratulations.<br />\r\n\tFor up to date info, please visit <a href="https://github.com/comic/grand-challenge.org">https://github.com/comic/grand-challenge.org</a></p>\r\n<p>\r\n\tOn COMIC framework anyone can sign up and create a project. A project is a collection of pages, data and software under a single name.<br />\r\n\tThe page you are reading now is the page "home" in the project "comic". The project "comic" is special in two ways: First, it is shown by default if you navigate to the root url and second, it\'s pages appear as a menu below each and every page of the framework. To make a project other than \'comic\' the main project, change the MAIN_PROJECT_NAME setting in comic/settings/00_default.conf.</p>\r\n<h2>\r\n\tUseful Code</h2>\r\n<p>\r\n\tAt the base level a project in COMIC framework is a collection of html pages under a single header. Most if the interesting functionality comes from using django <em>template tags</em> which you can use as functions inside you html code. They look like { % render_graph results.csv % } and are rendered by django when encountered. A list of template tags can be found on page "<a href="../template_tags">template tags</a>"</p>\r\n<p>\r\n\t\tcode for a sign in/ create new project button on a page:</p>\r\n<p>\r\n\t </p>\r\n<div style="text-align: center;background-color:#DFDFDF;float:none;margin-left:10px;margin-right:16px;margin-top:1px;padding:12px;width:164px;">\r\n\t\r\n\t<p>\r\n\t\t <a class="createNewComicSiteLink" href="{% url \'challenges:create\' %}">Create a new project</a></div>\r\n<p>There is a <a href=\'/site/testsite/\'>test site here</a>.</p><p> <h1>Site Stats</h1><div class=\'row no-gutters\'><div class=\'col-sm-4\'></div><div class=\'col-sm-4\'>{% allusers_statistics False %}</div><div class=\'col-sm-4\'></div></div> </p>\r\n',
)
Page.objects.create(
title="template_tags",
display_title="template tags",
challenge=challenge,
html='<p>\r\n\tBelow is a list of all template tags which can be used on any page of the framework. For example, the table below is rendered by inserting { % taglist % } into the html of the page. Some template tags use additional arguments. These are described in the right column.</p>\r\n<p>\r\n\t </p>\r\n<p>\r\n\t{% taglist %}</p>\r\n<p>\r\n\t </p>\r\n<p>\r\n\tIn addition, any <a href="https://docs.djangoproject.com/en/dev/ref/templates/builtins/">built-in django template tag</a> is also available.</p>\r\n',
)
demoadmin = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
demoparticipant = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
adminuser = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
adminuser.is_staff = True
adminuser.save()
demo = Challenge.objects.create(
short_name="demo",
description="demo project",
creator=demoadmin,
use_evaluation=True,
hidden=False,
)
demo.add_participant(demoparticipant)
Page.objects.create(
challenge=demo, title="all", permission_lvl="ALL"
)
Page.objects.create(
challenge=demo, title="reg", permission_lvl="REG"
)
Page.objects.create(
challenge=demo, title="adm", permission_lvl="ADM"
)
method = Method(challenge=demo, creator=demoadmin)
container = ContentFile(base64.b64decode(b""))
method.image.save("test.tar", container)
method.save()
submission = Submission(challenge=demo, creator=demoparticipant)
content = ContentFile(base64.b64decode(b""))
submission.file.save("test.csv", content)
submission.save()
job = Job.objects.create(submission=submission, method=method)
Result.objects.create(
metrics={
"acc": {"mean": 0.5, "std": 0.1},
"dice": {"mean": 0.71, "std": 0.05},
},
job=job,
)
demo.evaluation_config.score_title = "Accuracy ± std"
demo.evaluation_config.score_jsonpath = "acc.mean"
demo.evaluation_config.score_error_jsonpath = "acc.std"
demo.evaluation_config.extra_results_columns = [
{
"title": "Dice ± std",
"path": "dice.mean",
"error_path": "dice.std",
"order": "desc",
}
]
demo.evaluation_config.save()
ex_challenge = ExternalChallenge.objects.create(
creator=demoadmin,
homepage="https://www.example.com",
short_name="EXAMPLE2018",
title="Example External Challenge 2018",
description="An example of an external challenge",
event_name="Example Event",
event_url="https://www.example.com/2018",
publication_journal_name="Nature",
publication_url="https://doi.org/10.1038/s41586-018-0367-9",
hidden=False,
)
TaskType.objects.create(type="Segmentation")
TaskType.objects.create(type="Classification")
regions_structures = {
"Head and Neck": ["Brain", "Teeth"],
"Thorax": ["Lung"],
"Cardiac": ["Heart"],
"Abdomen": ["Liver", "Pancreas", "Kidney", "Spleen"],
"Pelvis": ["Prostate", "Cervix"],
"Spine": ["Spinal Cord"],
"Upper Limb": ["Hand"],
"Lower Limb": ["Knee"],
}
for region, structures in regions_structures.items():
r = BodyRegion.objects.create(region=region)
for structure in structures:
BodyStructure.objects.create(structure=structure, region=r)
modalities = (
"CT",
"MR",
"XR",
"PET",
"PET-CT",
"PET-MR",
"Mammography",
"CT-MR",
"US",
"TEM",
"Histology",
)
for modality in modalities:
ImagingModality.objects.create(modality=modality)
mr_modality = ImagingModality.objects.get(modality="MR")
ex_challenge.modalities.add(mr_modality)
ex_challenge.save()
retina_demo = UserenaSignup.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
active=True,
)
retina_group = Group.objects.get(
name=settings.RETINA_GRADERS_GROUP_NAME
)
retina_demo.groups.add(retina_group)