def test_method_validation(evaluation_image):
""" The validator should set the correct sha256 and set the ready bit """
container, sha256 = evaluation_image
method = MethodFactory(image__from_path=container)
# The method factory fakes the sha256 on creation
assert method.image_sha256 != sha256
assert method.ready == False
validate_method_async(method_pk=method.pk)
method = Method.objects.get(pk=method.pk)
assert method.image_sha256 == sha256
assert method.ready == True
def test_method_detail(client, TwoChallengeSets):
method = MethodFactory(
challenge=TwoChallengeSets.ChallengeSet1.challenge,
creator=TwoChallengeSets.ChallengeSet1.admin,
)
validate_admin_only_view(
viewname="evaluation:method-detail",
two_challenge_set=TwoChallengeSets,
reverse_kwargs={"pk": method.pk},
client=client,
)
def challenge_set_with_evaluation(ChallengeSet):
""" Creates a challenge with two methods.
To use this you must mark the test with @pytest.mark.django_db """
EvalChallengeSet = namedtuple(
"EvalChallengeSet", ["ChallengeSet", "method"]
)
ChallengeSet.challenge.use_evaluation = True
ChallengeSet.challenge.save()
method = MethodFactory(
challenge=ChallengeSet.challenge, creator=ChallengeSet.creator
)
return EvalChallengeSet(ChallengeSet, method)
def challenge_set_with_evaluation(challenge_set):
"""
Creates a challenge with two methods.
To use this you must mark the test with `@pytest.mark.django_db`.
"""
eval_challenge_set = namedtuple("eval_challenge_set",
["challenge_set", "method"])
challenge_set.challenge.use_evaluation = True
challenge_set.challenge.save()
method = MethodFactory(challenge=challenge_set.challenge,
creator=challenge_set.creator)
return eval_challenge_set(challenge_set, method)
def test_method_validation_not_a_docker_tar(submission_file):
""" Upload something that isnt a docker file should be invalid """
method = MethodFactory(image__from_path=submission_file)
assert method.ready == False
with pytest.raises(ValidationError):
validate_docker_image_async(pk=method.pk,
app_label=method._meta.app_label,
model_name=method._meta.model_name)
method = Method.objects.get(pk=method.pk)
assert method.ready == False
assert 'manifest.json not found' in method.status
def test_method_validation_invalid_dockefile(alpine_images):
""" Uploading two images in a tar archive should fail """
method = MethodFactory(image__from_path=alpine_images)
assert method.ready == False
with pytest.raises(ValidationError):
validate_docker_image_async(pk=method.pk,
app_label=method._meta.app_label,
model_name=method._meta.model_name)
method = Method.objects.get(pk=method.pk)
assert method.ready == False
assert 'should only have 1 image' in method.status
def test_method_validation_root_dockerfile(root_image):
"""Uploading two images in a tar archive should fail."""
method = MethodFactory(image__from_path=root_image)
assert method.ready is False
with pytest.raises(ValidationError):
validate_docker_image(
pk=method.pk,
app_label=method._meta.app_label,
model_name=method._meta.model_name,
)
method = Method.objects.get(pk=method.pk)
assert method.ready is False
assert "runs as root" in method.status
def test_submission_evaluation(
client, evaluation_image, submission_file, settings
):
# Override the celery settings
settings.task_eager_propagates = (True,)
settings.task_always_eager = (True,)
settings.broker_url = ("memory://",)
settings.backend = "memory"
# Upload a submission and create a job
dockerclient = docker.DockerClient(
base_url=settings.CONTAINER_EXEC_DOCKER_BASE_URL
)
eval_container, sha256 = evaluation_image
method = MethodFactory(
image__from_path=eval_container, image_sha256=sha256, ready=True
)
# We should not be able to download methods
response = client.get(method.image.url)
assert response.status_code == 404
num_containers_before = len(dockerclient.containers.list())
num_volumes_before = len(dockerclient.volumes.list())
# This will create a job, and we'll wait for it to be executed
submission = SubmissionFactory(
file__from_path=submission_file, challenge=method.challenge
)
# The evaluation method should clean up after itself
assert len(dockerclient.volumes.list()) == num_volumes_before
assert len(dockerclient.containers.list()) == num_containers_before
# The evaluation method should return the correct answer
assert len(submission.job_set.all()) == 1
assert submission.job_set.all()[0].result.metrics["acc"] == 0.5
# Try with a csv file
submission = SubmissionFactory(
file__from_path=Path(__file__).parent / "resources" / "submission.csv",
challenge=method.challenge,
)
assert len(submission.job_set.all()) == 1
assert submission.job_set.all()[0].result.metrics["acc"] == 0.5
def test_job_detail(client, TwoChallengeSets):
method = MethodFactory(
challenge=TwoChallengeSets.ChallengeSet1.challenge,
creator=TwoChallengeSets.ChallengeSet1.admin,
ready=True,
)
submission = SubmissionFactory(
challenge=TwoChallengeSets.ChallengeSet1.challenge,
creator=TwoChallengeSets.ChallengeSet1.participant,
)
job = JobFactory(method=method, submission=submission)
validate_admin_only_view(
viewname="evaluation:job-detail",
two_challenge_set=TwoChallengeSets,
reverse_kwargs={"pk": job.pk},
client=client,
)
def test_submission_evaluation(client, evaluation_image, submission_file,
settings):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
# Upload a submission and create an evaluation
dockerclient = docker.DockerClient(
base_url=settings.COMPONENTS_DOCKER_BASE_URL)
eval_container, sha256 = evaluation_image
method = MethodFactory(image__from_path=eval_container,
image_sha256=sha256,
ready=True)
# We should not be able to download methods
with pytest.raises(NotImplementedError):
_ = method.image.url
num_containers_before = len(dockerclient.containers.list())
num_volumes_before = len(dockerclient.volumes.list())
# This will create an evaluation, and we'll wait for it to be executed
submission = SubmissionFactory(predictions_file__from_path=submission_file,
challenge=method.challenge)
# The evaluation method should clean up after itself
assert len(dockerclient.volumes.list()) == num_volumes_before
assert len(dockerclient.containers.list()) == num_containers_before
# The evaluation method should return the correct answer
assert len(submission.evaluation_set.all()) == 1
assert (submission.evaluation_set.first().outputs.get(
interface__slug="metrics-json-file").value["acc"] == 0.5)
# Try with a csv file
submission = SubmissionFactory(
predictions_file__from_path=Path(__file__).parent / "resources" /
"submission.csv",
challenge=method.challenge,
)
assert len(submission.evaluation_set.all()) == 1
assert (submission.evaluation_set.first().outputs.get(
interface__slug="metrics-json-file").value["acc"] == 0.5)
def test_method_validation(evaluation_image):
"""The validator should set the correct sha256 and set the ready bit."""
container, sha256 = evaluation_image
method = MethodFactory(image__from_path=container)
# The method factory fakes the sha256 on creation
assert method.image_sha256 != sha256
assert method.ready is False
validate_docker_image(
pk=method.pk,
app_label=method._meta.app_label,
model_name=method._meta.model_name,
)
method = Method.objects.get(pk=method.pk)
assert method.image_sha256 == sha256
assert method.ready is True
def test_submission_evaluation(client, evaluation_image, submission_file):
# Upload a submission and create a job
dockerclient = docker.DockerClient(base_url=settings.DOCKER_BASE_URL)
user = UserFactory()
submission = SubmissionFactory(file__from_path=submission_file,
creator=user)
eval_container, sha256 = evaluation_image
method = MethodFactory(image__from_path=eval_container,
image_sha256=sha256,
ready=True)
# We should not be able to download methods
response = client.get(method.image.url)
assert response.status_code == 403
job = JobFactory(submission=submission, method=method)
num_containers_before = len(dockerclient.containers.list())
num_volumes_before = len(dockerclient.volumes.list())
res = evaluate_submission(job=job)
# The evaluation method should return the correct answer
assert res["acc"] == 0.5
# The evaluation method should clean up after itself
assert len(dockerclient.volumes.list()) == num_volumes_before
assert len(dockerclient.containers.list()) == num_containers_before
# Try with a csv file
submission = SubmissionFactory(
file__from_path=Path(__file__).parent / 'resources' / 'submission.csv',
creator=user,
)
job = JobFactory(submission=submission, method=method)
res = evaluate_submission(job=job)
assert res["acc"] == 0.5
