def submit(ctx, file, annotation, large, public, private):
"""
For uploading submission files to evalai:
- Invoked by running 'evalai challenge CHALLENGE phase PHASE submit --file FILE'
- For large files, add a '--large' option at the end of the command
For uploading test annotation files to evalai:
- Invoked by running "evalai challenge CHALLENGE phase PHASE submit --file FILE --annotation"
Arguments:
ctx (class click.Context) -- The context object which holds state of the invocation
file (str) -- the path of the file to be uploaded
annotations (boolean) -- flag to denote if file is a test annotation file
large (boolean) -- flag to denote if submission file is large (if large, presigned urls are used for uploads)
public (boolean) -- flag to denote if submission is public
private (boolean) -- flag to denote if submission is private
Returns:
None
"""
if public and private:
message = "\nError: Submission can't be public and private.\nPlease select either --public or --private"
notify_user(message, color="red")
else:
if annotation:
upload_file_using_presigned_url(ctx.phase_id, file, "annotation")
else:
submission_metadata = {}
if public:
submission_metadata["is_public"] = json.dumps(True)
elif private:
submission_metadata["is_public"] = json.dumps(False)
else:
submission_metadata["is_public"] = None
if click.confirm("Do you want to include the Submission Details?"):
submission_metadata["method_name"] = click.prompt(
style("Method Name", fg="yellow"), type=str, default=""
)
submission_metadata["method_description"] = click.prompt(
style("Method Description", fg="yellow"), type=str, default=""
)
submission_metadata["project_url"] = click.prompt(
style("Project URL", fg="yellow"), type=str, default=""
)
submission_metadata["publication_url"] = click.prompt(
style("Publication URL", fg="yellow"), type=str, default=""
)
if large:
upload_file_using_presigned_url(ctx.phase_id, file, "submission", submission_metadata)
else:
make_submission(ctx.challenge_id, ctx.phase_id, file, submission_metadata)
def participate(ctx, team):
"""
Participate in a challenge.
"""
"""
Invoked by running `evalai challenge CHALLENGE participate TEAM`
"""
terms_and_conditions_page_url = "{}{}".format(get_host_url(), URLS.terms_and_conditions_page.value)
terms_and_conditions_page_url = terms_and_conditions_page_url.format(ctx.challenge_id)
message = "Please refer challenge terms and conditions here: {}" \
"\n\nBy agreeing to participate in the challenge, you are agreeing to terms and conditions." \
"\n\nDo you accept challenge terms and conditions?".format(terms_and_conditions_page_url)
if click.confirm(message):
participate_in_a_challenge(ctx.challenge_id, team)
else:
message = "\nYou can't participate in the challenge without accepting terms and conditions"
notify_user(message, color="red")
def submit(ctx, file, annotation, large, public, private):
"""
For uploading submission files to evalai:
- Invoked by running 'evalai challenge CHALLENGE phase PHASE submit --file FILE'
- For large files, add a '--large' option at the end of the command
For uploading test annotation files to evalai:
- Invoked by running "evalai challenge CHALLENGE phase PHASE submit --file FILE --annotation"
Arguments:
ctx (class click.Context) -- The context object which holds state of the invocation
file (str) -- the path of the file to be uploaded
annotations (boolean) -- flag to denote if file is a test annotation file
large (boolean) -- flag to denote if submission file is large (if large, presigned urls are used for uploads)
public (boolean) -- flag to denote if submission is public
private (boolean) -- flag to denote if submission is private
Returns:
None
"""
if public and private:
message = "\nError: Submission can't be public and private.\nPlease select either --public or --private"
notify_user(message, color="red")
else:
if annotation:
upload_file_using_presigned_url(ctx.phase_id, file, "annotation")
else:
submission_metadata = {}
if public:
submission_metadata["is_public"] = json.dumps(True)
elif private:
submission_metadata["is_public"] = json.dumps(False)
else:
submission_metadata["is_public"] = None
if click.confirm("Do you want to include the Submission Details?"):
submission_metadata["method_name"] = click.prompt(style(
"Method Name", fg="yellow"),
type=str,
default="")
submission_metadata["method_description"] = click.prompt(
style("Method Description", fg="yellow"),
type=str,
default="",
)
submission_metadata["project_url"] = click.prompt(style(
"Project URL", fg="yellow"),
type=str,
default="")
submission_metadata["publication_url"] = click.prompt(
style("Publication URL", fg="yellow"),
type=str,
default="")
submission_meta_attributes = get_submission_meta_attributes(
ctx.challenge_id, ctx.phase_id)
submission_attribute_metadata = []
if (submission_meta_attributes
and len(submission_meta_attributes) > 0):
if click.confirm(
"Do you want to include the Submission Metadata?"):
for attribute in submission_meta_attributes:
attribute_type = attribute["type"]
attribute_name = attribute["name"]
attribute_description = attribute["description"]
attribute_required = attribute.get("required")
if attribute_required:
attribute_name = attribute_name + '*'
value = None
message = "{} ({})".format(attribute_name,
attribute_description)
if attribute_type == "text":
while True:
value = click.prompt(
style(message, fg="yellow"),
type=str,
default="",
)
if not attribute_required or value != "":
break
echo("Error: {} is a required field".format(
attribute["name"]))
if attribute_type == "boolean":
while True:
value = click.prompt(style(message,
fg="yellow"),
type=bool,
default="")
if not attribute_required or value != "":
break
echo("Error: {} is a required field".format(
attribute["name"]))
if attribute_type == "radio":
while True:
value = click.prompt(style(
"{}:\nChoices:{}".format(
message, attribute["options"]),
fg="yellow",
),
type=click.Choice(
attribute["options"]),
default="")
if not attribute_required or value != "":
break
echo("Error: {} is a required field".format(
attribute["name"]))
if attribute_type == "checkbox":
option_chosen = True
while option_chosen:
value = []
choices = click.prompt(style(
"{}:\nChoices(separated by comma):{}".
format(message, attribute["options"]),
fg="yellow",
),
type=str,
show_default=False,
default="")
if choices != "":
choices = [
choice.strip(" ")
for choice in choices.split(",")
]
else:
choices = []
option_chosen = False
if attribute_required and len(choices) == 0:
echo(
"Error: {} is a required field. Please select atleast one option"
.format(attribute["name"]))
option_chosen = True
for choice in choices:
if choice in attribute["options"]:
value.append(choice)
option_chosen = False
else:
echo(
"Error: Choose correct value(s) from the given options only"
)
option_chosen = True
break
submission_attribute_metadata.append(
{attribute_name: value})
if large:
upload_file_using_presigned_url(ctx.phase_id, file,
"submission",
submission_metadata)
else:
make_submission(
ctx.challenge_id,
ctx.phase_id,
file,
submission_metadata,
submission_attribute_metadata,
)
def push(image, phase, url, public, private):
"""
Push docker image to a particular challenge phase.
"""
"""
Invoked by `evalai push IMAGE:TAG -p PHASE_ID`.
"""
if len(image.split(":")) != 2:
message = "\nError: Please enter the tag name with image.\n\nFor eg: `evalai push ubuntu:latest --phase 123`"
notify_user(message, color="red")
sys.exit(1)
if public and private:
message = "\nError: Submission can't be public and private.\nPlease select either --public or --private"
notify_user(message, color="red")
sys.exit(1)
submission_metadata = {}
if public:
submission_metadata["is_public"] = json.dumps(True)
elif private:
submission_metadata["is_public"] = json.dumps(False)
tag = str(uuid.uuid4())
docker_client = docker.from_env()
try:
docker_image = docker_client.images.get(image)
except docker.errors.ImageNotFound:
message = "\nError: Image not found. Please enter the correct image name and tag."
notify_user(message, color="red")
sys.exit(1)
request_path = URLS.phase_details_using_slug.value
request_path = request_path.format(phase)
response = make_request(request_path, "GET")
challenge_pk = response.get("challenge")
phase_pk = response.get("id")
request_path = URLS.challenge_details.value
request_path = request_path.format(challenge_pk)
response = make_request(request_path, "GET")
max_docker_image_size = response.get("max_docker_image_size")
docker_image_size = docker_image.__dict__.get("attrs").get("VirtualSize")
if docker_image_size > max_docker_image_size:
max_docker_image_size = convert_bytes_to(max_docker_image_size, "gb")
message = "\nError: Image is too large. The maximum image size allowed is {} GB".format(
max_docker_image_size)
notify_user(message, color="red")
sys.exit(1)
request_path = URLS.get_aws_credentials.value
request_path = request_path.format(phase_pk)
response = make_request(request_path, "GET")
federated_user = response["success"]["federated_user"]
repository_uri = response["success"]["docker_repository_uri"]
# Production Environment
if ENVIRONMENT == "PRODUCTION":
AWS_ACCOUNT_ID = federated_user["FederatedUser"][
"FederatedUserId"].split(":")[0]
AWS_SERVER_PUBLIC_KEY = federated_user["Credentials"]["AccessKeyId"]
AWS_SERVER_SECRET_KEY = federated_user["Credentials"][
"SecretAccessKey"]
SESSION_TOKEN = federated_user["Credentials"]["SessionToken"]
ecr_client = boto3.client(
"ecr",
region_name="us-east-1",
aws_access_key_id=AWS_SERVER_PUBLIC_KEY,
aws_secret_access_key=AWS_SERVER_SECRET_KEY,
aws_session_token=SESSION_TOKEN,
)
token = ecr_client.get_authorization_token(
registryIds=[AWS_ACCOUNT_ID])
ecr_client = boto3.client("ecr", region_name="us-east-1")
username, password = (base64.b64decode(
token["authorizationData"][0]
["authorizationToken"]).decode().split(":"))
registry = token["authorizationData"][0]["proxyEndpoint"]
docker_client.login(username,
password,
registry=registry,
dockercfg_path=os.getcwd())
# Development and Test Environment
else:
repository_uri = "{0}/{1}".format(url, repository_uri.split("/")[1])
# Tag and push docker image and create a submission if successfully pushed
docker_client.images.get(image).tag("{}:{}".format(repository_uri, tag))
for line in docker_client.images.push(repository_uri,
tag,
stream=True,
decode=True):
if line.get("status") in ["Pushing", "Pushed"
] and line.get("progress"):
print("{id}: {status} {progress}".format(**line))
elif line.get("errorDetail"):
error = line.get("error")
notify_user(error, color="red")
elif line.get("aux"):
aux = line.get("aux")
pushed_image_tag = aux["Tag"]
submitted_image_uri = "{}:{}".format(repository_uri,
pushed_image_tag)
BASE_TEMP_DIR = tempfile.mkdtemp()
data = {"submitted_image_uri": submitted_image_uri}
submission_file_path = os.path.join(BASE_TEMP_DIR,
"submission.json")
with open(submission_file_path, "w") as outfile:
json.dump(data, outfile)
request_path = URLS.make_submission.value
request_path = request_path.format(challenge_pk, phase_pk)
response = make_request(request_path,
"POST",
submission_file_path,
data=submission_metadata)
shutil.rmtree(BASE_TEMP_DIR)
else:
print(" ".join("{}: {}".format(k, v) for k, v in line.items()
if k != "progressDetail"))
def push(image, phase):
"""
Push docker image to a particular challenge phase.
"""
"""
Invoked by `evalai push IMAGE:TAG -p PHASE_ID`.
"""
if len(image.split(":")) != 2:
message = "\nError: Please enter the tag name with image.\n\nFor eg: `evalai push ubuntu:latest --phase 123`"
notify_user(message, color="red")
sys.exit(1)
tag = image[1]
docker_client = docker.from_env()
try:
docker_image = docker_client.images.get(image)
except docker.errors.ImageNotFound:
message = (
"\nError: Image not found. Please enter the correct image name and tag."
)
notify_user(message, color="red")
sys.exit(1)
request_path = URLS.challenge_phase_details.value
request_path = request_path.format(phase)
response = make_request(request_path, "GET")
challenge_pk = response.get('challenge')
request_path = URLS.challenge_details.value
request_path = request_path.format(challenge_pk)
response = make_request(request_path, "GET")
max_docker_image_size = response.get('max_docker_image_size')
docker_image_size = docker_image.__dict__.get('attrs').get('VirtualSize')
if docker_image_size > max_docker_image_size:
max_docker_image_size = convert_bytes_to(max_docker_image_size, 'gb')
message = (
"\nError: Image is too large. The maximum image size allowed is {} GB".format(max_docker_image_size)
)
notify_user(message, color="red")
sys.exit(1)
request_path = URLS.get_aws_credentials.value
request_path = request_path.format(phase)
response = make_request(request_path, "GET")
federated_user = response["success"]["federated_user"]
repository_uri = response["success"]["docker_repository_uri"]
AWS_ACCOUNT_ID = federated_user["FederatedUser"]["FederatedUserId"].split(":")[0]
AWS_SERVER_PUBLIC_KEY = federated_user["Credentials"]["AccessKeyId"]
AWS_SERVER_SECRET_KEY = federated_user["Credentials"]["SecretAccessKey"]
SESSION_TOKEN = federated_user["Credentials"]["SessionToken"]
ecr_client = boto3.client(
"ecr",
region_name="us-east-1",
aws_access_key_id=AWS_SERVER_PUBLIC_KEY,
aws_secret_access_key=AWS_SERVER_SECRET_KEY,
aws_session_token=SESSION_TOKEN,
)
token = ecr_client.get_authorization_token(registryIds=[AWS_ACCOUNT_ID])
ecr_client = boto3.client("ecr", region_name="us-east-1")
username, password = (
base64.b64decode(token["authorizationData"][0]["authorizationToken"])
.decode()
.split(":")
)
registry = token["authorizationData"][0]["proxyEndpoint"]
docker_client.login(username, password, registry=registry, dockercfg_path=os.getcwd())
# Tag and push docker image and create a submission if successfully pushed
docker_client.images.get(image).tag("{}:{}".format(repository_uri, tag))
for line in docker_client.images.push(
repository_uri, tag, stream=True, decode=True
):
if line.get("status") in ["Pushing", "Pushed"] and line.get("progress"):
print("{id}: {status} {progress}".format(**line))
elif line.get("errorDetail"):
error = line.get("error")
notify_user(error, color="red")
elif line.get('aux'):
aux = line.get('aux')
pushed_image_tag = aux['Tag']
submitted_image_uri = '{}:{}'.format(repository_uri, pushed_image_tag)
BASE_TEMP_DIR = tempfile.mkdtemp()
data = {
"submitted_image_uri": submitted_image_uri
}
submission_file_path = os.path.join(BASE_TEMP_DIR, 'submission.json')
with open(submission_file_path, 'w') as outfile:
json.dump(data, outfile)
request_path = URLS.make_submission.value
request_path = request_path.format(challenge_pk, phase)
response = make_request(request_path, "POST", submission_file_path)
shutil.rmtree(BASE_TEMP_DIR)
else:
print(
" ".join(
"{}: {}".format(k, v)
for k, v in line.items()
if k != "progressDetail"
)
)
