def get_participant_or_host_team_challenges(url, teams):
"""
Returns the challenges corresponding to the participant or host teams
"""
challenges = []
for team in teams:
header = get_request_header()
try:
response = requests.get(url.format(team["id"]), headers=header)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code == 401:
validate_token(response.json())
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
challenges += response["results"]
return challenges
def display_challenges(url):
"""
Function to fetch & display the challenge list based on API
"""
header = get_request_header()
try:
response = requests.get(url, headers=header)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code == 401:
validate_token(response.json())
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
challenges = response["results"]
if len(challenges) != 0:
pretty_print_challenge_data(challenges)
else:
echo("Sorry, no challenges found.")
def submission_details_request(submission_id):
"""
Function to request details of a particular submission
"""
url = "{}{}".format(get_host_url(), URLS.get_submission.value)
url = url.format(submission_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenge CHALLENGE phase PHASE submissions` "
"to view your submission.\n".format(
response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
return response
def get_participant_or_host_teams(url):
"""
Returns the participant or host teams corresponding to the user
"""
header = get_request_header()
try:
response = requests.get(url, headers=header)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code == 401:
validate_token(response.json())
echo(style(err, fg="red", bold=True))
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
return response["results"]
def make_submission(challenge_id, phase_id, file, submission_metadata={}):
"""
Function to submit a file to a challenge
"""
url = "{}{}".format(get_host_url(), URLS.make_submission.value)
url = url.format(challenge_id, phase_id)
headers = get_request_header()
input_file = {"input_file": file}
data = {"status": "submitting"}
data = dict(data, **submission_metadata)
try:
response = requests.post(url,
headers=headers,
files=input_file,
data=data)
file.close()
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
"active phases.\n".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
if "input_file" in response.json():
echo(style(response.json()["input_file"][0], fg="red", bold=True))
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
echo(
style(
"\nYour file {} with the ID {} is successfully submitted.\n".
format(file.name, response["id"]),
fg="green",
bold=True,
))
echo(
style(
"You can use `evalai submission {}` to view this submission's status.\n"
.format(response["id"]),
bold=True,
fg="white"))
def display_challenge_phase_list(challenge_id):
"""
Function to display all challenge phases for a particular challenge.
"""
url = URLS.challenge_phase_list.value
url = "{}{}".format(get_host_url(), url)
url = url.format(challenge_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}".format(response.json()["error"]),
fg="red",
bold=True,
))
echo(
style(
"\nUse `evalai challenges` to fetch the active challenges.",
fg="red",
bold=True,
))
echo(
style(
"\nUse `evalai challenge CHALLENGE phases` to fetch the active phases.\n",
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
challenge_phases = response["results"]
pretty_print_all_challenge_phases(challenge_phases)
def participate_in_a_challenge(challenge_id, participant_team_id):
"""
Function to participate in a particular challenge.
"""
url = "{}{}".format(get_host_url(), URLS.participate_in_a_challenge.value)
url = url.format(challenge_id, participant_team_id)
headers = get_request_header()
headers["Content-Type"] = "application/json"
try:
response = requests.post(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai teams` to fetch your participant "
"teams.\n".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
if response.status_code == 201:
echo(
style(
"Your team id {} is now participating in this challenge.".
format(participant_team_id),
fg="green",
bold=True,
))
def display_challenge_phase_detail(challenge_id, phase_id, is_json):
"""
Function to print details of a challenge phase.
"""
url = URLS.challenge_phase_detail.value
url = "{}{}".format(get_host_url(), url)
url = url.format(challenge_id, phase_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
"active phases.\n".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
phase = response
if is_json:
phase_json = json.dumps(phase, indent=4, sort_keys=True)
echo(phase_json)
else:
pretty_print_challenge_phase_data(phase)
def display_ongoing_challenge_list():
"""
Displays the list of ongoing challenges from the backend
"""
url = "{}{}".format(get_host_url(), URLS.challenge_list.value)
header = get_request_header()
try:
response = requests.get(url, headers=header)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code == 401:
validate_token(response.json())
echo(style(err, fg="red", bold=True))
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
challenges = response["results"]
# Filter out past/unapproved/unpublished challenges.
challenges = list(
filter(
lambda challenge: validate_date_format(challenge["end_date"]) >
datetime.now() and challenge["approved_by_admin"] and challenge[
"published"],
challenges,
))
if len(challenges) != 0:
pretty_print_challenge_data(challenges)
else:
echo(style(
"Sorry, no challenges found.",
fg="red",
bold=True,
))
def get_submission_meta_attributes(challenge_id, phase_id):
"""
Function to get all submission_meta_attributes for a challenge phase
Parameters:
challenge_id (int): id of challenge to which submission is made
phase_id (int): id of challenge phase to which submission is made
Returns:
list: list of objects of submission meta attributes
"""
url = "{}{}".format(get_host_url(), URLS.challenge_phase_detail.value)
url = url.format(challenge_id, phase_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
"active phases.\n".format(response.json()),
fg="red",
bold=True,
)
)
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
)
)
sys.exit(1)
response = response.json()
return response["submission_meta_attributes"]
def display_leaderboard(challenge_id, phase_split_id):
"""
Function to display the Leaderboard of a particular CPS.
"""
url = "{}{}".format(get_host_url(), URLS.leaderboard.value)
url = url.format(phase_split_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"Error: {}".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(style(err, fg="red", bold=True))
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
results = response["results"]
if len(results) != 0:
attributes = results[0]["leaderboard__schema"]["labels"]
pretty_print_leaderboard_data(attributes, results)
else:
echo(
"Sorry, no Leaderboard results found.",
bold=True,
fg="red",
)
def display_challenge_phase_split_list(challenge_id):
"""
Function to display Challenge Phase Splits of a particular challenge.
"""
url = URLS.challenge_phase_split_detail.value
url = "{}{}".format(get_host_url(), url)
url = url.format(challenge_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
"active phases.\n".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(style(err, fg="red", bold=True))
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
phase_splits = response.json()
if len(phase_splits) != 0:
pretty_print_challenge_phase_split_data(phase_splits)
else:
echo(
style("Sorry, no Challenge Phase Splits found.",
fg="red",
bold=True))
def display_my_submission_details(challenge_id, phase_id, start_date, end_date):
"""
Function to display the details of a particular submission.
"""
url = URLS.my_submissions.value
url = "{}{}".format(get_host_url(), url)
url = url.format(challenge_id, phase_id)
headers = get_request_header()
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
"active phases.\n".format(response.json()["error"]),
fg="red",
bold=True,
)
)
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException as err:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
)
)
sys.exit(1)
response = response.json()
submissions = response["results"]
pretty_print_my_submissions_data(submissions, start_date, end_date)
def display_challenge_details(challenge):
"""
Function to display challenge details.
"""
url = URLS.challenge_details.value
url = "{}{}".format(get_host_url(), url)
url = url.format(challenge)
header = get_request_header()
try:
response = requests.get(url, headers=header)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}".format(response.json()["error"]),
fg="red",
bold=True,
))
echo(
style(
"\nUse `evalai challenges` to fetch the active challenges.\n",
fg="red",
bold=True,
))
else:
echo(style(err, fg="red", bold=True))
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
pretty_print_challenge_details(response)
def display_teams(is_host):
"""
Function to display all the participant or host teams of a user
"""
url = "{}{}"
headers = get_request_header()
if is_host:
url = url.format(get_host_url(), URLS.host_team_list.value)
else:
url = url.format(get_host_url(), URLS.participant_team_list.value)
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"Error: {}".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = response.json()
teams = response["results"]
if len(teams) != 0:
pretty_print_team_data(teams, is_host)
else:
echo("Sorry, no teams found.")
def download_file(url):
"""
Download the file using URL with bucket and key
"""
"""
Invoked by `evalai download_file URL`.
"""
parsed_url = urlparse.urlparse(url)
parsed_host_url = "{parsed_url.scheme}://{parsed_url.netloc}".format(
parsed_url=parsed_url)
is_correct_host = False
curr_host = get_host_url()
if parsed_host_url in EVALAI_API_URLS:
if parsed_host_url == curr_host:
is_correct_host = True
if is_correct_host:
bucket = urlparse.parse_qs(parsed_url.query).get("bucket")
key = urlparse.parse_qs(parsed_url.query).get("key")
if not bucket or not key:
echo(
style(
"\nThe bucket or key is missing in the url.\n",
fg="red",
bold=True,
))
sys.exit(1)
headers = get_request_header()
file_name = key[0].split("/")[-1]
URL = URLS.download_file.value
URL = "{}{}".format(get_host_url(), URL)
URL = URL.format(bucket[0], key[0])
try:
response = requests.get(URL, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException as e:
echo(
style(
"\nCould not connect to EvalAI API. Please check"
" the URL or if server is running\n",
bold=True,
fg="red",
))
sys.exit(1)
with open(file_name, "wb") as file:
total_file_length = int(response.headers.get("content-length"))
chunk_size = 1024
with click.progressbar(length=total_file_length,
label="Downloading file") as bar:
for data in response.iter_content(chunk_size=chunk_size):
file.write(data)
bar.update(chunk_size)
echo(
style(
"\nYour file {} is successfully downloaded.\n".format(
file_name),
fg="green",
bold=True,
))
else:
echo(
style(
"\nThe url doesn't match the EvalAI url. Please check the url.\n",
fg="red",
bold=True,
))
sys.exit(1)
def make_request(path, method, files=None, data=None):
url = "{}{}".format(get_host_url(), path)
headers = get_request_header()
if method == "GET":
try:
response = requests.get(url, headers=headers)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n".format(response.json().get("error")),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
return response.json()
elif method == "POST":
if files:
files = {"input_file": open(files, "rb")}
else:
files = None
data = {"status": "submitting"}
try:
response = requests.post(url,
headers=headers,
files=files,
data=data)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
echo(
style(
"\nError: {}\n"
"\nUse `evalai challenges` to fetch the active challenges.\n"
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
"active phases.\n".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
response = json.loads(response.text)
echo(
style(
"\nYour docker file is successfully submitted.\n",
fg="green",
bold=True,
))
echo(
style(
"You can use `evalai submission {}` to view this submission's status.\n"
.format(response.get("id")),
bold=True,
fg="white"))
return response
elif method == "PUT":
# TODO: Add support for PUT request
pass
elif method == "PATCH":
# TODO: Add support for PATCH request
pass
elif method == "DELETE":
# TODO: Add support for DELETE request
pass
def create_team(team_name, team_url, is_host):
"""
Function to create a new team by taking in the team name as input.
"""
url = "{}{}"
if is_host:
url = url.format(get_host_url(), URLS.create_host_team.value)
else:
url = url.format(get_host_url(), URLS.participant_team_list.value)
headers = get_request_header()
headers["Content-Type"] = "application/json"
data = {}
data["team_name"] = team_name
if team_url:
data["team_url"] = team_url
data = json.dumps(data)
try:
response = requests.post(url, headers=headers, data=data)
response.raise_for_status()
except requests.exceptions.HTTPError as err:
if response.status_code in EVALAI_ERROR_CODES:
validate_token(response.json())
if "team_name" in response.json().keys():
validate_token(response.json())
echo(
style(
"Error: {}".format(response.json()["team_name"][0]),
fg="red",
bold=True,
))
else:
echo(
style(
"Error: {}".format(response.json()["error"]),
fg="red",
bold=True,
))
else:
echo(err)
sys.exit(1)
except requests.exceptions.RequestException:
echo(
style(
"\nCould not establish a connection to EvalAI."
" Please check the Host URL.\n",
bold=True,
fg="red",
))
sys.exit(1)
if response.status_code == 201:
response = response.json()
if is_host:
echo(
style(
"\nYour host team {} was successfully created.\n".format(
response["team_name"]),
fg="green",
bold=True,
))
else:
echo(
style(
"\nYour participant team {} was successfully created.\n".
format(response["team_name"]),
fg="green",
bold=True,
))