def test_modify_label(seeded_database, client, test_project_data, test_queue,
test_labels, test_irr_queue, test_admin_queue):
'''
This tests the history table's ability to modify a label
'''
request_info = {"labelID": test_labels[0].pk, "labeling_time": 3}
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client)
# have a user annotate some data
data = get_assignments(client_profile, project, 1)[0]
assert data is not None
response = client.post('/api/annotate_data/' + str(data.pk) + '/',
request_info)
assert DataLabel.objects.filter(data=data).count() == 1
# call modify label to change it to something else
change_info = {
"dataID": data.pk,
"oldLabelID": test_labels[0].pk,
"labelID": test_labels[1].pk
}
response = client.post('/api/modify_label/' + str(data.pk) + '/',
change_info)
assert 'error' not in response.json() and 'detail' not in response.json()
# check that the label is updated and it's in the correct places
# check that there are no duplicate labels
assert DataLabel.objects.filter(data=data).count() == 1
assert DataLabel.objects.get(data=data).label.pk == test_labels[1].pk
# check it's in change log
assert LabelChangeLog.objects.filter(data=data).count() == 1
def test_modify_label_to_skip(seeded_database, client, test_project_data,
test_queue, test_irr_queue, test_labels,
test_admin_queue):
'''This tests the history table's ability to change labeled items
to skipped items.'''
request_info = {"labelID": test_labels[0].pk, "labeling_time": 3}
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client)
# have a user annotate some data
data = get_assignments(client_profile, project, 1)[0]
assert data is not None
response = client.post('/api/annotate_data/' + str(data.pk) + '/',
request_info)
assert DataLabel.objects.filter(data=data).count() == 1
# Call the change to skip function. Should now be in admin table, not be
# in history table.
change_info = {"dataID": data.pk, "oldLabelID": test_labels[0].pk}
response = client.post('/api/modify_label_to_skip/' + str(data.pk) + '/',
change_info)
assert 'error' not in response.json() and 'detail' not in response.json()
assert DataLabel.objects.filter(data=data).count() == 0
assert DataQueue.objects.filter(queue=test_admin_queue).count() == 1
# check it's in change log
assert LabelChangeLog.objects.filter(data=data,
new_label="skip").count() == 1
def test_coded_table(seeded_database, client, admin_client, test_project_data, test_queue, test_admin_queue, test_irr_queue, test_labels):
'''
This tests the table that displays the labeled table
'''
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
# first, check that it is empty
response = client.get('/api/data_coded_table/' + str(project.pk) + '/')
assert 'detail' in response.json() and "Invalid permission. Must be an admin" in response.json()[
'detail']
response = admin_client.get('/api/data_coded_table/' + str(project.pk) + '/').json()
assert len(response['data']) == 0
# label a few things, and check that they are in the table
data = get_assignments(client_profile, project, 3)
data_text = []
label_names = []
for i in range(3):
data_text.append(escape(data[i].text))
label_names.append(test_labels[i].name)
response = client.post('/api/annotate_data/' + str(data[i].pk) + '/', {
"labelID": test_labels[i].pk, "labeling_time": 1
})
response = admin_client.get('/api/data_coded_table/' + str(project.pk) + '/').json()
assert len(response['data']) == 3
for row in response['data']:
assert row['Text'] in data_text
assert row['Label'] in label_names
assert row['Coder'] == str(client_profile)
def test_predicted_table(seeded_database, admin_client, client, test_project_unlabeled_and_tfidf, test_queue, test_labels, test_irr_queue, test_admin_queue):
'''
This tests that the predicted table contains what it should
'''
project = test_project_unlabeled_and_tfidf
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
# first, check that it is empty
response = client.get('/api/data_predicted_table/' + str(project.pk) + '/')
assert 'detail' in response.json() and "Invalid permission. Must be an admin" in response.json()[
'detail']
response = admin_client.get('/api/data_predicted_table/' + str(project.pk) + '/').json()
assert len(response['data']) == 0
# label 15 things and check that it is still empty
data = get_assignments(client_profile, project, 15)
data_text = []
label_names = []
for i in range(15):
data_text.append(escape(data[i].text))
label_names.append(test_labels[i % 3].name)
response = client.post('/api/annotate_data/' + str(data[i].pk) + '/', {
"labelID": test_labels[i % 3].pk, "labeling_time": 1
})
response = admin_client.get('/api/data_predicted_table/' + str(project.pk) + '/').json()
assert len(response['data']) == 0
# label 15 more things and let the predictions be created
# check that the unlabeled items are in the table
data = get_assignments(client_profile, project, 15)
for i in range(15):
data_text.append(escape(data[i].text))
label_names.append(test_labels[i % 3].name)
response = client.post('/api/annotate_data/' + str(data[i].pk) + '/', {
"labelID": test_labels[i % 3].pk, "labeling_time": 1
})
response = admin_client.get('/api/data_predicted_table/' + str(project.pk) + '/').json()
training_set = TrainingSet.objects.get(
set_number=project.get_current_training_set().set_number - 1)
model = Model.objects.get(training_set=training_set)
# check that the table holds the predicted data
assert len(response['data']) == (DataPrediction.objects.filter(
data__project=project, model=model).count()) // len(test_labels)
# check that the table has the number of unlabeled data
assert len(response['data']) == (Data.objects.filter(
project=project).count() - DataLabel.objects.filter(data__project=project).count())
# check that the table does not have the labeled data
data_list = list(DataPrediction.objects.filter(data__project=project).values_list("data__text"))
for d in data_text:
assert d not in data_list
def test_unlabeled_table(
seeded_database,
client,
admin_client,
test_project_unlabeled_and_tfidf,
test_queue,
test_admin_queue,
test_irr_queue,
test_labels,
):
"""This tests that the unlabeled data table contains what it should."""
project = test_project_unlabeled_and_tfidf
# first, check that it has all the unlabeled data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client
)
response = client.get("/api/data_unlabeled_table/" + str(project.pk) + "/")
assert (
"detail" in response.json()
and "Invalid permission. Must be an admin" in response.json()["detail"]
)
response = admin_client.get(
"/api/data_unlabeled_table/" + str(project.pk) + "/"
).json()
assert "data" in response
assert (
len(response["data"])
== Data.objects.filter(project=project).count()
- DataQueue.objects.filter(data__project=project).count()
)
# label something. Check it is not in the table.
data = get_assignments(client_profile, project, 2)
response = client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{"labelID": test_labels[0].pk, "labeling_time": 1},
)
response = admin_client.get(
"/api/data_unlabeled_table/" + str(project.pk) + "/"
).json()
data_ids = [d["ID"] for d in response["data"]]
assert data[0].pk not in data_ids
# skip something. Check it is not in the table.
response = client.post("/api/skip_data/" + str(data[1].pk) + "/")
response = admin_client.get(
"/api/data_unlabeled_table/" + str(project.pk) + "/"
).json()
data_ids = [d["ID"] for d in response["data"]]
assert data[1].pk not in data_ids
def test_admin_table(
seeded_database,
admin_client,
client,
test_project_data,
test_queue,
test_irr_queue,
test_admin_queue,
test_labels,
):
"""This tests that the admin table holds the correct items."""
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client
)
# check that a non-admin can't get the table
response = client.get("/api/data_admin_table/" + str(project.pk) + "/").json()
assert (
"detail" in response
and "Invalid permission. Must be an admin" in response["detail"]
)
response = admin_client.get("/api/data_admin_table/" + str(project.pk) + "/").json()
# first, check that it is empty
assert len(response["data"]) == 0
# label something. Should still be empty.
data = get_assignments(client_profile, project, 2)
response = client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{"labelID": test_labels[0].pk, "labeling_time": 1},
)
response = admin_client.get("/api/data_admin_table/" + str(project.pk) + "/").json()
assert len(response["data"]) == 0
# skip something. Should be in the table.
response = client.post("/api/skip_data/" + str(data[1].pk) + "/")
response = admin_client.get("/api/data_admin_table/" + str(project.pk) + "/").json()
assert len(response["data"]) == 1
assert response["data"][0]["ID"] == data[1].pk
# admin annotate the data. Admin table should be empty again.
response = admin_client.post(
"/api/label_admin_label/" + str(data[1].pk) + "/",
{"labelID": test_labels[0].pk},
)
response = admin_client.get("/api/data_admin_table/" + str(project.pk) + "/").json()
assert len(response["data"]) == 0
def test_get_label_history(
seeded_database,
admin_client,
client,
test_project_data,
test_queue,
test_labels,
test_admin_queue,
test_irr_queue,
):
"""This tests the function that returns the elements that user has already
labeled."""
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client
)
# before anything has been labeled, the history table should be empty
response = admin_client.get("/api/get_label_history/" + str(project.pk) + "/")
assert response.json()["data"] == []
# skip an item. Should still be empty
data = get_assignments(client_profile, project, 2)
datum = data[0]
assert datum is not None
response = client.post("/api/skip_data/" + str(datum.pk) + "/")
assert "error" not in response.json() and "detail" not in response.json()
response = client.get("/api/get_label_history/" + str(project.pk) + "/")
assert response.json()["data"] == []
# have one user label something. Call label history on two users.
request_info = {"labelID": test_labels[0].pk, "labeling_time": 3}
datum = data[1]
response = client.post("/api/annotate_data/" + str(datum.pk) + "/", request_info)
assert "error" not in response.json() and "detail" not in response.json()
response_client = client.get("/api/get_label_history/" + str(project.pk) + "/")
assert response_client.json()["data"] != []
response_admin = admin_client.get("/api/get_label_history/" + str(project.pk) + "/")
assert response_admin.json()["data"] == []
# the label should be in the correct person's history
response_data = response_client.json()["data"][0]
assert response_data["id"] == datum.pk
assert response_data["labelID"] == test_labels[0].pk
def test_coded_table(
seeded_database,
client,
admin_client,
test_project_data,
test_queue,
test_admin_queue,
test_irr_queue,
test_labels,
):
"""This tests the table that displays the labeled table."""
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
# first, check that it is empty
response = client.get("/api/data_coded_table/" + str(project.pk) + "/")
assert ("detail" in response.json()
and "Invalid permission. Must be an admin"
in response.json()["detail"])
response = admin_client.get("/api/data_coded_table/" + str(project.pk) +
"/").json()
assert len(response["data"]) == 0
# label a few things, and check that they are in the table
data = get_assignments(client_profile, project, 3)
data_text = []
label_names = []
for i in range(3):
data_text.append(escape(data[i].text))
label_names.append(test_labels[i].name)
response = client.post(
"/api/annotate_data/" + str(data[i].pk) + "/",
{
"labelID": test_labels[i].pk,
"labeling_time": 1
},
)
response = admin_client.get("/api/data_coded_table/" + str(project.pk) +
"/").json()
assert len(response["data"]) == 3
for row in response["data"]:
assert row["Text"] in data_text
assert row["Label"] in label_names
assert row["Coder"] == str(client_profile)
def test_admin_table(seeded_database, admin_client, client, test_project_data,
test_queue, test_irr_queue, test_admin_queue,
test_labels):
'''
This tests that the admin table holds the correct items
'''
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
# check that a non-admin can't get the table
response = client.get('/api/data_admin_table/' + str(project.pk) +
'/').json()
assert 'detail' in response and 'Invalid permission. Must be an admin' in response[
'detail']
response = admin_client.get('/api/data_admin_table/' + str(project.pk) +
'/').json()
# first, check that it is empty
assert len(response['data']) == 0
# label something. Should still be empty.
data = get_assignments(client_profile, project, 2)
response = client.post('/api/annotate_data/' + str(data[0].pk) + '/', {
"labelID": test_labels[0].pk,
"labeling_time": 1
})
response = admin_client.get('/api/data_admin_table/' + str(project.pk) +
'/').json()
assert len(response['data']) == 0
# skip something. Should be in the table.
response = client.post('/api/skip_data/' + str(data[1].pk) + '/')
response = admin_client.get('/api/data_admin_table/' + str(project.pk) +
'/').json()
assert len(response['data']) == 1
assert response['data'][0]['ID'] == data[1].pk
# admin annotate the data. Admin table should be empty again.
response = admin_client.post(
'/api/label_admin_label/' + str(data[1].pk) + '/',
{"labelID": test_labels[0].pk})
response = admin_client.get('/api/data_admin_table/' + str(project.pk) +
'/').json()
assert len(response['data']) == 0
def test_admin_label(
seeded_database,
admin_client,
client,
test_project_data,
test_queue,
test_labels,
test_irr_queue,
test_admin_queue,
):
"""This tests the admin ability to label skipped items in the admin table."""
# fill queue. The admin queue should be empty
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client
)
assert DataQueue.objects.filter(queue=test_admin_queue).count() == 0
# have a normal client skip something and try to admin label. Should not
# be allowed
data = get_assignments(client_profile, project, 1)[0]
response = client.post("/api/skip_data/" + str(data.pk) + "/")
assert "error" not in response.json() and "detail" not in response.json()
payload = {"labelID": test_labels[0].pk}
response = client.post("/api/label_admin_label/" + str(data.pk) + "/", payload)
assert (
"detail" in response.json()
and "Invalid permission. Must be an admin" in response.json()["detail"]
)
# check datum is in proper places
assert DataQueue.objects.filter(data=data, queue=test_admin_queue).count() == 1
assert DataQueue.objects.filter(data=data, queue=test_queue).count() == 0
assert DataLabel.objects.filter(data=data).count() == 0
# Let admin label datum. Should work. Check it is now in proper places
response = admin_client.post(
"/api/label_admin_label/" + str(data.pk) + "/", payload
)
assert "error" not in response.json() and "detail" not in response.json()
assert DataQueue.objects.filter(data=data, queue=test_admin_queue).count() == 0
assert DataLabel.objects.filter(data=data).count() == 1
def test_label_distribution_inverted(seeded_database, admin_client, client,
test_project_data, test_queue,
test_irr_queue, test_labels,
test_admin_queue):
'''
This tests the api that produces the label counts chart for
the skew page. It is stacked differently than the previous.
'''
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
# at the beginning, should return empty list
response = client.get('/api/label_distribution/' + str(project.pk) + '/')
assert 'detail' in response.json(
) and "Invalid permission. Must be an admin" in response.json()['detail']
response = admin_client.get('/api/label_distribution/' + str(project.pk) +
'/')
assert len(response.json()) == 0
# have client label three things differently. Check values.
data = get_assignments(client_profile, project, 3)
response = client.post('/api/annotate_data/' + str(data[0].pk) + '/', {
"labelID": test_labels[0].pk,
"labeling_time": 3
})
response = client.post('/api/annotate_data/' + str(data[1].pk) + '/', {
"labelID": test_labels[1].pk,
"labeling_time": 3
})
response = client.post('/api/annotate_data/' + str(data[2].pk) + '/', {
"labelID": test_labels[2].pk,
"labeling_time": 3
})
assert DataLabel.objects.filter(data__in=data).count() == 3
response = admin_client.get('/api/label_distribution_inverted/' +
str(project.pk) + '/').json()
assert len(response) > 0
for row in response:
user = row['key']
temp_dict = row['values']
for label_row in temp_dict:
if user == str(client_profile):
assert label_row['y'] == 1
else:
assert user in [str(admin_profile), 'test_profile']
assert label_row['y'] == 0
# Have admin label three things the same. Check values.
data = get_assignments(admin_profile, project, 3)
response = admin_client.post('/api/annotate_data/' + str(data[0].pk) + '/',
{
"labelID": test_labels[0].pk,
"labeling_time": 3
})
response = admin_client.post('/api/annotate_data/' + str(data[1].pk) + '/',
{
"labelID": test_labels[0].pk,
"labeling_time": 3
})
response = admin_client.post('/api/annotate_data/' + str(data[2].pk) + '/',
{
"labelID": test_labels[0].pk,
"labeling_time": 3
})
response = admin_client.get('/api/label_distribution_inverted/' +
str(project.pk) + '/').json()
assert len(response) > 0
for row in response:
user = row['key']
temp_dict = row['values']
for label_row in temp_dict:
if user == str(client_profile):
assert label_row['y'] == 1
elif user == str(admin_profile):
if label_row['x'] == test_labels[0].name:
assert label_row['y'] == 3
else:
assert label_row['y'] == 0
else:
assert label_row['y'] == 0
def test_model_metrics(seeded_database, admin_client, client, test_project_unlabeled_and_tfidf, test_queue, test_admin_queue, test_irr_queue, test_labels):
'''
This function tests the model metrics api
'''
project = test_project_unlabeled_and_tfidf
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
# at the beginning, shouldn't have any
for metric in ['accuracy', 'f1', 'precision', 'recall']:
response = admin_client.get('/api/model_metrics/'
+ str(project.pk) + '/?metric=' + metric).json()
if len(response) == 1:
assert response[0]['key'] == 'Accuracy'
else:
assert len(response) == len(test_labels)
for temp_dict in response:
assert len(temp_dict['values']) == 0
# label 30 items. The model should run.
data = get_assignments(client_profile, project, 30)
for i in range(30):
response = client.post('/api/annotate_data/' + str(data[i].pk) + '/', {
"labelID": test_labels[i % 3].pk, "labeling_time": 1
})
assert DataLabel.objects.filter(data__in=data).count() == 30
# check that metrics were generated
# for metric in ['accuracy', 'f1', 'precision', 'recall']:
for metric in ['accuracy', 'f1', 'precision', 'recall']:
response = admin_client.get('/api/model_metrics/'
+ str(project.pk) + '/?metric=' + metric).json()
if len(response) == 1:
assert response[0]['key'] == 'Accuracy'
else:
assert len(response) == len(test_labels)
# check there is some value for the first run
for temp_dict in response:
assert len(temp_dict['values']) == 1
# do this again and check that a new metric is generated
fill_queue(test_queue, project.learning_method)
data = get_assignments(client_profile, project, 30)
for i in range(30):
response = client.post('/api/annotate_data/' + str(data[i].pk) + '/', {
"labelID": test_labels[i % 3].pk, "labeling_time": 1
})
fill_queue(test_queue, project.learning_method)
data = get_assignments(client_profile, project, 10)
for i in range(10):
response = client.post('/api/annotate_data/' + str(data[i].pk) + '/', {
"labelID": test_labels[i % 3].pk, "labeling_time": 1
})
for metric in ['accuracy', 'f1', 'precision', 'recall']:
response = admin_client.get('/api/model_metrics/'
+ str(project.pk) + '/?metric=' + metric).json()
if len(response) == 1:
assert response[0]['key'] == 'Accuracy'
else:
assert len(response) == len(test_labels)
# check there is some value for the first run
for temp_dict in response:
assert len(temp_dict['values']) == 2
def test_label_timing(seeded_database, admin_client, client, test_project_data, test_queue, test_admin_queue, test_irr_queue, test_labels):
# test that it starts out empty
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
response = client.get('/api/label_timing/' + str(project.pk) + '/')
assert 'detail' in response.json() and "Invalid permission. Must be an admin" in response.json()[
'detail']
response = admin_client.get('/api/label_timing/' + str(project.pk) + '/').json()
assert len(response['data']) == 0
assert response['yDomain'] == 0
# have the client label three data with time=1
data = get_assignments(client_profile, project, 3)
response = client.post('/api/annotate_data/' + str(data[0].pk) + '/', {
"labelID": test_labels[0].pk, "labeling_time": 1
})
response = client.post('/api/annotate_data/' + str(data[1].pk) + '/', {
"labelID": test_labels[1].pk, "labeling_time": 1
})
response = client.post('/api/annotate_data/' + str(data[2].pk) + '/', {
"labelID": test_labels[2].pk, "labeling_time": 1
})
# check that the quartiles are all the same
response = admin_client.get('/api/label_timing/' + str(project.pk) + '/').json()
assert len(response['data']) == 1
quarts = response['data'][0]['values']
assert quarts['Q1'] == 1 and quarts['Q2'] == 1 and quarts['Q3'] == 1
assert quarts['whisker_low'] == 1 and quarts['whisker_high'] == 1
assert response['yDomain'] == 11
# have the client label three data with 0, 3, 10. Check quartiles
data = get_assignments(client_profile, project, 3)
response = client.post('/api/annotate_data/' + str(data[0].pk) + '/', {
"labelID": test_labels[0].pk, "labeling_time": 0
})
response = client.post('/api/annotate_data/' + str(data[1].pk) + '/', {
"labelID": test_labels[1].pk, "labeling_time": 3
})
response = client.post('/api/annotate_data/' + str(data[2].pk) + '/', {
"labelID": test_labels[2].pk, "labeling_time": 10
})
response = admin_client.get('/api/label_timing/' + str(project.pk) + '/').json()
assert len(response['data']) == 1
quarts = response['data'][0]['values']
assert quarts['Q1'] == 1 and quarts['Q2'] == 1 and quarts['Q3'] == 3
assert quarts['whisker_low'] == 0 and quarts['whisker_high'] == 10
assert response['yDomain'] == 20
# have a client label with t=100. Check quartiles.
data = get_assignments(client_profile, project, 1)
response = client.post('/api/annotate_data/' + str(data[0].pk) + '/', {
"labelID": test_labels[0].pk, "labeling_time": 100
})
response = admin_client.get('/api/label_timing/' + str(project.pk) + '/').json()
quarts = response['data'][0]['values']
assert quarts['Q1'] == 1 and quarts['Q2'] == 1 and quarts['Q3'] == 10
assert quarts['whisker_low'] == 0 and quarts['whisker_high'] == 100
assert response['yDomain'] == 110
def test_label_timing(
seeded_database,
admin_client,
client,
test_project_data,
test_queue,
test_admin_queue,
test_irr_queue,
test_labels,
):
# test that it starts out empty
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
response = client.get("/api/label_timing/" + str(project.pk) + "/")
assert ("detail" in response.json()
and "Invalid permission. Must be an admin"
in response.json()["detail"])
response = admin_client.get("/api/label_timing/" + str(project.pk) +
"/").json()
assert len(response["data"]) == 0
assert response["yDomain"] == 0
# have the client label three data with time=1
data = get_assignments(client_profile, project, 3)
response = client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 1
},
)
response = client.post(
"/api/annotate_data/" + str(data[1].pk) + "/",
{
"labelID": test_labels[1].pk,
"labeling_time": 1
},
)
response = client.post(
"/api/annotate_data/" + str(data[2].pk) + "/",
{
"labelID": test_labels[2].pk,
"labeling_time": 1
},
)
# check that the quartiles are all the same
response = admin_client.get("/api/label_timing/" + str(project.pk) +
"/").json()
assert len(response["data"]) == 1
quarts = response["data"][0]["values"]
assert quarts["Q1"] == 1 and quarts["Q2"] == 1 and quarts["Q3"] == 1
assert quarts["whisker_low"] == 1 and quarts["whisker_high"] == 1
assert response["yDomain"] == 11
# have the client label three data with 0, 3, 10. Check quartiles
data = get_assignments(client_profile, project, 3)
response = client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 0
},
)
response = client.post(
"/api/annotate_data/" + str(data[1].pk) + "/",
{
"labelID": test_labels[1].pk,
"labeling_time": 3
},
)
response = client.post(
"/api/annotate_data/" + str(data[2].pk) + "/",
{
"labelID": test_labels[2].pk,
"labeling_time": 10
},
)
response = admin_client.get("/api/label_timing/" + str(project.pk) +
"/").json()
assert len(response["data"]) == 1
quarts = response["data"][0]["values"]
assert quarts["Q1"] == 1 and quarts["Q2"] == 1 and quarts["Q3"] == 3
assert quarts["whisker_low"] == 0 and quarts["whisker_high"] == 10
assert response["yDomain"] == 20
# have a client label with t=100. Check quartiles.
data = get_assignments(client_profile, project, 1)
response = client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 100
},
)
response = admin_client.get("/api/label_timing/" + str(project.pk) +
"/").json()
quarts = response["data"][0]["values"]
assert quarts["Q1"] == 1 and quarts["Q2"] == 1 and quarts["Q3"] == 10
assert quarts["whisker_low"] == 0 and quarts["whisker_high"] == 100
assert response["yDomain"] == 110
def test_label_distribution(
seeded_database,
admin_client,
client,
test_project_data,
test_queue,
test_labels,
test_irr_queue,
test_admin_queue,
):
"""This tests the api that produces the label counts chart for the admin page."""
project = test_project_data
client_profile, admin_profile = sign_in_and_fill_queue(
project, test_queue, client, admin_client)
response = client.get("/api/label_distribution/" + str(project.pk) + "/")
assert ("detail" in response.json()
and "Invalid permission. Must be an admin"
in response.json()["detail"])
# at the beginning, should return empty list
response = admin_client.get("/api/label_distribution/" + str(project.pk) +
"/")
assert len(response.json()) == 0
# have client label three things differently. Check values.
data = get_assignments(client_profile, project, 3)
response = client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 3
},
)
response = client.post(
"/api/annotate_data/" + str(data[1].pk) + "/",
{
"labelID": test_labels[1].pk,
"labeling_time": 3
},
)
response = client.post(
"/api/annotate_data/" + str(data[2].pk) + "/",
{
"labelID": test_labels[2].pk,
"labeling_time": 3
},
)
assert DataLabel.objects.filter(data__in=data).count() == 3
response = admin_client.get("/api/label_distribution/" + str(project.pk) +
"/").json()
assert len(response) > 0
for row in response:
temp_dict = row["values"]
for label_dict in temp_dict:
assert label_dict["x"] in [
str(admin_profile),
str(client_profile),
"test_profile",
]
if (label_dict["x"] == str(admin_profile)
or label_dict["x"] == "test_profile"):
assert label_dict["y"] == 0
else:
assert label_dict["y"] == 1
# Have admin label three things the same. Check values.
data = get_assignments(admin_profile, project, 3)
response = admin_client.post(
"/api/annotate_data/" + str(data[0].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 3
},
)
response = admin_client.post(
"/api/annotate_data/" + str(data[1].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 3
},
)
response = admin_client.post(
"/api/annotate_data/" + str(data[2].pk) + "/",
{
"labelID": test_labels[0].pk,
"labeling_time": 3
},
)
response = admin_client.get("/api/label_distribution/" + str(project.pk) +
"/").json()
assert len(response) > 0
for row in response:
label = row["key"]
temp_dict = row["values"]
for label_dict in temp_dict:
assert label_dict["x"] in [
str(admin_profile),
str(client_profile),
"test_profile",
]
if label_dict["x"] == str(admin_profile):
if label == test_labels[0].name:
assert label_dict["y"] == 3
else:
assert label_dict["y"] == 0
elif label_dict["x"] == "test_profile":
assert label_dict["y"] == 0
else:
assert label_dict["y"] == 1