def test_queue_refill(setup_celery, test_project_data, test_all_queues, test_profile, test_labels, test_redis, tmpdir, settings): ''' Check that the queues refill the way they should. Have one person label everything in a batch. Check that the queue refills but the irr queue now has twice the irr% * batch amount ''' project = test_project_data normal_queue, admin_queue, irr_queue = test_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) irr_count = math.ceil((project.percentage_irr / 100) * project.batch_size) non_irr_count = math.ceil( ((100 - project.percentage_irr) / 100) * project.batch_size) for i in range(non_irr_count): datum = assign_datum(test_profile, project, "normal") assert datum is not None label_data(test_labels[0], datum, test_profile, 3) check_and_trigger_model(datum, test_profile) for i in range(irr_count): datum = assign_datum(test_profile, project, "irr") assert datum is not None label_data(test_labels[0], datum, test_profile, 3) check_and_trigger_model(datum, test_profile) assert DataQueue.objects.filter( queue=normal_queue).count() == non_irr_count assert DataQueue.objects.filter(queue=irr_queue).count() == irr_count * 2
def test_cohens_kappa_perc_agreement_no_agreement( setup_celery, test_project_half_irr_data, test_half_irr_all_queues, test_profile, test_profile2, test_labels_half_irr, test_redis, tmpdir, settings, ): """This just tests the kappa and percent if nobody ever agreed.""" project = test_project_half_irr_data labels = test_labels_half_irr normal_queue, admin_queue, irr_queue = test_half_irr_all_queues fill_queue( normal_queue, "random", irr_queue, project.percentage_irr, project.batch_size ) # label 5 irr elements but disagree on all of them for i in range(5): datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[i % 3], datum, test_profile, 3) label_data(labels[(i + 1) % 3], datum, test_profile2, 3) kappa, perc = cohens_kappa(project) assert round(kappa, 3) == -0.471 assert perc == 0.0
def test_annotate_irr(setup_celery, test_project_half_irr_data, test_half_irr_all_queues, test_profile, test_profile2, test_profile3, test_labels_half_irr, test_redis, tmpdir, settings): ''' This tests the irr labeling workflow, and checks that the data is in the correct models ''' project = test_project_half_irr_data normal_queue, admin_queue, irr_queue = test_half_irr_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) # get an irr datum. One should exist. datum = assign_datum(test_profile, project, "irr") assert datum is not None # let one user label a datum. It should be in DataLabel, not be in IRRLog, # still be in IRR Queue label_data(test_labels_half_irr[0], datum, test_profile, 3) assert DataLabel.objects.filter(data=datum, profile=test_profile).count() > 0 assert IRRLog.objects.filter(data=datum, profile=test_profile).count() == 0 assert DataQueue.objects.filter(data=datum, queue=irr_queue).count() > 0 datum2 = assign_datum(test_profile2, project, "irr") assert datum.pk == datum2.pk datum3 = assign_datum(test_profile3, project, "irr") assert datum.pk == datum3.pk # let other user label the same datum. It should now be in datatable with # creater=profile, be in IRRLog (twice), not be in IRRQueue label_data(test_labels_half_irr[0], datum2, test_profile2, 3) assert DataLabel.objects.filter(data=datum2).count() == 1 assert DataLabel.objects.get(data=datum2).profile.pk == project.creator.pk assert IRRLog.objects.filter(data=datum2).count() == 2 assert DataQueue.objects.filter(data=datum2, queue=irr_queue).count() == 0 # let a third user label the first data something else. It should be in # IRRLog but not overwrite the label from before label_data(test_labels_half_irr[0], datum3, test_profile3, 3) assert IRRLog.objects.filter(data=datum3).count() == 3 assert DataLabel.objects.filter(data=datum3).count() == 1 assert DataLabel.objects.get(data=datum3).profile.pk == project.creator.pk # let two users disagree on a datum. It should be in the admin queue, # not in irr queue, not in datalabel, in irrlog twice second_datum = assign_datum(test_profile, project, "irr") # should be a new datum assert datum.pk != second_datum.pk second_datum2 = assign_datum(test_profile2, project, "irr") label_data(test_labels_half_irr[0], second_datum, test_profile, 3) label_data(test_labels_half_irr[1], second_datum2, test_profile2, 3) assert DataQueue.objects.filter(data=second_datum2, queue=admin_queue).count() == 1 assert DataQueue.objects.filter(data=second_datum2, queue=irr_queue).count() == 0 assert DataLabel.objects.filter(data=second_datum2).count() == 0 assert IRRLog.objects.filter(data=second_datum2).count() == 2
def test_skip_irr( setup_celery, test_project_half_irr_data, test_half_irr_all_queues, test_profile, test_profile2, test_profile3, test_labels_half_irr, test_redis, tmpdir, settings, ): """This tests the skip function, and see if the data is in the correct places.""" project = test_project_half_irr_data normal_queue, admin_queue, irr_queue = test_half_irr_all_queues fill_queue(normal_queue, "random", irr_queue, project.percentage_irr, project.batch_size) # get an irr datum. One should exist. datum = assign_datum(test_profile, project, "irr") assert datum is not None # let one user skip an irr datum. It should not be in adminqueue, should be in irr queue, # should be in irrlog, should be in irr queue, not be in datalabel skip_data(datum, test_profile) assert DataQueue.objects.filter(data=datum, queue=admin_queue).count() == 0 assert DataQueue.objects.filter(data=datum, queue=irr_queue).count() == 1 assert IRRLog.objects.filter(data=datum, profile=test_profile).count() == 1 assert DataLabel.objects.filter(data=datum, profile=test_profile).count() == 0 # let the other user skip the data. It should be in admin queue, # IRRlog (twice), and nowhere else. datum2 = assign_datum(test_profile2, project, "irr") assert datum.pk == datum2.pk skip_data(datum2, test_profile2) assert DataQueue.objects.filter(data=datum, queue=admin_queue).count() == 1 assert DataQueue.objects.filter(data=datum, queue=irr_queue).count() == 0 assert IRRLog.objects.filter(data=datum).count() == 2 assert DataLabel.objects.filter(data=datum).count() == 0 # have two users label an IRR datum then have a third user skip it. # It should be in the IRRLog but not in admin queue or anywhere else. second_datum = assign_datum(test_profile, project, "irr") second_datum2 = assign_datum(test_profile2, project, "irr") assert second_datum.pk != datum.pk assert second_datum.pk == second_datum2.pk second_datum3 = assign_datum(test_profile3, project, "irr") assert second_datum2.pk == second_datum3.pk label_data(test_labels_half_irr[0], second_datum, test_profile, 3) label_data(test_labels_half_irr[0], second_datum2, test_profile2, 3) skip_data(second_datum3, test_profile3) assert DataQueue.objects.filter(data=second_datum3, queue=admin_queue).count() == 0 assert DataQueue.objects.filter(data=second_datum3, queue=irr_queue).count() == 0 assert IRRLog.objects.filter(data=second_datum3).count() == 3 assert DataLabel.objects.filter(data=second_datum3).count() == 1
def test_model_task_redis_no_dupes_data_left_in_queue( test_project_labeled_and_tfidf, test_queue_labeled, test_irr_queue_labeled, test_admin_queue_labeled, test_redis, tmpdir, settings): project = test_project_labeled_and_tfidf initial_training_set = project.get_current_training_set().set_number queue = project.queue_set.get(type="normal") queue.length = 40 queue.save() irr_queue = project.queue_set.get(type="irr") irr_queue.length = 40 irr_queue.save() model_path_temp = tmpdir.listdir()[0].mkdir('model_pickles') settings.MODEL_PICKLE_PATH = str(model_path_temp) batch_size = project.batch_size fill_queue(queue, 'random', irr_queue, irr_percent=project.percentage_irr, batch_size=batch_size) labels = project.labels.all() for i in range(int(batch_size * ((100 - project.percentage_irr) / 100))): datum = assign_datum(project.creator, project) label_data(random.choice(labels), datum, project.creator, 3) tasks.send_model_task.delay(project.pk).get() assert project.get_current_training_set( ).set_number == initial_training_set + 1 redis_items = test_redis.lrange(redis_serialize_queue(queue), 0, -1) assert len(redis_items) == len(set(redis_items))
def test_assign_datum_profile_queue_returns_correct_datum( db, test_profile_queue, test_profile, test_profile_queue2, test_profile2, test_redis): fill_queue(test_profile_queue, orderby='random') fill_queue(test_profile_queue2, orderby='random') datum = assign_datum(test_profile, test_profile_queue.project) assert isinstance(datum, Data)
def test_assign_datum_project_queue_returns_datum(db, test_queue, test_profile, test_redis): """Assign a datum from a project-wide queue (null profile ID).""" fill_queue(test_queue, orderby="random") datum = assign_datum(test_profile, test_queue.project) # Make sure we got the datum assert isinstance(datum, Data)
def test_g_naivebayes_classifier( setup_celery, test_project_gnb_data_tfidf, test_gnb_labels, test_gnb_queue_list, test_profile, test_redis, tmpdir, settings, ): """This tests that a project with the Gaussian Naiive Bayes classifier can successfully train and give predictions for a model.""" normal_queue, admin_queue, irr_queue = test_gnb_queue_list labels = test_gnb_labels project = test_project_gnb_data_tfidf active_l = project.learning_method batch_size = project.batch_size initial_training_set = project.get_current_training_set() model_path_temp = tmpdir.listdir()[0].mkdir("model_pickles") settings.MODEL_PICKLE_PATH = str(model_path_temp) assert project.classifier == "gnb" assert active_l == "least confident" fill_queue(normal_queue, "random") assert DataQueue.objects.filter(queue=normal_queue).count() == batch_size for i in range(batch_size): datum = assign_datum(test_profile, project) label_data(labels[i % 3], datum, test_profile, 3) ret_str = check_and_trigger_model(datum) assert ret_str == "model running" # Assert model created and saved assert_obj_exists(Model, {"project": project}) model = Model.objects.get(project=project) assert os.path.isfile(model.pickle_path) assert model.pickle_path == os.path.join( str(model_path_temp), "project_" + str(project.pk) + "_training_" + str(initial_training_set.set_number) + ".pkl", ) # Assert predictions created predictions = DataPrediction.objects.filter(data__project=project) assert ( len(predictions) == Data.objects.filter(project=project, labelers=None).count() * project.labels.count() )
def test_get_assignments_one_existing_assignment(db, test_profile, test_project_data, test_queue, test_redis): fill_queue(test_queue, orderby='random') assigned_datum = assign_datum(test_profile, test_project_data) data = get_assignments(test_profile, test_project_data, 1) assert isinstance(data[0], Data) # We should just get the datum that was already assigned assert data[0] == assigned_datum
def test_assign_datum_project_queue_correct_assignment(db, test_queue, test_profile, test_redis): fill_queue(test_queue, orderby='random') datum = assign_datum(test_profile, test_queue.project) # Make sure the assignment is correct assignment = AssignedData.objects.filter(data=datum) assert len(assignment) == 1 assert assignment[0].profile == test_profile assert assignment[0].queue == test_queue assert assignment[0].assigned_timestamp is not None
def test_skip_data(db, test_profile, test_queue, test_admin_queue, test_redis): fill_queue(test_queue, orderby='random') project = test_queue.project datum = assign_datum(test_profile, project) move_skipped_to_admin_queue(datum, test_profile, project) # Make sure the assignment was removed assert not AssignedData.objects.filter( profile=test_profile, data=datum, queue=test_queue).exists() # make sure the item was re-assigned to the admin queue assert DataQueue.objects.filter(data=datum, queue=test_admin_queue).exists() # make sure not in normal queue assert not DataQueue.objects.filter(data=datum, queue=test_queue).exists()
def test_assign_datum_project_queue_pops_queues(db, test_queue, test_profile, test_redis): fill_queue(test_queue, orderby='random') datum = assign_datum(test_profile, test_queue.project) # Make sure the datum was removed from queues but not set assert test_redis.llen('queue:' + str(test_queue.pk)) == test_queue.length - 1 assert test_redis.scard('set:' + str(test_queue.pk)) == test_queue.length # but not from the db queue assert test_queue.data.count() == test_queue.length assert datum in test_queue.data.all()
def test_get_assignments_multiple_existing_assignments(db, test_profile, test_project_data, test_queue, test_redis): fill_queue(test_queue, orderby='random') assigned_data = [] for i in range(5): assigned_data.append(assign_datum(test_profile, test_project_data)) data = get_assignments(test_profile, test_project_data, 5) assert len(data) == 5 assert len(data) == len(assigned_data) for datum, assigned_datum in zip(data, assigned_data): assert isinstance(datum, Data) # We should just get the data that was already assigned assert len(data) == len(assigned_data)
def test_randomforest_classifier(setup_celery, test_project_randomforest_data_tfidf, test_randomforest_labels, test_randomforest_queue_list, test_profile, test_redis, tmpdir, settings): ''' This tests that a project with the random forest classifier can successfully train and give predictions for a model ''' normal_queue, admin_queue, irr_queue = test_randomforest_queue_list labels = test_randomforest_labels project = test_project_randomforest_data_tfidf active_l = project.learning_method batch_size = project.batch_size initial_training_set = project.get_current_training_set() model_path_temp = tmpdir.listdir()[0].mkdir('model_pickles') settings.MODEL_PICKLE_PATH = str(model_path_temp) assert project.classifier == "random forest" assert active_l == 'least confident' fill_queue(normal_queue, 'random') assert DataQueue.objects.filter(queue=normal_queue).count() == batch_size for i in range(batch_size): datum = assign_datum(test_profile, project) label_data(labels[i % 3], datum, test_profile, 3) ret_str = check_and_trigger_model(datum) assert ret_str == 'model running' # Assert model created and saved assert_obj_exists(Model, {'project': project}) model = Model.objects.get(project=project) assert os.path.isfile(model.pickle_path) assert model.pickle_path == os.path.join( str(model_path_temp), 'project_' + str(project.pk) + '_training_' + str(initial_training_set.set_number) + '.pkl') # Assert predictions created predictions = DataPrediction.objects.filter(data__project=project) assert len(predictions) == Data.objects.filter( project=project, labelers=None).count() * project.labels.count()
def test_check_and_trigger_model_first_labeled( setup_celery, test_project_data, test_labels, test_queue, test_profile ): initial_training_set = test_project_data.get_current_training_set() fill_queue(test_queue, orderby="random") datum = assign_datum(test_profile, test_queue.project) test_label = test_labels[0] label_data(test_label, datum, test_profile, 3) check = check_and_trigger_model(datum) assert check == "no trigger" assert test_project_data.get_current_training_set() == initial_training_set assert test_project_data.model_set.count() == 0 assert DataPrediction.objects.filter(data__project=test_project_data).count() == 0 assert DataUncertainty.objects.filter(data__project=test_project_data).count() == 0 assert DataQueue.objects.filter(queue=test_queue).count() == TEST_QUEUE_LEN - 1
def test_label_data(db, test_profile, test_queue, test_redis): fill_queue(test_queue, orderby='random') datum = assign_datum(test_profile, test_queue.project) test_label = Label.objects.create(name='test', project=test_queue.project) label_data(test_label, datum, test_profile, 3) # Make sure the label was properly recorded assert datum in test_profile.labeled_data.all() assert_obj_exists( DataLabel, { 'data': datum, 'profile': test_profile, 'label': test_label, 'time_to_label': 3 }) # Make sure the assignment was removed assert not AssignedData.objects.filter( profile=test_profile, data=datum, queue=test_queue).exists()
def test_label_data(db, test_profile, test_queue, test_redis): fill_queue(test_queue, orderby="random") datum = assign_datum(test_profile, test_queue.project) test_label = Label.objects.create(name="test", project=test_queue.project) label_data(test_label, datum, test_profile, 3) # Make sure the label was properly recorded assert datum in test_profile.labeled_data.all() assert_obj_exists( DataLabel, { "data": datum, "profile": test_profile, "label": test_label, "time_to_label": 3, }, ) # Make sure the assignment was removed assert not AssignedData.objects.filter( profile=test_profile, data=datum, queue=test_queue).exists()
def test_assign_datum_profile_queue_pops_queues(db, test_profile_queue, test_profile, test_profile_queue2, test_profile2, test_redis): fill_queue(test_profile_queue, orderby='random') fill_queue(test_profile_queue2, orderby='random') datum = assign_datum(test_profile, test_profile_queue.project) # Make sure the datum was removed from the correct queues but not sets assert test_redis.llen( 'queue:' + str(test_profile_queue.pk)) == test_profile_queue.length - 1 assert test_redis.scard( 'set:' + str(test_profile_queue.pk)) == test_profile_queue.length # ...but not the other queues assert test_profile_queue.data.count() == test_profile_queue.length assert datum in test_profile_queue.data.all() assert test_redis.llen( 'queue:' + str(test_profile_queue2.pk)) == test_profile_queue2.length assert test_redis.scard( 'set:' + str(test_profile_queue2.pk)) == test_profile_queue2.length assert test_profile_queue2.data.count() == test_profile_queue2.length
def test_fleiss_kappa_perc_agreement( setup_celery, test_project_all_irr_3_coders_data, test_all_irr_3_coders_all_queues, test_profile, test_profile2, test_profile3, test_labels_all_irr_3_coders, test_redis, tmpdir, settings): ''' This tests the results of the Fleiss's kappa function when fed different situations ''' project = test_project_all_irr_3_coders_data labels = test_labels_all_irr_3_coders normal_queue, admin_queue, irr_queue = test_all_irr_3_coders_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) # first check that an error is thrown if there is no data with pytest.raises(ValueError) as excinfo: fleiss_kappa(project) assert 'No irr data' in str(excinfo.value) # next, check that the same error happens if only two have labeled it datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[1], datum, test_profile2, 3) with pytest.raises(ValueError) as excinfo: fleiss_kappa(project) assert 'No irr data' in str(excinfo.value) # have everyone label a datum differenty # [1 1 1], kappa = -0.5, pa = 0 label_data(labels[2], datum, test_profile3, 3) kappa, perc = fleiss_kappa(project) assert round(kappa, 1) == -0.5 assert perc == 0.0 # have only two people label a datum the same and check that kappa is the same datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[0], datum, test_profile2, 3) kappa, perc = fleiss_kappa(project) assert round(kappa, 1) == -0.5 assert perc == 0.0 # have last person label datum the same # [[1 1 1],[3 0 0]], kappa = 0.0, pa = 0.5 label_data(labels[0], datum, test_profile3, 3) kappa, perc = fleiss_kappa(project) assert round(kappa, 2) == 0.0 assert perc == 0.5 # have two people agree and one disagree # [[1 1 1],[3 0 0],[2 1 0]], kappa = -0.13, pa=0.333 datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[0], datum, test_profile2, 3) label_data(labels[1], datum, test_profile3, 3) kappa, perc = fleiss_kappa(project) assert round(kappa, 2) == -0.13 assert round(perc, 2) == 0.33 # repeat previous step with slight variation # [[1 1 1],[3 0 0],[2 1 0],[1 2 0]], kappa = -0.08, pa=0.25 datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[1], datum, test_profile2, 3) label_data(labels[1], datum, test_profile3, 3) kappa, perc = fleiss_kappa(project) assert round(kappa, 2) == -0.08 assert round(perc, 2) == 0.25
def test_cohens_kappa_perc_agreement(setup_celery, test_project_half_irr_data, test_half_irr_all_queues, test_profile, test_profile2, test_labels_half_irr, test_redis, tmpdir, settings): ''' want to check several different configurations including empty, no agreement Should throw an error if no irr data processed yet ''' project = test_project_half_irr_data labels = test_labels_half_irr normal_queue, admin_queue, irr_queue = test_half_irr_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) # check that before anything is labeled, an error is thrown with pytest.raises(ValueError) as excinfo: cohens_kappa(project) assert 'No irr data' in str(excinfo.value) # have two labelers label two datum the same. for i in range(2): datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[0], datum, test_profile2, 3) # kappa requires at least two labels be represented with pytest.raises(ValueError) as excinfo: cohens_kappa(project) assert 'Need at least two labels represented' in str(excinfo.value) datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[1], datum, test_profile, 3) label_data(labels[1], datum, test_profile2, 3) # Now kappa should be 1 kappa, perc = cohens_kappa(project) assert kappa == 1.0 assert perc == 1.0 # have two labelers disagree on two datum check the value datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[1], datum, test_profile, 3) label_data(labels[2], datum, test_profile2, 3) datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[1], datum, test_profile2, 3) kappa, perc = cohens_kappa(project) assert round(kappa, 3) == 0.333 assert perc == 0.6
def test_percent_agreement_table(setup_celery, test_project_all_irr_3_coders_data, test_all_irr_3_coders_all_queues, test_profile, test_profile2, test_profile3, test_labels_all_irr_3_coders, test_redis, tmpdir, settings): ''' This tests the percent agreement table ''' project = test_project_all_irr_3_coders_data ProjectPermissions.objects.create(profile=test_profile2, project=project, permission='CODER') ProjectPermissions.objects.create(profile=test_profile3, project=project, permission='CODER') labels = test_labels_all_irr_3_coders normal_queue, admin_queue, irr_queue = test_all_irr_3_coders_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) table_data_perc = pd.DataFrame( perc_agreement_table_data(project))["Percent Agreement"].tolist() # first test that it has "No Samples" for the percent for all assert len(table_data_perc) == 3 assert (table_data_perc[0] == "No samples") and (table_data_perc[1] == "No samples") and (table_data_perc[2] == "No samples") # First have everyone give same label, should be 100% for all datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[0], datum, test_profile2, 3) label_data(labels[0], datum, test_profile3, 3) table_data_perc = pd.DataFrame( perc_agreement_table_data(project))["Percent Agreement"].tolist() assert (table_data_perc[0] == "100.0%") and (table_data_perc[1] == "100.0%") and (table_data_perc[2] == "100.0%") # Next have user1 = user2 != user3, Check values datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[0], datum, test_profile2, 3) label_data(labels[1], datum, test_profile3, 3) table_data_perc = pd.DataFrame( perc_agreement_table_data(project))["Percent Agreement"].tolist() # goes in the order [prof2,prof3], [prof2, prof], [prof3, prof] assert (table_data_perc[0] == "50.0%") and (table_data_perc[1] == "100.0%") and (table_data_perc[2] == "50.0%") # Next have all users skip. Should count as disagreement. datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") skip_data(datum, test_profile) skip_data(datum, test_profile2) skip_data(datum, test_profile3) table_data_perc = pd.DataFrame( perc_agreement_table_data(project))["Percent Agreement"].tolist() # goes in the order [prof2,prof3], [prof2, prof], [prof3, prof] assert (table_data_perc[0] == "33.3%") and (table_data_perc[1] == "66.7%") and (table_data_perc[2] == "33.3%") # Lastly have two users label. Should be the same as before datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") assign_datum(test_profile3, project, "irr") label_data(labels[0], datum, test_profile, 3) label_data(labels[0], datum, test_profile2, 3) table_data_perc = pd.DataFrame( perc_agreement_table_data(project))["Percent Agreement"].tolist() # goes in the order [prof2,prof3], [prof2, prof], [prof3, prof] assert (table_data_perc[0] == "33.3%") and (table_data_perc[1] == "66.7%") and (table_data_perc[2] == "33.3%")
def test_heatmap_data(setup_celery, test_project_half_irr_data, test_half_irr_all_queues, test_profile, test_profile2, test_labels_half_irr, test_redis, tmpdir, settings): ''' These tests check that the heatmap accurately reflects the data ''' project = test_project_half_irr_data ProjectPermissions.objects.create(profile=test_profile, project=project, permission='CODER') ProjectPermissions.objects.create(profile=test_profile2, project=project, permission='CODER') labels = test_labels_half_irr normal_queue, admin_queue, irr_queue = test_half_irr_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) combo1 = str(test_profile.pk) + "_" + str(test_profile2.pk) same1 = str(test_profile.pk) + "_" + str(test_profile.pk) same2 = str(test_profile2.pk) + "_" + str(test_profile2.pk) # don't label anything. The heatmap shoud have all zeros for user pair heatmap = irr_heatmap_data(project) assert combo1 in heatmap heatmap = heatmap[combo1] counts = pd.DataFrame(heatmap)["count"].tolist() assert np.all(np.equal(counts, [0] * len(counts))) # have one user skip 3 things and another label them. for i in range(3): datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[i], datum, test_profile, 3) skip_data(datum, test_profile2) # check that user1-user1 map is I3 heatmap = irr_heatmap_data(project) same_frame = pd.DataFrame(heatmap[same1]) assert same_frame.loc[(same_frame["label1"] == labels[0].name) & ( same_frame["label2"] == labels[0].name)]["count"].tolist()[0] == 1 assert same_frame.loc[(same_frame["label1"] == labels[1].name) & ( same_frame["label2"] == labels[1].name)]["count"].tolist()[0] == 1 assert same_frame.loc[(same_frame["label1"] == labels[2].name) & ( same_frame["label2"] == labels[2].name)]["count"].tolist()[0] == 1 assert np.sum(same_frame["count"].tolist()) == 3 # check the second user only has 3 in the skip-skip spot same_frame2 = pd.DataFrame(heatmap[same2]) assert same_frame2.loc[(same_frame2["label1"] == "Skip") & ( same_frame["label2"] == "Skip")]["count"].tolist()[0] == 3 assert np.sum(same_frame2["count"].tolist()) == 3 # check that the between-user heatmap has skip-label = 1 for each label heatmap = irr_heatmap_data(project) heatmap = pd.DataFrame(heatmap[combo1]) assert heatmap.loc[(heatmap["label1"] == labels[0].name) & (heatmap["label2"] == "Skip")]["count"].tolist()[0] == 1 assert heatmap.loc[(heatmap["label1"] == labels[1].name) & (heatmap["label2"] == "Skip")]["count"].tolist()[0] == 1 assert heatmap.loc[(heatmap["label1"] == labels[2].name) & (heatmap["label2"] == "Skip")]["count"].tolist()[0] == 1 assert np.sum(heatmap["count"].tolist()) == 3 # have users agree on 5 labels and datums, check heatmap for i in range(5): datum = assign_datum(test_profile, project, "irr") assign_datum(test_profile2, project, "irr") label_data(labels[i % 3], datum, test_profile, 3) label_data(labels[i % 3], datum, test_profile2, 3) heatmap = irr_heatmap_data(project) heatmap = pd.DataFrame(heatmap[combo1]) assert heatmap.loc[(heatmap["label1"] == labels[0].name) & ( heatmap["label2"] == labels[0].name)]["count"].tolist()[0] == 2 assert heatmap.loc[(heatmap["label1"] == labels[1].name) & ( heatmap["label2"] == labels[1].name)]["count"].tolist()[0] == 2 assert heatmap.loc[(heatmap["label1"] == labels[2].name) & ( heatmap["label2"] == labels[2].name)]["count"].tolist()[0] == 1 assert np.sum(heatmap["count"].tolist()) == 8 # have one user label something, show the heatmap hasn't changed datum = assign_datum(test_profile, project, "irr") label_data(labels[0], datum, test_profile, 3) heatmap = irr_heatmap_data(project) same_map = heatmap[same1] assert np.sum(pd.DataFrame(same_map)["count"].tolist()) == 8 heatmap = pd.DataFrame(heatmap[combo1]) assert np.sum(pd.DataFrame(heatmap)["count"].tolist()) == 8
def test_all_irr(setup_celery, test_project_all_irr_3_coders_data, test_all_irr_3_coders_all_queues, test_profile, test_profile2, test_profile3, test_labels_all_irr_3_coders, test_redis, tmpdir, settings): ''' This tests the case with 100% IRR and triple labeling required ''' project = test_project_all_irr_3_coders_data labels = test_labels_all_irr_3_coders normal_queue, admin_queue, irr_queue = test_all_irr_3_coders_all_queues fill_queue(normal_queue, 'random', irr_queue, project.percentage_irr, project.batch_size) # check the normal queue is empty and the irr queue is full assert DataQueue.objects.filter( queue=irr_queue).count() == project.batch_size assert DataQueue.objects.filter(queue=normal_queue).count() == 0 # check everything in the irr queue has irr_ind = true assert DataQueue.objects.filter( queue=irr_queue, data__irr_ind=True).count() == project.batch_size # have one person label three datum and check that they are still in the queue datum = assign_datum(test_profile, project, "irr") second_datum = assign_datum(test_profile, project, "irr") third_datum = assign_datum(test_profile, project, "irr") assert datum.pk != second_datum.pk assert third_datum.pk != second_datum.pk label_data(labels[0], datum, test_profile, 3) label_data(labels[0], second_datum, test_profile, 3) label_data(labels[0], third_datum, test_profile, 3) assert DataQueue.objects.filter( queue=irr_queue, data__in=[datum, second_datum, third_datum]).count() == 3 # have one person skip all three datum, and check that they are still in the irr queue, in irrlog, and in datalabel, but not in admin queue datum2 = assign_datum(test_profile2, project, "irr") second_datum2 = assign_datum(test_profile2, project, "irr") third_datum2 = assign_datum(test_profile2, project, "irr") assert datum.pk == datum2.pk assert second_datum.pk == second_datum2.pk assert third_datum.pk == third_datum2.pk skip_data(datum2, test_profile2) skip_data(second_datum2, test_profile2) skip_data(third_datum2, test_profile2) assert DataQueue.objects.filter( data__in=[datum2, second_datum2, third_datum2], queue=irr_queue).count() == 3 assert DataQueue.objects.filter( data__in=[datum2, second_datum2, third_datum2], queue=admin_queue).count() == 0 assert IRRLog.objects.filter( data__in=[datum2, second_datum2, third_datum2]).count() == 3 assert DataLabel.objects.filter( data__in=[datum2, second_datum2, third_datum2]).count() == 3 # have the third person label all three datum and check that they are in the log and admin queue, but not in irr queue or datalabel datum3 = assign_datum(test_profile3, project, "irr") second_datum3 = assign_datum(test_profile3, project, "irr") third_datum3 = assign_datum(test_profile3, project, "irr") assert datum.pk == datum3.pk assert second_datum.pk == second_datum3.pk assert third_datum.pk == third_datum3.pk label_data(labels[0], datum3, test_profile3, 3) label_data(labels[1], second_datum3, test_profile3, 3) label_data(labels[0], third_datum3, test_profile3, 3) assert DataQueue.objects.filter( data__in=[datum3, second_datum3, third_datum3], queue=irr_queue).count() == 0 assert DataQueue.objects.filter( data__in=[datum3, second_datum3, third_datum3], queue=admin_queue).count() == 3 assert IRRLog.objects.filter( data__in=[datum3, second_datum3, third_datum3]).count() == 9 assert DataLabel.objects.filter( data__in=[datum3, second_datum3, third_datum3]).count() == 0