def test_(self): def new_vote(worker, sample, label): return WorkerQualityVote(worker=worker, sample=sample, label=label) mv = MajorityVoting( job_id=self.job.id, votes_storage=DBVotesStorage(storage_id=self.job.id), ) mv.add_votes([]) self.assertEqual([], mv.extract_decisions()) WorkerQualityVote.objects.create( worker=self.workers[2], sample=self.sample, label=LABEL_YES, ) self.assertEqual(LABEL_YES, mv.extract_decisions()[0][1]) WorkerQualityVote.objects.bulk_create([ new_vote(self.workers[0], self.sample, LABEL_NO), new_vote(self.workers[1], self.sample, LABEL_NO) ]) self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1]) mv.reset() self.assertEqual([], mv.extract_decisions()) WorkerQualityVote.objects.bulk_create([ new_vote(self.workers[0], self.sample, LABEL_YES), new_vote(self.workers[1], self.sample, LABEL_YES), new_vote(self.workers[2], self.sample, LABEL_NO), new_vote(self.workers[3], self.sample, LABEL_NO), new_vote(self.workers[4], self.sample, LABEL_BROKEN), new_vote(self.workers[5], self.sample, LABEL_BROKEN), new_vote(self.workers[6], self.sample, LABEL_NO), new_vote(self.workers[7], self.sample, LABEL_NO), new_vote(self.workers[8], self.sample, LABEL_NO), ]) self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
def testProcessingVotes(self): def new_vote(worker, sample, label): return WorkerQualityVote(worker=worker, sample=sample, label=label) mv = MajorityVoting( job_id=self.job.id, votes_storage=DBVotesStorage(storage_id=self.job.id), ) worker = Worker.objects.create_tagasauris(external_id=21) WorkerJobAssociation.objects.create(job=self.job, worker=worker) Sample.objects.create_by_worker( url="http://google.com", source_val=self.workers[0].id, job_id=self.job.id, ) sample = Sample.objects.get(job=self.job, url='http://google.com') WorkerQualityVote.objects.bulk_create([ new_vote(self.workers[0], self.sample, LABEL_YES), new_vote(self.workers[1], self.sample, LABEL_YES), new_vote(self.workers[2], self.sample, LABEL_NO), new_vote(self.workers[3], self.sample, LABEL_NO), new_vote(self.workers[4], self.sample, LABEL_BROKEN), new_vote(self.workers[5], self.sample, LABEL_BROKEN), new_vote(self.workers[6], self.sample, LABEL_NO), new_vote(self.workers[7], self.sample, LABEL_NO), new_vote(self.workers[8], self.sample, LABEL_NO), ]) self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1]) WorkerQualityVote.objects.bulk_create([ new_vote(worker, sample, LABEL_YES), new_vote(self.workers[9], sample, LABEL_YES), new_vote(self.workers[10], sample, LABEL_YES), ]) self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES) self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1]) WorkerQualityVote.objects.bulk_create([ new_vote(self.workers[11], sample, LABEL_NO), new_vote(self.workers[12], sample, LABEL_YES), new_vote(self.workers[13], sample, LABEL_YES), ]) self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES) self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1]) WorkerQualityVote.objects.bulk_create([ new_vote(self.workers[14], sample, LABEL_NO), new_vote(self.workers[15], sample, LABEL_NO), new_vote(self.workers[16], sample, LABEL_YES), new_vote(self.workers[17], sample, LABEL_YES), new_vote(self.workers[18], sample, LABEL_YES), new_vote(self.workers[19], sample, LABEL_YES), ]) self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES) self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
def testQualityComputing(self): def new_vote(worker, sample, label): return WorkerQualityVote(worker=worker, sample=sample, label=label) worker = Worker.objects.create_tagasauris(external_id=20) Sample.objects.create_by_worker( url="http://google.com", source_val=worker.id, job_id=self.job.id, ) Sample.objects.create_by_worker( url="http://google.com/2", source_val=worker.id, job_id=self.job.id, ) sample1 = Sample.objects.get(job=self.job, url='http://google.com') sample2 = Sample.objects.get(job=self.job, url='http://google.com/2') votes = [] # 20 workers, 3 different voting sets # 7 sets of type 1 # 7 sets of type 2 # 6 sets of type 3 # = sample1's label is YES, sample2's label is NO for worker in self.workers: res = worker.id % 3 if res == 0: votes.append(new_vote(worker, sample1, LABEL_YES)) votes.append(new_vote(worker, sample2, LABEL_NO)) elif res == 1: votes.append(new_vote(worker, sample1, LABEL_YES)) votes.append(new_vote(worker, sample2, LABEL_NO)) elif res == 2: votes.append(new_vote(worker, sample1, LABEL_NO)) votes.append(new_vote(worker, sample2, LABEL_YES)) mv = MajorityVoting( job_id=self.job.id, votes_storage=DBVotesStorage(storage_id=self.job.id), ) WorkerQualityVote.objects.bulk_create(votes) mv.extract_decisions() for worker in self.workers: res = worker.id % 3 if res == 0: self.assertEqual( worker.get_estimated_quality_for_job(self.job), 1.0, ) elif res == 1: self.assertEqual( worker.get_estimated_quality_for_job(self.job), 1.0, ) elif res == 2: self.assertEqual( worker.get_estimated_quality_for_job(self.job), 0.0, ) # Check if different votes processing properly clean themselves. mv.reset() votes = [] # 20 workers, 3 different voting sets # 6 sets of type 1 # 7 sets of type 2 # 7 sets of type 3 # = sample1's label is YES, sample2's label is NO for worker in self.workers: res = worker.id % 3 if res == 2: votes.append(new_vote(worker, sample1, LABEL_YES)) votes.append(new_vote(worker, sample2, LABEL_NO)) elif res == 1: votes.append(new_vote(worker, sample1, LABEL_YES)) votes.append(new_vote(worker, sample2, LABEL_NO)) elif res == 0: votes.append(new_vote(worker, sample1, LABEL_NO)) votes.append(new_vote(worker, sample2, LABEL_YES)) mv = MajorityVoting( job_id=self.job.id, votes_storage=DBVotesStorage(storage_id=self.job.id), ) WorkerQualityVote.objects.bulk_create(votes) mv.extract_decisions() for worker in self.workers: res = worker.id % 3 if res == 2: self.assertEqual( worker.get_estimated_quality_for_job(self.job), 1, ) elif res == 1: self.assertEqual( worker.get_estimated_quality_for_job(self.job), 1, ) elif res == 0: self.assertEqual( worker.get_estimated_quality_for_job(self.job), 0, )
def create_algorithm(self, job, *args, **kwargs): return MajorityVoting(job_id=job.id, votes_storage=DBVotesStorage(storage_id=job.id))