def test_(self):
def new_vote(worker, sample, label):
return WorkerQualityVote(worker=worker, sample=sample, label=label)
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
mv.add_votes([])
self.assertEqual([], mv.extract_decisions())
WorkerQualityVote.objects.create(
worker=self.workers[2],
sample=self.sample,
label=LABEL_YES,
)
self.assertEqual(LABEL_YES, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[0], self.sample, LABEL_NO),
new_vote(self.workers[1], self.sample, LABEL_NO)
])
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
mv.reset()
self.assertEqual([], mv.extract_decisions())
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[0], self.sample, LABEL_YES),
new_vote(self.workers[1], self.sample, LABEL_YES),
new_vote(self.workers[2], self.sample, LABEL_NO),
new_vote(self.workers[3], self.sample, LABEL_NO),
new_vote(self.workers[4], self.sample, LABEL_BROKEN),
new_vote(self.workers[5], self.sample, LABEL_BROKEN),
new_vote(self.workers[6], self.sample, LABEL_NO),
new_vote(self.workers[7], self.sample, LABEL_NO),
new_vote(self.workers[8], self.sample, LABEL_NO),
])
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
def testProcessingVotes(self):
def new_vote(worker, sample, label):
return WorkerQualityVote(worker=worker, sample=sample, label=label)
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
worker = Worker.objects.create_tagasauris(external_id=21)
WorkerJobAssociation.objects.create(job=self.job, worker=worker)
Sample.objects.create_by_worker(
url="http://google.com",
source_val=self.workers[0].id,
job_id=self.job.id,
)
sample = Sample.objects.get(job=self.job, url='http://google.com')
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[0], self.sample, LABEL_YES),
new_vote(self.workers[1], self.sample, LABEL_YES),
new_vote(self.workers[2], self.sample, LABEL_NO),
new_vote(self.workers[3], self.sample, LABEL_NO),
new_vote(self.workers[4], self.sample, LABEL_BROKEN),
new_vote(self.workers[5], self.sample, LABEL_BROKEN),
new_vote(self.workers[6], self.sample, LABEL_NO),
new_vote(self.workers[7], self.sample, LABEL_NO),
new_vote(self.workers[8], self.sample, LABEL_NO),
])
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(worker, sample, LABEL_YES),
new_vote(self.workers[9], sample, LABEL_YES),
new_vote(self.workers[10], sample, LABEL_YES),
])
self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES)
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[11], sample, LABEL_NO),
new_vote(self.workers[12], sample, LABEL_YES),
new_vote(self.workers[13], sample, LABEL_YES),
])
self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES)
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[14], sample, LABEL_NO),
new_vote(self.workers[15], sample, LABEL_NO),
new_vote(self.workers[16], sample, LABEL_YES),
new_vote(self.workers[17], sample, LABEL_YES),
new_vote(self.workers[18], sample, LABEL_YES),
new_vote(self.workers[19], sample, LABEL_YES),
])
self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES)
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
def testQualityComputing(self):
def new_vote(worker, sample, label):
return WorkerQualityVote(worker=worker, sample=sample, label=label)
worker = Worker.objects.create_tagasauris(external_id=20)
Sample.objects.create_by_worker(
url="http://google.com",
source_val=worker.id,
job_id=self.job.id,
)
Sample.objects.create_by_worker(
url="http://google.com/2",
source_val=worker.id,
job_id=self.job.id,
)
sample1 = Sample.objects.get(job=self.job, url='http://google.com')
sample2 = Sample.objects.get(job=self.job, url='http://google.com/2')
votes = []
# 20 workers, 3 different voting sets
# 7 sets of type 1
# 7 sets of type 2
# 6 sets of type 3
# = sample1's label is YES, sample2's label is NO
for worker in self.workers:
res = worker.id % 3
if res == 0:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 1:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 2:
votes.append(new_vote(worker, sample1, LABEL_NO))
votes.append(new_vote(worker, sample2, LABEL_YES))
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
WorkerQualityVote.objects.bulk_create(votes)
mv.extract_decisions()
for worker in self.workers:
res = worker.id % 3
if res == 0:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1.0,
)
elif res == 1:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1.0,
)
elif res == 2:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
0.0,
)
# Check if different votes processing properly clean themselves.
mv.reset()
votes = []
# 20 workers, 3 different voting sets
# 6 sets of type 1
# 7 sets of type 2
# 7 sets of type 3
# = sample1's label is YES, sample2's label is NO
for worker in self.workers:
res = worker.id % 3
if res == 2:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 1:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 0:
votes.append(new_vote(worker, sample1, LABEL_NO))
votes.append(new_vote(worker, sample2, LABEL_YES))
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
WorkerQualityVote.objects.bulk_create(votes)
mv.extract_decisions()
for worker in self.workers:
res = worker.id % 3
if res == 2:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1,
)
elif res == 1:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1,
)
elif res == 0:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
0,
)
def testProcessingVotes(self):
def new_vote(worker, sample, label):
return WorkerQualityVote(worker=worker, sample=sample, label=label)
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
worker = Worker.objects.create_tagasauris(external_id=21)
WorkerJobAssociation.objects.create(job=self.job, worker=worker)
Sample.objects.create_by_worker(
url="http://google.com",
source_val=self.workers[0].id,
job_id=self.job.id,
)
sample = Sample.objects.get(job=self.job, url='http://google.com')
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[0], self.sample, LABEL_YES),
new_vote(self.workers[1], self.sample, LABEL_YES),
new_vote(self.workers[2], self.sample, LABEL_NO),
new_vote(self.workers[3], self.sample, LABEL_NO),
new_vote(self.workers[4], self.sample, LABEL_BROKEN),
new_vote(self.workers[5], self.sample, LABEL_BROKEN),
new_vote(self.workers[6], self.sample, LABEL_NO),
new_vote(self.workers[7], self.sample, LABEL_NO),
new_vote(self.workers[8], self.sample, LABEL_NO),
])
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(worker, sample, LABEL_YES),
new_vote(self.workers[9], sample, LABEL_YES),
new_vote(self.workers[10], sample, LABEL_YES),
])
self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES)
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[11], sample, LABEL_NO),
new_vote(self.workers[12], sample, LABEL_YES),
new_vote(self.workers[13], sample, LABEL_YES),
])
self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES)
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[14], sample, LABEL_NO),
new_vote(self.workers[15], sample, LABEL_NO),
new_vote(self.workers[16], sample, LABEL_YES),
new_vote(self.workers[17], sample, LABEL_YES),
new_vote(self.workers[18], sample, LABEL_YES),
new_vote(self.workers[19], sample, LABEL_YES),
])
self.assertEqual(mv.extract_decisions()[1][1], LABEL_YES)
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
def test_(self):
def new_vote(worker, sample, label):
return WorkerQualityVote(worker=worker, sample=sample, label=label)
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
mv.add_votes([])
self.assertEqual([], mv.extract_decisions())
WorkerQualityVote.objects.create(
worker=self.workers[2],
sample=self.sample,
label=LABEL_YES,
)
self.assertEqual(LABEL_YES, mv.extract_decisions()[0][1])
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[0], self.sample, LABEL_NO),
new_vote(self.workers[1], self.sample, LABEL_NO)
])
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
mv.reset()
self.assertEqual([], mv.extract_decisions())
WorkerQualityVote.objects.bulk_create([
new_vote(self.workers[0], self.sample, LABEL_YES),
new_vote(self.workers[1], self.sample, LABEL_YES),
new_vote(self.workers[2], self.sample, LABEL_NO),
new_vote(self.workers[3], self.sample, LABEL_NO),
new_vote(self.workers[4], self.sample, LABEL_BROKEN),
new_vote(self.workers[5], self.sample, LABEL_BROKEN),
new_vote(self.workers[6], self.sample, LABEL_NO),
new_vote(self.workers[7], self.sample, LABEL_NO),
new_vote(self.workers[8], self.sample, LABEL_NO),
])
self.assertEqual(LABEL_NO, mv.extract_decisions()[0][1])
def testQualityComputing(self):
def new_vote(worker, sample, label):
return WorkerQualityVote(worker=worker, sample=sample, label=label)
worker = Worker.objects.create_tagasauris(external_id=20)
Sample.objects.create_by_worker(
url="http://google.com",
source_val=worker.id,
job_id=self.job.id,
)
Sample.objects.create_by_worker(
url="http://google.com/2",
source_val=worker.id,
job_id=self.job.id,
)
sample1 = Sample.objects.get(job=self.job, url='http://google.com')
sample2 = Sample.objects.get(job=self.job, url='http://google.com/2')
votes = []
# 20 workers, 3 different voting sets
# 7 sets of type 1
# 7 sets of type 2
# 6 sets of type 3
# = sample1's label is YES, sample2's label is NO
for worker in self.workers:
res = worker.id % 3
if res == 0:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 1:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 2:
votes.append(new_vote(worker, sample1, LABEL_NO))
votes.append(new_vote(worker, sample2, LABEL_YES))
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
WorkerQualityVote.objects.bulk_create(votes)
mv.extract_decisions()
for worker in self.workers:
res = worker.id % 3
if res == 0:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1.0,
)
elif res == 1:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1.0,
)
elif res == 2:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
0.0,
)
# Check if different votes processing properly clean themselves.
mv.reset()
votes = []
# 20 workers, 3 different voting sets
# 6 sets of type 1
# 7 sets of type 2
# 7 sets of type 3
# = sample1's label is YES, sample2's label is NO
for worker in self.workers:
res = worker.id % 3
if res == 2:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 1:
votes.append(new_vote(worker, sample1, LABEL_YES))
votes.append(new_vote(worker, sample2, LABEL_NO))
elif res == 0:
votes.append(new_vote(worker, sample1, LABEL_NO))
votes.append(new_vote(worker, sample2, LABEL_YES))
mv = MajorityVoting(
job_id=self.job.id,
votes_storage=DBVotesStorage(storage_id=self.job.id),
)
WorkerQualityVote.objects.bulk_create(votes)
mv.extract_decisions()
for worker in self.workers:
res = worker.id % 3
if res == 2:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1,
)
elif res == 1:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
1,
)
elif res == 0:
self.assertEqual(
worker.get_estimated_quality_for_job(self.job),
0,
)
def create_algorithm(self, job, *args, **kwargs):
return MajorityVoting(job_id=job.id,
votes_storage=DBVotesStorage(storage_id=job.id))
