def configure_qualifications():
from mturk.models import MtQualification, MtQualificationAssignment
from mturk.utils import get_or_create_mturk_worker
#
# Substance Labeling
substance = MtQualification.objects.get_or_create(
slug="substance",
defaults={
'name':
"Material Naming Master",
'keywords':
"material,naming,substance,images",
'description':
"You are an expert at naming and identifying regions of material or texture in an image.",
})[0]
shape_substance_labels = dict(
MaterialShape.objects.filter(substance__isnull=False).values_list(
'id', 'substance_id'))
shape_name_labels = dict(
MaterialShape.objects.filter(name__isnull=False).values_list(
'id', 'name_id'))
for worker in UserProfile.objects.exclude(mturk_worker_id=''):
worker_substance_labels = ShapeSubstanceLabel.objects.filter(user=worker) \
.values_list('shape_id', 'substance_id')
worker_name_labels = MaterialShapeNameLabel.objects.filter(user=worker) \
.values_list('shape_id', 'name_id')
ngood = 0
nbad = 0
for (shape_id, worker_substance_label) in worker_substance_labels:
if shape_id in shape_substance_labels:
if worker_substance_label == shape_substance_labels[shape_id]:
ngood += 1
else:
nbad += 1
for (shape_id, worker_name_label) in worker_name_labels:
if shape_id in shape_name_labels:
if worker_name_label == shape_name_labels[shape_id]:
ngood += 1
else:
nbad += 1
if ngood + nbad > 0:
perc = float(ngood) / float(ngood + nbad)
if ngood >= 100:
if perc >= 0.85:
substance_asst, created = substance.assignments.get_or_create(
worker=worker)
print 'Granting substance to %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
substance_asst.set_value(1)
elif ngood + nbad >= 200 and perc < 0.75:
try:
substance.assignments.get(worker=worker).set_value(0)
print 'Revoking substance from %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
except MtQualificationAssignment.DoesNotExist:
pass
if perc < 0.75 and not worker.always_approve:
worker.block(reason=(
"For naming tasks, your accuracy is %s%%, which is too low. "
+ "Most workers have an accuracy above 85%%.") %
int(perc * 100))
print 'Blocking user %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
elif nbad >= 100 and not worker.always_approve:
worker.block(reason=(
"For naming tasks, your accuracy is %s%%, which is too low. "
+ "Most workers have an accuracy above 85%%.") %
int(perc * 100))
print 'Blocking user %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
#
# Material Segmentation
matseg = MtQualification.objects.get_or_create(
slug="mat_seg",
defaults={
'name':
"Material Segmentation Master",
'keywords':
"material,segmentation,drawing,images",
'description':
"You are an expert at drawing and identifying regions of a single type of material or texture in an image.",
})[0]
# adequate work but annoying to deal with
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A1YLO4LT02D2F0'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='AU8FIYQPHKD7I'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A26GDUUFGPAYUL'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A2428ZTX6LJU33'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A2DHCVFGQRKU5'))[0].set_value(0)
# great workers
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1)
good_users = dict(
Counter(
MaterialShape.objects.filter(correct=True).values_list(
'user__mturk_worker_id', flat=True)).most_common())
bad_users = dict(
Counter(
MaterialShape.objects.filter(correct=False).values_list(
'user__mturk_worker_id', flat=True)).most_common())
for (id, ngood) in good_users.iteritems():
if ngood > 50:
nbad = bad_users[id] if id in bad_users else 0
perc = float(ngood) / float(ngood + nbad)
if perc >= 0.8:
worker = UserProfile.objects.get(mturk_worker_id=id)
matseg_asst, created = matseg.assignments.get_or_create(
worker=worker)
if created or matseg_asst.value != 0:
print 'Granting mat_seg to %s (%s good, %s bad, %s%%)' % (
id, ngood, nbad, perc * 100)
matseg_asst.set_value(1)
elif perc < 0.6 and not worker.always_approve:
worker = UserProfile.objects.get(mturk_worker_id=id)
try:
matseg.assignments.get(worker=worker).set_value(0)
print 'Revoking mat_seg from %s (%s good, %s bad, %s%%)' % (
id, ngood, nbad, perc * 100)
except MtQualificationAssignment.DoesNotExist:
pass
# Best workers
matseg2 = MtQualification.objects.get_or_create(
slug="mat_seg2",
defaults={
'name':
"Material Segmentation Master 2",
'keywords':
"material,segmentation,drawing,images,2",
'description':
"You are a super expert at drawing and identifying regions of a single type of material or texture in an image.",
})[0]
matseg2.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1)
matseg2.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1)
#
# Grant quals to admin
if settings.MTURK_ADMIN_WORKER_ID:
admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID)
matseg.assignments.get_or_create(worker=admin_user)[0].set_value(1)
matseg2.assignments.get_or_create(worker=admin_user)[0].set_value(1)
substance.assignments.get_or_create(worker=admin_user)[0].set_value(1)
def configure_qualifications():
from mturk.models import MtQualification, MtQualificationAssignment
from mturk.utils import get_or_create_mturk_worker
#
# Substance Labeling
substance = MtQualification.objects.get_or_create(
slug="substance",
defaults={
'name': "Material Naming Master",
'keywords': "material,naming,substance,images",
'description': "You are an expert at naming and identifying regions of material or texture in an image.",
}
)[0]
shape_substance_labels = dict(
MaterialShape.objects.filter(substance__isnull=False).values_list('id', 'substance_id'))
shape_name_labels = dict(
MaterialShape.objects.filter(name__isnull=False).values_list('id', 'name_id'))
for worker in UserProfile.objects.exclude(mturk_worker_id=''):
worker_substance_labels = ShapeSubstanceLabel.objects.filter(user=worker) \
.values_list('shape_id', 'substance_id')
worker_name_labels = MaterialShapeNameLabel.objects.filter(user=worker) \
.values_list('shape_id', 'name_id')
ngood = 0
nbad = 0
for (shape_id, worker_substance_label) in worker_substance_labels:
if shape_id in shape_substance_labels:
if worker_substance_label == shape_substance_labels[shape_id]:
ngood += 1
else:
nbad += 1
for (shape_id, worker_name_label) in worker_name_labels:
if shape_id in shape_name_labels:
if worker_name_label == shape_name_labels[shape_id]:
ngood += 1
else:
nbad += 1
if ngood + nbad > 0:
perc = float(ngood) / float(ngood + nbad)
if ngood >= 100:
if perc >= 0.85:
substance_asst, created = substance.assignments.get_or_create(
worker=worker)
print 'Granting substance to %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
substance_asst.set_value(1)
elif ngood + nbad >= 200 and perc < 0.75:
try:
substance.assignments.get(worker=worker).set_value(0)
print 'Revoking substance from %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
except MtQualificationAssignment.DoesNotExist:
pass
if perc < 0.75 and not worker.always_approve:
worker.block(reason=("For naming tasks, your accuracy is %s%%, which is too low. " +
"Most workers have an accuracy above 85%%.") % int(perc * 100))
print 'Blocking user %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
elif nbad >= 100 and not worker.always_approve:
worker.block(reason=("For naming tasks, your accuracy is %s%%, which is too low. " +
"Most workers have an accuracy above 85%%.") % int(perc * 100))
print 'Blocking user %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
#
# Material Segmentation
matseg = MtQualification.objects.get_or_create(
slug="mat_seg",
defaults={
'name': "Material Segmentation Master",
'keywords': "material,segmentation,drawing,images",
'description': "You are an expert at drawing and identifying regions of a single type of material or texture in an image.",
}
)[0]
# adequate work but annoying to deal with
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A1YLO4LT02D2F0'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='AU8FIYQPHKD7I'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A26GDUUFGPAYUL'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A2428ZTX6LJU33'))[0].set_value(0)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A2DHCVFGQRKU5'))[0].set_value(0)
# great workers
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1)
matseg.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1)
good_users = dict(Counter(
MaterialShape.objects
.filter(correct=True)
.values_list('user__mturk_worker_id', flat=True)
).most_common())
bad_users = dict(Counter(
MaterialShape.objects
.filter(correct=False)
.values_list('user__mturk_worker_id', flat=True)
).most_common())
for (id, ngood) in good_users.iteritems():
if ngood > 50:
nbad = bad_users[id] if id in bad_users else 0
perc = float(ngood) / float(ngood + nbad)
if perc >= 0.8:
worker = UserProfile.objects.get(mturk_worker_id=id)
matseg_asst, created = matseg.assignments.get_or_create(
worker=worker)
if created or matseg_asst.value != 0:
print 'Granting mat_seg to %s (%s good, %s bad, %s%%)' % (id, ngood, nbad, perc * 100)
matseg_asst.set_value(1)
elif perc < 0.6 and not worker.always_approve:
worker = UserProfile.objects.get(mturk_worker_id=id)
try:
matseg.assignments.get(worker=worker).set_value(0)
print 'Revoking mat_seg from %s (%s good, %s bad, %s%%)' % (id, ngood, nbad, perc * 100)
except MtQualificationAssignment.DoesNotExist:
pass
# Best workers
matseg2 = MtQualification.objects.get_or_create(
slug="mat_seg2",
defaults={
'name': "Material Segmentation Master 2",
'keywords': "material,segmentation,drawing,images,2",
'description': "You are a super expert at drawing and identifying regions of a single type of material or texture in an image.",
}
)[0]
matseg2.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1)
matseg2.assignments.get_or_create(worker=UserProfile.objects.get(
mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1)
#
# Grant quals to admin
if settings.MTURK_ADMIN_WORKER_ID:
admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID)
matseg.assignments.get_or_create(worker=admin_user)[0].set_value(1)
matseg2.assignments.get_or_create(worker=admin_user)[0].set_value(1)
substance.assignments.get_or_create(worker=admin_user)[0].set_value(1)
def configure_qualifications():
from mturk.models import MtQualification, MtQualificationAssignment
from mturk.utils import get_or_create_mturk_worker
#
# BSDF matching
bsdfmatch = MtQualification.objects.get_or_create(
slug="bsdf_match",
defaults={
'name': "Appearance Matching Master",
'keywords': "appearance,matching,blob,graphics,BRDF",
'description': "You are an expert at matching the appearance of a synthetic blob and a shape in an image."
}
)[0]
good_users = dict(Counter(
ShapeBsdfLabel_wd.objects
.filter(color_correct=True, gloss_correct=True)
.values_list('user__mturk_worker_id', flat=True)
).most_common())
bad_users = dict(Counter(
ShapeBsdfLabel_wd.objects
.filter(Q(color_correct=False) | Q(gloss_correct=False))
.values_list('user__mturk_worker_id', flat=True)
).most_common())
for (id, ngood) in good_users.iteritems():
nbad = bad_users[id] if id in bad_users else 0
if ngood + nbad > 0:
perc = float(ngood) / float(ngood + nbad)
if ngood >= 30:
worker = UserProfile.objects.get(mturk_worker_id=id)
if perc >= 0.75:
bsdfmatch_asst, created = bsdfmatch.assignments.get_or_create(
worker=worker)
print 'Granting bsdf_match to %s (%s good, %s bad)' % (id, ngood, nbad)
bsdfmatch_asst.set_value(1)
elif perc < 0.1 and not worker.always_approve:
# worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " +
#"Most workers have an accuracy above 75%%.") % int(perc * 100))
print 'WOULD block user %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
elif perc < 0.5:
try:
bsdfmatch.assignments.get(worker=worker).set_value(0)
print 'Revoking bsdf_match from %s (%s good, %s bad)' % (id, ngood, nbad)
except MtQualificationAssignment.DoesNotExist:
pass
elif nbad >= 30 and perc < 0.1 and not worker.always_approve:
# worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " +
#"Most workers have an accuracy above 75%%.") % int(perc * 100))
print 'WOULD block user %s (%s good, %s bad, %s%%)' % (
worker.mturk_worker_id, ngood, nbad, perc * 100)
#
# Grant quals to admin
if settings.MTURK_ADMIN_WORKER_ID:
admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID)
bsdfmatch.assignments.get_or_create(worker=admin_user)[0].set_value(1)
def configure_qualifications():
from mturk.models import MtQualification, MtQualificationAssignment
from mturk.utils import get_or_create_mturk_worker
#
# BSDF matching
bsdfmatch = MtQualification.objects.get_or_create(
slug="bsdf_match",
defaults={
"name": "Appearance Matching Master",
"keywords": "appearance,matching,blob,graphics,BRDF",
"description": "You are an expert at matching the appearance of a synthetic blob and a shape in an image.",
},
)[0]
good_users = dict(
Counter(
ShapeBsdfLabel_wd.objects.filter(color_correct=True, gloss_correct=True).values_list(
"user__mturk_worker_id", flat=True
)
).most_common()
)
bad_users = dict(
Counter(
ShapeBsdfLabel_wd.objects.filter(Q(color_correct=False) | Q(gloss_correct=False)).values_list(
"user__mturk_worker_id", flat=True
)
).most_common()
)
for (id, ngood) in good_users.iteritems():
nbad = bad_users[id] if id in bad_users else 0
if ngood + nbad > 0:
perc = float(ngood) / float(ngood + nbad)
if ngood >= 30:
worker = UserProfile.objects.get(mturk_worker_id=id)
if perc >= 0.75:
bsdfmatch_asst, created = bsdfmatch.assignments.get_or_create(worker=worker)
print "Granting bsdf_match to %s (%s good, %s bad)" % (id, ngood, nbad)
bsdfmatch_asst.set_value(1)
elif perc < 0.1 and not worker.always_approve:
# worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " +
# "Most workers have an accuracy above 75%%.") % int(perc * 100))
print "WOULD block user %s (%s good, %s bad, %s%%)" % (worker.mturk_worker_id, ngood, nbad, perc * 100)
elif perc < 0.5:
try:
bsdfmatch.assignments.get(worker=worker).set_value(0)
print "Revoking bsdf_match from %s (%s good, %s bad)" % (id, ngood, nbad)
except MtQualificationAssignment.DoesNotExist:
pass
elif nbad >= 30 and perc < 0.1 and not worker.always_approve:
# worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " +
# "Most workers have an accuracy above 75%%.") % int(perc * 100))
print "WOULD block user %s (%s good, %s bad, %s%%)" % (worker.mturk_worker_id, ngood, nbad, perc * 100)
#
# Grant quals to admin
if settings.MTURK_ADMIN_WORKER_ID:
admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID)
bsdfmatch.assignments.get_or_create(worker=admin_user)[0].set_value(1)
