def configure_qualifications(): from mturk.models import MtQualification, MtQualificationAssignment from mturk.utils import get_or_create_mturk_worker # # Substance Labeling substance = MtQualification.objects.get_or_create( slug="substance", defaults={ 'name': "Material Naming Master", 'keywords': "material,naming,substance,images", 'description': "You are an expert at naming and identifying regions of material or texture in an image.", })[0] shape_substance_labels = dict( MaterialShape.objects.filter(substance__isnull=False).values_list( 'id', 'substance_id')) shape_name_labels = dict( MaterialShape.objects.filter(name__isnull=False).values_list( 'id', 'name_id')) for worker in UserProfile.objects.exclude(mturk_worker_id=''): worker_substance_labels = ShapeSubstanceLabel.objects.filter(user=worker) \ .values_list('shape_id', 'substance_id') worker_name_labels = MaterialShapeNameLabel.objects.filter(user=worker) \ .values_list('shape_id', 'name_id') ngood = 0 nbad = 0 for (shape_id, worker_substance_label) in worker_substance_labels: if shape_id in shape_substance_labels: if worker_substance_label == shape_substance_labels[shape_id]: ngood += 1 else: nbad += 1 for (shape_id, worker_name_label) in worker_name_labels: if shape_id in shape_name_labels: if worker_name_label == shape_name_labels[shape_id]: ngood += 1 else: nbad += 1 if ngood + nbad > 0: perc = float(ngood) / float(ngood + nbad) if ngood >= 100: if perc >= 0.85: substance_asst, created = substance.assignments.get_or_create( worker=worker) print 'Granting substance to %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) substance_asst.set_value(1) elif ngood + nbad >= 200 and perc < 0.75: try: substance.assignments.get(worker=worker).set_value(0) print 'Revoking substance from %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) except MtQualificationAssignment.DoesNotExist: pass if perc < 0.75 and not worker.always_approve: worker.block(reason=( "For naming tasks, your accuracy is %s%%, which is too low. " + "Most workers have an accuracy above 85%%.") % int(perc * 100)) print 'Blocking user %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) elif nbad >= 100 and not worker.always_approve: worker.block(reason=( "For naming tasks, your accuracy is %s%%, which is too low. " + "Most workers have an accuracy above 85%%.") % int(perc * 100)) print 'Blocking user %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) # # Material Segmentation matseg = MtQualification.objects.get_or_create( slug="mat_seg", defaults={ 'name': "Material Segmentation Master", 'keywords': "material,segmentation,drawing,images", 'description': "You are an expert at drawing and identifying regions of a single type of material or texture in an image.", })[0] # adequate work but annoying to deal with matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A1YLO4LT02D2F0'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='AU8FIYQPHKD7I'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A26GDUUFGPAYUL'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A2428ZTX6LJU33'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A2DHCVFGQRKU5'))[0].set_value(0) # great workers matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1) good_users = dict( Counter( MaterialShape.objects.filter(correct=True).values_list( 'user__mturk_worker_id', flat=True)).most_common()) bad_users = dict( Counter( MaterialShape.objects.filter(correct=False).values_list( 'user__mturk_worker_id', flat=True)).most_common()) for (id, ngood) in good_users.iteritems(): if ngood > 50: nbad = bad_users[id] if id in bad_users else 0 perc = float(ngood) / float(ngood + nbad) if perc >= 0.8: worker = UserProfile.objects.get(mturk_worker_id=id) matseg_asst, created = matseg.assignments.get_or_create( worker=worker) if created or matseg_asst.value != 0: print 'Granting mat_seg to %s (%s good, %s bad, %s%%)' % ( id, ngood, nbad, perc * 100) matseg_asst.set_value(1) elif perc < 0.6 and not worker.always_approve: worker = UserProfile.objects.get(mturk_worker_id=id) try: matseg.assignments.get(worker=worker).set_value(0) print 'Revoking mat_seg from %s (%s good, %s bad, %s%%)' % ( id, ngood, nbad, perc * 100) except MtQualificationAssignment.DoesNotExist: pass # Best workers matseg2 = MtQualification.objects.get_or_create( slug="mat_seg2", defaults={ 'name': "Material Segmentation Master 2", 'keywords': "material,segmentation,drawing,images,2", 'description': "You are a super expert at drawing and identifying regions of a single type of material or texture in an image.", })[0] matseg2.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1) matseg2.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1) # # Grant quals to admin if settings.MTURK_ADMIN_WORKER_ID: admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID) matseg.assignments.get_or_create(worker=admin_user)[0].set_value(1) matseg2.assignments.get_or_create(worker=admin_user)[0].set_value(1) substance.assignments.get_or_create(worker=admin_user)[0].set_value(1)
def configure_qualifications(): from mturk.models import MtQualification, MtQualificationAssignment from mturk.utils import get_or_create_mturk_worker # # Substance Labeling substance = MtQualification.objects.get_or_create( slug="substance", defaults={ 'name': "Material Naming Master", 'keywords': "material,naming,substance,images", 'description': "You are an expert at naming and identifying regions of material or texture in an image.", } )[0] shape_substance_labels = dict( MaterialShape.objects.filter(substance__isnull=False).values_list('id', 'substance_id')) shape_name_labels = dict( MaterialShape.objects.filter(name__isnull=False).values_list('id', 'name_id')) for worker in UserProfile.objects.exclude(mturk_worker_id=''): worker_substance_labels = ShapeSubstanceLabel.objects.filter(user=worker) \ .values_list('shape_id', 'substance_id') worker_name_labels = MaterialShapeNameLabel.objects.filter(user=worker) \ .values_list('shape_id', 'name_id') ngood = 0 nbad = 0 for (shape_id, worker_substance_label) in worker_substance_labels: if shape_id in shape_substance_labels: if worker_substance_label == shape_substance_labels[shape_id]: ngood += 1 else: nbad += 1 for (shape_id, worker_name_label) in worker_name_labels: if shape_id in shape_name_labels: if worker_name_label == shape_name_labels[shape_id]: ngood += 1 else: nbad += 1 if ngood + nbad > 0: perc = float(ngood) / float(ngood + nbad) if ngood >= 100: if perc >= 0.85: substance_asst, created = substance.assignments.get_or_create( worker=worker) print 'Granting substance to %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) substance_asst.set_value(1) elif ngood + nbad >= 200 and perc < 0.75: try: substance.assignments.get(worker=worker).set_value(0) print 'Revoking substance from %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) except MtQualificationAssignment.DoesNotExist: pass if perc < 0.75 and not worker.always_approve: worker.block(reason=("For naming tasks, your accuracy is %s%%, which is too low. " + "Most workers have an accuracy above 85%%.") % int(perc * 100)) print 'Blocking user %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) elif nbad >= 100 and not worker.always_approve: worker.block(reason=("For naming tasks, your accuracy is %s%%, which is too low. " + "Most workers have an accuracy above 85%%.") % int(perc * 100)) print 'Blocking user %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) # # Material Segmentation matseg = MtQualification.objects.get_or_create( slug="mat_seg", defaults={ 'name': "Material Segmentation Master", 'keywords': "material,segmentation,drawing,images", 'description': "You are an expert at drawing and identifying regions of a single type of material or texture in an image.", } )[0] # adequate work but annoying to deal with matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A1YLO4LT02D2F0'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='AU8FIYQPHKD7I'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A26GDUUFGPAYUL'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A2428ZTX6LJU33'))[0].set_value(0) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A2DHCVFGQRKU5'))[0].set_value(0) # great workers matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1) matseg.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1) good_users = dict(Counter( MaterialShape.objects .filter(correct=True) .values_list('user__mturk_worker_id', flat=True) ).most_common()) bad_users = dict(Counter( MaterialShape.objects .filter(correct=False) .values_list('user__mturk_worker_id', flat=True) ).most_common()) for (id, ngood) in good_users.iteritems(): if ngood > 50: nbad = bad_users[id] if id in bad_users else 0 perc = float(ngood) / float(ngood + nbad) if perc >= 0.8: worker = UserProfile.objects.get(mturk_worker_id=id) matseg_asst, created = matseg.assignments.get_or_create( worker=worker) if created or matseg_asst.value != 0: print 'Granting mat_seg to %s (%s good, %s bad, %s%%)' % (id, ngood, nbad, perc * 100) matseg_asst.set_value(1) elif perc < 0.6 and not worker.always_approve: worker = UserProfile.objects.get(mturk_worker_id=id) try: matseg.assignments.get(worker=worker).set_value(0) print 'Revoking mat_seg from %s (%s good, %s bad, %s%%)' % (id, ngood, nbad, perc * 100) except MtQualificationAssignment.DoesNotExist: pass # Best workers matseg2 = MtQualification.objects.get_or_create( slug="mat_seg2", defaults={ 'name': "Material Segmentation Master 2", 'keywords': "material,segmentation,drawing,images,2", 'description': "You are a super expert at drawing and identifying regions of a single type of material or texture in an image.", } )[0] matseg2.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A3LNPBZXHKKAA7'))[0].set_value(1) matseg2.assignments.get_or_create(worker=UserProfile.objects.get( mturk_worker_id='A1ENHFQSXOXG6I'))[0].set_value(1) # # Grant quals to admin if settings.MTURK_ADMIN_WORKER_ID: admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID) matseg.assignments.get_or_create(worker=admin_user)[0].set_value(1) matseg2.assignments.get_or_create(worker=admin_user)[0].set_value(1) substance.assignments.get_or_create(worker=admin_user)[0].set_value(1)
def configure_qualifications(): from mturk.models import MtQualification, MtQualificationAssignment from mturk.utils import get_or_create_mturk_worker # # BSDF matching bsdfmatch = MtQualification.objects.get_or_create( slug="bsdf_match", defaults={ 'name': "Appearance Matching Master", 'keywords': "appearance,matching,blob,graphics,BRDF", 'description': "You are an expert at matching the appearance of a synthetic blob and a shape in an image." } )[0] good_users = dict(Counter( ShapeBsdfLabel_wd.objects .filter(color_correct=True, gloss_correct=True) .values_list('user__mturk_worker_id', flat=True) ).most_common()) bad_users = dict(Counter( ShapeBsdfLabel_wd.objects .filter(Q(color_correct=False) | Q(gloss_correct=False)) .values_list('user__mturk_worker_id', flat=True) ).most_common()) for (id, ngood) in good_users.iteritems(): nbad = bad_users[id] if id in bad_users else 0 if ngood + nbad > 0: perc = float(ngood) / float(ngood + nbad) if ngood >= 30: worker = UserProfile.objects.get(mturk_worker_id=id) if perc >= 0.75: bsdfmatch_asst, created = bsdfmatch.assignments.get_or_create( worker=worker) print 'Granting bsdf_match to %s (%s good, %s bad)' % (id, ngood, nbad) bsdfmatch_asst.set_value(1) elif perc < 0.1 and not worker.always_approve: # worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " + #"Most workers have an accuracy above 75%%.") % int(perc * 100)) print 'WOULD block user %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) elif perc < 0.5: try: bsdfmatch.assignments.get(worker=worker).set_value(0) print 'Revoking bsdf_match from %s (%s good, %s bad)' % (id, ngood, nbad) except MtQualificationAssignment.DoesNotExist: pass elif nbad >= 30 and perc < 0.1 and not worker.always_approve: # worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " + #"Most workers have an accuracy above 75%%.") % int(perc * 100)) print 'WOULD block user %s (%s good, %s bad, %s%%)' % ( worker.mturk_worker_id, ngood, nbad, perc * 100) # # Grant quals to admin if settings.MTURK_ADMIN_WORKER_ID: admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID) bsdfmatch.assignments.get_or_create(worker=admin_user)[0].set_value(1)
def configure_qualifications(): from mturk.models import MtQualification, MtQualificationAssignment from mturk.utils import get_or_create_mturk_worker # # BSDF matching bsdfmatch = MtQualification.objects.get_or_create( slug="bsdf_match", defaults={ "name": "Appearance Matching Master", "keywords": "appearance,matching,blob,graphics,BRDF", "description": "You are an expert at matching the appearance of a synthetic blob and a shape in an image.", }, )[0] good_users = dict( Counter( ShapeBsdfLabel_wd.objects.filter(color_correct=True, gloss_correct=True).values_list( "user__mturk_worker_id", flat=True ) ).most_common() ) bad_users = dict( Counter( ShapeBsdfLabel_wd.objects.filter(Q(color_correct=False) | Q(gloss_correct=False)).values_list( "user__mturk_worker_id", flat=True ) ).most_common() ) for (id, ngood) in good_users.iteritems(): nbad = bad_users[id] if id in bad_users else 0 if ngood + nbad > 0: perc = float(ngood) / float(ngood + nbad) if ngood >= 30: worker = UserProfile.objects.get(mturk_worker_id=id) if perc >= 0.75: bsdfmatch_asst, created = bsdfmatch.assignments.get_or_create(worker=worker) print "Granting bsdf_match to %s (%s good, %s bad)" % (id, ngood, nbad) bsdfmatch_asst.set_value(1) elif perc < 0.1 and not worker.always_approve: # worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " + # "Most workers have an accuracy above 75%%.") % int(perc * 100)) print "WOULD block user %s (%s good, %s bad, %s%%)" % (worker.mturk_worker_id, ngood, nbad, perc * 100) elif perc < 0.5: try: bsdfmatch.assignments.get(worker=worker).set_value(0) print "Revoking bsdf_match from %s (%s good, %s bad)" % (id, ngood, nbad) except MtQualificationAssignment.DoesNotExist: pass elif nbad >= 30 and perc < 0.1 and not worker.always_approve: # worker.block(reason=("For blob matching tasks, your accuracy is %s%%, which is too low. " + # "Most workers have an accuracy above 75%%.") % int(perc * 100)) print "WOULD block user %s (%s good, %s bad, %s%%)" % (worker.mturk_worker_id, ngood, nbad, perc * 100) # # Grant quals to admin if settings.MTURK_ADMIN_WORKER_ID: admin_user = get_or_create_mturk_worker(settings.MTURK_ADMIN_WORKER_ID) bsdfmatch.assignments.get_or_create(worker=admin_user)[0].set_value(1)