def make_usernames_csv_for_turk():
print "Creating csv file of usernames for evaluation on Mechanical Turk..."
twitter_site = short_text_websites.get_twitter_site()
usernames_to_evaluate = crosssite_username_dataset_mgr.get_usernames_to_evaluate_mturk(twitter_site)
print str(len(usernames_to_evaluate))+" unconfirmed usernames available:"
print usernames_to_evaluate
# dump to csv
username_rows = [[username] for username in usernames_to_evaluate]
csv_util.write_to_spreadsheet(__usernames_to_judge_csv_path__, username_rows, None)
def update_usernames_csv_with_judgments():
''' Handles updating the cross-site-usernames
spreadsheet given new Mechanical Turker evaluations '''
print "Updating cross-site-usernames spreadsheet with Mechanical Turker "+\
"judgments about username matching from spreadsheet of results..."
twitter_site = short_text_websites.get_twitter_site()
evaluated_usernames = crosssite_username_dataset_mgr.get_usernames_to_evaluate_mturk(twitter_site)
judgments = {}
row_num = 0
rows_plus_headers = csv_util.query_csv_for_rows(__usernames_results_csv_path__, False)
for row in rows_plus_headers:
try:
if row_num==0: # row 0 is header
username_col = row.index('Input.user')
turkerID_col = row.index('WorkerId')
answer_col = row.index('Answer.Q1')
else:
judged_username = row[username_col]
if not judged_username in evaluated_usernames:
raise # this shouldn't happen so see what's going on..
if judged_username in judgments:
evaluations = judgments[judged_username]
else:
evaluations = Turker_Username_Evaluation(judged_username)
judgments[judged_username] = evaluations
workerID = row[turkerID_col]
judgment = row[answer_col]
if 'True'==judgment:
evaluations.add_true_eval(workerID)
elif 'False'==judgment:
evaluations.add_false_eval(workerID)
else:
evaluations.add_unknown_eval(workerID)
row_num = row_num+1
except:
continue # just ignore a problematic assignment row
# usernames that meet a threshold indicating that enough
# of a majority of turkers agreed this is the same individual
likely_usernames = []
conflicting_judgments = []
# usernames that all workers agreed belonged to a single
# person (ie ignore usernames that were rejected by any worker)
unanimous_confirmed_usernames = []
# Also get usernames that all workers unanimously
# agreed do NOT belong to a single individual
unanimous_nonmatch_usernames = []
for username in judgments:
if username in __MANUAL_OVERRIDES_TRUE__:
likely_usernames.append(username)
continue
evaluation = judgments[username]
eval_score = evaluation.get_eval_measure()
print "Match score for "+str(username)+": "+str(eval_score)
if eval_score>=__JUDGMENT_THRESHOLD__:
likely_usernames.append(username)
elif evaluation.get_number_true_evals()>0:
conflicting_judgments.append(username)
# Each username given to 5 turkers to evaluate.
if (evaluation.get_number_true_evals()>0 and
evaluation.get_number_false_evals()==0 and
evaluation.get_number_unknown_evals()==0):
# all turkers unanimously confirmed this username
unanimous_confirmed_usernames.append(username)
if (evaluation.get_number_false_evals()>0 and
evaluation.get_number_true_evals()==0 and
evaluation.get_number_unknown_evals()==0 and
not username.lower() in __MANUAL_OVERRIDES_FALSE_UNCLEAR__):
unanimous_nonmatch_usernames.append(username)
print "Judged "+str(len(judgments))+" usernames"
print "Likely matches"+str(len(likely_usernames))
print "Conflicting judgments:"+str(conflicting_judgments)
# Update the judgment cell in the spreadsheet for unanimously confirmed usernames
crosssite_username_dataset_mgr.update_confirmed_positive_usernames(twitter_site, likely_usernames)
print "Updated cross-site-usernames spreadsheet to reflect majority positive confirmations"
crosssite_username_dataset_mgr.update_confirmed_negative_usernames(twitter_site, unanimous_nonmatch_usernames)
print "Updated cross-site-usernames spreadsheet to reflect unanimous negative confirmations"
