def get_results(tid, assignmentid, workerid, hid):
try:
db = get_db()
programname = db.get_tasklet_program(tid)
key = HaCRSUtil.get_compositekey(tid, assignmentid, workerid, hid)
tasklet = db.get_full_tasklet(tid)
next_payout, next_transition, curr_payout, curr_transition = -1, -1, -1, -1
if tasklet == None:
return "No task found"
if tasklet['type'] == 'SEED':
pass
#render_me['payout_arr'] = tasklet['payout_arr']
config_path = os.path.dirname(__file__) + "/webconfig.ini"
config = HaCRSUtil.get_config(config_path)
# config = HaCRSUtil.get_config('webconfig.ini')
xfile = HaCRSUtil.find_result_file(config, programname, key)
results = json.loads(open(xfile).readlines()[-1])
next_payout, next_transition = HaCRSUtil.get_next_payout_border(tasklet['payout_arr'], results['new_transitions'])
curr_payout = HaCRSUtil.get_current_payout(tasklet['payout_arr'], results['new_transitions'])
results['next_payout'] = next_payout
results['next_transition'] = next_transition
results['curr_payout'] = curr_payout
results['min_payout'] = tasklet['amount']
return json.dumps(results)
except Exception as e:
return "{}"
def __init__(self):
self.config = HaCRSUtil.get_config('../config.ini')
HOST = self.config.get('mturk','host')
AWS_ACCESS_KEY_ID = self.config.get('mturk', 'access_key_id')
AWS_SECRET_ACCESS_KEY = self.config.get('mturk', 'secret_access_key')
self.MTconnection = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
host=HOST)
self.db = HaCRSDB()
self.con, self.cur = HaCRSUtil.get_db(self.config)
self.mt = HaCRSTurker()
def push_tasks_mturk(self):
frame_height = self.config.get('mturk', 'frameheight')
amount = 0.01
tasklets = self.db.get_unassigned_tasklets()
sdescription = self.config.get('mturk', 'shortdescr')
for tasklet in tasklets:
#print 'pushing!'
url = "https://cgcturk.hacked.jp/tasklet/{}/".format(tasklet['id'])
keywords = ["easy"]
questionform = ExternalQuestion(url, frame_height)
hit_result = self.MTconnection.create_hit(
title=HaCRSUtil.get_tasklet_name(tasklet),
description=sdescription,
keywords=keywords,
max_assignments=1,
question=questionform,
reward=Price(amount=amount),
response_groups=('Minimal', 'HITDetail'), # ?
)
assert len(hit_result) == 1
mturkid = self.db.create_mturk_resource(hit_result[0].HITId,
hit_result[0].HITGroupId)
self.db.add_mturk_tasklet_association(tasklet['id'], mturkid)
self.db.commit()
def create_seed_tasklet( ):
input_args = ['programname', 'total_transitions', 'previous_transitions']
try:
args = {}
for x in input_args:
assert x in request.get_json().keys()
args[x] = request.get_json()[x]
except Exception as e:
raise InvalidUsage('Required argument missing')
try:
db = get_db()
config = get_config()
pid = db.lookup_program(args['programname'])
assert pid != None
rating, base_pay, payout_array = HaCRSUtil.hit_info(int(args['total_transitions']) - int(args['previous_transitions']), int(args['total_transitions']))
tid = db.create_seed_tasklet(pid, base_pay, payout_array, rating)
#except Exception as e:
except IOError as e:
return repr(e)
raise InvalidUsage('Error creating tasklet: {}'.format(e))
try:
pass
#mt = get_mt()
# push tasklet to mturk!
#mturkid, hit_result = mt.push_tasklet_mturk(tid)
except Exception as e:
raise InvalidUsage()
return json.dumps({'tasklet_id': str(tid), 'base_pay': base_pay, 'rating': rating, 'payout_array': payout_array})
def get_config(self, conffile='config.ini'):
try:
config_path = os.path.dirname(__file__) + "/../config.ini"
self.config = HaCRSUtil.get_config(config_path)
#self.config = configparser.configparser()
#self.config.readfp(open(conffile))
self.config.readfp(open(config_path))
except Exception as e:
sys.stderr.write("Couldn't read config ({}): {}\n".format(
conffile, e))
sys.exit(1)
def create_tasklet(self, typ, program, base_pay_cents, keywords):
# to be safe
assert base_pay_cents < 2000
assert HaCRSUtil.is_tasklet_type(typ)
base_pay = round(float(base_pay_cents) / 100, 2)
self.cur.execute('insert into tasklets (type, program, amount, keywords ) values (%s, %s, %s, %s) returning id', [typ, program, base_pay, keywords])
taskletid = self.cur.fetchone()[0]
self.con.commit()
return taskletid
def get_merged_interactions(programname, xid):
config_path = os.path.dirname(__file__) + "/webconfig.ini"
config = HaCRSUtil.get_config(config_path)
# config = HaCRSUtil.get_config('webconfig.ini')
print('=========================== xid = ', xid)
if xid.startswith('AUTO'):
seed_dir = 'afl_seeds'
else:
seed_dir = 'minified_seeds'
xdir = "{}/{}/{}".format(config.get('general', 'resultsdir'), programname.replace(' ', '_'), seed_dir)
influence = json.loads(get_printable_string(resource_stream('mtweb', '{}/{}.influence.json'.format(xdir,xid)).read()))
output = make_printable_string(resource_stream('mtweb', '{}/{}.output'.format(xdir, xid)).read())
interactions = json.loads(get_printable_string(resource_stream('mtweb', '{}/{}.interaction.json'.format(xdir,xid)).read()))
print('pulled {}/{}.interaction.json'.format(xdir, xid))
#character_similarities = read_similarities_csv(open('{}/{}.character_similarities.csv'.format(xdir,xid)).read())
compartments = json.loads(get_printable_string(resource_stream('mtweb', '{}/{}.compartment_information.json'.format(xdir, xid)).read()))
print('pulled {}/{}.compartment_information.json'.format(xdir, xid))
in_out_merge = {}
try:
for interaction in interactions:
if interaction['type'] == 'write':
for offset in interaction['offsets']:
in_out_merge[str(len(in_out_merge))]={'value': output[offset], 'type': 'input'}
elif interaction['type'] == 'read':
for offset in interaction['offsets']:
in_out_merge[str(len(in_out_merge))]={'value': influence[str(offset)]['value'] ,
'reachable_strings': influence[str(offset)]['reachable_strings'] ,
'other_options': influence[str(offset)]['other_options'] ,
#'similarities': character_similarities[offset],
'compartment': find_compartment(compartments, offset),
'type': 'output'}
else:
sys.stderr.write('?\n')
except Exception as e:
import traceback
#print offset, len(output)
traceback.print_exc()
raise
#print 'dumping in_out_merge json'
return json.dumps(in_out_merge)
def showtid(tid, aid = None, hid=None, wid = None, showinput=None):
if aid == None:
aid = request.args.get("assignmentId", "")
if hid == None:
hid = request.args.get("hitId", "")
if wid == None:
wid = request.args.get("workerId", "")
if showinput == None:
showinput = request.args.get("showinput", "")
instructions = {}
instructions['SEED'] = 'instructions/seeding.html'
templates = {}
templates['SEED'] = 'templates/seedtasklet.html'
try:
assert len(tid) == 36
db = get_db()
config_path = os.path.dirname(__file__) + "/webconfig.ini"
config = HaCRSUtil.get_config(config_path)
# config = HaCRSUtil.get_config('webconfig.ini')
render_me = {}
tasklet = db.get_full_tasklet(tid)
if tasklet == None:
return "No task found (or not implemented)"
seeds = get_interactions(config, tasklet['program'])
if not ( aid == "" or aid == "ASSIGNMENT_ID_NOT_AVAILABLE"):
execution_id = uuid.uuid4()
turkerinfo = HaCRSUtil.get_compositekey(tid, aid, wid, hid)
password = random_vnc_password()
vr = VNCRunner(tasklet, turkerinfo, password)
vr_thread = Thread(target = vr_thread_runner, args = (vr, ))
vr_thread.start()
render_me['vrport'] = vr.useport
render_me['password'] = password
db.log_session_start(tid, hid, wid, aid, execution_id,
str(request.user_agent), request.remote_addr)
render_me['amazon_host'] = config.get('mt', 'host')
render_me['assignment_id'] = aid
render_me['tasklet_id'] = tid
render_me['hit_id'] = hid
render_me['difficulty'] = tasklet['keywords']
render_me['seeds'] = seeds
render_me['tasktype'] = tasklet['type']
render_me['programname'] = tasklet['program'].replace('_', ' ')
render_me['programinstructions'] = get_printable_string(get_program_description(config, tasklet['program']))
render_me['worker_id'] = wid
render_me['taskname'] = HaCRSUtil.get_tasklet_name(tasklet)
# render_me['instructions'] = open(instructions[tasklet['type']]).read()
render_me['instructions'] = resource_stream('mtweb', instructions[tasklet['type']]).read()
render_me['showinput'] = urllib.parse.quote(showinput)
return do_render(templates[tasklet['type']], render_me)
except Exception as e:
return "Couldn't fetch task: {}".format(e)
def __init__(self):
config_path = os.path.dirname(__file__) + "/../config.ini"
self.config = HaCRSUtil.get_config(config_path)
HOST = self.config.get('mturk','host')
self.con, self.cur = HaCRSUtil.get_db(self.config)
psycopg2.extras.register_uuid()
def get_config():
if not hasattr(g, 'config'):
config_path = os.path.dirname(__file__) + "/../config.ini"
g.config = HaCRSUtil.get_config(config_path)
# g.config = HaCRSUtil.get_config('../config.ini')
return g.config
def get_solve_ratio(self):
global EXPERIMENT_START
prog_maxcoverage = {}
taskid_earnings = {}
total_payout_with_bonus = 0
goalreached = 0
goalnotreached = 0
for program in json.load(open(self.config.get('general', 'programsjson'))):
prog_maxcoverage[program] = 0
program = None
seed_taskletid_solved = {}
empty = { 'easy': 0, 'medium': 0, 'hard': 0, 'very_hard': 0, 'priority': 0 }
for tasklet in self.db.get_seed_tasklets() + self.db.get_seek_tasklets():
if not str(tasklet['timestamp']).startswith(EXPERIMENT_START):
continue
for jfile in glob.glob('{}/{}/{}*/*.json'.format(self.config.get('general', 'resultsfolder'), tasklet['program'], str(tasklet['id']))):
try:
metadata = self.split_composite_key(jfile.split(os.path.sep)[jfile.split(os.path.sep).index('result.json')-1])
# fake keys
except Exception as e:
#print e
continue
# that's us
if metadata['workerid'] == 'A2PRAI0ABXN99X':
continue
results = json.loads(open(jfile).readlines()[-1])
tasklet = self.db.get_full_tasklet(metadata['tid'])
if tasklet == None:
#print "No tasklet for program {}".format(tasklet['program'])
continue
if tasklet['type'] == 'SEED':
payout = HaCRSUtil.get_current_payout(tasklet['payout_arr'], results['new_transitions'])
prog_maxcoverage[tasklet['program']] = max(prog_maxcoverage[tasklet['program']], results['coverage'])
elif tasklet['type'] == 'SEEK':
payout = tasklet['amount']
elif tasklet['type'] == 'DRILL':
payout = tasklet['amount']
total_payout_with_bonus += payout
# over-achieved
if tasklet['amount'] <= payout:
goalreached += 1
elif tasklet['amount'] > payout:
goalnotreached += 1
if tasklet['amount'] <= payout and tasklet['type'] == 'SEED':
hitinfos = self.db.get_hit_for_tasklet(tasklet['id'])
for hit in hitinfos:
assignment = self.mt.get_assignment_from_hit(hit)
if assignment and assignment.WorkerId in ['A10O5YR01H865K', 'A1HRHFU7KTS0KW', 'A1PUHCEBSOWETV']:
pass
seed_taskletid_solved[tasklet['id']] = True
if metadata['tid'] not in taskid_earnings.keys():
taskid_earnings[metadata['tid']] = {}
if metadata['workerid'] not in taskid_earnings[metadata['tid']].keys():
taskid_earnings[metadata['tid']][metadata['workerid']] = {}
taskid_earnings[metadata['tid']][metadata['workerid']] = {'payout': payout, 'amount': tasklet['amount']}
#print "Goal reached: {}, Goal not reached: {}". format(goalreached, goalnotreached)
#print "taskid_earnings"
pprint(taskid_earnings)
return taskid_earnings, prog_maxcoverage, seed_taskletid_solved
def get_seed_stats(self, seed_taskletid_solved):
global EXPERIMENT_START
unique_seed_workers = set()
worker_payouts_base = {}
worker_payouts_bonus = {}
worker_payouts_combined = {}
worker_solves = {}
tasklet_solved = set()
total_payout_base = 0
total_payout_bonus = 0
tasklet_difficulty = {}
program_solves = {}
for program in json.load(open(self.config.get('general', 'programsjson'))):
prog_maxcoverage[program] = 0
if program not in program_solves:
program_solves[program] = 0
program = None
total_payout = 0
for tasklet in self.db.get_seed_tasklets():
if not str(tasklet['timestamp']).startswith(EXPERIMENT_START):
continue
if tasklet['program'] in ['seed_training', 'A_Game_of_Chance']:
continue
if tasklet['id'] in seed_taskletid_solved.keys():
program_solves[tasklet['program']] += 1
for jfile in glob.glob('{}/{}/{}*/*.json'.format(self.config.get('general', 'resultsfolder'), tasklet['program'], str(tasklet['id']))):
try:
metadata = self.split_composite_key(jfile.split(os.path.sep)[jfile.split(os.path.sep).index('result.json')-1])
# fake keys
except Exception as e:
continue
# that's us
if metadata['workerid'] == 'A2PRAI0ABXN99X':
continue
results = json.loads(open(jfile).readlines()[-1])
tasklet = self.db.get_full_tasklet(metadata['tid'])
payout = HaCRSUtil.get_current_payout(tasklet['payout_arr'], results['new_transitions'])
prog_maxcoverage[tasklet['program']] = max(prog_maxcoverage[tasklet['program']], results['coverage'])
if payout > 0:
if metadata['workerid'] not in worker_solves.keys():
worker_solves[metadata['workerid']] = 0
worker_solves[metadata['workerid']] += 1
unique_seed_workers.add(metadata['workerid'])
if tasklet['keywords'] not in tasklet_difficulty:
tasklet_difficulty[tasklet['keywords']] = 0
tasklet_difficulty[tasklet['keywords']] += 1
tasklet_solved.add(tasklet['id'])
if metadata['workerid'] not in worker_payouts_base.keys():
worker_payouts_base[metadata['workerid']] = []
worker_payouts_bonus[metadata['workerid']] = []
worker_payouts_combined[metadata['workerid']] = []
worker_payouts_base[metadata['workerid']].append(tasklet['amount'])
total_payout_base += tasklet['amount']
if payout > tasklet['amount']:
total_payout_bonus += round(payout - tasklet['amount'], 2)
worker_payouts_bonus[metadata['workerid']].append(round(payout - tasklet['amount'], 2))
total_payout_bonus += round(payout - tasklet['amount'], 2)
worker_payouts_combined[metadata['workerid']].append(round(payout, 2))
total_payout += payout
pass
#print 'Total seed BASE payment: $ {}'.format(total_payout_base)
#print 'Total seed BONUS payment: $ {}'.format(total_payout_bonus)
#print 'Workers solving at least one SEED Task: {}'.format(len(unique_seed_workers))
#print 'Number of solved SEED tasks: {}'.format(len(tasklet_solved))
#print 'Number of tasklets by difficulty: {}'.format(tasklet_difficulty)
#print 'Busiest worker: {} solves'.format(max(worker_solves.values()))
#print 'Average worker throughput: {} solves'.format(round(sum(worker_solves.values()) / float(len(worker_solves.values())), 2))
pdb.set_trace()