def topKAverageWithTutorials(tasks, outcomes, workers, ps):
k = ps[0]
t = ps[1]
answers = []
correct = {}
mapping = {}
#do tutorials
tutorials = simulate.createBinaryTasks(t)
#ranked = sorted(correct.items(), key=operator.itemgetter(1), reverse=True)
#do tasks
totalTutorials = 0
for task in tasks:
availables = findAvailableWorkers(workers)
tops, currentTutorials = getKWorkers(availables, k, tutorials, outcomes)
totalTutorials += currentTutorials
#print tops
votes = {}
vote = None
max_vote = 0
for pick in tops:
answer = pick.doTask(task, outcomes)
key = str(answer)
votes.setdefault(key, 0)
votes[key] += 1
if votes[key] > max_vote:
max_vote = votes[key]
vote = answer
answers.append((vote, k))
#update
for worker in tops:
if worker.last_answer == vote:
worker.updateLearning(True)
else:
worker.updateLearning(False)
print totalTutorials, ' tutorials'
return answers
import matplotlib.pyplot as plot import simulate import learn import random import math from worker import Worker if __name__ == '__main__': p = 0 r_start = 1 r_end = 200 runs = 100 horizon = 200 threshold = 0.01 tasks = simulate.createBinaryTasks(horizon) outcomes = [True, False] ys1 = [] ys2 = [] tally = [] for r in range(r_start, r_end + 1): total = 0 maximum = 0 count = 0 worker = Worker(str(uuid.uuid1()), 0, p, r, 1, 1) worker.addNoise(0, 0.1) for i in range(0, runs): ts = [] cs = [] for j in range(0, horizon):
max_length = 500
xs = np.arange(1, max_length + 1, 1)
avgDiffR = []
avgDiffQ = []
diffR5 = []
diffR15 = []
diffROther = []
diffQ5 = []
diffQ15 = []
diffQOther = []
for length in range(1, max_length + 1):
resetWorkers(workers)
tasks = simulate.createBinaryTasks(length)
totalDiffR = 0.0
totalDiffQ = 0.0
totalDiffR5 = 0
totalDiffR15 = 0
totalDiffROther = 0
totalDiffQ5 = 0
totalDiffQ15 = 0
totalDiffQOther = 0
for worker in workers:
for task in tasks:
answer = worker.doTask(task, outcomes)
if answer == task:
worker.updateLearning(True)
else:
worker.updateLearning(False)
for i in range(0, len(tasks)):
aqs[i] = aqs[i] / float(runs)
ecqs[i] = ecqs[i] / float(runs)
eqs[i] = eqs[i] / float(runs)
errs[i] = errs[i] / float(runs)
fqs[i] = fqs[i] / float(runs)
result = {'cqs': cqs, 'qs': qs, 'aqs': aqs, 'ecqs': ecqs, 'eqs': eqs}
return result
if __name__ == '__main__':
print 'Regression Test'
worker = Worker(str(uuid.uuid1()), 0, 5, 20, 1, 1)
tasks = simulate.createBinaryTasks(800)
c1 = getCurves(worker, tasks, 5, 1)
c2 = getCurves(worker, tasks, 5, 0.9)
c3 = getCurves(worker, tasks, 5, 0.8)
c4 = getCurves(worker, tasks, 5, 0.7)
#initialize plots
#f, ax = plot.subplots(2, sharex=True)
x = np.arange(1, len(tasks) + 1, 1)
def dh(self, tasks, outcomes, workers, ps): #l is the horizon -> maximum number of workers to hire #s is the number of samples l = ps[0] s = ps[1] t = ps[2] #number of tutorials result = [] total_tasks = len(tasks) completed_tasks = 0 #running tutorials tutorials = simulate.createBinaryTasks(t) #step = 10 #step_counter = 0 #self.reset() #self.calculateAverageWorkerQuality(workers) #availables = getAvailableWorkers(workers) #rankedWorkers = self.randomRank(workers, total_tasks - completed_tasks, l) #self.rankWorkers(workers, total_tasks - completed_tasks)# - completed_tasks) #self.sample(s, l, outcomes, availables) #self.evaluate(outcomes) totalTutorials = 0 for task in tasks: #self.reset() #self.sample(s, l, task, outcomes, self.rankWorkers(workers, total_tasks - completed_tasks)) #continue #self.evaluate(outcomes) #print 'Evaluation done' #step_counter = (step_counter + 1) % step #if step_counter == 0: #this rerank workers and do sampling and evaluation again self.reset() self.calculateAverageWorkerQuality() #rankedWorkers = self.randomRank(workers, total_tasks - completed_tasks, l) #rankedWorkers = self.rankWorkers(workers, total_tasks - completed_tasks)# - completed_tasks) availables = self.getAvailableWorkers(workers) rankedWorkers = self.getRankedWorkers(availables, l, total_tasks - completed_tasks) self.sample(s, l, outcomes, rankedWorkers)#, rankedWorkers) self.evaluate(outcomes) self.hire_pointer = self.root last_hire = None last_answer = None hired = [] answers = [] while True: if len(hired) >= l: break else: next_worker = self.hireNext(last_hire, last_answer) if next_worker is None: break else: if next_worker.x < t: for tutorial in tutorials: answer = next_worker.doTask(tutorial, outcomes) if answer == tutorial: next_worker.updateLearning(True) else: next_worker.updateLearning(False) next_worker.learn() totalTutorials += t answer = next_worker.doTask(task, outcomes, next_worker.c) hired.append(next_worker) answers.append(answer) last_hire = next_worker last_answer = answer #hiring is done prediction, probability = self.aggregate(hired, answers, outcomes) #print '----------' #for i in range(0, len(hired)): # worker = hired[i] # answer = answers[i] # print answer, worker.getHybridQuality() #print prediction, probability, '|', task #print '-----' #print answers #for w in hired: # print w.getEstimatedQualityAtX(w.x) #print 'actual result', prediction, probability #print answers, prediction #update all hired workers self.update(hired, answers, prediction) result.append((prediction, len(hired))) #if len(hired) == 1: #print task, prediction, self.getCappedQuality(hired[0].getHybridQuality(), self.average_worker_quality, hired[0].x), hired[0].getHybridQuality(), hired[0].getQuality() completed_tasks += 1 print totalTutorials, ' tutorials' return result
