def topKAverageWithTutorials(tasks, outcomes, workers, ps): k = ps[0] t = ps[1] answers = [] correct = {} mapping = {} #do tutorials tutorials = simulate.createBinaryTasks(t) #ranked = sorted(correct.items(), key=operator.itemgetter(1), reverse=True) #do tasks totalTutorials = 0 for task in tasks: availables = findAvailableWorkers(workers) tops, currentTutorials = getKWorkers(availables, k, tutorials, outcomes) totalTutorials += currentTutorials #print tops votes = {} vote = None max_vote = 0 for pick in tops: answer = pick.doTask(task, outcomes) key = str(answer) votes.setdefault(key, 0) votes[key] += 1 if votes[key] > max_vote: max_vote = votes[key] vote = answer answers.append((vote, k)) #update for worker in tops: if worker.last_answer == vote: worker.updateLearning(True) else: worker.updateLearning(False) print totalTutorials, ' tutorials' return answers
import matplotlib.pyplot as plot import simulate import learn import random import math from worker import Worker if __name__ == '__main__': p = 0 r_start = 1 r_end = 200 runs = 100 horizon = 200 threshold = 0.01 tasks = simulate.createBinaryTasks(horizon) outcomes = [True, False] ys1 = [] ys2 = [] tally = [] for r in range(r_start, r_end + 1): total = 0 maximum = 0 count = 0 worker = Worker(str(uuid.uuid1()), 0, p, r, 1, 1) worker.addNoise(0, 0.1) for i in range(0, runs): ts = [] cs = [] for j in range(0, horizon):
max_length = 500 xs = np.arange(1, max_length + 1, 1) avgDiffR = [] avgDiffQ = [] diffR5 = [] diffR15 = [] diffROther = [] diffQ5 = [] diffQ15 = [] diffQOther = [] for length in range(1, max_length + 1): resetWorkers(workers) tasks = simulate.createBinaryTasks(length) totalDiffR = 0.0 totalDiffQ = 0.0 totalDiffR5 = 0 totalDiffR15 = 0 totalDiffROther = 0 totalDiffQ5 = 0 totalDiffQ15 = 0 totalDiffQOther = 0 for worker in workers: for task in tasks: answer = worker.doTask(task, outcomes) if answer == task: worker.updateLearning(True) else: worker.updateLearning(False)
for i in range(0, len(tasks)): aqs[i] = aqs[i] / float(runs) ecqs[i] = ecqs[i] / float(runs) eqs[i] = eqs[i] / float(runs) errs[i] = errs[i] / float(runs) fqs[i] = fqs[i] / float(runs) result = {'cqs': cqs, 'qs': qs, 'aqs': aqs, 'ecqs': ecqs, 'eqs': eqs} return result if __name__ == '__main__': print 'Regression Test' worker = Worker(str(uuid.uuid1()), 0, 5, 20, 1, 1) tasks = simulate.createBinaryTasks(800) c1 = getCurves(worker, tasks, 5, 1) c2 = getCurves(worker, tasks, 5, 0.9) c3 = getCurves(worker, tasks, 5, 0.8) c4 = getCurves(worker, tasks, 5, 0.7) #initialize plots #f, ax = plot.subplots(2, sharex=True) x = np.arange(1, len(tasks) + 1, 1)
def dh(self, tasks, outcomes, workers, ps): #l is the horizon -> maximum number of workers to hire #s is the number of samples l = ps[0] s = ps[1] t = ps[2] #number of tutorials result = [] total_tasks = len(tasks) completed_tasks = 0 #running tutorials tutorials = simulate.createBinaryTasks(t) #step = 10 #step_counter = 0 #self.reset() #self.calculateAverageWorkerQuality(workers) #availables = getAvailableWorkers(workers) #rankedWorkers = self.randomRank(workers, total_tasks - completed_tasks, l) #self.rankWorkers(workers, total_tasks - completed_tasks)# - completed_tasks) #self.sample(s, l, outcomes, availables) #self.evaluate(outcomes) totalTutorials = 0 for task in tasks: #self.reset() #self.sample(s, l, task, outcomes, self.rankWorkers(workers, total_tasks - completed_tasks)) #continue #self.evaluate(outcomes) #print 'Evaluation done' #step_counter = (step_counter + 1) % step #if step_counter == 0: #this rerank workers and do sampling and evaluation again self.reset() self.calculateAverageWorkerQuality() #rankedWorkers = self.randomRank(workers, total_tasks - completed_tasks, l) #rankedWorkers = self.rankWorkers(workers, total_tasks - completed_tasks)# - completed_tasks) availables = self.getAvailableWorkers(workers) rankedWorkers = self.getRankedWorkers(availables, l, total_tasks - completed_tasks) self.sample(s, l, outcomes, rankedWorkers)#, rankedWorkers) self.evaluate(outcomes) self.hire_pointer = self.root last_hire = None last_answer = None hired = [] answers = [] while True: if len(hired) >= l: break else: next_worker = self.hireNext(last_hire, last_answer) if next_worker is None: break else: if next_worker.x < t: for tutorial in tutorials: answer = next_worker.doTask(tutorial, outcomes) if answer == tutorial: next_worker.updateLearning(True) else: next_worker.updateLearning(False) next_worker.learn() totalTutorials += t answer = next_worker.doTask(task, outcomes, next_worker.c) hired.append(next_worker) answers.append(answer) last_hire = next_worker last_answer = answer #hiring is done prediction, probability = self.aggregate(hired, answers, outcomes) #print '----------' #for i in range(0, len(hired)): # worker = hired[i] # answer = answers[i] # print answer, worker.getHybridQuality() #print prediction, probability, '|', task #print '-----' #print answers #for w in hired: # print w.getEstimatedQualityAtX(w.x) #print 'actual result', prediction, probability #print answers, prediction #update all hired workers self.update(hired, answers, prediction) result.append((prediction, len(hired))) #if len(hired) == 1: #print task, prediction, self.getCappedQuality(hired[0].getHybridQuality(), self.average_worker_quality, hired[0].x), hired[0].getHybridQuality(), hired[0].getQuality() completed_tasks += 1 print totalTutorials, ' tutorials' return result