import random
import simplejson
from estimator import chao92, sChao92, nominal, vNominal, sNominal, remain_switch, gt_switch, extrapolation, extrapolation2, extrapolation3, vRemainSwitch, vRemainSwitch2, extrapolateFromSample, sampleCoverage, minTasks, minTasksToCleanAll
from datagen import generateDist, generateDataset, generateWeightedDataset, shuffleList
from dataload import simulatedData, loadCrowdFlowerData, loadRestaurant2, loadProduct, loadRestaurantExtSample, loadAddress
import pickle
from simulation import plotMulti, plotY1Y2, holdoutRealWorkers
##############################################
########### real-world datasets ##############
##############################################
#====total error====
estimators = [
lambda x: extrapolateFromSample(x, slist, 0.05) + obvious_err,
lambda x: vNominal(x) + obvious_err,
lambda x: sChao92(x, shift=1) + obvious_err,
lambda x: vRemainSwitch2(x) + obvious_err
]
#estimators = [lambda x: vNominal(x) + obvious_err, lambda x: chao92(x)+obvious_err]
legend = ["EXTRAPOL", "VOTING", "V-CHAO", "SWITCH"]
gt_list = [
lambda x: gt + obvious_err, lambda x: gt + obvious_err,
lambda x: gt + obvious_err, lambda x: gt + obvious_err
]
legend_gt = ["Ground Truth"]
#====switch====
estimators2 = [lambda x: remain_switch(x) - sNominal(x)]
estimators2a = [
lambda x: remain_switch(x, neg_switch=False) - sNominal(x,
import pickle from simulation import plotMulti, plotY1Y2, holdoutRealWorkers ###################################### ###########simulated dataset########## ###################################### logscale = False dirty = 0.2 n_items = 1000 n_rep = 10 print 'Sensitivity of Total Error Estimation' #estimators_sim = [vNominal, chao92, lambda x:sChao92(x,shift=1),lambda x:vRemainSwitch2(x)] #legend_sim = ['VOTING','Chao92','V-CHAO','SWITCH'] #gt_list_sim = [lambda x:gt,lambda x:gt,lambda x:gt,lambda x:gt] estimators_sim = [lambda x: vNominal(x), chao92, lambda x: vRemainSwitch2(x)] gt_list_sim = [lambda x: gt, lambda x: gt, lambda x: gt] legend_sim = ['VOTING', 'Chao92', 'SWITCH'] legend_gt = ["Ground Truth"] yaxis = 'SRMSE' #'Relative Error %' rel_err = True err_skew = False ''' title = 'Tradeoff: False Positives' recall = 0.1 n_worker=50 font = 20 Xs = [] Ys = [] GTs = []
plt.ylim(ymin=0,ymax=ymax)
print num_gt + num_estimators, legend_gt + legend
#plt.xlim(xmin=points[0][0], xmax=points[0][len(points[0])-1])
plt.legend(legend_gt,loc=loc)
plt.grid(True)
plt.savefig(filename,bbox_inches='tight')#,format='pdf')
##############################################
########### real-world datasets ##############
##############################################
#total error
#estimators = [lambda x: sChao92(x,shift=1),lambda x: vGoodToulmin(x,c=2),lambda x: vGoodToulmin(x,c=10),vNominal]
#legend = ["$vChao92 (s=1)$","$GT (m=2\sqrt{n})$","$GT (m=10\sqrt{n})$","$majority$"]
#estimators = [lambda x:vNominal(x)+obvious_err,lambda x:vNominal(x)+obvious_err-majority_fp(x,slist),lambda x: sChao92(x,shift=1)+obvious_err,lambda x:vRemainSwitch(x)+obvious_err,lambda x:nChao92(x,neg_support=False)+obvious_err, lambda x: nChao92(x,neg_support=True)+obvious_err]
#legend = ["Majority","Majority-FP","vChao92 (s=1)","sChao92+Majority","nChao92+","nChao92-"]
estimators = [lambda x:vNominal(x)+obvious_err,lambda x:vRemainSwitch(x)+obvious_err, lambda x: dChao92(x,1) + obvious_err, lambda x: dChao92(x,0)+obvious_err, lambda x: (dChao92(x,1)+dChao92(x,0))/2.+obvious_err,lambda x: unseen(x)+obvious_err]
legend = ["Majority","sChao92+Majority","dChao92(1)","dChao92(0)","dChao92(0/1)","unseen"]
estimators = [lambda x:vNominal(x)+obvious_err,lambda x:(dChao92(x,1)+dChao92(x,0))/2.+obvious_err,lambda x: unseen(x)+obvious_err, lambda x: twoPhase(x)+obvious_err]
legend = ["Majority","dChao92(0/1)","Unseen","twoPhase"]
majorities = [lambda x: majority_fp(x,slist), lambda x:majority_fn(x,slist)]
legend_maj = ['False Positive Majority','False Negative Majority']
#estimators = [vGoodToulmin,lambda x: sChao92(x,shift=1),vNominal]
gt_list = [lambda x: gt+obvious_err]
#legend=["fChao92","wChao92","vChao92(s=1)","Nominal (Voting)"]
legend_gt=["Ground Truth"]
#estimators_rel = [lambda x:100*float(math.fabs(chao92(x)-gt))/gt,lambda x:100*float(math.fabs(vNominal(x)-gt))/gt,lambda x: 100*float(math.fabs(sChao92(x,shift=1)-gt))/gt]
#switch
estimators2 = [lambda x: remain_switch(x,slist=slist) - sNominal(x)]
estimators2a = [lambda x: remain_switch(x,neg_switch=False) - sNominal(x,neg_switch=False)]
estimators2b = [lambda x: remain_switch(x,pos_switch=False) - sNominal(x,pos_switch=False)]
gt_list2 = [lambda x: gt_switch(x,slist)]