def plot_ccdf(self, women_values, men_values, labels, filename, xlabel, xlog, ylog): item_frequency_female = itemfreq(women_values) item_frequency_male = itemfreq(men_values) ccdf= 1 ut.plot_cdf(list([item_frequency_female, item_frequency_male]), labels, ['pink','blue'], filename, xlabel, ccdf, xlog, ylog)
def analyze_num_lang_edition(self, classname, people): path = "img/" + classname men = people[people.gender == "male"] women = people[people.gender == "female"] # create folder if not exist if not os.path.exists(path): os.makedirs(path) self.logfile.write("\n\n\n\ " + classname + "\n") edition_counts = np.append(women.edition_count.values, men.edition_count.values) labels = ["female (" + str(len(women.index)) + ")", "male (" + str(len(men.index)) + ")"] # print edition_counts # print type(edition_counts)probability that a score randomly drawn from population A will be greater than a score randomly drawn from population B. max_num_editions = np.max(edition_counts) # print max_num_editions top_men = [] men_percentage = 0 topk = range(10, 110, 10) if men.shape[0] > 0: men_percentage = [ men[men.edition_count == x].shape[0] / float(men.shape[0]) for x in range(1, max_num_editions) ] self.logfile.write("\n % of local men ") self.logfile.write(str(men[men.edition_count < 2].shape[0] / float(men.shape[0]))) self.logfile.write( "\n percentage men %s" % str(len(men.index) / float((len(women.index) + len(men.index)))) ) men_vals = men.edition_count.order(ascending=True).values women_percentage = 0 if women.shape[0] > 0: women_percentage = [ women[women.edition_count == x].shape[0] / float(women.shape[0]) for x in range(1, max_num_editions) ] self.logfile.write("\n\n % local women ") self.logfile.write(str(women[women.edition_count < 2].shape[0] / float(women.shape[0]))) self.logfile.write( "\n percentage women %s" % str(len(women.index) / float((len(women.index) + len(men.index)))) ) women_vals = women.edition_count.order(ascending=True).values if women.shape[0] > 0 and men.shape[0] > 0: j_m = 0 j_w = 0 U, p = stats.mstats.mannwhitneyu(women.edition_count, men.edition_count) ut.write_mannwithneyu(U, p, women.edition_count, men.edition_count, self.logfile) # for i in topk: # perc_men = len(men.edition_count.index) * i/100.0 # print "bottom %s percent of men are %s - mean %s - median %s"% (str(i), str(perc_men), np.mean(men_vals[:int(perc_men)]), np.median(men_vals[:int(perc_men)])) # top_men.append(men_vals[:int(perc_men)]) # perc_women = len(women.edition_count.index) * i/100.0 # print "bottom %s percent of women are %s - mean %s - median %s"% (str(i), str(perc_women), np.mean(women_vals[:int(perc_men)]), np.median(women_vals[:int(perc_men)])) # top_women.append(women_vals[:int(perc_women)]) # j_m += int(perc_men) # j_w += int(perc_women) # ut.plotTopk(top_women, top_men, ['pink', 'blue'], topk, path+'/numedition_gender_topk.png') ut.plot_percentage( women_percentage, men_percentage, ["pink", "blue"], range(1, max_num_editions), path + "/numedition_gender_percentage" + self.pre + "-" + self.post + ".png", ) self.genderBoxplots(women, men, labels, path) self.logfile.write("\n\n num women %s" % len(women.index)) self.logfile.write("\n num men %s" % len(men.index)) data = [women.edition_count.values, men.edition_count.values] # Compute the qth percentile of women and men q75 = [np.percentile(x, q=75) for x in data] self.logfile.write("\n third quartil (75th percentile): " + str(q75)) q95 = [np.percentile(x, q=95) for x in data] self.logfile.write("\n 95th percentile: " + str(q95)) q99 = [np.percentile(x, q=99) for x in data] self.logfile.write("\n 99th percentile: " + str(q99)) q99_women = q99[0] q99_men = q99[1] self.logfile.write("\n threshold women 99th percentile: %s" % q99_women) self.logfile.write("\n threshold men 99th percentile: %s" % q99_men) men_percentage = 0 th = np.min(q99) if men.shape[0] > 0: men_percentage = [men[men.edition_count == x].shape[0] / float(men.shape[0]) for x in range(1, int(th))] women_percentage = 0 if women.shape[0] > 0: women_percentage = [ women[women.edition_count == x].shape[0] / float(women.shape[0]) for x in range(1, int(th)) ] ut.plot_percentage( women_percentage, men_percentage, ["pink", "blue"], range(1, int(th)), path + "/numedition_gender_percentage" + self.pre + "-" + self.post + "_99.png", ) # RANDOM BASELINE FOR RATIO fake_ratios_norm = list() fake_ratios = list() for i in range(1, 1000): # print (people.gender.value_counts()) people["random_gender"] = pd.Series(np.random.permutation(people.gender.values), index=people.index) # print (people.random_gender.value_counts()) fake_men = people[people.random_gender == "male"] fake_women = people[people.random_gender == "female"] item_frequency_fake_female = itemfreq(np.array(fake_women["edition_count"].values.tolist())) item_frequency_fake_male = itemfreq(np.array(fake_men["edition_count"].values.tolist())) fake_ratios_norm.append(self.get_ratio(item_frequency_fake_female, item_frequency_fake_male, True)) fake_ratios.append(self.get_ratio(item_frequency_fake_female, item_frequency_fake_male, True)) item_frequency_female = itemfreq(np.array(women["edition_count"].values.tolist())) item_frequency_male = itemfreq(np.array(men["edition_count"].values.tolist())) ratio_norm = self.get_ratio(item_frequency_female, item_frequency_male, True) mean_fake_ratio_norm = {} mean_fake_ratio = {} for key in ratio_norm.keys(): vals = [] vals_norm = [] for dic1 in fake_ratios_norm: if key in dic1: vals_norm.append(dic1.get(key)) mean_fake_ratio_norm[key] = np.mean(vals_norm) for dic2 in fake_ratios: if key in dic2: vals.append(dic2.get(key)) mean_fake_ratio[key] = np.mean(vals) # if we plor normalized ratio we should take the log since otherwise upper boun dis 0 but ratio can become extremly small. # ut.plotratio(ratio_norm, mean_fake_ratio_norm, ['g','r--'], ['empirical gender', 'random gender'], self.pre+"-"+self.post, path+'/numedition_gender_ratio'+self.pre+"-"+self.post+'_norm.png', 'Num Editions', 'Male Proportion/Female Proportion', False, False) ratio = self.get_ratio(item_frequency_female, item_frequency_male, False) lowess = sm.nonparametric.lowess(ratio.values(), ratio.keys(), frac=0.1) ut.plotratio( ratio, lowess, mean_fake_ratio, ["b^", "g", "r--"], ["empirical gender", "lowess fit", "random gender"], self.pre + "-" + self.post, path + "/numedition_gender_ratio" + self.pre + "-" + self.post + ".png", "Num Editions", "Male/Female", False, False, ) # ratio = self.get_ratio(item_frequency_female, item_frequency_male, True) # ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'_norm.png', 'Num Editions', 'Female-Male-Ratio', False, False) # ratio = self.get_ratio(item_frequency_female, item_frequency_male, False) # ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'.png', 'Num Editions', 'Female-Male-Ratio', False, False) # ut.plot_rank_size(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ranksize_99.png', 'Rank', 'Num Editions', False, True) # print "Mann Withney U Test Frequ Dist:" # print stats.mstats.mannwhitneyu(item_frequency_female, item_frequency_male) # print stats.ranksums(item_frequency_female, item_frequency_male) ut.plot_cdf( list([item_frequency_female, item_frequency_male]), labels, ["pink", "blue"], path + "/numedition_gender_ccdf" + self.pre + "-" + self.post + ".png", "Num Editions", True, False, True, ) # ut.plot_cdf(list([item_frequency_female[:np.max(q95)], item_frequency_male[:np.max(q95)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_95.png', 'Num Editions', True, False, True) # ut.plot_cdf(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_99.png', 'Num Editions', True, False, True) self.logfile.write("\n\n men median(men mean), women median (women mean)") self.logfile.write( "\n " + str(np.median(men.edition_count.values)) + "(" + str(np.mean(men.edition_count.values)) + "), " + str(np.median(women.edition_count.values)) + "(" + str(np.mean(women.edition_count.values)) + ")" ) return { "class": classname, "median-men": np.median(men.edition_count.values), "mean-men": np.mean(men.edition_count.values), "sem-men": stats.sem(men.edition_count.values), "sem-women": stats.sem(women.edition_count.values), "median-women": np.median(women.edition_count.values), "mean-women": np.mean(women.edition_count.values), }
def analyze_num_lang_edition(self, classname, people): path = "img/" + classname men = people[people.gender == "male"] women = people[people.gender == "female"] #create folder if not exist if not os.path.exists(path): os.makedirs(path) self.logfile.write("\n\n\n\ " + classname + "\n") edition_counts = np.append(women.edition_count.values, men.edition_count.values) labels = [ 'female (' + str(len(women.index)) + ')', 'male (' + str(len(men.index)) + ')' ] #print edition_counts #print type(edition_counts)probability that a score randomly drawn from population A will be greater than a score randomly drawn from population B. max_num_editions = np.max(edition_counts) #print max_num_editions top_men = [] men_percentage = 0 topk = range(10, 110, 10) if (men.shape[0] > 0): men_percentage = [ men[men.edition_count == x].shape[0] / float(men.shape[0]) for x in range(1, max_num_editions) ] self.logfile.write("\n % of local men ") self.logfile.write( str(men[men.edition_count < 2].shape[0] / float(men.shape[0]))) self.logfile.write("\n percentage men %s" % str( len(men.index) / float((len(women.index) + len(men.index))))) men_vals = men.edition_count.order(ascending=True).values women_percentage = 0 if (women.shape[0] > 0): women_percentage = [ women[women.edition_count == x].shape[0] / float(women.shape[0]) for x in range(1, max_num_editions) ] self.logfile.write("\n\n % local women ") self.logfile.write( str(women[women.edition_count < 2].shape[0] / float(women.shape[0]))) self.logfile.write("\n percentage women %s" % str( len(women.index) / float((len(women.index) + len(men.index))))) women_vals = women.edition_count.order(ascending=True).values if (women.shape[0] > 0 and men.shape[0] > 0): j_m = 0 j_w = 0 U, p = stats.mstats.mannwhitneyu(women.edition_count, men.edition_count) ut.write_mannwithneyu(U, p, women.edition_count, men.edition_count, self.logfile) #for i in topk: # perc_men = len(men.edition_count.index) * i/100.0 # print "bottom %s percent of men are %s - mean %s - median %s"% (str(i), str(perc_men), np.mean(men_vals[:int(perc_men)]), np.median(men_vals[:int(perc_men)])) # top_men.append(men_vals[:int(perc_men)]) # perc_women = len(women.edition_count.index) * i/100.0 # print "bottom %s percent of women are %s - mean %s - median %s"% (str(i), str(perc_women), np.mean(women_vals[:int(perc_men)]), np.median(women_vals[:int(perc_men)])) # top_women.append(women_vals[:int(perc_women)]) # j_m += int(perc_men) # j_w += int(perc_women) #ut.plotTopk(top_women, top_men, ['pink', 'blue'], topk, path+'/numedition_gender_topk.png') ut.plot_percentage( women_percentage, men_percentage, ['pink', 'blue'], range(1, max_num_editions), path + '/numedition_gender_percentage' + self.pre + "-" + self.post + '.png') self.genderBoxplots(women, men, labels, path) self.logfile.write("\n\n num women %s" % len(women.index)) self.logfile.write("\n num men %s" % len(men.index)) data = [women.edition_count.values, men.edition_count.values] # Compute the qth percentile of women and men q75 = [np.percentile(x, q=75) for x in data] self.logfile.write("\n third quartil (75th percentile): " + str(q75)) q95 = [np.percentile(x, q=95) for x in data] self.logfile.write("\n 95th percentile: " + str(q95)) q99 = [np.percentile(x, q=99) for x in data] self.logfile.write("\n 99th percentile: " + str(q99)) q99_women = q99[0] q99_men = q99[1] self.logfile.write("\n threshold women 99th percentile: %s" % q99_women) self.logfile.write("\n threshold men 99th percentile: %s" % q99_men) men_percentage = 0 th = np.min(q99) if (men.shape[0] > 0): men_percentage = [ men[men.edition_count == x].shape[0] / float(men.shape[0]) for x in range(1, int(th)) ] women_percentage = 0 if (women.shape[0] > 0): women_percentage = [ women[women.edition_count == x].shape[0] / float(women.shape[0]) for x in range(1, int(th)) ] ut.plot_percentage( women_percentage, men_percentage, ['pink', 'blue'], range(1, int(th)), path + '/numedition_gender_percentage' + self.pre + "-" + self.post + '_99.png') # RANDOM BASELINE FOR RATIO fake_ratios_norm = list() fake_ratios = list() for i in range(1, 1000): #print (people.gender.value_counts()) people["random_gender"] = pd.Series(np.random.permutation( people.gender.values), index=people.index) #print (people.random_gender.value_counts()) fake_men = people[people.random_gender == "male"] fake_women = people[people.random_gender == "female"] item_frequency_fake_female = itemfreq( np.array(fake_women['edition_count'].values.tolist())) item_frequency_fake_male = itemfreq( np.array(fake_men['edition_count'].values.tolist())) fake_ratios_norm.append( self.get_ratio(item_frequency_fake_female, item_frequency_fake_male, True)) fake_ratios.append( self.get_ratio(item_frequency_fake_female, item_frequency_fake_male, True)) item_frequency_female = itemfreq( np.array(women['edition_count'].values.tolist())) item_frequency_male = itemfreq( np.array(men['edition_count'].values.tolist())) ratio_norm = self.get_ratio(item_frequency_female, item_frequency_male, True) mean_fake_ratio_norm = {} mean_fake_ratio = {} for key in ratio_norm.keys(): vals = [] vals_norm = [] for dic1 in fake_ratios_norm: if key in dic1: vals_norm.append(dic1.get(key)) mean_fake_ratio_norm[key] = np.mean(vals_norm) for dic2 in fake_ratios: if key in dic2: vals.append(dic2.get(key)) mean_fake_ratio[key] = np.mean(vals) # if we plor normalized ratio we should take the log since otherwise upper boun dis 0 but ratio can become extremly small. #ut.plotratio(ratio_norm, mean_fake_ratio_norm, ['g','r--'], ['empirical gender', 'random gender'], self.pre+"-"+self.post, path+'/numedition_gender_ratio'+self.pre+"-"+self.post+'_norm.png', 'Num Editions', 'Male Proportion/Female Proportion', False, False) ratio = self.get_ratio(item_frequency_female, item_frequency_male, False) lowess = sm.nonparametric.lowess(ratio.values(), ratio.keys(), frac=0.1) ut.plotratio( ratio, lowess, mean_fake_ratio, ['b^', 'g', 'r--'], ['empirical gender', 'lowess fit', 'random gender'], self.pre + "-" + self.post, path + '/numedition_gender_ratio' + self.pre + "-" + self.post + '.png', 'Num Editions', 'Male/Female', False, False) #ratio = self.get_ratio(item_frequency_female, item_frequency_male, True) #ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'_norm.png', 'Num Editions', 'Female-Male-Ratio', False, False) #ratio = self.get_ratio(item_frequency_female, item_frequency_male, False) #ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'.png', 'Num Editions', 'Female-Male-Ratio', False, False) #ut.plot_rank_size(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ranksize_99.png', 'Rank', 'Num Editions', False, True) #print "Mann Withney U Test Frequ Dist:" #print stats.mstats.mannwhitneyu(item_frequency_female, item_frequency_male) #print stats.ranksums(item_frequency_female, item_frequency_male) ut.plot_cdf( list([item_frequency_female, item_frequency_male]), labels, ['pink', 'blue'], path + '/numedition_gender_ccdf' + self.pre + "-" + self.post + '.png', 'Num Editions', True, False, True) #ut.plot_cdf(list([item_frequency_female[:np.max(q95)], item_frequency_male[:np.max(q95)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_95.png', 'Num Editions', True, False, True) #ut.plot_cdf(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_99.png', 'Num Editions', True, False, True) self.logfile.write( "\n\n men median(men mean), women median (women mean)") self.logfile.write("\n " + str(np.median(men.edition_count.values)) + '(' + str(np.mean(men.edition_count.values)) + '), ' + str(np.median(women.edition_count.values)) + '(' + str(np.mean(women.edition_count.values)) + ')') return { "class": classname, "median-men": np.median(men.edition_count.values), "mean-men": np.mean(men.edition_count.values), "sem-men": stats.sem(men.edition_count.values), "sem-women": stats.sem(women.edition_count.values), "median-women": np.median(women.edition_count.values), "mean-women": np.mean(women.edition_count.values) }