def plot_ccdf(self, women_values, men_values, labels, filename, xlabel, xlog, ylog):
item_frequency_female = itemfreq(women_values)
item_frequency_male = itemfreq(men_values)
ccdf= 1
ut.plot_cdf(list([item_frequency_female, item_frequency_male]), labels, ['pink','blue'], filename, xlabel, ccdf, xlog, ylog)
def analyze_num_lang_edition(self, classname, people):
path = "img/" + classname
men = people[people.gender == "male"]
women = people[people.gender == "female"]
# create folder if not exist
if not os.path.exists(path):
os.makedirs(path)
self.logfile.write("\n\n\n\ " + classname + "\n")
edition_counts = np.append(women.edition_count.values, men.edition_count.values)
labels = ["female (" + str(len(women.index)) + ")", "male (" + str(len(men.index)) + ")"]
# print edition_counts
# print type(edition_counts)probability that a score randomly drawn from population A will be greater than a score randomly drawn from population B.
max_num_editions = np.max(edition_counts)
# print max_num_editions
top_men = []
men_percentage = 0
topk = range(10, 110, 10)
if men.shape[0] > 0:
men_percentage = [
men[men.edition_count == x].shape[0] / float(men.shape[0]) for x in range(1, max_num_editions)
]
self.logfile.write("\n % of local men ")
self.logfile.write(str(men[men.edition_count < 2].shape[0] / float(men.shape[0])))
self.logfile.write(
"\n percentage men %s" % str(len(men.index) / float((len(women.index) + len(men.index))))
)
men_vals = men.edition_count.order(ascending=True).values
women_percentage = 0
if women.shape[0] > 0:
women_percentage = [
women[women.edition_count == x].shape[0] / float(women.shape[0]) for x in range(1, max_num_editions)
]
self.logfile.write("\n\n % local women ")
self.logfile.write(str(women[women.edition_count < 2].shape[0] / float(women.shape[0])))
self.logfile.write(
"\n percentage women %s" % str(len(women.index) / float((len(women.index) + len(men.index))))
)
women_vals = women.edition_count.order(ascending=True).values
if women.shape[0] > 0 and men.shape[0] > 0:
j_m = 0
j_w = 0
U, p = stats.mstats.mannwhitneyu(women.edition_count, men.edition_count)
ut.write_mannwithneyu(U, p, women.edition_count, men.edition_count, self.logfile)
# for i in topk:
# perc_men = len(men.edition_count.index) * i/100.0
# print "bottom %s percent of men are %s - mean %s - median %s"% (str(i), str(perc_men), np.mean(men_vals[:int(perc_men)]), np.median(men_vals[:int(perc_men)]))
# top_men.append(men_vals[:int(perc_men)])
# perc_women = len(women.edition_count.index) * i/100.0
# print "bottom %s percent of women are %s - mean %s - median %s"% (str(i), str(perc_women), np.mean(women_vals[:int(perc_men)]), np.median(women_vals[:int(perc_men)]))
# top_women.append(women_vals[:int(perc_women)])
# j_m += int(perc_men)
# j_w += int(perc_women)
# ut.plotTopk(top_women, top_men, ['pink', 'blue'], topk, path+'/numedition_gender_topk.png')
ut.plot_percentage(
women_percentage,
men_percentage,
["pink", "blue"],
range(1, max_num_editions),
path + "/numedition_gender_percentage" + self.pre + "-" + self.post + ".png",
)
self.genderBoxplots(women, men, labels, path)
self.logfile.write("\n\n num women %s" % len(women.index))
self.logfile.write("\n num men %s" % len(men.index))
data = [women.edition_count.values, men.edition_count.values]
# Compute the qth percentile of women and men
q75 = [np.percentile(x, q=75) for x in data]
self.logfile.write("\n third quartil (75th percentile): " + str(q75))
q95 = [np.percentile(x, q=95) for x in data]
self.logfile.write("\n 95th percentile: " + str(q95))
q99 = [np.percentile(x, q=99) for x in data]
self.logfile.write("\n 99th percentile: " + str(q99))
q99_women = q99[0]
q99_men = q99[1]
self.logfile.write("\n threshold women 99th percentile: %s" % q99_women)
self.logfile.write("\n threshold men 99th percentile: %s" % q99_men)
men_percentage = 0
th = np.min(q99)
if men.shape[0] > 0:
men_percentage = [men[men.edition_count == x].shape[0] / float(men.shape[0]) for x in range(1, int(th))]
women_percentage = 0
if women.shape[0] > 0:
women_percentage = [
women[women.edition_count == x].shape[0] / float(women.shape[0]) for x in range(1, int(th))
]
ut.plot_percentage(
women_percentage,
men_percentage,
["pink", "blue"],
range(1, int(th)),
path + "/numedition_gender_percentage" + self.pre + "-" + self.post + "_99.png",
)
# RANDOM BASELINE FOR RATIO
fake_ratios_norm = list()
fake_ratios = list()
for i in range(1, 1000):
# print (people.gender.value_counts())
people["random_gender"] = pd.Series(np.random.permutation(people.gender.values), index=people.index)
# print (people.random_gender.value_counts())
fake_men = people[people.random_gender == "male"]
fake_women = people[people.random_gender == "female"]
item_frequency_fake_female = itemfreq(np.array(fake_women["edition_count"].values.tolist()))
item_frequency_fake_male = itemfreq(np.array(fake_men["edition_count"].values.tolist()))
fake_ratios_norm.append(self.get_ratio(item_frequency_fake_female, item_frequency_fake_male, True))
fake_ratios.append(self.get_ratio(item_frequency_fake_female, item_frequency_fake_male, True))
item_frequency_female = itemfreq(np.array(women["edition_count"].values.tolist()))
item_frequency_male = itemfreq(np.array(men["edition_count"].values.tolist()))
ratio_norm = self.get_ratio(item_frequency_female, item_frequency_male, True)
mean_fake_ratio_norm = {}
mean_fake_ratio = {}
for key in ratio_norm.keys():
vals = []
vals_norm = []
for dic1 in fake_ratios_norm:
if key in dic1:
vals_norm.append(dic1.get(key))
mean_fake_ratio_norm[key] = np.mean(vals_norm)
for dic2 in fake_ratios:
if key in dic2:
vals.append(dic2.get(key))
mean_fake_ratio[key] = np.mean(vals)
# if we plor normalized ratio we should take the log since otherwise upper boun dis 0 but ratio can become extremly small.
# ut.plotratio(ratio_norm, mean_fake_ratio_norm, ['g','r--'], ['empirical gender', 'random gender'], self.pre+"-"+self.post, path+'/numedition_gender_ratio'+self.pre+"-"+self.post+'_norm.png', 'Num Editions', 'Male Proportion/Female Proportion', False, False)
ratio = self.get_ratio(item_frequency_female, item_frequency_male, False)
lowess = sm.nonparametric.lowess(ratio.values(), ratio.keys(), frac=0.1)
ut.plotratio(
ratio,
lowess,
mean_fake_ratio,
["b^", "g", "r--"],
["empirical gender", "lowess fit", "random gender"],
self.pre + "-" + self.post,
path + "/numedition_gender_ratio" + self.pre + "-" + self.post + ".png",
"Num Editions",
"Male/Female",
False,
False,
)
# ratio = self.get_ratio(item_frequency_female, item_frequency_male, True)
# ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'_norm.png', 'Num Editions', 'Female-Male-Ratio', False, False)
# ratio = self.get_ratio(item_frequency_female, item_frequency_male, False)
# ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'.png', 'Num Editions', 'Female-Male-Ratio', False, False)
# ut.plot_rank_size(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ranksize_99.png', 'Rank', 'Num Editions', False, True)
# print "Mann Withney U Test Frequ Dist:"
# print stats.mstats.mannwhitneyu(item_frequency_female, item_frequency_male)
# print stats.ranksums(item_frequency_female, item_frequency_male)
ut.plot_cdf(
list([item_frequency_female, item_frequency_male]),
labels,
["pink", "blue"],
path + "/numedition_gender_ccdf" + self.pre + "-" + self.post + ".png",
"Num Editions",
True,
False,
True,
)
# ut.plot_cdf(list([item_frequency_female[:np.max(q95)], item_frequency_male[:np.max(q95)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_95.png', 'Num Editions', True, False, True)
# ut.plot_cdf(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_99.png', 'Num Editions', True, False, True)
self.logfile.write("\n\n men median(men mean), women median (women mean)")
self.logfile.write(
"\n "
+ str(np.median(men.edition_count.values))
+ "("
+ str(np.mean(men.edition_count.values))
+ "), "
+ str(np.median(women.edition_count.values))
+ "("
+ str(np.mean(women.edition_count.values))
+ ")"
)
return {
"class": classname,
"median-men": np.median(men.edition_count.values),
"mean-men": np.mean(men.edition_count.values),
"sem-men": stats.sem(men.edition_count.values),
"sem-women": stats.sem(women.edition_count.values),
"median-women": np.median(women.edition_count.values),
"mean-women": np.mean(women.edition_count.values),
}
def analyze_num_lang_edition(self, classname, people):
path = "img/" + classname
men = people[people.gender == "male"]
women = people[people.gender == "female"]
#create folder if not exist
if not os.path.exists(path):
os.makedirs(path)
self.logfile.write("\n\n\n\ " + classname + "\n")
edition_counts = np.append(women.edition_count.values,
men.edition_count.values)
labels = [
'female (' + str(len(women.index)) + ')',
'male (' + str(len(men.index)) + ')'
]
#print edition_counts
#print type(edition_counts)probability that a score randomly drawn from population A will be greater than a score randomly drawn from population B.
max_num_editions = np.max(edition_counts)
#print max_num_editions
top_men = []
men_percentage = 0
topk = range(10, 110, 10)
if (men.shape[0] > 0):
men_percentage = [
men[men.edition_count == x].shape[0] / float(men.shape[0])
for x in range(1, max_num_editions)
]
self.logfile.write("\n % of local men ")
self.logfile.write(
str(men[men.edition_count < 2].shape[0] / float(men.shape[0])))
self.logfile.write("\n percentage men %s" % str(
len(men.index) / float((len(women.index) + len(men.index)))))
men_vals = men.edition_count.order(ascending=True).values
women_percentage = 0
if (women.shape[0] > 0):
women_percentage = [
women[women.edition_count == x].shape[0] /
float(women.shape[0]) for x in range(1, max_num_editions)
]
self.logfile.write("\n\n % local women ")
self.logfile.write(
str(women[women.edition_count < 2].shape[0] /
float(women.shape[0])))
self.logfile.write("\n percentage women %s" % str(
len(women.index) / float((len(women.index) + len(men.index)))))
women_vals = women.edition_count.order(ascending=True).values
if (women.shape[0] > 0 and men.shape[0] > 0):
j_m = 0
j_w = 0
U, p = stats.mstats.mannwhitneyu(women.edition_count,
men.edition_count)
ut.write_mannwithneyu(U, p, women.edition_count, men.edition_count,
self.logfile)
#for i in topk:
# perc_men = len(men.edition_count.index) * i/100.0
# print "bottom %s percent of men are %s - mean %s - median %s"% (str(i), str(perc_men), np.mean(men_vals[:int(perc_men)]), np.median(men_vals[:int(perc_men)]))
# top_men.append(men_vals[:int(perc_men)])
# perc_women = len(women.edition_count.index) * i/100.0
# print "bottom %s percent of women are %s - mean %s - median %s"% (str(i), str(perc_women), np.mean(women_vals[:int(perc_men)]), np.median(women_vals[:int(perc_men)]))
# top_women.append(women_vals[:int(perc_women)])
# j_m += int(perc_men)
# j_w += int(perc_women)
#ut.plotTopk(top_women, top_men, ['pink', 'blue'], topk, path+'/numedition_gender_topk.png')
ut.plot_percentage(
women_percentage, men_percentage, ['pink', 'blue'],
range(1, max_num_editions),
path + '/numedition_gender_percentage' + self.pre + "-" +
self.post + '.png')
self.genderBoxplots(women, men, labels, path)
self.logfile.write("\n\n num women %s" % len(women.index))
self.logfile.write("\n num men %s" % len(men.index))
data = [women.edition_count.values, men.edition_count.values]
# Compute the qth percentile of women and men
q75 = [np.percentile(x, q=75) for x in data]
self.logfile.write("\n third quartil (75th percentile): " +
str(q75))
q95 = [np.percentile(x, q=95) for x in data]
self.logfile.write("\n 95th percentile: " + str(q95))
q99 = [np.percentile(x, q=99) for x in data]
self.logfile.write("\n 99th percentile: " + str(q99))
q99_women = q99[0]
q99_men = q99[1]
self.logfile.write("\n threshold women 99th percentile: %s" %
q99_women)
self.logfile.write("\n threshold men 99th percentile: %s" %
q99_men)
men_percentage = 0
th = np.min(q99)
if (men.shape[0] > 0):
men_percentage = [
men[men.edition_count == x].shape[0] / float(men.shape[0])
for x in range(1, int(th))
]
women_percentage = 0
if (women.shape[0] > 0):
women_percentage = [
women[women.edition_count == x].shape[0] /
float(women.shape[0]) for x in range(1, int(th))
]
ut.plot_percentage(
women_percentage, men_percentage, ['pink', 'blue'],
range(1, int(th)), path + '/numedition_gender_percentage' +
self.pre + "-" + self.post + '_99.png')
# RANDOM BASELINE FOR RATIO
fake_ratios_norm = list()
fake_ratios = list()
for i in range(1, 1000):
#print (people.gender.value_counts())
people["random_gender"] = pd.Series(np.random.permutation(
people.gender.values),
index=people.index)
#print (people.random_gender.value_counts())
fake_men = people[people.random_gender == "male"]
fake_women = people[people.random_gender == "female"]
item_frequency_fake_female = itemfreq(
np.array(fake_women['edition_count'].values.tolist()))
item_frequency_fake_male = itemfreq(
np.array(fake_men['edition_count'].values.tolist()))
fake_ratios_norm.append(
self.get_ratio(item_frequency_fake_female,
item_frequency_fake_male, True))
fake_ratios.append(
self.get_ratio(item_frequency_fake_female,
item_frequency_fake_male, True))
item_frequency_female = itemfreq(
np.array(women['edition_count'].values.tolist()))
item_frequency_male = itemfreq(
np.array(men['edition_count'].values.tolist()))
ratio_norm = self.get_ratio(item_frequency_female,
item_frequency_male, True)
mean_fake_ratio_norm = {}
mean_fake_ratio = {}
for key in ratio_norm.keys():
vals = []
vals_norm = []
for dic1 in fake_ratios_norm:
if key in dic1:
vals_norm.append(dic1.get(key))
mean_fake_ratio_norm[key] = np.mean(vals_norm)
for dic2 in fake_ratios:
if key in dic2:
vals.append(dic2.get(key))
mean_fake_ratio[key] = np.mean(vals)
# if we plor normalized ratio we should take the log since otherwise upper boun dis 0 but ratio can become extremly small.
#ut.plotratio(ratio_norm, mean_fake_ratio_norm, ['g','r--'], ['empirical gender', 'random gender'], self.pre+"-"+self.post, path+'/numedition_gender_ratio'+self.pre+"-"+self.post+'_norm.png', 'Num Editions', 'Male Proportion/Female Proportion', False, False)
ratio = self.get_ratio(item_frequency_female, item_frequency_male,
False)
lowess = sm.nonparametric.lowess(ratio.values(),
ratio.keys(),
frac=0.1)
ut.plotratio(
ratio, lowess, mean_fake_ratio, ['b^', 'g', 'r--'],
['empirical gender', 'lowess fit', 'random gender'],
self.pre + "-" + self.post, path + '/numedition_gender_ratio' +
self.pre + "-" + self.post + '.png', 'Num Editions',
'Male/Female', False, False)
#ratio = self.get_ratio(item_frequency_female, item_frequency_male, True)
#ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'_norm.png', 'Num Editions', 'Female-Male-Ratio', False, False)
#ratio = self.get_ratio(item_frequency_female, item_frequency_male, False)
#ut.plotline(list(ratio.keys()), list(ratio.values()), ['pink','blue'], path+'/numedition_gender_ratio'+self.pre+'.png', 'Num Editions', 'Female-Male-Ratio', False, False)
#ut.plot_rank_size(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ranksize_99.png', 'Rank', 'Num Editions', False, True)
#print "Mann Withney U Test Frequ Dist:"
#print stats.mstats.mannwhitneyu(item_frequency_female, item_frequency_male)
#print stats.ranksums(item_frequency_female, item_frequency_male)
ut.plot_cdf(
list([item_frequency_female, item_frequency_male]), labels,
['pink', 'blue'], path + '/numedition_gender_ccdf' + self.pre +
"-" + self.post + '.png', 'Num Editions', True, False, True)
#ut.plot_cdf(list([item_frequency_female[:np.max(q95)], item_frequency_male[:np.max(q95)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_95.png', 'Num Editions', True, False, True)
#ut.plot_cdf(list([item_frequency_female[:np.max(q99)], item_frequency_male[:np.max(q99)]]), labels, ['pink','blue'], path+'/numedition_gender_ccdf_99.png', 'Num Editions', True, False, True)
self.logfile.write(
"\n\n men median(men mean), women median (women mean)")
self.logfile.write("\n " +
str(np.median(men.edition_count.values)) + '(' +
str(np.mean(men.edition_count.values)) + '), ' +
str(np.median(women.edition_count.values)) +
'(' + str(np.mean(women.edition_count.values)) +
')')
return {
"class": classname,
"median-men": np.median(men.edition_count.values),
"mean-men": np.mean(men.edition_count.values),
"sem-men": stats.sem(men.edition_count.values),
"sem-women": stats.sem(women.edition_count.values),
"median-women": np.median(women.edition_count.values),
"mean-women": np.mean(women.edition_count.values)
}
