def map_col_vals_to_ints(df, column_to_change, return_map=False): cols = df.columns unik_vals_map = ch_col_names( pd.DataFrame(df[column_to_change].unique()).reset_index(), ['tmp_new_col', column_to_change]) df = pd.merge(df, unik_vals_map) df = rm_cols_if_present(df, column_to_change) df = ch_col_names(df, column_to_change, 'tmp_new_col') if return_map: return df[cols], else: return df[cols]
def map_col_vals_to_ints(df, column_to_change, return_map=False): cols = df.columns unik_vals_map = ch_col_names( pd.DataFrame(df[column_to_change].unique()).reset_index(), ['tmp_new_col', column_to_change]) df = pd.merge(df, unik_vals_map) df = rm_cols_if_present(df, column_to_change) df = ch_col_names(df, column_to_change, 'tmp_new_col') if return_map: return df[cols], else: return df[cols]
def add_query_fanout_scores(query_report_df): if 'destination' not in query_report_df.columns: query_report_df = add_destination(query_report_df) ad_group_fanout = mk_query_fanout_scores(query_report_df,target='ad_group',statVars='impressions',keep_statVars=False) ad_group_fanout = daf_ch.ch_col_names(ad_group_fanout, ['ad_group_imps_freq_fanout_ratio','ad_group_count_fanout_ratio'], ['impressions_freq_fanout_ratio','impressions_count_fanout_ratio']) destination_fanout = mk_query_fanout_scores(query_report_df,target='destination',statVars='impressions',keep_statVars=False) destination_fanout = daf_ch.ch_col_names(destination_fanout, ['destination_imps_freq_fanout_ratio','destination_count_fanout_ratio'], ['impressions_freq_fanout_ratio','impressions_count_fanout_ratio']) query_report_df = query_report_df.merge(ad_group_fanout,on=['search_term','ad_group']) query_report_df = query_report_df.merge(destination_fanout,on=['search_term','destination']) return query_report_df
def get_kw_duplicates_01(df, dup_def='kw_lower', gr_keys=['match_type', 'ad_group', 'campaign']): """ old function to get kw_duplicates probably better to use kw_dup_diagnosis """ if dup_def == 'all': # get kw_lower dup_count df d = get_kw_duplicates_01(df, dup_def='kw_lower', gr_keys=gr_keys) d = daf_ch.ch_col_names(d, 'dup_count', 'lower_dups') del d['kw_lower'] # get kw_lower_ascii dup_count df dd = get_kw_duplicates_01(df, dup_def='kw_lower_ascii', gr_keys=gr_keys) dd = daf_ch.ch_col_names(dd, 'dup_count', 'ascii_dups') del dd['kw_lower_ascii'] # merge d and dd d = d.merge(dd, how='outer') # get kw_lower_ascii_ordered dup_count df dd = get_kw_duplicates_01(df, dup_def='kw_lower_ascii_ordered', gr_keys=gr_keys) dd = daf_ch.ch_col_names(dd, 'dup_count', 'order_dups') del dd['kw_lower_ascii_ordered'] # merge d and dd d = d.merge(dd, how='outer') # replace nans with 0s d = d.fillna(0) return d else: df = df.copy() # make a copy if dup_def == 'kw_lower': d = add_col(df, 'kw_lower', overwrite=False) gr_keys = oc.union(['kw_lower'], gr_keys) elif dup_def == 'kw_lower_ascii': d = add_col(df, 'kw_lower_ascii', overwrite=False) gr_keys = oc.union(['kw_lower_ascii'], gr_keys) elif dup_def == 'kw_lower_ascii_ordered': d = df[df.match_type == 'Broad'] d = add_col(d, 'kw_lower_ascii_ordered', overwrite=False) gr_keys = oc.union((['kw_lower_ascii_ordered'], gr_keys)) else: raise ValueError("don't know how to handle that dup_def") assert_dependencies(d, gr_keys, "to get duplicates") return get_duplicates(d, gr_keys, keep_count=True)
def termdoc_to_doc_counts(term_doc_df, doc_var=None, term_var='term', count_var='count'): term_doc_df, doc_var, term_var = __process_term_doc_var_names__(term_doc_df, doc_var=doc_var, term_var=term_var) # keep only doc and terms, and one copy of any (doc,term) pair term_doc_df = term_doc_df[[doc_var, term_var]].drop_duplicates(cols=[doc_var, term_var]).reset_index(drop=True) # group by terms term_doc_df = term_doc_df[[term_var]].groupby(term_var).count() term_doc_df = daf_ch.ch_col_names(term_doc_df, count_var, term_var) return term_doc_df
def termdoc_to_doc_counts(term_doc_df, doc_var=None, term_var='term', count_var='count'): term_doc_df, doc_var, term_var = __process_term_doc_var_names__(term_doc_df, doc_var=doc_var, term_var=term_var) # keep only doc and terms, and one copy of any (doc,term) pair term_doc_df = term_doc_df[[doc_var, term_var]].drop_duplicates(cols=[doc_var, term_var]).reset_index(drop=True) # group by terms term_doc_df = term_doc_df[[term_var]].groupby(term_var).count() term_doc_df = daf_ch.ch_col_names(term_doc_df, count_var, term_var) return term_doc_df
def add_target_scores(query_report_df): vars_to_keep = ['search_term','impressions','destination','ad_group','destination_imps_freq_fanout_ratio','ad_group_imps_freq_fanout_ratio'] query_report_df = add_query_fanout_scores(query_report_df) query_report_df = query_report_df[vars_to_keep] query_report_df = daf_ch.ch_col_names(query_report_df, ['ad_group_score','destination_score'], ['ad_group_imps_freq_fanout_ratio','destination_imps_freq_fanout_ratio']) query_report_df = query_report_df.sort(columns=['search_term','destination_score','ad_group_score']) return query_report_df
def from_count_df_to_count(cls, count_df, count_col='pval'): """ Creates a potential from a dataframe specifying point counts (where the count column name is specified by count_col """ pot_vars = list(colloc.setdiff(count_df.columns, [count_col])) tb = count_df[pot_vars+[count_col]].groupby(pot_vars).sum().reset_index() tb = ch_col_names(tb, 'pval', count_col) return Pot(tb)
def from_count_df_to_count(cls, count_df, count_col='pval'): """ Creates a potential from a dataframe specifying point counts (where the count column name is specified by count_col """ pot_vars = list(colloc.setdiff(count_df.columns, [count_col])) tb = count_df[pot_vars + [count_col]].groupby(pot_vars).sum().reset_index() tb = ch_col_names(tb, 'pval', count_col) return Pot(tb)
def from_points_to_bins(cls, pts, **kwargs): """ Creates a potential from a dataframe specifying point counts (where the count column name is specified by count_col """ if isinstance(pts, pd.DataFrame): tb = group_and_count(pts) tb = ch_col_names(tb, 'pval', 'count') return Pot(tb)
def __process_term_doc_var_names__(term_doc_df, doc_var=None, term_var='term'): if term_var != 'term': term_doc_df = daf_ch.ch_col_names(term_doc_df, [term_var], [term_var]) cols = term_doc_df.columns if doc_var is None: # try to guess it if len(cols) != 2: raise ValueError("In order to guess the doc_var, there needs to be only two columns") else: doc_var = list(set(cols)-set([term_var]))[0] return (term_doc_df, doc_var, term_var)
def from_points_to_bins(cls, pts, **kwargs): """ Creates a potential from a dataframe specifying point counts (where the count column name is specified by count_col """ if isinstance(pts, pd.DataFrame): tb = group_and_count(pts) tb = ch_col_names(tb, 'pval', 'count') return Pot(tb)
def __process_term_doc_var_names__(term_doc_df, doc_var=None, term_var='term'): if term_var != 'term': term_doc_df = daf_ch.ch_col_names(term_doc_df, [term_var], [term_var]) cols = term_doc_df.columns if doc_var is None: # try to guess it if len(cols) != 2: raise ValueError("In order to guess the doc_var, there needs to be only two columns") else: doc_var = list(set(cols)-set([term_var]))[0] return (term_doc_df, doc_var, term_var)
def get_kw_duplicates_01(df,dup_def='kw_lower',gr_keys=['match_type','ad_group','campaign']): """ old function to get kw_duplicates probably better to use kw_dup_diagnosis """ if dup_def=='all': # get kw_lower dup_count df d = get_kw_duplicates_01(df,dup_def='kw_lower',gr_keys=gr_keys) d = daf_ch.ch_col_names(d,'dup_count','lower_dups') del d['kw_lower'] # get kw_lower_ascii dup_count df dd = get_kw_duplicates_01(df,dup_def='kw_lower_ascii',gr_keys=gr_keys) dd = daf_ch.ch_col_names(dd,'dup_count','ascii_dups') del dd['kw_lower_ascii'] # merge d and dd d = d.merge(dd,how='outer') # get kw_lower_ascii_ordered dup_count df dd = get_kw_duplicates_01(df,dup_def='kw_lower_ascii_ordered',gr_keys=gr_keys) dd = daf_ch.ch_col_names(dd,'dup_count','order_dups') del dd['kw_lower_ascii_ordered'] # merge d and dd d = d.merge(dd,how='outer') # replace nans with 0s d = d.fillna(0) return d else: df = df.copy() # make a copy if dup_def=='kw_lower': d = add_col(df,'kw_lower',overwrite=False) gr_keys = oc.union(['kw_lower'],gr_keys) elif dup_def=='kw_lower_ascii': d = add_col(df,'kw_lower_ascii',overwrite=False) gr_keys = oc.union(['kw_lower_ascii'],gr_keys) elif dup_def=='kw_lower_ascii_ordered': d = df[df.match_type=='Broad'] d = add_col(d,'kw_lower_ascii_ordered',overwrite=False) gr_keys = oc.union((['kw_lower_ascii_ordered'],gr_keys)) else: raise ValueError("don't know how to handle that dup_def") assert_dependencies(d,gr_keys,"to get duplicates") return get_duplicates(d,gr_keys,keep_count=True)
def mk_df_of_travel_domains(): # set up resources html_folder = '/D/Dropbox/dev/py/data/html/google_results_tests/' file_list = [ 'hotel - 100 Google Search Results.html', 'find hotel deals - 100 Google Search Results.html', 'hotel travel sites - 100 Google Search Results.html', 'find hotels - 100 Google Search Results.html', 'hotel paris - 100 Google Search Results.html', 'hotel rome - 100 Google Search Results.html', 'hotel london - 100 Google Search Results.html', 'hotel nyc - 100 Google Search Results.html', 'hotels in france - 100 Google Search Results.html', 'hotels in italy - 100 Google Search Results.html' ] filepath_list = [os.path.join(html_folder, f) for f in file_list] # parse all this r = [google.mk_gresult_tag_dict(f) for f in filepath_list] r = [google.parse_tag_dict(f) for f in r] # make domain lists org_domain_list = [] ads_domain_list = [] tads_domain_list = [] for rr in r: rrr = rr['organic_results_list'] org_domain_list = org_domain_list + [ x['domain'] for x in rrr if 'domain' in x ] rrr = rr['rhs_ads_list'] ads_domain_list = ads_domain_list + [ x['disp_url_domain'] for x in rrr if 'disp_url_domain' in x ] rrr = rr['top_ads_list'] ads_domain_list = ads_domain_list + [ x['disp_url_domain'] for x in rrr if 'disp_url_domain' in x ] domain_list = org_domain_list + ads_domain_list print("number of org_domain_list entries = %d" % len(org_domain_list)) print("number of ads_domain_list entries = %d" % len(ads_domain_list)) print("number of (all) domain_list entries = %d" % len(domain_list)) # make a dataframe counting the number of times we encouter each domain df = pd.DataFrame(domain_list, columns=['domain']) dg = df.groupby('domain').count() #agg([('domain_count','len')]) dg = daf_ch.ch_col_names(dg, 'count', 'domain') thresh = 4 print("length before removing count<%d entries = %d" % (thresh, len(dg))) dg = dg[dg['count'] >= thresh] print("length before removing count<%d entries = %d" % (thresh, len(dg))) dg['frequency'] = dg['count'] / float(max(dg['count'])) dg = dg.sort(columns=['count'], ascending=False) dg.head(30) # return this! return dg
def termdoc_to_term_idf(term_doc_df, doc_var=None, term_var='term'): # processing input term_doc_df, doc_var, term_var = __process_term_doc_var_names__(term_doc_df, doc_var=doc_var, term_var=term_var) # get the number of docs num_of_docs = len(np.unique(term_doc_df[doc_var])) # get doc_counts term_doc_df = termdoc_to_doc_counts(term_doc_df, doc_var, term_var) # # keep only doc and terms, and one copy of any (doc,term) pair # term_doc_df = term_doc_df[[doc_var,'term']].drop_duplicates(cols=[doc_var, 'term']).reset_index(drop=True) # # group by terms # term_doc_df = term_doc_df[['term']].groupby('term').count() term_doc_df['term'] = \ semantics_math.idf_log10(num_of_docs_containing_term=np.array(term_doc_df['term']), num_of_docs=float(num_of_docs)) return daf_ch.ch_col_names(term_doc_df, ['stat'], [term_var])
def termdoc_to_term_idf(term_doc_df, doc_var=None, term_var='term'): # processing input term_doc_df, doc_var, term_var = __process_term_doc_var_names__(term_doc_df, doc_var=doc_var, term_var=term_var) # get the number of docs num_of_docs = len(np.unique(term_doc_df[doc_var])) # get doc_counts term_doc_df = termdoc_to_doc_counts(term_doc_df, doc_var, term_var) # # keep only doc and terms, and one copy of any (doc,term) pair # term_doc_df = term_doc_df[[doc_var,'term']].drop_duplicates(cols=[doc_var, 'term']).reset_index(drop=True) # # group by terms # term_doc_df = term_doc_df[['term']].groupby('term').count() term_doc_df['term'] = \ semantics_math.idf_log10(num_of_docs_containing_term=np.array(term_doc_df['term']), num_of_docs=float(num_of_docs)) return daf_ch.ch_col_names(term_doc_df, ['stat'], [term_var])
def mk_df_of_travel_domains(): # set up resources html_folder = '/D/Dropbox/dev/py/data/html/google_results_tests/' file_list = ['hotel - 100 Google Search Results.html', 'find hotel deals - 100 Google Search Results.html', 'hotel travel sites - 100 Google Search Results.html', 'find hotels - 100 Google Search Results.html', 'hotel paris - 100 Google Search Results.html', 'hotel rome - 100 Google Search Results.html', 'hotel london - 100 Google Search Results.html', 'hotel nyc - 100 Google Search Results.html', 'hotels in france - 100 Google Search Results.html', 'hotels in italy - 100 Google Search Results.html' ] filepath_list = [os.path.join(html_folder,f) for f in file_list] # parse all this r = [google.mk_gresult_tag_dict(f) for f in filepath_list] r = [google.parse_tag_dict(f) for f in r] # make domain lists org_domain_list = [] ads_domain_list = [] tads_domain_list = [] for rr in r: rrr = rr['organic_results_list'] org_domain_list = org_domain_list + [x['domain'] for x in rrr if x.has_key('domain')] rrr = rr['rhs_ads_list'] ads_domain_list = ads_domain_list + [x['disp_url_domain'] for x in rrr if x.has_key('disp_url_domain')] rrr = rr['top_ads_list'] ads_domain_list = ads_domain_list + [x['disp_url_domain'] for x in rrr if x.has_key('disp_url_domain')] domain_list = org_domain_list + ads_domain_list print "number of org_domain_list entries = %d" % len(org_domain_list) print "number of ads_domain_list entries = %d" % len(ads_domain_list) print "number of (all) domain_list entries = %d" % len(domain_list) # make a dataframe counting the number of times we encouter each domain df = pd.DataFrame(domain_list,columns=['domain']) dg = df.groupby('domain').count() #agg([('domain_count','len')]) dg = daf_ch.ch_col_names(dg,'count','domain') thresh = 4 print "length before removing count<%d entries = %d" % (thresh,len(dg)) dg = dg[dg['count']>=thresh] print "length before removing count<%d entries = %d" % (thresh,len(dg)) dg['frequency'] = dg['count']/float(max(dg['count'])) dg = dg.sort(columns=['count'],ascending=False) dg.head(30) # return this! return dg
def termdoc_to_termdoc_count(term_doc_df, doc_var=None, term_var='term', count_var='count'): # processing input term_doc_df, doc_var, term_var = __process_term_doc_var_names__(term_doc_df, doc_var=doc_var, term_var=term_var) term_doc_df = term_doc_df.groupby([doc_var, term_var]).count() term_doc_df = daf_ch.ch_col_names(term_doc_df, count_var, term_var) return term_doc_df
def termdoc_to_termdoc_count(term_doc_df, doc_var=None, term_var='term', count_var='count'): # processing input term_doc_df, doc_var, term_var = __process_term_doc_var_names__(term_doc_df, doc_var=doc_var, term_var=term_var) term_doc_df = term_doc_df.groupby([doc_var, term_var]).count() term_doc_df = daf_ch.ch_col_names(term_doc_df, count_var, term_var) return term_doc_df
def ch_col_names(self, df): return daf_ch.ch_col_names(df, self.variable_map.values(), self.variable_map.keys())
def id_year_of_birth(raw_life_course_data, **kwargs): d = raw_life_course_data[['id', 'year']].groupby('id').min() return daf_ch.ch_col_names(d, 'yob', 'year').reset_index()
def add_is_unik(): tok_str_count = daf_ch.ch_col_names( tok_lists[['tok_str']].groupby('tok_str').count(), 'tok_str')
def add_is_unik(): tok_str_count = daf_ch.ch_col_names(tok_lists[['tok_str']].groupby('tok_str').count(), 'tok_str')
def mk_awe_cols(df): old_cols = df.columns new_cols = map(awe_col_replacer, old_cols) return daf_ch.ch_col_names(df, new_cols, old_cols)