def crosslingual(): all_data = pd.concat([df for key, df in lang_data.items()]) ### VA-->BE5 model on all_data model = get_model(n_inputs=len(VA), n_outputs=len(BE5), source_lexicon=all_data[VA]) model.fit(words=all_data.index, labels=all_data[BE5]) ###combining chinese data def get_zh(): zh1 = prepare_data.load_yu16() zh2 = prepare_data.load_yao16() zh = pd.concat([zh1, zh2]) del zh1 del zh2 return drop_duplicates(zh) settings = [{ 'name': 'it_Montefinese_BE', 'load': prepare_data.load_montefinese14 }, { 'name': 'pt_Soares_BE', 'load': prepare_data.load_soares12 }, { 'name': 'nl_Moors_BE', 'load': prepare_data.load_moors13 }, { 'name': 'id_Sianipar_BE', 'load': prepare_data.load_sianipar16 }, { 'name': 'zh_Yu_Yao_BE', 'load': get_zh }, { 'name': 'fr_Monnier_BE', 'load': prepare_data.load_monnier14 }, { 'name': 'gr_Palogiannidi_BE', 'load': prepare_data.load_palogiannidi16 }, { 'name': 'fn_Eilola_BE', 'load': prepare_data.load_eilola10 }, { 'name': 'sv_Davidson_BE', 'load': prepare_data.load_davidson14 }] num_of_new_entries = pd.DataFrame(columns=['N']) for s in settings: print(s['name']) source_lex = s['load']() lex = model.lexicon_creation(words=source_lex.index, features=source_lex[VA]) save_tsv(lex, 'lexicons/{}.tsv'.format(s['name'])) num_of_new_entries.loc[s['name']] = len(lex) save_tsv(num_of_new_entries, 'new_entries_crosslingual.tsv')
def monolingual(): def create_lexicon(source_rep, target_rep, train_data, target_data): model = get_model(n_inputs=len(source_rep), n_outputs=len(target_rep), source_lexicon=train_data[source_rep]) model.fit(words=train_data.index, labels=train_data[target_rep]) lex = model.lexicon_creation(words=target_data.index, features=target_data[source_rep]) return lex #### for the monolingual set-up, we selected the data sets with the #### highest accuracy in the monolingual evaluation (estimated to be the cleanest data) settings = [{ 'name': 'Warriner_BE', 'source_rep': VAD, 'target_rep': BE5, 'train_data': prepare_data.get_english_anew(), 'target_data': prepare_data.load_warriner13 }, { 'name': 'Stadthagen_Dominance', 'source_rep': BE5, 'target_rep': VAD, 'train_data': prepare_data.get_spanish_hinojosa(), 'target_data': prepare_data.load_stadthagen17 }, { 'name': 'Vo_BE', 'source_rep': VA, 'target_rep': BE5, 'train_data': prepare_data.get_german_bawl(), 'target_data': prepare_data.load_vo09 }, { 'name': 'Imbir_BE', 'source_rep': VA, 'target_rep': BE5, 'train_data': prepare_data.get_polish_nawl(), 'target_data': prepare_data.load_imbir16 }] for s in settings: lex = create_lexicon(source_rep=s['source_rep'], target_rep=s['target_rep'], train_data=s['train_data'], target_data=s['target_data']()) save_tsv(lex, 'lexicons/{}.tsv'.format(s['name']))
def crossvalidate(self, words, labels, k_splits, outpath): ''' Performs crossvalidation with each of the models given to this instance of the Evaluator class. The different models are tested on identical train/test splits which allows for using paired t-tests. ''' if not os.path.isdir(outpath): os.makedirs(outpath) words = pd.Series(words) assert (len(words)==len(labels)), 'Words and labels have unequal'+\ 'length.' assert (k_splits > 1), 'Crossvalidation makes no sense for k<2!' results={key:pd.DataFrame(columns=labels.columns)\ for key in list(self.models)} k = 1 for train_index, test_index in KFold(n_splits=k_splits, shuffle=True).\ split(labels): print('k=' + str(k)) train_labels = labels.iloc[train_index] test_labels = labels.iloc[test_index] train_words = words.iloc[train_index] test_words = words.iloc[test_index] for model_name in list(self.models): print(model_name) model = self.models[model_name] model.fit(train_words, train_labels) preds = model.predict(test_words) preds = pd.DataFrame(preds, columns=labels.columns) results[model_name].loc[k] = util.eval(test_labels, preds) model.initialize() #resets the model. k += 1 # Averaging results print('\n') for m_name in list(results): results[m_name] = util.average_results_df(results[m_name]) util.save_tsv(df=results[m_name], path=outpath + '/' + m_name + '.tsv')
'bawl': load_vo09(), 'nawl': load_riegel15(), 'imbir': load_imbir16() } pairs = { 'anew': 'warriner', 'redondo': 'stadthagen_va', 'hinojosa': 'stadthagen_va', 'ferre': 'stadthagen_be', 'hinojosa': 'stadthagen_be', 'schmidtke': 'bawl', 'nawl': 'imbir' } results = {} for first in pairs.keys(): second = pairs[first] result = compute_isr(data[first], data[second]) print(results) results['{}-{}'.format(first, second)] = result df = pd.DataFrame(index=results.keys(), columns=VAD + BE5 + ['N']) for key, value in results.items(): df.loc[key] = value print(df) save_tsv(df, 'results.tsv')
import pandas as pd from main import data from framework.util import save_tsv df = pd.DataFrame(columns=['N']) for s in data.SETTINGS: df.loc[s.name] = len(s.load_data()) df.rename(index=data.IN_PAPER_NAMES, inplace=True) print(df) save_tsv(df, 'corpus_statistics.tsv')
settings = [ 'English_ANEW_Stevenson', 'German_Schmidtke', 'Polish_Imbir', 'Spanish_Hinojosa', 'Spanish_Redondo' ] results_be = pd.DataFrame(index=settings, columns=BE5) for s in settings: for c in BE5: result=util.get_average_result_from_df('results/be2vad/lm/{}/diff_{}.tsv'\ .format(s,c)) results_be.loc[s, c] = result results_vad = pd.DataFrame(index=settings, columns=VAD) for s in settings: for d in VAD: result=util.get_average_result_from_df('results/vad2be/lm/{}/diff_{}.tsv'\ .format(s,d)) results_vad.loc[s, d] = result df = results_vad.join(results_be, how='inner') df.loc['Average'] = df.mean(axis=0) df = df * (-1) #change sign util.save_tsv(df, 'overview_with_hinojosa.tsv') df.to_latex('overview_with_hinojosa.tex', float_format=util.no_zeros_formatter) df.drop(['Spanish_Hinojosa', 'Average'], inplace=True) df.loc['Average'] = df.mean(axis=0) util.save_tsv(df, 'overview.tsv') df.to_latex('overview.tex', float_format=util.no_zeros_formatter)
def undo_spearman_brown(x): ''' Reverts Spearman-Brown adjustment for k=2 (the case of adjusting split-half reliability). ''' return 1 / ((2 / x) - (2 - 1)) def spearman_brown_adjustment(r, k): r_adjusted = (k * r) / (1 + (k - 1) * r) return r_adjusted save_tsv(df.drop('CORRECTED', axis=1), 'shr_as_reported.tsv') # print(df) ''' Normalize split-half reliabilities to N=10. That is, apply spearman_brown_ adjustment with k=10/N if the score has already been normalized (e.g., N=30 then each half contains ratings from 15 participants. Then the predicted reliability for N=30 was reported). Otherwise, if the reported split-half-reliabilities has not already been adjusted, k is set to 10/(N/2). E.g., N=20, then 10 ratings are in each split, so k=10/(20/2)=1, so the reported reliabilities remain unchanged. ''' N_star = 10 #10 for i in range(df.shape[0]): if df['CORRECTED'][i] == 1: # for j in range(df.shape[1]): # df.iloc[i,j]=undo_spearman_brown(df.iloc[i,j])
for var in list(source_lexicon): models[var] = framework.models.SKlearn_Mapping_Model( base_model=base_model, source_lexicon=source_lexicon.drop(var, axis=1)) # Run actual evaluation ev = framework.models.Evaluator(models=models) ev.crossvalidate(words=target_lexicon.index, labels=target_lexicon, k_splits=k_fold, outpath='results/{}/{}/{}/'.format( curr_dir, base_model_name, setting.name)) ### compute difference to full model: df_full = util.load_tsv('results/{}/{}/{}/full.tsv'.format( curr_dir, base_model_name, setting.name)) print(df_full) for var in list(source_lexicon): df_var = util.load_tsv('results/{}/{}/{}/{}.tsv'.format( curr_dir, base_model_name, setting.name, var)) print(df_var) df_diff = df_var - df_full print(df_diff) util.save_tsv(df=df_diff, path='results/{}/{}/{}/diff_{}.tsv'.format( curr_dir, base_model_name, setting.name, var)) ### compute average values average_subdirs('results/be2vad/lm') average_subdirs('results/vad2be/lm')
source_lexicon = pd.concat( [train_lex[source_rep], test_lex[source_rep]]) target_lexicon = test_lex[target_rep] n_in = source_lexicon.shape[1] n_out = target_lexicon.shape[1] my_model = framework.models.Mapping_Model( layers=[n_in] + MY_MODEL['hidden_layers'] + [n_out], activation=MY_MODEL['activation'], dropout_hidden=MY_MODEL['dropout_hidden'], train_steps=MY_MODEL['train_steps'], batch_size=MY_MODEL['batch_size'], optimizer=MY_MODEL['optimizer'], source_lexicon=source_lexicon, verbose=1) my_model.fit(train_lex.index, train_lex[target_rep]) preds = my_model.predict(target_lexicon.index) preds = pd.DataFrame(preds, columns=list(target_lexicon)) result = util.eval(preds, target_lexicon) print(result) results[d].loc[setting.name] = result print(results[d]) results['vad2be']['Avg_BE'] = results['vad2be'].mean(axis=1) results['be2vad']['Avg_VA'] = results['be2vad'].mean(axis=1) results = pd.concat([results[key] for key in list(DIRECTIONS)], axis=1) util.save_tsv(results, 'results.tsv')
models = ['baseline', 'reference_LM', 'Reference_KNN', 'my_model'] VARS = VAD + BE5 df = pd.DataFrame(index=[setting.name for setting in SETTINGS], columns=VARS) for d in directions: for s in SETTINGS: results = load_tsv('results/{}/{}/my_model.tsv'.format(d, s.name)) for var in VARS: if var in list(results): df.loc[s.name, var] = results.loc['Average', var] df.rename(index=IN_PAPER_NAMES, inplace=True) df.rename(index=str, columns=SHORT_COLUMNS, inplace=True) save_tsv(df, 'overview_individual.tsv') # read normalized split half reliabilites to make larger values bold df_shr = load_tsv('../../analysis/shr/shr_normalized.tsv') df_greater = df > df_shr df_lesser = df < df_shr print(df_greater) print(df_lesser) outperformed = 0 not_outperformed = 0 # add cell colour df = df.round(3) print(df)
df=pd.DataFrame(index=[setting.name for setting in SETTINGS], columns=['be2vad_'+model for model in models]+\ ['vad2be_'+model for model in models]) for d in directions: for setting in SETTINGS: for model in models: perf = get_average_result_from_df('results/{}/{}/{}.tsv'.format( d, setting.name, model)) df.loc[setting.name, d + '_' + model] = perf df.loc['Average'] = df.mean(axis=0) df.rename(index=IN_PAPER_NAMES, inplace=True) save_tsv(df, 'overview.tsv', dec=3) string = df.to_latex(float_format=no_zeros_formatter) lines = string.split('\n') lines[0] = '\\begin{tabular}{|l|rrrr|rrrr|}' lines = [ '%%%%%% Automatic Python output from {} &%%%%%%%%%%'.format( datetime.datetime.now()) ] + lines lines[-1] = '%%%%%%%%%%%%%%%%%%%%%%%%' lines.insert( 3, '{} & \multicolumn{4}{c|}{BE2VAD} & \multicolumn{4}{c|}{VAD2BE} \\\\') lines[4]=lines[4].replace('be2vad\_','').replace('vad2be\_', '').\ replace('Reference\_','').replace('reference\_','').replace('my\_model','FFNN').\