def plot_l1_with_code_switches(results_dir='../../simulations/messageless/messageless_balanced', languages=['en', 'es']): df = pd.read_csv(f'{results_dir}/performance_per_lang.csv', index_col=None, header=0, skipinitialspace=True, dtype={'epoch': int}) plt = Plotter(results_dir=results_dir) for l in languages: plt.performance(df[df.switch_from == l], fname=f'l1_performance_{l}', include_code_switches=True, max_epochs=40)
def pairwise_cross_model_comparison(create_files, fname='cog_models_merged', only_last_epoch=True, results_dir='../../simulations/cog_paper/cognate_percentage/pairwise_training/' 'evaluation', models=('10/cog', '15/cog', '20/cog', '25/cog', '30/cog')): if create_files: create_all_model_csv_files(results_dir, models=models, create_csv=True, only_last_epoch=only_last_epoch) all_df = [] for m in models: m = m.replace('/', '_') print(m, f'{results_dir}/{m}_models_merged.csv') df = pd.read_csv(f'{results_dir}/{m}_models_merged.csv') df['model_name'] = m.split('_')[0] all_df.append(df) sentences_to_test = pd.concat(all_df, sort=False) only_common_sentences = True if only_common_sentences: count_correct = sentences_to_test.groupby('sentence_idx').count() num_per_unique_sentence = len(models) * 2 # models * (cognate + non_cognate sentences) sentences_to_test = sentences_to_test.loc[sentences_to_test['sentence_idx'].isin( count_correct[count_correct['model_name'] == num_per_unique_sentence].index)] sentences_to_test.to_csv(f'{results_dir}/{fname}.csv') group_models(results_dir, f'{fname}.csv') plt = Plotter(results_dir=results_dir) plt.plot_cognate_last_epoch(df_name=f'count_{fname}.csv', ci=68, xaxis_step=5, info_to_plot=('code_switched',), lineplot=True)
def group_models(results_dir, fname, make_exclusive=False, per_switch_direction=False): print(f'{results_dir}/{fname}') df = pd.read_csv(f'{results_dir}/{fname}', dtype={'switched_before': int, 'switched_at': int, 'switched_right_after': int, 'switched_second_after': int, 'switched_after_anywhere': int, 'concept_position': int, 'first_switch_position': int}) if make_exclusive: df['switched_after_anywhere'] = df.apply(lambda x: (0 if ((x['switched_right_after'] == 1) or (x['switched_second_after'] == 1) or (x['switched_at'] == 1) or (x['switched_before'] == 1)) else x['switched_after_anywhere']), axis=1) df['switched_second_after'] = df.apply(lambda x: (0 if ((x['switched_right_after'] == 1) or (x['switched_at'] == 1) or (x['switched_before'] == 1)) else x['switched_second_after']), axis=1) df['switched_right_after'] = df.apply( lambda x: (0 if (x['switched_at'] == 1) or (x['switched_before'] == 1) else x['switched_right_after']), axis=1) df['switched_at'] = df.apply(lambda x: 0 if x['switched_before'] == 1 else x['switched_at'], axis=1) df.to_csv(f'{results_dir}/{"exclusive_" if make_exclusive else ""}{"per_lang" if per_switch_direction else ""}' f'{fname}') groups = ['epoch', 'network_num', 'model'] if 'model_name' in df: groups.append('model_name') if per_switch_direction: groups.append('switch_from') gb = df.groupby(groups).apply(lambda dft: pd.Series( {'code_switched': dft.is_code_switched.sum(), 'total_sentences': len(dft.meaning), 'switched_before': dft.switched_before.sum(), 'switched_at': dft.switched_at.sum(), 'switched_right_after': dft.switched_right_after.sum(), 'switched_second_after': dft.switched_second_after.sum(), 'switched_after_anywhere': dft.switched_after_anywhere.sum(), 'code_switched_percentage': dft.is_code_switched.sum() * 100 / len(dft.meaning), 'switched_before_percentage': (dft.switched_before.sum() * 100 / len(dft.meaning)), 'switched_at_percentage': dft.switched_at.sum() * 100 / len(dft.meaning), 'switched_right_after_percentage': (dft.switched_right_after.sum() * 100 / len(dft.meaning)), 'switched_second_after_percentage': (dft.switched_second_after.sum() * 100 / len(dft.meaning)), 'switched_after_anywhere_percentage': (dft.switched_after_anywhere.sum() * 100 / len(dft.meaning)) })) gb.to_csv(f'{results_dir}/{"exclusive_" if make_exclusive else ""}{"per_lang" if per_switch_direction else ""}' f'count_{fname}')
def non_pairwise_cross_model_comparison(results_dir='../../simulations/cog_paper/cognate_percentage/' 'non_pairwise_training/', create_csv=True, num_simulations=70, remove_incorrect_meaning=True): if create_csv: df_per_test_set = [] for fname in ['non_cognate_evaluation', 'cognate_evaluation']: models = [i for i in range(0 if 'non' in fname else 10, num_simulations + 1, 10)] num_per_unique_sentence = len(models) all_dfs = [] for m in models: print(m) df = pd.read_csv(f'{results_dir}/{fname}/{m}/all_results.csv') df = df[df.epoch == df.epoch.max()] if remove_incorrect_meaning: df.drop(df.loc[df.meaning == 0].index, inplace=True) if fname == 'cognate_evaluation': assert len(df) == len(df[df.target_has_cognate == True]) df['model_name'] = m df['sentence_idx'] = df.apply(lambda x: hash_string(f'{x.message}{x.network_num}'), axis=1) df['test_set'] = fname all_dfs.append(df) non_pairwise_merged = pd.concat(all_dfs, sort=False) only_common_sentences = True if only_common_sentences: count_correct = non_pairwise_merged.groupby('sentence_idx').count() print(num_per_unique_sentence, 'total:', len(non_pairwise_merged)) non_pairwise_merged = non_pairwise_merged.loc[non_pairwise_merged['sentence_idx'].isin( count_correct[count_correct['model_name'] == num_per_unique_sentence].index)] print('clean:', len(non_pairwise_merged)) df_per_test_set.append(non_pairwise_merged) test_sets_merged = pd.concat(df_per_test_set, sort=False) test_sets_merged.to_csv(f'{results_dir}/non_pairwise_cognate_non_cognate.csv') gb = test_sets_merged.groupby(['epoch', 'network_num', 'model_name', 'test_set']).apply( lambda dft: pd.Series({'code_switched': dft.is_code_switched.sum(), 'total_sentences': len(dft.meaning), 'code_switched_percentage': dft.is_code_switched.sum() * 100 / len(dft.meaning) })) gb.to_csv(f'{results_dir}/non_pairwise_test_sets_grouped' f'{f"_include_incorrect" if not remove_incorrect_meaning else ""}.csv') plt = Plotter(results_dir=results_dir) plt.plot_cognate_last_epoch(df_name='non_pairwise_test_sets_grouped.csv', hue='test_set', include_annotations=False, lineplot=True, ci=68, xrow='model_name')
def plot_l1_l2_performance_cognate_models(results_dir='../../simulations/cog_paper/'): plt = Plotter(results_dir=results_dir) for testset in ['generic']: # ['bos', 'eos', 'generic']: for model in ['balanced', 'enes', 'esen']: for m in ['cog1', 'cog2']: print(f'{results_dir}/{testset}/{model}/{m}/performance_per_lang.csv') df = pd.read_csv(f'{results_dir}/{testset}/{model}/{m}/performance_per_lang.csv', index_col=None, header=0, skipinitialspace=True, dtype={'epoch': int}) l2_lang = model[-2:] if model != 'balanced' else None lang = [model[-4:-2]] if model != 'balanced' else ['en', 'es'] if model != 'balanced': df.l2_epoch = 10 print(lang, l2_lang) for l in lang: plt.performance(df[df.switch_from == l], fname=f'l1_{testset}_{model}_{m}_{l}', include_code_switches=True) if l2_lang: plt.l2_performance(df, l2_lang=l2_lang, fname=f'l2_{testset}_{model}_{m}_{l2_lang}', include_code_switches=True)
def create_all_model_csv_files(results_dir, only_last_epoch=True, create_csv=True, models=('ff',), per_switch_direction=False): if create_csv: for m in models: create_model_csv_files(results_dir, models=(f'{m}1', f'{m}2'), only_last_epoch=only_last_epoch) for m in models: m = m.replace("/", "_") fnames = [f'{results_dir}/all_epochs_{m}1_{m}2.csv', f'{results_dir}/all_epochs_{m}2_{m}1.csv'] all_df = [] for fname in fnames: if os.path.isfile(fname): df = pd.read_csv(fname) all_df.append(df) columns = ['epoch', 'network_num', 'sentence_idx', 'model', 'meaning', 'is_code_switched', 'switched_type', 'switched_before', 'switched_at', 'switched_right_after', 'switched_second_after', 'switched_after_anywhere', 'first_switch_position', 'switch_from', 'produced_sentence', 'target_sentence', 'produced_pos', 'target_pos'] pd.concat(all_df, sort=False).to_csv(f'{results_dir}/{m}_full_models_merged.csv', index=False, columns=columns) group_models(results_dir, f'{m}_models_merged.csv', per_switch_direction=per_switch_direction)
def create_model_csv_files(results_dir, models=('cog1', 'cog2'), model_name='cognate', only_last_epoch=False, only_compare_with_sentences_not_containing_any_cognates=True, only_test_cognates=True): """ :param results_dir: :param models: :param model_name: cognate or false_friend :param only_last_epoch: :param only_compare_with_sentences_not_containing_any_cognates: :return: """ cognate_list = None print('Cognate list file name:', cognate_list_fname) if cognate_list_fname: with open(f'{results_dir}/{cognate_list_fname}', encoding='utf-8') as f: cognate_list = [line.rstrip('\n').lower() for line in f] for model_pair in [models, list(reversed(models))]: print(model_pair) target_concepts = None if cognate_list: with open(f'{results_dir}/{model_pair[0]}/input/cognates.in', encoding='utf-8') as f: cognates_in_simulation = [line.rstrip('\n').lower() for line in f] target_concepts = '|'.join(set(cognate_list).intersection(cognates_in_simulation)) print('target concepts:', target_concepts) elif only_test_cognates: with open(f'{results_dir}/{model_pair[0]}/input/cognates.in', encoding='utf-8') as f: cognates_in_simulation = [line.rstrip('\n').lower() for line in f] target_concepts = '|'.join(cognates_in_simulation) print('target concepts:', target_concepts) if cognate_list and not target_concepts: break cog_or_ff_df = pd.read_csv(f'{results_dir}/{model_pair[0]}/all_results.csv') cog_or_ff_df.loc[:, 'model'] = model_name no_cog_or_ff_df = pd.read_csv(f'{results_dir}/{model_pair[1]}/all_results.csv') no_cog_or_ff_df.loc[:, 'model'] = f'non_{model_name}' if only_last_epoch: cog_or_ff_df = cog_or_ff_df[cog_or_ff_df.epoch == cog_or_ff_df.epoch.max()] print(len(cog_or_ff_df)) no_cog_or_ff_df = no_cog_or_ff_df[no_cog_or_ff_df.epoch == no_cog_or_ff_df.epoch.max()] print(len(no_cog_or_ff_df)) if 'false_friend' in model_name: sentences_with = cog_or_ff_df.loc[cog_or_ff_df.target_has_false_friend == True].copy() else: sentences_with = cog_or_ff_df.loc[cog_or_ff_df.target_has_cognate == True].copy() if target_concepts: sentences_with = sentences_with.loc[sentences_with.target_sentence.str.contains(target_concepts)] # assign sentence_idx: sentences_with['sentence_idx'] = sentences_with.apply(lambda x: hash_string(f'{x.message}{x.network_num}'), axis=1) no_cog_or_ff_df['sentence_idx'] = no_cog_or_ff_df.apply(lambda x: hash_string(f'{x.message}{x.network_num}'), axis=1) sentences_without = no_cog_or_ff_df.copy() # Remove sentences with incorrect meaning or those in which the word of interest was produced at the end. sentences_with.drop(sentences_with.loc[sentences_with.meaning == 0].index, inplace=True) print(len(sentences_without)) sentences_without.drop(sentences_without.loc[sentences_without.meaning == 0].index, inplace=True) print('correct meaning:', len(sentences_without)) if only_compare_with_sentences_not_containing_any_cognates: sentences_without.drop(sentences_without.loc[sentences_without.target_has_cognate == True].index, inplace=True) print('without: not containing any cognates', len(sentences_without), '---') sentences_to_test = pd.concat([sentences_with, sentences_without], sort=False) sentences_to_test.loc[sentences_to_test.concept_position == 'None', 'concept_position'] = -1 sentences_to_test.loc[sentences_to_test.first_switch_position == 'None', 'first_switch_position'] = -1 # sentences that are correctly produced among all models fname = f'all_epochs_{"_".join(model_pair).replace("/", "_")}.csv' # remove sentences that have not been produced by both cognate/non-cognate models count_correct = sentences_to_test.groupby('sentence_idx').count() sentences_to_test = sentences_to_test.loc[sentences_to_test['sentence_idx'].isin( count_correct[count_correct['model'] == len(model_pair)].index)] print('final sentences to test:', len(sentences_to_test)) # -1 because indexing of other columns starts from 0, not 1 sentences_to_test.to_csv(f'{results_dir}/{fname}') group_models(results_dir, fname)