def _eval_lex_model(label_est_samples, valid_samples) -> float:
estimated_labelprops = {
"estimated": calculate_labelprops(
label_est_samples,
_DATADEF.n_classes,
_DATADEF.domain_names,
)
}
datadef = get_datadef(_DATASET_NAME)
datadef.load_labelprops_func = lambda _split: estimated_labelprops[
_split
]
metrics = eval_lexicon_model(
model,
datadef,
valid_samples,
vocab,
use_source_individual_norm=_LEXICON_CONFIG[
"use_source_individual_norm"
],
labelprop_split="estimated", # match _load_labelprops_func()
)
return metrics["valid_f1"]
import sys
from os.path import basename, join, realpath
import torch
from config import LEXICON_DIR
from experiments.datadef.zoo import get_datadef
from modapt.eval import reduce_and_save_metrics
from modapt.lexicon import eval_lexicon_model, run_lexicon_experiment
from modapt.model.logreg_config.grid_search import load_logreg_model_config_all_archs
from modapt.utils import read_txt_as_str_list, save_json
_DATASET_NAME = sys.argv[1]
_REG = float(sys.argv[2])
_DATADEF = get_datadef(_DATASET_NAME)
_SCRIPT_PATH = realpath(__file__)
_EXPERIMENT_NAME = basename(_SCRIPT_PATH).replace(".py", "")
_SAVE_ROOT = join(LEXICON_DIR, _DATASET_NAME, _EXPERIMENT_NAME)
_ARCH2CONFIG = load_logreg_model_config_all_archs(_DATADEF.n_classes,
_DATADEF.n_sources)
for arch, config in _ARCH2CONFIG.items():
config["reg"] = _REG # override with cmd arg
print("\n")
print("+" * 30)
print(arch)
print("+" * 30)
# py <script> <savedir>
import sys
from os import makedirs
from os.path import join
from random import Random
import pandas as pd
from config import RANDOM_SEED
from experiments.datadef.zoo import get_datadef
from modapt.dataset.common import to_df
_DATADEF = get_datadef("sentiment")
_SAVEDIR = sys.argv[1]
_N_TRAIN_SAMPLE_PER_DOMAIN = 100
_N_VALID_LABELED = 20
_N_VALID_UNLABELED = 20
_RNG = Random(RANDOM_SEED)
_TRAIN_DOMAINS = ["airline", "imdb", "senti140", "sst"]
_VALID_DOMAIN = "amazon"
makedirs(_SAVEDIR, exist_ok=True)
train_samples = []
for d in _TRAIN_DOMAINS:
samples = _DATADEF.load_splits_func([d], ["train"])["train"]
_RNG.shuffle(samples)
train_samples.extend(samples[:_N_TRAIN_SAMPLE_PER_DOMAIN])
train_df = to_df(train_samples)
train_df.to_csv(join(_SAVEDIR, "train.csv"))
from os.path import join
import pandas as pd
from config import DATA_DIR
from experiments.datadef import zoo
from modapt.utils import load_json, save_json
FRAMING_DATA_DIR = join(DATA_DIR, "framing_labeled")
ISSUES = zoo.get_datadef("framing").domain_names
if __name__ == "__main__":
stats = []
for issue in ISSUES:
print(">>", issue)
data = load_json(join(FRAMING_DATA_DIR, f"{issue}_labeled.json"))
ids = list(data.keys())
testsets = load_json(join(FRAMING_DATA_DIR, f"{issue}_test_sets.json"))
testsets = {setname: set(ids) for setname, ids in testsets.items()}
trainsets = {}
# relevance train set: any sample not in test set relevance
trainsets["relevance"] = list({
id
for id in data if (id in ids and id not in testsets["relevance"])
})
# primary frame trainset: any sample not in testset primary frame, and has non null primary fram
trainsets["primary_frame"] = list({
id
]
def get_valid_accs(metrics):
return np.array([
max(
metrics[source]["mean"].get("f1", 0),
metrics[source]["mean"].get("valid_f1", 0),
metrics[source]["mean"].get("valid_f1.best", 0),
) for source in _DATADEF.domain_names
])
for datasetname in _DATASETS:
rows = {}
_DATADEF = get_datadef(datasetname)
source2labelprops = _DATADEF.load_labelprops_func(
"train" if "test" not in _METRIC_FILENAME else "test")
# acc if naively choose most common label from each source
most_common_acc = np.array([a.max()
for a in source2labelprops.values()]).mean()
rows["most_common"] = {"acc": round(most_common_acc, 3), "delta_std": "-"}
lexicon_metrics = load_json(
join(LEXICON_DIR, datasetname, _EXP_NAME, _METRIC_FILENAME))
lexicon_base_accs = np.array([
lexicon_metrics[_LEXICON_BASE_ARCH][source]["mean"]["valid_f1"]
for source in _DATADEF.domain_names
])
rows[_LEXICON_BASE_ARCH] = {
