def import_model(config_dict: ConfigDict) -> LanguageModel:
"""
Import a model from a json file.
Parameters
----------
config_dict : ConfigDict
Dictionary containing the model and init_states configuration.
Returns
--------
A LanguageModel created from the given files
"""
model_type = config_dict["model"].pop("type")
module = import_module("diagnnose.model_wrappers")
model_constructor: Type[LanguageModel] = getattr(module, model_type)
model: LanguageModel = model_constructor(**config_dict["model"])
vocab_path = get_vocab_from_config(config_dict)
model.set_init_states(vocab_path=vocab_path, **config_dict["init_states"])
config_dict["model"]["type"] = model_type
return model
if __name__ == "__main__":
arg_groups = {
"activations", "corpus", "decompose", "init_states", "model",
"plot_attention", "vocab", "extract"
}
arg_parser, required_args = create_arg_parser(arg_groups)
config_dict = create_config_dict(arg_parser, required_args)
# load the model
model: LanguageModel = import_model(config_dict)
# load the corpus
corpus: Corpus = import_corpus(
vocab_path=get_vocab_from_config(config_dict), **config_dict["corpus"])
# set fix shapley to false
if "fix_shapley" not in config_dict["decompose"]:
config_dict["decompose"]["fix_shapley"] = False
# get attention?
attention = CDAttention(
model,
corpus,
cd_config=config_dict["decompose"],
plot_config=config_dict["plot_attention"],
)
print("Creating example plot")
attention.plot_by_sen_id([2], avg_decs=True)
from diagnnose.config.arg_parser import create_arg_parser
from diagnnose.config.setup import create_config_dict
from diagnnose.downstream.suite import DownstreamSuite
from diagnnose.models.import_model import import_model
from diagnnose.models.lm import LanguageModel
from diagnnose.vocab import get_vocab_from_config
if __name__ == "__main__":
arg_groups = {"model", "vocab", "downstream", "init_states"}
arg_parser, required_args = create_arg_parser(arg_groups)
config_dict = create_config_dict(arg_parser, required_args, arg_groups)
model: LanguageModel = import_model(config_dict)
vocab_path = get_vocab_from_config(config_dict)
assert vocab_path is not None, "vocab_path should be provided"
suite = DownstreamSuite(
config_dict["downstream"]["config"],
vocab_path,
device=config_dict["model"].get("device", "cpu"),
print_results=True,
)
suite.run(model)
def calc_diff_scores(config,
lm: LanguageModel) -> Dict[str, Dict[str, Tensor]]:
scores = {}
for corpus_type in ["unamb", "stereo"]:
scores[corpus_type] = {}
for condition in ["FM", "MF"]:
corpus_name = f"{corpus_type}_{condition}.txt"
corpus_path = os.path.join(config["corpus"]["path"], corpus_name)
corpus: Corpus = import_corpus(
corpus_path,
vocab_path=get_vocab_from_config(config),
header_from_first_line=True,
)
if config["activations"].get("activations_dir", None) is not None:
activations_dir = os.path.join(
config["activations"]["activations_dir"],
corpus_type,
condition.lower(),
)
else:
activations_dir = None
sen_ids = slice(0, len(corpus))
factory = DecomposerFactory(
lm,
activations_dir or TMP_DIR,
create_new_activations=(activations_dir is None),
corpus=corpus,
sen_ids=sen_ids,
)
ref_types = [ex.ref_type for ex in corpus.examples]
classes = create_winobias_classes(ref_types, corpus)
decomposer = factory.create(sen_ids)
lens = decomposer.final_index - 1
final_hidden = decomposer.activation_dict[decomposer.toplayer,
"hx"][range(len(corpus)),
lens +
1].unsqueeze(2)
full_probs = torch.bmm(lm.decoder_w[classes],
final_hidden).squeeze()
full_probs += lm.decoder_b[classes]
obj_idx_start = torch.tensor(
[ex.obj_idx_start - 1 for ex in corpus])
obj_idx_end = torch.tensor([ex.obj_idx + 1 for ex in corpus])
ranges = [
(0, 2),
(2, obj_idx_start),
(obj_idx_start, obj_idx_end),
(obj_idx_end, lens + 1),
]
scores[corpus_type][condition] = torch.zeros(4)
for i, (start, stop) in enumerate(ranges):
config["decompose"].update({"start": start, "stop": stop})
rel_dec = decomposer.decompose(
**config["decompose"])["relevant"]
final_rel_dec = rel_dec[range(len(corpus)), lens].unsqueeze(2)
rel_probs = torch.bmm(lm.decoder_w[classes],
final_rel_dec).squeeze()
rel_probs /= full_probs
prob_diffs = rel_probs[:, 0] - rel_probs[:, 1]
scores[corpus_type][condition][i] = torch.mean(prob_diffs)
print(corpus_type, condition, scores[corpus_type][condition])
return scores