def test_final_ds():
acs = list(Path("data/labels").glob("*.json"))
final_dsp = "out/standardDsTmp.zip"
with TemporaryDirectory() as temp_dir:
_, _, ac_dsps = replay_ac(acs, temp_dir)
rawdata = load_datazip(ac_dsps[-1])
rawdata_ref = load_datazip(final_dsp)
assert rawdata.y_test_tags == rawdata_ref.y_test_tags
def test_split_ref():
final_dsp = "out/standardDsTmp.zip"
src = "out/standard"
with TemporaryDirectory() as temp_dir:
dsps = mk_standardDs(final_dsp, temp_dir, plot=False)
for idx, dsp in enumerate(dsps):
rawdata = load_datazip(dsp)
ref_rawdata = load_datazip(f"{src}/standardDs{idx}.zip")
assert rawdata.y_test_tags == ref_rawdata.y_test_tags
def train_eval(
dsp,
output_p,
upsample=200,
keep_key=True,
over=5,
cv=True,
):
max_len = 150
marker = Path(dsp).stem
fn = f"{output_p}/{marker}tag_stat.csv"
ds = load_datazip(dsp)
tag_stat = rawdata_stat(ds)
tag_stat.to_csv(fn)
mlb = MultiLabelBinarizer().fit(ds.y_tags)
if not cv:
single_train_eval(ds, max_len, upsample, keep_key, over, mlb, output_p,
marker)
else:
for idx, tmp_ds in enumerate(kf_flow(ds)):
single_train_eval(
tmp_ds,
max_len,
upsample,
keep_key,
over,
mlb,
output_p,
marker + f"cv{idx}",
)
def form_random_ds(
standard_dsps: List[str],
eval_ret="mona_j.csv",
unlabelled_p="unlabelled.json",
outdir=".",
train_size=400,
):
df = pd.read_csv(eval_ret).drop_duplicates(subset=["ID", "Judge"],
keep="last")
indices = df["ID"].to_list()
sampled_cases = load_json(unlabelled_p)
add_text = [sampled_cases[idx] for idx in indices]
y_true_ = df["eval"].map(lambda x: x.split(", ")).to_list()
dsps = []
for i, base_path in enumerate(standard_dsps):
ds = load_datazip(base_path)
random_idx = random.sample(list(range(len(indices))), train_size)
x_train_dict = [add_text[idx] for idx in random_idx]
y_train_tags = [y_true_[idx] for idx in random_idx]
ds.x_train_dict = x_train_dict
ds.y_train_tags = y_train_tags
ds.x_dict = x_train_dict + ds.x_test_dict
ds.y_tags = y_train_tags + ds.y_test_tags
dsp = dump_datazip(ds, f"{outdir}/randomDs{i}.zip")
dsps.append(dsp)
print(dsp)
return dsps
def test_random_ds():
standard_dsps = [f"out/standard/standardDs{idx}.zip" for idx in range(3)]
eval_ret = "data/evaluation/mona_j.csv"
unlabelled_p = "data/unlabelled.json"
src = "out/random"
with TemporaryDirectory() as temp_dir:
dsps = mk_randomDs(
standard_dsps,
eval_ret=eval_ret,
unlabelled_p=unlabelled_p,
dst=temp_dir,
plot=False,
)
for idx, dsp in enumerate(dsps):
rawdata = load_datazip(dsp)
ref_rawdata = load_datazip(f"{src}/randomDs{idx}.zip")
assert rawdata.y_test_tags == ref_rawdata.y_test_tags
def mk_standardDs(final_dsp="standardDsTmp.zip", dst="out/standard", plot=False):
dsps = dataset_split(final_dsp, dst)
if plot:
for idx, dsp in enumerate(dsps):
rawdata = load_datazip(dsp)
tag_stat = rawdata_stat(rawdata)
tag_stat.to_csv(f"{dst}/data_stat{idx}.csv")
fig = plot_tag_stat(tag_stat)
fig.write_image(f"{dst}/data_stat{idx}.pdf")
return dsps
def test_prediction():
model = StandaloneModel.from_path("TagModel", keep_key=False, max_len=100)
cases = load_json("data/eval.json")
ds = load_datazip("data/dataset.zip")
mlb = MultiLabelBinarizer().fit(ds.y_tags)
indexes = [9, 15]
used = [cases[idx]["text"] for idx in indexes]
prob = model.predict_prob(used, mlb)
tags = model.predict_tags(used, mlb)
pred_out = mlb.transform(tags)
build_eval_json(used, prob, pred_out)
def dataset_split(final_dsp, dst="."):
ds = load_datazip(final_dsp)
standard_dsps = []
for i in range(3):
output = f"{dst}/standardDs{i}.zip"
split_and_dump_dataset(ds.x_dict,
ds.y_tags,
train_first=False,
output=output)
standard_dsps.append(output)
return standard_dsps
def main(
dataset_p: str,
unlabelled_p: str,
model_p: str,
outdir: str,
plot=True,
train=False,
):
if train:
ds = load_datazip(dataset_p)
train_main_model(ds, model_p=model_p)
if plot:
make_figures(model_p, dataset_p, unlabelled_p, dst=outdir)
def mk_randomDs(
standard_dsps,
eval_ret="data/evaluation/mona_j.csv",
unlabelled_p="data/unlabelled.json",
dst="out/random",
plot=False,
):
dsps = form_random_ds(standard_dsps, eval_ret, unlabelled_p, outdir=dst)
if plot:
for idx, dsp in enumerate(dsps):
rawdata = load_datazip(dsp)
tag_stat = rawdata_stat(rawdata)
tag_stat.to_csv(f"{dst}/random_data_stat{idx}.csv")
fig = plot_tag_stat(tag_stat)
fig.write_image(f"{dst}/random_data_stat{idx}.pdf")
return dsps
def run_experiment(
output_p="output",
dataset_path="stdDs.zip",
upsample=200,
keep_key=True,
over=None,
step=50,
metrics_only=False,
):
if over is None:
over = -1 if upsample == -1 else 5
output_p = f"{output_p}/{'keepKey_' if keep_key else 'noKey_'}{upsample}"
os.makedirs(output_p, exist_ok=True)
ds = load_datazip(dataset_path)
mlb = MultiLabelBinarizer().fit(ds.y_tags)
max_len = 150
for size in range(step, len(ds.x_train_dict) + 1, step):
if metrics_only:
model = StandaloneModel.from_path(f"{output_p}/model",
keep_key=keep_key,
max_len=max_len)
eval_model(model, ds, over, mlb, output_p, size)
else:
part_ds = slice_dataset(ds, size)
params = Params(part_ds, max_len, upsample, 0.5,
"bert-base-uncased", keep_key, 10, mlb)
print(len(part_ds.x_train_dict))
pipeline = Pipeline(params)
model_p = pipeline.train(output_dir=output_p)
model = StandaloneModel(pipeline.model,
pipeline.tokenizer,
keep_key=keep_key,
max_len=max_len)
eval_model(model, part_ds, over, mlb, output_p, size)
if size >= len(ds.x_train_dict):
pipeline.trainer.save_model(f"{output_p}/model")
del pipeline
shutil.rmtree(model_p)
del model
gc.collect()
with torch.no_grad():
torch.cuda.empty_cache()
def _add_training(
self,
eval_ret: str,
base_path,
idx_marker=1,
):
outdir = self.outdir
unlabelled_p = self.unlabelled_p
ds = load_datazip(base_path)
df = pd.read_csv(eval_ret).drop_duplicates(subset=["ID", "Judge"],
keep="last")
indices = df["ID"].to_list()
sampled_cases = load_json(unlabelled_p)
add_texts = [sampled_cases[idx] for idx in indices]
y_true_ = df["eval"].map(lambda x: x.split(", ")).to_list()
ds.x_train_dict = ds.x_train_dict + add_texts
ds.y_train_tags = ds.y_train_tags + y_true_
ds.x_dict = ds.x_dict + add_texts
ds.y_tags = ds.y_tags + y_true_
dsp = dump_datazip(ds, f"{outdir}/dataset{idx_marker}.zip")
print(dsp)
return ds
def __init__(
self,
init_model_p,
eval_ret="mona_j.csv",
dataset_p="stdDs.zip",
ori_eval_p="outputsS/eval.json",
unlabelled_p="outputsK/unlabelled.json",
outdir="feedbackM",
):
self.init_model_p = init_model_p
self.eval_ret = eval_ret
self.dataset_p = dataset_p
self.ori_eval_p = ori_eval_p
self.unlabelled_p = unlabelled_p
self.outdir = outdir
Path(self.outdir).mkdir(exist_ok=True, parents=True)
shutil.copyfile(self.dataset_p, f"{outdir}/dataset0.zip")
shutil.copyfile(self.ori_eval_p, f"{outdir}/eval0.json")
self.ds = load_datazip(self.dataset_p)
self.mlb = MultiLabelBinarizer().fit(self.ds.y_tags)
self.df = pd.read_csv(eval_ret).drop_duplicates(subset=["ID", "Judge"],
keep="last")
model,
mlb,
unlabelled_p="outputsK/unlabelled.json",
outdir="outputs",
over=5,
marker="",
skip_state=False,
):
sampled_cases = load_json(unlabelled_p)
if not skip_state:
sampled_state = get_unlabelled_state(model, sampled_cases, mlb)
dump_state(sampled_state, state_p=f"{outdir}/unstate{marker}.pkl")
unstate_df = dimension_reduction(sampled_state, "TSNE", n_components=2)
unstate_df.to_csv(f"{outdir}/unlabel_tsne{marker}.csv")
fig = state_plot(unstate_df, 12)
fig.write_image(f"{outdir}/unlabelled_TSNE{marker}.pdf")
fig.write_html(f"{outdir}/unlabel_tsne{marker}.html")
preds = model.over_predict(sampled_cases, n=over)
thresh_items = label_output(preds)
pred_prob = [list(zip(mlb.classes_, pred)) for pred in preds]
eval_json = build_eval_json(sampled_cases, pred_prob, thresh_items)
dump_json(f"{outdir}/eval{marker}.json", eval_json)
if __name__ == "__main__":
ds = load_datazip("dataset.zip")
mlb = MultiLabelBinarizer().fit(ds.y_tags)
model_p = "lab4/keepKey_200/model"
model = StandaloneModel.from_path(model_p, keep_key=True, max_len=150)
form_eval(model, mlb)
def sample_evaluation_from_path(cases_p, dsp):
all_cases = load_json(cases_p)
dataset = load_datazip(dsp)
return sample_evaluation(all_cases, dataset)
def make_figures(model_p: str, dsp: str, unlabelled_p: str, dst="figF"):
dataset = load_datazip(dsp)
model = StandaloneModel.from_path(model_p, keep_key=True, max_len=150)
over = 5
mlb = MultiLabelBinarizer().fit(dataset.y_tags)
os.makedirs(dst, exist_ok=True)
# Rawdata_stat
fn = f"{dst}/data_stat.csv"
if os.path.exists(fn):
tag_stat = pd.read_csv(fn, index_col=0)
else:
tag_stat = rawdata_stat(dataset)
tag_stat.to_csv(fn)
fig = plot_tag_stat(tag_stat)
fig.write_image(f"{dst}/data_stat.pdf")
# Unlabelled
fn = f"{dst}/unlabel_tsne.csv"
if os.path.exists(fn):
unstate_df = pd.read_csv(fn, index_col=0)
else:
sampled_cases = load_json(unlabelled_p)
sampled_state = get_unlabelled_state(model, sampled_cases, mlb)
dump_state(sampled_state, state_p=f"{dst}/unstate.pkl")
unstate_df = dimension_reduction(sampled_state, "TSNE", n_components=2)
unstate_df.to_csv(fn)
preds = model.over_predict(sampled_cases, n=over)
thresh_items = label_output(preds)
pred_prob = [list(zip(mlb.classes_, pred)) for pred in preds]
eval_json = build_eval_json(sampled_cases, pred_prob, thresh_items)
dump_json(f"{dst}/eval.json", eval_json)
fig = state_plot(unstate_df, 12)
fig.write_image(f"{dst}/unlabel_tsne.pdf")
fig.write_html(f"{dst}/unlabel_tsne.html")
# Labelled
fn = f"{dst}/label_tsne.csv"
if os.path.exists(fn):
state_df = pd.read_csv(fn, index_col=0)
else:
states = get_tag_states(model, dataset, mlb)
state_df = dimension_reduction(states, "TSNE", n_components=2)
state_df.to_csv(fn)
fig = state_plot(state_df, 12)
fig.write_image(f"{dst}/label_tsne.pdf")
fig.write_html(f"{dst}/label_tsne.html")
# Performance
eval_model(model, dataset, over, mlb, dst, "")
# Top key
fn = f"{dst}/top_key.json"
if os.path.exists(fn):
top_key = load_json(fn)
else:
maskExplainer = MaskExplainer(mlb)
top = top_keywords(maskExplainer, model, dataset.x_dict)
top_key = filter_top(top, dataset.y_tags, thresh=20)
dump_json(fn, top_key)
fig = kw_plot(top_key)
fig.write_image(f"{dst}/knockout_result.pdf")
def test_upsampling(dsp):
ds = load_datazip(dsp)
new_x1, _ = upsampling(ds.x_train_dict, ds.y_train_tags, target=200)
new_x2, _ = upsampling(ds.x_train_dict, ds.y_train_tags, target=-200)
print(len(new_x1))
assert len(new_x1) == len(new_x2)