def main(model1='lstm', criterion1='bald', model2='cnn', criterion2='bald', model_seed=0,
domain='alarm', data_seed=0, batchsize=20, max_epoch=100, patience=20, tot_acq=160, log_dir='logs'):
m_dict = {'lstm': 'LSTM', 'cnn': 'CNN', 'aoe': 'AOE', 'roberta': 'RoBERTa'}
c_dict = {'max-entropy': 'Max-Entropy', 'bald': 'BALD'}
optimal_order1, _, _ = load_optimal(log_dir, model1, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)
optimal_order2, _, _ = load_optimal(log_dir, model2, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)
plot_relative_order(optimal_order1, optimal_order2,
f'{m_dict[model1]}\nOptimal', f'{m_dict[model2]}\nOptimal')
plt.savefig(f'../figures/intent_classification/relative_orders/{model1}_{model2}_optimal.pdf',
bbox_inches='tight')
heuristic_order1 = load_baseline(criterion1, 'test', model1, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)['order']
heuristic_order2 = load_baseline(criterion2, 'test', model2, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)['order']
plot_relative_order(heuristic_order1, heuristic_order2,
f'{m_dict[model1]}\n{c_dict[criterion1]}', f'{m_dict[model2]}\n{c_dict[criterion2]}')
plt.savefig(f'../figures/intent_classification/relative_orders/{model1}_{model2}_heuristic.pdf',
bbox_inches='tight')
def main(criterion, model='lstm', model_seed=0, domain='alarm', data_seed=0,
batchsize=20, max_epoch=100, patience=20, tot_acq=160, evaluation_set='test', gpu_idx=0, log_dir='logs'):
assert model != 'roberta', 'IDMR for RoBERTa model is not implemented'
data = pickle.load(open('data/TOP.pkl', 'rb'))[domain]['seeds'][data_seed]
N_warmstart = len(data['warmstart'])
N = len(data['pool'])
try:
idmr_curve = load_baseline(f'idmr-{criterion}', evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)['curve']
except KeyError:
lens_proportions = None
accessible_set = data['warmstart'] + data['train_valid'] + data['pool']
accessible_sents, _ = zip(*accessible_set)
lens = [len(nltk.word_tokenize(s)) for s in accessible_sents]
lens_ct = Counter(lens)
cts = np.array([lens_ct[l] for l in range(max(lens_ct.keys()) + 1)])
props = cts / sum(cts)
groups = group_proportions(props, 0.08)
lens_proportions = dict()
for i, g in enumerate(groups):
lo, hi = min(g), max(g)
sum_props = sum(props[p] for p in range(lo, hi + 1))
if i == len(groups) - 1:
hi = 100
lens_proportions[(lo, hi)] = sum_props
idmr_curve, idmr_order = idmr(data, criterion, evaluation_set, lens_proportions, model, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq, gpu_idx)
print(idmr_curve)
print(np.mean(idmr_curve))
store_baseline(idmr_curve, idmr_order, f'idmr-{criterion}', evaluation_set, model, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)
plt.figure()
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
baselines = [('max-entropy', 'Max-Entropy', 0), ('bald', 'BALD', 1), ('random', 'Random', 4)]
if criterion == 'max-entropy':
baselines.append(('idmr-max-entropy', 'IDMR Max-Ent.', 6))
elif criterion == 'bald':
baselines.append(('idmr-bald', 'IDMR BALD', 6))
optimal_order, _, _ = load_optimal(log_dir, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model, model_seed, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq, None, None, baselines)
xmin1, xmax1, ymin1, ymax1 = plt.axis()
plt.xticks(np.linspace(N_warmstart, tot_acq + N_warmstart, 5))
plt.xlabel('# Data Points')
plt.ylabel('F1')
plt.title('Input Distribution-Matching Regularization')
plt.savefig(f'../figures/intent_classification/idmr_{criterion}.pdf', bbox_inches='tight')
def main(criterion, evaluation_set, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq, log_dir, gpu_idx):
data = pickle.load(open('data/restaurant.pkl', 'rb'))['seeds'][data_seed]
N_warmstart = len(data['warmstart'])
try:
idmr_curve = load_baseline(f'idmr-{criterion}', evaluation_set, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)['curve']
except KeyError:
N_pool = len(data['pool'])
accessible_set = data['warmstart'] + data['train_valid'] + data['pool']
accessible_sents, _ = zip(*accessible_set)
lens = [len(s) for s in accessible_sents]
lens_ct = Counter(lens)
max_len = max(lens_ct.keys())
cts = np.array([lens_ct[l] for l in range(max_len + 1)])
props = cts / sum(cts)
groups = group_proportions(props, 0.13)
lens_proportions = dict()
for i, g in enumerate(groups):
lo, hi = min(g), max(g)
sum_props = sum(props[p] for p in range(lo, hi + 1))
if i == len(groups) - 1:
hi = 100
lens_proportions[(lo, hi)] = sum_props
idmr_curve, idmr_order = idmr(data, evaluation_set, criterion, lens_proportions, model_seed,
batchsize, max_epoch, patience, tot_acq, gpu_idx)
store_baseline(idmr_curve, idmr_order, f'idmr-{criterion}', evaluation_set, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)
print(idmr_curve)
print(np.mean(idmr_curve))
plt.figure()
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
display_name = {'min-confidence': 'Min-Confidence', 'normalized-min-confidence': 'Norm.-Min-Conf.',
'longest': 'Longest'}[criterion]
baselines = [(criterion, display_name, 0), (f'idmr-{criterion}', f'IDMR-{display_name.replace("idence", ".")}', 6),
('random', 'Random', 4)]
plot_curves(optimal_order, xs, evaluation_set, model_seed, model_seed, data_seed, batchsize, max_epoch, patience,
tot_acq, None, None, baselines)
plt.tight_layout()
plt.savefig(f'../figures/named_entity_recognition/idmr_{criterion}.pdf', bbox_inches='tight')
def main(model_seed=0,
data_seed=0,
batchsize=25,
max_epoch=100,
patience=20,
tot_acq=300,
evaluation_set='test',
log_dir='logs',
tsne_dim=3,
num_clusters=5,
gpu_idx=0):
data = load_data(data_seed)
N_warmstart = len(data['warmstart'])
try:
idmr_curve = load_baseline('idmr-max-entropy', evaluation_set,
model_seed, data_seed, batchsize, max_epoch,
patience, tot_acq)['curve']
except KeyError:
data = load_data(data_seed)
idmr_curve, idmr_order = idmr(data, evaluation_set, model_seed,
batchsize, max_epoch, patience, tot_acq,
tsne_dim, num_clusters, gpu_idx)
store_baseline(idmr_curve, idmr_order, 'idmr-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
print(idmr_curve)
print(np.mean(idmr_curve))
plt.figure()
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
baselines = [('max-entropy', 'Max-Entropy', 0), ('bald', 'BALD', 1),
('random', 'Random', 4),
('idmr-max-entropy', 'IDMR Max-Ent.', 6)]
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model_seed, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq, None, None,
baselines)
plt.title('IDMR Performance Curve')
plt.savefig('../figures/object_classification/idmr.pdf',
bbox_inches='tight')
def compute_quality(criterion, search_model, search_seed, eval_model,
eval_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq, use_gpus, workers_per_gpu, log_dir):
if criterion == 'random':
curve = load_baseline('random', 'test', eval_model, eval_seed, domain,
data_seed, batchsize, max_epoch, patience,
tot_acq)['curve']
return np.mean(curve)
spec1 = f'{criterion} {search_model} {search_seed}'
spec2 = f'{eval_model} {eval_seed}'
spec3 = f'{domain} {data_seed} {batchsize} {max_epoch} {patience} {tot_acq}'
spec = f'{spec1} | {spec2} | {spec3}'
if criterion == 'optimal':
order, _, _ = load_optimal(log_dir, search_model, search_seed, domain,
data_seed, batchsize, max_epoch, patience,
tot_acq)
else:
order = load_baseline(criterion, 'test', search_model, eval_seed,
domain, data_seed, batchsize, max_epoch,
patience, tot_acq)['order']
with shelve.open('statistics/model_transfer.shv') as transfer:
if spec in transfer and transfer[spec][
'order'] == order: # fresh copy of cache
return np.mean(transfer[spec]['curve'])
# not present or stale copy of cache
train_args = Namespace(model=eval_model,
model_seed=eval_seed,
domain=domain,
data_seed=data_seed,
evaluation_set='test',
batchsize=batchsize,
max_epoch=max_epoch,
patience=patience,
tot_acq=tot_acq,
use_gpus=use_gpus,
workers_per_gpu=workers_per_gpu)
scheduler = TrainScheduler(train_args)
curve = scheduler.evaluate_order(order)
transfer[spec] = {'curve': curve, 'order': order}
return np.mean(curve)
def main(criterion, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq, log_dir):
data = pickle.load(open('data/restaurant.pkl', 'rb'))['seeds'][data_seed]
N_warmstart = len(data['warmstart'])
N_pool = len(data['pool'])
warmstart_sents, warmstart_tags = zip(*data['warmstart'])
pool_sents, pool_tags = zip(*data['pool'])
test_sents, test_tags = zip(*data['test'])
display_name = {'min-confidence': 'Min-Confidence', 'normalized-min-confidence': 'Norm.-Min-Confidence',
'longest': 'Longest'}[criterion]
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)
heuristic_order = load_baseline(criterion, 'test', model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)['order']
random.seed(0)
random_order = random.sample(range(N_pool), N_pool)[:tot_acq]
lens_ct = Counter([len(t) for t in test_sents])
max_len = max(lens_ct.keys())
lens_props = [lens_ct[i] / sum(lens_ct.values()) for i in range(max_len + 1)]
len_groups = group_proportions(lens_props, 0.13)
num_groups = len(len_groups)
group_sizes = np.array([sum([lens_ct[l] for l in len_group]) for len_group in len_groups])
len_ref = np.cumsum(group_sizes / sum(group_sizes))
len_ref = list(len_ref.flat)
len_ref.insert(0, 0)
tag_groups = [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16]]
tag_groups_dict = {v: i for i, vs in enumerate(tag_groups) for v in vs}
_, test_tags = zip(*data['test'])
test_tags = [tag_groups_dict[t] for t in sum(map(list, test_tags), [])]
test_tag_counts = Counter(test_tags)
test_tag_counts = np.array([test_tag_counts[i] for i in range(len(tag_groups))])
tag_ref = np.cumsum(test_tag_counts / sum(test_tag_counts))
tag_ref = list(tag_ref.flat)
tag_ref.insert(0, 0)
fig = plt.figure(figsize=[9, 6])
gs = GridSpec(ncols=5, nrows=2, width_ratios=[10, 0.7, 10, 0.7, 10], wspace=0.05, hspace=0.05)
xs = np.linspace(N_warmstart, N_warmstart + tot_acq, 6)
plt.subplot(gs[0, 0])
plot_len_proportion(optimal_order, warmstart_sents, pool_sents, len_groups, len_ref, tot_acq)
plt.xticks([])
plt.ylabel('Length Distribution')
plt.title('Optimal Order')
plt.subplot(gs[0, 1])
plot_ref_meter(len_ref)
plt.subplot(gs[0, 2])
plot_len_proportion(heuristic_order, warmstart_sents, pool_sents, len_groups, len_ref, tot_acq)
plt.xticks([])
plt.yticks([])
plt.title(f'{display_name} Order')
plt.subplot(gs[0, 3])
plot_ref_meter(len_ref)
plt.subplot(gs[0, 4])
plot_len_proportion(random_order, warmstart_sents, pool_sents, len_groups, len_ref, tot_acq)
plt.xticks([])
plt.yticks([])
plt.title('Random Order')
plt.subplot(gs[1, 0])
plot_tag_proportion(optimal_order, warmstart_tags, pool_tags, tag_ref, tag_groups_dict, tot_acq)
plt.ylabel('Tag Distribution')
plt.xlabel('# Data Points')
plt.subplot(gs[1, 1])
plot_ref_meter(tag_ref)
plt.subplot(gs[1, 2])
plot_tag_proportion(heuristic_order, warmstart_tags, pool_tags, tag_ref, tag_groups_dict, tot_acq)
plt.yticks([])
plt.xlabel('# Data Points')
plt.subplot(gs[1, 3])
plot_ref_meter(tag_ref)
plt.subplot(gs[1, 4])
plot_tag_proportion(random_order, warmstart_tags, pool_tags, tag_ref, tag_groups_dict, tot_acq)
plt.xticks(xs)
plt.yticks([])
plt.xlabel('# Data Points')
plt.savefig(f'../figures/named_entity_recognition/distribution_vis_{criterion}.pdf', bbox_inches='tight')
def main(model='lstm',
model_seed=0,
domain='alarm',
data_seed=0,
batchsize=20,
max_epoch=100,
patience=20,
tot_acq=160,
log_dir='logs'):
data = pickle.load(open('data/TOP.pkl', 'rb'))[domain]
num_labels = int(len(data['intent_label_mapping']) / 2)
data = data['seeds'][data_seed]
N = len(data['pool'])
N_warmstart = len(data['warmstart'])
optimal_order, _, _ = load_optimal(log_dir, model, model_seed, domain,
data_seed, batchsize, max_epoch,
patience, tot_acq)
criterions = [('max-entropy', 'Max-Entropy'), ('bald', 'BALD')]
heuristic_orders = [
load_baseline(c, 'test', model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)['order']
for c, _ in criterions
]
random.seed(0)
random_order = random.sample(range(N), N)[:tot_acq]
warmstart_sents, warmstart_labels = zip(*data['warmstart'])
warmstart_sents = [nltk.word_tokenize(sent) for sent in warmstart_sents]
pool_sents, pool_labels = zip(*data['pool'])
pool_sents = [nltk.word_tokenize(sent) for sent in pool_sents]
test_sents, test_labels = zip(*data['test'])
test_sents = [nltk.word_tokenize(sent) for sent in test_sents]
lens_ct = Counter([len(t) for t in test_sents])
max_len = max(lens_ct.keys())
lens_props = [
lens_ct[i] / sum(lens_ct.values()) for i in range(max_len + 1)
]
len_groups = group_adjacent(lens_props, 0.08)
print('Length grouping:', len_groups)
group_sizes = np.array(
[sum([lens_ct[l] for l in len_group]) for len_group in len_groups])
len_ref_cdf = np.cumsum(group_sizes / sum(group_sizes))
len_ref_cdf = list(len_ref_cdf.flat)
len_ref_cdf.insert(0, 0)
test_label_counts = Counter(test_labels)
test_label_counts = np.array(
[test_label_counts[i] for i in range(num_labels)])
label_ref_cdf = np.cumsum(test_label_counts / sum(test_label_counts))
label_ref_cdf = list(label_ref_cdf.flat)
label_ref_cdf.insert(0, 0)
fig = plt.figure(figsize=[12, 5.5])
gs = GridSpec(ncols=7,
nrows=2,
width_ratios=[10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05,
hspace=0.05)
xs = range(N_warmstart, tot_acq + N_warmstart + 1)
fig.add_subplot(gs[0, 0])
plot_length_distribution(optimal_order, warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title('Optimal')
plt.xticks([])
plt.ylabel('Sentence Length Distribution')
fig.add_subplot(gs[0, 1])
plot_ref_meter(len_ref_cdf)
fig.add_subplot(gs[0, 2])
plot_length_distribution(heuristic_orders[0], warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title(f'{criterions[0][1]}')
plt.xticks([])
plt.yticks([])
fig.add_subplot(gs[0, 3])
plot_ref_meter(len_ref_cdf)
fig.add_subplot(gs[0, 4])
plot_length_distribution(heuristic_orders[1], warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title(f'{criterions[1][1]}')
plt.xticks([])
plt.yticks([])
fig.add_subplot(gs[0, 5])
plot_ref_meter(len_ref_cdf)
fig.add_subplot(gs[0, 6])
plot_length_distribution(random_order, warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title('Random')
plt.xticks([])
plt.yticks([])
fig.add_subplot(gs[1, 0])
plot_label_distribution(optimal_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.ylabel('Label Distribution')
fig.add_subplot(gs[1, 1])
plot_ref_meter(label_ref_cdf)
fig.add_subplot(gs[1, 2])
plot_label_distribution(heuristic_orders[0], warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.yticks([])
fig.add_subplot(gs[1, 3])
plot_ref_meter(label_ref_cdf)
fig.add_subplot(gs[1, 4])
plot_label_distribution(heuristic_orders[1], warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.yticks([])
fig.add_subplot(gs[1, 5])
plot_ref_meter(label_ref_cdf)
fig.add_subplot(gs[1, 6])
plot_label_distribution(random_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.yticks([])
plt.savefig('../figures/intent_classification/distribution_vis.pdf',
bbox_inches='tight')
def main(model_seed=0,
data_seed=0,
batchsize=25,
max_epoch=100,
patience=20,
tot_acq=300,
evaluation_set='test',
gpu_idx=0,
log_dir='logs'):
data = load_data(data_seed)
num_labels = 10
N_warmstart = len(data['warmstart'])
N_pool = len(data['pool'])
_, warmstart_y = zip(*data['warmstart'])
_, pool_y = zip(*data['pool'])
_, eval_y = zip(*data[evaluation_set])
try:
l1 = load_baseline(f'odmr-l1-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l1_curve, l1_order = l1['curve'], l1['order']
print(l1_curve, np.mean(l1_curve))
l2 = load_baseline(f'odmr-l2-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l2_curve, l2_order = l2['curve'], l2['order']
print(l2_curve, np.mean(l2_curve))
l3 = load_baseline(f'odmr-l3-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l3_curve, l3_order = l3['curve'], l3['order']
print(l3_curve, np.mean(l3_curve))
l4 = load_baseline(f'odmr-l4-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l4_curve, l4_order = l4['curve'], l4['order']
print(l4_curve, np.mean(l4_curve))
except KeyError:
l1_curve, l1_order = odmr(data,
evaluation_set,
'test',
True,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l1_curve, np.mean(l1_curve))
store_baseline(l1_curve, l1_order, f'odmr-l1-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
l2_curve, l2_order = odmr(data,
evaluation_set,
'accessible',
True,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l2_curve, np.mean(l2_curve))
store_baseline(l2_curve, l2_order, f'odmr-l2-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
l3_curve, l3_order = odmr(data,
evaluation_set,
'test',
False,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l3_curve, np.mean(l3_curve))
store_baseline(l3_curve, l3_order, f'odmr-l3-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
l4_curve, l4_order = odmr(data,
evaluation_set,
'accessible',
False,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l4_curve, np.mean(l4_curve))
store_baseline(l4_curve, l4_order, f'odmr-l4-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
plt.figure(figsize=[20, 4])
gs = GridSpec(ncols=9,
nrows=1,
width_ratios=[10, 0.3, 10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05)
plt.subplot(gs[0, 0])
baselines = [('max-entropy', 'Max-Entropy', 0),
('odmr-l1-max-entropy', 'Test + True', 1),
('odmr-l2-max-entropy', 'Acce + True', 2),
('odmr-l3-max-entropy', 'Test + Pred', 6),
('odmr-l4-max-entropy', 'Acce + Pred', 8),
('random', 'Random', 4)]
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
optimal_order, optimal_quality, _ = load_optimal(log_dir, model_seed,
data_seed, batchsize,
max_epoch, patience,
tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model_seed, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq, None, None,
baselines)
plt.xlabel('# Data Points')
plt.ylabel('Accuracy')
plt.title('Object Classification')
plt.subplot(gs[0, 2])
plot_label_proportion(l1_order, warmstart_y, pool_y, eval_y)
plt.title('Test + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 3])
plot_ref_meter(eval_y)
plt.subplot(gs[0, 4])
plot_label_proportion(l2_order, warmstart_y, pool_y, eval_y)
plt.title('Accessible + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 5])
plot_ref_meter(eval_y)
plt.subplot(gs[0, 6])
plot_label_proportion(l3_order, warmstart_y, pool_y, eval_y)
plt.title('Test + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 7])
plot_ref_meter(eval_y)
plt.subplot(gs[0, 8])
plot_label_proportion(l4_order, warmstart_y, pool_y, eval_y)
plt.title('Accessible + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.savefig('../figures/object_classification/odmr.pdf',
bbox_inches='tight')
def main(search_model_seed,
eval_model_seed=None,
data_seed=0,
tot_acq=300,
batchsize=25,
max_epoch=100,
patience=20,
use_gpus='all',
workers_per_gpu=2,
log_dir='logs'):
if eval_model_seed is None:
eval_model_seed = search_model_seed
N_warmstart = 50
optimal_order, optimal_quality, _ = load_optimal(log_dir,
search_model_seed,
data_seed, batchsize,
max_epoch, patience,
tot_acq)
print(f'optimal quality in log: {optimal_quality}')
plt.figure(figsize=[7.5, 4])
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
baselines = [('max-entropy', 'Max-Entropy', 0), ('bald', 'BALD', 1),
('batchbald', 'BatchBALD', 2), ('random', 'Random', 4)]
plot_args = [
search_model_seed, eval_model_seed, data_seed, batchsize, max_epoch,
patience, tot_acq, use_gpus, workers_per_gpu, baselines
]
plt.subplot(1, 2, 1)
valid_curve = plot_curves(optimal_order, xs, 'valid', *plot_args)
xmin1, xmax1, ymin1, ymax1 = plt.axis()
plt.xticks(np.linspace(N_warmstart, tot_acq + N_warmstart, 5))
plt.xlabel('# Data Points')
plt.ylabel('Accuracy')
plt.title('Validation Set $\\mathcal{D}^V$')
ax1 = plt.gca()
plt.subplot(1, 2, 2)
test_curve = plot_curves(optimal_order, xs, 'test', *plot_args)
xmin2, xmax2, ymin2, ymax2 = plt.axis()
plt.yticks([])
plt.xticks(np.linspace(N_warmstart, tot_acq + N_warmstart, 5))
plt.xlabel('# Data Points')
ax2 = plt.gca()
plt.title('Test Set $\\mathcal{D}^T$')
ax1.set_xlim(min(xmin1, xmin2), max(xmax1, xmax2))
ax1.set_ylim(min(ymin1, ymin2), max(ymax1, ymax2))
ax2.set_xlim(min(xmin1, xmin2), max(xmax1, xmax2))
ax2.set_ylim(min(ymin1, ymin2), max(ymax1, ymax2))
plt.tight_layout()
fn = f'../figures/object_classification/perf_curves/s{search_model_seed}_e{eval_model_seed}.pdf'
plt.savefig(abs_path(fn), bbox_inches='tight')
print(
f'Validation quality: {np.mean(valid_curve)}; Test quality: {np.mean(test_curve)}'
)
spec = f'{search_model_seed} {eval_model_seed} {data_seed} {batchsize} {max_epoch} {patience} {tot_acq}'
with shelve.open(abs_path('statistics/perf_curves.shv')) as curves:
curves[spec] = {
'valid': valid_curve,
'test': test_curve,
'order': optimal_order
}
def main(criterion,
model='lstm',
model_seed=0,
domain='alarm',
data_seed=0,
batchsize=20,
max_epoch=100,
patience=20,
tot_acq=160,
evaluation_set='test',
gpu_idx=0,
log_dir='logs'):
data = pickle.load(open('data/TOP.pkl', 'rb'))[domain]
num_labels = int(len(data['intent_label_mapping']) / 2)
data = data['seeds'][data_seed]
_, warmstart_labels = zip(*data['warmstart'])
_, pool_labels = zip(*data['pool'])
N_warmstart = len(data['warmstart'])
N = len(data['pool'])
try:
l1 = load_baseline(f'odmr-l1-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l1_curve, l1_order = l1['curve'], l1['order']
print(l1_curve, np.mean(l1_curve))
l2 = load_baseline(f'odmr-l2-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l2_curve, l2_order = l2['curve'], l2['order']
print(l2_curve, np.mean(l2_curve))
l3 = load_baseline(f'odmr-l3-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l3_curve, l3_order = l3['curve'], l3['order']
print(l3_curve, np.mean(l3_curve))
l4 = load_baseline(f'odmr-l4-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l4_curve, l4_order = l4['curve'], l4['order']
print(l4_curve, np.mean(l4_curve))
except KeyError:
l1_curve, l1_order = odmr(data,
criterion,
evaluation_set,
'test',
True,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l1_curve, np.mean(l1_curve))
store_baseline(l1_curve, l1_order, f'odmr-l1-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
l2_curve, l2_order = odmr(data,
criterion,
evaluation_set,
'accessible',
True,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l2_curve, np.mean(l2_curve))
store_baseline(l2_curve, l2_order, f'odmr-l2-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
l3_curve, l3_order = odmr(data,
criterion,
evaluation_set,
'test',
False,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l3_curve, np.mean(l3_curve))
store_baseline(l3_curve, l3_order, f'odmr-l3-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
l4_curve, l4_order = odmr(data,
criterion,
evaluation_set,
'accessible',
False,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l4_curve, np.mean(l4_curve))
store_baseline(l4_curve, l4_order, f'odmr-l4-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
plt.figure(figsize=[20, 4])
gs = GridSpec(ncols=9,
nrows=1,
width_ratios=[10, 0.3, 10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05)
plt.subplot(gs[0, 0])
if criterion == 'max-entropy':
baselines = [('max-entropy', 'Max-Entropy', 0),
('odmr-l1-max-entropy', 'Test + True', 1),
('odmr-l2-max-entropy', 'Acce + True', 2),
('odmr-l3-max-entropy', 'Test + Pred', 6),
('odmr-l4-max-entropy', 'Acce + Pred', 8),
('random', 'Random', 4)]
elif criterion == 'bald':
baselines = [('bald', 'BALD', 0), ('odmr-l1-bald', 'Test + True', 1),
('odmr-l2-bald', 'Acce + True', 2),
('odmr-l3-bald', 'Test + Pred', 6),
('odmr-l4-bald', 'Acce + Pred', 8),
('random', 'Random', 4)]
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
optimal_order, optimal_quality, _ = load_optimal(log_dir, model,
model_seed, domain,
data_seed, batchsize,
max_epoch, patience,
tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model, model_seed,
model_seed, domain, data_seed, batchsize, max_epoch, patience,
tot_acq, None, None, baselines)
plt.xlabel('# Data Points')
plt.ylabel('F1')
plt.title('Intent Classification')
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1))
_, test_labels = zip(*data['test'])
test_label_counts = Counter(test_labels)
test_label_counts = np.array(
[test_label_counts[i] for i in range(num_labels)])
label_ref_cdf = np.cumsum(test_label_counts / sum(test_label_counts))
label_ref_cdf = list(label_ref_cdf.flat)
label_ref_cdf.insert(0, 0)
plt.subplot(gs[0, 2])
plot_label_distribution(l1_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Test + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 3])
plot_ref_meter(label_ref_cdf)
plt.subplot(gs[0, 4])
plot_label_distribution(l2_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Accessible + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 5])
plot_ref_meter(label_ref_cdf)
plt.subplot(gs[0, 6])
plot_label_distribution(l3_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Test + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 7])
plot_ref_meter(label_ref_cdf)
plt.subplot(gs[0, 8])
plot_label_distribution(l4_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Accessible + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.savefig(f'../figures/intent_classification/odmr_{criterion}.pdf',
bbox_inches='tight')
def main(model_seed, data_seed, batchsize, max_epoch, patience, tot_acq,
log_dir):
data = load_data(data_seed)
warmstart = data['warmstart']
pool = data['pool']
test = data['test']
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)
bald_order = load_baseline('bald', 'test', model_seed, data_seed,
batchsize, max_epoch, patience,
tot_acq)['order']
batchbald_order = load_baseline('batchbald', 'test', model_seed, data_seed,
batchsize, max_epoch, patience,
tot_acq)['order']
random.seed(0)
random_order = random.sample(range(len(pool)), len(pool))[:tot_acq]
warmstart_X, warmstart_y = zip(*warmstart)
warmstart_X = np.array(warmstart_X)
pool_X, pool_y = zip(*pool)
pool_X = np.array(pool_X)
test_X, test_y = zip(*test)
test_X = np.array(test_X)
all_X = np.vstack((warmstart_X, pool_X, test_X[:2000]))
all_X = all_X.reshape(len(all_X), -1)
pca = PCA(n_components=100, random_state=np.random.RandomState(0))
tsne = TSNE(n_components=2,
n_jobs=-1,
random_state=np.random.RandomState(0))
all_X_2d = tsne.fit_transform(pca.fit_transform(all_X))
warmstart_X_2d, pool_X_2d, test_X_2d = np.array_split(
all_X_2d,
[len(warmstart_X), len(warmstart_X) + len(pool_X)])
plt.figure(figsize=[10, 7])
gs = GridSpec(ncols=7,
nrows=2,
width_ratios=[10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05,
height_ratios=[3, 2],
hspace=0.15)
plt.subplot(gs[0, :3])
plot_tsne(bald_order, warmstart_X_2d, warmstart_y, pool_X_2d, pool_y,
test_X_2d, test_y[:2000], 5, batchsize)
plt.title('BALD Order')
plt.subplot(gs[0, 4:])
plot_tsne(batchbald_order, warmstart_X_2d, warmstart_y, pool_X_2d, pool_y,
test_X_2d, test_y[:2000], 5, batchsize)
plt.title('BatchBALD Order')
plt.subplot(gs[1, 0])
plot_label_proportion(optimal_order, warmstart_y, pool_y, test_y)
plt.title('Optimal Order')
plt.ylabel('Label Distribution')
plt.subplot(gs[1, 1])
plot_ref_meter(test_y)
plt.subplot(gs[1, 2])
plot_label_proportion(bald_order, warmstart_y, pool_y, test_y)
plt.title(f'BALD Order')
plt.yticks([])
plt.subplot(gs[1, 3])
plot_ref_meter(test_y)
plt.subplot(gs[1, 4])
plot_label_proportion(batchbald_order, warmstart_y, pool_y, test_y)
plt.title(f'BatchBALD Order')
plt.yticks([])
plt.subplot(gs[1, 5])
plot_ref_meter(test_y)
plt.subplot(gs[1, 6])
plot_label_proportion(random_order, warmstart_y, pool_y, test_y)
plt.title('Random Order')
plt.yticks([])
plt.savefig(
f'../figures/object_classification/distribution_vis_bald_batchbald.pdf',
bbox_inches='tight')