def main(model_seeds=[0, 1, 2, 3, 4],
data_seed=0,
batchsize=25,
max_epoch=100,
patience=20,
tot_acq=300,
criterions=['max-entropy', 'bald', 'batchbald']):
optimal = []
with shelve.open('statistics/perf_curves.shv') as curves:
for seed in model_seeds:
spec = f'{seed} {seed} {data_seed} {batchsize} {max_epoch} {patience} {tot_acq}'
curve = curves[spec]['test']
optimal.append(np.mean(curve))
print(f'optimal quality: {np.mean(optimal)}')
for criterion in criterions:
heuristic = []
for seed in model_seeds:
curve = load_baseline(criterion, 'test', seed, data_seed,
batchsize, max_epoch, patience,
tot_acq)['curve']
heuristic.append(np.mean(curve))
print(f'{criterion} quality: {np.mean(heuristic)}')
random = []
for seed in model_seeds:
curve = load_baseline('random', 'test', seed, data_seed, batchsize,
max_epoch, patience, tot_acq)['curve']
random.append(np.mean(curve))
print(f'Random AUC: {np.mean(random)}')
def main(model_seed=0, data_seed=0, batchsize=25, max_epoch=100, patience=20,
tot_acq=250, criterions=['min-confidence', 'normalized-min-confidence', 'longest']):
with shelve.open('statistics/perf_curves.shv') as curves:
spec = f'{model_seed} {model_seed} {data_seed} {batchsize} {max_epoch} {patience} {tot_acq}'
curve = curves[spec]['test']
print(f'optimal quality: {np.mean(curve)}')
for criterion in criterions:
curve = load_baseline(criterion, 'test', model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)['curve']
print(f'{criterion} quality: {np.mean(curve)}')
curve = load_baseline('random', 'test', model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)['curve']
print(f'Random AUC: {np.mean(curve)}')
def plot_curves(order, xs, evaluation_set, search_model_seed, eval_model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq, use_gpus,
workers_per_gpu, baselines):
legends = []
spec = f'{search_model_seed} {eval_model_seed} {data_seed} {batchsize} {max_epoch} {patience} {tot_acq}'
with shelve.open(abs_path('statistics/perf_curves.shv')) as curves:
if spec in curves and curves[spec]['order'] == order:
optimal_curve = curves[spec][evaluation_set]
else:
eval_args = Namespace(model_seed=eval_model_seed,
data_seed=data_seed,
batchsize=batchsize,
max_epoch=max_epoch,
patience=patience,
evaluation_set=evaluation_set,
tot_acq=tot_acq,
use_gpus=use_gpus,
workers_per_gpu=workers_per_gpu)
scheduler = TrainScheduler(eval_args)
optimal_curve = scheduler.evaluate_order(order)
plt.plot(xs, optimal_curve, 'C3-o')
legends.append(f'Optimal: {np.mean(optimal_curve):0.3f}')
for name, display_name, color in baselines:
try:
curve = load_baseline(name, evaluation_set, eval_model_seed,
data_seed, batchsize, max_epoch, patience,
tot_acq)['curve']
plt.plot(xs, curve, f'C{color}-o')
legends.append(f'{display_name}: {np.mean(curve):0.3f}')
except:
print(f'{display_name} not found')
plt.legend(legends)
return optimal_curve
def get_baseline_model(args):
vocab = utils.load_vocab(args.vocab_json)
if args.baseline_start_from is not None:
model, kwargs = utils.load_baseline(args.baseline_start_from)
elif args.model_type == 'LSTM':
kwargs = {
'vocab': vocab,
'rnn_wordvec_dim': args.rnn_wordvec_dim,
'rnn_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout,
'fc_dims': parse_int_list(args.classifier_fc_dims),
'fc_use_batchnorm': args.classifier_batchnorm == 1,
'fc_dropout': args.classifier_dropout,
}
model = LstmModel(**kwargs)
elif args.model_type == 'CNN+LSTM':
kwargs = {
'vocab': vocab,
'rnn_wordvec_dim': args.rnn_wordvec_dim,
'rnn_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout,
'cnn_feat_dim': parse_int_list(args.feature_dim),
'cnn_num_res_blocks': args.cnn_num_res_blocks,
'cnn_res_block_dim': args.cnn_res_block_dim,
'cnn_proj_dim': args.cnn_proj_dim,
'cnn_pooling': args.cnn_pooling,
'fc_dims': parse_int_list(args.classifier_fc_dims),
'fc_use_batchnorm': args.classifier_batchnorm == 1,
'fc_dropout': args.classifier_dropout,
}
model = CnnLstmModel(**kwargs)
elif args.model_type == 'CNN+LSTM+SA':
kwargs = {
'vocab': vocab,
'rnn_wordvec_dim': args.rnn_wordvec_dim,
'rnn_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout,
'cnn_feat_dim': parse_int_list(args.feature_dim),
'stacked_attn_dim': args.stacked_attn_dim,
'num_stacked_attn': args.num_stacked_attn,
'fc_dims': parse_int_list(args.classifier_fc_dims),
'fc_use_batchnorm': args.classifier_batchnorm == 1,
'fc_dropout': args.classifier_dropout,
}
model = CnnLstmSaModel(**kwargs)
if model.rnn.token_to_idx != vocab['question_token_to_idx']:
# Make sure new vocab is superset of old
for k, v in model.rnn.token_to_idx.items():
assert k in vocab['question_token_to_idx']
assert vocab['question_token_to_idx'][k] == v
for token, idx in vocab['question_token_to_idx'].items():
model.rnn.token_to_idx[token] = idx
kwargs['vocab'] = vocab
model.rnn.expand_vocab(vocab['question_token_to_idx'])
model.cuda()
model.train()
return model, kwargs
def main(model1='lstm', criterion1='bald', model2='cnn', criterion2='bald', model_seed=0,
domain='alarm', data_seed=0, batchsize=20, max_epoch=100, patience=20, tot_acq=160, log_dir='logs'):
m_dict = {'lstm': 'LSTM', 'cnn': 'CNN', 'aoe': 'AOE', 'roberta': 'RoBERTa'}
c_dict = {'max-entropy': 'Max-Entropy', 'bald': 'BALD'}
optimal_order1, _, _ = load_optimal(log_dir, model1, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)
optimal_order2, _, _ = load_optimal(log_dir, model2, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)
plot_relative_order(optimal_order1, optimal_order2,
f'{m_dict[model1]}\nOptimal', f'{m_dict[model2]}\nOptimal')
plt.savefig(f'../figures/intent_classification/relative_orders/{model1}_{model2}_optimal.pdf',
bbox_inches='tight')
heuristic_order1 = load_baseline(criterion1, 'test', model1, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)['order']
heuristic_order2 = load_baseline(criterion2, 'test', model2, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)['order']
plot_relative_order(heuristic_order1, heuristic_order2,
f'{m_dict[model1]}\n{c_dict[criterion1]}', f'{m_dict[model2]}\n{c_dict[criterion2]}')
plt.savefig(f'../figures/intent_classification/relative_orders/{model1}_{model2}_heuristic.pdf',
bbox_inches='tight')
def compute_quality(criterion, search_model, search_seed, eval_model,
eval_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq, use_gpus, workers_per_gpu, log_dir):
if criterion == 'random':
curve = load_baseline('random', 'test', eval_model, eval_seed, domain,
data_seed, batchsize, max_epoch, patience,
tot_acq)['curve']
return np.mean(curve)
spec1 = f'{criterion} {search_model} {search_seed}'
spec2 = f'{eval_model} {eval_seed}'
spec3 = f'{domain} {data_seed} {batchsize} {max_epoch} {patience} {tot_acq}'
spec = f'{spec1} | {spec2} | {spec3}'
if criterion == 'optimal':
order, _, _ = load_optimal(log_dir, search_model, search_seed, domain,
data_seed, batchsize, max_epoch, patience,
tot_acq)
else:
order = load_baseline(criterion, 'test', search_model, eval_seed,
domain, data_seed, batchsize, max_epoch,
patience, tot_acq)['order']
with shelve.open('statistics/model_transfer.shv') as transfer:
if spec in transfer and transfer[spec][
'order'] == order: # fresh copy of cache
return np.mean(transfer[spec]['curve'])
# not present or stale copy of cache
train_args = Namespace(model=eval_model,
model_seed=eval_seed,
domain=domain,
data_seed=data_seed,
evaluation_set='test',
batchsize=batchsize,
max_epoch=max_epoch,
patience=patience,
tot_acq=tot_acq,
use_gpus=use_gpus,
workers_per_gpu=workers_per_gpu)
scheduler = TrainScheduler(train_args)
curve = scheduler.evaluate_order(order)
transfer[spec] = {'curve': curve, 'order': order}
return np.mean(curve)
def main(criterion, model='lstm', model_seed=0, domain='alarm', data_seed=0,
batchsize=20, max_epoch=100, patience=20, tot_acq=160, evaluation_set='test', gpu_idx=0, log_dir='logs'):
assert model != 'roberta', 'IDMR for RoBERTa model is not implemented'
data = pickle.load(open('data/TOP.pkl', 'rb'))[domain]['seeds'][data_seed]
N_warmstart = len(data['warmstart'])
N = len(data['pool'])
try:
idmr_curve = load_baseline(f'idmr-{criterion}', evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)['curve']
except KeyError:
lens_proportions = None
accessible_set = data['warmstart'] + data['train_valid'] + data['pool']
accessible_sents, _ = zip(*accessible_set)
lens = [len(nltk.word_tokenize(s)) for s in accessible_sents]
lens_ct = Counter(lens)
cts = np.array([lens_ct[l] for l in range(max(lens_ct.keys()) + 1)])
props = cts / sum(cts)
groups = group_proportions(props, 0.08)
lens_proportions = dict()
for i, g in enumerate(groups):
lo, hi = min(g), max(g)
sum_props = sum(props[p] for p in range(lo, hi + 1))
if i == len(groups) - 1:
hi = 100
lens_proportions[(lo, hi)] = sum_props
idmr_curve, idmr_order = idmr(data, criterion, evaluation_set, lens_proportions, model, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq, gpu_idx)
print(idmr_curve)
print(np.mean(idmr_curve))
store_baseline(idmr_curve, idmr_order, f'idmr-{criterion}', evaluation_set, model, model_seed,
domain, data_seed, batchsize, max_epoch, patience, tot_acq)
plt.figure()
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
baselines = [('max-entropy', 'Max-Entropy', 0), ('bald', 'BALD', 1), ('random', 'Random', 4)]
if criterion == 'max-entropy':
baselines.append(('idmr-max-entropy', 'IDMR Max-Ent.', 6))
elif criterion == 'bald':
baselines.append(('idmr-bald', 'IDMR BALD', 6))
optimal_order, _, _ = load_optimal(log_dir, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model, model_seed, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq, None, None, baselines)
xmin1, xmax1, ymin1, ymax1 = plt.axis()
plt.xticks(np.linspace(N_warmstart, tot_acq + N_warmstart, 5))
plt.xlabel('# Data Points')
plt.ylabel('F1')
plt.title('Input Distribution-Matching Regularization')
plt.savefig(f'../figures/intent_classification/idmr_{criterion}.pdf', bbox_inches='tight')
def main(criterion, evaluation_set, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq, log_dir, gpu_idx):
data = pickle.load(open('data/restaurant.pkl', 'rb'))['seeds'][data_seed]
N_warmstart = len(data['warmstart'])
try:
idmr_curve = load_baseline(f'idmr-{criterion}', evaluation_set, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)['curve']
except KeyError:
N_pool = len(data['pool'])
accessible_set = data['warmstart'] + data['train_valid'] + data['pool']
accessible_sents, _ = zip(*accessible_set)
lens = [len(s) for s in accessible_sents]
lens_ct = Counter(lens)
max_len = max(lens_ct.keys())
cts = np.array([lens_ct[l] for l in range(max_len + 1)])
props = cts / sum(cts)
groups = group_proportions(props, 0.13)
lens_proportions = dict()
for i, g in enumerate(groups):
lo, hi = min(g), max(g)
sum_props = sum(props[p] for p in range(lo, hi + 1))
if i == len(groups) - 1:
hi = 100
lens_proportions[(lo, hi)] = sum_props
idmr_curve, idmr_order = idmr(data, evaluation_set, criterion, lens_proportions, model_seed,
batchsize, max_epoch, patience, tot_acq, gpu_idx)
store_baseline(idmr_curve, idmr_order, f'idmr-{criterion}', evaluation_set, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)
print(idmr_curve)
print(np.mean(idmr_curve))
plt.figure()
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
display_name = {'min-confidence': 'Min-Confidence', 'normalized-min-confidence': 'Norm.-Min-Conf.',
'longest': 'Longest'}[criterion]
baselines = [(criterion, display_name, 0), (f'idmr-{criterion}', f'IDMR-{display_name.replace("idence", ".")}', 6),
('random', 'Random', 4)]
plot_curves(optimal_order, xs, evaluation_set, model_seed, model_seed, data_seed, batchsize, max_epoch, patience,
tot_acq, None, None, baselines)
plt.tight_layout()
plt.savefig(f'../figures/named_entity_recognition/idmr_{criterion}.pdf', bbox_inches='tight')
def main(model_seed=0,
data_seed=0,
batchsize=25,
max_epoch=100,
patience=20,
tot_acq=300,
evaluation_set='test',
log_dir='logs',
tsne_dim=3,
num_clusters=5,
gpu_idx=0):
data = load_data(data_seed)
N_warmstart = len(data['warmstart'])
try:
idmr_curve = load_baseline('idmr-max-entropy', evaluation_set,
model_seed, data_seed, batchsize, max_epoch,
patience, tot_acq)['curve']
except KeyError:
data = load_data(data_seed)
idmr_curve, idmr_order = idmr(data, evaluation_set, model_seed,
batchsize, max_epoch, patience, tot_acq,
tsne_dim, num_clusters, gpu_idx)
store_baseline(idmr_curve, idmr_order, 'idmr-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
print(idmr_curve)
print(np.mean(idmr_curve))
plt.figure()
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
baselines = [('max-entropy', 'Max-Entropy', 0), ('bald', 'BALD', 1),
('random', 'Random', 4),
('idmr-max-entropy', 'IDMR Max-Ent.', 6)]
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model_seed, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq, None, None,
baselines)
plt.title('IDMR Performance Curve')
plt.savefig('../figures/object_classification/idmr.pdf',
bbox_inches='tight')
def main(criterion, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq, log_dir):
data = pickle.load(open('data/restaurant.pkl', 'rb'))['seeds'][data_seed]
N_warmstart = len(data['warmstart'])
N_pool = len(data['pool'])
warmstart_sents, warmstart_tags = zip(*data['warmstart'])
pool_sents, pool_tags = zip(*data['pool'])
test_sents, test_tags = zip(*data['test'])
display_name = {'min-confidence': 'Min-Confidence', 'normalized-min-confidence': 'Norm.-Min-Confidence',
'longest': 'Longest'}[criterion]
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)
heuristic_order = load_baseline(criterion, 'test', model_seed, data_seed, batchsize, max_epoch, patience, tot_acq)['order']
random.seed(0)
random_order = random.sample(range(N_pool), N_pool)[:tot_acq]
lens_ct = Counter([len(t) for t in test_sents])
max_len = max(lens_ct.keys())
lens_props = [lens_ct[i] / sum(lens_ct.values()) for i in range(max_len + 1)]
len_groups = group_proportions(lens_props, 0.13)
num_groups = len(len_groups)
group_sizes = np.array([sum([lens_ct[l] for l in len_group]) for len_group in len_groups])
len_ref = np.cumsum(group_sizes / sum(group_sizes))
len_ref = list(len_ref.flat)
len_ref.insert(0, 0)
tag_groups = [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16]]
tag_groups_dict = {v: i for i, vs in enumerate(tag_groups) for v in vs}
_, test_tags = zip(*data['test'])
test_tags = [tag_groups_dict[t] for t in sum(map(list, test_tags), [])]
test_tag_counts = Counter(test_tags)
test_tag_counts = np.array([test_tag_counts[i] for i in range(len(tag_groups))])
tag_ref = np.cumsum(test_tag_counts / sum(test_tag_counts))
tag_ref = list(tag_ref.flat)
tag_ref.insert(0, 0)
fig = plt.figure(figsize=[9, 6])
gs = GridSpec(ncols=5, nrows=2, width_ratios=[10, 0.7, 10, 0.7, 10], wspace=0.05, hspace=0.05)
xs = np.linspace(N_warmstart, N_warmstart + tot_acq, 6)
plt.subplot(gs[0, 0])
plot_len_proportion(optimal_order, warmstart_sents, pool_sents, len_groups, len_ref, tot_acq)
plt.xticks([])
plt.ylabel('Length Distribution')
plt.title('Optimal Order')
plt.subplot(gs[0, 1])
plot_ref_meter(len_ref)
plt.subplot(gs[0, 2])
plot_len_proportion(heuristic_order, warmstart_sents, pool_sents, len_groups, len_ref, tot_acq)
plt.xticks([])
plt.yticks([])
plt.title(f'{display_name} Order')
plt.subplot(gs[0, 3])
plot_ref_meter(len_ref)
plt.subplot(gs[0, 4])
plot_len_proportion(random_order, warmstart_sents, pool_sents, len_groups, len_ref, tot_acq)
plt.xticks([])
plt.yticks([])
plt.title('Random Order')
plt.subplot(gs[1, 0])
plot_tag_proportion(optimal_order, warmstart_tags, pool_tags, tag_ref, tag_groups_dict, tot_acq)
plt.ylabel('Tag Distribution')
plt.xlabel('# Data Points')
plt.subplot(gs[1, 1])
plot_ref_meter(tag_ref)
plt.subplot(gs[1, 2])
plot_tag_proportion(heuristic_order, warmstart_tags, pool_tags, tag_ref, tag_groups_dict, tot_acq)
plt.yticks([])
plt.xlabel('# Data Points')
plt.subplot(gs[1, 3])
plot_ref_meter(tag_ref)
plt.subplot(gs[1, 4])
plot_tag_proportion(random_order, warmstart_tags, pool_tags, tag_ref, tag_groups_dict, tot_acq)
plt.xticks(xs)
plt.yticks([])
plt.xlabel('# Data Points')
plt.savefig(f'../figures/named_entity_recognition/distribution_vis_{criterion}.pdf', bbox_inches='tight')
def main(model='lstm',
model_seed=0,
domain='alarm',
data_seed=0,
batchsize=20,
max_epoch=100,
patience=20,
tot_acq=160,
log_dir='logs'):
data = pickle.load(open('data/TOP.pkl', 'rb'))[domain]
num_labels = int(len(data['intent_label_mapping']) / 2)
data = data['seeds'][data_seed]
N = len(data['pool'])
N_warmstart = len(data['warmstart'])
optimal_order, _, _ = load_optimal(log_dir, model, model_seed, domain,
data_seed, batchsize, max_epoch,
patience, tot_acq)
criterions = [('max-entropy', 'Max-Entropy'), ('bald', 'BALD')]
heuristic_orders = [
load_baseline(c, 'test', model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)['order']
for c, _ in criterions
]
random.seed(0)
random_order = random.sample(range(N), N)[:tot_acq]
warmstart_sents, warmstart_labels = zip(*data['warmstart'])
warmstart_sents = [nltk.word_tokenize(sent) for sent in warmstart_sents]
pool_sents, pool_labels = zip(*data['pool'])
pool_sents = [nltk.word_tokenize(sent) for sent in pool_sents]
test_sents, test_labels = zip(*data['test'])
test_sents = [nltk.word_tokenize(sent) for sent in test_sents]
lens_ct = Counter([len(t) for t in test_sents])
max_len = max(lens_ct.keys())
lens_props = [
lens_ct[i] / sum(lens_ct.values()) for i in range(max_len + 1)
]
len_groups = group_adjacent(lens_props, 0.08)
print('Length grouping:', len_groups)
group_sizes = np.array(
[sum([lens_ct[l] for l in len_group]) for len_group in len_groups])
len_ref_cdf = np.cumsum(group_sizes / sum(group_sizes))
len_ref_cdf = list(len_ref_cdf.flat)
len_ref_cdf.insert(0, 0)
test_label_counts = Counter(test_labels)
test_label_counts = np.array(
[test_label_counts[i] for i in range(num_labels)])
label_ref_cdf = np.cumsum(test_label_counts / sum(test_label_counts))
label_ref_cdf = list(label_ref_cdf.flat)
label_ref_cdf.insert(0, 0)
fig = plt.figure(figsize=[12, 5.5])
gs = GridSpec(ncols=7,
nrows=2,
width_ratios=[10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05,
hspace=0.05)
xs = range(N_warmstart, tot_acq + N_warmstart + 1)
fig.add_subplot(gs[0, 0])
plot_length_distribution(optimal_order, warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title('Optimal')
plt.xticks([])
plt.ylabel('Sentence Length Distribution')
fig.add_subplot(gs[0, 1])
plot_ref_meter(len_ref_cdf)
fig.add_subplot(gs[0, 2])
plot_length_distribution(heuristic_orders[0], warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title(f'{criterions[0][1]}')
plt.xticks([])
plt.yticks([])
fig.add_subplot(gs[0, 3])
plot_ref_meter(len_ref_cdf)
fig.add_subplot(gs[0, 4])
plot_length_distribution(heuristic_orders[1], warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title(f'{criterions[1][1]}')
plt.xticks([])
plt.yticks([])
fig.add_subplot(gs[0, 5])
plot_ref_meter(len_ref_cdf)
fig.add_subplot(gs[0, 6])
plot_length_distribution(random_order, warmstart_sents, pool_sents,
len_groups, len_ref_cdf, xs)
plt.title('Random')
plt.xticks([])
plt.yticks([])
fig.add_subplot(gs[1, 0])
plot_label_distribution(optimal_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.ylabel('Label Distribution')
fig.add_subplot(gs[1, 1])
plot_ref_meter(label_ref_cdf)
fig.add_subplot(gs[1, 2])
plot_label_distribution(heuristic_orders[0], warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.yticks([])
fig.add_subplot(gs[1, 3])
plot_ref_meter(label_ref_cdf)
fig.add_subplot(gs[1, 4])
plot_label_distribution(heuristic_orders[1], warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.yticks([])
fig.add_subplot(gs[1, 5])
plot_ref_meter(label_ref_cdf)
fig.add_subplot(gs[1, 6])
plot_label_distribution(random_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.xlabel('# Data Points')
plt.xticks(np.linspace(N_warmstart, N_warmstart + tot_acq, 5))
plt.yticks([])
plt.savefig('../figures/intent_classification/distribution_vis.pdf',
bbox_inches='tight')
def main(model_seed=0,
data_seed=0,
batchsize=25,
max_epoch=100,
patience=20,
tot_acq=300,
evaluation_set='test',
gpu_idx=0,
log_dir='logs'):
data = load_data(data_seed)
num_labels = 10
N_warmstart = len(data['warmstart'])
N_pool = len(data['pool'])
_, warmstart_y = zip(*data['warmstart'])
_, pool_y = zip(*data['pool'])
_, eval_y = zip(*data[evaluation_set])
try:
l1 = load_baseline(f'odmr-l1-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l1_curve, l1_order = l1['curve'], l1['order']
print(l1_curve, np.mean(l1_curve))
l2 = load_baseline(f'odmr-l2-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l2_curve, l2_order = l2['curve'], l2['order']
print(l2_curve, np.mean(l2_curve))
l3 = load_baseline(f'odmr-l3-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l3_curve, l3_order = l3['curve'], l3['order']
print(l3_curve, np.mean(l3_curve))
l4 = load_baseline(f'odmr-l4-max-entropy', evaluation_set, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq)
l4_curve, l4_order = l4['curve'], l4['order']
print(l4_curve, np.mean(l4_curve))
except KeyError:
l1_curve, l1_order = odmr(data,
evaluation_set,
'test',
True,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l1_curve, np.mean(l1_curve))
store_baseline(l1_curve, l1_order, f'odmr-l1-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
l2_curve, l2_order = odmr(data,
evaluation_set,
'accessible',
True,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l2_curve, np.mean(l2_curve))
store_baseline(l2_curve, l2_order, f'odmr-l2-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
l3_curve, l3_order = odmr(data,
evaluation_set,
'test',
False,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l3_curve, np.mean(l3_curve))
store_baseline(l3_curve, l3_order, f'odmr-l3-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
l4_curve, l4_order = odmr(data,
evaluation_set,
'accessible',
False,
model_seed,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
smoothing=1)
print(l4_curve, np.mean(l4_curve))
store_baseline(l4_curve, l4_order, f'odmr-l4-max-entropy',
evaluation_set, model_seed, data_seed, batchsize,
max_epoch, patience, tot_acq)
plt.figure(figsize=[20, 4])
gs = GridSpec(ncols=9,
nrows=1,
width_ratios=[10, 0.3, 10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05)
plt.subplot(gs[0, 0])
baselines = [('max-entropy', 'Max-Entropy', 0),
('odmr-l1-max-entropy', 'Test + True', 1),
('odmr-l2-max-entropy', 'Acce + True', 2),
('odmr-l3-max-entropy', 'Test + Pred', 6),
('odmr-l4-max-entropy', 'Acce + Pred', 8),
('random', 'Random', 4)]
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
optimal_order, optimal_quality, _ = load_optimal(log_dir, model_seed,
data_seed, batchsize,
max_epoch, patience,
tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model_seed, model_seed,
data_seed, batchsize, max_epoch, patience, tot_acq, None, None,
baselines)
plt.xlabel('# Data Points')
plt.ylabel('Accuracy')
plt.title('Object Classification')
plt.subplot(gs[0, 2])
plot_label_proportion(l1_order, warmstart_y, pool_y, eval_y)
plt.title('Test + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 3])
plot_ref_meter(eval_y)
plt.subplot(gs[0, 4])
plot_label_proportion(l2_order, warmstart_y, pool_y, eval_y)
plt.title('Accessible + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 5])
plot_ref_meter(eval_y)
plt.subplot(gs[0, 6])
plot_label_proportion(l3_order, warmstart_y, pool_y, eval_y)
plt.title('Test + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 7])
plot_ref_meter(eval_y)
plt.subplot(gs[0, 8])
plot_label_proportion(l4_order, warmstart_y, pool_y, eval_y)
plt.title('Accessible + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.savefig('../figures/object_classification/odmr.pdf',
bbox_inches='tight')
def render(self, mode='human', smooth_kernel_size=5, baseline=None):
"""
Renders current state as a figure.
"""
assert mode == 'human'
sns.set(style='whitegrid')
colors = [
'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple'
]
plt.ion()
plt.figure(0, dpi=40)
plt.clf()
experiments = [
self._info_list_to_plot_metrics(
self.info_list,
label='Auto-learned',
smooth_kernel_size=smooth_kernel_size)
]
if baseline is None:
try:
experiments.append(
self._info_list_to_plot_metrics(
utils.load_baseline(self.dataset + '_' +
self.architecture),
label='Baseline',
smooth_kernel_size=smooth_kernel_size))
except:
if not self.displayed_load_error:
print(
'Error: failed to load baseline experiment data. Run baselines.py to generate.'
)
self.displayed_load_error = True
else:
experiments.append(
self._info_list_to_plot_metrics(
baseline.alrs.info_list,
label='Baseline',
smooth_kernel_size=smooth_kernel_size))
plt.subplot(1, 3, 1)
for i, (timeline, train_losses, val_losses, learning_rates,
smoothed_train_losses, smoothed_val_losses,
smoothed_learning_rates, label) in enumerate(experiments):
if smoothed_train_losses is not None:
plt.plot(timeline,
np.log(train_losses),
color=colors[i],
alpha=0.25)
plt.plot(timeline,
np.log(smoothed_train_losses),
color=colors[i],
label=label)
else:
plt.plot(timeline,
np.log(train_losses),
color=colors[i],
label=label)
plt.xlabel('Train steps')
plt.ylabel('Log training loss')
plt.legend(loc='upper right')
plt.subplot(1, 3, 2)
for i, (timeline, train_losses, val_losses, learning_rates,
smoothed_train_losses, smoothed_val_losses,
smoothed_learning_rates, label) in enumerate(experiments):
if smoothed_val_losses is not None:
plt.plot(timeline,
np.log(val_losses),
color=colors[i],
alpha=0.25)
plt.plot(timeline,
np.log(smoothed_val_losses),
color=colors[i],
label=label)
else:
plt.plot(timeline,
np.log(val_losses),
color=colors[i],
label=label)
plt.xlabel('Train steps')
plt.ylabel('Log validation loss')
plt.legend(loc='upper right')
plt.subplot(1, 3, 3)
for i, (timeline, train_losses, val_losses, learning_rates,
smoothed_train_losses, smoothed_val_losses,
smoothed_learning_rates, label) in enumerate(experiments):
if i == 0 and smoothed_learning_rates is not None:
plt.plot(timeline, learning_rates, color=colors[i], alpha=0.25)
plt.plot(timeline,
smoothed_learning_rates,
color=colors[i],
label=label)
else:
plt.plot(timeline,
learning_rates,
color=colors[i],
label=label)
plt.xlabel('Train steps')
plt.ylabel('Learning rate')
plt.legend(loc='upper right')
last_step = len(self.info_list) == (self.num_train_steps //
self.update_freq)
plt.tight_layout()
if last_step:
path = 'results/'
if not os.path.exists(path): os.makedirs(path)
plt.savefig(path + 'experiment.png')
plt.show()
plt.draw()
plt.pause(5 if last_step else 0.001)
def main(criterion,
model='lstm',
model_seed=0,
domain='alarm',
data_seed=0,
batchsize=20,
max_epoch=100,
patience=20,
tot_acq=160,
evaluation_set='test',
gpu_idx=0,
log_dir='logs'):
data = pickle.load(open('data/TOP.pkl', 'rb'))[domain]
num_labels = int(len(data['intent_label_mapping']) / 2)
data = data['seeds'][data_seed]
_, warmstart_labels = zip(*data['warmstart'])
_, pool_labels = zip(*data['pool'])
N_warmstart = len(data['warmstart'])
N = len(data['pool'])
try:
l1 = load_baseline(f'odmr-l1-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l1_curve, l1_order = l1['curve'], l1['order']
print(l1_curve, np.mean(l1_curve))
l2 = load_baseline(f'odmr-l2-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l2_curve, l2_order = l2['curve'], l2['order']
print(l2_curve, np.mean(l2_curve))
l3 = load_baseline(f'odmr-l3-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l3_curve, l3_order = l3['curve'], l3['order']
print(l3_curve, np.mean(l3_curve))
l4 = load_baseline(f'odmr-l4-{criterion}', evaluation_set, model,
model_seed, domain, data_seed, batchsize, max_epoch,
patience, tot_acq)
l4_curve, l4_order = l4['curve'], l4['order']
print(l4_curve, np.mean(l4_curve))
except KeyError:
l1_curve, l1_order = odmr(data,
criterion,
evaluation_set,
'test',
True,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l1_curve, np.mean(l1_curve))
store_baseline(l1_curve, l1_order, f'odmr-l1-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
l2_curve, l2_order = odmr(data,
criterion,
evaluation_set,
'accessible',
True,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l2_curve, np.mean(l2_curve))
store_baseline(l2_curve, l2_order, f'odmr-l2-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
l3_curve, l3_order = odmr(data,
criterion,
evaluation_set,
'test',
False,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l3_curve, np.mean(l3_curve))
store_baseline(l3_curve, l3_order, f'odmr-l3-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
l4_curve, l4_order = odmr(data,
criterion,
evaluation_set,
'accessible',
False,
model,
model_seed,
domain,
batchsize,
max_epoch,
patience,
tot_acq,
gpu_idx,
num_labels,
smoothing=1)
print(l4_curve, np.mean(l4_curve))
store_baseline(l4_curve, l4_order, f'odmr-l4-{criterion}',
evaluation_set, model, model_seed, domain, data_seed,
batchsize, max_epoch, patience, tot_acq)
plt.figure(figsize=[20, 4])
gs = GridSpec(ncols=9,
nrows=1,
width_ratios=[10, 0.3, 10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05)
plt.subplot(gs[0, 0])
if criterion == 'max-entropy':
baselines = [('max-entropy', 'Max-Entropy', 0),
('odmr-l1-max-entropy', 'Test + True', 1),
('odmr-l2-max-entropy', 'Acce + True', 2),
('odmr-l3-max-entropy', 'Test + Pred', 6),
('odmr-l4-max-entropy', 'Acce + Pred', 8),
('random', 'Random', 4)]
elif criterion == 'bald':
baselines = [('bald', 'BALD', 0), ('odmr-l1-bald', 'Test + True', 1),
('odmr-l2-bald', 'Acce + True', 2),
('odmr-l3-bald', 'Test + Pred', 6),
('odmr-l4-bald', 'Acce + Pred', 8),
('random', 'Random', 4)]
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1, batchsize))
optimal_order, optimal_quality, _ = load_optimal(log_dir, model,
model_seed, domain,
data_seed, batchsize,
max_epoch, patience,
tot_acq)
plot_curves(optimal_order, xs, evaluation_set, model, model_seed,
model_seed, domain, data_seed, batchsize, max_epoch, patience,
tot_acq, None, None, baselines)
plt.xlabel('# Data Points')
plt.ylabel('F1')
plt.title('Intent Classification')
xs = list(range(N_warmstart, N_warmstart + tot_acq + 1))
_, test_labels = zip(*data['test'])
test_label_counts = Counter(test_labels)
test_label_counts = np.array(
[test_label_counts[i] for i in range(num_labels)])
label_ref_cdf = np.cumsum(test_label_counts / sum(test_label_counts))
label_ref_cdf = list(label_ref_cdf.flat)
label_ref_cdf.insert(0, 0)
plt.subplot(gs[0, 2])
plot_label_distribution(l1_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Test + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 3])
plot_ref_meter(label_ref_cdf)
plt.subplot(gs[0, 4])
plot_label_distribution(l2_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Accessible + Groundtruth')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 5])
plot_ref_meter(label_ref_cdf)
plt.subplot(gs[0, 6])
plot_label_distribution(l3_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Test + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.subplot(gs[0, 7])
plot_ref_meter(label_ref_cdf)
plt.subplot(gs[0, 8])
plot_label_distribution(l4_order, warmstart_labels, pool_labels,
num_labels, label_ref_cdf, xs)
plt.title('Accessible + Predicted')
plt.xlabel('# Data Points')
plt.yticks([])
plt.savefig(f'../figures/intent_classification/odmr_{criterion}.pdf',
bbox_inches='tight')
import setproctitle
import pickle as pkl
import utils
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
if __name__ == '__main__':
args = utils.parse_args()
setproctitle.setproctitle('PPO2-ALRS')
env = utils.make_alrs_env(args, test=True, baseline=True)
baseline = utils.values_from_list_of_dicts(
utils.load_baseline(args.dataset + '_' + args.architecture), 'lr')
test_loss, test_acc = utils.AvgLoss(), utils.AvgLoss()
num_runs = 10
for run in range(num_runs):
env.reset()
done = False
step = 0
alrs = env.venv.envs[0].env
while not done:
action = np.array(baseline[step] / alrs.lr).reshape(1, )
_, _, done, _ = env.step(action)
def main(model_seed, data_seed, batchsize, max_epoch, patience, tot_acq,
log_dir):
data = load_data(data_seed)
warmstart = data['warmstart']
pool = data['pool']
test = data['test']
optimal_order, _, _ = load_optimal(log_dir, model_seed, data_seed,
batchsize, max_epoch, patience, tot_acq)
bald_order = load_baseline('bald', 'test', model_seed, data_seed,
batchsize, max_epoch, patience,
tot_acq)['order']
batchbald_order = load_baseline('batchbald', 'test', model_seed, data_seed,
batchsize, max_epoch, patience,
tot_acq)['order']
random.seed(0)
random_order = random.sample(range(len(pool)), len(pool))[:tot_acq]
warmstart_X, warmstart_y = zip(*warmstart)
warmstart_X = np.array(warmstart_X)
pool_X, pool_y = zip(*pool)
pool_X = np.array(pool_X)
test_X, test_y = zip(*test)
test_X = np.array(test_X)
all_X = np.vstack((warmstart_X, pool_X, test_X[:2000]))
all_X = all_X.reshape(len(all_X), -1)
pca = PCA(n_components=100, random_state=np.random.RandomState(0))
tsne = TSNE(n_components=2,
n_jobs=-1,
random_state=np.random.RandomState(0))
all_X_2d = tsne.fit_transform(pca.fit_transform(all_X))
warmstart_X_2d, pool_X_2d, test_X_2d = np.array_split(
all_X_2d,
[len(warmstart_X), len(warmstart_X) + len(pool_X)])
plt.figure(figsize=[10, 7])
gs = GridSpec(ncols=7,
nrows=2,
width_ratios=[10, 0.7, 10, 0.7, 10, 0.7, 10],
wspace=0.05,
height_ratios=[3, 2],
hspace=0.15)
plt.subplot(gs[0, :3])
plot_tsne(bald_order, warmstart_X_2d, warmstart_y, pool_X_2d, pool_y,
test_X_2d, test_y[:2000], 5, batchsize)
plt.title('BALD Order')
plt.subplot(gs[0, 4:])
plot_tsne(batchbald_order, warmstart_X_2d, warmstart_y, pool_X_2d, pool_y,
test_X_2d, test_y[:2000], 5, batchsize)
plt.title('BatchBALD Order')
plt.subplot(gs[1, 0])
plot_label_proportion(optimal_order, warmstart_y, pool_y, test_y)
plt.title('Optimal Order')
plt.ylabel('Label Distribution')
plt.subplot(gs[1, 1])
plot_ref_meter(test_y)
plt.subplot(gs[1, 2])
plot_label_proportion(bald_order, warmstart_y, pool_y, test_y)
plt.title(f'BALD Order')
plt.yticks([])
plt.subplot(gs[1, 3])
plot_ref_meter(test_y)
plt.subplot(gs[1, 4])
plot_label_proportion(batchbald_order, warmstart_y, pool_y, test_y)
plt.title(f'BatchBALD Order')
plt.yticks([])
plt.subplot(gs[1, 5])
plot_ref_meter(test_y)
plt.subplot(gs[1, 6])
plot_label_proportion(random_order, warmstart_y, pool_y, test_y)
plt.title('Random Order')
plt.yticks([])
plt.savefig(
f'../figures/object_classification/distribution_vis_bald_batchbald.pdf',
bbox_inches='tight')
