def make_removals_plot_lfw(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H0 = Jobs.group(['spec.order'],
{'exp_key': exp_key0, 'state':2},
{'losses': []},
'function(d, o){o.losses.push(d.result.loss);}')
order_choices = params.order_choices
ords0 = pluck(H0, 'spec.order')
reinds = [ords0.index(_o) for _o in order_choices]
H0 = [H0[_r] for _r in reinds]
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H = Jobs.group(['spec.desc.order'],
{'exp_key': exp_key, 'state':2},
{'losses': []},
'function(d, o){o.losses.push(d.result.loss);}')
order_choices_removals = params.order_choices_removals
ords = pluck(H, 'spec.desc.order')
reinds = [ords.index(_o) for _o in order_choices_removals]
H = [H[_r] for _r in reinds]
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsEvaluation/hyperopt.theano_bandit_algos.TheanoRandom'
for (ind,l) in enumerate(Jobs.find({'exp_key': exp_key, 'state': 2}, fields=['result.order_results'])):
print(ind)
for _ind in range(len(l['result']['order_results'])):
y = l['result']['order_results'][_ind]['loss']
H[_ind]['losses'].append(y)
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels0 = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) for (Before, After) in order_choices]
order_labels = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) for (Before, After) in order_choices_removals]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18,8))
plt.boxplot([1-np.array(h['losses']) for h in H0 + H])
means = [1-np.array(h['losses']).mean() for h in H0 + H]
plt.plot(range(1,len(H0)+len(H)+1), means, color='green')
plt.scatter(range(1,len(H0)+len(H)+1), means)
plt.xticks(range(1,len(ords0 + ords)+1), order_labels0 + order_labels, rotation=60)
plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5, linestyle='--', color='green', linewidth=2)
plt.title('Model form removals on LFW Verification Task', y=.95, fontsize=15)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
if save:
plt.savefig('model_exploration_removal_boxplots_lfw.png')
def make_plot_lfw_reorder_other(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditModelExplorationOther/hyperopt.Random'
H = Jobs.group(['spec.order'],
{'exp_key': exp_key, 'state':2,
'spec.preproc.size.0':250
},
{'losses': []},
'function(d, o){o.losses.push(d.result.loss);}')
order_choices = params.order_choices
ords = pluck(H, 'spec.order')
reinds = [ords.index(_o) for _o in order_choices]
H = [H[_r] for _r in reinds]
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) for (Before, After) in order_choices]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18,8))
plt.boxplot([1-np.array(h['losses']) for h in H])
means = [1-np.array(h['losses']).mean() for h in H]
plt.plot(range(1,len(H)+1), means, color='green')
plt.scatter(range(1,len(H)+1), means)
plt.xticks(range(1,len(ords)+1), order_labels, rotation=60)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
def make_standardfirstdifferent_removals_plot_lfw(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H0 = Jobs.group(['spec.order'],
{'exp_key': exp_key0, 'state':2, 'spec.order': [[], ['activ','lpool','lnorm']]},
{'losses': []},
'function(d, o){o.losses.push(d.result.loss);}')
order_choices = [[[], ['activ','lpool','lnorm']]]
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditStandardFirstDifferentRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H = Jobs.group(['spec.desc.order'],
{'exp_key': exp_key, 'state':2},
{'losses': []},
'function(d, o){o.losses.push(d.result.loss);}')
standard_removed_orders_first_different = params.standard_removed_orders_first_different
standard_removed_orders_first_different = map(list, standard_removed_orders_first_different)
inds = [0, 6, 1, 7, 2, 8, 3, 9, 4, 10, 5, 11]
standard_removed_orders_first_different = [standard_removed_orders_first_different[ind] for ind in inds]
ords = pluck(H, 'spec.desc.order')
reinds = [ords.index(_o) for _o in standard_removed_orders_first_different]
H = [H[_r] for _r in reinds]
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels0 = [','.join([od[b] for b in After]) for (Before, After) in order_choices]
order_labels = [','.join([od[b] for b in A]) + '*' + ','.join([od[b] for b in C])for [[B,A],[D,C]] in standard_removed_orders_first_different]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18,8))
plt.boxplot([1-np.array(h['losses']) for h in H0 + H])
means = [1-np.array(h['losses']).mean() for h in H0 + H]
plt.plot(range(1,len(H0)+len(H)+1), means, color='green')
plt.scatter(range(1,len(H0)+len(H)+1), means)
plt.xticks(range(1,len(order_labels0) + len(order_labels) + 1), order_labels0 + order_labels, rotation=60)
plt.axvline(len(order_labels0) + .5, linestyle='--', color='green', linewidth=2)
plt.title('Model form first-different removals on LFW Verification Task', y=.95, fontsize=15)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
if save:
plt.savefig('model_exploration_firstdifferent_removal_boxplots_lfw.png')
def make_plot_lfw_reorder_other(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditModelExplorationOther/hyperopt.Random'
H = Jobs.group(['spec.order'], {
'exp_key': exp_key,
'state': 2,
'spec.preproc.size.0': 250
}, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')
order_choices = params.order_choices
ords = pluck(H, 'spec.order')
reinds = [ords.index(_o) for _o in order_choices]
H = [H[_r] for _r in reinds]
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels = [
','.join([od[b]
for b in Before]) + '|' + ','.join([od[b] for b in After])
for (Before, After) in order_choices
]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18, 8))
plt.boxplot([1 - np.array(h['losses']) for h in H])
means = [1 - np.array(h['losses']).mean() for h in H]
plt.plot(range(1, len(H) + 1), means, color='green')
plt.scatter(range(1, len(H) + 1), means)
plt.xticks(range(1, len(ords) + 1), order_labels, rotation=60)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
def make_removals_plot_lfw(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H0 = Jobs.group(['spec.order'], {
'exp_key': exp_key0,
'state': 2
}, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')
order_choices = params.order_choices
ords0 = pluck(H0, 'spec.order')
reinds = [ords0.index(_o) for _o in order_choices]
H0 = [H0[_r] for _r in reinds]
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H = Jobs.group(['spec.desc.order'], {
'exp_key': exp_key,
'state': 2
}, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')
order_choices_removals = params.order_choices_removals
ords = pluck(H, 'spec.desc.order')
reinds = [ords.index(_o) for _o in order_choices_removals]
H = [H[_r] for _r in reinds]
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsEvaluation/hyperopt.theano_bandit_algos.TheanoRandom'
for (ind, l) in enumerate(
Jobs.find({
'exp_key': exp_key,
'state': 2
},
fields=['result.order_results'])):
print(ind)
for _ind in range(len(l['result']['order_results'])):
y = l['result']['order_results'][_ind]['loss']
H[_ind]['losses'].append(y)
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels0 = [
','.join([od[b]
for b in Before]) + '|' + ','.join([od[b] for b in After])
for (Before, After) in order_choices
]
order_labels = [
','.join([od[b]
for b in Before]) + '|' + ','.join([od[b] for b in After])
for (Before, After) in order_choices_removals
]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18, 8))
plt.boxplot([1 - np.array(h['losses']) for h in H0 + H])
means = [1 - np.array(h['losses']).mean() for h in H0 + H]
plt.plot(range(1, len(H0) + len(H) + 1), means, color='green')
plt.scatter(range(1, len(H0) + len(H) + 1), means)
plt.xticks(range(1,
len(ords0 + ords) + 1),
order_labels0 + order_labels,
rotation=60)
plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5,
linestyle='--',
color='green',
linewidth=2)
plt.title('Model form removals on LFW Verification Task',
y=.95,
fontsize=15)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
if save:
plt.savefig('model_exploration_removal_boxplots_lfw.png')
def make_removals_plot_synthetic(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key0 = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom/fewer_training_examples'
H0 = Jobs.group(['spec.order'], {
'exp_key': exp_key0,
'state': 2
}, {
'losses': [],
'count': 0
}, 'function(d, o){o.losses.push(d.result.loss); o.count += 1}')
H0e = Jobs.group(['spec.order'], {
'exp_key': exp_key0,
'state': 3
}, {'count': 0}, 'function(d, o){o.count += 1;}')
order_choices = params.order_choices
ords0 = pluck(H0, 'spec.order')
reinds = [ords0.index(_o) for _o in order_choices]
H0 = [H0[_r] for _r in reinds]
ords0e = pluck(H0e, 'spec.order')
reinds_e = [ords0e.index(_o) for _o in order_choices]
H0e = [H0e[_r] for _r in reinds_e]
exp_key = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H = Jobs.group(['spec.desc.order'], {
'exp_key': exp_key,
'state': 2
}, {
'losses': [],
'count': 0
}, 'function(d, o){o.losses.push(d.result.loss); o.count += 1}')
He = Jobs.group(['spec.desc.order'], {
'exp_key': exp_key,
'state': 3
}, {'count': 0}, 'function(d, o){o.count += 1;}')
order_choices_removals = params.order_choices_removals
ords = pluck(H, 'spec.desc.order')
reinds = [ords.index(_o) for _o in order_choices_removals]
H = [H[_r] for _r in reinds]
ordse = pluck(He, 'spec.desc.order')
reinds_e = [
ordse.index(_o) if _o in ordse else None
for _o in order_choices_removals
]
He = [He[_r] if _r is not None else {'count': 0} for _r in reinds_e]
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels0 = [
','.join([od[b]
for b in Before]) + '|' + ','.join([od[b] for b in After]) +
' (' + str(he['count']) + '/' + str(h['count']) + ')'
for (Before, After), he, h in zip(order_choices, H0e, H0)
]
order_labels = [
','.join([od[b]
for b in Before]) + '|' + ','.join([od[b] for b in After]) +
' (' + str(he['count']) + '/' + str(h['count']) + ')'
for (Before, After), he, h in zip(order_choices_removals, He, H)
]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18, 8))
plt.boxplot([1 - np.array(h['losses']) for h in H0 + H])
means = [1 - np.array(h['losses']).mean() for h in H0 + H]
plt.plot(range(1, len(H0) + len(H) + 1), means, color='green')
plt.scatter(range(1, len(H0) + len(H) + 1), means)
plt.xticks(range(1,
len(ords0 + ords) + 1),
order_labels0 + order_labels,
rotation=60)
plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5,
linestyle='--',
color='green',
linewidth=2)
plt.title('Model form removals on Synthetic Categorization Task',
y=.95,
fontsize=15)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
if save:
plt.savefig('model_exploration_removal_boxplots_Synthetic.png')
def make_standardfirstdifferent_removals_plot_lfw(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H0 = Jobs.group(
['spec.order'], {
'exp_key': exp_key0,
'state': 2,
'spec.order': [[], ['activ', 'lpool', 'lnorm']]
}, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')
order_choices = [[[], ['activ', 'lpool', 'lnorm']]]
exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditStandardFirstDifferentRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H = Jobs.group(['spec.desc.order'], {
'exp_key': exp_key,
'state': 2
}, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')
standard_removed_orders_first_different = params.standard_removed_orders_first_different
standard_removed_orders_first_different = map(
list, standard_removed_orders_first_different)
inds = [0, 6, 1, 7, 2, 8, 3, 9, 4, 10, 5, 11]
standard_removed_orders_first_different = [
standard_removed_orders_first_different[ind] for ind in inds
]
ords = pluck(H, 'spec.desc.order')
reinds = [ords.index(_o) for _o in standard_removed_orders_first_different]
H = [H[_r] for _r in reinds]
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels0 = [
','.join([od[b] for b in After]) for (Before, After) in order_choices
]
order_labels = [
','.join([od[b] for b in A]) + '*' + ','.join([od[b] for b in C])
for [[B, A], [D, C]] in standard_removed_orders_first_different
]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18, 8))
plt.boxplot([1 - np.array(h['losses']) for h in H0 + H])
means = [1 - np.array(h['losses']).mean() for h in H0 + H]
plt.plot(range(1, len(H0) + len(H) + 1), means, color='green')
plt.scatter(range(1, len(H0) + len(H) + 1), means)
plt.xticks(range(1,
len(order_labels0) + len(order_labels) + 1),
order_labels0 + order_labels,
rotation=60)
plt.axvline(len(order_labels0) + .5,
linestyle='--',
color='green',
linewidth=2)
plt.title('Model form first-different removals on LFW Verification Task',
y=.95,
fontsize=15)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
if save:
plt.savefig(
'model_exploration_firstdifferent_removal_boxplots_lfw.png')
def make_removals_plot_synthetic(save=False):
conn = pm.Connection()
db = conn['hyperopt']
Jobs = db['jobs']
exp_key0 = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom/fewer_training_examples'
H0 = Jobs.group(['spec.order'],
{'exp_key': exp_key0, 'state':2},
{'losses': [], 'count': 0},
'function(d, o){o.losses.push(d.result.loss); o.count += 1}')
H0e = Jobs.group(['spec.order'],
{'exp_key': exp_key0, 'state':3},
{'count': 0},
'function(d, o){o.count += 1;}')
order_choices = params.order_choices
ords0 = pluck(H0, 'spec.order')
reinds = [ords0.index(_o) for _o in order_choices]
H0 = [H0[_r] for _r in reinds]
ords0e = pluck(H0e, 'spec.order')
reinds_e = [ords0e.index(_o) for _o in order_choices]
H0e = [H0e[_r] for _r in reinds_e]
exp_key = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'
H = Jobs.group(['spec.desc.order'],
{'exp_key': exp_key, 'state':2},
{'losses': [], 'count': 0},
'function(d, o){o.losses.push(d.result.loss); o.count += 1}')
He = Jobs.group(['spec.desc.order'],
{'exp_key': exp_key, 'state':3},
{'count': 0},
'function(d, o){o.count += 1;}')
order_choices_removals = params.order_choices_removals
ords = pluck(H, 'spec.desc.order')
reinds = [ords.index(_o) for _o in order_choices_removals]
H = [H[_r] for _r in reinds]
ordse = pluck(He, 'spec.desc.order')
reinds_e = [ordse.index(_o) if _o in ordse else None for _o in order_choices_removals]
He = [He[_r] if _r is not None else {'count': 0} for _r in reinds_e]
od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
order_labels0 = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) + ' (' + str(he['count']) + '/' + str(h['count']) + ')' for (Before, After), he, h in zip(order_choices,H0e, H0)]
order_labels = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) + ' (' + str(he['count']) + '/' + str(h['count']) + ')' for (Before, After), he, h in zip(order_choices_removals, He, H)]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(18,8))
plt.boxplot([1-np.array(h['losses']) for h in H0 + H])
means = [1-np.array(h['losses']).mean() for h in H0 + H]
plt.plot(range(1,len(H0)+len(H)+1), means, color='green')
plt.scatter(range(1,len(H0)+len(H)+1), means)
plt.xticks(range(1,len(ords0 + ords)+1), order_labels0 + order_labels, rotation=60)
plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5, linestyle='--', color='green', linewidth=2)
plt.title('Model form removals on Synthetic Categorization Task', y=.95, fontsize=15)
plt.ylabel('Absolute performance')
plt.xlabel('Architecture tag')
if save:
plt.savefig('model_exploration_removal_boxplots_Synthetic.png')