示例#1
0
def make_removals_plot_lfw(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']

    exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
    H0 = Jobs.group(['spec.order'],
                   {'exp_key': exp_key0, 'state':2},
                   {'losses': []},
                   'function(d, o){o.losses.push(d.result.loss);}')

    order_choices = params.order_choices
    ords0 = pluck(H0, 'spec.order')
    reinds = [ords0.index(_o) for _o in order_choices]
    H0 = [H0[_r] for _r in reinds]
    
    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'

    H = Jobs.group(['spec.desc.order'],
                   {'exp_key': exp_key, 'state':2},
                   {'losses': []},
                   'function(d, o){o.losses.push(d.result.loss);}')
        
    order_choices_removals = params.order_choices_removals
    ords = pluck(H, 'spec.desc.order')
    reinds = [ords.index(_o) for _o in order_choices_removals]
    H = [H[_r] for _r in reinds]
    
    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsEvaluation/hyperopt.theano_bandit_algos.TheanoRandom'
    for (ind,l) in enumerate(Jobs.find({'exp_key': exp_key, 'state': 2}, fields=['result.order_results'])):
        print(ind)
        for _ind in range(len(l['result']['order_results'])):
            y = l['result']['order_results'][_ind]['loss']
            H[_ind]['losses'].append(y)
                   
    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels0 = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) for (Before, After) in order_choices]
    order_labels = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) for (Before, After) in order_choices_removals]

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18,8))
    plt.boxplot([1-np.array(h['losses']) for h in H0 + H])
    means = [1-np.array(h['losses']).mean() for h in H0 + H]
    plt.plot(range(1,len(H0)+len(H)+1), means, color='green')
    plt.scatter(range(1,len(H0)+len(H)+1), means)
    
    plt.xticks(range(1,len(ords0 + ords)+1), order_labels0 + order_labels, rotation=60)
    plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
    plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5, linestyle='--', color='green', linewidth=2)
    
    plt.title('Model form removals on LFW Verification Task', y=.95, fontsize=15)
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
    if save:
        plt.savefig('model_exploration_removal_boxplots_lfw.png')
示例#2
0
def make_plot_lfw_reorder_other(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']
    
    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditModelExplorationOther/hyperopt.Random'

    H = Jobs.group(['spec.order'],
                   {'exp_key': exp_key, 'state':2, 
                    'spec.preproc.size.0':250
                   },
                   {'losses': []},
                   'function(d, o){o.losses.push(d.result.loss);}')
        
    order_choices = params.order_choices
    ords = pluck(H, 'spec.order')
    reinds = [ords.index(_o) for _o in order_choices]
    H = [H[_r] for _r in reinds]

    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) for (Before, After) in order_choices]
 
    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18,8))
    plt.boxplot([1-np.array(h['losses']) for h in H])
    means = [1-np.array(h['losses']).mean() for h in H]
    plt.plot(range(1,len(H)+1), means, color='green')
    plt.scatter(range(1,len(H)+1), means)
    
    plt.xticks(range(1,len(ords)+1),  order_labels, rotation=60)
    
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
示例#3
0
def make_standardfirstdifferent_removals_plot_lfw(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']


    exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
    H0 = Jobs.group(['spec.order'],
                   {'exp_key': exp_key0, 'state':2, 'spec.order': [[], ['activ','lpool','lnorm']]},
                   {'losses': []},
                   'function(d, o){o.losses.push(d.result.loss);}')
                   
    order_choices = [[[], ['activ','lpool','lnorm']]]

    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditStandardFirstDifferentRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'

    H = Jobs.group(['spec.desc.order'],
                   {'exp_key': exp_key, 'state':2},
                   {'losses': []},
                   'function(d, o){o.losses.push(d.result.loss);}')
        
    standard_removed_orders_first_different = params.standard_removed_orders_first_different
    standard_removed_orders_first_different = map(list, standard_removed_orders_first_different)
    inds = [0, 6, 1, 7, 2, 8, 3, 9, 4, 10, 5, 11]
    standard_removed_orders_first_different = [standard_removed_orders_first_different[ind] for ind in inds]
    
    ords = pluck(H, 'spec.desc.order')
    reinds = [ords.index(_o) for _o in standard_removed_orders_first_different]
    H = [H[_r] for _r in reinds]

    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels0 = [','.join([od[b] for b in After]) for (Before, After) in order_choices]
    order_labels = [','.join([od[b] for b in A]) + '*' +  ','.join([od[b] for b in C])for [[B,A],[D,C]] in standard_removed_orders_first_different]
 

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18,8))
    plt.boxplot([1-np.array(h['losses']) for h in H0 + H])
    means = [1-np.array(h['losses']).mean() for h in H0 + H]
    plt.plot(range(1,len(H0)+len(H)+1), means, color='green')
    plt.scatter(range(1,len(H0)+len(H)+1), means)
    plt.xticks(range(1,len(order_labels0) + len(order_labels) + 1), order_labels0 + order_labels, rotation=60)
    plt.axvline(len(order_labels0) + .5, linestyle='--', color='green', linewidth=2)


    plt.title('Model form first-different removals on LFW Verification Task', y=.95, fontsize=15)
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
    if save:
        plt.savefig('model_exploration_firstdifferent_removal_boxplots_lfw.png')
示例#4
0
def make_plot_lfw_reorder_other(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']

    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditModelExplorationOther/hyperopt.Random'

    H = Jobs.group(['spec.order'], {
        'exp_key': exp_key,
        'state': 2,
        'spec.preproc.size.0': 250
    }, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')

    order_choices = params.order_choices
    ords = pluck(H, 'spec.order')
    reinds = [ords.index(_o) for _o in order_choices]
    H = [H[_r] for _r in reinds]

    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels = [
        ','.join([od[b]
                  for b in Before]) + '|' + ','.join([od[b] for b in After])
        for (Before, After) in order_choices
    ]

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18, 8))
    plt.boxplot([1 - np.array(h['losses']) for h in H])
    means = [1 - np.array(h['losses']).mean() for h in H]
    plt.plot(range(1, len(H) + 1), means, color='green')
    plt.scatter(range(1, len(H) + 1), means)

    plt.xticks(range(1, len(ords) + 1), order_labels, rotation=60)

    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
示例#5
0
def make_removals_plot_lfw(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']

    exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
    H0 = Jobs.group(['spec.order'], {
        'exp_key': exp_key0,
        'state': 2
    }, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')

    order_choices = params.order_choices
    ords0 = pluck(H0, 'spec.order')
    reinds = [ords0.index(_o) for _o in order_choices]
    H0 = [H0[_r] for _r in reinds]

    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'

    H = Jobs.group(['spec.desc.order'], {
        'exp_key': exp_key,
        'state': 2
    }, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')

    order_choices_removals = params.order_choices_removals
    ords = pluck(H, 'spec.desc.order')
    reinds = [ords.index(_o) for _o in order_choices_removals]
    H = [H[_r] for _r in reinds]

    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditRemovalsEvaluation/hyperopt.theano_bandit_algos.TheanoRandom'
    for (ind, l) in enumerate(
            Jobs.find({
                'exp_key': exp_key,
                'state': 2
            },
                      fields=['result.order_results'])):
        print(ind)
        for _ind in range(len(l['result']['order_results'])):
            y = l['result']['order_results'][_ind]['loss']
            H[_ind]['losses'].append(y)

    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels0 = [
        ','.join([od[b]
                  for b in Before]) + '|' + ','.join([od[b] for b in After])
        for (Before, After) in order_choices
    ]
    order_labels = [
        ','.join([od[b]
                  for b in Before]) + '|' + ','.join([od[b] for b in After])
        for (Before, After) in order_choices_removals
    ]

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18, 8))
    plt.boxplot([1 - np.array(h['losses']) for h in H0 + H])
    means = [1 - np.array(h['losses']).mean() for h in H0 + H]
    plt.plot(range(1, len(H0) + len(H) + 1), means, color='green')
    plt.scatter(range(1, len(H0) + len(H) + 1), means)

    plt.xticks(range(1,
                     len(ords0 + ords) + 1),
               order_labels0 + order_labels,
               rotation=60)
    plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
    plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5,
                linestyle='--',
                color='green',
                linewidth=2)

    plt.title('Model form removals on LFW Verification Task',
              y=.95,
              fontsize=15)
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
    if save:
        plt.savefig('model_exploration_removal_boxplots_lfw.png')
示例#6
0
def make_removals_plot_synthetic(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']

    exp_key0 = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom/fewer_training_examples'
    H0 = Jobs.group(['spec.order'], {
        'exp_key': exp_key0,
        'state': 2
    }, {
        'losses': [],
        'count': 0
    }, 'function(d, o){o.losses.push(d.result.loss); o.count += 1}')

    H0e = Jobs.group(['spec.order'], {
        'exp_key': exp_key0,
        'state': 3
    }, {'count': 0}, 'function(d, o){o.count += 1;}')

    order_choices = params.order_choices
    ords0 = pluck(H0, 'spec.order')
    reinds = [ords0.index(_o) for _o in order_choices]
    H0 = [H0[_r] for _r in reinds]
    ords0e = pluck(H0e, 'spec.order')
    reinds_e = [ords0e.index(_o) for _o in order_choices]
    H0e = [H0e[_r] for _r in reinds_e]

    exp_key = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'

    H = Jobs.group(['spec.desc.order'], {
        'exp_key': exp_key,
        'state': 2
    }, {
        'losses': [],
        'count': 0
    }, 'function(d, o){o.losses.push(d.result.loss); o.count += 1}')
    He = Jobs.group(['spec.desc.order'], {
        'exp_key': exp_key,
        'state': 3
    }, {'count': 0}, 'function(d, o){o.count += 1;}')

    order_choices_removals = params.order_choices_removals
    ords = pluck(H, 'spec.desc.order')
    reinds = [ords.index(_o) for _o in order_choices_removals]
    H = [H[_r] for _r in reinds]
    ordse = pluck(He, 'spec.desc.order')
    reinds_e = [
        ordse.index(_o) if _o in ordse else None
        for _o in order_choices_removals
    ]
    He = [He[_r] if _r is not None else {'count': 0} for _r in reinds_e]

    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels0 = [
        ','.join([od[b]
                  for b in Before]) + '|' + ','.join([od[b] for b in After]) +
        ' (' + str(he['count']) + '/' + str(h['count']) + ')'
        for (Before, After), he, h in zip(order_choices, H0e, H0)
    ]
    order_labels = [
        ','.join([od[b]
                  for b in Before]) + '|' + ','.join([od[b] for b in After]) +
        ' (' + str(he['count']) + '/' + str(h['count']) + ')'
        for (Before, After), he, h in zip(order_choices_removals, He, H)
    ]

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18, 8))
    plt.boxplot([1 - np.array(h['losses']) for h in H0 + H])
    means = [1 - np.array(h['losses']).mean() for h in H0 + H]
    plt.plot(range(1, len(H0) + len(H) + 1), means, color='green')
    plt.scatter(range(1, len(H0) + len(H) + 1), means)

    plt.xticks(range(1,
                     len(ords0 + ords) + 1),
               order_labels0 + order_labels,
               rotation=60)
    plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
    plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5,
                linestyle='--',
                color='green',
                linewidth=2)

    plt.title('Model form removals on Synthetic Categorization Task',
              y=.95,
              fontsize=15)
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
    if save:
        plt.savefig('model_exploration_removal_boxplots_Synthetic.png')
示例#7
0
def make_standardfirstdifferent_removals_plot_lfw(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']

    exp_key0 = 'lfw_model_exploration.LFWBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom'
    H0 = Jobs.group(
        ['spec.order'], {
            'exp_key': exp_key0,
            'state': 2,
            'spec.order': [[], ['activ', 'lpool', 'lnorm']]
        }, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')

    order_choices = [[[], ['activ', 'lpool', 'lnorm']]]

    exp_key = 'thor_model_exploration.model_exploration_bandits.LFWBanditStandardFirstDifferentRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'

    H = Jobs.group(['spec.desc.order'], {
        'exp_key': exp_key,
        'state': 2
    }, {'losses': []}, 'function(d, o){o.losses.push(d.result.loss);}')

    standard_removed_orders_first_different = params.standard_removed_orders_first_different
    standard_removed_orders_first_different = map(
        list, standard_removed_orders_first_different)
    inds = [0, 6, 1, 7, 2, 8, 3, 9, 4, 10, 5, 11]
    standard_removed_orders_first_different = [
        standard_removed_orders_first_different[ind] for ind in inds
    ]

    ords = pluck(H, 'spec.desc.order')
    reinds = [ords.index(_o) for _o in standard_removed_orders_first_different]
    H = [H[_r] for _r in reinds]

    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels0 = [
        ','.join([od[b] for b in After]) for (Before, After) in order_choices
    ]
    order_labels = [
        ','.join([od[b] for b in A]) + '*' + ','.join([od[b] for b in C])
        for [[B, A], [D, C]] in standard_removed_orders_first_different
    ]

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18, 8))
    plt.boxplot([1 - np.array(h['losses']) for h in H0 + H])
    means = [1 - np.array(h['losses']).mean() for h in H0 + H]
    plt.plot(range(1, len(H0) + len(H) + 1), means, color='green')
    plt.scatter(range(1, len(H0) + len(H) + 1), means)
    plt.xticks(range(1,
                     len(order_labels0) + len(order_labels) + 1),
               order_labels0 + order_labels,
               rotation=60)
    plt.axvline(len(order_labels0) + .5,
                linestyle='--',
                color='green',
                linewidth=2)

    plt.title('Model form first-different removals on LFW Verification Task',
              y=.95,
              fontsize=15)
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
    if save:
        plt.savefig(
            'model_exploration_firstdifferent_removal_boxplots_lfw.png')
示例#8
0
def make_removals_plot_synthetic(save=False):
    conn = pm.Connection()
    db = conn['hyperopt']
    Jobs = db['jobs']

    exp_key0 = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditModelExploration/hyperopt.theano_bandit_algos.TheanoRandom/fewer_training_examples'
    H0 = Jobs.group(['spec.order'],
                   {'exp_key': exp_key0, 'state':2},
                   {'losses': [], 'count': 0},
                   'function(d, o){o.losses.push(d.result.loss); o.count += 1}')

    H0e = Jobs.group(['spec.order'],
                   {'exp_key': exp_key0, 'state':3},
                   {'count': 0},
                   'function(d, o){o.count += 1;}')

    order_choices = params.order_choices
    ords0 = pluck(H0, 'spec.order')
    reinds = [ords0.index(_o) for _o in order_choices]
    H0 = [H0[_r] for _r in reinds]
    ords0e = pluck(H0e, 'spec.order')
    reinds_e = [ords0e.index(_o) for _o in order_choices]
    H0e = [H0e[_r] for _r in reinds_e]    
    
    exp_key = 'thor_model_exploration.model_exploration_bandits.SyntheticBanditRemovalsExploration/hyperopt.theano_bandit_algos.TheanoRandom'

    H = Jobs.group(['spec.desc.order'],
                   {'exp_key': exp_key, 'state':2},
                   {'losses': [], 'count': 0},
                   'function(d, o){o.losses.push(d.result.loss); o.count += 1}')
    He = Jobs.group(['spec.desc.order'],
                   {'exp_key': exp_key, 'state':3},
                   {'count': 0},
                   'function(d, o){o.count += 1;}')
        
    order_choices_removals = params.order_choices_removals
    ords = pluck(H, 'spec.desc.order')
    reinds = [ords.index(_o) for _o in order_choices_removals]
    H = [H[_r] for _r in reinds]
    ordse = pluck(He, 'spec.desc.order')
    reinds_e = [ordse.index(_o) if _o in ordse else None for _o in order_choices_removals]
    He = [He[_r] if _r is not None else {'count': 0} for _r in reinds_e]    
                       
    od = {'lpool': 'p', 'activ': 'a', 'lnorm': 'n'}
    order_labels0 = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) + ' (' + str(he['count']) + '/' + str(h['count']) + ')' for (Before, After), he, h in zip(order_choices,H0e, H0)]
    order_labels = [','.join([od[b] for b in Before]) + '|' + ','.join([od[b] for b in After]) + ' (' + str(he['count']) + '/' + str(h['count']) + ')' for (Before, After), he, h in zip(order_choices_removals, He, H)]

    import matplotlib.pyplot as plt
    fig = plt.figure(figsize=(18,8))
    plt.boxplot([1-np.array(h['losses']) for h in H0 + H])
    means = [1-np.array(h['losses']).mean() for h in H0 + H]
    plt.plot(range(1,len(H0)+len(H)+1), means, color='green')
    plt.scatter(range(1,len(H0)+len(H)+1), means)
    
    plt.xticks(range(1,len(ords0 + ords)+1), order_labels0 + order_labels, rotation=60)
    plt.axvline(len(ords0) + .5, linestyle='--', color='green', linewidth=2)
    plt.axvline(len(ords0) + len(params.order_choices_single_removals) + .5, linestyle='--', color='green', linewidth=2)
    
    plt.title('Model form removals on Synthetic Categorization Task', y=.95, fontsize=15)
    plt.ylabel('Absolute performance')
    plt.xlabel('Architecture tag')
    if save:
        plt.savefig('model_exploration_removal_boxplots_Synthetic.png')