示例#1
0
    -
        order: [3, 4] #overall avg recall at 10%
    -
        order: [5, 6] #overall_groups at 10%
    -
        order: [7, 8] #overall_groups at 30%
    -
        order: [9, 10] #director3 at 30%
    -
        order: [11, 12] #director2 at 30%
    -
        order: [13, 14] #director1 at 30%
"""

import yaml
res = select_model.run(args['experiment_id'], yaml.load(selector),
                       data.query('fold != "2017-12-26"'))

# +
import seaborn as sns

selector = res['selectors'][0]

fig, axis = plt.subplots(figsize=(15, 10), ncols=1, nrows=1)

df = explode(selector['data'])
model_name = df['name'].unique()[0]
df = pd.concat([df, data.query('name in ("higher_than_x", "random")')])
df['new_name'] = df.apply(lambda x: x['name'] + '_' + str(x['learner_id']), 1)
translate = {
    'higher_than_x': 'Ordered by Value Baseline',
    'random': 'Random Baseline',
示例#2
0
from model_selection import plotting, select_model
from utils.utils import connect_to_database

con = connect_to_database()

# Add here your parameters:
experiment_id = 1129688681
model_selector = 'good_selector'

args = dict(experiment_id=experiment_id)

plotting.overall_performance_per_fold(args, thresh=0)

res = select_model.run(
    args['experiment_id'],
    Path().cwd().parent / 'model_selectors' / (model_selector + '.yaml'))

for df in res['selectors']:
    print(df['learner_stats'])

good_learners = res['selectors'][3]['learner_ids']

# ## Feature Importance

query = """
select feature, avg(abs(importance)) importance
from experiments.feature_importances
where learner_id = {learner_id}
and importance != 0
group by feature
示例#3
0
import sys
from pathlib import Path
import os

source_path = str(
    Path(os.path.abspath('joaoc-model-selector.py')).parent.parent / 'src')
if source_path not in sys.path:
    sys.path.insert(0, source_path)
# sys.path.insert(0, '../utils')

import model_selection.select_model as sm

# !pwd

res = sm.run(
    3124936745,
    '/home/joao.carabetta/Documents/dncp/model_selectors/template.yaml')

# +
import pickle
# Packages
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
import seaborn as sns
# from pipeline.data_persistence import persist_local
from itertools import cycle

color = cycle(cm.get_cmap('tab20', 20).colors)
示例#4
0
            type: top
            value: 10
        statistic: std
        higher_is_better: false
        

selectors:
    -
        order: [1] 
    -
        order: [2] 
    -
        order: [3, 4]
""")

metric_selected, data, max_fold = select_model.run(macro_experiment_id,
                                                   selector)

# ## Recall by Quality Reviews
#
# <div class="alert alert-block alert-info">
# The <b>Selected Models</b> line has an interval. The intervall boundaries are the minimum and maximum of all selected models.
# </div>

plotting.plot_metrics(metric_selected,
                      data,
                      max_fold,
                      baselines,
                      metric='recall')

# ## Precision by Quality Reviews
示例#5
0
            type: threshold
            value: 0.01
        statistic: mean
        higher_is_better: true
        

selectors:
    -
        order: [1]
    -
        order: [2]
    -
        order: [3]
"""

res = select_model.run(args['experiment_id'], yaml.load(selector), data)


# +
nrows = len(res['selectors'])

fig, ax = plt.subplots(figsize=(15, nrows * 10),
                         ncols=1, nrows=nrows)

for i, selector in enumerate(res['selectors']):
    
    axis = ax[i]

    df = explode(selector['data'])
    df = pd.concat([df, data.query('name in ("higher_than_x", "random")')])
    df['new_name'] = df.apply(lambda x: x['name'] + '_' + str(x['learner_id']), 1)