def run_cross_validation(settings, targets, classifiers, pipelines):
print 'Cross-validation task'
print 'Targets', ', '.join(targets)
print 'Pipelines:\n ', '\n '.join([p.get_name() for p in pipelines])
print 'Classifiers', ', '.join([c[1] for c in classifiers])
run_prepare_data_for_cross_validation(settings, targets, pipelines)
# run on pool first, then show results after
pool = Pool(settings.N_jobs)
for i, pipeline in enumerate(pipelines):
for j, (classifier, classifier_name) in enumerate(classifiers):
for k, target in enumerate(targets):
progress_str = 'P=%d/%d C=%d/%d T=%d/%d' % (i+1, len(pipelines), j+1, len(classifiers), k+1, len(targets))
cross_validation_score(settings, target, pipeline, classifier, classifier_name,
strategy=cross_validation_strategy, pool=pool, progress_str=progress_str, return_data=False, quiet=True)
pool.close()
pool.join()
summaries = []
best = {}
for p_num, pipeline in enumerate(pipelines):
for c_num, (classifier, classifier_name) in enumerate(classifiers):
mean_scores = []
median_scores = []
datas = []
for target in targets:
print 'Running %s pipeline %s classifier %s' % (target, pipeline.get_name(), classifier_name)
data = cross_validation_score(settings, target, pipeline, classifier, classifier_name,
strategy=cross_validation_strategy, quiet=True)
datas.append(data)
if data.mean_score != data.median_score:
print '%.3f (mean)' % data.mean_score, data.mean_scores
print '%.3f (median)' % data.median_score, data.median_scores
else:
print '%.3f' % data.mean_score
mean_scores.append(data.mean_score)
median_scores.append(data.median_score)
best_score = best.get(target, [0, None, None, None])[0]
cur_score = max(data.mean_score, data.median_score)
if cur_score > best_score:
best[target] = [cur_score, pipeline, classifier, classifier_name]
name = 'p=%d c=%d %s mean %s' % (p_num, c_num, classifier_name, pipeline.get_name())
summary = get_score_summary(name, mean_scores)
summaries.append((summary, np.mean(mean_scores)))
print summary
name = 'p=%d c=%d %s median %s' % (p_num, c_num, classifier_name, pipeline.get_name())
summary = get_score_summary(name, median_scores)
summaries.append((summary, np.mean(median_scores)))
print summary
print_results(summaries)
print '\nbest'
for target in targets:
pipeline = best[target][1]
classifier_name = best[target][3]
print target, best[target][0], classifier_name, pipeline.get_names()
def run_cross_validation(settings, targets, pipelines, mask_range, split_ratios, classifiers):
pool = Pool(settings.N_jobs)
for i, pipeline in enumerate(pipelines):
for j, (classifier, classifier_name) in enumerate(classifiers):
for k, target in enumerate(targets):
pool.apply_async(cross_validation_score, [settings, target, pipeline, classifier, classifier_name], {'quiet': True})
for split_num, split_ratio in enumerate(split_ratios):
masks = generate_feature_masks(settings, target, pipeline, np.max(mask_range), split_ratio, random_state=0, quiet=True)
for mask_num, mask in enumerate(masks):
progress_str = 'P=%d/%d C=%d/%d T=%d/%d S=%d/%d M=%d/%d' % (i+1, len(pipelines), j+1, len(classifiers), k+1, len(targets), split_num+1, len(split_ratios), mask_num+1, len(masks))
cross_validation_score(settings, target, pipeline, classifier, classifier_name, feature_mask=mask, quiet=True, return_data=False, pool=pool, progress_str=progress_str)
pool.close()
pool.join()
print 'Finished cross validation mp'
summaries = []
for p_num, pipeline in enumerate(pipelines):
for classifier, classifier_name in classifiers:
scores_full = []
scores_masked = [[[] for y in mask_range] for x in split_ratios]
for i, target in enumerate(targets):
run_prepare_data_for_cross_validation(settings, [target], [pipeline], quiet=True)
data = cross_validation_score(settings, target, pipeline, classifier, classifier_name, pool=None, quiet=True)
scores_full.append(data.mean_score)
for split_index, split_ratio in enumerate(split_ratios):
masks = generate_feature_masks(settings, target, pipeline, np.max(mask_range), split_ratio, random_state=0, quiet=True)
for mask_index, num_masks in enumerate(mask_range):
predictions = []
y_cvs = None
for mask in masks[0:num_masks]:
data = cross_validation_score(settings, target, pipeline, classifier, classifier_name, feature_mask=mask, pool=None, quiet=True)
predictions.append(data.mean_predictions)
if y_cvs is None:
y_cvs = data.y_cvs
else:
for y_cv_1, y_cv_2 in zip(y_cvs, data.y_cvs):
assert np.alltrue(y_cv_1 == y_cv_2)
predictions = np.mean(predictions, axis=0)
scores = [roc_auc_score(y_cv, p) for p, y_cv in zip(predictions, y_cvs)]
score = np.mean(scores)
scores_masked[split_index][mask_index].append(score)
summary = get_score_summary('%s p=%d full' % (classifier_name, p_num), scores_full, np.mean(scores_full), targets)
summaries.append((summary, np.mean(scores_full)))
for split_index, split_ratio in enumerate(split_ratios):
for mask_index, num_masks in enumerate(mask_range):
scores = scores_masked[split_index][mask_index]
summary = get_score_summary('%s p=%d split_ratio=%s masks=%d' % (classifier_name, p_num, split_ratio, num_masks), scores, np.mean(scores), targets)
summaries.append((summary, np.mean(scores)))
print summary
print_results(summaries)
def run_cross_validation(settings, targets, pipelines, mask_range, split_ratios, classifiers):
pool = Pool(settings.N_jobs)
for i, pipeline in enumerate(pipelines):
for j, (classifier, classifier_name) in enumerate(classifiers):
for k, target in enumerate(targets):
pool.apply_async(cross_validation_score, [settings, target, pipeline, classifier, classifier_name], {'quiet': True})
for split_num, split_ratio in enumerate(split_ratios):
masks = generate_feature_masks(settings, target, pipeline, np.max(mask_range), split_ratio, random_state=0, quiet=True)
for mask_num, mask in enumerate(masks):
progress_str = 'P=%d/%d C=%d/%d T=%d/%d S=%d/%d M=%d/%d' % (i+1, len(pipelines), j+1, len(classifiers), k+1, len(targets), split_num+1, len(split_ratios), mask_num+1, len(masks))
cross_validation_score(settings, target, pipeline, classifier, classifier_name, feature_mask=mask, quiet=True, return_data=False, pool=pool, progress_str=progress_str)
pool.close()
pool.join()
print('Finished cross validation mp')
summaries = []
for p_num, pipeline in enumerate(pipelines):
for classifier, classifier_name in classifiers:
scores_full = []
scores_masked = [[[] for y in mask_range] for x in split_ratios]
for i, target in enumerate(targets):
run_prepare_data_for_cross_validation(settings, [target], [pipeline], quiet=True)
data = cross_validation_score(settings, target, pipeline, classifier, classifier_name, pool=None, quiet=True)
scores_full.append(data.mean_score)
for split_index, split_ratio in enumerate(split_ratios):
masks = generate_feature_masks(settings, target, pipeline, np.max(mask_range), split_ratio, random_state=0, quiet=True)
for mask_index, num_masks in enumerate(mask_range):
predictions = []
y_cvs = None
for mask in masks[0:num_masks]:
data = cross_validation_score(settings, target, pipeline, classifier, classifier_name, feature_mask=mask, pool=None, quiet=True)
predictions.append(data.mean_predictions)
if y_cvs is None:
y_cvs = data.y_cvs
else:
for y_cv_1, y_cv_2 in zip(y_cvs, data.y_cvs):
assert np.alltrue(y_cv_1 == y_cv_2)
predictions = np.mean(predictions, axis=0)
scores = [roc_auc_score(y_cv, p) for p, y_cv in zip(predictions, y_cvs)]
score = np.mean(scores)
scores_masked[split_index][mask_index].append(score)
summary = get_score_summary('%s p=%d full' % (classifier_name, p_num), scores_full)
summaries.append((summary, np.mean(scores_full)))
for split_index, split_ratio in enumerate(split_ratios):
for mask_index, num_masks in enumerate(mask_range):
scores = scores_masked[split_index][mask_index]
summary = get_score_summary('%s p=%d split_ratio=%s masks=%d' % (classifier_name, p_num, split_ratio, num_masks), scores)
summaries.append((summary, np.mean(scores)))
print(summary)
print_results(summaries)
def evaluate_fitness_score(settings, target, pipeline, classifier, classifier_name, quiet, arg):
individual, best_score = arg
if np.sum(individual) == 0:
score = 0.0
else:
score = float(cross_validation_score(settings, target, pipeline, classifier, classifier_name,
strategy=cross_validation_strategy, feature_mask=individual, quiet=True).mean_score)
if score > best_score:
if not quiet: print score, np.sum(individual)
return score,
def evaluate_fitness_score(settings, target, pipeline, classifier,
classifier_name, quiet, arg):
individual, best_score = arg
if np.sum(individual) == 0:
score = 0.0
else:
score = float(
cross_validation_score(settings,
target,
pipeline,
classifier,
classifier_name,
strategy=cross_validation_strategy,
feature_mask=individual,
quiet=True).mean_score)
if score > best_score:
if not quiet: print score, np.sum(individual)
return score,
def process_target(settings,
target,
pipeline,
classifier,
classifier_name,
ratio,
ngen,
quiet,
threshold=400):
# make results repeatable
random.seed(0)
num_features, num_training_examples = get_pipeline_data(
settings, target, pipeline)
# Using sub-feature selection for the human patients appears to perform worse than
# using full feature set. My guess is that perhaps there is not enough training samples
# for this technique to work effectively. So do not run GA if there are too few training
# samples. The threshold parameter can be tweaked with more testing.
if num_training_examples < threshold:
score = float(
cross_validation_score(settings,
target,
pipeline,
classifier,
classifier_name,
strategy=cross_validation_strategy,
quiet=True).mean_score)
return score, [[1] * num_features]
num_wanted_features = int(num_features * ratio)
if not quiet: print 'ratio', ratio
if not quiet: print 'num features', num_features
if not quiet: print 'num wanted features', num_wanted_features
if not quiet: print target, classifier_name
pool = Pool(settings.N_jobs)
toolbox = base.Toolbox()
toolbox.register("map", pool.map)
toolbox.register("attr_bool", random_bool, ratio)
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_bool, num_features)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("evaluate", evaluate_fitness_score, settings, target,
pipeline, classifier, classifier_name, quiet)
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=0.05)
toolbox.register("select", tools.selTournament, tournsize=3)
pop = toolbox.population(n=30)
CXPB, MUTPB, NGEN = 0.5, 0.2, ngen
best_score = 0
best_feature_mask = None
all_feature_masks = {}
# Evaluate the entire population
if not quiet: print 'evaluating pop %d' % len(pop)
fitnesses = toolbox.map(toolbox.evaluate, [(ind, 1.0) for ind in pop])
if not quiet: print 'done evaluating'
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
all_feature_masks[calc_feature_mask_string(ind)] = (list(ind), fit[0])
# calc first best
fits = [ind.fitness.values[0] for ind in pop]
best_index = np.argmax(fits)
score = fits[best_index]
if score > best_score:
best_score = score
best_feature_mask = pop[best_index]
if not quiet: print 'new best', best_score, np.sum(best_feature_mask)
# Begin the evolution
for g in range(NGEN):
if not quiet: print("-- %s: Generation %i --" % (target, g))
# Select the next generation individuals
offspring = toolbox.select(pop, int(len(pop)))
# Clone the selected individuals
offspring = list(toolbox.map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate,
[(ind, best_score) for ind in invalid_ind])
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
all_feature_masks[calc_feature_mask_string(ind)] = (list(ind),
fit[0])
if not quiet:
print(" Evaluated %i individuals (pop size %d)" %
(len(invalid_ind), len(offspring)))
# The population is entirely replaced by the offspring
pop[:] = offspring
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
best_index = np.argmax(fits)
all_f = [np.sum(ind) for ind in pop]
if not quiet:
print ' %s, %s, %s (%d-%d)' % (target, fits[best_index],
np.sum(pop[best_index]),
np.min(all_f), np.max(all_f))
length = len(pop)
mean = sum(fits) / length
if not quiet: print(" Min %s" % min(fits))
if not quiet: print(" Max %s" % max(fits))
if not quiet: print(" Avg %s" % mean)
score = fits[best_index]
if score > best_score:
best_score = score
best_feature_mask = pop[best_index]
if not quiet:
print 'new best', best_score, np.sum(best_feature_mask)
if not quiet: print("-- End of (successful) evolution --")
best_ind = tools.selBest(pop, 1)[0]
if not quiet:
print "-- Finished --\n%s\n%s\n%s" % (
target, best_ind.fitness.values[0], best_ind)
pop = list(all_feature_masks.values())
pop.sort(cmp=lambda x1, x2: cmp(x2[1], x1[1]))
sorted_pop = [ind for ind, score in pop]
print target, 'best', pop[0][1], 'worst', pop[-1][1]
return best_score, sorted_pop
def process_target(settings, target, pipeline, classifier, classifier_name, ratio, ngen, quiet, threshold=400):
# make results repeatable
random.seed(0)
num_features, num_training_examples = get_pipeline_data(settings, target, pipeline)
# Using sub-feature selection for the human patients appears to perform worse than
# using full feature set. My guess is that perhaps there is not enough training samples
# for this technique to work effectively. So do not run GA if there are too few training
# samples. The threshold parameter can be tweaked with more testing.
if num_training_examples < threshold:
score = float(cross_validation_score(settings, target, pipeline, classifier, classifier_name,
strategy=cross_validation_strategy, quiet=True).mean_score)
return score, [[1] * num_features]
num_wanted_features = int(num_features * ratio)
if not quiet: print 'ratio', ratio
if not quiet: print 'num features', num_features
if not quiet: print 'num wanted features', num_wanted_features
if not quiet: print target, classifier_name
pool = Pool(settings.N_jobs)
toolbox = base.Toolbox()
toolbox.register("map", pool.map)
toolbox.register("attr_bool", random_bool, ratio)
toolbox.register("individual", tools.initRepeat, creator.Individual, toolbox.attr_bool, num_features)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("evaluate", evaluate_fitness_score, settings, target, pipeline, classifier, classifier_name, quiet)
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=0.05)
toolbox.register("select", tools.selTournament, tournsize=3)
pop = toolbox.population(n=30)
CXPB, MUTPB, NGEN = 0.5, 0.2, ngen
best_score = 0
best_feature_mask = None
all_feature_masks = {}
# Evaluate the entire population
if not quiet: print 'evaluating pop %d' % len(pop)
fitnesses = toolbox.map(toolbox.evaluate, [(ind, 1.0) for ind in pop])
if not quiet: print 'done evaluating'
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
all_feature_masks[calc_feature_mask_string(ind)] = (list(ind), fit[0])
# calc first best
fits = [ind.fitness.values[0] for ind in pop]
best_index = np.argmax(fits)
score = fits[best_index]
if score > best_score:
best_score = score
best_feature_mask = pop[best_index]
if not quiet: print 'new best', best_score, np.sum(best_feature_mask)
# Begin the evolution
for g in range(NGEN):
if not quiet: print("-- %s: Generation %i --" % (target, g))
# Select the next generation individuals
offspring = toolbox.select(pop, int(len(pop)))
# Clone the selected individuals
offspring = list(toolbox.map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, [(ind, best_score) for ind in invalid_ind])
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
all_feature_masks[calc_feature_mask_string(ind)] = (list(ind), fit[0])
if not quiet: print(" Evaluated %i individuals (pop size %d)" % (len(invalid_ind), len(offspring)))
# The population is entirely replaced by the offspring
pop[:] = offspring
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
best_index = np.argmax(fits)
all_f = [np.sum(ind) for ind in pop]
if not quiet: print ' %s, %s, %s (%d-%d)' % (target, fits[best_index], np.sum(pop[best_index]), np.min(all_f), np.max(all_f))
length = len(pop)
mean = sum(fits) / length
if not quiet: print(" Min %s" % min(fits))
if not quiet: print(" Max %s" % max(fits))
if not quiet: print(" Avg %s" % mean)
score = fits[best_index]
if score > best_score:
best_score = score
best_feature_mask = pop[best_index]
if not quiet: print 'new best', best_score, np.sum(best_feature_mask)
if not quiet: print("-- End of (successful) evolution --")
best_ind = tools.selBest(pop, 1)[0]
if not quiet: print "-- Finished --\n%s\n%s\n%s" % (target, best_ind.fitness.values[0], best_ind)
pop = list(all_feature_masks.values())
pop.sort(cmp=lambda x1, x2: cmp(x2[1], x1[1]))
sorted_pop = [ind for ind, score in pop]
print target, 'best', pop[0][1], 'worst', pop[-1][1]
return best_score, sorted_pop
def run_cross_validation(settings, targets, classifiers, pipelines):
print 'Cross-validation task'
print 'Targets', ', '.join(targets)
print 'Pipelines:\n ', '\n '.join([p.get_name() for p in pipelines])
print 'Classifiers', ', '.join([c[1] for c in classifiers])
run_prepare_data_for_cross_validation(settings, targets, pipelines)
# run on pool first, then show results after
pool = Pool(settings.N_jobs)
for i, pipeline in enumerate(pipelines):
for j, (classifier, classifier_name) in enumerate(classifiers):
for k, target in enumerate(targets):
progress_str = 'P=%d/%d C=%d/%d T=%d/%d' % (
i + 1, len(pipelines), j + 1, len(classifiers), k + 1,
len(targets))
cross_validation_score(settings,
target,
pipeline,
classifier,
classifier_name,
strategy=cross_validation_strategy,
pool=pool,
progress_str=progress_str,
return_data=False,
quiet=True)
pool.close()
pool.join()
summaries = []
best = {}
for p_num, pipeline in enumerate(pipelines):
for c_num, (classifier, classifier_name) in enumerate(classifiers):
mean_scores = []
median_scores = []
datas = []
for target in targets:
print 'Running %s pipeline %s classifier %s' % (
target, pipeline.get_name(), classifier_name)
data = cross_validation_score(
settings,
target,
pipeline,
classifier,
classifier_name,
strategy=cross_validation_strategy,
quiet=True)
datas.append(data)
if data.mean_score != data.median_score:
print '%.3f (mean)' % data.mean_score, data.mean_scores
print '%.3f (median)' % data.median_score, data.median_scores
else:
print '%.3f' % data.mean_score
mean_scores.append(data.mean_score)
median_scores.append(data.median_score)
best_score = best.get(target, [0, None, None, None])[0]
cur_score = max(data.mean_score, data.median_score)
if cur_score > best_score:
best[target] = [
cur_score, pipeline, classifier, classifier_name
]
name = 'p=%d c=%d %s mean %s' % (p_num, c_num, classifier_name,
pipeline.get_name())
summary = get_score_summary(name, mean_scores)
summaries.append((summary, np.mean(mean_scores)))
print summary
name = 'p=%d c=%d %s median %s' % (p_num, c_num, classifier_name,
pipeline.get_name())
summary = get_score_summary(name, median_scores)
summaries.append((summary, np.mean(median_scores)))
print summary
print_results(summaries)
print '\nbest'
for target in targets:
pipeline = best[target][1]
classifier_name = best[target][3]
print target, best[target][0], classifier_name, pipeline.get_names()
