def combine_results(results):
'''
Given a list of result dictionaries, return analysis information such as
the median values of each statistic as well as the median absolute
deviation.
Parameters:
- ``results``: A list of dictionaries containing similar key values.
'''
combined = {}
# Collect the successful runs
successful = [result for result in results if result['success']]
# Combine like keys across all runs
for result in results:
for key, value in result.iteritems():
try:
combined[key].append(value)
except KeyError:
combined[key] = [value]
# Analyze the values for each key
for key, value in combined.items():
try:
combined[key] = util.median_deviation(value)
except TypeError:
del combined[key]
try:
combined['success'] = len(successful) / float(len(results)), 0
except ZeroDivisionError:
combined['success'] = 0, 0
return combined
def combine_results(results):
'''
Given a list of result dictionaries, return analysis information such as
the median values of each statistic as well as the median absolute
deviation.
Parameters:
- ``results``: A list of dictionaries containing similar key values.
'''
combined = {}
# Collect the successful runs
successful = [result for result in results if result['success']]
# Combine like keys across all runs
for result in results:
for key, value in result.iteritems():
try:
combined[key].append(value)
except KeyError:
combined[key] = [value]
# Analyze the values for each key
for key, value in combined.items():
try:
combined[key] = util.median_deviation(value)
except TypeError:
del combined[key]
try:
combined['success'] = len(successful) / float(len(results)), 0
except ZeroDivisionError:
combined['success'] = 0, 0
return combined
# Determine the settings from the filename
problem, dup, ordering, nodes, mut, seed = base.split('_')
with open_file_method(filename)(filename, 'r') as f:
data = json.load(f)
version = dup, ordering, nodes, mut
if (dup, ordering) == ('skip', 'normal'):
control_group = version
statify[version].append(data[1]['evals'])
active[version].append(data[1]['phenotype'])
best = data[1]['bests'][-1]
test = data[1]['test_inputs']
individual = Individual.reconstruct_individual(best, test)
simplified = individual.new(Individual.simplify)
reduced[version].append(len(simplified.active))
filecount += 1
except ValueError:
print filename, "FAILED"
# Kruskal's requires a rectangular matrix
rect = make_rectangular(statify.values(), 10000001)
print 'Files Successfully Loaded', filecount
print 'Kruskal Wallis', kruskalwallis(rect)
for version, data in statify.iteritems():
print '--------- %s ---------' % str(version)
print "MES, MAD", median_deviation(data)
print 'Active', median_deviation(active[version])
print 'Reduced', median_deviation(reduced[version])
print 'Mann Whitney U against Control',
print mannwhitneyu(statify[control_group], data)
# Determine the settings from the filename
problem, dup, ordering, nodes, mut, seed = base.split('_')
with open_file_method(filename)(filename, 'r') as f:
data = json.load(f)
version = dup, ordering, nodes, mut
if (dup, ordering) == ('skip', 'normal'):
control_group = version
statify[version].append(data[1]['evals'])
active[version].append(data[1]['phenotype'])
best = data[1]['bests'][-1]
test = data[1]['test_inputs']
individual = Individual.reconstruct_individual(best, test)
simplified = individual.new(Individual.simplify)
reduced[version].append(len(simplified.active))
filecount += 1
except ValueError:
print(filename, "FAILED")
# Kruskal's requires a rectangular matrix
rect = make_rectangular(list(statify.values()), 10000001)
print('Files Successfully Loaded', filecount)
print('Kruskal Wallis', kruskalwallis(rect))
for version, data in statify.items():
print('--------- %s ---------' % str(version))
print("MES, MAD", median_deviation(data))
print('Active', median_deviation(active[version]))
print('Reduced', median_deviation(reduced[version]))
print('Mann Whitney U against Control', end=' ')
print(mannwhitneyu(statify[control_group], data))
outname = base.split('_')[:3]
with open_file_method(filename)(filename, 'r') as f:
data = json.load(f)
graph_length = float(data[1]['bests'][-1]['graph_length'])
percentages.append(data[1]['unused'] / graph_length)
stripped = {}
best_worst = max(best_worst, data[1]['bests'][0]['fitness'])
for best in data[1]['bests']:
stripped[best['fitness']] = list(map(int, best['never_active']))
levels.add(best['fitness'])
storage.append(stripped)
filecount += 1
except ValueError:
print(filename, "FAILED")
print("Loaded", filecount)
print("Median final never active", median_deviation(percentages))
# Limits the number of unique fitness levels to plot
ysteps = 40
scan_lines = [x for x in sorted(levels) if x >= best_worst]
if len(scan_lines) > ysteps:
step = float(len(scan_lines)) / ysteps
print('Step', step, 'highest', int(step * (ysteps - 1)), len(scan_lines))
scan_lines = [scan_lines[int(x * step)] for x in range(ysteps)]
# Ensure that the maximum fitness is always added to the scan lines
if scan_lines[-1] != max(levels):
scan_lines.append(max(levels))
combined = defaultdict(list)
for stored in storage:
index = 0
fitnesses, values = list(zip(*sorted(stored.items())))
outname = base.split('_')[:3]
with open_file_method(filename)(filename, 'r') as f:
data = json.load(f)
graph_length = float(data[1]['bests'][-1]['graph_length'])
percentages.append(data[1]['unused'] / graph_length)
stripped = {}
best_worst = max(best_worst, data[1]['bests'][0]['fitness'])
for best in data[1]['bests']:
stripped[best['fitness']] = map(int, best['never_active'])
levels.add(best['fitness'])
storage.append(stripped)
filecount += 1
except ValueError:
print filename, "FAILED"
print "Loaded", filecount
print "Median final never active", median_deviation(percentages)
# Limits the number of unique fitness levels to plot
ysteps = 40
scan_lines = [x for x in sorted(levels) if x >= best_worst]
if len(scan_lines) > ysteps:
step = float(len(scan_lines)) / ysteps
print 'Step', step, 'highest', int(step * (ysteps - 1)), len(scan_lines)
scan_lines = [scan_lines[int(x * step)] for x in range(ysteps)]
# Ensure that the maximum fitness is always added to the scan lines
if scan_lines[-1] != max(levels):
scan_lines.append(max(levels))
combined = defaultdict(list)
for stored in storage:
index = 0
fitnesses, values = zip(*sorted(stored.items()))
from os import path
from collections import defaultdict
from util import pretty_name, median_deviation
from scipy.stats.mstats import kruskalwallis
if __name__ == '__main__':
# Run through all of the files gathering different seeds into lists
statify = defaultdict(list)
active = defaultdict(list)
filecount = 0
for filename in sys.argv[1:]:
base = path.basename(filename)
try:
problem, nodes, version, seed = base.split('_')
with open(filename, 'r') as f:
data = json.load(f)
statify[version].append(data[1]['evals'])
active[version].append(data[1]['phenotype'])
filecount += 1
except ValueError:
print filename, "FAILED"
print 'Files Successfully Loaded', filecount
print 'Kruskal Wallis', kruskalwallis(statify.values())
for version, data in statify.iteritems():
print '--------- %s ---------' % pretty_name[version]
print "MES, MAD", median_deviation(data)
print 'Active', median_deviation(active[version])
print 'Mann Whitney U against Normal',
print stats.mannwhitneyu(statify['normal'], data)
