def collect_results(self, parameters, score):
# Get the particle for these parameters.
for particle in self.swarm:
if particle.parameters == parameters:
break
else:
raise Exception("Unrecognized parameters!")
if isinstance(score, Exception) or math.isnan(score):
# Program crashed, replace this particle.
if particle.best is not None:
particle.parameters = ParameterSet(particle.best)
elif self.best is not None:
particle.parameters = ParameterSet(self.best)
else:
particle.parameters = ParameterSet(self.lab.default_parameters)
particle.initialize_velocities(self.swarm)
particle.update_position()
else:
# Update with results of this particles evaluation.
if self.best_score is not None:
self.best_score *= 1 - score_decay_rate / len(self.swarm)
if self.best is None or score > self.best_score:
self.best = ParameterSet(particle.parameters)
self.best_score = score
print("New global best score %g." % score)
particle.update(score, self.best)
particle.lock = False
self.save()
def _parse(self, report):
if not report.strip():
raise ValueError("Empty lab report file!")
sections = report.split(self.section_divider)
self.header = sections[0]
default_parameters = '\n'.join( sections[1].split('\n')[1:-1] )
cli = sections[1].split('\n')[-1].strip('$ ').split()
sorted_pval_table = sections[2]
experiment_sections = sections[3:]
file_defaults = ParameterSet(default_parameters)
# Consistency check for parameters & experiment argv.
if file_defaults != self.default_parameters or cli != self.argv:
while True:
q = input("Default parameters or invocation have changed, options:\n" +
" old - Ignore the new/given, use what's on file.\n" +
" new - Use the new/given, overwrites the old file!\n" +
" abort.\n" +
">>> ")
q = q.strip().lower()
if q == 'old':
self.default_parameters = file_defaults
self.argv = cli
break
elif q == 'new':
shutil.copy(self.lab_report, self.lab_report + '.backup')
break
elif q in ('abort', 'exit', 'quit') or q in 'aeq':
sys.exit()
[Experiment(self, s) for s in experiment_sections if s.strip()]
def _parse(self, string):
if "Notes:" in string:
string, _, self.notes = string.partition('Notes:')
# Reconstruct the parameters.
self.modifications = []
for change in re.findall(r"^[Mm]od.*:(.*)$", string, re.MULTILINE):
path, eq, value = change.partition('=')
self.modifications.append((path, value))
self.parameters = ParameterSet(self.lab.default_parameters)
for path, value in self.modifications:
self.parameters.apply(path, value)
if "Hash: " in string:
# Override hash(self) with whats on file since this is reconstructed
# from defaults + modifications, and the defaults might have changed.
self._hash = int(re.search("Hash: (.*)", string).groups()[0],
base=16)
if "Journal: " in string:
self.journal = re.search("Journal: (.*)", string).groups()[0]
if "Attempts: " in string:
self.attempts = int(
re.search("Attempts: (.*)", string).groups()[0])
if "Scores: " in string:
self.scores = re.search("Scores: (.*)", string).groups()[0].strip()
self.scores = [
float(s.strip()) for s in self.scores.split(',') if s.strip()
]
assert (len(self.scores) <= self.attempts
) # Attempts may fail and not return a score.
def initialize_velocities(self, swarm=None):
# Make a new parameter structure for the velocity data.
self.velocities = ParameterSet(self.parameters)
# Iterate through every field in the structure.
for path in self.parameters.enumerate():
value = self.parameters.get(path)
if swarm is not None:
# Analyse the other particle velocities, so that the new
# velocity is not too large or too small.
data = [p.velocities.get(path) for p in swarm if p is not self]
velocity = np.random.normal(np.mean(data), np.std(data))
else:
# New swarm, start with a large random velocity.
max_percent_change = .10
uniform = 2 * random.random() - 1
if isinstance(value, float):
velocity = value * uniform * max_percent_change
elif isinstance(value, int):
if abs(value) < 1. / max_percent_change:
velocity = uniform # Parameters are rounded, so 50% chance this will mutate.
else:
velocity = value * uniform * max_percent_change
else:
raise NotImplementedError()
self.velocities.apply(path, velocity)
def __init__(self, initial_parameters, swarm=None):
self.parameters = ParameterSet(initial_parameters)
self.best = None
self.best_score = None
self.age = 0
self.initialize_velocities(swarm)
self.lock = False
def __init__(
self,
lab,
string=None,
modifications=None,
parameters=None,
):
""" """
self.lab = lab
self.attempts = 0
self.scores = []
self.notes = ' '
# Load or create this experiment's data.
if string is not None:
self._parse(string)
elif modifications is not None:
self.parameters = ParameterSet(self.lab.default_parameters)
for path, value in modifications:
self.parameters.apply(path, value)
elif parameters is not None:
self.parameters = ParameterSet(parameters)
else:
raise TypeError("Not enough arguments to Experiment.__init__()")
self.parameters = self.parameters.typecast(self.lab.structure)
self.modifications = self.lab.default_parameters.diff(self.parameters)
if hash(self) not in self.lab.experiment_ids:
self.lab.experiments.append(self)
self.lab.experiment_ids[hash(self)] = self
else:
existing = self.lab.experiment_ids[hash(self)]
if existing.parameters == self.parameters:
raise ValueError("Duplicate Parameters, Hash %X" % hash(self))
else:
raise SystemExit("Hash Collision!")
# Start a journal file for this experiment.
if not hasattr(self, 'journal'):
self.journal = os.path.join(self.lab.ae_directory,
"%X.journal" % hash(self))
with open(self.journal, 'a') as file:
file.write('Experiment Journal For Parameters:\n')
file.write(str(self.parameters) + '\n')
file.write('Hash: %X\n' % hash(self))
file.write('Command Line Invocation: $ ' +
' '.join(self.lab.argv) + '\n')
def update(self, score, global_best):
self.age += 1
if self.best_score is not None:
self.best_score *= 1 - score_decay_rate
if self.best is None or score > self.best_score:
self.best = ParameterSet(self.parameters)
self.best_score = score
print("New particle best score %g." % self.best_score)
self.update_position()
self.update_velocity(global_best)
def get_experiment(self, parameters):
"""
Returns Experiment instance for the given parameters. If one does not
already exist for these parameter then it is created.
"""
if isinstance( parameters, Experiment ):
return parameters
p = ParameterSet( parameters ).typecast( self.structure )
h = hash(p)
if h in self.experiment_ids:
return self.experiment_ids[h]
else:
return Experiment(self, parameters=p)
def _load_experiment_module(self, experiment_module):
"""
Argument experiment_module is command line argument 0, specifying the
file path to the experiment module.
"""
self.path, experiment_module = os.path.split(experiment_module)
self.name, dot_py = os.path.splitext(experiment_module)
assert(dot_py == '.py')
self.module_reload = 'import sys; sys.path.append("%s"); '%self.path
self.module_reload += 'import %s; '%self.name
exec_globals = {}
exec(self.module_reload, exec_globals)
self.module = exec_globals[self.name]
self.default_parameters = ParameterSet(self.module.default_parameters)
self.structure = self.default_parameters.get_types()
def merge(self, lab, ideas):
""" Take several experiments and return the best combination of them. """
# Marshal all of the modifications together.
ideas = sorted(ideas, key=lambda x: -x.mean())
paths = []
values = []
for x in ideas:
for path, value in x.modifications:
if path in paths:
continue # Higher scoring experiments take precedence.
paths.append(path)
values.append(value)
# Create and get the experiment object.
params = ParameterSet(lab.default_parameters)
for p, v in zip(paths, values):
params.apply(p, v)
return lab.get_experiment(params)
def __init__(self, laboratory, args):
self.lab = laboratory
# Get a list of every parameter to experiment with.
target_parameters = []
for start in args.grid_search.split(','):
node = self.lab.default_parameters.get(start)
subtree = ParameterSet.enumerate(node)
target_parameters.extend(start + end for end in subtree)
# Suggest modifications to each parameter.
self.experiments = []
for path in target_parameters:
value = self.lab.default_parameters.get(path)
for mod in self.mod_funcs:
params = ParameterSet(self.lab.default_parameters)
params.apply(path, mod(value))
X = self.lab.get_experiment(params)
if not X.notes.strip():
X.notes += "Suggested by Grid Search.\n"
self.experiments.append(X)
self.lab.save(
) # Write all of the new grid-search experiments to the lab report.