def mutate_microbial_diversity_distance(songs, conf, i_night=None,
datasaver=None):
"""microbial genetic algorithm.
fitness objective: pressure for diversity: maximize the mean distance
from the others songs"""
if datasaver is None:
datasaver = QuietDataSaver()
songs = np.asarray(songs)
nb_replay = conf['replay']
rng = conf['rng_obj']
mat_neigh_dist = generate_mat_neighbours_distances(songs)
# datasaver.add(cond="init_mat_neigh_dist",
# mat_neigh_dist=mat_neigh_dist)
for i in range(nb_replay):
picked_songs = rng.choice(len(songs), size=2, replace=False)
best_id = np.argmax(np.mean(mat_neigh_dist[picked_songs], axis=1))
loser_id = 1 - best_id
new_song = songs[picked_songs[best_id]].mutate()
songs[picked_songs[loser_id]] = new_song
mat_neigh_dist = update_mat_neighbours_distances(mat_neigh_dist,
picked_songs[loser_id],
songs)
# datasaver.add(cond="after_a_replay", i_night=i_night, i_replay=i,
# i_loser=picked_songs[loser_id],
# i_winner=picked_songs[best_id],
# mat_neigh_dist=mat_neigh_dist)
return songs
def optimise_gesture_whole(songs, goal, conf, datasaver=None, iday=None):
"""Optimise gestures randomly from the song models with dummy algorithm.
Include the previous and next gesture in the evaluation to remove
border issues.
"""
measure = conf['measure_obj']
comp = conf['comp_obj']
train_per_day = conf['train_per_day']
rng = conf['rng_obj']
tutor_song = songs[0].song # just for the assert
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
for itrain in range(train_per_day):
isong = rng.randint(len(songs))
song = songs[isong]
ig = rng.randint(len(song.gestures))
s = song.gen_sound()
assert len(tutor_song) == len(s), "%d %d" % (len(tutor_song), len(s))
c = measure(s)
pre_score = comp(goal, c)
logger.info(
'{}/{}: fit gesture {} of song {} (length {}, score {})'.format(
itrain + 1, train_per_day, ig, isong, len(s), pre_score))
res, hill_score = fit_gesture_whole(goal, song, ig, conf)
# datasaver.add(pre_score=pre_score,
# new_score=hill_score, isong=isong, ig=ig)
songs[isong].gestures[ig][1] = deepcopy(res)
logger.info('new score {}'.format(hill_score))
assert pre_score >= hill_score, "{} >= {} est faux".format(
pre_score, hill_score)
return songs
def mutate_microbial_diversity(songs, goal, cur_day, nb_day,
conf, datasaver=None):
"""Microbial GA implementation for the songs."""
if datasaver is None:
datasaver = QuietDataSaver()
songs = np.asarray(songs)
measure = conf['measure_obj']
comp = conf['comp_obj']
nb_replay = conf['replay']
rng = conf['rng_obj']
threshold = conf.get('diversity_threshold', 2000)
# bloat_weight = conf.get('bloat_weight', 0)
diversity_weight = conf.get('diversity_weight', 1)
# diversity_decay = conf.get('decay', None)
# if diversity_decay == 'linear':
# diversity_weight = diversity_weight * (1 - (cur_day / (nb_day - 1)))
for i in range(nb_replay):
picked_songs = rng.choice(len(songs), size=2, replace=False)
scores = get_scores(goal, songs[picked_songs], measure, comp)
nb_similar = genetic_neighbours(songs[picked_songs], songs, threshold)
# TODO: to choose: if used or not
# Penalise if too much gestures
# nb_gestures = np.array([len(songs[picked_songs[0]].gestures),
# len(songs[picked_songs[1]].gestures)])
# best_id = np.argmin(scores * (nb_similar**diversity_weight)
# * (nb_gestures**bloat_weight))
best_id = np.argmin(scores * (nb_similar**diversity_weight))
loser_id = 1 - best_id # if best_id == 0: loser_id = 1, else: loser_id = 0
songs[picked_songs[loser_id]] = songs[picked_songs[best_id]].mutate()
return songs
def optimise_gesture_whole_local_search(songs,
goal,
conf,
datasaver=None,
iday=None):
"""
Optimise gestures randomly from the song models
with a stochastic local search
"""
measure = conf['measure_obj']
comp = conf['comp_obj']
train_per_day = conf['train_per_day']
rng = conf['rng_obj']
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
improvement_cpt = np.zeros(train_per_day)
for itrain in range(train_per_day):
isong = rng.randint(len(songs))
song = songs[isong]
ig = rng.randint(len(song.gestures))
s = song.gen_sound()
c = measure(s)
pre_score = comp(goal, c)
logger.info(
'{}/{}: fit gesture {} of song {} (length {}, score {})'.format(
itrain + 1, train_per_day, ig, isong, len(s), pre_score))
res, hill_score = fit_gesture_whole_local_search(goal, song, ig, conf)
songs[isong].gestures[ig][1] = deepcopy(res)
# datasaver.add(iday=iday, itrain=itrain, isong=isong, ig=ig,
# pre_score=pre_score, new_score=hill_score) # too much verbose
logger.info('new score {}'.format(hill_score))
assert pre_score >= hill_score, "{} >= {} est faux".format(
pre_score, hill_score)
if hill_score < pre_score:
improvement_cpt[itrain] += 1
datasaver.add(label='day',
cond='after_day_learning',
improvement_cpt=improvement_cpt)
return songs
def extend_pop(songs, conf, datasaver=None):
"""Extend the size of a population."""
if datasaver is None:
datasaver = QuietDataSaver()
new_pop_size = conf['night_concurrent']
rng = conf['rng_obj']
songs = np.asarray(songs)
size_pop_to_add = new_pop_size - len(songs)
night_pop = rng.choice(songs, size=size_pop_to_add, replace=True)
night_pop = np.array([song.mutate() for song in night_pop])
night_pop = np.concatenate((songs, night_pop))
return night_pop
def mutate_best_models_elite(songs, goal, conf,
datasaver=None):
"""Elite selection and mutation of the best models.
Keep the best mutations after each replay, parents are present in the
selection.
"""
if datasaver is None:
datasaver = QuietDataSaver()
measure = conf['measure_obj']
comp = conf['comp_obj']
nb_replay = conf['replay']
rng = conf['rng_obj']
nb_conc_song = len(songs)
pscore = get_scores(goal, songs, measure, comp)
score = pscore
nb_conc_night = nb_conc_song * 2
# make night_songs an array to do list indexes.
night_songs = np.array(songs)
for dummy_i in range(nb_replay):
fitness = len(night_songs) - rank(score)
night_songs = np.random.choice(night_songs, size=nb_conc_night,
p=fitness/np.sum(fitness))
night_songs = np.array([song.mutate() for song in night_songs])
score = get_scores(goal, night_songs, measure, comp)
fitness = len(night_songs) - rank(score)
isongs = rng.choice(len(night_songs),
size=nb_conc_song, replace=False,
p=fitness/np.sum(fitness))
night_songs = night_songs[isongs]
score = score[isongs]
nsongs = night_songs.tolist()
datasaver.add(prev_songs=songs, prev_scores=pscore, new_songs=nsongs,
new_scores=score)
return nsongs
def mutate_best_models_dummy(songs, goal, measure, comp, nb_replay,
datasaver=None, rng=None):
"""Dummy selection and mutation of the best models."""
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
nb_conc_song = len(songs)
night_songs = np.array(songs)
pscore = get_scores(goal, songs, measure, comp)
nb_conc_night = nb_conc_song * 2
fitness = len(night_songs) - rank(pscore)
night_songs = np.random.choice(night_songs, size=nb_conc_night,
p=fitness/np.sum(fitness))
night_songs = np.array([song.mutate(nb_replay) for song in night_songs])
score = get_scores(goal, night_songs, measure, comp)
fitness = len(night_songs) - rank(score)
isongs = rng.choice(len(night_songs),
size=nb_conc_song, replace=False,
p=fitness/np.sum(fitness))
nsongs = night_songs[isongs].tolist()
datasaver.add(prev_songs=songs, prev_scores=pscore, new_songs=nsongs,
new_scores=score[isongs])
return nsongs
def mutate_microbial(songs, goal, conf, datasaver=None):
"""Microbial GA implementation for the songs."""
if datasaver is None:
datasaver = QuietDataSaver()
songs = np.asarray(songs)
measure = conf['measure_obj']
comp = conf['comp_obj']
nb_replay = conf['replay']
rng = conf['rng_obj']
for i in range(nb_replay):
picked_songs = rng.choice(len(songs), size=2, replace=False)
scores = get_scores(goal, songs[picked_songs], measure, comp)
best = np.argmin(scores)
loser = 1 - best # if best = 0, loser = 1, else: loser = 0
songs[picked_songs[loser]] = songs[picked_songs[best]].mutate()
return songs
def optimise_gesture_padded(songs,
tutor_song,
conf,
datasaver=None,
iday=None):
"""Optimise gestures randomly from the song models with dummy algorithm.
Include the previous and next gesture in the evaluation to remove
border issues.
"""
nb_pad = conf.get('nb_pad', 2)
measure = conf['measure_obj']
comp = conf['comp_obj']
train_per_day = conf['train_per_day']
rng = conf['rng_obj']
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
for itrain in range(train_per_day):
isong = rng.randint(len(songs))
song = songs[isong]
ig = rng.randint(len(song.gestures))
if ig - nb_pad >= 0: # TODO: a confirmer : le padding permet de prendre un geste plus long pour eviter les effets de bords. Oui a priori
start = song.gestures[ig - nb_pad][0]
else:
start = 0
end = song.gesture_end(
ig + nb_pad
) # the potential index out of bound is handled by the gesture_end function
logger.info('{}/{}: fit gesture {} of song {} (length {})'.format(
itrain + 1, train_per_day, ig, isong, end - start))
g = measure(tutor_song[start:end])
range_ = range(max(0, ig - nb_pad),
min(len(song.gestures) - 1, ig + nb_pad) + 1)
s = song.gen_sound(range_)
assert len(
tutor_song[start:end]) == len(s), "%d %d" % (end - start, len(s))
c = measure(s)
pre_score = comp(g, c)
res, hill_score = fit_gesture_padded(tutor_song, song, ig, conf)
# datasaver.add(pre_score=pre_score,
# new_score=hill_score, isong=isong, ig=ig)
songs[isong].gestures[ig][1] = deepcopy(res)
assert pre_score >= hill_score, "{} >= {} est faux".format(
pre_score, hill_score)
return songs
def optimise_root_mean_square_error(songs,
tutor_song,
conf,
datasaver=None,
iday=None):
"""Optimises the gestures so that each sample of the song model
is as close as possible to the corresponding sample in the tutor song.
Only use the root mean square error and not some features of the song.
"""
comp = conf["comp_obj"]
train_per_day = conf["train_per_day"]
nb_iter = conf["iter_per_train"]
rng = conf["rng_obj"]
deviation = np.diag(conf["dev"])
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
for itrain in range(train_per_day):
isong = rng.randint(len(songs))
song = songs[isong]
ig = rng.randint(len(song.gestures))
s = song.gen_sound()
pre_score = comp(tutor_song, s)
logger.info(
'{}/{}: fit gesture {} of song {} (length {}, score {})'.format(
itrain + 1, train_per_day, ig, isong, len(s), pre_score))
best_gest = deepcopy(song.gestures[ig][1])
best_score = pre_score
i = 0
while best_score > 0.01 and i < nb_iter:
new_gest = rng.multivariate_normal(best_gest, deviation)
new_sound = _padded_gen_sound(song, range(0, len(song.gestures)),
ig, new_gest)
new_score = comp(tutor_song, new_sound)
if new_score < best_score:
best_score = new_score
best_gest = new_gest
i += 1
song.gestures[ig][1] = deepcopy(best_gest)
# datasaver.add(iday=iday, itrain=itrain, isong=isong, ig=ig,
# pre_score=pre_score, new_score=best_score) # too much verbose
logger.info("new score {}".format(best_score))
return songs
def optimise_gesture_dummy(songs,
tutor_song,
measure,
comp,
train_per_day=10,
nb_iter_per_train=10,
datasaver=None,
rng=None):
"""Optimise gestures randomly from the song models with dummy algorithm."""
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
for itrain in range(train_per_day):
isong = rng.randint(len(songs))
song = songs[isong]
ig = rng.randint(len(song.gestures))
start = song.gestures[ig][0]
try:
end = song.gestures[ig + 1][0]
except IndexError: # We have picked the last gesture
end = len(tutor_song)
logger.info('{}/{}: fit gesture {} of song {} (length {})'.format(
itrain + 1, train_per_day, ig, isong, end - start))
prior = deepcopy(song.gestures[ig][1])
g = measure(tutor_song[start:end])
s = gen_sound(prior,
end - start,
falpha=lambda x, p: only_sin(x, p, nb_sin=3),
fbeta=lambda x, p: only_sin(x, p, nb_sin=1),
falpha_nb_args=13)
c = measure(s)
pre_score = comp(g, c)
res, hill_score = fit_gesture_hill(tutor_song[start:end].copy(),
measure,
comp,
start_prior=prior,
nb_iter=nb_iter_per_train,
temp=None,
rng=rng)
# datasaver.add(pre_score=pre_score,
# new_score=hill_score, isong=isong, ig=ig)
songs[isong].gestures[ig][1] = deepcopy(res)
assert pre_score >= hill_score
return songs
def optimise_proportional_training(songs,
goal,
conf,
datasaver=None,
iday=None):
"""
Optimise gestures randomly from the song models
with a stochastic local search
The number of trainings for this day is proportional to
the total number of gestures in the day songs
"""
measure = conf['measure_obj']
comp = conf['comp_obj']
rng = conf['rng_obj']
train_multiplier = conf.get('train_multiplier', 3)
if datasaver is None:
datasaver = QuietDataSaver()
if rng is None:
rng = np.random.RandomState()
train_per_day = 0
for song in songs:
train_per_day += len(song.gestures)
train_per_day *= train_multiplier
train_per_day = round(
train_per_day) # avoid possible float value with train_multiplier < 1
datasaver.add(label='day',
cond='define_number_of_trainings',
train_per_day=train_per_day)
improvement_cpt = np.zeros(train_per_day)
for itrain in range(train_per_day):
isong = rng.randint(len(songs))
song = songs[isong]
ig = rng.randint(len(song.gestures))
s = song.gen_sound()
c = measure(s)
pre_score = comp(goal, c)
logger.info(
'{}/{}: fit gesture {} of song {} (length {}, score {})'.format(
itrain + 1, train_per_day, ig, isong, len(s), pre_score))
res, hill_score = fit_gesture_whole_local_search(goal, song, ig, conf)
songs[isong].gestures[ig][1] = deepcopy(res)
# datasaver.add(iday=iday, itrain=itrain, isong=isong, ig=ig,
# pre_score=pre_score, new_score=hill_score) # too much verbose
logger.info('new score {}'.format(hill_score))
assert pre_score >= hill_score, "{} >= {} est faux".format(
pre_score, hill_score)
if hill_score < pre_score:
improvement_cpt[itrain] += 1
datasaver.add(label='day',
cond='after_day_learning',
improvement_cpt=improvement_cpt)
return songs
def fit_song(tutor_song, conf, datasavers=None):
"""Fit a song with a day and a night phase.
This function returns a list of SongModel.
The fit is split in two phases: A day part and a night part. The day part
is a simple optimisation algorithm within gesture. The night part
is a restructuring algorithm. See details in the modules
`song_model.SongModel`, `day_optimisers` and `night_optimisers`
Parameters
----------
tutor_song : 1D array_like
The tutor song that the algorithm will try to reproduce.
It will be normalized between -1 and +1 internally.
You don't need to do it yourself.
conf : dict
The dictionnary of all the parameters needed for the run.
Values that are required with `fit_song are`:
'dlm': The day learning model key from DAY_LEARNING_MODELS dict.
'nlm': The night learning model key from NIGHT_LEARNING_MODELS dict.
'days': The number of day for a run
'concurrent': The number of concurrent songs during the day.
'comp_obj': a callable for the comparison.
'rng_obj': a `numpy.RandomState` object for the random generation.
'measure_obj': a callable to measure song features.
'comp_obj', 'rng_obj' and 'measure_obj' are not importable from json
files, but can be built easily by reading arguments like 'seed' or keys
from the configuration files, like 'dlm' and 'nlm'.
The required values depend on the day learning model and night
learning model picked.
Returns
-------
songmodels : List[SongModel]
The songmodels at the end of the training.
See also
--------
song_model.SongModel
day_optimisers
night_optimisers
"""
tutor_song = normalize_and_center(tutor_song)
tutor_feat = bsa.all_song_features(tutor_song,
bsa.SR,
freq_range=bsa.FREQ_RANGE,
fft_step=bsa.FFT_STEP,
fft_size=bsa.FFT_SIZE)
conf['measure_obj'] = lambda x: bsa_measure(
x, bsa.SR, coefs=conf['coefs'], tutor_feat=tutor_feat)
day_optimisation = DAY_LEARNING_MODELS[conf['dlm']]
night_optimisation = NIGHT_LEARNING_MODELS[conf['nlm']]
nb_day = conf['days']
nb_conc_song = conf['concurrent']
measure = conf['measure_obj']
comp = conf['comp_obj']
rng = conf['rng_obj']
nb_split = conf.get('split', 10)
# muta_proba is a list of 3 values: [P(deletion), P(division), P(movement)]
muta_proba = conf['muta_proba']
songs = [
SongModel(song=tutor_song,
priors=conf['prior'],
nb_split=nb_split,
rng=rng,
muta_proba=muta_proba) for i in range(nb_conc_song)
]
goal = measure(tutor_song)
if datasavers is None:
datasavers = {}
datasavers["standard"] = QuietDataSaver()
datasavers["day"] = QuietDataSaver()
datasavers["night"] = QuietDataSaver()
datasavers["standard"].add(moment='Start',
songs=songs,
scores=get_scores(goal, songs, measure, comp))
cond1 = conf['dlm'] == 'optimise_gesture_whole'
cond2 = conf['dlm'] == 'optimise_gesture_whole_local_search'
cond3 = conf['dlm'] == 'optimise_proportional_training'
if cond1 or cond2 or cond3:
target = goal
else:
# case where conf['dlm'] == 'optimise_root_mean_square_error'
target = tutor_song
for iday in range(nb_day):
logger.info('*\t*\t*\tDay {} of {}\t*\t*\t*'.format(iday + 1, nb_day))
with datasavers["day"].set_context('day_optim'):
songs = day_optimisation(songs,
target,
conf,
datasaver=datasavers["day"],
iday=iday)
datasavers["day"].flush(
) # Write the data in several times, otherwise it is too big and cause MemoryError. It means it has to be extract from the pickle file differently
score = get_scores(goal, songs, measure, comp)
if iday + 1 != nb_day:
logger.debug(score)
datasavers["standard"].add(moment='before_night',
songs=songs,
scores=score)
logger.info('z\tz\tz\tNight\tz\tz\tz')
with datasavers["standard"].set_context('night_optim'):
with datasavers["night"].set_context('replay'):
if conf['nlm'] == "no_night":
pass
# kind of "multi objective diversity" by minimising the number of neighbours of a song and also keep good song with low error distance
elif conf['nlm'] == "mutate_microbial_diversity_uniform":
songs = night_optimisation(songs,
goal,
iday,
nb_day,
conf,
datasavers=datasavers)
# only diversity by maximising the metric between songs, metric not symmetric
elif conf[
'nlm'] == "mutate_microbial_diversity_continuous_uniform":
songs = night_optimisation(songs,
conf,
i_night=iday,
datasavers=datasavers)
# only diversity by maximising the distance between songs, using a symmetrical distance
elif conf[
'nlm'] == "mutate_microbial_diversity_distance_uniform":
songs = night_optimisation(songs,
conf,
i_night=iday,
datasavers=datasavers)
else:
songs = night_optimisation(
songs,
goal,
conf,
datasaver=datasavers["standard"])
score = get_scores(goal, songs, measure, comp)
datasavers["night"].flush(
) # Write the data in several times, otherwise it is too big and cause MemoryError. It means it has to be extract from the pickle file differently
datasavers["standard"].add(moment='after_night',
songs=songs,
scores=score)
datasavers["standard"].write()
datasavers["standard"].add(moment='End',
songs=songs,
scores=get_scores(goal, songs, measure, comp))
return songs