def run_robustness_exp(max_gens, pop_size, archive_size, crossover_rate,
mutation_rate, mutation_value_rate, stations, **kwargs):
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in wave_watch_results(path_to_results='../../samples/ww-res/', stations=stations)]
train_model = FakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=stations,
error=error_rmse_all,
forecasts_path='../../../wind-postproc/out',
noise_run=0)
test_model = model_all_stations()
history, archive_history = SPEA2(
params=SPEA2.Params(max_gens=max_gens,
pop_size=pop_size,
archive_size=archive_size,
crossover_rate=crossover_rate,
mutation_rate=mutation_rate,
mutation_value_rate=mutation_value_rate),
init_population=initial_pop_lhs,
objectives=partial(calculate_objectives_interp, train_model),
crossover=crossover,
mutation=mutation).solution(verbose=False)
exptime2 = str(datetime.datetime.now().time()).replace(":", "-")
save_archive_history(archive_history, f'rob-exp-bl-{exptime2}.csv')
params = history.last().genotype
forecasts = []
if 'save_figures' in kwargs and kwargs['save_figures'] is True:
closest_hist = test_model.closest_params(params)
closest_params_set_hist = SWANParams(drf=closest_hist[0],
cfw=closest_hist[1],
stpm=closest_hist[2])
for row in grid.rows:
if set(row.model_params.params_list()) == set(
closest_params_set_hist.params_list()):
drf_idx, cfw_idx, stpm_idx = test_model.params_idxs(
row.model_params)
forecasts = test_model.grid[drf_idx, cfw_idx, stpm_idx]
break
plot_results(forecasts=forecasts,
observations=wave_watch_results(
path_to_results='../../samples/ww-res/',
stations=ALL_STATIONS),
baseline=default_params_forecasts(test_model),
save=True,
file_path=kwargs['figure_path'])
return history.last()
def run_genetic_opt(max_gens, pop_size, archive_size, crossover_rate,
mutation_rate, mutation_value_rate, stations, **kwargs):
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in wave_watch_results(path_to_results='../../samples/ww-res/', stations=stations)]
train_model = FidelityFakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=stations,
error=error_rmse_all,
forecasts_path='../../../wind-fidelity/*')
test_range = (0, 1)
test_model = model_all_stations(forecasts_range=test_range)
operators = default_operators()
history, archive_history = DefaultSPEA2(
params=DefaultSPEA2.Params(max_gens=max_gens,
pop_size=pop_size,
archive_size=archive_size,
crossover_rate=crossover_rate,
mutation_rate=mutation_rate,
mutation_value_rate=mutation_value_rate),
objectives=partial(calculate_objectives_interp, train_model),
evolutionary_operators=operators).solution(verbose=False)
exptime2 = str(datetime.datetime.now().time()).replace(":", "-")
# save_archive_history(archive_history, f'rob-exp-bl-{exptime2}.csv')
params = history.last().genotype
if 'save_figures' in kwargs and kwargs['save_figures'] is True:
params = test_model.closest_params(params)
closest_params = SWANParams(drf=params[0],
cfw=params[1],
stpm=params[2],
fidelity_time=params[3],
fidelity_space=params[4])
drf_idx, cfw_idx, stpm_idx, fid_time_idx, fid_space_idx = test_model.params_idxs(
closest_params)
forecasts = test_model.grid[drf_idx, cfw_idx, stpm_idx, fid_time_idx,
fid_space_idx]
plot_results(forecasts=forecasts,
observations=wave_watch_results(
path_to_results='../../samples/ww-res/',
stations=ALL_STATIONS),
baseline=default_params_forecasts(test_model),
save=True,
file_path=kwargs['figure_path'],
values_range=test_range)
return history.last()
def error_grid(noise_case=0):
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
stations = [1, 2, 3, 4, 5, 6, 7, 8, 9]
ww3_obs_all = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=stations)]
model_all = FakeModel(grid_file=grid,
observations=ww3_obs_all,
stations_to_out=stations,
error=error_rmse_all,
forecasts_path='../../../wind-postproc/out',
noise_run=noise_case)
with open(f'../../samples/params_rmse_{noise_case}.csv',
mode='w',
newline='') as csv_file:
writer = csv.writer(csv_file, delimiter=',')
header = ['DRF', 'CFW', 'STPM']
error_columns = [f'ERROR_K{station}' for station in stations]
header.extend(error_columns)
writer.writerow(header)
for row in grid.rows:
metrics = model_all.output(params=row.model_params)
row_to_write = row.model_params.params_list()
row_to_write.extend(metrics)
writer.writerow(row_to_write)
def run_evolution(sur_points, time_delta, space_delta, point_for_retrain, gens_to_change_fidelity, max_gens, pop_size,
archive_size, iter_id, deadline):
train_stations = [1, 2, 3]
initial_fidelity = (180, 56)
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=train_stations)]
train_model = FidelityFakeModel(grid_file=grid, observations=ww3_obs,
stations_to_out=train_stations, error=error_rmse_all,
forecasts_path='../../../2fidelity/*', forecasts_range=(0, 1),
sur_points=sur_points,
is_surrogate=True, initial_fidelity=initial_fidelity)
operators = default_operators()
handler = FidelityHandler(surrogates=train_model.surrogates_by_stations, time_delta=time_delta,
space_delta=space_delta,
point_for_retrain=point_for_retrain, gens_to_change_fidelity=gens_to_change_fidelity)
dyn_params = SPEA2.Params(max_gens=max_gens, pop_size=pop_size, archive_size=archive_size,
crossover_rate=0.7, mutation_rate=0.7,
mutation_value_rate=[0.1, 0.01, 0.001])
history, _, _ = DynamicSPEA2(
params=dyn_params,
objectives=partial(calculate_objectives_interp, train_model),
evolutionary_operators=operators,
fidelity_handler=handler).solution(verbose=True)
best = history.last()
fieldnames = ['ID', 'IterId', 'SetId', 'drf', 'cfw', 'stpm',
'rmse_all', 'rmse_peak', 'mae_all', 'mae_peak', 'deadline']
with open(os.path.join("C:\\metaopt2", "res-nondd.csv"), 'a', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
row_to_write = {'ID': run_id, 'IterId': iter_id, 'SetId': 0,
'drf': best.genotype.drf,
'cfw': best.genotype.cfw,
'stpm': best.genotype.stpm,
'deadline': deadline}
metrics = all_error_metrics(best.genotype, models_to_tests)
for metric_name in metrics.keys():
stations_metrics = metrics[metric_name]
stations_to_write = {}
for station_idx in range(len(stations_metrics)):
key = f'st{station_idx + 1}'
stations_to_write.update({key: stations_metrics[station_idx]})
row_to_write.update({metric_name: stations_to_write})
writer.writerow(row_to_write)
return history.last().error_value
def model_all_stations():
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=ALL_STATIONS)]
model = FakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=ALL_STATIONS,
error=error_rmse_all,
forecasts_path='../../../wind-postproc/out',
noise_run=0)
return model
def model_all_stations(forecasts_range=(0, 1)):
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=ALL_STATIONS)]
model = FidelityFakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=ALL_STATIONS,
error=error_rmse_all,
forecasts_path='../../../2fidelity/*',
forecasts_range=forecasts_range)
return model
def run_evolution():
train_stations = [1]
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=train_stations)]
initial_fidelity = (180, 56)
sur_points = 5
train_model = FidelityFakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=train_stations,
error=error_rmse_all,
forecasts_path='../../../2fidelity/*',
forecasts_range=(0, 1),
sur_points=sur_points,
is_surrogate=True,
initial_fidelity=initial_fidelity)
operators = default_operators()
handler = FidelityHandler(surrogates=train_model.surrogates_by_stations,
time_delta=15,
space_delta=7,
point_for_retrain=3,
gens_to_change_fidelity=10)
dyn_params = SPEA2.Params(max_gens=100,
pop_size=10,
archive_size=5,
crossover_rate=0.7,
mutation_rate=0.7,
mutation_value_rate=[0.1, 0.01, 0.001])
ex_time = DynamicSPEA2PerfModel.get_execution_time(sur_points,
initial_fidelity,
dyn_params, handler)
print(ex_time)
_, _, points_by_fid = DynamicSPEA2(
params=dyn_params,
objectives=partial(calculate_objectives_interp, train_model),
evolutionary_operators=operators,
fidelity_handler=handler,
points_by_fidelity=default_points_by_fidelity(size=5)).solution(
verbose=True)
points_by_fid
def prepare_all_fake_models():
errors = [error_rmse_all]
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
noises = [0, 1, 2, 15, 16, 17, 25, 26]
for noise in noises:
for err in errors:
for stations in stations_for_run_set:
print(
f'configure model for: noise = {noise}; error = {err}; stations = {stations}'
)
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=stations)]
_ = FakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=stations,
error=err,
forecasts_path=f'../../../wind-postproc/out',
noise_run=noise)
def run_genetic_opt(max_gens, pop_size, archive_size, crossover_rate,
mutation_rate, mutation_value_rate, sur_points, stations,
**kwargs):
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in wave_watch_results(path_to_results='../../samples/ww-res/', stations=stations)]
train_model = FidelityFakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=stations,
error=error_rmse_all,
forecasts_path='../../../2fidelity/*',
is_surrogate=True,
sur_points=sur_points,
initial_fidelity=(180, 56))
test_range = (0, 1)
test_model = model_all_stations(forecasts_range=test_range)
operators = default_operators()
handler = FidelityHandler(surrogates=train_model.surrogates_by_stations,
time_delta=30,
space_delta=14,
point_for_retrain=3,
gens_to_change_fidelity=10)
history, archive_history, _ = DynamicSPEA2(
params=SPEA2.Params(max_gens=max_gens,
pop_size=30,
archive_size=10,
crossover_rate=crossover_rate,
mutation_rate=mutation_rate,
mutation_value_rate=mutation_value_rate),
objectives=partial(calculate_objectives_interp, train_model),
evolutionary_operators=operators,
fidelity_handler=handler).solution(verbose=False)
exptime2 = str(datetime.datetime.now().time()).replace(":", "-")
# save_archive_history(archive_history, f'rob-exp-bl-{exptime2}.csv')
params = history.last().genotype
return history.last()
def init_models_to_tests():
metrics = {'rmse_all': error_rmse_all,
'rmse_peak': error_rmse_peak,
'mae_all': error_mae_all,
'mae_peak': error_mae_peak}
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=ALL_STATIONS)]
models = {}
for metric_name in metrics.keys():
model = FidelityFakeModel(grid_file=grid, observations=ww3_obs, stations_to_out=ALL_STATIONS,
error=metrics[metric_name],
forecasts_path='../../../2fidelity/*')
models[metric_name] = model
return models
def grid_rmse():
# fake, grid = real_obs_config()
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
stations = [1, 2, 3]
ww3_obs = \
[obs.time_series() for obs in wave_watch_results(path_to_results='../../samples/ww-res/', stations=stations)]
fake = FakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=stations,
error=error_rmse_peak,
forecasts_path='../../../wind-noice-runs/results_fixed/0',
noise_run=0)
errors_total = []
m_error = pow(10, 9)
with open('../../samples/params_rmse.csv', mode='w',
newline='') as csv_file:
writer = csv.writer(csv_file, delimiter=',')
writer.writerow([
'ID', 'DRF', 'CFW', 'STPM', 'RMSE_K1', 'RMSE_K2', 'RMSE_K3',
'TOTAL_RMSE'
])
for row in grid.rows:
error = fake.output(params=row.model_params)
print(grid.rows.index(row), error)
if m_error > rmse(error):
m_error = rmse(error)
print(f"new min: {m_error}; {row.model_params.params_list()}")
errors_total.append(rmse(error))
row_to_write = row.model_params.params_list()
row_to_write.extend(error)
row_to_write.append(rmse(error))
writer.writerow(row_to_write)
print(f'min total rmse: {min(errors_total)}')
def optimize_test(train_stations,
max_gens,
pop_size,
archive_size,
crossover_rate,
mutation_rate,
mutation_value_rate,
sur_points,
plot_figures=True):
train_range = (0, 1)
test_range = (0, 1)
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=train_stations)]
error = error_rmse_all
train_model = FidelityFakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=train_stations,
error=error,
forecasts_path='../../../2fidelity/*',
forecasts_range=train_range,
is_surrogate=True,
sur_points=sur_points)
operators = default_operators()
history, archive_history = DefaultSPEA2(
params=DefaultSPEA2.Params(max_gens,
pop_size=pop_size,
archive_size=archive_size,
crossover_rate=crossover_rate,
mutation_rate=mutation_rate,
mutation_value_rate=mutation_value_rate),
objectives=partial(calculate_objectives_interp, train_model),
evolutionary_operators=operators).solution(verbose=True)
params = history.last().genotype
if plot_figures:
test_model = model_all_stations(forecasts_range=test_range)
params = test_model.closest_params(params)
closest_params = SWANParams(drf=params[0],
cfw=params[1],
stpm=params[2],
fidelity_time=params[3],
fidelity_space=params[4])
drf_idx, cfw_idx, stpm_idx, fid_time_idx, fid_space_idx = test_model.params_idxs(
closest_params)
forecasts = test_model.grid[drf_idx, cfw_idx, stpm_idx, fid_time_idx,
fid_space_idx]
plot_results(forecasts=forecasts,
observations=wave_watch_results(
path_to_results='../../samples/ww-res/',
stations=ALL_STATIONS),
baseline=default_params_forecasts(test_model))
plot_population_movement(archive_history, grid)
return history.last().error_value
def run_evolution(sur_points, time_delta, space_delta, point_for_retrain,
gens_to_change_fidelity, max_gens, pop_size, archive_size):
train_stations = [1]
initial_fidelity = (180, 56)
#sur_points = 5
dyn_params = SPEA2.Params(max_gens=max_gens,
pop_size=pop_size,
archive_size=archive_size,
crossover_rate=0.7,
mutation_rate=0.7,
mutation_value_rate=[0.1, 0.01, 0.001])
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=train_stations)]
train_model = FidelityFakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=train_stations,
error=error_rmse_all,
forecasts_path='../../../2fidelity/*',
forecasts_range=(0, 1),
sur_points=sur_points,
is_surrogate=True,
initial_fidelity=initial_fidelity)
operators = default_operators()
handler = FidelityHandler(surrogates=train_model.surrogates_by_stations,
time_delta=15,
space_delta=7,
point_for_retrain=3,
gens_to_change_fidelity=10)
dyn_params = SPEA2.Params(max_gens=max_gens,
pop_size=pop_size,
archive_size=archive_size,
crossover_rate=0.7,
mutation_rate=0.7,
mutation_value_rate=[0.1, 0.01, 0.001])
# ex_time = DynamicSPEA2PerfModel.get_execution_time(sur_points, initial_fidelity, dyn_params, handler)
# print(ex_time)
history, _, _ = DynamicSPEA2(
params=dyn_params,
objectives=partial(calculate_objectives_interp, train_model),
evolutionary_operators=operators,
fidelity_handler=handler).solution(verbose=True)
fieldnames = [
'ID', 'deadline', 'sur_points', 'gens_to_change_fidelity', 'max_gens',
'pop_size', 'error'
]
with open(os.path.join("C:\\metaopt1", "res1.csv"), 'a',
newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
row_to_write = {
'ID': 0,
'deadline': deadline,
'sur_points': sur_points,
'gens_to_change_fidelity': gens_to_change_fidelity,
'max_gens': max_gens,
'pop_size': pop_size,
'error': history.last().error_value
}
writer.writerow(row_to_write)
return history.last().error_value
def optimize_by_ww3_obs(train_stations,
max_gens,
pop_size,
archive_size,
crossover_rate,
mutation_rate,
mutation_value_rate,
iter_ind,
plot_figures=True):
grid = CSVGridFile('../../samples/wind-exp-params-new.csv')
ww3_obs = \
[obs.time_series() for obs in
wave_watch_results(path_to_results='../../samples/ww-res/', stations=train_stations)]
error = error_rmse_all
train_model = FakeModel(grid_file=grid,
observations=ww3_obs,
stations_to_out=train_stations,
error=error,
forecasts_path='../../../wind-postproc/out',
noise_run=0)
history, archive_history = SPEA2(
params=SPEA2.Params(max_gens,
pop_size=pop_size,
archive_size=archive_size,
crossover_rate=crossover_rate,
mutation_rate=mutation_rate,
mutation_value_rate=mutation_value_rate),
init_population=initial_pop_lhs,
objectives=partial(calculate_objectives_interp, train_model),
crossover=crossover,
mutation=mutation).solution(verbose=True)
params = history.last().genotype
save_archive_history(archive_history, f'history-{iter_ind}.csv')
if plot_figures:
test_model = model_all_stations()
closest_hist = test_model.closest_params(params)
closest_params_set_hist = SWANParams(drf=closest_hist[0],
cfw=closest_hist[1],
stpm=closest_hist[2])
forecasts = []
for row in grid.rows:
if set(row.model_params.params_list()) == set(
closest_params_set_hist.params_list()):
drf_idx, cfw_idx, stpm_idx = test_model.params_idxs(
row.model_params)
forecasts = test_model.grid[drf_idx, cfw_idx, stpm_idx]
if grid.rows.index(row) < 100:
print("!!!")
print("index : %d" % grid.rows.index(row))
break
plot_results(forecasts=forecasts,
observations=wave_watch_results(
path_to_results='../../samples/ww-res/',
stations=ALL_STATIONS),
baseline=default_params_forecasts(test_model))
plot_population_movement(archive_history, grid)
return history
