def freeze_all_but_crossability(self):
for layer in self.model.layers:
if not layer.name == "crossability_block" and not layer.__class__.__name__ == "InputLayer":
layer.trainable = False
Verbose.print_text(
1,
"[INFO] " + layer.name.ljust(26) + " set to non-trainable")
def __export_spots_forecasts(
spots_and_prediction,
filename): #[Spot(name, lat, lon, prediction), ...]
Verbose.print_arguments()
content = """
{
"type": "FeatureCollection",
"features": [
"""
sep = ""
for ks, s in enumerate(spots_and_prediction):
content += sep + """
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [""" + str(s.lon) + "," + str(s.lat) + """]
},
"properties": {
"id": \"""" + str(s.id) + """\",
"name": \"""" + Forecast.__fix_spots_name(s.name).replace(
'"', '\\"') + """\",
"nbFlights": """ + str(s.nbFlights) + """,
"flyability": """ + str(s.prediction) + """
}
}
"""
sep = ","
content += "]}"
with open(filename, "w") as fout:
fout.write(content)
def __set_trained(self, dictContent): Verbose.print_arguments() modelContent = ModelContent() for kd,d in dictContent.items(): modelContent.add(kd, d) self.trainedModel.new(modelContent, self.wind_dim, self.other_dim, self.humidity_dim, self.nb_altitudes, self.model_type)
def save_all_weights(self):
Verbose.print_arguments()
if self.modelClass == ModelCells:
os.makedirs(self.dirLayerWeights(), exist_ok=True)
# shared
for layer in [
"flyability_block", "crossability_block",
"wind_block_cells", "wind_flyability_block",
"humidity_flyability_block"
]:
arrs = self.model.get_layer(layer).get_weights()
for kArr, arr in enumerate(arrs):
np.save(self.filesLayerWeights(layer + "_%d" % kArr), arr)
# population
populationWeights = self.model.get_layer(
"population_block").get_weights()
np.save(self.filesLayerWeights("population_date"),
populationWeights[0]) # date factor
if ModelSettings.optimize_dow:
assert (len(populationWeights) == 3)
np.save(self.filesLayerWeights("population_dow"),
populationWeights[1]) # DOW factor
else:
assert (len(populationWeights) == 2)
# specific
for c, cell in enumerate(self.modelContent.cells()):
for sr in range(self.modelContent.super_resolution *
self.modelContent.super_resolution):
c_sr = c * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
cell_sr = cell * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
np.save(
self.filesLayerWeights("population_alt_cell_%d" %
cell_sr), populationWeights[-1]
[c_sr, :]) # population at each altitude
else:
for c, cell in enumerate(self.modelContent.cells()):
if self.modelContent.nbSpots(cell) > 0:
np.save(
self.filesLayerWeights("population_0_spots__cell_%d" %
cell),
self.model.get_layer("population__cell_%d" %
cell).get_weights()[0])
windWeights = self.model.get_layer(
"wind_block_spots__cell_%d" % cell).get_weights()
np.save(
self.filesLayerWeights("wind_block_spots_0__cell_%d" %
cell), windWeights[0])
np.save(
self.filesLayerWeights("wind_block_spots_1__cell_%d" %
cell), windWeights[1])
def compute_cells_forecasts(models_directory, problem_formulation, meteo_matrix): Verbose.print_arguments() predict = Predict(models_directory, ModelType.CELLS, problem_formulation) predict.set_meteo_data(meteo_matrix, GfsData().parameters_vector_all) predict.set_trained_cells() predict.trainedModel.load_all_weights() predict.set_prediction_population() NN_X = predict.get_X(range(meteo_matrix.shape[0])) return predict.trainedModel.model.predict(NN_X)
def download_forecast(forecastTime, grid, h, forecastHour, path):
forecastTime = forecastTime[0:4 + 2 + 2] + "%2F" + forecastTime[-2:]
url = "https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_" + grid + ".pl?file=gfs.t" + (
"%02d" % forecastHour
) + "z.pgrb2." + grid + ".f" + ("%03d" % h) + "&" + GfsData(
).g_grib_url_levels + "&" + GfsData(
).g_grib_url_meteovars + "&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fgfs." + forecastTime
Verbose.print_text(1, url)
Forecast.__download(url, path)
def evaluate(self, days_to_use=None):
if days_to_use is None:
days_to_use = range(self.nb_days)
Verbose.print_text(1, "[INFO] Evaluation: "+ \
str(self.trained_model.model.evaluate(x = [x[days_to_use,...] for x in self.all_X],
y = [y[days_to_use,...] for y in self.all_Y],
verbose = 0)))
def check_need_to_update_forecast(self, valid_date, last_forecast_time):
last_forecast_time_file = os.path.join(self.last_forecast_time_file_dir, valid_date.strftime('%Y-%m-%d'))
if os.path.isfile(last_forecast_time_file):
with open(last_forecast_time_file, "r") as file:
contentLastForecastTime = file.read()
if contentLastForecastTime >= last_forecast_time:
Verbose.print_text(1, "forecast is up-to-date")
return False # my forecast is up-to-date
return True
def __check_meteo_files(meteo_files):
ok = True
for mf in meteo_files:
ok = ok and os.path.isfile(mf) and os.path.getsize(mf) > 5000
if ok:
Verbose.print_text(1, "[OK ] Meteo files found")
return True
else:
Verbose.print_text(1, "[WARNING] Meteo files not found")
return False
def compile(self, metrics=[]):
Verbose.print_arguments()
if self.problem_formulation == ProblemFormulation.CLASSIFICATION:
self.model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
loss='binary_crossentropy',
metrics=["accuracy"])
else: # REGRESSION
self.model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
loss='mean_absolute_error',
metrics=[])
def set_population_value(self, popuValue):
Verbose.print_arguments()
for layer in self.model.layers:
if layer.name.startswith("population"):
Verbose.print_text(
1, "[INFO] Setting population value %f in layer '%s'" %
(popuValue, layer.name))
populationWeights = layer.get_weights()
populationWeights[-1][:, :] = popuValue
layer.set_weights(populationWeights)
def __load_or_compute_data(self):
Verbose.print_arguments()
if self.data_not_loaded():
try:
self.__load()
except:
pass
if self.data_not_loaded():
Verbose.print_text(
0, "[WARNING] data_not_loaded, __computeSpotsInformation")
self.__compute_spots_information()
self.__save()
def set_trained(self, cells, super_resolution=1, load_weights=True):
Verbose.print_arguments()
model_content = ModelContent()
if self.model_type == ModelType.CELLS:
model_content.set_super_resolution(super_resolution)
for c in cells:
if self.model_type == ModelType.SPOTS:
# All the spots of the cells
model_content.add(c, [s for s in range(len(self.Y_spots[c]))])
else:
# All the cell
model_content.add(c, -1)
# Maybe the cell has no spots, which will crash at network creation
if self.model_type == ModelType.SPOTS and model_content.total_nb_spots(
) == 0:
return False
#===================================================================
# Create model and load weights
#===================================================================
tf.keras.backend.clear_session(
) # https://github.com/keras-team/keras/issues/3579
self.trained_model.new(model_content,
wind_dim=self.wind_dim,
other_dim=self.X_other[0].shape[-1],
humidity_dim=self.X_humidity[0].shape[-1],
nb_altitudes=self.X_wind[0].shape[-1] //
self.wind_dim,
model_type=self.model_type)
# Re-load shared and specific weights if exists
if load_weights:
self.trained_model.load_all_weights()
#===================================================================
# Model intputs/outputs
#===================================================================
self.all_X = self.__get_X(model_content)
self.all_Y = self.__get_Y(model_content)
return True
def __get_Y(self, model_content):
Verbose.print_arguments()
if self.model_type == ModelType.CELLS:
Y = self.flights_data.get_flights_by_altitude_matrix(
self.all_cells, self.nb_altitudes,
model_content.super_resolution,
self.problem_formulation == ProblemFormulation.REGRESSION)
nb_models = 4
all_Y = [
[] for model in range(nb_models)
] # flyability, crossability, wind-flyability, rain-flyability
for cell in [
c * model_content.super_resolution *
model_content.super_resolution + sc
for c in model_content.cells()
for sc in range(model_content.super_resolution *
model_content.super_resolution)
]:
for model in range(nb_models):
all_Y[model] += [
Y[model][[
cell * self.nb_days + d
for d in range(self.nb_days)
], :]
]
return [np.stack(all_Y[model], 1) for model in range(nb_models)]
else: # spots
all_Y = [] # 1 per cell
for c in model_content.cells():
if len(self.Y_spots[c]) > 0:
all_Y += [
np.stack([
self.Y_spots[c][s] for s in model_content.spots(c)
], 1)
] # Y_spots: [cell][spot] -> array of shape (nb_days,)
# all_Y[cell] = np array of shape (nb_days, nbSpots)
return all_Y
def set_meteo_data(self, meteo_matrix, parameters_vector_all):
Verbose.print_arguments()
# Parameters prefixed by hour
parameters_vector_all_with_hours = [(h, ) + p for h in [6, 12, 18]
for p in parameters_vector_all]
meteoParamsOther, meteoParamsWind, meteoParamsPrecipitation = TrainedModel.meteoParams(
)
# Params for other and wind
rainIdx = [[
parameters_vector_all_with_hours.index(pX)
for pX in meteoParamsPrecipitation[h]
] for h in range(3)]
otherIdx = [[
parameters_vector_all_with_hours.index(pX)
for pX in meteoParamsOther[h]
] for h in range(3)]
windIdx = [[
parameters_vector_all_with_hours.index(pX)
for pX in meteoParamsWind[h]
] for h in range(3)]
self.X_rain = [meteo_matrix[:, rainIdx[h]] for h in range(3)]
self.X_other = [meteo_matrix[:, otherIdx[h]] for h in range(3)]
self.X_wind = [
Utils.convert_wind_matrix(meteo_matrix[:, windIdx[h]],
self.wind_dim) for h in range(3)
]
# Load and apply normalization
normalization_mean_other, normalization_std_other, normalization_mean_rain, normalization_std_rain = BinObj.load(
"normalization_%s" % str(self.model_type).split(".")[-1],
self.models_directory)
for h in range(3):
Utils.apply_normalization(self.X_other[h],
normalization_mean_other,
normalization_std_other)
Utils.apply_normalization(self.X_rain[h], normalization_mean_rain,
normalization_std_rain)
def new(self, model_content, wind_dim, other_dim, humidity_dim,
nb_altitudes, model_type):
Verbose.print_arguments()
if model_type == ModelType.CELLS:
self.modelClass = ModelCells
initialization = None
else:
self.modelClass = ModelSpots
initialization = {}
initialization['date_factor'] = np.load(
self.filesLayerWeights("population_date") + ".npy")
if ModelSettings.optimize_dow:
initialization['dow_factor'] = np.load(
self.filesLayerWeights("population_dow") + ".npy")
self.model = self.modelClass.createNewModel(self.problem_formulation,
model_content, wind_dim,
other_dim, humidity_dim,
nb_altitudes,
initialization)
self.modelContent = model_content
self.nbWindAltitudes = nb_altitudes
def __download(url, path): Verbose.print_arguments() r = requests.get(url, stream=True) downloaded_size = 0 tmpPath = path + ".downloading" with open(tmpPath, 'wb') as f: for chunk in r.iter_content(chunk_size=None): if chunk: if downloaded_size // (1024 * 1024) != (downloaded_size + len(chunk)) // (1024 * 1024): Verbose.print_text(1, str(round(downloaded_size/(1024.0 * 1024.0),1)) +"Mo", True) downloaded_size += len(chunk) f.write(chunk) os.rename(tmpPath, path) Verbose.print_text(1, "")
def __compute_spots_forecasts(self, models_directory, problem_formulation,
lats, lons, meteo_matrix, filename):
Verbose.print_arguments()
predict = Predict(models_directory, ModelType.SPOTS,
problem_formulation)
predict.set_meteo_data(meteo_matrix, GfsData().parameters_vector_all)
#=============================================================
# depend de GribReader.get_values_array(self, params, crops):
forecastCellsLine = {}
line = 0
for crop in self.crops:
for iLat in range(crop[0], crop[1]):
for iLon in range(crop[2], crop[3]):
forecastCellsLine[(iLat, iLon)] = line
line += 1
#=============================================================
# Compute or load cells_and_spots
#=============================================================
filename_cells_and_spots = "Forecast_cellsAndSpots_" + "_".join(
[str(crop[d]) for crop in self.crops for d in range(4)])
if not BinObj.exists(filename_cells_and_spots):
Verbose.print_text(
0,
"Generating precomputation file because of new crop, it may crash on the server... To be computed on my computer"
)
cells_and_spots = {}
# find forecast cell for each spot
# C'est comme le cellsAndSpots de Train sauf que les cellules sont les cells de forecast (32942 cells)
spots_data = SpotsData()
spots = spots_data.getSpots(range(80))
for kc, cell_spots in enumerate(spots):
for ks, spot in enumerate(cell_spots):
iCell = (np.abs(lats - spot.lat).argmin(),
np.abs(lons - spot.lon).argmin())
cellLine = forecastCellsLine[iCell]
kcks_spot = (
(kc, ks), spot.toDict()) # (training cell, ks)
if not cellLine in cells_and_spots:
cells_and_spots[cellLine] = [kcks_spot]
else:
cells_and_spots[cellLine] += [kcks_spot]
BinObj.save(cells_and_spots, filename_cells_and_spots)
else:
cells_and_spots = BinObj.load(filename_cells_and_spots)
#=============================================================
# Create a model with 1 cell of 1 spot
#=============================================================
predict.set_trained_spots()
#=============================================================
# Compute prediction for each spot, one by one
#=============================================================
spots_and_prediction = []
for kcslst, cslst in enumerate(cells_and_spots.items()):
meteoLine, spotsLst = cslst
for cs in spotsLst:
modelContent = ModelContent()
modelContent.add(cs[0][0], cs[0][1])
predict.trainedModel.load_all_weights(
modelContent) # TODO do not reload shared weights
predict.set_prediction_population()
NN_X = predict.get_X([meteoLine])
prediction = predict.trainedModel.model.predict(NN_X)[0][0]
spots_and_prediction += [
Spot((cs[1]['name'], cs[1]['lat'], cs[1]['lon']),
cs[1]['id'], cs[1]['nbFlights'], prediction)
] #[cs[1]]
#=============================================================
# Export all results
#=============================================================
Forecast.__export_spots_forecasts(spots_and_prediction, filename)
def main(self):
last_forecast_times = Forecast.get_last_forecast_times(
self.GFS_resolution)
Verbose.print_text(1,
"last_forecast_times: " + str(last_forecast_times))
for last_forecast_time in last_forecast_times[0:4]:
last_forecast_time_dt = datetime.datetime(
int(last_forecast_time[0:4]), int(last_forecast_time[4:6]),
int(last_forecast_time[6:8]))
last_forecast_hour = int(last_forecast_time[-2:]) # 0, 6, 12, 18
Verbose.print_text(
2, "last_forecast_hour: " + str(last_forecast_hour))
Verbose.print_text(
2, "last_forecast_time: " + str(last_forecast_time))
#==================================================================
# Update each day
#==================================================================
for days in range(self.nb_days):
day_datetime = last_forecast_time_dt + datetime.timedelta(
days=days)
strdate = day_datetime.strftime("%Y-%m-%d")
tiles_dir_this_day = self.tiles_dir + "/" + strdate
#======================================================================================
# Skip if no need to update
#======================================================================================
if not self.check_need_to_update_forecast(
day_datetime, last_forecast_time):
continue
#======================================================================================
# Make tiles dir
#======================================================================================
os.makedirs(tiles_dir_this_day, exist_ok=True)
#======================================================================================
# Download / update next hours grib files
#======================================================================================
if self.forced_meteo_files is None:
meteo_files = []
l_h = [(hh + 24 * days - last_forecast_hour)
for hh in [6, 12, 18]
if (hh + 24 * days - last_forecast_hour) >= 0]
for kh, h in enumerate(l_h):
forecast_datetime_with_hours = datetime.datetime(
int(last_forecast_time[0:4]),
int(last_forecast_time[4:6]),
int(last_forecast_time[6:8]),
int(last_forecast_time[8:10]))
valid_datetime = forecast_datetime_with_hours + datetime.timedelta(
hours=h)
meteo_file = self.destination_forecast_file % valid_datetime.strftime(
"%Y-%m-%d-%H")
Verbose.print_text(1, "download " + meteo_file)
if not self.DEBUG_MODE or not os.path.exists(
meteo_file):
self.download_forecast(last_forecast_time,
self.GFS_resolution, h,
last_forecast_hour,
meteo_file)
meteo_files += [meteo_file]
#======================================================================================
# Re-create a complete list of 3 files even with those not updated (in the past)
#
# params:
# - destinationForecastFile
# input:
# - dayDateTime
# output:
# - meteo_files
#======================================================================================
meteo_files = [
self.destination_forecast_file %
(day_datetime +
datetime.timedelta(hours=h)).strftime("%Y-%m-%d-%H")
for h in [6, 12, 18]
]
Verbose.print_text(1, str(meteo_files))
else: # self.forced_meteo_files is not None
# ======================================================================================
# Use forced meteo files
#
# params:
# - forced_meteo_files
# ======================================================================================
assert (len(self.forced_meteo_files) == 3)
meteo_files = [
"/tmp/forced_meteo_06", "/tmp/forced_meteo_12",
"/tmp/forced_meteo_18"
]
for m in range(len(meteo_files)):
shutil.copyfile(self.forced_meteo_files[m],
meteo_files[m])
#======================================================================================
# Skip if meteo files are missing
#======================================================================================
if not self.__check_meteo_files(meteo_files):
if not self.DEBUG_MODE:
Forecast.clean_meteo_files(meteo_files)
else:
for mf in meteo_files:
if os.path.isfile(
mf) and os.path.getsize(mf) <= 5000:
Forecast.remove_file(mf)
continue
#======================================================================================
# PROGRESS: start computation
#======================================================================================
self.set_progress(10, strdate)
#======================================================================================
# Read weather data
#======================================================================================
distinct_latitudes, distinct_longitudes, meteo_matrix = ForecastData.readWeatherData(
meteo_files, self.crops)
#======================================================================================
# Compute and generate the predictions file for tiler
#======================================================================================
ForecastAndAnl.compute_prediction_file_cells( # predictions
ForecastAndAnl.compute_cells_forecasts(
self.models_directory, self.problem_formulation,
meteo_matrix),
# other params
self.prediction_filename_for_tiler,
distinct_latitudes,
distinct_longitudes,
np.copy(meteo_matrix),
self.crops,
self.meteoParams,
self.grid_desc_predictions)
#======================================================================================
# Compute spots prediction
#======================================================================================
if True:
self.__compute_spots_forecasts(
self.models_directory, self.problem_formulation,
distinct_latitudes, distinct_longitudes,
np.copy(meteo_matrix),
os.path.join(self.tiles_dir,
os.path.join(strdate, "spots.json")))
#======================================================================================
# Clean files
#======================================================================================
if not self.DEBUG_MODE:
Forecast.clean_meteo_files(meteo_files)
#========================================================================
# PROGRESS: end of prediction computation
#========================================================================
self.set_progress(20, strdate)
#========================================================================
# Render tiles
#========================================================================
if self.render_tiles:
ForecastAndAnl.generate_tiler_argument_file(
self.tiler_arguments_filename,
self.prediction_filename_for_tiler,
tiles_dir_this_day,
self.tiler_cache_dir,
self.geo_json_borders,
self.min_tiles_zoom,
self.max_tiles_zoom,
self.background_tiles_dir,
True,
"",
self.skipped_tiles,
generateTranspaVersion=True)
call([self.tiler_program, self.tiler_arguments_filename])
#========================================================================
# Set last update date
#========================================================================
self.save_last_update_time(day_datetime, last_forecast_time)
#========================================================================
# PROGRESS: end for this date
#========================================================================
self.set_progress(100, strdate)
def train(self,
lr_schedule,
use_validation_set=False,
train_crossability_only=False):
Verbose.print_arguments()
self.trained_model.compile([]) # re-compile for removing metrics
#self.trained_model.model.summary()
if self.model_type == ModelType.CELLS:
if train_crossability_only:
train.trained_model.freeze_all_but_crossability()
else:
train.trained_model.unfreeze_all()
#===================================================================
# Training/validation set split
#===================================================================
if use_validation_set:
if False: # random validation set
validation_split = 0.2
permutation = np.random.permutation(self.nb_days)
days_to_use_validation = permutation[
0:int(round(self.nb_days * validation_split))]
else: # non-random and well distributed validation set
days_to_use_validation = np.array(range(0, self.nb_days, 2))
else:
if True: # during process tuning, still check the second optim with validation data
days_to_use_validation = np.array(range(0, self.nb_days, 6))
else:
days_to_use_validation = np.array([])
days_to_use_training = np.array([
s for s in range(self.nb_days) if s not in days_to_use_validation
])
#===================================================================
# Run training
#===================================================================
history = self.trained_model.model.fit(
x=[x[days_to_use_training, ...] for x in self.all_X],
y=[y[days_to_use_training, ...] for y in self.all_Y],
validation_data=(
[x[days_to_use_validation, ...] for x in self.all_X],
[y[days_to_use_validation, ...] for y in self.all_Y])
if days_to_use_validation.size > 0 else None,
epochs=lr_schedule[2],
batch_size=32,
shuffle=True,
verbose=0,
callbacks=[
self.schedule_with_params(lr_init=lr_schedule[0],
lr_end=lr_schedule[1],
nb_epochs=lr_schedule[2]),
MyTrainingLogger(
self.model_type, self.models_directory + "/" +
str(self.model_type).split(".")[-1] + ".log")
])
if 'val_loss' in history.history:
return np.mean(history.history['val_loss'][-5:])
else:
return None
def save(self):
Verbose.print_arguments()
self.trained_model.save_all_weights()
# main
##########################################################################################################
if __name__ == "__main__":
problem_formulation = ProblemFormulation.CLASSIFICATION
model_dir = "./bin/models/%s_1.0.0" % str(problem_formulation).split(
".")[-1]
#######################################################################
# Check if script is already running
#######################################################################
process = Utils.get_elapsed_time("forecast.py")
if len(process) > 1:
Verbose.print_text(0, "[WARNING] already running: " + str(process))
for p in process:
if p[1] > 4 * 3600:
Verbose.print_text(
0, "[WARNING] running for more than 3h, killing it !")
os.kill(p[0], signal.SIGTERM)
else:
sys.exit(0)
#######################################################################
# RUN
#######################################################################
forecast = Forecast(model_dir, problem_formulation)
forecast.main()
def createNewModel(cls, problem_formulation, model_content, wind_dim, other_dim, humidity_dim, nb_altitudes, initialization): Verbose.print_arguments() # Check initialization assert(not initialization is None) for k in initialization: assert(k in ["date_factor", "dow_factor"]) Verbose.print_text(1, "initialization "+ str(initialization)) # ============================================================================================================== # Shared variables # ============================================================================================================== # Not trainable var_date_factor = tf.keras.backend.constant(value=initialization['date_factor'], name="var_date_factor") if ModelSettings.optimize_dow: var_dow_factor = tf.keras.backend.constant(value=initialization['dow_factor'], name="var_dow_factor") else: var_dow_factor = tf.keras.backend.constant(value=ModelSettings.dow_init, name="var_dow_factor") # ============================================================================================================== # Inputs # ============================================================================================================== input_date = tf.keras.layers.Input(shape=(1,), name="in_date") input_dow = tf.keras.layers.Input(shape=(7,), name="in_dow") input_mountainess = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes), name="in_mountainess") input_other = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, other_dim), name="in_other") input_humidity = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, humidity_dim), name="in_rain") input_wind = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes, 3, wind_dim), name="in_wind") all_inputs = [input_date, input_dow, input_mountainess, input_other, input_humidity, input_wind] # ============================================================================================================== # Blocks # ============================================================================================================== flyabilityModel = get_flyability_block(other_dim, humidity_dim, name="flyability_block") flyabilityModel.trainable = False # freeze the block # ============================================================================================================== # Iterations over cells # ============================================================================================================== all_outputs = [] # 1 by cell for kc,c in enumerate(model_content.cells()): nb_spots = len(model_content.spots(c)) if nb_spots > 0: # inputs for this cell input_wind_this_cell = tf.keras.layers.Lambda(lambda x: x[:,kc,...])(input_wind) input_other_this_cell = tf.keras.layers.Lambda(lambda x: x[:,kc,...])(input_other) input_humidity_this_cell = tf.keras.layers.Lambda(lambda x: x[:,kc,...])(input_humidity) # blocks population_block = get_population_block(problem_formulation, var_date_factor, var_dow_factor, 1, name="population__cell_%d"%c) wind_block = get_wind_block_spots(nb_spots, name="wind_block_spots__cell_%d"%c) # Flyability wind_prediction = wind_block(input_wind_this_cell) wind_prediction = tf.keras.layers.Lambda(lambda x: tf.keras.backend.reshape(x, (-1, 1, nb_spots, 3)))(wind_prediction) flyability_prediction = encapsulate_flyability(flyabilityModel, 1, nb_spots, other_dim, humidity_dim, [wind_prediction, input_other_this_cell, input_humidity_this_cell]) # Apply population flown_prediction = population_block([flyability_prediction, input_date, input_dow]) flown_prediction = tf.keras.layers.Lambda(lambda x: tf.keras.backend.reshape(x, (-1, nb_spots)))(flown_prediction) all_outputs += [flown_prediction] #========================================================================================= # Make model #========================================================================================= return tf.keras.models.Model(all_inputs, all_outputs)
def createNewModel(cls, problem_formulation, model_content, wind_dim, other_dim, humidity_dim, nb_altitudes, initialization): Verbose.print_arguments() # ============================================================================================================== # Shared variables # ============================================================================================================== var_date_factor = tf.keras.backend.variable(np.array([[1.275]], dtype=np.float), name="var_date_factor") if ModelSettings.optimize_dow: var_dow_factor = tf.keras.backend.variable(np.array(ModelSettings.dow_init, dtype=np.float), name="var_dow_factor") else: var_dow_factor = tf.keras.backend.constant(np.array(ModelSettings.dow_init, dtype=np.float), name="var_dow_factor") # ============================================================================================================== # Inputs # ============================================================================================================== input_date = tf.keras.layers.Input(shape=(1,), name="in_date") input_dow = tf.keras.layers.Input(shape=(7,), name="in_dow") input_mountainess = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes), name="in_mountainess") input_other = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, other_dim), name="in_other") input_humidity = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, humidity_dim), name="in_rain") input_wind = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes, 3, wind_dim), name="in_wind") allInputs = [input_date, input_dow, input_mountainess, input_other, input_humidity, input_wind] # ============================================================================================================== # Blocks # ============================================================================================================== wind_block = get_wind_block_cells(name="wind_block_cells") flyability_block = get_flyability_block(other_dim, humidity_dim, name="flyability_block") population_block = get_population_block(problem_formulation, var_date_factor, var_dow_factor, model_content.super_resolution, name="population_block") crossability_block = get_crossability_block(other_dim, humidity_dim, nb_altitudes, wind_dim, model_content, name="crossability_block") wind_flyability_block = get_wind_flyability_block(nb_altitudes, model_content, name="wind_flyability_block") humidity_flyability_block = get_humidity_flyability_block(nb_altitudes, model_content, humidity_dim, name="humidity_flyability_block") # ============================================================================================================== # Flyability/crossability # ============================================================================================================== wind_prediction = wind_block([input_mountainess, input_wind]) flyability_prediction = encapsulate_flyability(flyability_block, model_content.nbCells(), nb_altitudes, other_dim, humidity_dim, [wind_prediction, input_other, input_humidity]) crossability_prediction = crossability_block([flyability_prediction, wind_prediction, input_other, input_humidity]) wind_flyability_prediction = wind_flyability_block(wind_prediction) humidity_flyability_prediction = humidity_flyability_block(input_humidity) #========================================================================================= # Apply population #========================================================================================= flown_prediction = population_block([flyability_prediction, input_date, input_dow]) crossed_prediction = population_block([crossability_prediction, input_date, input_dow]) wind_flown_prediction = population_block([wind_flyability_prediction, input_date, input_dow]) humidity_flown_prediction = population_block([humidity_flyability_prediction, input_date, input_dow]) #========================================================================================= # Make model #========================================================================================= return tf.keras.models.Model(allInputs, [flown_prediction, crossed_prediction, wind_flown_prediction, humidity_flown_prediction])
def load_all_weights(self, modelContentToLoad=None, wind_bias=-1.0):
Verbose.print_arguments()
# self.modelContent est le contenu du network, il définit le nom des layers
# modelContentToLoad définit le nom des fichiers de weights à utiliser
if modelContentToLoad is None:
modelContentToLoad = self.modelContent
else:
assert (self.modelContent.sameStructure(modelContentToLoad))
if self.modelClass == ModelCells:
layers = [
"flyability_block", "crossability_block", "wind_block_cells",
"wind_flyability_block", "humidity_flyability_block"
]
else:
layers = ["flyability_block"]
#===============================================================================================================
# shared
#===============================================================================================================
# Flyability, Fufu, Wind, WindFlyability, RainFlyability
for layer in layers:
try:
arrs = self.model.get_layer(layer).get_weights()
for kArr, arr in enumerate(arrs):
arr = np.load(
self.filesLayerWeights(layer + "_%d" % kArr) + ".npy")
if wind_bias > 0.0 and layer == "wind_block_cells":
arr = wind_bias * arr
arrs[kArr] = arr
self.model.get_layer(layer).set_weights(arrs)
except:
pass
if self.modelClass == ModelCells:
populationWeights = self.model.get_layer(
"population_block").get_weights() #[(1,), (7,), (5,)]
try:
populationWeights[0] = np.load(
self.filesLayerWeights("population_date") +
".npy") # date factor (1,)
if ModelSettings.optimize_dow:
assert (len(populationWeights) == 3)
populationWeights[1] = np.load(
self.filesLayerWeights("population_dow") +
".npy") # DOW factor (7,)
else:
assert (len(populationWeights) == 2)
except:
Verbose.print_text(
1,
"[ERROR] Could not load weights population_date population_dow"
)
raise
# ===============================================================================================================
# specific
# ===============================================================================================================
if self.modelClass == ModelCells:
for c, cell in enumerate(
self.modelContent.cells()): # TODO super_resolution
for sr in range(self.modelContent.super_resolution *
self.modelContent.super_resolution):
c_sr = c * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
cell_sr = cell * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
try:
populationWeights[-1][c_sr, :] = np.load(
self.filesLayerWeights(
"population_alt_cell_%d" % cell_sr) +
".npy") # population at each altitude
except:
Verbose.print_text(
1,
"[ERROR] Could not load weights for population cell %d"
% cell_sr)
raise
self.model.get_layer("population_block").set_weights(
populationWeights)
else:
Verbose.print_text(1,
"modelContent :" + str(self.modelContent))
Verbose.print_text(1,
"modelContentToLoad:" + str(modelContentToLoad))
for c, cell in enumerate(self.modelContent.cells()):
cellToLoad = modelContentToLoad.cells()[c]
spotsToLoad = modelContentToLoad.spots(cellToLoad)
try:
self.model.get_layer(
"population__cell_%d" % cell).set_weights([
np.load(
self.filesLayerWeights(
"population_0_spots__cell_%d" %
cellToLoad) + ".npy")
])
except:
Verbose.print_text(
1, "[WARNING] Could not load weights for " +
("population__cell_%d" % cell))
try:
# load weights for all spots
WindSpots_cell_0 = np.load(
self.filesLayerWeights(
"wind_block_spots_0__cell_%d" % cellToLoad) +
".npy") # shape (nbSpots, nbWindDim)
WindSpots_cell_1 = np.load(
self.filesLayerWeights(
"wind_block_spots_1__cell_%d" % cellToLoad) +
".npy") # shape ( 1, nbWindDim)
# keep only spots of modelContent
WindSpots_cell_0 = WindSpots_cell_0[spotsToLoad, :]
WindSpots_cell_1 = WindSpots_cell_1[spotsToLoad, :]
self.model.get_layer(
"wind_block_spots__cell_%d" % cell).set_weights(
[WindSpots_cell_0, WindSpots_cell_1])
except:
Verbose.print_text(
1, "[WARNING] Could not load weights for " +
("wind_block_spots__cell_%d" % cell))
def __get_X(self, model_content): # The same for the 2 models
Verbose.print_arguments()
all_X = []
# 0) date (nb_days,)
all_X += [self.X_date]
# 1) DOW hot vectors (nb_days, 7)
all_X += [self.X_dow]
# 2) mountainess (nb_days, nb_cells, nb_altitudes)
all_X += [
np.repeat(
np.array([
self.flights_data.cellKAltitudeMountainess[c][alt]
for c in model_content.cells()
for alt in range(self.nb_altitudes)
],
dtype=np.float).reshape(1, model_content.nbCells(),
self.nb_altitudes),
self.nb_days, 0)
]
# 3) other (nb_days, nb_cells, nbHours, dimOther)
all_X += [
np.stack([
np.stack([
self.X_other[h]
[[c * self.nb_days + d for d in range(self.nb_days)], :]
for c in model_content.cells()
], 1) for h in range(3)
], 2)
]
# 4) rain (nb_days, nb_cells, nbHours, dimRain)
all_X += [
np.stack([
np.stack([
self.X_humidity[h]
[[c * self.nb_days + d for d in range(self.nb_days)], :]
for c in model_content.cells()
], 1) for h in range(3)
], 2)
]
# 5) wind (nb_days, nb_cells, nb_altitudes, nbHours, dimWind)
all_X += [
np.stack([
np.stack([
np.stack([
self.X_wind[h]
[[c * self.nb_days + d for d in range(self.nb_days)],
alt * self.wind_dim:(alt + 1) * self.wind_dim]
for c in model_content.cells()
], 1) for alt in range(self.nb_altitudes)
], 2) for h in range(3)
], 3)
]
# debug, print shapes
if False:
for ix in range(len(all_X)):
print(str(ix).rjust(3), all_X[ix].shape)
return all_X
def set_prediction_population(self): Verbose.print_arguments() prediction_popu = self.__get_prediction_population() self.trainedModel.set_population_value(prediction_popu)
def set_trained_spots(self):
Verbose.print_arguments()
self.__set_trained({0: [0]})
def set_trained_cells(self):
Verbose.print_arguments()
self.__set_trained({0: [-1]})
def compute_prediction_file_cells( cells_forecasts, prediction_filename_for_tiler, distinct_latitudes, distinct_longitudes, meteo_matrix, crops, meteo_params, grid_desc_predictions, export_for_tiler=True): Verbose.print_arguments() # ====================================================================================== # Retrieve geopotential height at 12h indices in meteoParams # # inputs: # - meteoParams # - parameters_geopotential_height # outputs: # - geopotential_height_indices # ====================================================================================== geopotential_height_indices = [] for geopotential_height_param in GfsData().parameters_geopotential_height: for kParam, param in enumerate(meteo_params): if param == (12,) + geopotential_height_param: geopotential_height_indices += [kParam] break assert (len(geopotential_height_indices) == len(GfsData().parameters_geopotential_height)) # ====================================================================================== # Retrieve wind at 12h # # inputs: # - meteoParams # - parameters_wind # outputs: # - wind_indices # ====================================================================================== wind_indices = [] for wind_param in GfsData().parameters_wind: for kParam, param in enumerate(meteo_params): if param == (12,) + wind_param: wind_indices += [kParam] break assert (len(wind_indices) == len(GfsData().parameters_wind)) # ======================================================================== # Create and fill prediction grid # # params: # - grid_desc_predictions # inputs: # - distinct_latitudes # - distinct_longitudes # - meteo_matrix # - geopotential_height_indices # - predictions # outputs: # - prediction_grid # ======================================================================== prediction_grid = GridLatLon(grid_desc_predictions[0], grid_desc_predictions[1], grid_desc_predictions[2], grid_desc_predictions[3]) p = 0 for crop in crops: # number of data rectangles to read in the GRIB grid lat_range = (crop[0], crop[1]) lon_range = (crop[2], crop[3]) Verbose.print_text(1, "lats: " + str(distinct_latitudes[lat_range[0]]) +" , "+ str(distinct_latitudes[lat_range[1]-1])) Verbose.print_text(1, "lons: " + str(distinct_longitudes[lon_range[0]]) +" , "+ str(distinct_longitudes[lon_range[1]-1])) for ilat in range(lat_range[0], lat_range[1]): for ilon in range(lon_range[0], lon_range[1]): predictions_data_sample = () # Add geopential heights (not predicted, simply meteo variables) predictions_data_sample += (meteo_matrix[p, geopotential_height_indices[0]], # 0 meteo_matrix[p, geopotential_height_indices[1]], # 1 meteo_matrix[p, geopotential_height_indices[2]], # 2 meteo_matrix[p, geopotential_height_indices[3]], # 3 meteo_matrix[p, geopotential_height_indices[4]]) # 4 predictions_data_sample += (cells_forecasts[0][p,0,0], # flyability at 1000 # 5 cells_forecasts[0][p,0,1], # flyability at 900 # 6 cells_forecasts[0][p,0,2], # flyability at 800 # 7 cells_forecasts[0][p,0,3], # flyability at 700 # 8 cells_forecasts[0][p,0,4], # flyability at 600 # 9 np.mean(cells_forecasts[1][p,0,0:5]),# crossability # 10 cells_forecasts[2][p,0,0], # wind at 1000 # 11 cells_forecasts[2][p,0,1], # wind at 900 # 12 cells_forecasts[2][p,0,2], # wind at 800 # 13 cells_forecasts[2][p,0,3], # wind at 700 # 14 cells_forecasts[2][p,0,4], # wind at 600 # 15 np.mean(cells_forecasts[3][p,0,0:5]), # humidity # 16 0.0) # Attractiveness # 17 predictions_data_sample += (meteo_matrix[p, wind_indices[0 + 0]], # U 1000 12h # 18 meteo_matrix[p, wind_indices[2 + 0]], # U 900 12h # 19 meteo_matrix[p, wind_indices[4 + 0]], # U 800 12h # 20 meteo_matrix[p, wind_indices[6 + 0]], # U 700 12h # 21 meteo_matrix[p, wind_indices[8 + 0]], # U 600 12h # 22 meteo_matrix[p, wind_indices[0 + 1]], # V 1000 12h # 23 meteo_matrix[p, wind_indices[2 + 1]], # V 900 12h # 24 meteo_matrix[p, wind_indices[4 + 1]], # V 800 12h # 25 meteo_matrix[p, wind_indices[6 + 1]], # V 700 12h # 26 meteo_matrix[p, wind_indices[8 + 1]]) # V 600 12h # 27 # Push the sample prediction_grid.add(distinct_latitudes[ilat], distinct_longitudes[ilon], predictions_data_sample) p += 1 # ======================================================================== # Export predictions for tiler # # params: # - predictionFilenameForTiler # inputs: # - prediction_grid # ======================================================================== accessors = [lambda x, ix=ixVal: sum([xx[ix] for xx in x]) / len(x) for ixVal in range(len(predictions_data_sample))] if export_for_tiler: prediction_grid.export_data_for_tiler(prediction_filename_for_tiler, accessors) else: prediction_grid.export_json(prediction_filename_for_tiler, accessors)
