def __export_spots_forecasts(
spots_and_prediction,
filename): #[Spot(name, lat, lon, prediction), ...]
Verbose.print_arguments()
content = """
{
"type": "FeatureCollection",
"features": [
"""
sep = ""
for ks, s in enumerate(spots_and_prediction):
content += sep + """
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [""" + str(s.lon) + "," + str(s.lat) + """]
},
"properties": {
"id": \"""" + str(s.id) + """\",
"name": \"""" + Forecast.__fix_spots_name(s.name).replace(
'"', '\\"') + """\",
"nbFlights": """ + str(s.nbFlights) + """,
"flyability": """ + str(s.prediction) + """
}
}
"""
sep = ","
content += "]}"
with open(filename, "w") as fout:
fout.write(content)
def __set_trained(self, dictContent): Verbose.print_arguments() modelContent = ModelContent() for kd,d in dictContent.items(): modelContent.add(kd, d) self.trainedModel.new(modelContent, self.wind_dim, self.other_dim, self.humidity_dim, self.nb_altitudes, self.model_type)
def save_all_weights(self):
Verbose.print_arguments()
if self.modelClass == ModelCells:
os.makedirs(self.dirLayerWeights(), exist_ok=True)
# shared
for layer in [
"flyability_block", "crossability_block",
"wind_block_cells", "wind_flyability_block",
"humidity_flyability_block"
]:
arrs = self.model.get_layer(layer).get_weights()
for kArr, arr in enumerate(arrs):
np.save(self.filesLayerWeights(layer + "_%d" % kArr), arr)
# population
populationWeights = self.model.get_layer(
"population_block").get_weights()
np.save(self.filesLayerWeights("population_date"),
populationWeights[0]) # date factor
if ModelSettings.optimize_dow:
assert (len(populationWeights) == 3)
np.save(self.filesLayerWeights("population_dow"),
populationWeights[1]) # DOW factor
else:
assert (len(populationWeights) == 2)
# specific
for c, cell in enumerate(self.modelContent.cells()):
for sr in range(self.modelContent.super_resolution *
self.modelContent.super_resolution):
c_sr = c * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
cell_sr = cell * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
np.save(
self.filesLayerWeights("population_alt_cell_%d" %
cell_sr), populationWeights[-1]
[c_sr, :]) # population at each altitude
else:
for c, cell in enumerate(self.modelContent.cells()):
if self.modelContent.nbSpots(cell) > 0:
np.save(
self.filesLayerWeights("population_0_spots__cell_%d" %
cell),
self.model.get_layer("population__cell_%d" %
cell).get_weights()[0])
windWeights = self.model.get_layer(
"wind_block_spots__cell_%d" % cell).get_weights()
np.save(
self.filesLayerWeights("wind_block_spots_0__cell_%d" %
cell), windWeights[0])
np.save(
self.filesLayerWeights("wind_block_spots_1__cell_%d" %
cell), windWeights[1])
def compute_cells_forecasts(models_directory, problem_formulation, meteo_matrix): Verbose.print_arguments() predict = Predict(models_directory, ModelType.CELLS, problem_formulation) predict.set_meteo_data(meteo_matrix, GfsData().parameters_vector_all) predict.set_trained_cells() predict.trainedModel.load_all_weights() predict.set_prediction_population() NN_X = predict.get_X(range(meteo_matrix.shape[0])) return predict.trainedModel.model.predict(NN_X)
def compile(self, metrics=[]):
Verbose.print_arguments()
if self.problem_formulation == ProblemFormulation.CLASSIFICATION:
self.model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
loss='binary_crossentropy',
metrics=["accuracy"])
else: # REGRESSION
self.model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
loss='mean_absolute_error',
metrics=[])
def set_population_value(self, popuValue):
Verbose.print_arguments()
for layer in self.model.layers:
if layer.name.startswith("population"):
Verbose.print_text(
1, "[INFO] Setting population value %f in layer '%s'" %
(popuValue, layer.name))
populationWeights = layer.get_weights()
populationWeights[-1][:, :] = popuValue
layer.set_weights(populationWeights)
def __load_or_compute_data(self):
Verbose.print_arguments()
if self.data_not_loaded():
try:
self.__load()
except:
pass
if self.data_not_loaded():
Verbose.print_text(
0, "[WARNING] data_not_loaded, __computeSpotsInformation")
self.__compute_spots_information()
self.__save()
def __download(url, path): Verbose.print_arguments() r = requests.get(url, stream=True) downloaded_size = 0 tmpPath = path + ".downloading" with open(tmpPath, 'wb') as f: for chunk in r.iter_content(chunk_size=None): if chunk: if downloaded_size // (1024 * 1024) != (downloaded_size + len(chunk)) // (1024 * 1024): Verbose.print_text(1, str(round(downloaded_size/(1024.0 * 1024.0),1)) +"Mo", True) downloaded_size += len(chunk) f.write(chunk) os.rename(tmpPath, path) Verbose.print_text(1, "")
def set_trained(self, cells, super_resolution=1, load_weights=True):
Verbose.print_arguments()
model_content = ModelContent()
if self.model_type == ModelType.CELLS:
model_content.set_super_resolution(super_resolution)
for c in cells:
if self.model_type == ModelType.SPOTS:
# All the spots of the cells
model_content.add(c, [s for s in range(len(self.Y_spots[c]))])
else:
# All the cell
model_content.add(c, -1)
# Maybe the cell has no spots, which will crash at network creation
if self.model_type == ModelType.SPOTS and model_content.total_nb_spots(
) == 0:
return False
#===================================================================
# Create model and load weights
#===================================================================
tf.keras.backend.clear_session(
) # https://github.com/keras-team/keras/issues/3579
self.trained_model.new(model_content,
wind_dim=self.wind_dim,
other_dim=self.X_other[0].shape[-1],
humidity_dim=self.X_humidity[0].shape[-1],
nb_altitudes=self.X_wind[0].shape[-1] //
self.wind_dim,
model_type=self.model_type)
# Re-load shared and specific weights if exists
if load_weights:
self.trained_model.load_all_weights()
#===================================================================
# Model intputs/outputs
#===================================================================
self.all_X = self.__get_X(model_content)
self.all_Y = self.__get_Y(model_content)
return True
def __get_Y(self, model_content):
Verbose.print_arguments()
if self.model_type == ModelType.CELLS:
Y = self.flights_data.get_flights_by_altitude_matrix(
self.all_cells, self.nb_altitudes,
model_content.super_resolution,
self.problem_formulation == ProblemFormulation.REGRESSION)
nb_models = 4
all_Y = [
[] for model in range(nb_models)
] # flyability, crossability, wind-flyability, rain-flyability
for cell in [
c * model_content.super_resolution *
model_content.super_resolution + sc
for c in model_content.cells()
for sc in range(model_content.super_resolution *
model_content.super_resolution)
]:
for model in range(nb_models):
all_Y[model] += [
Y[model][[
cell * self.nb_days + d
for d in range(self.nb_days)
], :]
]
return [np.stack(all_Y[model], 1) for model in range(nb_models)]
else: # spots
all_Y = [] # 1 per cell
for c in model_content.cells():
if len(self.Y_spots[c]) > 0:
all_Y += [
np.stack([
self.Y_spots[c][s] for s in model_content.spots(c)
], 1)
] # Y_spots: [cell][spot] -> array of shape (nb_days,)
# all_Y[cell] = np array of shape (nb_days, nbSpots)
return all_Y
def set_meteo_data(self, meteo_matrix, parameters_vector_all):
Verbose.print_arguments()
# Parameters prefixed by hour
parameters_vector_all_with_hours = [(h, ) + p for h in [6, 12, 18]
for p in parameters_vector_all]
meteoParamsOther, meteoParamsWind, meteoParamsPrecipitation = TrainedModel.meteoParams(
)
# Params for other and wind
rainIdx = [[
parameters_vector_all_with_hours.index(pX)
for pX in meteoParamsPrecipitation[h]
] for h in range(3)]
otherIdx = [[
parameters_vector_all_with_hours.index(pX)
for pX in meteoParamsOther[h]
] for h in range(3)]
windIdx = [[
parameters_vector_all_with_hours.index(pX)
for pX in meteoParamsWind[h]
] for h in range(3)]
self.X_rain = [meteo_matrix[:, rainIdx[h]] for h in range(3)]
self.X_other = [meteo_matrix[:, otherIdx[h]] for h in range(3)]
self.X_wind = [
Utils.convert_wind_matrix(meteo_matrix[:, windIdx[h]],
self.wind_dim) for h in range(3)
]
# Load and apply normalization
normalization_mean_other, normalization_std_other, normalization_mean_rain, normalization_std_rain = BinObj.load(
"normalization_%s" % str(self.model_type).split(".")[-1],
self.models_directory)
for h in range(3):
Utils.apply_normalization(self.X_other[h],
normalization_mean_other,
normalization_std_other)
Utils.apply_normalization(self.X_rain[h], normalization_mean_rain,
normalization_std_rain)
def new(self, model_content, wind_dim, other_dim, humidity_dim,
nb_altitudes, model_type):
Verbose.print_arguments()
if model_type == ModelType.CELLS:
self.modelClass = ModelCells
initialization = None
else:
self.modelClass = ModelSpots
initialization = {}
initialization['date_factor'] = np.load(
self.filesLayerWeights("population_date") + ".npy")
if ModelSettings.optimize_dow:
initialization['dow_factor'] = np.load(
self.filesLayerWeights("population_dow") + ".npy")
self.model = self.modelClass.createNewModel(self.problem_formulation,
model_content, wind_dim,
other_dim, humidity_dim,
nb_altitudes,
initialization)
self.modelContent = model_content
self.nbWindAltitudes = nb_altitudes
def set_trained_cells(self):
Verbose.print_arguments()
self.__set_trained({0: [-1]})
def __compute_spots_forecasts(self, models_directory, problem_formulation,
lats, lons, meteo_matrix, filename):
Verbose.print_arguments()
predict = Predict(models_directory, ModelType.SPOTS,
problem_formulation)
predict.set_meteo_data(meteo_matrix, GfsData().parameters_vector_all)
#=============================================================
# depend de GribReader.get_values_array(self, params, crops):
forecastCellsLine = {}
line = 0
for crop in self.crops:
for iLat in range(crop[0], crop[1]):
for iLon in range(crop[2], crop[3]):
forecastCellsLine[(iLat, iLon)] = line
line += 1
#=============================================================
# Compute or load cells_and_spots
#=============================================================
filename_cells_and_spots = "Forecast_cellsAndSpots_" + "_".join(
[str(crop[d]) for crop in self.crops for d in range(4)])
if not BinObj.exists(filename_cells_and_spots):
Verbose.print_text(
0,
"Generating precomputation file because of new crop, it may crash on the server... To be computed on my computer"
)
cells_and_spots = {}
# find forecast cell for each spot
# C'est comme le cellsAndSpots de Train sauf que les cellules sont les cells de forecast (32942 cells)
spots_data = SpotsData()
spots = spots_data.getSpots(range(80))
for kc, cell_spots in enumerate(spots):
for ks, spot in enumerate(cell_spots):
iCell = (np.abs(lats - spot.lat).argmin(),
np.abs(lons - spot.lon).argmin())
cellLine = forecastCellsLine[iCell]
kcks_spot = (
(kc, ks), spot.toDict()) # (training cell, ks)
if not cellLine in cells_and_spots:
cells_and_spots[cellLine] = [kcks_spot]
else:
cells_and_spots[cellLine] += [kcks_spot]
BinObj.save(cells_and_spots, filename_cells_and_spots)
else:
cells_and_spots = BinObj.load(filename_cells_and_spots)
#=============================================================
# Create a model with 1 cell of 1 spot
#=============================================================
predict.set_trained_spots()
#=============================================================
# Compute prediction for each spot, one by one
#=============================================================
spots_and_prediction = []
for kcslst, cslst in enumerate(cells_and_spots.items()):
meteoLine, spotsLst = cslst
for cs in spotsLst:
modelContent = ModelContent()
modelContent.add(cs[0][0], cs[0][1])
predict.trainedModel.load_all_weights(
modelContent) # TODO do not reload shared weights
predict.set_prediction_population()
NN_X = predict.get_X([meteoLine])
prediction = predict.trainedModel.model.predict(NN_X)[0][0]
spots_and_prediction += [
Spot((cs[1]['name'], cs[1]['lat'], cs[1]['lon']),
cs[1]['id'], cs[1]['nbFlights'], prediction)
] #[cs[1]]
#=============================================================
# Export all results
#=============================================================
Forecast.__export_spots_forecasts(spots_and_prediction, filename)
def createNewModel(cls, problem_formulation, model_content, wind_dim, other_dim, humidity_dim, nb_altitudes, initialization): Verbose.print_arguments() # Check initialization assert(not initialization is None) for k in initialization: assert(k in ["date_factor", "dow_factor"]) Verbose.print_text(1, "initialization "+ str(initialization)) # ============================================================================================================== # Shared variables # ============================================================================================================== # Not trainable var_date_factor = tf.keras.backend.constant(value=initialization['date_factor'], name="var_date_factor") if ModelSettings.optimize_dow: var_dow_factor = tf.keras.backend.constant(value=initialization['dow_factor'], name="var_dow_factor") else: var_dow_factor = tf.keras.backend.constant(value=ModelSettings.dow_init, name="var_dow_factor") # ============================================================================================================== # Inputs # ============================================================================================================== input_date = tf.keras.layers.Input(shape=(1,), name="in_date") input_dow = tf.keras.layers.Input(shape=(7,), name="in_dow") input_mountainess = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes), name="in_mountainess") input_other = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, other_dim), name="in_other") input_humidity = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, humidity_dim), name="in_rain") input_wind = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes, 3, wind_dim), name="in_wind") all_inputs = [input_date, input_dow, input_mountainess, input_other, input_humidity, input_wind] # ============================================================================================================== # Blocks # ============================================================================================================== flyabilityModel = get_flyability_block(other_dim, humidity_dim, name="flyability_block") flyabilityModel.trainable = False # freeze the block # ============================================================================================================== # Iterations over cells # ============================================================================================================== all_outputs = [] # 1 by cell for kc,c in enumerate(model_content.cells()): nb_spots = len(model_content.spots(c)) if nb_spots > 0: # inputs for this cell input_wind_this_cell = tf.keras.layers.Lambda(lambda x: x[:,kc,...])(input_wind) input_other_this_cell = tf.keras.layers.Lambda(lambda x: x[:,kc,...])(input_other) input_humidity_this_cell = tf.keras.layers.Lambda(lambda x: x[:,kc,...])(input_humidity) # blocks population_block = get_population_block(problem_formulation, var_date_factor, var_dow_factor, 1, name="population__cell_%d"%c) wind_block = get_wind_block_spots(nb_spots, name="wind_block_spots__cell_%d"%c) # Flyability wind_prediction = wind_block(input_wind_this_cell) wind_prediction = tf.keras.layers.Lambda(lambda x: tf.keras.backend.reshape(x, (-1, 1, nb_spots, 3)))(wind_prediction) flyability_prediction = encapsulate_flyability(flyabilityModel, 1, nb_spots, other_dim, humidity_dim, [wind_prediction, input_other_this_cell, input_humidity_this_cell]) # Apply population flown_prediction = population_block([flyability_prediction, input_date, input_dow]) flown_prediction = tf.keras.layers.Lambda(lambda x: tf.keras.backend.reshape(x, (-1, nb_spots)))(flown_prediction) all_outputs += [flown_prediction] #========================================================================================= # Make model #========================================================================================= return tf.keras.models.Model(all_inputs, all_outputs)
def createNewModel(cls, problem_formulation, model_content, wind_dim, other_dim, humidity_dim, nb_altitudes, initialization): Verbose.print_arguments() # ============================================================================================================== # Shared variables # ============================================================================================================== var_date_factor = tf.keras.backend.variable(np.array([[1.275]], dtype=np.float), name="var_date_factor") if ModelSettings.optimize_dow: var_dow_factor = tf.keras.backend.variable(np.array(ModelSettings.dow_init, dtype=np.float), name="var_dow_factor") else: var_dow_factor = tf.keras.backend.constant(np.array(ModelSettings.dow_init, dtype=np.float), name="var_dow_factor") # ============================================================================================================== # Inputs # ============================================================================================================== input_date = tf.keras.layers.Input(shape=(1,), name="in_date") input_dow = tf.keras.layers.Input(shape=(7,), name="in_dow") input_mountainess = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes), name="in_mountainess") input_other = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, other_dim), name="in_other") input_humidity = tf.keras.layers.Input(shape=(model_content.nbCells(), 3, humidity_dim), name="in_rain") input_wind = tf.keras.layers.Input(shape=(model_content.nbCells(), nb_altitudes, 3, wind_dim), name="in_wind") allInputs = [input_date, input_dow, input_mountainess, input_other, input_humidity, input_wind] # ============================================================================================================== # Blocks # ============================================================================================================== wind_block = get_wind_block_cells(name="wind_block_cells") flyability_block = get_flyability_block(other_dim, humidity_dim, name="flyability_block") population_block = get_population_block(problem_formulation, var_date_factor, var_dow_factor, model_content.super_resolution, name="population_block") crossability_block = get_crossability_block(other_dim, humidity_dim, nb_altitudes, wind_dim, model_content, name="crossability_block") wind_flyability_block = get_wind_flyability_block(nb_altitudes, model_content, name="wind_flyability_block") humidity_flyability_block = get_humidity_flyability_block(nb_altitudes, model_content, humidity_dim, name="humidity_flyability_block") # ============================================================================================================== # Flyability/crossability # ============================================================================================================== wind_prediction = wind_block([input_mountainess, input_wind]) flyability_prediction = encapsulate_flyability(flyability_block, model_content.nbCells(), nb_altitudes, other_dim, humidity_dim, [wind_prediction, input_other, input_humidity]) crossability_prediction = crossability_block([flyability_prediction, wind_prediction, input_other, input_humidity]) wind_flyability_prediction = wind_flyability_block(wind_prediction) humidity_flyability_prediction = humidity_flyability_block(input_humidity) #========================================================================================= # Apply population #========================================================================================= flown_prediction = population_block([flyability_prediction, input_date, input_dow]) crossed_prediction = population_block([crossability_prediction, input_date, input_dow]) wind_flown_prediction = population_block([wind_flyability_prediction, input_date, input_dow]) humidity_flown_prediction = population_block([humidity_flyability_prediction, input_date, input_dow]) #========================================================================================= # Make model #========================================================================================= return tf.keras.models.Model(allInputs, [flown_prediction, crossed_prediction, wind_flown_prediction, humidity_flown_prediction])
def load_all_weights(self, modelContentToLoad=None, wind_bias=-1.0):
Verbose.print_arguments()
# self.modelContent est le contenu du network, il définit le nom des layers
# modelContentToLoad définit le nom des fichiers de weights à utiliser
if modelContentToLoad is None:
modelContentToLoad = self.modelContent
else:
assert (self.modelContent.sameStructure(modelContentToLoad))
if self.modelClass == ModelCells:
layers = [
"flyability_block", "crossability_block", "wind_block_cells",
"wind_flyability_block", "humidity_flyability_block"
]
else:
layers = ["flyability_block"]
#===============================================================================================================
# shared
#===============================================================================================================
# Flyability, Fufu, Wind, WindFlyability, RainFlyability
for layer in layers:
try:
arrs = self.model.get_layer(layer).get_weights()
for kArr, arr in enumerate(arrs):
arr = np.load(
self.filesLayerWeights(layer + "_%d" % kArr) + ".npy")
if wind_bias > 0.0 and layer == "wind_block_cells":
arr = wind_bias * arr
arrs[kArr] = arr
self.model.get_layer(layer).set_weights(arrs)
except:
pass
if self.modelClass == ModelCells:
populationWeights = self.model.get_layer(
"population_block").get_weights() #[(1,), (7,), (5,)]
try:
populationWeights[0] = np.load(
self.filesLayerWeights("population_date") +
".npy") # date factor (1,)
if ModelSettings.optimize_dow:
assert (len(populationWeights) == 3)
populationWeights[1] = np.load(
self.filesLayerWeights("population_dow") +
".npy") # DOW factor (7,)
else:
assert (len(populationWeights) == 2)
except:
Verbose.print_text(
1,
"[ERROR] Could not load weights population_date population_dow"
)
raise
# ===============================================================================================================
# specific
# ===============================================================================================================
if self.modelClass == ModelCells:
for c, cell in enumerate(
self.modelContent.cells()): # TODO super_resolution
for sr in range(self.modelContent.super_resolution *
self.modelContent.super_resolution):
c_sr = c * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
cell_sr = cell * self.modelContent.super_resolution * self.modelContent.super_resolution + sr
try:
populationWeights[-1][c_sr, :] = np.load(
self.filesLayerWeights(
"population_alt_cell_%d" % cell_sr) +
".npy") # population at each altitude
except:
Verbose.print_text(
1,
"[ERROR] Could not load weights for population cell %d"
% cell_sr)
raise
self.model.get_layer("population_block").set_weights(
populationWeights)
else:
Verbose.print_text(1,
"modelContent :" + str(self.modelContent))
Verbose.print_text(1,
"modelContentToLoad:" + str(modelContentToLoad))
for c, cell in enumerate(self.modelContent.cells()):
cellToLoad = modelContentToLoad.cells()[c]
spotsToLoad = modelContentToLoad.spots(cellToLoad)
try:
self.model.get_layer(
"population__cell_%d" % cell).set_weights([
np.load(
self.filesLayerWeights(
"population_0_spots__cell_%d" %
cellToLoad) + ".npy")
])
except:
Verbose.print_text(
1, "[WARNING] Could not load weights for " +
("population__cell_%d" % cell))
try:
# load weights for all spots
WindSpots_cell_0 = np.load(
self.filesLayerWeights(
"wind_block_spots_0__cell_%d" % cellToLoad) +
".npy") # shape (nbSpots, nbWindDim)
WindSpots_cell_1 = np.load(
self.filesLayerWeights(
"wind_block_spots_1__cell_%d" % cellToLoad) +
".npy") # shape ( 1, nbWindDim)
# keep only spots of modelContent
WindSpots_cell_0 = WindSpots_cell_0[spotsToLoad, :]
WindSpots_cell_1 = WindSpots_cell_1[spotsToLoad, :]
self.model.get_layer(
"wind_block_spots__cell_%d" % cell).set_weights(
[WindSpots_cell_0, WindSpots_cell_1])
except:
Verbose.print_text(
1, "[WARNING] Could not load weights for " +
("wind_block_spots__cell_%d" % cell))
def __get_X(self, model_content): # The same for the 2 models
Verbose.print_arguments()
all_X = []
# 0) date (nb_days,)
all_X += [self.X_date]
# 1) DOW hot vectors (nb_days, 7)
all_X += [self.X_dow]
# 2) mountainess (nb_days, nb_cells, nb_altitudes)
all_X += [
np.repeat(
np.array([
self.flights_data.cellKAltitudeMountainess[c][alt]
for c in model_content.cells()
for alt in range(self.nb_altitudes)
],
dtype=np.float).reshape(1, model_content.nbCells(),
self.nb_altitudes),
self.nb_days, 0)
]
# 3) other (nb_days, nb_cells, nbHours, dimOther)
all_X += [
np.stack([
np.stack([
self.X_other[h]
[[c * self.nb_days + d for d in range(self.nb_days)], :]
for c in model_content.cells()
], 1) for h in range(3)
], 2)
]
# 4) rain (nb_days, nb_cells, nbHours, dimRain)
all_X += [
np.stack([
np.stack([
self.X_humidity[h]
[[c * self.nb_days + d for d in range(self.nb_days)], :]
for c in model_content.cells()
], 1) for h in range(3)
], 2)
]
# 5) wind (nb_days, nb_cells, nb_altitudes, nbHours, dimWind)
all_X += [
np.stack([
np.stack([
np.stack([
self.X_wind[h]
[[c * self.nb_days + d for d in range(self.nb_days)],
alt * self.wind_dim:(alt + 1) * self.wind_dim]
for c in model_content.cells()
], 1) for alt in range(self.nb_altitudes)
], 2) for h in range(3)
], 3)
]
# debug, print shapes
if False:
for ix in range(len(all_X)):
print(str(ix).rjust(3), all_X[ix].shape)
return all_X
def train(self,
lr_schedule,
use_validation_set=False,
train_crossability_only=False):
Verbose.print_arguments()
self.trained_model.compile([]) # re-compile for removing metrics
#self.trained_model.model.summary()
if self.model_type == ModelType.CELLS:
if train_crossability_only:
train.trained_model.freeze_all_but_crossability()
else:
train.trained_model.unfreeze_all()
#===================================================================
# Training/validation set split
#===================================================================
if use_validation_set:
if False: # random validation set
validation_split = 0.2
permutation = np.random.permutation(self.nb_days)
days_to_use_validation = permutation[
0:int(round(self.nb_days * validation_split))]
else: # non-random and well distributed validation set
days_to_use_validation = np.array(range(0, self.nb_days, 2))
else:
if True: # during process tuning, still check the second optim with validation data
days_to_use_validation = np.array(range(0, self.nb_days, 6))
else:
days_to_use_validation = np.array([])
days_to_use_training = np.array([
s for s in range(self.nb_days) if s not in days_to_use_validation
])
#===================================================================
# Run training
#===================================================================
history = self.trained_model.model.fit(
x=[x[days_to_use_training, ...] for x in self.all_X],
y=[y[days_to_use_training, ...] for y in self.all_Y],
validation_data=(
[x[days_to_use_validation, ...] for x in self.all_X],
[y[days_to_use_validation, ...] for y in self.all_Y])
if days_to_use_validation.size > 0 else None,
epochs=lr_schedule[2],
batch_size=32,
shuffle=True,
verbose=0,
callbacks=[
self.schedule_with_params(lr_init=lr_schedule[0],
lr_end=lr_schedule[1],
nb_epochs=lr_schedule[2]),
MyTrainingLogger(
self.model_type, self.models_directory + "/" +
str(self.model_type).split(".")[-1] + ".log")
])
if 'val_loss' in history.history:
return np.mean(history.history['val_loss'][-5:])
else:
return None
def save(self):
Verbose.print_arguments()
self.trained_model.save_all_weights()
def set_prediction_population(self): Verbose.print_arguments() prediction_popu = self.__get_prediction_population() self.trainedModel.set_population_value(prediction_popu)
def compute_prediction_file_cells( cells_forecasts, prediction_filename_for_tiler, distinct_latitudes, distinct_longitudes, meteo_matrix, crops, meteo_params, grid_desc_predictions, export_for_tiler=True): Verbose.print_arguments() # ====================================================================================== # Retrieve geopotential height at 12h indices in meteoParams # # inputs: # - meteoParams # - parameters_geopotential_height # outputs: # - geopotential_height_indices # ====================================================================================== geopotential_height_indices = [] for geopotential_height_param in GfsData().parameters_geopotential_height: for kParam, param in enumerate(meteo_params): if param == (12,) + geopotential_height_param: geopotential_height_indices += [kParam] break assert (len(geopotential_height_indices) == len(GfsData().parameters_geopotential_height)) # ====================================================================================== # Retrieve wind at 12h # # inputs: # - meteoParams # - parameters_wind # outputs: # - wind_indices # ====================================================================================== wind_indices = [] for wind_param in GfsData().parameters_wind: for kParam, param in enumerate(meteo_params): if param == (12,) + wind_param: wind_indices += [kParam] break assert (len(wind_indices) == len(GfsData().parameters_wind)) # ======================================================================== # Create and fill prediction grid # # params: # - grid_desc_predictions # inputs: # - distinct_latitudes # - distinct_longitudes # - meteo_matrix # - geopotential_height_indices # - predictions # outputs: # - prediction_grid # ======================================================================== prediction_grid = GridLatLon(grid_desc_predictions[0], grid_desc_predictions[1], grid_desc_predictions[2], grid_desc_predictions[3]) p = 0 for crop in crops: # number of data rectangles to read in the GRIB grid lat_range = (crop[0], crop[1]) lon_range = (crop[2], crop[3]) Verbose.print_text(1, "lats: " + str(distinct_latitudes[lat_range[0]]) +" , "+ str(distinct_latitudes[lat_range[1]-1])) Verbose.print_text(1, "lons: " + str(distinct_longitudes[lon_range[0]]) +" , "+ str(distinct_longitudes[lon_range[1]-1])) for ilat in range(lat_range[0], lat_range[1]): for ilon in range(lon_range[0], lon_range[1]): predictions_data_sample = () # Add geopential heights (not predicted, simply meteo variables) predictions_data_sample += (meteo_matrix[p, geopotential_height_indices[0]], # 0 meteo_matrix[p, geopotential_height_indices[1]], # 1 meteo_matrix[p, geopotential_height_indices[2]], # 2 meteo_matrix[p, geopotential_height_indices[3]], # 3 meteo_matrix[p, geopotential_height_indices[4]]) # 4 predictions_data_sample += (cells_forecasts[0][p,0,0], # flyability at 1000 # 5 cells_forecasts[0][p,0,1], # flyability at 900 # 6 cells_forecasts[0][p,0,2], # flyability at 800 # 7 cells_forecasts[0][p,0,3], # flyability at 700 # 8 cells_forecasts[0][p,0,4], # flyability at 600 # 9 np.mean(cells_forecasts[1][p,0,0:5]),# crossability # 10 cells_forecasts[2][p,0,0], # wind at 1000 # 11 cells_forecasts[2][p,0,1], # wind at 900 # 12 cells_forecasts[2][p,0,2], # wind at 800 # 13 cells_forecasts[2][p,0,3], # wind at 700 # 14 cells_forecasts[2][p,0,4], # wind at 600 # 15 np.mean(cells_forecasts[3][p,0,0:5]), # humidity # 16 0.0) # Attractiveness # 17 predictions_data_sample += (meteo_matrix[p, wind_indices[0 + 0]], # U 1000 12h # 18 meteo_matrix[p, wind_indices[2 + 0]], # U 900 12h # 19 meteo_matrix[p, wind_indices[4 + 0]], # U 800 12h # 20 meteo_matrix[p, wind_indices[6 + 0]], # U 700 12h # 21 meteo_matrix[p, wind_indices[8 + 0]], # U 600 12h # 22 meteo_matrix[p, wind_indices[0 + 1]], # V 1000 12h # 23 meteo_matrix[p, wind_indices[2 + 1]], # V 900 12h # 24 meteo_matrix[p, wind_indices[4 + 1]], # V 800 12h # 25 meteo_matrix[p, wind_indices[6 + 1]], # V 700 12h # 26 meteo_matrix[p, wind_indices[8 + 1]]) # V 600 12h # 27 # Push the sample prediction_grid.add(distinct_latitudes[ilat], distinct_longitudes[ilon], predictions_data_sample) p += 1 # ======================================================================== # Export predictions for tiler # # params: # - predictionFilenameForTiler # inputs: # - prediction_grid # ======================================================================== accessors = [lambda x, ix=ixVal: sum([xx[ix] for xx in x]) / len(x) for ixVal in range(len(predictions_data_sample))] if export_for_tiler: prediction_grid.export_data_for_tiler(prediction_filename_for_tiler, accessors) else: prediction_grid.export_json(prediction_filename_for_tiler, accessors)
def set_trained_spots(self):
Verbose.print_arguments()
self.__set_trained({0: [0]})