def __load(self):
self.spots = BinObj.load("spots_merged") # ("name", lat, lon)
self.spots_by_cell = BinObj.load("spots_by_cell")
self.flights_by_spot = BinObj.load("flights_by_spot")
self.flights_by_cell_day_spot = BinObj.load("flights_by_cell_day_spot")
self.meteo_days = BinObj.load("meteo_days")
def __init__(self):
self.cell_resolution = 1.0
self.flights_by_cell_day = BinObj.load(
"flights_by_cell_day"
) # [cell in range(0,318257))] -> [('yyyy-mm-dd hh:mm:ss', (score, alt, plaf, lat, lon, takeoff_alt, mountainess)), ...]
self.cellKAltitudeMountainess = BinObj.load(
"mountainess_by_cell_alt"
) # [cell][kAlt] -> mountainess (in [0,1])
self.nb_cells = len(BinObj.load("sorted_cells_latlon"))
self.nb_days = len(self.flights_by_cell_day) // self.nb_cells
def __init__(self):
self.sorted_cells = BinObj.load(
"sorted_cells") # [kCell] -> (row, col)
self.meteo_days = BinObj.load("meteo_days")
self.meteo_params = BinObj.load("meteo_params")
self.meteo_content_by_cell_day = BinObj.load(
"meteo_content_by_cell_day")
self.nb_days = len(self.meteo_days)
self.nb_cells = len(self.sorted_cells)
def __save(self):
assert (not self.data_not_loaded())
BinObj.save(self.spots, "spots_merged")
BinObj.save(self.spots_by_cell, "spots_by_cell")
BinObj.save(self.flights_by_spot, "flights_by_spot")
BinObj.save(self.flights_by_cell_day_spot, "flights_by_cell_day_spot")
def __init__(self, models_directory, problem_formulation):
if Forecast.on_the_server():
root = "/home/antoine/GIT/Paraglidable/"
self.DEBUG_MODE = False
self.last_forecast_time_file_dir = "/tmp/lastForecastTime"
self.downloaded_forecasts_dir = "/tmp/forecasts"
self.prediction_filename_for_tiler = "/tmp/predictions.txt"
self.tiler_arguments_filename = "/tmp/tilerArguments.json"
self.tiles_dir = "/var/www/html/data/tiles"
self.tiler_program = root+"tiler/Tiler/Tiler"
self.tiler_cache_dir = root+"tiler/_cache"
self.background_tiles_dir = root+"tiler/background_tiles"
self.geo_json_borders = root+"tiler/data/Europe_africa_med_red.geo.json"
self.skipped_tiles = root+"tiler/data/skippedTiles.txt"
self.min_tiles_zoom = 5
self.max_tiles_zoom = 9
self.render_tiles = True
self.forced_meteo_files = None
elif Forecast.in_docker():
root = "/workspaces/Paraglidable/"
self.DEBUG_MODE = False
self.last_forecast_time_file_dir = "/tmp/lastForecastTime"
self.downloaded_forecasts_dir = "/tmp/forecasts"
self.prediction_filename_for_tiler = "/tmp/predictions.txt"
self.tiler_arguments_filename = "/tmp/tilerArguments.json"
self.tiles_dir = root+"www/data/tiles"
self.tiler_program = root+"tiler/Tiler/Tiler"
self.tiler_cache_dir = root+"tiler/_cache"
self.background_tiles_dir = root+"tiler/background_tiles"
self.geo_json_borders = root+"tiler/data/Europe_africa_med_red.geo.json"
self.skipped_tiles = root+"tiler/data/skippedTiles.txt"
self.min_tiles_zoom = 5
self.max_tiles_zoom = 8
self.render_tiles = True
self.forced_meteo_files = None
else:
sys.exit(1)
self.destination_forecast_file = os.path.join(self.downloaded_forecasts_dir, "%s")
self.problem_formulation = problem_formulation
self.grid_desc_predictions = (0.25, 0.25, -0.125, -0.125) # can be different from meteo grid
self.GFS_resolution = "0p25"
self.nb_days = 10
# Coordinates of the 2 data rectangles taken into the GRIB file
# Will fail when the grid will change in the GRIB files
# TODO use lat/lon bbox and get indices using distinctLatitudes and distinctLongitudes
# Le crop doit être un peu plus grand que celui du Tiler C++
# Il faut ajouter les tiles d'elevation aussi
self.crops = [(93, 274, 0, 140), (93, 274, 1394, 1440)]
os.makedirs(self.last_forecast_time_file_dir, exist_ok=True)
os.makedirs(self.downloaded_forecasts_dir, exist_ok=True)
self.meteoParams = BinObj.load("meteo_params")
self.models_directory = models_directory
def __init__(self, models_directory, model_type, problem_formulation):
self.model_type = model_type
self.problem_formulation = problem_formulation
# Params
self.wind_dim = 8
self.nb_altitudes = 5
self.all_cells = [c for c in range(80)]
self.models_directory = models_directory
# model
self.trained_model = TrainedModel(models_directory,
problem_formulation)
# All the training data
self.__loadTrainingData()
# Compute and save normalization at 12h
normalization_mean_other, normalization_std_other = Utils.compute_normalization_coeffs(
self.X_other[1])
normalization_mean_humidity, normalization_std_humidity = Utils.compute_normalization_coeffs(
self.X_humidity[1])
BinObj.save([
normalization_mean_other, normalization_std_other,
normalization_mean_humidity, normalization_std_humidity
], "normalization_%s" % str(self.model_type).split(".")[-1],
self.models_directory)
# Apply normalization
for h in range(3):
Utils.apply_normalization(self.X_other[h],
normalization_mean_other,
normalization_std_other)
Utils.apply_normalization(self.X_humidity[h],
normalization_mean_humidity,
normalization_std_humidity)
# wind is not normalized
# Training data for the wanted cells
self.all_X = None
self.all_Y = None
def __compute_spots_forecasts(self, models_directory, problem_formulation,
lats, lons, meteo_matrix, filename):
Verbose.print_arguments()
predict = Predict(models_directory, ModelType.SPOTS,
problem_formulation)
predict.set_meteo_data(meteo_matrix, GfsData().parameters_vector_all)
#=============================================================
# depend de GribReader.get_values_array(self, params, crops):
forecastCellsLine = {}
line = 0
for crop in self.crops:
for iLat in range(crop[0], crop[1]):
for iLon in range(crop[2], crop[3]):
forecastCellsLine[(iLat, iLon)] = line
line += 1
#=============================================================
# Compute or load cells_and_spots
#=============================================================
filename_cells_and_spots = "Forecast_cellsAndSpots_" + "_".join(
[str(crop[d]) for crop in self.crops for d in range(4)])
if not BinObj.exists(filename_cells_and_spots):
Verbose.print_text(
0,
"Generating precomputation file because of new crop, it may crash on the server... To be computed on my computer"
)
cells_and_spots = {}
# find forecast cell for each spot
# C'est comme le cellsAndSpots de Train sauf que les cellules sont les cells de forecast (32942 cells)
spots_data = SpotsData()
spots = spots_data.getSpots(range(80))
for kc, cell_spots in enumerate(spots):
for ks, spot in enumerate(cell_spots):
iCell = (np.abs(lats - spot.lat).argmin(),
np.abs(lons - spot.lon).argmin())
cellLine = forecastCellsLine[iCell]
kcks_spot = (
(kc, ks), spot.toDict()) # (training cell, ks)
if not cellLine in cells_and_spots:
cells_and_spots[cellLine] = [kcks_spot]
else:
cells_and_spots[cellLine] += [kcks_spot]
BinObj.save(cells_and_spots, filename_cells_and_spots)
else:
cells_and_spots = BinObj.load(filename_cells_and_spots)
#=============================================================
# Create a model with 1 cell of 1 spot
#=============================================================
predict.set_trained_spots()
#=============================================================
# Compute prediction for each spot, one by one
#=============================================================
spots_and_prediction = []
for kcslst, cslst in enumerate(cells_and_spots.items()):
meteoLine, spotsLst = cslst
for cs in spotsLst:
modelContent = ModelContent()
modelContent.add(cs[0][0], cs[0][1])
predict.trainedModel.load_all_weights(
modelContent) # TODO do not reload shared weights
predict.set_prediction_population()
NN_X = predict.get_X([meteoLine])
prediction = predict.trainedModel.model.predict(NN_X)[0][0]
spots_and_prediction += [
Spot((cs[1]['name'], cs[1]['lat'], cs[1]['lon']),
cs[1]['id'], cs[1]['nbFlights'], prediction)
] #[cs[1]]
#=============================================================
# Export all results
#=============================================================
Forecast.__export_spots_forecasts(spots_and_prediction, filename)
if nb_tiles != 118: # already computed
try:
meteo_files = [[
f for f in glob.glob(src_anl_dir + day.strftime("%Y-%m/") +
day.strftime("gfsanl_3_%Y%m%d_") +
("%02d00" % h) + "*.grb*")
if ".params" not in f
][0] for h in [6, 12, 18]]
#print(meteo_files, tiles_dir_this_day)
#continue
problem_formulation = ProblemFormulation.CLASSIFICATION
models_directory = "bin/models/CLASSIFICATION_1.0.0/"
meteoParams = BinObj.load("meteo_params")
grid_desc_predictions = (1.0, 1.0, -0.5, -0.5
) # can be different from meteo grid
crops = [(93 // 4, 274 // 4, 0, 136 // 4),
(93 // 4, 274 // 4, 1394 // 4, 1440 // 4)]
#======================================================================================
# Read weather data
#======================================================================================
distinct_latitudes, distinct_longitudes, meteo_matrix = ForecastData.readWeatherData(
meteo_files, crops)
#======================================================================================
# Compute and generate the predictions file for tiler
#======================================================================================
def set_meteo_data(self, meteo_matrix, parameters_vector_all):
Verbose.print_arguments()
# Parameters prefixed by hour
parameters_vector_all_with_hours = [(h,) + p for h in [6, 12, 18] for p in parameters_vector_all]
meteoParamsOther, meteoParamsWind, meteoParamsPrecipitation = TrainedModel.meteoParams()
# Params for other and wind
rainIdx = [[parameters_vector_all_with_hours.index(pX) for pX in meteoParamsPrecipitation[h]] for h in range(3)]
otherIdx = [[parameters_vector_all_with_hours.index(pX) for pX in meteoParamsOther[h]] for h in range(3)]
windIdx = [[parameters_vector_all_with_hours.index(pX) for pX in meteoParamsWind[h]] for h in range(3)]
self.X_rain = [ meteo_matrix[:, rainIdx[h] ] for h in range(3)]
self.X_other = [ meteo_matrix[:, otherIdx[h]] for h in range(3)]
self.X_wind = [Utils.convert_wind_matrix(meteo_matrix[:, windIdx[h] ], self.wind_dim) for h in range(3)]
# Load and apply normalization
normalization_mean_other, normalization_std_other, normalization_mean_rain, normalization_std_rain = BinObj.load("normalization_%s"%str(self.model_type).split(".")[-1], self.models_directory)
for h in range(3):
Utils.apply_normalization(self.X_other[h], normalization_mean_other, normalization_std_other)
Utils.apply_normalization(self.X_rain[h], normalization_mean_rain, normalization_std_rain)
def __compute_spots_information(self):
self.meteo_days = BinObj.load("meteo_days")
cells = BinObj.load("sorted_cells_latlon")
flights_by_cell_day = BinObj.load(
"flights_by_cell_day"
) # [cell] -> [('yyyy-mm-dd hh:mm:ss', (score, alt, plaf, lat, lon, takeoff_alt, mountainess)), ...]
self.spots_before_merge = BinObj.load("spots") # ("name", lat, lon)
self.spots = self.__fusion_of_close_spots(
self.spots_before_merge) # ("name", lat, lon)
cells_resolution = 1.0
spotsWidth = 0.1
nb_days = len(self.meteo_days)
nb_cells = len(cells)
nb_spots = len(self.spots)
#=====================================================================================
# Assign spots to cells
#=====================================================================================
self.spots_by_cell = [[] for c in range(nb_cells)
] # [kCell] -> [kSpot1, kSpot2, ...]
for ks, s in enumerate(self.spots):
for kc, c in enumerate(cells):
if abs(s[1] - c[0]) <= cells_resolution / 2.0 and abs(
s[2] - c[1]) <= cells_resolution / 2.0:
self.spots_by_cell[kc] += [ks]
#=====================================================================================
# Assign flights to cells
#=====================================================================================
flights_by_cell = [[
f for d in range(nb_days)
for f in flights_by_cell_day[d * nb_cells + c]
] for c in range(nb_cells)]
#=====================================================================================
# Assign flights to spots
#=====================================================================================
self.flights_by_spot = [[] for s in range(nb_spots)]
for kc in tqdm.tqdm(range(nb_cells)):
for f in flights_by_cell[kc]:
closest_dist = 1000000.0
closest_spot = -1
for s in self.spots_by_cell[kc]:
dlat = (self.spots[s][1] - f[1][3])
dlon = (self.spots[s][2] - f[1][4])
dist = dlat**2 + dlon**2
if dist < closest_dist and abs(dlat) <= spotsWidth and abs(
dlon) <= spotsWidth:
closest_spot = s
closest_dist = dist
self.flights_by_spot[closest_spot] += [f]
#=====================================================================================
# flights_by_cell_day_spot[kCell][kDay][kSpot] = list of flights
#=====================================================================================
# strday -> kday
strDay_to_kDay = {}
for kday, day in enumerate(self.meteo_days):
strDay_to_kDay[str(day)] = kday
self.flights_by_cell_day_spot = [[{} for d in range(nb_days)]
for c in range(nb_cells)]
for kc in tqdm.tqdm(range(nb_cells)):
for ks in self.spots_by_cell[kc]:
for f in self.flights_by_spot[ks]:
kd = strDay_to_kDay[str(f[0]).split(" ")[0]]
if ks not in self.flights_by_cell_day_spot[kc][kd]:
self.flights_by_cell_day_spot[kc][kd][ks] = []
self.flights_by_cell_day_spot[kc][kd][ks] += [f]
def __init__(self):
self.nb_days = len(BinObj.load("meteo_days"))
self.nb_cells = len(
BinObj.load("sorted_cells")) # [kCell] -> (row, col)
def __init__(self):
self.meteo_days = BinObj.load("meteo_days")
self.nb_days = len(self.meteo_days)