def thick_cloud_test(is_land, vis, lir, cli_mu, cli_epsilon, quick_test=False): ''' Probably the most important test :param is_land: :param lir: :param cli_mu: :param cli_epsilon: :param quick_test :return: ''' if quick_test: return opaque_cloud_test(cli_mu, cli_epsilon) else: not_coast = decrease_connectivity_2(decrease_connectivity_2(is_land)) | \ decrease_connectivity_2(decrease_connectivity_2(~is_land)) mask = (cli_mu == -10) # WARNING: ERROR IN THE NEXT LINES, TO BE FIXED !! thresh_coherence_lir_land = compute_lir_texture_land_threshold() sd_lir = texture_test(lir, mask) heterogeneous = (sd_lir > thresh_coherence_lir_land) cli_epsilon_var = get_cloud_index_positive_variability_7d(sd_lir, mask, read_satellite_step()) thresh_epsilon_var = 0.2 del sd_lir broken_cloud_not_coasts = not_coast & heterogeneous & (cli_epsilon_var > thresh_epsilon_var) del cli_epsilon_var opaque = opaque_cloud_test(cli_mu, cli_epsilon) fog = opaque & (vis > 0.5) opaque_heterogeneous_cloud = heterogeneous & opaque return broken_cloud_not_coasts | opaque_heterogeneous_cloud | fog
def check_gaussian_hypothesis(latitudes, longitudes, begin, end, method='none'): from tomas_outputs import reduce_tomas_2_classes, get_tomas_outputs from decision_tree import reduce_two_classes, get_classes_v1_point, reduce_classes from get_data import get_features snow = get_features('visible', latitudes, longitudes, begin, end, 'abstract')[:, :, :, 0] # snow=feat[:,:,:,0] # var=feat[:,:,:,1] # del feat bb = get_bbox(latitudes[0], latitudes[-1], longitudes[0], longitudes[-1]) # visualize_map_time(snow, bb, vmin=0, vmax=1) dmask = dawn_day_test(get_zenith_angle(get_times_utc(begin, end, read_satellite_step(), 1), latitudes, longitudes)) if method == 'tomas': cloud = (reduce_tomas_2_classes(get_tomas_outputs( begin, end, latitudes[0], latitudes[-1], longitudes[0], longitudes[-1] )) == 1) snow = snow[~cloud] # visualize_map_time(cloud, bb) elif method == 'ped': classes = reduce_classes(get_classes_v1_point( latitudes, longitudes, begin, end )) cloud = (reduce_two_classes(classes) == 1) # visualize_map_time(cloud, bb) del classes snow = snow[~cloud] snow = snow[dmask & (snow > -9)] visualize_hist(snow.flatten(), 'level of snow', precision=100)
def maybe_cloud_after_all(is_land, is_supposed_free, vis): ''' the "unstable albedo test" classifies as cloud the previously "allegedly cloud-free and snow-free pixels" which are much brighter than expected :param is_land: :param is_supposed_free: :param vis: :return: a boolean matrix which is True if the pixel passes this visible cloud test but not the previous cloud tests ''' # apply only for a few consecutive days is_supposed_free_for_long = is_supposed_free & np.roll(is_supposed_free, -1) & np.roll(is_supposed_free, 2) & \ is_supposed_free & np.roll(is_supposed_free, -2) & np.roll(is_supposed_free, 2) (slots, lats, lons) = vis.shape slot_per_day = get_nb_slots_per_day(read_satellite_step(), 1) entire_days = slots / slot_per_day assert entire_days < 11, 'please do not apply this test on strictly more than 10 days' vis_copy = vis.copy() vis_copy[~is_supposed_free_for_long] = 100 supposed_clear_sky = np.min( apply_rolling_on_time(vis_copy, 5, 'mean').reshape((entire_days, slot_per_day, lats, lons)), axis=0) del vis_copy vis = vis.reshape((entire_days, slot_per_day, lats, lons)) # from quick_visualization import visualize_map_time # visualize_map_time(supposed_clear_sky, typical_bbox(seed)) # visualize_map_time(is_supposed_free & land_visible_test(is_land, vis, supposed_clear_sky).reshape((slots, lats, lons)), typical_bbox()) return is_supposed_free & land_visible_test(is_land, vis, supposed_clear_sky).reshape((slots, lats, lons))
def read_channels(channels, latitudes, longitudes, dfb_beginning, dfb_ending, slot_step=1): dir, pattern = read_channels_dir_and_pattern() satellite = read_satellite_name() satellite_step = read_satellite_step() nb_slots = get_nb_slots_per_day(satellite_step, slot_step) patterns = [ pattern.replace("{SATELLITE}", satellite).replace('{CHANNEL}', chan) for chan in channels ] nb_days = dfb_ending - dfb_beginning + 1 content = np.empty( (nb_slots * nb_days, len(latitudes), len(longitudes), len(patterns))) start = read_start_slot() for k in range(len(patterns)): pattern = patterns[k] chan = channels[k] dataset = DataSet.read( dirs=dir, extent={ 'latitude': latitudes, 'longitude': longitudes, 'dfb': { 'start': dfb_beginning, 'end': dfb_ending, "end_inclusive": True, 'start_inclusive': True, }, 'slot': np.arange(start, start + nb_slots, step=slot_step) }, file_pattern=pattern, variable_name=chan, fill_value=np.nan, interpolation='N', max_processes=0, ) data = dataset['data'].data day_slot_b = 0 day_slot_e = nb_slots for day in range(nb_days): content[day_slot_b:day_slot_e, :, :, k] = data[day] day_slot_b += nb_slots day_slot_e += nb_slots return content
def prepare_angles_features_bom_labels(seed, selected_slots): beginning, ending, latitude_beginning, latitude_end, longitude_beginning, longitude_end = typical_input( seed) times = get_times_utc(beginning, ending, read_satellite_step(), slot_step=1) latitudes, longitudes = get_latitudes_longitudes(latitude_beginning, latitude_end, longitude_beginning, longitude_end) a, b, c = len(times), len(latitudes), len(longitudes) nb_features_ = 8 features = np.empty((a, b, c, nb_features_)) angles, vis, ndsi, mask = get_features('visible', latitudes, longitudes, beginning, ending, output_level='ndsi', slot_step=1, gray_scale=False) test_angles = dawn_day_test(angles) land_mask = typical_land_mask(seed) mask = ((test_angles | land_mask) | mask) ndsi[mask] = -10 features[:, :, :, 5] = test_angles features[:, :, :, 6] = land_mask del test_angles, land_mask, mask features[:, :, :, 3] = vis features[:, :, :, 4] = ndsi del vis, ndsi features[:, :, :, :3] = get_features('infrared', latitudes, longitudes, beginning, ending, output_level='abstract', slot_step=1, gray_scale=False)[:, :, :, :3] features[:, :, :, 7] = (typical_static_classifier(seed) >= 2) if selected_slots is not None: return angles[selected_slots], features[selected_slots], selected_slots return angles, features, selected_slots
def read_classes(latitudes, longitudes, dfb_beginning, dfb_ending, slot_step=1): dir, pattern = read_indexes_dir_and_pattern('classes') satellite_step = read_satellite_step() nb_slots = get_nb_slots_per_day(satellite_step, slot_step) nb_days = dfb_ending - dfb_beginning + 1 content = np.empty((nb_slots * nb_days, len(latitudes), len(longitudes))) dataset = DataSet.read( dirs=dir, extent={ 'latitude': latitudes, 'longitude': longitudes, 'dfb': { 'start': dfb_beginning, 'end': dfb_ending, "end_inclusive": True, 'start_inclusive': True, }, 'slot': { "enumeration": np.arange(0, nb_slots, step=slot_step), "override_type": "slot" }, }, file_pattern=pattern, variable_name='Classes', fill_value=np.nan, interpolation='N', max_processes=0, ) data = dataset['data'].data day_slot_b = 0 day_slot_e = nb_slots for day in range(nb_days): content[day_slot_b:day_slot_e, :, :] = data[day] day_slot_b += nb_slots day_slot_e += nb_slots return content
def recognize_cloud_shade(vis, real_cloud_mask, cos_zen, th=0.2): nb_slots_per_day = get_nb_slots_per_day(read_satellite_step(), slot_step=1) (nb_slots, nb_lats, nb_lons) = np.shape(vis)[0:3] nb_days = nb_slots / nb_slots_per_day detected_cloud_shade = np.zeros_like(vis, dtype=bool) for slot in range(nb_slots_per_day): for lat in range(nb_lats): for lon in range(nb_lons): l = [] for day in range(nb_days): s = day * nb_slots_per_day + slot if not real_cloud_mask[s, lat, lon]: l.append(vis[s, lat, lon]) if len(l) > 0: supposed_albedo = np.median(l) for day in range(nb_days): s = day * nb_slots_per_day + slot if not real_cloud_mask[s, lat, lon] and (supposed_albedo - vis[s, lat, lon]) \ > th * cos_zen[s, lat, lon]: # not real_cloud_mask[s, lat, lon]: detected_cloud_shade[s, lat, lon] = True return detected_cloud_shade
def train_solar_model(zen, classes, features, method_learning, meta_method, pca_components, training_rate): t_beg = time() nb_days_training = len(zen) / get_nb_slots_per_day(read_satellite_step(), 1) select = mask_temporally_stratified_samples( zen, training_rate, coef_randomization * nb_days_training) features = reshape_features(features) select = select.flatten() nb_features = features.shape[-1] if pca_components is not None: nb_features = pca_components features = immediate_pca(features, pca_components) var = features[:, 0][select] training = np.empty((len(var), nb_features)) training[:, 0] = var for k in range(1, nb_features): training[:, k] = features[:, k][select] del var if method_learning == 'knn': estimator = create_knn() elif method_learning == 'bayes': estimator = create_naive_bayes() elif method_learning == 'mlp': estimator = create_neural_network() elif method_learning == 'forest': estimator = create_random_forest() else: estimator = create_decision_tree() if meta_method == 'bagging': estimator = create_bagging_estimator(estimator) model = fit_model(estimator, training, classes.flatten()[select]) del training t_train = time() print 'time training:', t_train - t_beg save(path_, model) t_save = time() print 'time save:', t_save - t_train
def prepare_temperature_mask(lats, lons, beginning, ending, slot_step=1): ''' Create a temperature mask which has the same temporal sampling than spectral channels :param lats: latitudes array :param lons: longitudes array :param beginning: dfb beginning sampling :param ending: dfb ending sampling :param slot_step: slot sampling chosen by the user (probably 1) :return: ''' satellite_step = read_satellite_step() nb_slots = get_nb_slots_per_day(satellite_step, slot_step) * (ending - beginning + 1) temperatures = read_temperature_forecast(lats, lons, beginning, ending) to_return = empty((nb_slots, len(lats), len(lons))) for slot in range(nb_slots): try: nearest_temp_meas = int(0.5 + satellite_step * slot_step * slot / 60) to_return[slot] = temperatures[nearest_temp_meas] + 273.15 except IndexError: nearest_temp_meas = int(satellite_step * slot_step * slot / 60) to_return[slot] = temperatures[nearest_temp_meas] + 273.15 return to_return
if __name__ == '__main__': beginning = 13517 + 60 nb_days = 2 ending = beginning + nb_days - 1 latitude_beginning = 40. latitude_ending = 45. longitude_beginning = 125. longitude_ending = 130. latitudes, longitudes = get_latitudes_longitudes(latitude_beginning, latitude_ending, longitude_beginning, longitude_ending) times = get_times_utc(beginning, ending, read_satellite_step(), 1) nb_slots_per_day = get_nb_slots_per_day(read_satellite_step(), 1) res = 1 / 33. w = len(longitudes) h = len(latitudes) bb = bounding_box(xmin=longitude_beginning, xmax=longitude_ending, ymin=latitude_beginning, ymax=latitude_ending, resolution=res, width=w, height=h) coef = (2 * np.pi / 360.) # for lon in lons_1d:
def read_labels(label_type, lat_beginning, lat_ending, lon_beginning, lon_ending, dfb_beginning, dfb_ending, slot_step=1, keep_holes=True): ''' this function assume labels are "well named", starting with YYYYMMDDHHMMSS where SS=0 60*HH+MM is a multiple of satellite time step :param label_type is 'CSP' (=clear sy mask) or '' :param dfb_beginning: :param dfb_ending: :param slot_step: :return: ''' label_type = label_type.upper() assert label_type in ['CSP', 'CT'], 'the type of labels you asked for does not exist' satellite_step = read_satellite_step() nb_slots_per_day = get_nb_slots_per_day(satellite_step, slot_step) res = 1 / 33. nb_lats = int((lat_ending - lat_beginning) / res) nb_lons = int((lon_ending - lon_beginning) / res) dir_ = read_labels_dir(label_type) var_ = {'CSP': 'clear_sky_probability', 'CT': 'cloud_type'}[label_type] if read_satellite_name() == 'H08': lonmin = 115. latmax = 60 if read_satellite_name() == 'GOES16': lonmin = -10. latmax = 60 lat_beginning_ind = int((latmax - lat_ending) / res) lat_ending_ind = int((latmax - lat_beginning) / res) lon_beginning_ind = int((lon_beginning - lonmin) / res) lon_ending_ind = int((lon_ending - lonmin) / res) selected_slots = [] if keep_holes: shape_ = ((dfb_ending - dfb_beginning + 1) * nb_slots_per_day, nb_lats, nb_lons) to_return = -10 * ones(shape_) else: to_return = [] sat_name = read_satellite_name() if sat_name == 'GOES16': suffixe = '_LATLON-GOES16.nc' elif sat_name == 'H08': suffixe = '_LATLON-HIMAWARI8-AHI.nc' for dfb in range(dfb_beginning, dfb_ending + 1): pre_pattern = dfb2yyyymmdd(dfb) for slot_of_the_day in range(nb_slots_per_day): try: real_slot = slot_of_the_day + (dfb - dfb_beginning) * nb_slots_per_day total_minutes = satellite_step * slot_step * slot_of_the_day hours, minutes = total_minutes / 60, total_minutes % 60 if len(str(hours)) == 1: hours = '0' + str(hours) if len(str(minutes)) == 1: minutes = '0' + str(minutes) filename = pre_pattern + str(hours) + str(minutes) + '00-' + label_type + suffixe content = Dataset(str(os.path.join(dir_, filename))) if keep_holes: to_return[real_slot] = \ content.variables[var_][lat_beginning_ind: lat_ending_ind, lon_beginning_ind:lon_ending_ind] else: to_return.append( content.variables[var_][lat_beginning_ind: lat_ending_ind, lon_beginning_ind:lon_ending_ind]) selected_slots.append(real_slot) except Exception as e: # the data for this slot does not exist or has not been load print e pass print to_return[to_return != -10] return asarray(to_return), selected_slots
def flagged_cloud_and_thermally_stable(is_land, dawn_day_clouds, lir_observed, fir_observed, past_lir_observed, past_fir_observed): number_slots_45_min = 45. / read_satellite_step() number_slots_1_hour = 60. / read_satellite_step() return roll(dawn_day_clouds, number_slots_45_min) & roll(dawn_day_clouds, number_slots_1_hour) & \ thermal_stability_test(is_land, lir_observed, fir_observed, past_lir_observed, past_fir_observed)
def get_classes_v1_point(latitudes, longitudes, beginning, ending, slot_step=1, shades_detection=False): visible_features = get_features( 'visible', latitudes, longitudes, beginning, ending, 'abstract', slot_step, gray_scale=False, ) infrared_features = get_features( 'infrared', latitudes, longitudes, beginning, ending, 'abstract', slot_step, gray_scale=False, ) # classes: classified_cli, snow over the ground, other (ground, sea...), unknown visibles = get_features( 'visible', latitudes, longitudes, beginning, ending, 'channel', slot_step, gray_scale=False, ) angles = get_zenith_angle( get_times_utc(beginning, ending, read_satellite_step(), 1), latitudes, longitudes) # bright = (classify_brightness(visible_features[:, :, :, 0]) == 1) & (visibles[:,:,:,1] > 0.25*angles) # bright = (visible_features[:, :, :, 0] > 0.3) & (visibles[:,:,:,1] > 0.25*angles) bright = segmentation('watershed-3d', visible_features[:, :, :, 0], thresh_method='static', static=0.3) # & (visibles[:,:,:,1] > 0.2*angles) # negative_variable_brightness = (classifiy_brightness_variability(visible_features[:, :, :, 1]) == 1) # positive_variable_brightness = (classifiy_brightness_variability(visible_features[:, :, :, 2]) == 1) negative_variable_brightness = (visible_features[:, :, :, 1] > 0.2) & bright positive_variable_brightness = (visible_features[:, :, :, 2] > 0.2) & bright # from classification_cloud_index import classify_cloud_covertness, classify_cloud_variability cold = (infrared_features[:, :, :, 2] == 1) thin_clouds = (infrared_features[:, :, :, 0] > 0.2) obvious_clouds = (infrared_features[:, :, :, 1] > 10) # obvious_clouds = (classify_cloud_covertness(infrared_features[:, :, :, 0]) == 1) & (infrared_features[:, :, :, 1] > 1) snow = bright & ~negative_variable_brightness & ~positive_variable_brightness & ~obvious_clouds & ~cold del bright (nb_slots, nb_latitudes, nb_longitudes) = np.shape(visible_features)[0:3] classes = np.zeros((nb_slots, nb_latitudes, nb_longitudes)) begin_affectation = time() classes[(visibles[:, :, :, 1] > 0.5 * angles) & (visibles[:, :, :, 0] > 0.1 * angles)] = 3 # before all the other classes (VERY IMPORTANT) classes[(infrared_features[:, :, :, 0] == -10 )] = 13 # before all the other classes (important) classes[(visible_features[:, :, :, 3] == 1 )] = 12 # before all the other classes (important) classes[snow] = 5 # class ground snow or ice classes[positive_variable_brightness] = 6 classes[negative_variable_brightness] = 7 classes[obvious_clouds] = 1 classes[thin_clouds] = 2 # classes[bright & ~negative_variable_brightness & warm] = 10 # classes[bright & negative_variable_brightness & warm] = 9 classes[cold] = 8 # classes[obvious_clouds & bright] = 3 if shades_detection: cloudy = (classes != 0) & (classes != 5) print 'launch shades analysis' shades_detection = recognize_cloud_shade( visibles[:, :, :, 1], cloudy, get_zenith_angle( get_times_utc(beginning, ending, read_satellite_step(), slot_step=1), latitudes, longitudes)) print 'completed shades analysis' classes[shades_detection] = 11 print 'time affectation', time() - begin_affectation print 'allegedly uncovered lands' print 'obvious clouds:1' print 'thin clouds:2' print 'visible but undecided:3' print 'slight clouds and bright:4' print 'snowy:5' print 'snowy clouds:6' print 'covered snow:7' print 'cold:8' # print 'hot bright corpses:9' # print 'hot bright variable corpses:10' print 'foggy:11' print 'sea clouds identified by visibility:12' #### WARNING: what about icy lakes??? #### # print 'suspect high snow index (over sea / around sunset or sunrise):13' print 'undefined:13' return classes
def get_classes_v2_image(latitudes, longitudes, beginning, ending, slot_step=1, method='otsu-3d', shades_detection=False): visible_features = get_features( 'visible', latitudes, longitudes, beginning, ending, True, slot_step, gray_scale=True, ) infrared_features = get_features( 'infrared', latitudes, longitudes, beginning, ending, True, slot_step, gray_scale=True, ) if method in ['watershed-2d', 'watershed-3d']: visible = get_features('visible', latitudes, longitudes, beginning, ending, 'abstract', slot_step, gray_scale=False) # visualize_map_time(segmentation_otsu_2d(vis), bbox) bright = (segmentation_otsu_2d(visible_features[:, :, :, 0]) & (visible[:, :, :, 1] > 0.35)) # visualize_map_time(bright, bbox) bright = segmentation(method, bright, thresh_method='binary') else: visible = get_features('visible', latitudes, longitudes, beginning, ending, 'abstract', slot_step, gray_scale=False) visible = segmentation( method, visible, 1, ) bright = (segmentation(method, visible_features[:, :, :, 0]) & visible) # negative_variable_brightness = visible_features[:, :, :, 1] > 25 # positive_variable_brightness = visible_features[:, :, :, 2] > 25 negative_variable_brightness = segmentation(method, visible_features[:, :, :, 1], thresh_method='static', static=30) positive_variable_brightness = segmentation(method, visible_features[:, :, :, 2], thresh_method='static', static=30) # slight_clouds = segmentation(method, infrared_features[:, :, :, 1]) # obvious_clouds = (infrared_features[:, :, :, 0] == 1) obvious_clouds = segmentation( method, infrared_features[:, :, :, 0]) & segmentation( method, infrared_features[:, :, :, 1], thresh_method='static', static=20) cold = (infrared_features[:, :, :, 2] == 1) # warm = (infrared_features[:, :, :, 2] == 1) # foggy: low snow index, good vis (nb_slots, nb_latitudes, nb_longitudes) = np.shape(visible_features)[0:3] classes = np.zeros((nb_slots, nb_latitudes, nb_longitudes)) # clouds = obvious_clouds | slight_clouds begin_affectation = time() classes[(infrared_features[:, :, :, 0] == -10 )] = 13 # before all the other classes (important) classes[(visible_features[:, :, :, 3] == 1 )] = 12 # before all the other classes (important) classes[bright & ~obvious_clouds & ~negative_variable_brightness] = 5 # class ground snow or ice classes[bright & positive_variable_brightness] = 6 classes[bright & negative_variable_brightness] = 7 # classes[bright & ~negative_variable_brightness & warm] = 10 # classes[bright & negative_variable_brightness & warm] = 9 classes[cold] = 8 # WARNING: slight clouds AND obvious clouds => obvious clouds # classes[slight_clouds & bright] = 4 # classes[slight_clouds & ~bright] = 2 classes[obvious_clouds & ~bright] = 1 classes[obvious_clouds & bright] = 3 if shades_detection: cloudy = (classes != 0) & (classes != 5) print 'launch shades analysis' shades_detection = recognize_cloud_shade( visible[:, :, :, 1], cloudy, get_zenith_angle( get_times_utc(beginning, ending, read_satellite_step(), slot_step=1), latitudes, longitudes)) print 'completed shades analysis' classes[shades_detection] = 11 # classes[bright & (infrared_features[:, :, :, 3] == 1)] = 7 # = cold and bright. opaque obvious_clouds or cold obvious_clouds over snowy stuff # classes[persistent_snow & (obvious_clouds | cold_opaque_clouds)] = 4 # classes[foggy] = 11 print 'time affectation', time() - begin_affectation print 'uncovered lands: 0' print 'obvious clouds:1' print 'slight clouds or sunrise/sunset clouds:2' print 'clouds and bright:3' print 'slight clouds and bright:4' print 'snowy:5' print 'snowy clouds:6' print 'covered snow:7' print 'cold not bright (cold thin water clouds?):8' # print 'hot bright corpses:9' # print 'hot bright variable corpses:10' print 'shades:11' print 'sea clouds identified by visibility:12' #### WARNING: what about icy lakes??? #### # print 'suspect high snow index (over sea / around sunset or sunrise):13' print 'undefined:13' return classes
def get_features( type_channels, latitudes, longitudes, dfb_beginning, dfb_ending, output_level, slot_step=1, gray_scale=False, ): """ Wrapper returning the predictors for cloud-classification model, using the desired type of inpit light channels :param type_channels: :param latitudes: :param longitudes: :param dfb_beginning: :param dfb_ending: :param output_level: :param slot_step: :param gray_scale: :return: """ satellite_step = read_satellite_step() is_land = read_land_mask(latitudes, longitudes) temperatures = read_temperature_forecast(latitudes, longitudes, dfb_beginning, dfb_ending) times = get_times_utc(dfb_beginning, dfb_ending, satellite_step, slot_step) channels = read_channels_names(type_channels) if type_channels == 'visible': content_visible = read_channels(channels, latitudes, longitudes, dfb_beginning, dfb_ending, slot_step) return visible_outputs( times, latitudes, longitudes, is_land, content_visible, satellite_step, slot_step, output_level, gray_scale, ) elif type_channels == 'infrared': content_infrared = read_channels(channels, latitudes, longitudes, dfb_beginning, dfb_ending, slot_step) return infrared_outputs( times, latitudes, longitudes, temperatures, content_infrared, satellite_step, slot_step, output_level, gray_scale, ) else: raise AttributeError( 'The type of channels should be \'visible\' or \'infrared\'')