def low_pass_filter_3d(array, cutoff, omega=0):
(a, b, c) = np.shape(array)
filt = np.empty((b, c))
filtered_spatial = np.empty((a, b, c))
# grs = empty((a, b, c, 4))
# grsq = empty((a,b,c,2))
for k in range(len(array)):
fft2 = fftpack.fft2(array[k, :, :])
filt[abs(fft2) < cutoff - omega] = 1
filt[abs(fft2) > cutoff + omega] = 0
mask = (cutoff - omega < abs(fft2)) & (abs(fft2) < cutoff + omega)
filt[mask] = 0.5 * (1 - np.sin(np.pi * (abs(fft2[mask]) - cutoff) /
(2 * omega)))
fft2 = fft2 * filt
filtered_spatial[k] = fftpack.ifft2(fft2)
return filtered_spatial
def read_channels(channels,
latitudes,
longitudes,
dfb_beginning,
dfb_ending,
slot_step=1):
dir, pattern = read_channels_dir_and_pattern()
satellite = read_satellite_name()
satellite_step = read_satellite_step()
nb_slots = get_nb_slots_per_day(satellite_step, slot_step)
patterns = [
pattern.replace("{SATELLITE}", satellite).replace('{CHANNEL}', chan)
for chan in channels
]
nb_days = dfb_ending - dfb_beginning + 1
content = np.empty(
(nb_slots * nb_days, len(latitudes), len(longitudes), len(patterns)))
start = read_start_slot()
for k in range(len(patterns)):
pattern = patterns[k]
chan = channels[k]
dataset = DataSet.read(
dirs=dir,
extent={
'latitude': latitudes,
'longitude': longitudes,
'dfb': {
'start': dfb_beginning,
'end': dfb_ending,
"end_inclusive": True,
'start_inclusive': True,
},
'slot': np.arange(start, start + nb_slots, step=slot_step)
},
file_pattern=pattern,
variable_name=chan,
fill_value=np.nan,
interpolation='N',
max_processes=0,
)
data = dataset['data'].data
day_slot_b = 0
day_slot_e = nb_slots
for day in range(nb_days):
content[day_slot_b:day_slot_e, :, :, k] = data[day]
day_slot_b += nb_slots
day_slot_e += nb_slots
return content
def time_smoothing(array_3D_to_smoothen, nb_neighbours_smoothing=5):
smoothing = nb_neighbours_smoothing > 0
if smoothing:
time_start_smoothing = time.time()
shape = np.shape(array_3D_to_smoothen)
array = np.empty(shape)
for k in range(nb_neighbours_smoothing,
shape[0] - nb_neighbours_smoothing):
array[k] = np.mean(
array_3D_to_smoothen[k - nb_neighbours_smoothing:k +
nb_neighbours_smoothing + 1])
time_stop_smoothing = time.time()
print 'time smoothing', str(time_stop_smoothing - time_start_smoothing)
return array / (1 + 2 * nb_neighbours_smoothing)
else:
return array_3D_to_smoothen
def prepare_angles_features_bom_labels(seed, selected_slots):
beginning, ending, latitude_beginning, latitude_end, longitude_beginning, longitude_end = typical_input(
seed)
times = get_times_utc(beginning,
ending,
read_satellite_step(),
slot_step=1)
latitudes, longitudes = get_latitudes_longitudes(latitude_beginning,
latitude_end,
longitude_beginning,
longitude_end)
a, b, c = len(times), len(latitudes), len(longitudes)
nb_features_ = 8
features = np.empty((a, b, c, nb_features_))
angles, vis, ndsi, mask = get_features('visible',
latitudes,
longitudes,
beginning,
ending,
output_level='ndsi',
slot_step=1,
gray_scale=False)
test_angles = dawn_day_test(angles)
land_mask = typical_land_mask(seed)
mask = ((test_angles | land_mask) | mask)
ndsi[mask] = -10
features[:, :, :, 5] = test_angles
features[:, :, :, 6] = land_mask
del test_angles, land_mask, mask
features[:, :, :, 3] = vis
features[:, :, :, 4] = ndsi
del vis, ndsi
features[:, :, :, :3] = get_features('infrared',
latitudes,
longitudes,
beginning,
ending,
output_level='abstract',
slot_step=1,
gray_scale=False)[:, :, :, :3]
features[:, :, :, 7] = (typical_static_classifier(seed) >= 2)
if selected_slots is not None:
return angles[selected_slots], features[selected_slots], selected_slots
return angles, features, selected_slots
def read_classes(latitudes,
longitudes,
dfb_beginning,
dfb_ending,
slot_step=1):
dir, pattern = read_indexes_dir_and_pattern('classes')
satellite_step = read_satellite_step()
nb_slots = get_nb_slots_per_day(satellite_step, slot_step)
nb_days = dfb_ending - dfb_beginning + 1
content = np.empty((nb_slots * nb_days, len(latitudes), len(longitudes)))
dataset = DataSet.read(
dirs=dir,
extent={
'latitude': latitudes,
'longitude': longitudes,
'dfb': {
'start': dfb_beginning,
'end': dfb_ending,
"end_inclusive": True,
'start_inclusive': True,
},
'slot': {
"enumeration": np.arange(0, nb_slots, step=slot_step),
"override_type": "slot"
},
},
file_pattern=pattern,
variable_name='Classes',
fill_value=np.nan,
interpolation='N',
max_processes=0,
)
data = dataset['data'].data
day_slot_b = 0
day_slot_e = nb_slots
for day in range(nb_days):
content[day_slot_b:day_slot_e, :, :] = data[day]
day_slot_b += nb_slots
day_slot_e += nb_slots
return content
def read_temperature_forecast(latitudes, longitudes, dfb_beginning,
dfb_ending):
dir, pattern = read_mask_dir_and_pattern('temperature_forecast')
nb_days = dfb_ending - dfb_beginning + 1
content = np.empty((24 * nb_days, len(latitudes), len(longitudes)))
dataset = DataSet.read(
dirs=dir,
extent={
'latitude': latitudes % 180,
'longitude': longitudes % 360,
'day': {
'start': dfb_beginning,
'end': dfb_ending,
"end_inclusive": True,
'start_inclusive': True,
},
'time': {
"enumeration": np.linspace(0., 24., num=24, endpoint=False),
"override_type": "hours"
},
},
file_pattern=pattern,
variable_name='temp_2m',
fill_value=np.nan,
interpolation='N',
max_processes=0,
)
data = dataset['data'].data
day_slot_b = 0
day_slot_e = 24
for day in range(nb_days):
content[day_slot_b:day_slot_e, :, :] = data[day]
day_slot_b += 24
day_slot_e += 24
return content
def train_solar_model(zen, classes, features, method_learning, meta_method,
pca_components, training_rate):
t_beg = time()
nb_days_training = len(zen) / get_nb_slots_per_day(read_satellite_step(),
1)
select = mask_temporally_stratified_samples(
zen, training_rate, coef_randomization * nb_days_training)
features = reshape_features(features)
select = select.flatten()
nb_features = features.shape[-1]
if pca_components is not None:
nb_features = pca_components
features = immediate_pca(features, pca_components)
var = features[:, 0][select]
training = np.empty((len(var), nb_features))
training[:, 0] = var
for k in range(1, nb_features):
training[:, k] = features[:, k][select]
del var
if method_learning == 'knn':
estimator = create_knn()
elif method_learning == 'bayes':
estimator = create_naive_bayes()
elif method_learning == 'mlp':
estimator = create_neural_network()
elif method_learning == 'forest':
estimator = create_random_forest()
else:
estimator = create_decision_tree()
if meta_method == 'bagging':
estimator = create_bagging_estimator(estimator)
model = fit_model(estimator, training, classes.flatten()[select])
del training
t_train = time()
print 'time training:', t_train - t_beg
save(path_, model)
t_save = time()
print 'time save:', t_save - t_train
def prepare_angles_features_ped_labels(seed, keep_holes=True):
'''
:param latitude_beginning:
:param latitude_end:
:param longitude_beginning:
:param longitude_end:
:param beginning:
:param ending:
:param output_level:
:param seed:
:param keep_holes:
:return:
'''
beginning, ending, latitude_beginning, latitude_end, longitude_beginning, longitude_end = typical_input(
seed)
latitudes, longitudes = get_latitudes_longitudes(latitude_beginning,
latitude_end,
longitude_beginning,
longitude_end)
if keep_holes:
labels, selected_slots = read_labels_keep_holes(
'csp', latitude_beginning, latitude_end, longitude_beginning,
longitude_end, beginning, ending)
else:
labels, selected_slots = read_labels_remove_holes(
'csp', latitude_beginning, latitude_end, longitude_beginning,
longitude_end, beginning, ending)
angles, vis, ndsi, mask = get_features('visible',
latitudes,
longitudes,
beginning,
ending,
output_level='ndsi',
slot_step=1,
gray_scale=False)
a, b, c = angles.shape
nb_features_ = 8
features = np.empty((a, b, c, nb_features_))
test_angles = dawn_day_test(angles)
land_mask = typical_land_mask(seed)
ndsi[((test_angles | land_mask) | mask)] = -10
features[:, :, :, 5] = test_angles
features[:, :, :, 6] = land_mask
features[:, :, :, 3] = vis
features[:, :, :, 4] = ndsi
del vis, ndsi
cli_mu, cli_epsilon, mask_input_cli = get_features('infrared',
latitudes,
longitudes,
beginning,
ending,
output_level='cli',
slot_step=1,
gray_scale=False)
mask = ((test_angles | land_mask) | mask)
cli_mu[mask] = -10
cli_epsilon[mask] = -10
features[:, :, :, 0] = cli_mu
features[:, :, :, 1] = cli_epsilon
del mask, test_angles, land_mask, cli_mu, cli_epsilon
features[:, :, :, 2] = get_features('infrared',
latitudes,
longitudes,
beginning,
ending,
output_level='channel',
slot_step=1,
gray_scale=False)[:, :, :, 1]
if selected_slots is not None:
features[selected_slots, :, :, 7] = labels
return angles[selected_slots], features[selected_slots], selected_slots
else:
features[:, :, :, 7] = labels
return angles, features, selected_slots