def get_snow_composite(latitudes, longitudes, dfb_beginning, dfb_ending, slot_step):
visible_features = get_features('visible', latitudes, longitudes, dfb_beginning, dfb_ending, False, slot_step)
vis = visible_features[:, :, :, 1]
sir = visible_features[:, :, :, 0]
mir = get_features('infrared', latitudes, longitudes, dfb_beginning, dfb_ending, False, slot_step)[:, :, :, 1]
mir -= 250
mir /= 100
(nb_slots, nb_latitudes, nb_longitudes) = np.shape(vis)
vis[vis < 0] = 0
sir[sir < 0] = 0
mir[mir < 0] = 0
composite = np.empty((nb_slots, nb_latitudes, nb_longitudes, 3))
composite[:, :, :, 0] = vis
composite[:, :, :, 1] = sir
composite[:, :, :, 2] = mir
return 255 * composite
def check_gaussian_hypothesis(latitudes, longitudes, begin, end, method='none'):
from tomas_outputs import reduce_tomas_2_classes, get_tomas_outputs
from decision_tree import reduce_two_classes, get_classes_v1_point, reduce_classes
from get_data import get_features
snow = get_features('visible', latitudes, longitudes, begin, end, 'abstract')[:, :, :, 0]
# snow=feat[:,:,:,0]
# var=feat[:,:,:,1]
# del feat
bb = get_bbox(latitudes[0], latitudes[-1], longitudes[0], longitudes[-1])
# visualize_map_time(snow, bb, vmin=0, vmax=1)
dmask = dawn_day_test(get_zenith_angle(get_times_utc(begin, end, read_satellite_step(), 1), latitudes, longitudes))
if method == 'tomas':
cloud = (reduce_tomas_2_classes(get_tomas_outputs(
begin, end, latitudes[0], latitudes[-1], longitudes[0], longitudes[-1]
)) == 1)
snow = snow[~cloud]
# visualize_map_time(cloud, bb)
elif method == 'ped':
classes = reduce_classes(get_classes_v1_point(
latitudes, longitudes, begin, end
))
cloud = (reduce_two_classes(classes) == 1)
# visualize_map_time(cloud, bb)
del classes
snow = snow[~cloud]
snow = snow[dmask & (snow > -9)]
visualize_hist(snow.flatten(), 'level of snow', precision=100)
def prepare_angles_features_bom_labels(seed, selected_slots):
beginning, ending, latitude_beginning, latitude_end, longitude_beginning, longitude_end = typical_input(
seed)
times = get_times_utc(beginning,
ending,
read_satellite_step(),
slot_step=1)
latitudes, longitudes = get_latitudes_longitudes(latitude_beginning,
latitude_end,
longitude_beginning,
longitude_end)
a, b, c = len(times), len(latitudes), len(longitudes)
nb_features_ = 8
features = np.empty((a, b, c, nb_features_))
angles, vis, ndsi, mask = get_features('visible',
latitudes,
longitudes,
beginning,
ending,
output_level='ndsi',
slot_step=1,
gray_scale=False)
test_angles = dawn_day_test(angles)
land_mask = typical_land_mask(seed)
mask = ((test_angles | land_mask) | mask)
ndsi[mask] = -10
features[:, :, :, 5] = test_angles
features[:, :, :, 6] = land_mask
del test_angles, land_mask, mask
features[:, :, :, 3] = vis
features[:, :, :, 4] = ndsi
del vis, ndsi
features[:, :, :, :3] = get_features('infrared',
latitudes,
longitudes,
beginning,
ending,
output_level='abstract',
slot_step=1,
gray_scale=False)[:, :, :, :3]
features[:, :, :, 7] = (typical_static_classifier(seed) >= 2)
if selected_slots is not None:
return angles[selected_slots], features[selected_slots], selected_slots
return angles, features, selected_slots
def infrared_low_cloud_composite(latitudes, longitudes, dfb_beginning, dfb_ending, slot_step):
infrared_features = get_features('infrared', latitudes, longitudes, dfb_beginning, dfb_ending, 'channel', slot_step)
(nb_slots, nb_latitudes, nb_longitudes) = np.shape(infrared_features)[0:3]
composite = np.zeros((nb_slots, nb_latitudes, nb_longitudes, 3))
try:
fir = infrared_features[:, :, :, 2]
fir = round(2 * (fir - 200))
composite[:, :, :, 0] = fir
except IndexError:
print 'Computing '
pass
lir = infrared_features[:, :, :, 1]
mir = infrared_features[:, :, :, 0]
mir = round(2 * (mir - 200))
lir = round(2 * (lir - 200))
composite[:, :, :, 1] = lir
composite[:, :, :, 2] = mir
composite[(lir <= 0) | (lir > 250)] = 0
return 255 * composite
nb_days = 3
ending = beginning + nb_days - 1
latitude_beginning = 40.
latitude_end = 45.
longitude_beginning = 125.
longitude_end = 130.
latitudes, longitudes = get_latitudes_longitudes(latitude_beginning,
latitude_end,
longitude_beginning,
longitude_end)
print_date_from_dfb(beginning, ending)
vis = get_features('visible', latitudes, longitudes, beginning, ending,
False)[:, :, :, 1]
# classes = get_classes_v1_point(latitudes, longitudes, beginning, ending, slot_step=1)
classes = get_classes_v2_image(latitudes,
longitudes,
beginning,
ending,
slot_step=1,
method='watershed-3d')
comparision = comparision_visible(vis, classes)
bbox = get_bbox(latitude_beginning, latitude_end, longitude_beginning,
longitude_end)
visualize_map_time(
classes,
import numpy as np
import matplotlib.pyplot as plt
import get_data as gd
X=[]
X = gd.get_features()
X=np.array(X)
f1_rice=[]
f2_rice=[]
for i in range(0,500):
f1_rice.append(X[i][1])
f2_rice.append(X[i][2])
f1_sugar=[]
f2_sugar=[]
for j in range(500,900):
f1_sugar.append(X[j][1])
f2_sugar.append(X[j][2])
f1_maize=[]
f2_maize=[]
for j in range(900,1298):
f1_maize.append(X[j][1])
f2_maize.append(X[j][2])
#plt.plot(f1_rice,'ro',f1_maize,'bs',f1_sugar,'g^',alpha=0.25)
plt.plot(f1_rice,f2_rice,'ro',f1_maize,f2_maize,'bs',f1_sugar,f2_sugar,'g^',alpha=0.25)
plt.show()
def get_classes_v1_point(latitudes,
longitudes,
beginning,
ending,
slot_step=1,
shades_detection=False):
visible_features = get_features(
'visible',
latitudes,
longitudes,
beginning,
ending,
'abstract',
slot_step,
gray_scale=False,
)
infrared_features = get_features(
'infrared',
latitudes,
longitudes,
beginning,
ending,
'abstract',
slot_step,
gray_scale=False,
)
# classes: classified_cli, snow over the ground, other (ground, sea...), unknown
visibles = get_features(
'visible',
latitudes,
longitudes,
beginning,
ending,
'channel',
slot_step,
gray_scale=False,
)
angles = get_zenith_angle(
get_times_utc(beginning, ending, read_satellite_step(), 1), latitudes,
longitudes)
# bright = (classify_brightness(visible_features[:, :, :, 0]) == 1) & (visibles[:,:,:,1] > 0.25*angles)
# bright = (visible_features[:, :, :, 0] > 0.3) & (visibles[:,:,:,1] > 0.25*angles)
bright = segmentation('watershed-3d',
visible_features[:, :, :, 0],
thresh_method='static',
static=0.3) # & (visibles[:,:,:,1] > 0.2*angles)
# negative_variable_brightness = (classifiy_brightness_variability(visible_features[:, :, :, 1]) == 1)
# positive_variable_brightness = (classifiy_brightness_variability(visible_features[:, :, :, 2]) == 1)
negative_variable_brightness = (visible_features[:, :, :, 1] >
0.2) & bright
positive_variable_brightness = (visible_features[:, :, :, 2] >
0.2) & bright
# from classification_cloud_index import classify_cloud_covertness, classify_cloud_variability
cold = (infrared_features[:, :, :, 2] == 1)
thin_clouds = (infrared_features[:, :, :, 0] > 0.2)
obvious_clouds = (infrared_features[:, :, :, 1] > 10)
# obvious_clouds = (classify_cloud_covertness(infrared_features[:, :, :, 0]) == 1) & (infrared_features[:, :, :, 1] > 1)
snow = bright & ~negative_variable_brightness & ~positive_variable_brightness & ~obvious_clouds & ~cold
del bright
(nb_slots, nb_latitudes, nb_longitudes) = np.shape(visible_features)[0:3]
classes = np.zeros((nb_slots, nb_latitudes, nb_longitudes))
begin_affectation = time()
classes[(visibles[:, :, :, 1] > 0.5 * angles)
& (visibles[:, :, :, 0] > 0.1 *
angles)] = 3 # before all the other classes (VERY IMPORTANT)
classes[(infrared_features[:, :, :, 0] == -10
)] = 13 # before all the other classes (important)
classes[(visible_features[:, :, :, 3] == 1
)] = 12 # before all the other classes (important)
classes[snow] = 5 # class ground snow or ice
classes[positive_variable_brightness] = 6
classes[negative_variable_brightness] = 7
classes[obvious_clouds] = 1
classes[thin_clouds] = 2
# classes[bright & ~negative_variable_brightness & warm] = 10
# classes[bright & negative_variable_brightness & warm] = 9
classes[cold] = 8
# classes[obvious_clouds & bright] = 3
if shades_detection:
cloudy = (classes != 0) & (classes != 5)
print 'launch shades analysis'
shades_detection = recognize_cloud_shade(
visibles[:, :, :, 1], cloudy,
get_zenith_angle(
get_times_utc(beginning,
ending,
read_satellite_step(),
slot_step=1), latitudes, longitudes))
print 'completed shades analysis'
classes[shades_detection] = 11
print 'time affectation', time() - begin_affectation
print 'allegedly uncovered lands'
print 'obvious clouds:1'
print 'thin clouds:2'
print 'visible but undecided:3'
print 'slight clouds and bright:4'
print 'snowy:5'
print 'snowy clouds:6'
print 'covered snow:7'
print 'cold:8'
# print 'hot bright corpses:9'
# print 'hot bright variable corpses:10'
print 'foggy:11'
print 'sea clouds identified by visibility:12' #### WARNING: what about icy lakes??? ####
# print 'suspect high snow index (over sea / around sunset or sunrise):13'
print 'undefined:13'
return classes
def get_classes_v2_image(latitudes,
longitudes,
beginning,
ending,
slot_step=1,
method='otsu-3d',
shades_detection=False):
visible_features = get_features(
'visible',
latitudes,
longitudes,
beginning,
ending,
True,
slot_step,
gray_scale=True,
)
infrared_features = get_features(
'infrared',
latitudes,
longitudes,
beginning,
ending,
True,
slot_step,
gray_scale=True,
)
if method in ['watershed-2d', 'watershed-3d']:
visible = get_features('visible',
latitudes,
longitudes,
beginning,
ending,
'abstract',
slot_step,
gray_scale=False)
# visualize_map_time(segmentation_otsu_2d(vis), bbox)
bright = (segmentation_otsu_2d(visible_features[:, :, :, 0]) &
(visible[:, :, :, 1] > 0.35))
# visualize_map_time(bright, bbox)
bright = segmentation(method, bright, thresh_method='binary')
else:
visible = get_features('visible',
latitudes,
longitudes,
beginning,
ending,
'abstract',
slot_step,
gray_scale=False)
visible = segmentation(
method,
visible,
1,
)
bright = (segmentation(method, visible_features[:, :, :, 0]) & visible)
# negative_variable_brightness = visible_features[:, :, :, 1] > 25
# positive_variable_brightness = visible_features[:, :, :, 2] > 25
negative_variable_brightness = segmentation(method,
visible_features[:, :, :, 1],
thresh_method='static',
static=30)
positive_variable_brightness = segmentation(method,
visible_features[:, :, :, 2],
thresh_method='static',
static=30)
# slight_clouds = segmentation(method, infrared_features[:, :, :, 1])
# obvious_clouds = (infrared_features[:, :, :, 0] == 1)
obvious_clouds = segmentation(
method, infrared_features[:, :, :, 0]) & segmentation(
method,
infrared_features[:, :, :, 1],
thresh_method='static',
static=20)
cold = (infrared_features[:, :, :, 2] == 1)
# warm = (infrared_features[:, :, :, 2] == 1)
# foggy: low snow index, good vis
(nb_slots, nb_latitudes, nb_longitudes) = np.shape(visible_features)[0:3]
classes = np.zeros((nb_slots, nb_latitudes, nb_longitudes))
# clouds = obvious_clouds | slight_clouds
begin_affectation = time()
classes[(infrared_features[:, :, :, 0] == -10
)] = 13 # before all the other classes (important)
classes[(visible_features[:, :, :, 3] == 1
)] = 12 # before all the other classes (important)
classes[bright & ~obvious_clouds
& ~negative_variable_brightness] = 5 # class ground snow or ice
classes[bright & positive_variable_brightness] = 6
classes[bright & negative_variable_brightness] = 7
# classes[bright & ~negative_variable_brightness & warm] = 10
# classes[bright & negative_variable_brightness & warm] = 9
classes[cold] = 8
# WARNING: slight clouds AND obvious clouds => obvious clouds
# classes[slight_clouds & bright] = 4
# classes[slight_clouds & ~bright] = 2
classes[obvious_clouds & ~bright] = 1
classes[obvious_clouds & bright] = 3
if shades_detection:
cloudy = (classes != 0) & (classes != 5)
print 'launch shades analysis'
shades_detection = recognize_cloud_shade(
visible[:, :, :, 1], cloudy,
get_zenith_angle(
get_times_utc(beginning,
ending,
read_satellite_step(),
slot_step=1), latitudes, longitudes))
print 'completed shades analysis'
classes[shades_detection] = 11
# classes[bright & (infrared_features[:, :, :, 3] == 1)] = 7 # = cold and bright. opaque obvious_clouds or cold obvious_clouds over snowy stuff
# classes[persistent_snow & (obvious_clouds | cold_opaque_clouds)] = 4
# classes[foggy] = 11
print 'time affectation', time() - begin_affectation
print 'uncovered lands: 0'
print 'obvious clouds:1'
print 'slight clouds or sunrise/sunset clouds:2'
print 'clouds and bright:3'
print 'slight clouds and bright:4'
print 'snowy:5'
print 'snowy clouds:6'
print 'covered snow:7'
print 'cold not bright (cold thin water clouds?):8'
# print 'hot bright corpses:9'
# print 'hot bright variable corpses:10'
print 'shades:11'
print 'sea clouds identified by visibility:12' #### WARNING: what about icy lakes??? ####
# print 'suspect high snow index (over sea / around sunset or sunrise):13'
print 'undefined:13'
return classes
print 'classification time: ', time() - t_begin
visualize_map_time(classes_ped,
bbox,
vmin=0,
vmax=nb_classes - 1,
title=method + ' Classes 0-' + str(nb_classes - 1) +
' from' + str(date_begin))
# statistics_classes(classes_ped, display_now=True)
# visualize_map_time(reduce_classes(classes_ped), bbox, vmin=0, vmax=4, title=method + ' Classes 0-' + str(5 - 1) +
# ' from' + str(date_begin))
visualize_map_time(comparision_visible(
get_features('visible', latitudes, longitudes, beginning, ending,
'channel')[:, :, :, 1], classes_ped),
bbox,
vmin=-1,
vmax=1,
title='comparision visible')
classes_ped = reduce_two_classes(classes_ped)
visualize_map_time(classes_ped,
bbox,
vmin=0,
vmax=1,
title='ped-' + method + ' Classes 0-' + str(1) +
' from' + str(date_begin))
# raise Exception('stop here for now pliz')
classes_tomas = get_tomas_outputs(beginning, ending, latitude_beginning,
latitude_end, longitude_beginning,
longitude_beginning = 125.
longitude_end = 130.
latitudes, longitudes = get_latitudes_longitudes(latitude_beginning, latitude_end,
longitude_beginning, longitude_end)
date_begin, date_end = print_date_from_dfb(beginning, ending)
print beginning, ending
type_channels = 0 # 0: infrared, 1: visible
if type_channels == 0:
infra = get_features(
'infrared',
latitudes,
longitudes,
beginning,
ending,
output_level=True,
slot_step=1,
gray_scale=False,
)
cli = infra[:, :, :, 0]
unbiased = infra[:, :, :, 1]
infrared_features = get_features(
'infrared',
latitudes,
longitudes,
beginning,
ending,
output_level=False,
feat] = equalize_histogram_2d(features[slot, :, :,
feat])
return features
if __name__ == '__main__':
types_channel = ['infrared', 'visible']
compute_indexes = True
chan = 0
channel_number = 1
type_channels = types_channel[channel_number]
beginning, ending, latitude_beginning, latitude_end, longitude_beginning, longitude_end = typical_input(
seed=0)
date_begin, date_end = print_date_from_dfb(beginning, ending)
lat, lon = get_latitudes_longitudes(latitude_beginning, latitude_end,
longitude_beginning, longitude_end)
# from quick_visualization import get_bbox
# bbox = get_bbox(latitude_beginning, latitude_end, longitude_beginning, longitude_end)
features = get_features(type_channels,
lat,
lon,
beginning,
ending,
compute_indexes,
slot_step=1,
gray_scale=True)
segmentation_watershed_2d(features[16, :, :, chan])
segmentation_otsu_2d(features, chan)
def prepare_angles_features_ped_labels(seed, keep_holes=True):
'''
:param latitude_beginning:
:param latitude_end:
:param longitude_beginning:
:param longitude_end:
:param beginning:
:param ending:
:param output_level:
:param seed:
:param keep_holes:
:return:
'''
beginning, ending, latitude_beginning, latitude_end, longitude_beginning, longitude_end = typical_input(
seed)
latitudes, longitudes = get_latitudes_longitudes(latitude_beginning,
latitude_end,
longitude_beginning,
longitude_end)
if keep_holes:
labels, selected_slots = read_labels_keep_holes(
'csp', latitude_beginning, latitude_end, longitude_beginning,
longitude_end, beginning, ending)
else:
labels, selected_slots = read_labels_remove_holes(
'csp', latitude_beginning, latitude_end, longitude_beginning,
longitude_end, beginning, ending)
angles, vis, ndsi, mask = get_features('visible',
latitudes,
longitudes,
beginning,
ending,
output_level='ndsi',
slot_step=1,
gray_scale=False)
a, b, c = angles.shape
nb_features_ = 8
features = np.empty((a, b, c, nb_features_))
test_angles = dawn_day_test(angles)
land_mask = typical_land_mask(seed)
ndsi[((test_angles | land_mask) | mask)] = -10
features[:, :, :, 5] = test_angles
features[:, :, :, 6] = land_mask
features[:, :, :, 3] = vis
features[:, :, :, 4] = ndsi
del vis, ndsi
cli_mu, cli_epsilon, mask_input_cli = get_features('infrared',
latitudes,
longitudes,
beginning,
ending,
output_level='cli',
slot_step=1,
gray_scale=False)
mask = ((test_angles | land_mask) | mask)
cli_mu[mask] = -10
cli_epsilon[mask] = -10
features[:, :, :, 0] = cli_mu
features[:, :, :, 1] = cli_epsilon
del mask, test_angles, land_mask, cli_mu, cli_epsilon
features[:, :, :, 2] = get_features('infrared',
latitudes,
longitudes,
beginning,
ending,
output_level='channel',
slot_step=1,
gray_scale=False)[:, :, :, 1]
if selected_slots is not None:
features[selected_slots, :, :, 7] = labels
return angles[selected_slots], features[selected_slots], selected_slots
else:
features[:, :, :, 7] = labels
return angles, features, selected_slots
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
from sklearn.preprocessing import StandardScaler
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import get_data
pd.set_option('display.max_rows', None)
df_features = get_data.get_features()
df_labels = get_data.get_labels()
df_features_validate = get_data.get_validation_features()
df_labels_validate = get_data.get_validation_labels()
df_features_test = get_data.get_test_features()
# Concatenate datasets
dataset_whole = pd.concat(
[df_features, df_features_validate, df_features_test])
# Fix errors
dataset_whole['title'] = dataset_whole['title'].fillna('N/A')
def remove_errors(self):
if type(self) == str and self.isdigit() == False:
return 0
else:
return self
dataset_whole['year'] = dataset_whole['year'].apply(remove_errors).apply(int)