def __init__(self,
labels,
dataset,
ids,
rasters,
size_patch=64,
transform=None,
add_all=True,
limit=-1,
reset_extractor=False,
**kwargs):
self.labels = labels
self.ids = ids
self.dataset = dataset
self.limit = limit
global ENVIRONMENTAL_DATASET_EXTRACTOR
if ENVIRONMENTAL_DATASET_EXTRACTOR is None or reset_extractor:
self.extractor = PatchExtractor(rasters,
size=size_patch,
verbose=True)
if add_all:
self.extractor.add_all()
ENVIRONMENTAL_DATASET_EXTRACTOR = self.extractor
else:
self.extractor = ENVIRONMENTAL_DATASET_EXTRACTOR
self.transform = transform
def __init__(self, root_dir, labels, dataset, ids, extractor=None):
self.extractor = extractor
self.labels = labels
self.ids = ids
self.dataset = dataset
if extractor is None:
self.extractor = PatchExtractor(root_dir, size=64, verbose=True)
self.extractor.add_all()
else:
self.extractor = extractor
def raster_characteristics(source):
"""
print infos about the rasters
:param source:
:return:
"""
r = check_source(source)
rasters = r['rasters']
extractor = PatchExtractor(rasters)
extractor.add_all()
print_statistics(str(extractor))
class EnvironmentalIGNDataset(Dataset):
def __init__(self, labels, dataset, ids, rasters, patches, size_patch=64, extractor=None, transform=None,
add_all=True, limit=-1):
self.extractor = extractor
self.labels = labels
self.ids = ids
self.dataset = dataset
self.patches = patches
self.limit = limit
if extractor is None:
self.extractor = PatchExtractor(rasters, size=size_patch, verbose=True)
if add_all:
self.extractor.add_all()
else:
self.extractor = extractor
self.transform = transform
def file_exists(self, idx):
return os.path.isfile(self.path(idx))
def path(self, idx):
image_id = str(int(self.ids[idx]))
return os.path.join(self.patches, image_id[-2:], image_id[-4:-2], image_id + '.npy')
def __len__(self):
return len(self.labels) if self.limit == -1 else min(len(self.labels), self.limit)
def __getitem__(self, idx):
ign_patch = np.transpose(np.load(self.path(idx)), (2, 0, 1))
tensor = self.extractor[self.dataset[idx]]
tensor = np.concatenate([tensor, ign_patch], axis=0)
if self.transform is not None:
tensor = self.transform(tensor).copy()
return torch.from_numpy(tensor).float(), self.labels[idx]
@property
def named_dimensions(self):
return [r.name for r in self.extractor.rasters] + ['IGN']
def __repr__(self):
return self.__str__()
def __str__(self):
return self.__class__.__name__ + '(size: {})'.format(len(self))
def __init__(self, labels, dataset, ids, rasters, patches, size_patch=64, extractor=None, transform=None,
add_all=True, limit=-1):
self.extractor = extractor
self.labels = labels
self.ids = ids
self.dataset = dataset
self.patches = patches
self.limit = limit
if extractor is None:
self.extractor = PatchExtractor(rasters, size=size_patch, verbose=True)
if add_all:
self.extractor.add_all()
else:
self.extractor = extractor
self.transform = transform
class GeoLifeClefDataset(Dataset):
def __init__(self, root_dir, labels, dataset, ids, extractor=None):
self.extractor = extractor
self.labels = labels
self.ids = ids
self.dataset = dataset
if extractor is None:
self.extractor = PatchExtractor(root_dir, size=64, verbose=True)
self.extractor.add_all()
else:
self.extractor = extractor
def __len__(self):
return len(self.labels)
def __getitem__(self, idx):
tensor = self.extractor[self.dataset[idx]]
tensor = random_rotation(tensor).copy()
return torch.from_numpy(tensor).float(), self.labels[idx]
class GeoLifeClefDataset(Dataset):
def __init__(self,
root_dir,
labels,
dataset,
ids,
extractor=None,
nb_labels=3336,
second_neihbour=True):
self.labels = labels
self.ids = ids
self.dataset = dataset
self.nb_labels = nb_labels
self.extractor = extractor
self.pos_multipoints = None
self.kdtree = None
self.multipoints = None
self.second_neihbour = second_neihbour
if extractor is None:
self.extractor = PatchExtractor(root_dir, size=64, verbose=True)
self.extractor.add_all()
else:
self.extractor = extractor
def __len__(self):
return len(self.labels)
def __getitem__(self, idx):
pos = [list(self.dataset[idx])]
dist, neighbours = self.kdtree.query(pos, k=2)
if dist[0][0] == 0 and self.second_neihbour:
mp_pos = self.pos_multipoints[neighbours[0][1]]
else:
mp_pos = self.pos_multipoints[neighbours[0][0]]
coocs = self.multipoints[mp_pos]
tensor = self.extractor[self.dataset[idx]]
return (torch.from_numpy(tensor).float(),
torch.from_numpy(coocs).float()), self.labels[idx]
def pplot(latitude,
longitude,
source,
resolution=1.,
style=special_parameters.plt_style,
nb_cols=5,
alpha=1.):
"""
patch plot
:param style:
:param latitude:
:param longitude:
:param source:
:param resolution:
:return:
"""
r = check_source(source)
rasters = r['rasters']
extractor = PatchExtractor(rasters, resolution=resolution)
extractor.add_all()
extractor.plot(item=(latitude, longitude),
return_fig=True,
style=style,
nb_cols=nb_cols,
alpha=alpha)
class GeoLifeClefDataset(Dataset):
def __init__(self,
root_dir,
labels,
dataset,
ids,
n_neighbours=200,
extractor=None,
nb_labels=3336):
self.labels = labels
self.ids = ids
self.dataset = dataset
self.n_neighbours = n_neighbours
self.nb_labels = nb_labels
self.extractor = extractor
self.kdtree = None
self.train_dataset = None
if extractor is None:
self.extractor = PatchExtractor(root_dir, size=64, verbose=True)
self.extractor.add_all()
else:
self.extractor = extractor
def __len__(self):
return len(self.labels)
def __getitem__(self, idx):
pos = [list(self.dataset[idx])]
dist, neighbours = self.kdtree.query(pos, k=self.n_neighbours)
coocs = np.zeros(self.nb_labels)
for n in neighbours[0]:
if idx != self.train_dataset.ids[n]:
coocs[self.train_dataset.labels[n]] += 1
tensor = self.extractor[self.dataset[idx]]
return (torch.from_numpy(tensor).float(),
torch.from_numpy(coocs).float()), self.labels[idx]
def __init__(self,
root_dir,
labels,
dataset,
ids,
n_neighbours=200,
extractor=None,
nb_labels=3336):
self.labels = labels
self.ids = ids
self.dataset = dataset
self.n_neighbours = n_neighbours
self.nb_labels = nb_labels
self.extractor = extractor
self.kdtree = None
self.train_dataset = None
if extractor is None:
self.extractor = PatchExtractor(root_dir, size=64, verbose=True)
self.extractor.add_all()
else:
self.extractor = extractor
def __init__(self,
root_dir,
labels,
dataset,
ids,
extractor=None,
nb_labels=3336,
second_neihbour=True):
self.labels = labels
self.ids = ids
self.dataset = dataset
self.nb_labels = nb_labels
self.extractor = extractor
self.pos_multipoints = None
self.kdtree = None
self.multipoints = None
self.second_neihbour = second_neihbour
if extractor is None:
self.extractor = PatchExtractor(root_dir, size=64, verbose=True)
self.extractor.add_all()
else:
self.extractor = extractor
class EnvironmentalDataset(Dataset):
def __init__(self,
labels,
dataset,
ids,
rasters,
size_patch=64,
transform=None,
add_all=True,
limit=-1,
reset_extractor=False,
**kwargs):
self.labels = labels
self.ids = ids
self.dataset = dataset
self.limit = limit
global ENVIRONMENTAL_DATASET_EXTRACTOR
if ENVIRONMENTAL_DATASET_EXTRACTOR is None or reset_extractor:
self.extractor = PatchExtractor(rasters,
size=size_patch,
verbose=True)
if add_all:
self.extractor.add_all()
ENVIRONMENTAL_DATASET_EXTRACTOR = self.extractor
else:
self.extractor = ENVIRONMENTAL_DATASET_EXTRACTOR
self.transform = transform
def __len__(self):
return len(self.labels) if self.limit == -1 else min(
len(self.labels), self.limit)
def __getitem__(self, idx):
if type(self.extractor) is not bool:
tensor = self.extractor[self.dataset[idx]]
if self.transform is not None:
tensor = self.transform(tensor).copy()
return torch.from_numpy(tensor).float(), self.labels[idx]
else:
return self.dataset[idx], self.labels[idx]
def numpy(self):
"""
:return: a numpy dataset of 1D vectors
"""
return np.array([
torch.flatten(self[i][0]).numpy() for i in range(len(self))
]), self.labels
@deprecated()
def get_vectors(self):
vec = []
for idx, data in enumerate(self.dataset):
vector = self.extractor[self.dataset[idx]]
if self.transform is not None:
vector = self.transform(vector).copy()
vector = list(vector)
vec.append(vector)
return vec
def __repr__(self):
return self.__str__()
def __str__(self):
return self.__class__.__name__ + '(size: {})'.format(len(self))
def plot_raster(rasters,
occurrences,
dataset_class,
validation_size=0,
test_size=1,
label_name='Label',
id_name='id',
splitter=train_test_split,
filters=tuple(),
online_filters=tuple(),
postprocessing=tuple(),
save_index=False,
limit=None,
raster="alti",
**kwargs):
_, _, grid_points = _occurrence_loader(rasters,
occurrences,
dataset_class,
validation_size,
test_size,
label_name,
id_name,
splitter,
filters,
online_filters,
postprocessing,
save_index,
limit,
extractor=PatchExtractor(
rasters, size=1, verbose=True),
**kwargs)
grid_points.extractor.append(raster)
r = np.zeros((len(grid_points.dataset), 1), dtype=float)
for i, data in enumerate(grid_points.dataset):
value = grid_points[i][0].numpy()
r[i, 0] = min(value[0], 2000)
print(r)
max_val = np.max(r)
"""
viridis = matplotlib.cm.get_cmap('inferno', max_val)
newcolors = viridis(np.linspace(0, 1, max_val))
pink = np.array([248 / 256, 24 / 256, 148 / 256, 1])
newcolors[1500:, :] = pink
newcmp = ListedColormap(newcolors)
newcmp.set_bad('grey', 1.)
top = matplotlib.cm.get_cmap('inferno', 2000)
bottom = matplotlib.cm.get_cmap('Blues', max_val-2000)
newcolors = np.vstack((top(np.linspace(0, 1, 2000)),
bottom(np.linspace(0, 1, max_val-2000))))
white = np.array([1, 1, 1, 1])
newcolors[2000:, :] = white
newcmp = ListedColormap(newcolors, name='OrangeBlue')
newcmp.set_bad('grey', 1.)
"""
plot_on_map(r,
grid_points.ids,
n_cols=1,
n_rows=1,
figsize=5,
log_scale=False,
mean_size=1,
selected=(0, ),
legend=(raster, ),
output=raster)