def _reproject(self, eopatch, src_raster):
"""
Reprojects the raster data from Geopedia's CRS (POP_WEB) to EOPatch's CRS.
"""
height, width = src_raster.shape
dst_raster = np.ones((height, width), dtype=self.raster_dtype)
src_bbox = transform_bbox(eopatch.bbox, CRS.POP_WEB)
src_transform = rasterio.transform.from_bounds(*src_bbox,
width=width,
height=height)
dst_bbox = eopatch.bbox
dst_transform = rasterio.transform.from_bounds(*dst_bbox,
width=width,
height=height)
rasterio.warp.reproject(
src_raster,
dst_raster,
src_transform=src_transform,
src_crs={'init': CRS.ogc_string(CRS.POP_WEB)},
src_nodata=0,
dst_transform=dst_transform,
dst_crs={'init': CRS.ogc_string(eopatch.bbox.crs)},
dst_nodata=self.no_data_val)
return dst_raster
def test_custom_crs(self):
for incorrect_value in ['string', -1, 999, None]:
with self.assertRaises(ValueError):
CRS(incorrect_value)
for correct_value in [3035, 'EPSG:3035', 10000]:
CRS(CRS(correct_value))
new_enum_value = str(correct_value).lower().strip('epsg: ')
self.assertTrue(CRS.has_value(new_enum_value))
def __init__(self, feature, folder=None, *, band_indices=None, date_indices=None, crs=None, fail_on_missing=True,
**kwargs):
"""
:param feature: Feature which will be exported
:type feature: (FeatureType, str)
:param folder: A directory containing image files or a path of an image file
:type folder: str
:param band_indices: Bands to be added to tiff image. Bands are represented by their 0-based index as tuple
in the inclusive interval form `(start_band, end_band)` or as list in the form
`[band_1, band_2,...,band_n]`.
:type band_indices: tuple or list or None
:param date_indices: Dates to be added to tiff image. Dates are represented by their 0-based index as tuple
in the inclusive interval form `(start_date, end_date)` or a list in the form `[date_1, date_2,...,date_n]`.
:type date_indices: tuple or list or None
:param crs: CRS in which to reproject the feature before writing it to GeoTiff
:type crs: CRS or str or None
:param fail_on_missing: should the pipeline fail if a feature is missing or just log warning and return
:type fail_on_missing: bool
:param image_dtype: Type of data to be exported into tiff image
:type image_dtype: numpy.dtype
:param no_data_value: Value of pixels of tiff image with no data in EOPatch
:type no_data_value: int or float
"""
super().__init__(feature, folder=folder, **kwargs)
self.band_indices = band_indices
self.date_indices = date_indices
self.crs = None if crs is None else CRS(crs)
self.fail_on_missing = fail_on_missing
def process(self, arguments):
data_as_list = self.validate_parameter(arguments,
"data",
required=True,
allowed_types=[list])
dims = self.validate_parameter(arguments,
"dims",
required=True,
allowed_types=[list])
coords = self.validate_parameter(arguments,
"coords",
allowed_types=[dict],
default={})
if "t" in coords:
coords["t"] = [
datetime.strptime(d, '%Y-%m-%d %H:%M:%S') for d in coords["t"]
]
data = xr.DataArray(
np.array(data_as_list, dtype=np.float),
coords=coords,
dims=dims,
attrs={
"band_aliases": {},
"bbox": BBox((
12.0,
45.0,
13.0,
46.0,
), CRS(4326)),
},
)
self.logger.info(data)
return data
def _repr_value(value):
"""Creates a representation string for different types of data.
:param value: data in any type
:return: representation string
:rtype: str
"""
if isinstance(value, np.ndarray):
return f"{EOPatch._repr_value_class(value)}(shape={value.shape}, dtype={value.dtype})"
if isinstance(value, gpd.GeoDataFrame):
crs = CRS(value.crs).ogc_string() if value.crs else value.crs
return f"{EOPatch._repr_value_class(value)}(columns={list(value)}, length={len(value)}, crs={crs})"
if isinstance(value, (list, tuple, dict)) and value:
repr_str = str(value)
if len(repr_str) <= MAX_DATA_REPR_LEN:
return repr_str
l_bracket, r_bracket = ("[",
"]") if isinstance(value, list) else ("(",
")")
if isinstance(value, (list, tuple)) and len(value) > 2:
repr_str = f"{l_bracket}{repr(value[0])}, ..., {repr(value[-1])}{r_bracket}"
if len(repr_str) > MAX_DATA_REPR_LEN and isinstance(
value, (list, tuple)) and len(value) > 1:
repr_str = f"{l_bracket}{repr(value[0])}, ...{r_bracket}"
if len(repr_str) > MAX_DATA_REPR_LEN:
repr_str = str(type(value))
return f"{repr_str}, length={len(value)}"
return repr(value)
def execute(self, eopatch, *, filename=None):
""" Execute method
:param eopatch: input EOPatch
:type eopatch: EOPatch
:param filename: filename of tiff file or None if entire path has already been specified in `folder` parameter
of task initialization.
:type filename: str or None
:return: Unchanged input EOPatch
:rtype: EOPatch
"""
feature_type, feature_name = next(self.feature(eopatch))
array = eopatch[feature_type][feature_name]
array_sub = self._get_bands_subset(array)
if feature_type.is_time_dependent():
array_sub = self._get_dates_subset(array_sub, eopatch.timestamp)
else:
# add temporal dimension
array_sub = np.expand_dims(array_sub, axis=0)
if not feature_type.is_spatial():
# add height and width dimensions
array_sub = np.expand_dims(np.expand_dims(array_sub, axis=1),
axis=1)
time_dim, height, width, band_dim = array_sub.shape
index = time_dim * band_dim
dst_transform = rasterio.transform.from_bounds(*eopatch.bbox,
width=width,
height=height)
dst_crs = {'init': CRS.ogc_string(eopatch.bbox.crs)}
image_dtype = array_sub.dtype if self.image_dtype is None else self.image_dtype
if image_dtype == np.int64:
image_dtype = np.int32
warnings.warn(
'Data from feature {} cannot be exported to tiff with dtype numpy.int64. Will export as '
'numpy.int32 instead'.format((feature_type, feature_name)))
# Write it out to a file
with rasterio.open(self._get_file_path(filename),
'w',
driver='GTiff',
width=width,
height=height,
count=index,
dtype=image_dtype,
nodata=self.no_data_value,
transform=dst_transform,
crs=dst_crs) as dst:
output_array = array_sub.astype(image_dtype)
output_array = np.moveaxis(output_array, -1,
1).reshape(index, height, width)
dst.write(output_array)
return eopatch
def test_repr(self):
crs_values = (
(CRS.POP_WEB, "CRS('3857')"),
(CRS.WGS84, "CRS('4326')"),
(CRS.UTM_33N, "CRS('32633')"),
(CRS.UTM_33S, "CRS('32733')"),
(CRS('3857'), "CRS('3857')"),
(CRS('4326'), "CRS('4326')"),
(CRS('32633'), "CRS('32633')"),
(CRS('32733'), "CRS('32733')"),
)
for crs, crs_repr in crs_values:
with self.subTest(msg=crs_repr):
self.assertEqual(crs_repr,
repr(crs),
msg="Expected {}, got {}".format(
crs_repr, repr(crs)))
def setUpClass(cls):
super().setUpClass()
with open(os.path.join(cls.INPUT_FOLDER, "test_fis_results.txt"), 'r') as file:
results = [ast.literal_eval(line.strip()) for line in file]
bbox = BBox([14.00, 45.00, 14.03, 45.03], crs=CRS.WGS84)
geometry1 = Geometry(Polygon([(465888.877326859, 5079639.436138632),
(465885.3413983975, 5079641.524618266),
(465882.9542217017, 5079647.166043535),
(465888.8780175466, 5079668.703676634),
(465888.877326859, 5079639.436138632)]),
CRS(32633))
geometry2 = Geometry('POLYGON((-5.13 48, -5.23 48.09, -5.13 48.17, -5.03 48.08, -5.13 48))', CRS.WGS84)
cls.test_cases = [
cls.FisTestCase('geometry',
FisRequest(layer='TRUE-COLOR-S2-L1C',
geometry_list=[geometry1],
time=('2017-1-1', '2017-2-1'),
resolution="50m",
histogram_type=HistogramType.STREAMING,
bins=5),
raw_result=results[0],
result_length=1),
cls.FisTestCase('bbox',
FisRequest(layer='BANDS-S2-L1C',
geometry_list=[bbox],
time='2017-1-1',
resolution="50m",
maxcc=0.2,
custom_url_params={
CustomUrlParam.ATMFILTER: "ATMCOR",
CustomUrlParam.DOWNSAMPLING: "BICUBIC",
CustomUrlParam.UPSAMPLING: "BICUBIC"}
),
raw_result=results[1],
result_length=1),
cls.FisTestCase('list',
FisRequest(data_source=DataSource.LANDSAT8,
layer='BANDS-L8',
geometry_list=[bbox, geometry1],
time=('2017-1-1', '2017-1-10'),
resolution="100m",
bins=32, data_folder=cls.OUTPUT_FOLDER),
raw_result=results[2], result_length=2,
save_data=True),
cls.FisTestCase('Polygon in WGS84',
FisRequest(layer='TRUE-COLOR-S2-L1C',
geometry_list=[geometry2],
time=('2017-10-1', '2017-10-2'),
resolution="60m",
bins=11, histogram_type=HistogramType.EQUALFREQUENCY),
raw_result=results[3], result_length=1),
]
for test_case in cls.test_cases:
test_case.collect_data()
def _get_rasterization_shapes(self, eopatch, group_classes=False):
""" Returns a generator of pairs of geometrical shapes and values. In rasterization process each shape will be
rasterized to it's corresponding value.
If there are no such geometries it will return `None`
:param group_classes: If true, function returns a zip that iterates through cascaded unions of polygons of the
same class, otherwise zip iterates through all polygons regardless of their class.
:type group_classes: boolean
"""
vector_data = self._get_vector_data(eopatch)
gpd_crs = vector_data.crs
# This special case has to be handled because of WGS84 and lat-lon order:
if isinstance(gpd_crs, pyproj.CRS):
gpd_crs = gpd_crs.to_epsg()
vector_data_crs = CRS(gpd_crs)
if eopatch.bbox.crs is not vector_data_crs:
warnings.warn('Vector data is not in the same CRS as EOPatch, this task will re-project vector data for '
'each execution', RuntimeWarning)
vector_data = vector_data.to_crs(eopatch.bbox.crs.pyproj_crs())
bbox_poly = eopatch.bbox.geometry
vector_data = vector_data[vector_data.geometry.intersects(bbox_poly)].copy(deep=True)
if vector_data.empty:
return None
if self.buffer:
vector_data.geometry = vector_data.geometry.buffer(self.buffer)
vector_data = vector_data[~vector_data.is_empty]
# vector_data could be empty as a result of (negative) buffer
if vector_data.empty:
return None
if not vector_data.geometry.is_valid.all():
warnings.warn('Given vector polygons contain some invalid geometries, they will be fixed', RuntimeWarning)
vector_data.geometry = vector_data.geometry.buffer(0)
if vector_data.geometry.has_z.any():
warnings.warn('Given vector polygons contain some 3D geometries, they will be projected to 2D',
RuntimeWarning)
vector_data.geometry = vector_data.geometry.apply(lambda geo: shapely.wkt.loads(geo.to_wkt()))
if self.values_column is None:
return zip(vector_data.geometry, [self.values] * len(vector_data.index))
if self.values is not None:
values = [self.values] if isinstance(self.values, (int, float)) else self.values
vector_data = vector_data[vector_data[self.values_column].isin(values)]
if group_classes:
classes = np.unique(vector_data[self.values_column])
grouped = (vector_data.geometry[vector_data[self.values_column] == cl] for cl in classes)
grouped = (shapely.ops.cascaded_union(group) for group in grouped)
return zip(grouped, classes)
return zip(vector_data.geometry, vector_data[self.values_column])
def _convert_bbox(spatial_extent):
crs = spatial_extent.get('crs', 4326)
return BBox(
(spatial_extent['west'],
spatial_extent['south'],
spatial_extent['east'],
spatial_extent['north'],),
CRS(crs), # we support whatever sentinelhub-py supports
)
def test_ogc_string(self):
crs_values = ((CRS.POP_WEB, 'EPSG:3857'), (CRS.WGS84, 'EPSG:4326'),
(CRS.UTM_33N, 'EPSG:32633'), (CRS.UTM_33S, 'EPSG:32733'))
for crs, epsg in crs_values:
with self.subTest(msg=epsg):
ogc_str = CRS.ogc_string(crs)
self.assertEqual(epsg,
ogc_str,
msg="Expected {}, got {}".format(
epsg, ogc_str))
def generate_slo_shapefile(path):
DATA_FOLDER = os.path.join('data')
area = gpd.read_file(os.path.join(DATA_FOLDER, 'svn_buffered.geojson'))
# area = gpd.read_file(os.path.join(DATA_FOLDER, 'austria.geojson'))
# Convert CRS to UTM_33N
country_crs = CRS.UTM_33N
area = area.to_crs(crs={'init': CRS.ogc_string(country_crs)})
# Get the country's shape in polygon format
country_shape = area.geometry.values.tolist()[-1]
# Plot country
area.plot()
plt.axis('off')
# Create the splitter to obtain a list of bboxes
bbox_splitter = BBoxSplitter([country_shape], country_crs,
(25 * 3, 17 * 3))
bbox_list = np.array(bbox_splitter.get_bbox_list())
info_list = np.array(bbox_splitter.get_info_list())
path_out = path + '/shapefiles'
if not os.path.isdir(path_out):
os.makedirs(path_out)
geometry = [Polygon(bbox.get_polygon()) for bbox in bbox_list]
idxs_x = [info['index_x'] for info in info_list]
idxs_y = [info['index_y'] for info in info_list]
gdf = gpd.GeoDataFrame({
'index_x': idxs_x,
'index_y': idxs_y
},
crs={'init': CRS.ogc_string(country_crs)},
geometry=geometry)
shapefile_name = path_out + '/slovenia.shp'
gdf.to_file(shapefile_name)
return gdf, bbox_list
def test_bbox_to_repr(self):
x1, y1, x2, y2 = 45.0, 12.0, 47.0, 14.0
bbox = BBox(((x1, y1), (x2, y2)), crs=CRS('4326'))
expect_repr = "BBox((({}, {}), ({}, {})), crs=CRS('4326'))".format(
x1, y1, x2, y2)
self.assertEqual(
repr(bbox),
expect_repr,
msg="String representations not matching: expected {}, got {}".
format(expect_repr, repr(bbox)))
def test_crs_parsing(self):
test_cases = [
(4326, CRS.WGS84),
('4326', CRS.WGS84),
('EPSG:3857', CRS.POP_WEB),
({
'init': 'EPSG:32638'
}, CRS.UTM_38N),
(pyproj.CRS('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'),
CRS.WGS84),
(pyproj.CRS(3857), CRS.POP_WEB),
]
for parse_value, expected_result in test_cases:
with self.subTest(msg=str(parse_value)):
parsed_result = CRS(parse_value)
self.assertEqual(parsed_result, expected_result)
with self.assertRaises(ValueError):
CRS(pyproj.CRS(4326))
def _preprocess_vector_data(self, vector_data, bbox, timestamps):
"""Applies preprocessing steps on a dataframe with geometries and potential values and timestamps"""
columns_to_keep = ["geometry"]
if self._rasterize_per_timestamp:
columns_to_keep.append("TIMESTAMP")
if self.values_column is not None:
columns_to_keep.append(self.values_column)
vector_data = vector_data[columns_to_keep]
if self._rasterize_per_timestamp:
vector_data = vector_data[vector_data.TIMESTAMP.isin(timestamps)]
if self.values_column is not None and self.values is not None:
values = [self.values] if isinstance(self.values,
(int, float)) else self.values
vector_data = vector_data[vector_data[self.values_column].isin(
values)]
gpd_crs = vector_data.crs
# This special case has to be handled because of WGS84 and lat-lon order:
if isinstance(gpd_crs, pyproj.CRS):
gpd_crs = gpd_crs.to_epsg()
vector_data_crs = CRS(gpd_crs)
if bbox.crs is not vector_data_crs:
warnings.warn(
"Vector data is not in the same CRS as EOPatch, this task will re-project vector data for "
"each execution",
EORuntimeWarning,
)
vector_data = vector_data.to_crs(bbox.crs.pyproj_crs())
bbox_poly = bbox.geometry
vector_data = vector_data[vector_data.geometry.intersects(
bbox_poly)].copy(deep=True)
if self.buffer:
vector_data.geometry = vector_data.geometry.buffer(self.buffer)
vector_data = vector_data[~vector_data.is_empty]
if not vector_data.geometry.is_valid.all():
warnings.warn(
"Given vector polygons contain some invalid geometries, they will be fixed",
EORuntimeWarning)
vector_data.geometry = vector_data.geometry.buffer(0)
if vector_data.geometry.has_z.any():
warnings.warn(
"Given vector polygons contain some 3D geometries, they will be projected to 2D",
EORuntimeWarning)
vector_data.geometry = vector_data.geometry.map(
functools.partial(shapely.ops.transform,
lambda *args: args[:2]))
return vector_data
def execute(self, eopatch=None, *, filename=None):
""" Execute method which adds a new feature to the EOPatch
:param eopatch: input EOPatch or None if a new EOPatch should be created
:type eopatch: EOPatch or None
:param filename: filename of tiff file or None if entire path has already been specified in `folder` parameter
of task initialization.
:type filename: str or None
:return: New EOPatch with added raster layer
:rtype: EOPatch
"""
feature_type, feature_name = next(self.feature())
if eopatch is None:
eopatch = EOPatch()
with rasterio.open(self._get_file_path(filename)) as source:
data_bbox = BBox(source.bounds, CRS(source.crs.to_epsg()))
if eopatch.bbox is None:
eopatch.bbox = data_bbox
reading_window = self._get_reading_window(source.width,
source.height, data_bbox,
eopatch.bbox)
data = source.read(window=reading_window,
boundless=True,
fill_value=self.no_data_value)
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
'Cannot import as a time-dependant feature because the number of tiff image channels '
'is not divisible by the number of timestamps')
data = data.reshape((times, channels // times) + data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = data
return eopatch
def _get_task(task_data):
window = json.loads(task_data['window'])
return Task(
task_id=task_data['task_id'],
bbox=BBox(
bbox=json.loads(task_data['bbox']), # TODO: why is string?
crs=CRS(task_data['crs'])),
acq_time=dt.datetime.strptime(task_data['datetime'], '%Y-%m-%d'),
window_shape=[window['height'], window['width']],
data_list=json.loads(task_data['data']),
vector_data=task_data['vector_data'])
def dataset_crs(self):
"""Provides a "crs" of dataset, loads it lazily (i.e. the first time it is needed)
:return: Dataset's CRS
:rtype: CRS
"""
if self._dataset_crs is None:
srid = self.geodb_client.get_collection_srid(collection=self.geodb_collection, database=self.geodb_db)
self._dataset_crs = CRS(f"epsg:{srid}")
return self._dataset_crs
def test_utm(self):
known_values = ((13, 46, '32633'), (13, 0, '32633'),
(13, -45, '32733'), (13, 0, '32633'),
(13, -0.0001, '32733'), (13, -46, '32733'))
for known_val in known_values:
lng, lat, epsg = known_val
with self.subTest(msg=epsg):
crs = CRS.get_utm_from_wgs84(lng, lat)
self.assertEqual(
epsg,
crs.value,
msg="Expected {}, got {} for lng={},lat={}".format(
epsg, crs.value, str(lng), str(lat)))
def _decode(self, file, path):
"""Loads from a file and decodes content."""
file_format = MimeType(fs.path.splitext(path)[1].strip("."))
if file_format is MimeType.NPY:
return np.load(file)
if file_format is MimeType.GPKG:
dataframe = gpd.read_file(file)
if dataframe.crs is not None:
# Trying to preserve a standard CRS and passing otherwise
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=SHUserWarning)
dataframe.crs = CRS(dataframe.crs).pyproj_crs()
except ValueError:
pass
if "TIMESTAMP" in dataframe:
dataframe.TIMESTAMP = pd.to_datetime(dataframe.TIMESTAMP)
return dataframe
if file_format in [MimeType.JSON, MimeType.GEOJSON]:
json_data = json.load(file)
if self.feature_type is FeatureType.BBOX:
return Geometry.from_geojson(json_data).bbox
return json_data
if file_format is MimeType.PICKLE:
warnings.warn(
f"File {self.path} with data of type {self.feature_type} is in pickle format which is deprecated "
"since eo-learn version 1.0. Please re-save this EOPatch with the new eo-learn version to "
"update the format. In newer versions this backward compatibility will be removed.",
EODeprecationWarning,
)
data = pickle.load(file)
# There seems to be an issue in geopandas==0.8.1 where unpickling GeoDataFrames, which were saved with an
# old geopandas version, loads geometry column into a pandas.Series instead geopandas.GeoSeries. Because
# of that it is missing a crs attribute which is only attached to the entire GeoDataFrame
if isinstance(data, GeoDataFrame) and not isinstance(
data.geometry, GeoSeries):
data = data.set_geometry("geometry")
return data
raise ValueError(f"Unsupported data type for file {path}.")
def execute(self, eopatch, *, filename):
feature_type, feature_name = next(self.feature(eopatch))
array = eopatch[feature_type][feature_name]
array_sub = self._get_bands_subset(array)
if feature_type in [
FeatureType.DATA, FeatureType.MASK, FeatureType.SCALAR
]:
array_sub = self._get_dates_subset(array_sub, eopatch.timestamp)
if feature_type is FeatureType.SCALAR:
# add height and width dimensions
array_sub = np.expand_dims(np.expand_dims(array_sub, axis=1),
axis=1)
else:
# add temporal dimension
array_sub = np.expand_dims(array_sub, axis=0)
if feature_type is FeatureType.SCALAR_TIMELESS:
# add height and width dimensions
array_sub = np.expand_dims(np.expand_dims(array_sub, axis=1),
axis=1)
time_dim, height, width, band_dim = array_sub.shape
index = time_dim * band_dim
dst_transform = rasterio.transform.from_bounds(*eopatch.bbox,
width=width,
height=height)
dst_crs = {'init': CRS.ogc_string(eopatch.bbox.crs)}
# Write it out to a file.
with rasterio.open(os.path.join(self.folder, filename),
'w',
driver='GTiff',
width=width,
height=height,
count=index,
dtype=self.image_dtype,
nodata=self.no_data_value,
transform=dst_transform,
crs=dst_crs) as dst:
output_array = array_sub.astype(self.image_dtype)
output_array = np.moveaxis(output_array, -1,
1).reshape(index, height, width)
dst.write(output_array)
return eopatch
def test_crs_parsing(self):
test_cases = [
(4326, CRS.WGS84),
('4326', CRS.WGS84),
('EPSG:3857', CRS.POP_WEB),
({
'init': 'EPSG:32638'
}, CRS.UTM_38N),
(pyproj.CRS('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'),
CRS.WGS84),
('urn:ogc:def:crs:epsg::32631', CRS.UTM_31N),
('urn:ogc:def:crs:OGC::CRS84', CRS.WGS84),
(pyproj.CRS(3857), CRS.POP_WEB),
(pyproj.CRS(3857), CRS.POP_WEB),
]
for parse_value, expected_result in test_cases:
with self.subTest(msg=str(parse_value)):
parsed_result = CRS(parse_value)
self.assertEqual(parsed_result, expected_result)
with self.assertWarns(SHUserWarning):
wgs84 = CRS(pyproj.CRS(4326))
self.assertEqual(wgs84, CRS.WGS84)
def _load_vector_data(self, bbox):
""" Loads vector data from geopedia table
"""
prepared_bbox = bbox.transform_bounds(CRS.POP_WEB) if bbox else None
geopedia_iterator = GeopediaFeatureIterator(layer=self.geopedia_table,
bbox=prepared_bbox,
offset=0,
gpd_session=None,
config=self.config,
**self.geopedia_kwargs)
geopedia_features = list(geopedia_iterator)
geometry = geopedia_features[0].get('geometry')
if not geometry:
raise ValueError(
f'Geopedia table "{self.geopedia_table}" does not contain geometries!'
)
self.dataset_crs = CRS(
geometry['crs']['properties']['name']) # always WGS84
return gpd.GeoDataFrame.from_features(
geopedia_features, crs=self.dataset_crs.pyproj_crs())
class GeopediaVectorImportTask(_BaseVectorImportTask):
""" A task for importing `Geopedia <https://geopedia.world>`__ features into EOPatch vector features
"""
def __init__(self,
feature,
geopedia_table,
reproject=True,
clip=False,
**kwargs):
"""
:param feature: A vector feature into which to import data
:type feature: (FeatureType, str)
:param geopedia_table: A Geopedia table from which to retrieve features
:type geopedia_table: str or int
:param reproject: Should the geometries be transformed to coordinate reference system of the requested bbox?
:type reproject: bool, default = True
:param clip: Should the geometries be clipped to the requested bbox, or should be geometries kept as they are?
:type clip: bool, default = False
:param kwargs: Additional args that will be passed to `GeopediaFeatureIterator`
"""
self.geopedia_table = geopedia_table
self.geopedia_kwargs = kwargs
self.dataset_crs = None
super().__init__(feature=feature, reproject=reproject, clip=clip)
def _load_vector_data(self, bbox):
""" Loads vector data from geopedia table
"""
prepared_bbox = bbox.transform_bounds(CRS.POP_WEB) if bbox else None
geopedia_iterator = GeopediaFeatureIterator(layer=self.geopedia_table,
bbox=prepared_bbox,
offset=0,
gpd_session=None,
config=self.config,
**self.geopedia_kwargs)
geopedia_features = list(geopedia_iterator)
geometry = geopedia_features[0].get('geometry')
if not geometry:
raise ValueError(
f'Geopedia table "{self.geopedia_table}" does not contain geometries!'
)
self.dataset_crs = CRS(
geometry['crs']['properties']['name']) # always WGS84
return gpd.GeoDataFrame.from_features(
geopedia_features, crs=self.dataset_crs.pyproj_crs())
def __init__(
self,
feature,
folder=None,
*,
band_indices=None,
date_indices=None,
crs=None,
fail_on_missing=True,
compress=None,
**kwargs,
):
"""
:param feature: Feature which will be exported
:type feature: (FeatureType, str)
:param folder: A directory containing image files or a path of an image file.
If the file extension of the image file is not provided, it will default to ".tif".
If a "*" wildcard or a datetime.strftime substring (e.g. "%Y%m%dT%H%M%S") is provided in the image file,
an EOPatch feature will be split over multiple GeoTiffs each corresponding to a timestamp,
and the stringified datetime will be appended to the image file name.
:type folder: str
:param band_indices: Bands to be added to tiff image. Bands are represented by their 0-based index as tuple
in the inclusive interval form `(start_band, end_band)` or as list in the form
`[band_1, band_2,...,band_n]`.
:type band_indices: tuple or list or None
:param date_indices: Dates to be added to tiff image. Dates are represented by their 0-based index as tuple
in the inclusive interval form `(start_date, end_date)` or a list in the form `[date_1, date_2,...,date_n]`.
:type date_indices: tuple or list or None
:param crs: CRS in which to reproject the feature before writing it to GeoTiff
:type crs: CRS or str or None
:param fail_on_missing: should the pipeline fail if a feature is missing or just log warning and return
:type fail_on_missing: bool
:param compress: the type of compression that rasterio should apply to exported image.
:type compress: str or None
:param image_dtype: Type of data to be exported into tiff image
:type image_dtype: numpy.dtype
:param no_data_value: Value of pixels of tiff image with no data in EOPatch
:type no_data_value: int or float
:param config: A configuration object containing AWS credentials
:type config: SHConfig
"""
super().__init__(feature, folder=folder, **kwargs)
self.band_indices = band_indices
self.date_indices = date_indices
self.crs = None if crs is None else CRS(crs)
self.fail_on_missing = fail_on_missing
self.compress = compress
def _load_from_bbox(self, eopatch=None, bbox=None):
feature_type, feature_name = next(self.feature())
if eopatch is None:
eopatch = GeogeniusEOPatch()
if bbox is not None:
eopatch.bbox = bbox
data_bounds = self.geogenius_image.bounds
data_bbox = BBox(
(data_bounds[0], data_bounds[1], data_bounds[2], data_bounds[3]),
CRS(self.geogenius_image.proj))
if eopatch.bbox is None:
eopatch.bbox = data_bbox
if data_bbox.geometry.intersects(eopatch.bbox.geometry):
data = self.geogenius_image.aoi(bbox=[
eopatch.bbox.min_x, eopatch.bbox.min_y, eopatch.bbox.max_x,
eopatch.bbox.max_y
])
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
'Cannot import as a time-dependant feature because the number of tiff image channels '
'is not divisible by the number of timestamps')
data = data.reshape((times, channels // times) +
data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = _DaskArrayLoader(data)
return eopatch
else:
raise ValueError(
"AOI does not intersect image: {} not in {}".format(
self.geogenius_image.bounds, eopatch.bbox))
def setUpClass(cls):
super().setUpClass()
polygon = shapely.geometry.Polygon([(465888.8773268595, 5079639.43613863),
(465885.3413983975, 5079641.52461826),
(465882.9542217017, 5079647.16604353),
(465888.8780175466, 5079668.70367663),
(465888.877326859, 5079639.436138632)])
cls.wkt_string = 'MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), ' \
'(30 20, 20 15, 20 25, 30 20)))'
cls.geometry1 = Geometry(polygon, CRS(32633))
cls.geometry2 = Geometry(cls.wkt_string, CRS.WGS84)
cls.bbox = BBox(bbox=[14.00, 45.00, 14.03, 45.03], crs=CRS.WGS84)
cls.bbox_collection = BBoxCollection([cls.bbox, BBox('46,13,47,20', CRS.WGS84)])
cls.geometry_list = [cls.geometry1, cls.geometry2, cls.bbox, cls.bbox_collection]
def _load_from_pixelbox(self, eopatch=None, pixelbox=None):
self._check_pixelbox(pixelbox)
feature_type, feature_name = next(self.feature())
if eopatch is None:
eopatch = GeogeniusEOPatch()
min_pixel_x, min_pixel_y, max_pixel_x, max_pixel_y = pixelbox
data = self.geogenius_image[:, min_pixel_y:max_pixel_y,
min_pixel_x:max_pixel_x]
real_y_pixel, real_x_pixel = data.shape[1:]
pad_x = (max_pixel_x - min_pixel_x) - real_x_pixel
pad_y = (max_pixel_y - min_pixel_y) - real_y_pixel
data_bounds = data.bounds
data_bbox = BBox(
(data_bounds[0], data_bounds[1], data_bounds[2], data_bounds[3]),
CRS(self.geogenius_image.proj))
eopatch.bbox = data_bbox
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
'Cannot import as a time-dependant feature because the number of tiff image channels '
'is not divisible by the number of timestamps')
data = data.reshape((times, channels // times) + data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = _DaskArrayLoader(data,
pad_x=pad_x,
pad_y=pad_y)
return eopatch
def execute(self, eopatch, *, filename):
array = eopatch.get_feature(self.feature_type, self.feature_name)
if self.band_count == 1:
array = array[..., 0]
dst_shape = array.shape
dst_transform = rasterio.transform.from_bounds(*eopatch.bbox, width=dst_shape[1], height=dst_shape[0])
dst_crs = {'init': CRS.ogc_string(eopatch.bbox.crs)}
# Write it out to a file.
with rasterio.open(filename, 'w', driver='GTiff',
width=dst_shape[1], height=dst_shape[0],
count=self.band_count, dtype=self.image_dtype, nodata=self.no_data_value,
transform=dst_transform, crs=dst_crs) as dst:
dst.write(array.astype(self.image_dtype), indexes=self.band_count)
return eopatch
def get_random_bbox(self, area_geometry):
"""
:param area_geometry: Geometry object which has to be in UTM CRS
:type area_geometry: sentinelhub.Geometry
"""
if not CRS.is_utm(area_geometry.crs):
raise ValueError(
'Geometry object has to be in UTM CRS for sampling')
reduced_geo = self._expand_geo_shape(area_geometry.geometry,
1 / self.resolution)
sampled_rectangle = random_sample(
reduced_geo, (self.window_shape[0] + 2 * self.buffer,
self.window_shape[1] + 2 * self.buffer))
expanded_rectangle = self._expand_geo_shape(sampled_rectangle,
self.resolution)
return BBox(expanded_rectangle.bounds, crs=area_geometry.crs)
