def test_custom_crs(self):
for incorrect_value in ['string', -1, 999, None]:
with self.assertRaises(ValueError):
CRS(incorrect_value)
for correct_value in [3035, 'EPSG:3035', 10000]:
CRS(CRS(correct_value))
new_enum_value = str(correct_value).lower().strip('epsg: ')
self.assertTrue(CRS.has_value(new_enum_value))
def __init__(self, feature, folder=None, *, band_indices=None, date_indices=None, crs=None, fail_on_missing=True,
**kwargs):
"""
:param feature: Feature which will be exported
:type feature: (FeatureType, str)
:param folder: A directory containing image files or a path of an image file
:type folder: str
:param band_indices: Bands to be added to tiff image. Bands are represented by their 0-based index as tuple
in the inclusive interval form `(start_band, end_band)` or as list in the form
`[band_1, band_2,...,band_n]`.
:type band_indices: tuple or list or None
:param date_indices: Dates to be added to tiff image. Dates are represented by their 0-based index as tuple
in the inclusive interval form `(start_date, end_date)` or a list in the form `[date_1, date_2,...,date_n]`.
:type date_indices: tuple or list or None
:param crs: CRS in which to reproject the feature before writing it to GeoTiff
:type crs: CRS or str or None
:param fail_on_missing: should the pipeline fail if a feature is missing or just log warning and return
:type fail_on_missing: bool
:param image_dtype: Type of data to be exported into tiff image
:type image_dtype: numpy.dtype
:param no_data_value: Value of pixels of tiff image with no data in EOPatch
:type no_data_value: int or float
"""
super().__init__(feature, folder=folder, **kwargs)
self.band_indices = band_indices
self.date_indices = date_indices
self.crs = None if crs is None else CRS(crs)
self.fail_on_missing = fail_on_missing
def process(self, arguments):
data_as_list = self.validate_parameter(arguments,
"data",
required=True,
allowed_types=[list])
dims = self.validate_parameter(arguments,
"dims",
required=True,
allowed_types=[list])
coords = self.validate_parameter(arguments,
"coords",
allowed_types=[dict],
default={})
if "t" in coords:
coords["t"] = [
datetime.strptime(d, '%Y-%m-%d %H:%M:%S') for d in coords["t"]
]
data = xr.DataArray(
np.array(data_as_list, dtype=np.float),
coords=coords,
dims=dims,
attrs={
"band_aliases": {},
"bbox": BBox((
12.0,
45.0,
13.0,
46.0,
), CRS(4326)),
},
)
self.logger.info(data)
return data
def _repr_value(value):
"""Creates a representation string for different types of data.
:param value: data in any type
:return: representation string
:rtype: str
"""
if isinstance(value, np.ndarray):
return f"{EOPatch._repr_value_class(value)}(shape={value.shape}, dtype={value.dtype})"
if isinstance(value, gpd.GeoDataFrame):
crs = CRS(value.crs).ogc_string() if value.crs else value.crs
return f"{EOPatch._repr_value_class(value)}(columns={list(value)}, length={len(value)}, crs={crs})"
if isinstance(value, (list, tuple, dict)) and value:
repr_str = str(value)
if len(repr_str) <= MAX_DATA_REPR_LEN:
return repr_str
l_bracket, r_bracket = ("[",
"]") if isinstance(value, list) else ("(",
")")
if isinstance(value, (list, tuple)) and len(value) > 2:
repr_str = f"{l_bracket}{repr(value[0])}, ..., {repr(value[-1])}{r_bracket}"
if len(repr_str) > MAX_DATA_REPR_LEN and isinstance(
value, (list, tuple)) and len(value) > 1:
repr_str = f"{l_bracket}{repr(value[0])}, ...{r_bracket}"
if len(repr_str) > MAX_DATA_REPR_LEN:
repr_str = str(type(value))
return f"{repr_str}, length={len(value)}"
return repr(value)
def test_repr(self):
crs_values = (
(CRS.POP_WEB, "CRS('3857')"),
(CRS.WGS84, "CRS('4326')"),
(CRS.UTM_33N, "CRS('32633')"),
(CRS.UTM_33S, "CRS('32733')"),
(CRS('3857'), "CRS('3857')"),
(CRS('4326'), "CRS('4326')"),
(CRS('32633'), "CRS('32633')"),
(CRS('32733'), "CRS('32733')"),
)
for crs, crs_repr in crs_values:
with self.subTest(msg=crs_repr):
self.assertEqual(crs_repr,
repr(crs),
msg="Expected {}, got {}".format(
crs_repr, repr(crs)))
def setUpClass(cls):
super().setUpClass()
with open(os.path.join(cls.INPUT_FOLDER, "test_fis_results.txt"), 'r') as file:
results = [ast.literal_eval(line.strip()) for line in file]
bbox = BBox([14.00, 45.00, 14.03, 45.03], crs=CRS.WGS84)
geometry1 = Geometry(Polygon([(465888.877326859, 5079639.436138632),
(465885.3413983975, 5079641.524618266),
(465882.9542217017, 5079647.166043535),
(465888.8780175466, 5079668.703676634),
(465888.877326859, 5079639.436138632)]),
CRS(32633))
geometry2 = Geometry('POLYGON((-5.13 48, -5.23 48.09, -5.13 48.17, -5.03 48.08, -5.13 48))', CRS.WGS84)
cls.test_cases = [
cls.FisTestCase('geometry',
FisRequest(layer='TRUE-COLOR-S2-L1C',
geometry_list=[geometry1],
time=('2017-1-1', '2017-2-1'),
resolution="50m",
histogram_type=HistogramType.STREAMING,
bins=5),
raw_result=results[0],
result_length=1),
cls.FisTestCase('bbox',
FisRequest(layer='BANDS-S2-L1C',
geometry_list=[bbox],
time='2017-1-1',
resolution="50m",
maxcc=0.2,
custom_url_params={
CustomUrlParam.ATMFILTER: "ATMCOR",
CustomUrlParam.DOWNSAMPLING: "BICUBIC",
CustomUrlParam.UPSAMPLING: "BICUBIC"}
),
raw_result=results[1],
result_length=1),
cls.FisTestCase('list',
FisRequest(data_source=DataSource.LANDSAT8,
layer='BANDS-L8',
geometry_list=[bbox, geometry1],
time=('2017-1-1', '2017-1-10'),
resolution="100m",
bins=32, data_folder=cls.OUTPUT_FOLDER),
raw_result=results[2], result_length=2,
save_data=True),
cls.FisTestCase('Polygon in WGS84',
FisRequest(layer='TRUE-COLOR-S2-L1C',
geometry_list=[geometry2],
time=('2017-10-1', '2017-10-2'),
resolution="60m",
bins=11, histogram_type=HistogramType.EQUALFREQUENCY),
raw_result=results[3], result_length=1),
]
for test_case in cls.test_cases:
test_case.collect_data()
def _get_rasterization_shapes(self, eopatch, group_classes=False):
""" Returns a generator of pairs of geometrical shapes and values. In rasterization process each shape will be
rasterized to it's corresponding value.
If there are no such geometries it will return `None`
:param group_classes: If true, function returns a zip that iterates through cascaded unions of polygons of the
same class, otherwise zip iterates through all polygons regardless of their class.
:type group_classes: boolean
"""
vector_data = self._get_vector_data(eopatch)
gpd_crs = vector_data.crs
# This special case has to be handled because of WGS84 and lat-lon order:
if isinstance(gpd_crs, pyproj.CRS):
gpd_crs = gpd_crs.to_epsg()
vector_data_crs = CRS(gpd_crs)
if eopatch.bbox.crs is not vector_data_crs:
warnings.warn('Vector data is not in the same CRS as EOPatch, this task will re-project vector data for '
'each execution', RuntimeWarning)
vector_data = vector_data.to_crs(eopatch.bbox.crs.pyproj_crs())
bbox_poly = eopatch.bbox.geometry
vector_data = vector_data[vector_data.geometry.intersects(bbox_poly)].copy(deep=True)
if vector_data.empty:
return None
if self.buffer:
vector_data.geometry = vector_data.geometry.buffer(self.buffer)
vector_data = vector_data[~vector_data.is_empty]
# vector_data could be empty as a result of (negative) buffer
if vector_data.empty:
return None
if not vector_data.geometry.is_valid.all():
warnings.warn('Given vector polygons contain some invalid geometries, they will be fixed', RuntimeWarning)
vector_data.geometry = vector_data.geometry.buffer(0)
if vector_data.geometry.has_z.any():
warnings.warn('Given vector polygons contain some 3D geometries, they will be projected to 2D',
RuntimeWarning)
vector_data.geometry = vector_data.geometry.apply(lambda geo: shapely.wkt.loads(geo.to_wkt()))
if self.values_column is None:
return zip(vector_data.geometry, [self.values] * len(vector_data.index))
if self.values is not None:
values = [self.values] if isinstance(self.values, (int, float)) else self.values
vector_data = vector_data[vector_data[self.values_column].isin(values)]
if group_classes:
classes = np.unique(vector_data[self.values_column])
grouped = (vector_data.geometry[vector_data[self.values_column] == cl] for cl in classes)
grouped = (shapely.ops.cascaded_union(group) for group in grouped)
return zip(grouped, classes)
return zip(vector_data.geometry, vector_data[self.values_column])
def _convert_bbox(spatial_extent):
crs = spatial_extent.get('crs', 4326)
return BBox(
(spatial_extent['west'],
spatial_extent['south'],
spatial_extent['east'],
spatial_extent['north'],),
CRS(crs), # we support whatever sentinelhub-py supports
)
def test_bbox_to_repr(self):
x1, y1, x2, y2 = 45.0, 12.0, 47.0, 14.0
bbox = BBox(((x1, y1), (x2, y2)), crs=CRS('4326'))
expect_repr = "BBox((({}, {}), ({}, {})), crs=CRS('4326'))".format(
x1, y1, x2, y2)
self.assertEqual(
repr(bbox),
expect_repr,
msg="String representations not matching: expected {}, got {}".
format(expect_repr, repr(bbox)))
def test_crs_parsing(self):
test_cases = [
(4326, CRS.WGS84),
('4326', CRS.WGS84),
('EPSG:3857', CRS.POP_WEB),
({
'init': 'EPSG:32638'
}, CRS.UTM_38N),
(pyproj.CRS('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'),
CRS.WGS84),
(pyproj.CRS(3857), CRS.POP_WEB),
]
for parse_value, expected_result in test_cases:
with self.subTest(msg=str(parse_value)):
parsed_result = CRS(parse_value)
self.assertEqual(parsed_result, expected_result)
with self.assertRaises(ValueError):
CRS(pyproj.CRS(4326))
def _preprocess_vector_data(self, vector_data, bbox, timestamps):
"""Applies preprocessing steps on a dataframe with geometries and potential values and timestamps"""
columns_to_keep = ["geometry"]
if self._rasterize_per_timestamp:
columns_to_keep.append("TIMESTAMP")
if self.values_column is not None:
columns_to_keep.append(self.values_column)
vector_data = vector_data[columns_to_keep]
if self._rasterize_per_timestamp:
vector_data = vector_data[vector_data.TIMESTAMP.isin(timestamps)]
if self.values_column is not None and self.values is not None:
values = [self.values] if isinstance(self.values,
(int, float)) else self.values
vector_data = vector_data[vector_data[self.values_column].isin(
values)]
gpd_crs = vector_data.crs
# This special case has to be handled because of WGS84 and lat-lon order:
if isinstance(gpd_crs, pyproj.CRS):
gpd_crs = gpd_crs.to_epsg()
vector_data_crs = CRS(gpd_crs)
if bbox.crs is not vector_data_crs:
warnings.warn(
"Vector data is not in the same CRS as EOPatch, this task will re-project vector data for "
"each execution",
EORuntimeWarning,
)
vector_data = vector_data.to_crs(bbox.crs.pyproj_crs())
bbox_poly = bbox.geometry
vector_data = vector_data[vector_data.geometry.intersects(
bbox_poly)].copy(deep=True)
if self.buffer:
vector_data.geometry = vector_data.geometry.buffer(self.buffer)
vector_data = vector_data[~vector_data.is_empty]
if not vector_data.geometry.is_valid.all():
warnings.warn(
"Given vector polygons contain some invalid geometries, they will be fixed",
EORuntimeWarning)
vector_data.geometry = vector_data.geometry.buffer(0)
if vector_data.geometry.has_z.any():
warnings.warn(
"Given vector polygons contain some 3D geometries, they will be projected to 2D",
EORuntimeWarning)
vector_data.geometry = vector_data.geometry.map(
functools.partial(shapely.ops.transform,
lambda *args: args[:2]))
return vector_data
def execute(self, eopatch=None, *, filename=None):
""" Execute method which adds a new feature to the EOPatch
:param eopatch: input EOPatch or None if a new EOPatch should be created
:type eopatch: EOPatch or None
:param filename: filename of tiff file or None if entire path has already been specified in `folder` parameter
of task initialization.
:type filename: str or None
:return: New EOPatch with added raster layer
:rtype: EOPatch
"""
feature_type, feature_name = next(self.feature())
if eopatch is None:
eopatch = EOPatch()
with rasterio.open(self._get_file_path(filename)) as source:
data_bbox = BBox(source.bounds, CRS(source.crs.to_epsg()))
if eopatch.bbox is None:
eopatch.bbox = data_bbox
reading_window = self._get_reading_window(source.width,
source.height, data_bbox,
eopatch.bbox)
data = source.read(window=reading_window,
boundless=True,
fill_value=self.no_data_value)
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
'Cannot import as a time-dependant feature because the number of tiff image channels '
'is not divisible by the number of timestamps')
data = data.reshape((times, channels // times) + data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = data
return eopatch
def _get_task(task_data):
window = json.loads(task_data['window'])
return Task(
task_id=task_data['task_id'],
bbox=BBox(
bbox=json.loads(task_data['bbox']), # TODO: why is string?
crs=CRS(task_data['crs'])),
acq_time=dt.datetime.strptime(task_data['datetime'], '%Y-%m-%d'),
window_shape=[window['height'], window['width']],
data_list=json.loads(task_data['data']),
vector_data=task_data['vector_data'])
def dataset_crs(self):
"""Provides a "crs" of dataset, loads it lazily (i.e. the first time it is needed)
:return: Dataset's CRS
:rtype: CRS
"""
if self._dataset_crs is None:
srid = self.geodb_client.get_collection_srid(collection=self.geodb_collection, database=self.geodb_db)
self._dataset_crs = CRS(f"epsg:{srid}")
return self._dataset_crs
def _decode(self, file, path):
"""Loads from a file and decodes content."""
file_format = MimeType(fs.path.splitext(path)[1].strip("."))
if file_format is MimeType.NPY:
return np.load(file)
if file_format is MimeType.GPKG:
dataframe = gpd.read_file(file)
if dataframe.crs is not None:
# Trying to preserve a standard CRS and passing otherwise
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=SHUserWarning)
dataframe.crs = CRS(dataframe.crs).pyproj_crs()
except ValueError:
pass
if "TIMESTAMP" in dataframe:
dataframe.TIMESTAMP = pd.to_datetime(dataframe.TIMESTAMP)
return dataframe
if file_format in [MimeType.JSON, MimeType.GEOJSON]:
json_data = json.load(file)
if self.feature_type is FeatureType.BBOX:
return Geometry.from_geojson(json_data).bbox
return json_data
if file_format is MimeType.PICKLE:
warnings.warn(
f"File {self.path} with data of type {self.feature_type} is in pickle format which is deprecated "
"since eo-learn version 1.0. Please re-save this EOPatch with the new eo-learn version to "
"update the format. In newer versions this backward compatibility will be removed.",
EODeprecationWarning,
)
data = pickle.load(file)
# There seems to be an issue in geopandas==0.8.1 where unpickling GeoDataFrames, which were saved with an
# old geopandas version, loads geometry column into a pandas.Series instead geopandas.GeoSeries. Because
# of that it is missing a crs attribute which is only attached to the entire GeoDataFrame
if isinstance(data, GeoDataFrame) and not isinstance(
data.geometry, GeoSeries):
data = data.set_geometry("geometry")
return data
raise ValueError(f"Unsupported data type for file {path}.")
def test_crs_parsing(self):
test_cases = [
(4326, CRS.WGS84),
('4326', CRS.WGS84),
('EPSG:3857', CRS.POP_WEB),
({
'init': 'EPSG:32638'
}, CRS.UTM_38N),
(pyproj.CRS('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'),
CRS.WGS84),
('urn:ogc:def:crs:epsg::32631', CRS.UTM_31N),
('urn:ogc:def:crs:OGC::CRS84', CRS.WGS84),
(pyproj.CRS(3857), CRS.POP_WEB),
(pyproj.CRS(3857), CRS.POP_WEB),
]
for parse_value, expected_result in test_cases:
with self.subTest(msg=str(parse_value)):
parsed_result = CRS(parse_value)
self.assertEqual(parsed_result, expected_result)
with self.assertWarns(SHUserWarning):
wgs84 = CRS(pyproj.CRS(4326))
self.assertEqual(wgs84, CRS.WGS84)
def __init__(
self,
feature,
folder=None,
*,
band_indices=None,
date_indices=None,
crs=None,
fail_on_missing=True,
compress=None,
**kwargs,
):
"""
:param feature: Feature which will be exported
:type feature: (FeatureType, str)
:param folder: A directory containing image files or a path of an image file.
If the file extension of the image file is not provided, it will default to ".tif".
If a "*" wildcard or a datetime.strftime substring (e.g. "%Y%m%dT%H%M%S") is provided in the image file,
an EOPatch feature will be split over multiple GeoTiffs each corresponding to a timestamp,
and the stringified datetime will be appended to the image file name.
:type folder: str
:param band_indices: Bands to be added to tiff image. Bands are represented by their 0-based index as tuple
in the inclusive interval form `(start_band, end_band)` or as list in the form
`[band_1, band_2,...,band_n]`.
:type band_indices: tuple or list or None
:param date_indices: Dates to be added to tiff image. Dates are represented by their 0-based index as tuple
in the inclusive interval form `(start_date, end_date)` or a list in the form `[date_1, date_2,...,date_n]`.
:type date_indices: tuple or list or None
:param crs: CRS in which to reproject the feature before writing it to GeoTiff
:type crs: CRS or str or None
:param fail_on_missing: should the pipeline fail if a feature is missing or just log warning and return
:type fail_on_missing: bool
:param compress: the type of compression that rasterio should apply to exported image.
:type compress: str or None
:param image_dtype: Type of data to be exported into tiff image
:type image_dtype: numpy.dtype
:param no_data_value: Value of pixels of tiff image with no data in EOPatch
:type no_data_value: int or float
:param config: A configuration object containing AWS credentials
:type config: SHConfig
"""
super().__init__(feature, folder=folder, **kwargs)
self.band_indices = band_indices
self.date_indices = date_indices
self.crs = None if crs is None else CRS(crs)
self.fail_on_missing = fail_on_missing
self.compress = compress
def _load_from_bbox(self, eopatch=None, bbox=None):
feature_type, feature_name = next(self.feature())
if eopatch is None:
eopatch = GeogeniusEOPatch()
if bbox is not None:
eopatch.bbox = bbox
data_bounds = self.geogenius_image.bounds
data_bbox = BBox(
(data_bounds[0], data_bounds[1], data_bounds[2], data_bounds[3]),
CRS(self.geogenius_image.proj))
if eopatch.bbox is None:
eopatch.bbox = data_bbox
if data_bbox.geometry.intersects(eopatch.bbox.geometry):
data = self.geogenius_image.aoi(bbox=[
eopatch.bbox.min_x, eopatch.bbox.min_y, eopatch.bbox.max_x,
eopatch.bbox.max_y
])
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
'Cannot import as a time-dependant feature because the number of tiff image channels '
'is not divisible by the number of timestamps')
data = data.reshape((times, channels // times) +
data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = _DaskArrayLoader(data)
return eopatch
else:
raise ValueError(
"AOI does not intersect image: {} not in {}".format(
self.geogenius_image.bounds, eopatch.bbox))
def setUpClass(cls):
super().setUpClass()
polygon = shapely.geometry.Polygon([(465888.8773268595, 5079639.43613863),
(465885.3413983975, 5079641.52461826),
(465882.9542217017, 5079647.16604353),
(465888.8780175466, 5079668.70367663),
(465888.877326859, 5079639.436138632)])
cls.wkt_string = 'MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), ' \
'(30 20, 20 15, 20 25, 30 20)))'
cls.geometry1 = Geometry(polygon, CRS(32633))
cls.geometry2 = Geometry(cls.wkt_string, CRS.WGS84)
cls.bbox = BBox(bbox=[14.00, 45.00, 14.03, 45.03], crs=CRS.WGS84)
cls.bbox_collection = BBoxCollection([cls.bbox, BBox('46,13,47,20', CRS.WGS84)])
cls.geometry_list = [cls.geometry1, cls.geometry2, cls.bbox, cls.bbox_collection]
def _load_from_pixelbox(self, eopatch=None, pixelbox=None):
self._check_pixelbox(pixelbox)
feature_type, feature_name = next(self.feature())
if eopatch is None:
eopatch = GeogeniusEOPatch()
min_pixel_x, min_pixel_y, max_pixel_x, max_pixel_y = pixelbox
data = self.geogenius_image[:, min_pixel_y:max_pixel_y,
min_pixel_x:max_pixel_x]
real_y_pixel, real_x_pixel = data.shape[1:]
pad_x = (max_pixel_x - min_pixel_x) - real_x_pixel
pad_y = (max_pixel_y - min_pixel_y) - real_y_pixel
data_bounds = data.bounds
data_bbox = BBox(
(data_bounds[0], data_bounds[1], data_bounds[2], data_bounds[3]),
CRS(self.geogenius_image.proj))
eopatch.bbox = data_bbox
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
'Cannot import as a time-dependant feature because the number of tiff image channels '
'is not divisible by the number of timestamps')
data = data.reshape((times, channels // times) + data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = _DaskArrayLoader(data,
pad_x=pad_x,
pad_y=pad_y)
return eopatch
def _load_data(self, filename):
""" Load data from file """
data = read_data(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
filename))
self.source_dict = {}
for source_info in data['sources']:
source = SourceType(source_info['source_type'])
source_info = to_python(source_info)
if source is SourceType.S2_L1C_ARCHIVE:
self.source_dict[source_info['id']] = Source(**source_info)
elif source is SourceType.GEOPEDIA_WB:
self.source_dict[source_info['id']] = Source(**source_info)
else:
raise NotImplementedError(
'Support for source {} is not implemented')
self.campaign_collection = {}
for campaign_info in data['campaigns']:
campaign_info = to_python(campaign_info)
# TODO: add parsing of Box for local back-end
if 'bbox' not in campaign_info:
campaign_info['bbox'] = BBox(bbox=((0, 0), (0, 0)),
crs=CRS(4326))
for param in ['input_source', 'output_source']:
campaign_info[param] = self.source_dict[campaign_info[param]
['id']]
self.campaign_collection[campaign_info['id']] = Campaign(
**campaign_info)
self.user_collection = {
user_info['id']: User(**to_python(user_info))
for user_info in data['users']
}
def _repr_value(value):
"""Creates a representation string for different types of data.
:param value: data in any type
:return: representation string
:rtype: str
"""
if isinstance(value, np.ndarray):
return '{}(shape={}, dtype={})'.format(
EOPatch._repr_value_class(value), value.shape, value.dtype)
if isinstance(value, gpd.GeoDataFrame):
crs = CRS(value.crs).ogc_string() if value.crs else value.crs
return f'{EOPatch._repr_value_class(value)}(' \
f'columns={list(value)}, ' \
f'length={len(value)}, ' \
f'crs={crs})'
if isinstance(value, (list, tuple, dict)) and value:
repr_str = str(value)
if len(repr_str) <= MAX_DATA_REPR_LEN:
return repr_str
bracket_str = '[{}]' if isinstance(value, list) else '({})'
if isinstance(value, (list, tuple)) and len(value) > 2:
repr_str = bracket_str.format('{}, ..., {}'.format(
repr(value[0]), repr(value[-1])))
if len(repr_str) > MAX_DATA_REPR_LEN and isinstance(
value, (list, tuple)) and len(value) > 1:
repr_str = bracket_str.format('{}, ...'.format(repr(value[0])))
if len(repr_str) > MAX_DATA_REPR_LEN:
repr_str = str(type(value))
return '{}, length={}'.format(repr_str, len(value))
return repr(value)
def _load_vector_data(self, bbox):
""" Loads vector data from geopedia table
"""
prepared_bbox = bbox.transform_bounds(CRS.POP_WEB) if bbox else None
geopedia_iterator = GeopediaFeatureIterator(layer=self.geopedia_table,
bbox=prepared_bbox,
offset=0,
gpd_session=None,
config=self.config,
**self.geopedia_kwargs)
geopedia_features = list(geopedia_iterator)
geometry = geopedia_features[0].get('geometry')
if not geometry:
raise ValueError(
f'Geopedia table "{self.geopedia_table}" does not contain geometries!'
)
self.dataset_crs = CRS(
geometry['crs']['properties']['name']) # always WGS84
return gpd.GeoDataFrame.from_features(
geopedia_features, crs=self.dataset_crs.pyproj_crs())
def _get_rasterization_shapes(self, eopatch):
""" Returns a generator of pairs of geometrical shapes and values. In rasterization process each shape will be
rasterized to it's corresponding value.
If there are no such geometries it will return `None`
"""
vector_data = self._get_vector_data(eopatch)
if 'init' not in vector_data.crs:
raise ValueError('Cannot recognize CRS of vector data')
vector_data_crs = CRS(vector_data.crs['init'])
if eopatch.bbox.crs is not vector_data_crs:
raise ValueError("Vector data and EOPatch should be in the same CRS, found '{}' and '{}'"
"".format(eopatch.bbox.crs, vector_data_crs))
bbox_poly = eopatch.bbox.geometry
vector_data = vector_data[vector_data.geometry.intersects(bbox_poly)].copy(deep=True)
if vector_data.empty:
return None
if not vector_data.geometry.is_valid.all():
warnings.warn('Given vector polygons contain some invalid geometries, they will be fixed', RuntimeWarning)
vector_data.geometry = vector_data.geometry.buffer(0)
if vector_data.geometry.has_z.any():
warnings.warn('Given vector polygons contain some 3D geometries, they will be projected to 2D',
RuntimeWarning)
vector_data.geometry = vector_data.geometry.apply(lambda geo: shapely.wkt.loads(geo.to_wkt()))
if self.values_column is None:
return zip(vector_data.geometry, [self.values] * len(vector_data.index))
if self.values is not None:
values = [self.values] if isinstance(self.values, (int, float)) else self.values
vector_data = vector_data[vector_data[self.values_column].isin(values)]
return zip(vector_data.geometry, vector_data[self.values_column])
def test_multipart_geometry(self):
evalscript = """
//VERSION=3
function setup() {
return {
input: ["B02", "B03", "B04", "dataMask"],
output: { id:"default", bands: 3}
}
}
function updateOutputMetadata(scenes, inputMetadata, outputMetadata) {
var sum = (r_count + g_count + b_count);
var r_rat = r_count / sum;
var g_rat = g_count / sum;
var b_rat = b_count / sum;
outputMetadata.userData = { "rgb_ratios": [r_rat, g_rat, b_rat] }
}
var r_count = 0;
var g_count = 0;
var b_count = 0;
function evaluatePixel(sample) {
b_count += sample.B02;
g_count += sample.B03;
r_count += sample.B04;
if (sample.dataMask == 0) {
return [1,1,1];
}
return [sample.B04*2.5, sample.B03*2.5, sample.B02*2.5];
}
"""
points = [(3983945.16471594, 4455475.78793186),
(3983888.00256275, 4455757.47439827),
(3983881.86585896, 4455756.21837259),
(3983806.23113651, 4456128.92147268),
(3983795.43856837, 4456181.52922753),
(3983782.49665288, 4456243.16761979),
(3983769.24786918, 4456304.74059236),
(3983755.69254332, 4456366.24660331),
(3983741.83100914, 4456427.68411298),
(3983731.89973217, 4456470.84795843),
(3983633.33670483, 4456895.81023678),
(3983639.43692276, 4456897.23726002),
(3983537.58701916, 4457336.35298979),
(3983531.486563, 4457334.92584801),
(3983451.81567033, 4457678.40439185),
(3983444.09684707, 4457713.91738361),
(3983433.21703553, 4457774.95105094),
(3983425.2423303, 4457836.4359853),
(3983420.19086095, 4457898.23280495),
(3983418.07413054, 4457960.2014162),
(3983418.89698951, 4458022.20133071),
(3983422.65762374, 4458084.09198476),
(3983429.05535825, 4458143.30899922),
(3983435.27377231, 4458142.55298424),
(3983439.97457434, 4458175.43769638),
(3983450.97468474, 4458236.14788553),
(3983466.88315168, 4458303.87476693),
(3983460.83517165, 4458305.51224157),
(3983466.80900589, 4458327.76588705),
(3983484.9991527, 4458387.02138291),
(3983505.97749719, 4458445.340412),
(3983538.67409472, 4458522.43435024),
(3983584.70089337, 4458635.18822735),
(3983780.40768297, 4459048.67824218),
(3983801.72985096, 4459096.84527808),
(3983883.42859759, 4459278.64097453),
(3984316.01202946, 4460214.51826613),
(3984398.97672295, 4460080.53793049),
(3984534.50220822, 4459799.86484374),
(3984577.77550522, 4459774.02321167),
(3984491.40157364, 4459687.94895666),
(3984776.22996932, 4459142.13379129),
(3984819.68594039, 4459029.12887873),
(3984907.71921624, 4458981.665405),
(3984888.9490588, 4458770.02890185),
(3985209.2168573, 4458503.41559024),
(3985821.45298221, 4458006.99923219),
(3985788.76207523, 4457880.30735337),
(3985793.50611539, 4457877.12247581),
(3985784.68739608, 4457859.48509427),
(3985732.13693102, 4457697.05635426),
(3985820.89433686, 4457656.86419316),
(3985677.94930497, 4457315.34906349),
(3985611.18897298, 4457337.80151946),
(3985327.61285454, 4457451.86990929),
(3985146.68294768, 4456972.64460213),
(3985446.37981687, 4456852.84034971),
(3985488.11295695, 4456837.9565739),
(3985384.27368677, 4456550.32595766),
(3985005.77351172, 4455718.96868536),
(3984372.83691021, 4455665.6888113),
(3984231.62160324, 4455623.03272949),
(3984096.30921154, 4455487.68759209),
(3983945.16471594, 4455475.78793186)]
sgeo = Polygon(points)
crs = CRS('epsg:3857')
geo = Geometry(sgeo, crs=crs)
bbox = BBox(sgeo.bounds, crs=crs)
size = bbox_to_dimensions(bbox, 10)
request = SentinelHubRequest(
evalscript=evalscript,
input_data=[
SentinelHubRequest.input_data(
data_source=DataSource.SENTINEL2_L1C,
time_interval=('2017-11-15T07:12:03',
'2017-12-15T07:12:04'),
)
],
responses=[
SentinelHubRequest.output_response('default', MimeType.TIFF),
SentinelHubRequest.output_response('userdata', MimeType.JSON)
],
bbox=bbox,
geometry=geo,
size=size)
tar = request.get_data(max_threads=3)[0]
img = tar['default.tif']
self.assertEqual(img.shape, (382, 181, 3))
self.test_numpy_data(img, exp_min=25, exp_max=255, exp_mean=144.89)
json_data = tar['userdata.json']
self.assertTrue('rgb_ratios' in json_data)
expected_ratios = [
0.29098381560041126, 0.3227735909047216, 0.3862425934948671
]
self.assertTrue(np.allclose(json_data['rgb_ratios'], expected_ratios))
def _read_crs(self):
""" Reads information about CRS from a dataset
"""
with fiona.open(self.path, **self.fiona_kwargs) as features:
self._dataset_crs = CRS(features.crs)
def execute(self, eopatch=None, *, filename=None):
"""Execute method which adds a new feature to the EOPatch
:param eopatch: input EOPatch or None if a new EOPatch should be created
:type eopatch: EOPatch or None
:param filename: filename of tiff file or None if entire path has already been specified in `folder` parameter
of task initialization.
:type filename: str, list of str or None
:return: New EOPatch with added raster layer
:rtype: EOPatch
"""
feature_type, feature_name = self.feature
if eopatch is None:
eopatch = EOPatch()
filesystem, filename_paths = self._get_filesystem_and_paths(
filename, eopatch.timestamp, create_paths=False)
with filesystem:
data = []
for path in filename_paths:
with filesystem.openbin(path, "r") as file_handle:
with rasterio.open(file_handle) as src:
boundless = True
if eopatch.bbox is None:
boundless = False
eopatch.bbox = BBox(src.bounds,
CRS(src.crs.to_epsg()))
read_window = self._get_reading_window(
src, eopatch.bbox)
data.append(
src.read(window=read_window,
boundless=boundless,
fill_value=self.no_data_value))
data = np.concatenate(data, axis=0)
if self.image_dtype is not None:
data = data.astype(self.image_dtype)
if not feature_type.is_spatial():
data = data.flatten()
if feature_type.is_timeless():
data = np.moveaxis(data, 0, -1)
else:
channels = data.shape[0]
times = self.timestamp_size
if times is None:
times = len(eopatch.timestamp) if eopatch.timestamp else 1
if channels % times != 0:
raise ValueError(
"Cannot import as a time-dependant feature because the number of tiff image channels "
"is not divisible by the number of timestamps")
data = data.reshape((times, channels // times) + data.shape[1:])
data = np.moveaxis(data, 1, -1)
eopatch[feature_type][feature_name] = data
return eopatch
This source code is licensed under the MIT license found in the LICENSE
file in the root directory of this source tree.
"""
import pytest
from eolearn.core import FeatureType
from eolearn.io import GeoDBVectorImportTask, GeopediaVectorImportTask, VectorImportTask
from sentinelhub import BBox, CRS
@pytest.mark.parametrize(argnames='reproject, clip, n_features, bbox, crs',
ids=['simple', 'bbox', 'bbox_full', 'bbox_smaller'],
argvalues=[(False, False, 193, None, None),
(False, False, 193,
BBox([857000, 6521500, 861000, 6525500],
CRS('epsg:2154')), None),
(True, True, 193,
BBox([657089, 5071037, 661093, 5075039],
CRS.UTM_31N), CRS.UTM_31N),
(True, True, 125,
BBox([657690, 5071637, 660493, 5074440],
CRS.UTM_31N), CRS.UTM_31N)])
class TestVectorImportTask:
""" Class for testing vector imports from local file, s3 bucket object and layer from Geopedia
"""
def test_import_local_file(self, gpkg_file, reproject, clip, n_features,
bbox, crs):
self._test_import(bbox, clip, crs, gpkg_file, n_features, reproject)
def test_import_s3_file(self, s3_gpkg_file, reproject, clip, n_features,
bbox, crs):
def get_crs(tile_info):
return CRS(tile_info['properties']['crs'].rsplit(':', 1)[1])
ndwi,
ndbi,
save,
)
time_interval = {
'18/19': ['2018-11-01', '2019-05-01'],
'19/20': ['2019-11-01', '2020-05-01'],
}
downloaded = os.listdir(SAVE_PATH)
downloaded = list(map(lambda x: int(x.split('_')[1]), downloaded))
execution_args = []
for id, row in gdf.loc[~gdf.index.isin(downloaded), :].iterrows():
bbox = row.geometry.bounds
bbox = BBox(bbox, CRS('32720'))
execution_args.append({
add_data: {
'bbox': bbox,
'time_interval': time_interval[row.Campania]
},
save: {
'eopatch_folder': f'eopatch_{id}'
}
})
executor = EOExecutor(workflow, execution_args)
executor.run(workers=None, multiprocess=True)
executor.make_report()
