def test_iterator(self):
for test_case in self.test_cases:
with self.subTest(msg='Test case {}'.format(test_case.name)):
params = test_case.request
gpd_iter = GeopediaFeatureIterator(**params)
for idx, feature in enumerate(gpd_iter):
self.assertTrue(
isinstance(feature, dict),
'Expected at dictionary, got {}'.format(type(feature)))
if idx >= test_case.min_features - 1:
break
self.assertEqual(
gpd_iter.index, test_case.min_features,
'Expected at least {} features, got {}'.format(
test_case.min_features, gpd_iter.index))
if test_case.min_size:
self.assertTrue(
test_case.min_size <= len(gpd_iter),
'There should be at least {} features available, '
'got {}'.format(test_case.min_size,
gpd_iter.get_size()))
def setUpClass(cls):
super().setUpClass()
bbox = BBox(bbox=[(2947363, 4629723), (3007595, 4669471)],
crs=CRS.POP_WEB)
bbox = bbox.transform(CRS.WGS84)
query_filter1 = 'f12458==32632'
query_filter2 = 'f12458==32635'
cls.test_cases = [
TestCaseContainer('All features',
GeopediaFeatureIterator(
1749, gpd_session=GeopediaSession()),
min_features=100,
min_size=1609),
TestCaseContainer('BBox filter',
GeopediaFeatureIterator('1749', bbox=bbox),
min_features=21),
TestCaseContainer('Query Filter',
GeopediaFeatureIterator(
'ttl1749', query_filter=query_filter1),
min_features=76),
TestCaseContainer('Both filters - No data',
GeopediaFeatureIterator(
1749, bbox=bbox, query_filter=query_filter1),
min_features=0),
TestCaseContainer('Both filters - Some data',
GeopediaFeatureIterator(
1749, bbox=bbox, query_filter=query_filter2),
min_features=21)
]
def get_austria_crop_geopedia_idx_to_crop_id_mapping():
"""
Returns mapping between Geopedia's crop index and crop id for Austria.
:return: pandas DataFrame with 'crop_geopedia_idx' and corresponding crop id
:rtype: pandas.DataFrame
"""
gpd_session = GeopediaSession()
to_crop_id = list(
GeopediaFeatureIterator(layer='2032', gpd_session=gpd_session))
to_crop_id = [{
'crop_geopedia_idx': code['id'],
**code['properties']
} for code in to_crop_id]
to_crop_id = pd.DataFrame(to_crop_id)
to_crop_id['crop_geopedia_idx'] = pd.to_numeric(
to_crop_id.crop_geopedia_idx)
to_crop_id.rename(index=str,
columns={"SNAR_BEZEI": "SNAR_BEZEI_NAME"},
inplace=True)
to_crop_id.rename(index=str,
columns={"crop_geopedia_idx": "SNAR_BEZEI"},
inplace=True)
return to_crop_id
def execute(self, eopatch):
# convert to 3857 CRS
bbox_3857 = eopatch.bbox.transform(CRS.POP_WEB)
# get iterator over features
gpd_iter = GeopediaFeatureIterator(layer=self.layer, bbox=bbox_3857)
features = list(gpd_iter)
if len(features):
gdf = gpd.GeoDataFrame.from_features(features)
gdf.crs = {'init': 'epsg:4326'}
# convert back to EOPatch CRS
gdf = gdf.to_crs({'init': f'epsg:{eopatch.bbox.crs.value}'})
if self.year:
# Filter by years
gdf = gdf.loc[gdf[self.year_col_name].isin([self.year])]
if self.drop_duplicates:
sel = gdf.drop('geometry', axis=1)
sel = sel.drop_duplicates()
gdf = gdf.loc[sel.index]
eopatch[self.feature_type][self.feature_name] = gdf
return eopatch
def query_columns(self, column_names, conditions, return_all=True):
""" The method makes a query to Geopedia table with filter conditions on values. It returns filtered table
content.
Example:
query_columns(['input_source_id','is_done'], ['=1', '=False'])
:param column_names: Names of table columns to apply query
:type column_names: str or list of str
:param conditions: Logical conditions to be applied to corresponding columns
:type conditions: str or list of str
:param return_all: Whether to return all elements satisfying query or only the first one. Default is all.
:type return_all: bool
:return: Items from Geopedia table with properties
:rtype: GeopediaRowData or list(GeopediaRowData)
"""
column_names = [column_names] if isinstance(column_names,
str) else column_names
conditions = [conditions] if isinstance(conditions,
str) else conditions
if len(column_names) != len(conditions):
raise ValueError(
"Name of columns and conditions must be of same length")
field_ids = [self.get_field_id(name) for name in column_names]
query = ' && '.join(
[col + expr for col, expr in zip(field_ids, conditions)])
gpd_iterator = GeopediaFeatureIterator(self.id,
query_filter=query,
gpd_session=self.gpd_session)
return self._return_query_results(gpd_iterator, query, return_all)
def test_size_before_iteration(self):
for test_case in self.test_cases:
if not test_case.min_features:
continue
with self.subTest(msg='Test case {}'.format(test_case.name)):
params = test_case.request
gpd_iter1 = GeopediaFeatureIterator(**params)
_ = gpd_iter1.get_size()
first_feature1 = next(gpd_iter1)
gpd_iter2 = GeopediaFeatureIterator(**params)
first_feature2 = next(gpd_iter2)
self.assertEqual(first_feature1, first_feature2)
def test_item_count(self):
gpd_iter = GeopediaFeatureIterator(1749, bbox=self.bbox)
data = list(gpd_iter)
minimal_data_len = 21
self.assertTrue(
len(data) >= minimal_data_len,
'Expected at least {} results, got {}'.format(
minimal_data_len, len(data)))
def get_danish_crop_geopedia_idx_to_crop_id_mapping():
"""
Returns mapping between Geopedia's crop index and crop id for Austria.
:return: pandas DataFrame with 'crop_geopedia_idx' and corresponding crop id
:rtype: pandas.DataFrame
"""
gpd_session = GeopediaSession()
to_crop_id = list(GeopediaFeatureIterator(layer='2050', gpd_session=gpd_session))
to_crop_id = [{'crop_geopedia_idx': code['id'], **code['properties']} for code in to_crop_id]
to_crop_id = pd.DataFrame(to_crop_id)
to_crop_id['crop_geopedia_idx'] = pd.to_numeric(to_crop_id.crop_geopedia_idx)
return to_crop_id
def test_without_bbox(self):
gpd_iter = GeopediaFeatureIterator(1749)
minimal_data_len = 1000
for idx, class_item in enumerate(gpd_iter):
self.assertTrue(
isinstance(class_item, dict),
'Expected at dictionary, got {}'.format(type(class_item)))
if idx >= minimal_data_len - 1:
break
self.assertEqual(
gpd_iter.index, minimal_data_len, 'Expected at least {} results, '
'got {}'.format(minimal_data_len, gpd_iter.index))
def get_crop_features(table_id):
"""
Returns DataFrame of crops for table_id from Geopedia
:return: pandas DataFrame
:rtype: pandas.DataFrame
"""
gpd_session = GeopediaSession()
crop_iterator = GeopediaFeatureIterator(layer=table_id,
gpd_session=gpd_session)
to_crop_id = [{
'crop_geopedia_idx': code['id'],
**code['properties']
} for code in crop_iterator]
df = pd.DataFrame(to_crop_id)
df['crop_geopedia_idx'] = pd.to_numeric(df.crop_geopedia_idx)
return df
def query_rows(self, row_ids):
""" The method makes a query to Geopedia table for specified rows. It returns table content for those rows.
Note: If input is a single ID it will return a single result, but if input is a list of IDs it will return a
list of results.
:param row_ids: IDs of queried rows
:type row_ids: int or list(int)
:return: Data about one or multiple Geopedia rows
:rtype: GeopediaRowData or list(GeopediaRowData)
"""
return_all = not isinstance(row_ids, (int, str))
row_ids = row_ids if return_all else [row_ids]
query = ' || '.join(
['id{} = {}'.format(self.id, row_id) for row_id in row_ids])
gpd_iterator = GeopediaFeatureIterator(self.id,
query_filter=query,
gpd_session=self.gpd_session)
return self._return_query_results(gpd_iterator, query, return_all)
def _load_vector_data(self, bbox):
""" Loads vector data from geopedia table
"""
prepared_bbox = bbox.transform_bounds(CRS.POP_WEB) if bbox else None
geopedia_iterator = GeopediaFeatureIterator(layer=self.geopedia_table,
bbox=prepared_bbox,
offset=0,
gpd_session=None,
config=self.config,
**self.geopedia_kwargs)
geopedia_features = list(geopedia_iterator)
geometry = geopedia_features[0].get('geometry')
if not geometry:
raise ValueError(
f'Geopedia table "{self.geopedia_table}" does not contain geometries!'
)
self.dataset_crs = CRS(
geometry['crs']['properties']['name']) # always WGS84
return gpd.GeoDataFrame.from_features(
geopedia_features, crs=self.dataset_crs.pyproj_crs())
def get_layer_item_list(layer, interval=None):
if interval:
list(islice(GeopediaFeatureIterator(layer), *interval))
return list(GeopediaFeatureIterator(layer))