def test_get_collections(self):
catalogue = Catalogue()
collections = list(catalogue.get_collections())
collection_ids = [collection.id for collection in collections]
self.assertIn("urn:eop:VITO:CGS_S1_GRD_L1", collection_ids)
self.assertIn("urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1", collection_ids)
self.assertIn("urn:eop:VITO:CGS_S1_SLC_L1", collection_ids)
def test_download_unauthenticated(self):
catalogue = Catalogue()
products = list(
catalogue.get_products("urn:eop:VITO:COP_DEM_GLO_30M_COG",
limit=1))
self.assertFalse(
catalogue._is_authorized_to_download_http(products[0].data[0]))
def test_get_product_dir(self):
product = terracatalogueclient.Product(
id=
'urn:eop:VITO:TERRASCOPE_S2_NDVI_V2:S2A_20150704T101006_32TML_NDVI_10M_V200',
title='S2A_20150704T101006_32TML_NDVI_10M_V200',
geojson=None,
geometry=None,
bbox=None,
beginningDateTime=None,
endingDateTime=None,
properties=None,
data=None,
related=None,
previews=None,
alternates=None)
self.assertEqual('/tmp/S2A_20150704T101006_32TML_NDVI_10M_V200',
Catalogue._get_product_dir('/tmp/', product))
product = terracatalogueclient.Product(
id='VI_20161001T092022_S2A_T34SDG-010m_V100_FAPAR',
title=
'Vegetation Indices 2017-ongoing (raster 010m) - version 1 : FAPAR T34SDG 20161001T092022',
geojson=None,
geometry=None,
bbox=None,
beginningDateTime=None,
endingDateTime=None,
properties=None,
data=None,
related=None,
previews=None,
alternates=None)
self.assertEqual('/tmp/VI_20161001T092022_S2A_T34SDG-010m_V100_FAPAR',
Catalogue._get_product_dir('/tmp/', product))
def test_get_products_unsupported_parameter(self):
catalogue = Catalogue()
products = catalogue.get_products("urn:eop:VITO:CGS_S1_SLC_L1",
test="test")
self.assertRaises(
SearchException, list, products
) # getting items from the generator and putting them in a list raises the error
def test_get_collections(self):
catalogue = Catalogue(self.config_hrvpp)
collections = list(catalogue.get_collections())
collection_ids = [collection.id for collection in collections]
self.assertIn(
"copernicus_r_utm-wgs84_10_m_hrvpp-vi_p_2017-ongoing_v01_r01",
collection_ids)
def test_convert_parameters_time(self):
params = {
"start": dt.date(2020, 1, 1),
"end": dt.datetime(2021, 2, 20, 10, 11, 12)
}
Catalogue._convert_parameters(params)
self.assertEqual("2020-01-01", params['start'])
self.assertEqual("2021-02-20T10:11:12Z", params['end'])
def test_get_products_bbox(self):
catalogue = Catalogue()
bbox = {'west': 4.1, 'south': 50, "east": 5.5, "north": 51}
products = catalogue.get_products("urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1",
bbox=bbox,
start="2020-01-01",
end="2020-05-01")
self.assertTrue(len(list(products))) # check if list is not empty
def test_get_products_geometry(self):
catalogue = Catalogue()
geom = box(4, 50, 6, 51)
products = catalogue.get_products("urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1",
geometry=geom,
start="2020-01-01",
end="2020-05-01")
self.assertTrue(len(list(products))) # check if list is not empty
def test_get_collections_by_platform(self):
catalogue = Catalogue()
collections = list(catalogue.get_collections(platform="SENTINEL-1"))
self.assertTrue(len(collections) > 0)
for c in collections:
self.assertTrue(
any(a['platform']['platformShortName'] == "SENTINEL-1"
for a in c.properties['acquisitionInformation']))
def test_download_method_s3(self):
with patch.object(Catalogue,
"_download_file_s3") as mock_download_file_s3:
catalogue = Catalogue()
href = "s3://b7bef1640a4a4ecca1e8ca04a70cd472:hr-vpp-products-vi-101-201610/01/VI_20161001T092022_S2A_T34SDG-010m_V101_QFLAG2.tif"
length = 495702
product_file = ProductFile(href, length)
catalogue.download_file(product_file, "/tmp")
mock_download_file_s3.assert_called_once()
def test_convert_parameters_bbox(self):
expected = "4.1,50,5.5,51"
params = {'bbox': [4.1, 50, 5.5, 51]}
Catalogue._convert_parameters(params)
self.assertEqual(expected, params['bbox'])
params = {'bbox': {'west': 4.1, 'south': 50, "east": 5.5, "north": 51}}
Catalogue._convert_parameters(params)
self.assertEqual(expected, params['bbox'])
def test_get_products_title(self):
catalogue = Catalogue()
title = "S2A_20200101T142731_19HBV_FAPAR_20M_V200"
products = list(
catalogue.get_products("urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
title=title))
# expect only one product with the same title
self.assertEqual(1, len(products))
self.assertEqual(title, products[0].title)
def test_download_method_http(self):
with patch.object(Catalogue,
"_download_file_http") as mock_download_file_http:
catalogue = Catalogue()
href = "https://phenology.vgt.vito.be/download/VI_V101/2016/10/01/VI_20161001T092022_S2A_T34SDG-010m_V101_QFLAG2.tif"
length = 495702
product_file = ProductFile(href, length)
catalogue.download_file(product_file, "/tmp")
mock_download_file_http.assert_called_once()
def test_get_products(self):
catalogue = Catalogue(self.config_hrvpp)
tileId = "31UGS"
products = list(
catalogue.get_products(
"copernicus_r_utm-wgs84_10_m_hrvpp-vi_p_2017-ongoing_v01_r01",
tileId=tileId,
start="2021-01-01",
end="2021-01-31"))
self.assertTrue(products) # check if list is not empty
def test_download_xml(self):
catalogue = Catalogue()
title = "S2A_20200101T142731_19HBV_FAPAR_20M_V200"
products = list(
catalogue.get_products("urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
title=title))
product = products[0]
with tempfile.TemporaryDirectory() as dir:
catalogue.download_file(product.alternates[0], dir)
self.assertTrue(os.path.isfile(os.path.join(dir, f"{title}.xml")))
def test_get_collections_by_bbox(self):
catalogue = Catalogue()
collections = list(
catalogue.get_collections(bbox={
'west': 4.1,
'south': 50,
"east": 5.5,
"north": 51
}))
self.assertTrue(len(collections) > 0)
def test_download_non_interactive(self):
# note that this will only work when production download service uses WekEO IdP for authentication
catalogue = Catalogue(self.config_hrvpp).authenticate_non_interactive(
os.getenv("WEKEO_USERNAME"), os.getenv("WEKEO_PASSWORD"))
tileId = "31UGS"
products = catalogue.get_products(
"copernicus_r_utm-wgs84_10_m_hrvpp-vi_p_2017-ongoing_v01_r01",
tileId=tileId,
start="2021-01-01",
end="2021-01-31",
accessedFrom="HTTP")
with tempfile.TemporaryDirectory() as tempdir:
catalogue.download_product(next(products), tempdir)
def test_get_products_date(self):
catalogue = Catalogue()
start = dt.date(2020, 1, 1)
end = dt.date(2020, 1, 31)
products = catalogue.get_products(
"urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
start=start,
end=end,
tileId="31UFS")
for p in products:
self.assertTrue(
dt.datetime(2019, 12, 31) <= p.beginningDateTime <=
dt.datetime(2020, 2, 1))
def test_get_products_productGroupId(self):
catalogue = Catalogue()
productGroupId = "S-America_NDVI"
products = list(
catalogue.get_products("urn:ogc:def:EOP:VITO:VGT_S10",
productGroupId=productGroupId,
start="2001-01-01",
end="2001-12-31"))
self.assertTrue(products)
for p in products:
self.assertEqual(
productGroupId,
p.properties["productInformation"]["productGroupId"])
def test_get_products_multi_page(self):
catalogue = Catalogue()
count = catalogue.get_product_count(
"urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
start="2020-01-01",
end="2020-12-31",
tileId="31UFS")
products = catalogue.get_products(
"urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
start="2020-01-01",
end="2020-12-31",
tileId="31UFS")
self.assertEqual(count, len(list(products)))
def test_download_force(self):
with patch.object(Catalogue,
"download_product") as mock_download_product:
catalogue = Catalogue()
products = list(
catalogue.get_products(
collection="urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
start="2021-02-01",
end="2021-02-28",
tileId="31UGS",
resolution=20))
catalogue.download_products(products, '/tmp', force=True)
mock_download_product.assert_called()
def test_get_products_modificationDate(self):
catalogue = Catalogue()
today = dt.date.today()
modificationDate = (today - dt.timedelta(weeks=4), None)
products = list(
catalogue.get_products("urn:eop:VITO:TERRASCOPE_S2_NDVI_V2",
tileId="31UFS",
modificationDate=modificationDate))
self.assertTrue(products)
for p in products:
updated = _parse_date(p.properties['updated'])
self.assertTrue(
dt.datetime.combine(modificationDate[0],
dt.datetime.min.time()) <= updated)
def test_get_products_publicationDate(self):
catalogue = Catalogue()
publicationDate = (dt.date(2021, 2,
20), dt.datetime(2021, 2, 22, 23, 59, 59))
products = list(
catalogue.get_products(
"urn:eop:VITO:TERRASCOPE_S1_SLC_COHERENCE_V1",
publicationDate=publicationDate))
self.assertTrue(products)
for p in products:
published = _parse_date(p.properties['published'])
self.assertTrue(
dt.datetime.combine(publicationDate[0], dt.datetime.min.time())
<= published <= publicationDate[1])
def test_get_products_cloudCover(self):
catalogue = Catalogue()
cloudCoverMax = 60
products = list(
catalogue.get_products("urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
cloudCover=cloudCoverMax,
start="2021-02-01",
end="2021-02-28",
tileId="31UFS"))
self.assertTrue(products) # check if list is not empty
for p in products:
self.assertGreaterEqual(
cloudCoverMax,
p.properties['productInformation']['cloudCover'])
def test_download_file_type_filter(self):
product = Product(
"urn:eop:VITO:TERRASCOPE_S2_NDVI_V2:S2A_20150704T101006_32TML_NDVI_10M_V200",
"S2A_20150704T101006_32TML_NDVI_10M_V200",
None,
None, [8.2009294, 40.5584724, 9.1170316, 41.5517851],
_parse_date("2015-07-04T10:10:06.027Z"),
_parse_date("2015-07-04T10:10:06.027Z"),
None,
data=[
ProductFile(
"https://services.terrascope.be/download/Sentinel2/NDVI_V2/2015/07/04/S2A_20150704T101006_32TML_NDVI_V200/S2A_20150704T101006_32TML_NDVI_10M_V200.tif",
39500389, "NDVI_10M", "image/tiff")
],
related=[
ProductFile(
"https://services.terrascope.be/download/Sentinel2/NDVI_V2/2015/07/04/S2A_20150704T101006_32TML_NDVI_V200/S2A_20150704T101006_32TML_SCENECLASSIFICATION_20M_V200.tif",
1521972, "SCENECLASSIFICATION_20M", "image/tiff",
"QUALITY")
],
previews=[
ProductFile(
"https://services.terrascope.be/download/Sentinel2/NDVI_V2/2015/07/04/S2A_20150704T101006_32TML_NDVI_V200/S2A_20150704T101006_32TML_NDVI_QUICKLOOK_V200.tif",
182760, None, "image/tiff", "QUICKLOOK")
],
alternates=[
ProductFile(
"https://services.terrascope.be/download/Sentinel2/NDVI_V2/2015/07/04/S2A_20150704T101006_32TML_NDVI_V200/S2A_20150704T101006_32TML_NDVI_10M_V200.xml",
32523, "Inspire metadata", "application/vnd.iso.19139+xml")
])
with patch.object(Catalogue, "download_file") as mock_download_file:
catalogue = Catalogue()
tmp_dir = "/tmp"
download_dir = os.path.join(tmp_dir, product.id.split(":")[-1])
# only download data files
catalogue.download_product(product, tmp_dir, ProductFileType.DATA)
mock_download_file.assert_called_once_with(product.data[0],
download_dir)
mock_download_file.reset_mock()
# download all files
catalogue.download_products([product], tmp_dir, force=True)
self.assertEqual(4, mock_download_file.call_count)
for pf in product.data + product.related + product.previews + product.alternates:
self.assertIn(((pf, download_dir), ),
mock_download_file.call_args_list)
mock_download_file.reset_mock()
# combine file types
catalogue.download_product(
product, tmp_dir,
ProductFileType.DATA | ProductFileType.RELATED)
self.assertEqual(2, mock_download_file.call_count)
for pf in product.data + product.related:
self.assertIn(((pf, download_dir), ),
mock_download_file.call_args_list)
def test_download_abort(self):
with patch.object(Catalogue, "download_product") as mock_download_product, \
patch("builtins.input") as mock_input:
mock_input.return_value = "n"
catalogue = Catalogue()
products = list(
catalogue.get_products(
collection="urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
start="2021-02-01",
end="2021-02-28",
tileId="31UGS",
resolution=20))
catalogue.download_products(products, '/tmp')
mock_download_product.assert_not_called()
def test_get_product_count_invalid_parameter_values(self):
catalogue = Catalogue()
self.assertRaises(SearchException,
catalogue.get_product_count,
"urn:eop:VITO:CGS_S1_SLC_L1",
orbitDirection="test",
geometry="polygon")
def test_get_products_tileId(self):
catalogue = Catalogue()
tileId = "31UGS"
products = list(
catalogue.get_products("urn:eop:VITO:TERRASCOPE_S2_FAPAR_V2",
tileId=tileId,
start="2021-01-01",
end="2021-01-31"))
self.assertTrue(products) # check if list is not empty
for p in products:
acquisitionParameters = next(
iter([
i['acquisitionParameters']
for i in p.properties['acquisitionInformation']
if 'acquisitionParameters' in i
]))
self.assertEqual(tileId, acquisitionParameters['tileId'])
def test_get_products_relativeOrbitNumber(self):
catalogue = Catalogue()
relativeOrbitNumber = 37
products = list(
catalogue.get_products("urn:eop:VITO:CGS_S1_SLC_L1",
relativeOrbitNumber=relativeOrbitNumber,
start="2020-01-01",
end="2020-05-01"))
self.assertTrue(products) # check if list is not empty
for p in products:
acquisitionParameters = next(
iter([
i['acquisitionParameters']
for i in p.properties['acquisitionInformation']
if 'acquisitionParameters' in i
]))
self.assertEqual(relativeOrbitNumber,
acquisitionParameters['relativeOrbitNumber'])
def test_get_products_orbitDirection(self):
catalogue = Catalogue()
orbitDirection = "DESCENDING"
products = list(
catalogue.get_products("urn:eop:VITO:CGS_S1_SLC_L1",
orbitDirection=orbitDirection,
start="2020-01-01",
end="2020-05-01"))
self.assertTrue(products) # check if list is not empty
for p in products:
acquisitionParameters = next(
iter([
i['acquisitionParameters']
for i in p.properties['acquisitionInformation']
if 'acquisitionParameters' in i
]))
self.assertEqual(orbitDirection,
acquisitionParameters['orbitDirection'])