def test_tile_metadata_with_geometry_check(self): def geometry_check(meta): if meta['latitude_band'] == 'N': return False return True f = open('tests/samples/tileInfo.json', 'rb') tile_info = json.loads(f.read().decode(), object_pairs_hook=OrderedDict) tile = tile_metadata(tile_info, metadata_to_dict('tests/samples/metadata.xml'), geometry_check) assert isinstance(tile, OrderedDict) assert tile[ 'thumbnail'] == 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/56/X/NF/2016/3/16/0/preview.jp2' assert tile[ 'tile_name'] == 'S2A_OPER_MSI_L1C_TL_SGS__20160316T054120_A003818_T56XNF_N02.01' assert tile['utm_zone'] == 56 assert tile['data_coverage_percentage'] == 65.58 assert tile['sensing_orbit_direction'] == 'DESCENDING' assert len(tile['download_links']['aws_s3']) == 13 # Make sure bands urls are left padded d_link = tile['download_links']['aws_s3'][0].split('.')[-2].split('/') assert d_link[-1] == 'B01'
def test_metadata_to_dict_new_format(self): xml = open('tests/samples/metadata_new.xml') product = metadata_to_dict(xml) assert isinstance(product, OrderedDict) assert 'band_list' in product assert 'tiles' in product assert product['spacecraft_name'] == 'Sentinel-2A' assert len(product['band_list']) == 13 tiles = list(iterkeys(product['tiles'])) assert len(tiles) == 1
def test_metadata_to_dict_new_format(self): xml = open('tests/samples/metadata_new.xml') product = metadata_to_dict(xml) assert isinstance(product, OrderedDict) assert 'band_list' in product assert 'tiles' in product assert product['spacecraft_name'] == 'Sentinel-2A' assert len(product['band_list']) == 13 tiles = list(iterkeys(product['tiles'])) assert len(tiles) == 1
def test_tile_metadata(self): f = open('tests/samples/tileInfo.json', 'rb') tile_info = json.loads(f.read().decode(), object_pairs_hook=OrderedDict) tile = tile_metadata(tile_info, metadata_to_dict('tests/samples/metadata.xml')) assert isinstance(tile, OrderedDict) assert tile['thumbnail'] == 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/56/X/NF/2016/3/16/0/preview.jp2' assert tile['tile_name'] == 'S2A_OPER_MSI_L1C_TL_SGS__20160316T054120_A003818_T56XNF_N02.01' assert tile['utm_zone'] == 56 assert tile['data_coverage_percentage'] == 65.58 assert tile['sensing_orbit_direction'] == 'DESCENDING' assert len(tile['download_links']['aws_s3']) == 13 # Make sure bands urls are left padded d_link = tile['download_links']['aws_s3'][0].split('.')[-2].split('/') assert d_link[-1] == 'B01'
def product_metadata(product, dst_folder, counter=None, writers=[file_writer], geometry_check=None): """ Extract metadata for a specific product """ if not counter: counter = { 'products': 0, 'saved_tiles': 0, 'skipped_tiles': 0, 'skipped_tiles_paths': [] } s3_url = 'http://sentinel-s2-l1c.s3.amazonaws.com' product_meta_link = '{0}/{1}'.format(s3_url, product['metadata']) product_info = requests.get(product_meta_link, stream=True) product_metadata = metadata_to_dict(product_info.raw) product_metadata['product_meta_link'] = product_meta_link counter['products'] += 1 for tile in product['tiles']: tile_info = requests.get('{0}/{1}'.format(s3_url, tile)) try: metadata = tile_metadata(tile_info.json(), copy(product_metadata), geometry_check) for w in writers: w(dst_folder, metadata) logger.info('Saving to disk: %s' % metadata['tile_name']) counter['saved_tiles'] += 1 except JSONDecodeError: logger.warning('Tile: %s was not found and skipped' % tile) counter['skipped_tiles'] += 1 counter['skipped_tiles_paths'].append(tile) return counter
def test_tile_metadata(self): f = open('tests/samples/tileInfo.json', 'rb') tile_info = json.loads(f.read().decode(), object_pairs_hook=OrderedDict) tile = tile_metadata(tile_info, metadata_to_dict('tests/samples/metadata.xml')) assert isinstance(tile, OrderedDict) assert tile[ 'thumbnail'] == 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/56/X/NF/2016/3/16/0/preview.jp2' assert tile[ 'tile_name'] == 'S2A_OPER_MSI_L1C_TL_SGS__20160316T054120_A003818_T56XNF_N02.01' assert tile['utm_zone'] == 56 assert tile['data_coverage_percentage'] == 65.58 assert tile['sensing_orbit_direction'] == 'DESCENDING' assert len(tile['download_links']['aws_s3']) == 13 assert tile['tile_origin']['crs']['properties'][ 'name'] == 'urn:ogc:def:crs:EPSG:8.9:4326' # Make sure bands urls are left padded d_link = tile['download_links']['aws_s3'][0].split('.')[-2].split('/') assert d_link[-1] == 'B01'