def metadata_to_dict(metadata):
""" Looks at metadata.xml file of sentinel product and extract useful keys
Returns a python dict """
tree = etree.parse(metadata)
root = tree.getroot()
meta = OrderedDict()
keys = [
'SPACECRAFT_NAME',
'PRODUCT_STOP_TIME',
'Cloud_Coverage_Assessment',
'PROCESSING_LEVEL',
'PRODUCT_TYPE',
'PROCESSING_BASELINE',
'SENSING_ORBIT_NUMBER',
'SENSING_ORBIT_DIRECTION',
'PRODUCT_FORMAT',
]
# grab important keys from the file
for key in keys:
try:
meta[key.lower()] = root.findall('.//' + key)[0].text
except IndexError:
meta[key.lower()] = None
meta['product_cloud_coverage_assessment'] = float(
meta.pop('cloud_coverage_assessment'))
meta['sensing_orbit_number'] = int(meta['sensing_orbit_number'])
# get tile list
meta['tiles'] = get_tiles_list(root.findall('.//Product_Organisation')[0])
# get available bands
if root.findall('.//Band_List'):
bands = root.findall('.//Band_List')[0]
meta['band_list'] = []
for b in bands:
band = b.text.replace('B', '')
if len(band) == 1:
band = 'B' + pad(band, 2)
else:
band = b.text
meta['band_list'].append(band)
else:
bands = root.findall('.//Spectral_Information_List')[0]
meta['band_list'] = []
for b in bands:
band = b.attrib['physicalBand'].replace('B', '')
if len(band) == 1:
band = 'B' + pad(band, 2)
else:
band = b.attrib['physicalBand']
meta['band_list'].append(band)
return meta
def metadata_to_dict(metadata):
""" Looks at metadata.xml file of sentinel product and extract useful keys
Returns a python dict """
tree = etree.parse(metadata)
root = tree.getroot()
meta = OrderedDict()
keys = [
'SPACECRAFT_NAME',
'PRODUCT_STOP_TIME',
'Cloud_Coverage_Assessment',
'PROCESSING_LEVEL',
'PRODUCT_TYPE',
'PROCESSING_BASELINE',
'SENSING_ORBIT_NUMBER',
'SENSING_ORBIT_DIRECTION',
'PRODUCT_FORMAT',
]
# grab important keys from the file
for key in keys:
try:
meta[key.lower()] = root.findall('.//' + key)[0].text
except IndexError:
meta[key.lower()] = None
meta['product_cloud_coverage_assessment'] = float(meta.pop('cloud_coverage_assessment'))
meta['sensing_orbit_number'] = int(meta['sensing_orbit_number'])
# get tile list
meta['tiles'] = get_tiles_list(root.findall('.//Product_Organisation')[0])
# get available bands
if root.findall('.//Band_List'):
bands = root.findall('.//Band_List')[0]
meta['band_list'] = []
for b in bands:
band = b.text.replace('B', '')
if len(band) == 1:
band = 'B' + pad(band, 2)
else:
band = b.text
meta['band_list'].append(band)
else:
bands = root.findall('.//Spectral_Information_List')[0]
meta['band_list'] = []
for b in bands:
band = b.attrib['physicalBand'].replace('B', '')
if len(band) == 1:
band = 'B' + pad(band, 2)
else:
band = b.attrib['physicalBand']
meta['band_list'].append(band)
return meta
def tile_metadata(tile, product, geometry_check=None):
""" Generate metadata for a given tile
- geometry_check is a function the determines whether to calculate the geometry by downloading
B01 and override provided geometry in tilejson. The meta object is passed to this function.
The function return a True or False response.
"""
grid = 'T{0}{1}{2}'.format(pad(tile['utmZone'], 2), tile['latitudeBand'], tile['gridSquare'])
meta = OrderedDict({
'tile_name': product['tiles'][grid]
})
logger.info('%s Processing tile %s' % (threading.current_thread().name, tile['path']))
meta['date'] = tile['timestamp'].split('T')[0]
meta['thumbnail'] = '{1}/{0}/preview.jp2'.format(tile['path'], s3_url)
# remove unnecessary keys
product.pop('tiles')
tile.pop('datastrip')
bands = product.pop('band_list')
for k, v in iteritems(tile):
meta[camelcase_underscore(k)] = v
meta.update(product)
# construct download links
links = ['{2}/{0}/{1}.jp2'.format(meta['path'], b, s3_url) for b in bands]
meta['download_links'] = {
'aws_s3': links
}
meta['original_tile_meta'] = '{0}/{1}/tileInfo.json'.format(s3_url, meta['path'])
def internal_latlon(meta):
keys = ['tile_origin', 'tile_geometry', 'tile_data_geometry']
for key in keys:
if key in meta:
meta[key] = to_latlon(meta[key])
return meta
# change coordinates to wsg4 degrees
if geometry_check:
if geometry_check(meta):
meta = get_tile_geometry_from_s3(meta)
else:
meta = internal_latlon(meta)
else:
meta = internal_latlon(meta)
# rename path key to aws_path
meta['aws_path'] = meta.pop('path')
return meta
def tile_metadata(tile, product, geometry_check=None):
""" Generate metadata for a given tile
- geometry_check is a function the determines whether to calculate the geometry by downloading
B01 and override provided geometry in tilejson. The meta object is passed to this function.
The function return a True or False response.
"""
grid = 'T{0}{1}{2}'.format(pad(tile['utmZone'], 2), tile['latitudeBand'],
tile['gridSquare'])
meta = OrderedDict({'tile_name': product['tiles'][grid]})
logger.info('%s Processing tile %s' %
(threading.current_thread().name, tile['path']))
meta['date'] = tile['timestamp'].split('T')[0]
meta['thumbnail'] = '{1}/{0}/preview.jp2'.format(tile['path'], s3_url)
# remove unnecessary keys
product.pop('tiles')
tile.pop('datastrip')
bands = product.pop('band_list')
for k, v in iteritems(tile):
meta[camelcase_underscore(k)] = v
meta.update(product)
# construct download links
links = ['{2}/{0}/{1}.jp2'.format(meta['path'], b, s3_url) for b in bands]
meta['download_links'] = {'aws_s3': links}
meta['original_tile_meta'] = '{0}/{1}/tileInfo.json'.format(
s3_url, meta['path'])
def internal_latlon(meta):
keys = ['tile_origin', 'tile_geometry', 'tile_data_geometry']
for key in keys:
if key in meta:
meta[key] = to_latlon(meta[key])
return meta
# change coordinates to wsg4 degrees
if geometry_check:
if geometry_check(meta):
meta = get_tile_geometry_from_s3(meta)
else:
meta = internal_latlon(meta)
else:
meta = internal_latlon(meta)
# rename path key to aws_path
meta['aws_path'] = meta.pop('path')
return meta
def amazon_s3_url_sentinel2(path, band, suffix='B', frmt='jp2'):
"""
Return an amazon s3 url for a sentinel2 scene band
:param sat:
Expects an object created by landsat_scene_interpreter function
:type sat:
dict
:param filename:
The filename that has to be downloaded from Amazon
:type filename:
String
:returns:
(String) The URL to a S3 file
"""
return '{0}{1}/{2}{3}.{4}'.format(S3_SENTINEL, path, suffix, pad(band, 2),
frmt)
def amazon_s3_url(cls, path, band, suffix='B', frmt='jp2'):
"""
Return an amazon s3 url for a sentinel2 scene band
:param path:
:param band:
:param suffix:
:param frmt:
:returns:
(String) The URL to a S3 file
"""
return '{0}{1}/{2}{3}.{4}'.format(
cls.S3_SENTINEL,
path,
suffix,
pad(band, 2),
frmt
)
def test_pad_left(self):
self.assertEqual(pad('1', 3), '001')
self.assertEqual(pad('QA', 2), 'QA')
self.assertEqual(pad('his', 4, char='t'), 'this')
def test_pad_right(self):
self.assertEqual(pad('1', 3, direction='right'), '100')
self.assertEqual(pad('QA', 2, direction='right'), 'QA')
self.assertEqual(pad('his', 4, char='t', direction='right'), 'hist')