def metadata_to_dict(metadata): """ Looks at metadata.xml file of sentinel product and extract useful keys Returns a python dict """ tree = etree.parse(metadata) root = tree.getroot() meta = OrderedDict() keys = [ 'SPACECRAFT_NAME', 'PRODUCT_STOP_TIME', 'Cloud_Coverage_Assessment', 'PROCESSING_LEVEL', 'PRODUCT_TYPE', 'PROCESSING_BASELINE', 'SENSING_ORBIT_NUMBER', 'SENSING_ORBIT_DIRECTION', 'PRODUCT_FORMAT', ] # grab important keys from the file for key in keys: try: meta[key.lower()] = root.findall('.//' + key)[0].text except IndexError: meta[key.lower()] = None meta['product_cloud_coverage_assessment'] = float( meta.pop('cloud_coverage_assessment')) meta['sensing_orbit_number'] = int(meta['sensing_orbit_number']) # get tile list meta['tiles'] = get_tiles_list(root.findall('.//Product_Organisation')[0]) # get available bands if root.findall('.//Band_List'): bands = root.findall('.//Band_List')[0] meta['band_list'] = [] for b in bands: band = b.text.replace('B', '') if len(band) == 1: band = 'B' + pad(band, 2) else: band = b.text meta['band_list'].append(band) else: bands = root.findall('.//Spectral_Information_List')[0] meta['band_list'] = [] for b in bands: band = b.attrib['physicalBand'].replace('B', '') if len(band) == 1: band = 'B' + pad(band, 2) else: band = b.attrib['physicalBand'] meta['band_list'].append(band) return meta
def metadata_to_dict(metadata): """ Looks at metadata.xml file of sentinel product and extract useful keys Returns a python dict """ tree = etree.parse(metadata) root = tree.getroot() meta = OrderedDict() keys = [ 'SPACECRAFT_NAME', 'PRODUCT_STOP_TIME', 'Cloud_Coverage_Assessment', 'PROCESSING_LEVEL', 'PRODUCT_TYPE', 'PROCESSING_BASELINE', 'SENSING_ORBIT_NUMBER', 'SENSING_ORBIT_DIRECTION', 'PRODUCT_FORMAT', ] # grab important keys from the file for key in keys: try: meta[key.lower()] = root.findall('.//' + key)[0].text except IndexError: meta[key.lower()] = None meta['product_cloud_coverage_assessment'] = float(meta.pop('cloud_coverage_assessment')) meta['sensing_orbit_number'] = int(meta['sensing_orbit_number']) # get tile list meta['tiles'] = get_tiles_list(root.findall('.//Product_Organisation')[0]) # get available bands if root.findall('.//Band_List'): bands = root.findall('.//Band_List')[0] meta['band_list'] = [] for b in bands: band = b.text.replace('B', '') if len(band) == 1: band = 'B' + pad(band, 2) else: band = b.text meta['band_list'].append(band) else: bands = root.findall('.//Spectral_Information_List')[0] meta['band_list'] = [] for b in bands: band = b.attrib['physicalBand'].replace('B', '') if len(band) == 1: band = 'B' + pad(band, 2) else: band = b.attrib['physicalBand'] meta['band_list'].append(band) return meta
def tile_metadata(tile, product, geometry_check=None): """ Generate metadata for a given tile - geometry_check is a function the determines whether to calculate the geometry by downloading B01 and override provided geometry in tilejson. The meta object is passed to this function. The function return a True or False response. """ grid = 'T{0}{1}{2}'.format(pad(tile['utmZone'], 2), tile['latitudeBand'], tile['gridSquare']) meta = OrderedDict({ 'tile_name': product['tiles'][grid] }) logger.info('%s Processing tile %s' % (threading.current_thread().name, tile['path'])) meta['date'] = tile['timestamp'].split('T')[0] meta['thumbnail'] = '{1}/{0}/preview.jp2'.format(tile['path'], s3_url) # remove unnecessary keys product.pop('tiles') tile.pop('datastrip') bands = product.pop('band_list') for k, v in iteritems(tile): meta[camelcase_underscore(k)] = v meta.update(product) # construct download links links = ['{2}/{0}/{1}.jp2'.format(meta['path'], b, s3_url) for b in bands] meta['download_links'] = { 'aws_s3': links } meta['original_tile_meta'] = '{0}/{1}/tileInfo.json'.format(s3_url, meta['path']) def internal_latlon(meta): keys = ['tile_origin', 'tile_geometry', 'tile_data_geometry'] for key in keys: if key in meta: meta[key] = to_latlon(meta[key]) return meta # change coordinates to wsg4 degrees if geometry_check: if geometry_check(meta): meta = get_tile_geometry_from_s3(meta) else: meta = internal_latlon(meta) else: meta = internal_latlon(meta) # rename path key to aws_path meta['aws_path'] = meta.pop('path') return meta
def tile_metadata(tile, product, geometry_check=None): """ Generate metadata for a given tile - geometry_check is a function the determines whether to calculate the geometry by downloading B01 and override provided geometry in tilejson. The meta object is passed to this function. The function return a True or False response. """ grid = 'T{0}{1}{2}'.format(pad(tile['utmZone'], 2), tile['latitudeBand'], tile['gridSquare']) meta = OrderedDict({'tile_name': product['tiles'][grid]}) logger.info('%s Processing tile %s' % (threading.current_thread().name, tile['path'])) meta['date'] = tile['timestamp'].split('T')[0] meta['thumbnail'] = '{1}/{0}/preview.jp2'.format(tile['path'], s3_url) # remove unnecessary keys product.pop('tiles') tile.pop('datastrip') bands = product.pop('band_list') for k, v in iteritems(tile): meta[camelcase_underscore(k)] = v meta.update(product) # construct download links links = ['{2}/{0}/{1}.jp2'.format(meta['path'], b, s3_url) for b in bands] meta['download_links'] = {'aws_s3': links} meta['original_tile_meta'] = '{0}/{1}/tileInfo.json'.format( s3_url, meta['path']) def internal_latlon(meta): keys = ['tile_origin', 'tile_geometry', 'tile_data_geometry'] for key in keys: if key in meta: meta[key] = to_latlon(meta[key]) return meta # change coordinates to wsg4 degrees if geometry_check: if geometry_check(meta): meta = get_tile_geometry_from_s3(meta) else: meta = internal_latlon(meta) else: meta = internal_latlon(meta) # rename path key to aws_path meta['aws_path'] = meta.pop('path') return meta
def amazon_s3_url_sentinel2(path, band, suffix='B', frmt='jp2'): """ Return an amazon s3 url for a sentinel2 scene band :param sat: Expects an object created by landsat_scene_interpreter function :type sat: dict :param filename: The filename that has to be downloaded from Amazon :type filename: String :returns: (String) The URL to a S3 file """ return '{0}{1}/{2}{3}.{4}'.format(S3_SENTINEL, path, suffix, pad(band, 2), frmt)
def amazon_s3_url(cls, path, band, suffix='B', frmt='jp2'): """ Return an amazon s3 url for a sentinel2 scene band :param path: :param band: :param suffix: :param frmt: :returns: (String) The URL to a S3 file """ return '{0}{1}/{2}{3}.{4}'.format( cls.S3_SENTINEL, path, suffix, pad(band, 2), frmt )
def test_pad_left(self): self.assertEqual(pad('1', 3), '001') self.assertEqual(pad('QA', 2), 'QA') self.assertEqual(pad('his', 4, char='t'), 'this')
def test_pad_right(self): self.assertEqual(pad('1', 3, direction='right'), '100') self.assertEqual(pad('QA', 2, direction='right'), 'QA') self.assertEqual(pad('his', 4, char='t', direction='right'), 'hist')