Пример #1
0
def remove_products_from_xml(self):
    """Remove the specified products from the XML file

    The file is read into memory, processed, and written back out with out
    the specified products.
    """
    # Create and load the metadata object
    espa_metadata = Metadata(xml_filename=self._xml_filename)

    # Search for and remove the items
    for band in espa_metadata.xml_object.bands.band:
        if band.attrib['product'] in products_to_remove:
            # Business logic to always keep the radsat_qa band if bt,
            # or toa, or sr output was chosen
            if (band.attrib['name'] == ''
                    and (options['include_sr'] or options['include_sr_toa']
                         or options['include_sr_thermal'])):
                continue
            else:
                self.remove_band_from_xml(band)

    # Validate the XML
    espa_metadata.validate()

    # Write it to the XML file
    espa_metadata.write(xml_filename=self._xml_filename)
class TestMetadata(unittest.TestCase):
    """Test a few things, and expand on it someday"""

    def setUp(self):

        xml_filename = 'unittests/test.xml'
        self.mm = Metadata(xml_filename=xml_filename)
        self.mm.parse()

    def tearDown(self):
        pass

    def test_find_version(self):
        self.assertEqual(self.mm.xml_object.get('version'), '2.0')

    def test_find_corners(self):
        self.assertEqual(self.mm.xml_object.global_metadata.corner[0]
                         .attrib['location'], 'UL')
        self.assertEqual(self.mm.xml_object.global_metadata.corner[1]
                         .attrib['location'], 'LR')

    def test_find_band_names(self):
        self.assertEqual(self.mm.xml_object.bands.band[0].get('name'),
                         'band1')
        self.assertEqual(self.mm.xml_object.bands.band[1].get('name'),
                         'band2')
        self.assertEqual(self.mm.xml_object.bands.band[2].get('name'),
                         'band3')
        self.assertEqual(self.mm.xml_object.bands.band[3].get('name'),
                         'band4')
        self.assertEqual(self.mm.xml_object.bands.band[4].get('name'),
                         'band5')
        self.assertEqual(self.mm.xml_object.bands.band[5].get('name'),
                         'band6')

    def test_write_success(self):
        # Also tests for successful validation
        self.mm.write(xml_filename='walnuts_pass.xml')
        self.assertTrue(os.path.exists('walnuts_pass.xml') == 1)
        os.unlink('walnuts_pass.xml')

    def test_validation_fail(self):
        myE = objectify.ElementMaker(annotate=False, namespace=None,
                                     nsmap=None)
        self.mm.xml_object.animals = myE.root()
        self.mm.xml_object.animals.tiger = myE.frog('white')
        self.mm.xml_object.animals.frog = myE.frog('green')

        with self.assertRaises(XMLError):
            self.mm.validate()
class TestMetadata(unittest.TestCase):
    """Test a few things, and expand on it someday"""
    def setUp(self):

        xml_filename = 'unittests/test.xml'
        self.mm = Metadata(xml_filename=xml_filename)
        self.mm.parse()

    def tearDown(self):
        pass

    def test_find_version(self):
        self.assertEqual(self.mm.xml_object.get('version'), '2.0')

    def test_find_corners(self):
        self.assertEqual(
            self.mm.xml_object.global_metadata.corner[0].attrib['location'],
            'UL')
        self.assertEqual(
            self.mm.xml_object.global_metadata.corner[1].attrib['location'],
            'LR')

    def test_find_band_names(self):
        self.assertEqual(self.mm.xml_object.bands.band[0].get('name'), 'band1')
        self.assertEqual(self.mm.xml_object.bands.band[1].get('name'), 'band2')
        self.assertEqual(self.mm.xml_object.bands.band[2].get('name'), 'band3')
        self.assertEqual(self.mm.xml_object.bands.band[3].get('name'), 'band4')
        self.assertEqual(self.mm.xml_object.bands.band[4].get('name'), 'band5')
        self.assertEqual(self.mm.xml_object.bands.band[5].get('name'), 'band6')

    def test_write_success(self):
        # Also tests for successful validation
        self.mm.write(xml_filename='walnuts_pass.xml')
        self.assertTrue(os.path.exists('walnuts_pass.xml') == 1)
        os.unlink('walnuts_pass.xml')

    def test_validation_fail(self):
        myE = objectify.ElementMaker(annotate=False,
                                     namespace=None,
                                     nsmap=None)
        self.mm.xml_object.animals = myE.root()
        self.mm.xml_object.animals.tiger = myE.frog('white')
        self.mm.xml_object.animals.frog = myE.frog('green')

        with self.assertRaises(XMLError):
            self.mm.validate()
Пример #4
0
def buildMetadata(metadata_filename, bit_counts, clip_extents, tile_id,
                  metadata_locs, production_timestamp, tiled_filenames,
                  segment, region, lng_count):
    """Build tile metadata starting from the source L2 metadata files."""
    logger.debug('Buildmetadata: Entered')

    SceneTagsToBeKept = ('data_provider', 'satellite', 'instrument',
                         'acquisition_date', 'scene_center_time',
                         'level1_production_date', 'wrs', 'product_id',
                         'lpgs_metadata_file')

    L1Tuple = []
    L2Tuple = []
    for i, metafilenames in enumerate(metadata_locs):
        if i < lng_count:
            l2tree = ET.parse(metafilenames['L2XML'])
            l2tree = l2tree.getroot()
            L2Tuple.append(l2tree)
            L1Tuple.append(landsat.read_metadatas(metafilenames['L1MTL']))

    # read any namespace from the 1st L2 scene
    namespace = ''
    if ('{' in l2tree.tag):
        endPos = (l2tree.tag).find('}')
        namespace = l2tree.tag[:endPos+1]

    # Start the output xml
    outRoot = ET.Element('ard_metadata')
    outTileMetadata = ET.SubElement(outRoot, 'tile_metadata')
    outTileGlobal = ET.SubElement(outTileMetadata, 'global_metadata')

    # data_provider - use L2 scene
    for child in l2tree.find(namespace + 'global_metadata'):
        if (child.tag == namespace + 'data_provider'):
            outDataProvider = ET.SubElement(outTileGlobal, 'data_provider',
                                            child.attrib)
            outDataProvider.text = child.text

    # satellite - new
    gm_satellite = ET.SubElement(outTileGlobal, 'satellite')
    satellite_strs = {
        'LT04': 'LANDSAT_4',
        'LT05': 'LANDSAT_5',
        'LE07': 'LANDSAT_7',
        'LC08': 'LANDSAT_8',
    }
    gm_satellite.text = satellite_strs[tile_id[:4]]

    # instrument - new
    gm_instrument = ET.SubElement(outTileGlobal, 'instrument')
    instrument_strs = {
        'LT04': 'TM',
        'LT05': 'TM',
        'LE07': 'ETM',
        'LC08': 'OLI/TIRS_Combined',
    }
    gm_instrument.text = instrument_strs[tile_id[:4]]

    # Level 1 Collection - new
    gm_l1coll = ET.SubElement(outTileGlobal, 'level1_collection')
    gm_l1coll.text = tile_id[34:36]

    # ARD Version - new
    gm_ardVersion = ET.SubElement(outTileGlobal, 'ard_version')
    gm_ardVersion.text = tile_id[38:40]

    # Region - new
    gm_region = ET.SubElement(outTileGlobal, 'region')
    gm_region.text = tile_id[5:7]

    # acquisition date - use L2 scene
    for child in l2tree.find(namespace + 'global_metadata'):
        if (child.tag == namespace + 'acquisition_date'):
            outAcqDate = ET.SubElement(outTileGlobal, 'acquisition_date',
                                       child.attrib)
            outAcqDate.text = child.text

    # tile_id - new
    gm_productid = ET.SubElement(outTileGlobal, 'product_id')
    gm_productid.text = tile_id

    # tile_production_date - new
    gm_tilepd = ET.SubElement(outTileGlobal, 'production_date')
    gm_tilepd.text = production_timestamp

    # bounding coordinates - modify L2 scene
    logger.debug('Buildmetadata: Ready for bounding_coordinates')
    horiz = tile_id[8:11]
    vertical = tile_id[11:14]
    newBoundingCoordsStr = getGeographicBoundingCoordinates(horiz,
                                                            vertical, region)
    tempBoundingElement = ET.fromstring(newBoundingCoordsStr)

    gm_bounding = ET.SubElement(outTileGlobal, tempBoundingElement.tag,
                                tempBoundingElement.attrib)

    for child in tempBoundingElement:
        gm_bounding_child = ET.SubElement(gm_bounding, child.tag, child.attrib)
        gm_bounding_child.text = child.text

    # projection information - modify L2 scene
    logger.debug('Buildmetadata: Ready for projection information')
    newProjInfo = global_createProjInfo(clip_extents, region)
    tempProjElement = ET.fromstring(newProjInfo)

    gm_ProjInfo = ET.SubElement(outTileGlobal, tempProjElement.tag,
                                tempProjElement.attrib)

    for child in tempProjElement:
        gm_proj_child = ET.SubElement(gm_ProjInfo, child.tag, child.attrib)
        gm_proj_child.text = child.text
        if (child.tag == "albers_proj_params"):
            for projChild in child:
                gm_proj_grandchild = ET.SubElement(gm_proj_child,
                                                   projChild.tag,
                                                   projChild.attrib)
                gm_proj_grandchild.text = projChild.text

    # orientation_angle - use L2 scene
    for child in l2tree.find(namespace + 'global_metadata'):
        if (child.tag == namespace + 'orientation_angle'):
            outOrientation = ET.SubElement(outTileGlobal, 'orientation_angle',
                                           child.attrib)
            outOrientation.text = child.text

    # tile_grid - new
    gm_tileid = ET.SubElement(outTileGlobal, 'tile_grid')
    gm_tileid.set('v', tile_id[11:14])
    gm_tileid.set('h', tile_id[8:11])

    # scene_count - new
    gm_sc = ET.SubElement(outTileGlobal, 'scene_count')
    gm_sc.text = str(lng_count)

    qa_percents = createPixelTypeTuple(bit_counts)

    # cloud_cover - new
    gm_cc = ET.SubElement(outTileGlobal, 'cloud_cover')
    gm_cc.text = qa_percents['cloud_cover']

    # cloud_shadow - new
    gm_cs = ET.SubElement(outTileGlobal, 'cloud_shadow')
    gm_cs.text = qa_percents['cloud_shadow']

    # snow_ice - new
    gm_si = ET.SubElement(outTileGlobal, 'snow_ice')
    gm_si.text = qa_percents['snow_ice']

    # fill - new
    gm_fill = ET.SubElement(outTileGlobal, 'fill')
    gm_fill.text = qa_percents['fill']

    #
    # Build all of the bands for the tile
    #
    # This group of tags originate from
    # a Level 2 metadata file.  This section will
    # describe the tile bands - most of the
    # information is already correct, but
    # anything tile related will have to be changed.
    #
    outTileBands = ET.SubElement(outTileMetadata, 'bands')

    # add lineage band
    lineageStr = createLineageSection(tile_id, production_timestamp)
    tempLineageElement = ET.fromstring(lineageStr)

    bands_lineage = ET.SubElement(outTileBands, tempLineageElement.tag,
                                  tempLineageElement.attrib)

    for child in tempLineageElement:
        bands_lineage_child = ET.SubElement(bands_lineage, child.tag,
                                            child.attrib)
        bands_lineage_child.text = child.text

    # Loop through all of the bands in the L2 file.
    # Each band will need to be modified to reflect the
    # characteristics of the tile.
    bandsElement = l2tree.find(namespace + 'bands')

    included_newnames = list()
    for curBand in bandsElement:
        oldBandStr = ET.tostring(curBand)
        newNameOnly, newBandStr = fixTileBand2(tile_id, tiled_filenames,
                                               production_timestamp,
                                               oldBandStr)

        if newBandStr is None:
            logger.debug('Skipping band not in current XML group')
            continue
        included_newnames.append(newNameOnly)

        tempBandElement = ET.fromstring(newBandStr)

        bands_band = ET.SubElement(outTileBands, tempBandElement.tag,
                                   tempBandElement.attrib)

        for child in tempBandElement:
            bands_band_child = ET.SubElement(bands_band, child.tag,
                                             child.attrib)
            bands_band_child.text = child.text
            if (child.tag in ["bitmap_description", "class_values"]):
                for bandChild in child:
                    bands_band_grandchild = ET.SubElement(bands_band_child,
                                                          bandChild.tag,
                                                          bandChild.attrib)
                    bands_band_grandchild.text = bandChild.text

    logger.debug('Buildmetadata: finished tile bands')

    #
    # "Global" and "bands" have now been created for the new tiles.
    #
    #  Next modify the scene metadata for each contributing scene.
    #  We'll have to read some values from the Level 1 (MTL.txt) file.
    #
    for i in range(lng_count):
        sceneRoot = (L2Tuple[i])

        #  Read some values from the Level 1 (MTL.txt) file.
        request_id = getL1Value(L1Tuple[i], "REQUEST_ID")
        scene_id = getL1Value(L1Tuple[i], "LANDSAT_SCENE_ID")
        elev_src = getL1Value(L1Tuple[i], "ELEVATION_SOURCE")

        if any(tile_id.startswith(x) for x in ('LT04', 'LT05', 'LE07')):
            sensor_mode = getL1Value(L1Tuple[i], "SENSOR_MODE")
            ephemeris_type = getL1Value(L1Tuple[i], "EPHEMERIS_TYPE")

        cpf_name = getL1Value(L1Tuple[i], "CPF_NAME")
        geometric_rmse_model = getL1Value(L1Tuple[i],
                                          "GEOMETRIC_RMSE_MODEL")
        geometric_rmse_model_x = getL1Value(L1Tuple[i],
                                            "GEOMETRIC_RMSE_MODEL_X")
        geometric_rmse_model_y = getL1Value(L1Tuple[i],
                                            "GEOMETRIC_RMSE_MODEL_Y")

        # opening tags for each scene
        outSceneMetadata = ET.SubElement(outRoot, 'scene_metadata')
        outSceneIndex = ET.SubElement(outSceneMetadata, 'index')
        outSceneIndex.text = str(i+1)
        outSceneGlobal = ET.SubElement(outSceneMetadata, 'global_metadata')

        # Regurgitate the L2 scene information,
        # interspursing some additional L1 info
        # along the way
        for child in sceneRoot.find(namespace + 'global_metadata'):
            newTag = (child.tag).replace('ns0:', '')
            newTag = (child.tag).replace(namespace, '')

            if (newTag in SceneTagsToBeKept):
                outGeneric = ET.SubElement(outSceneGlobal, newTag,
                                           child.attrib)
                outGeneric.text = child.text

            if (newTag == 'wrs'):
                outGeneric = ET.SubElement(outSceneGlobal, 'request_id')
                outGeneric.text = request_id
                outGeneric = ET.SubElement(outSceneGlobal, 'scene_id')
                outGeneric.text = scene_id

            if (newTag == 'product_id'):
                outGeneric = ET.SubElement(outSceneGlobal, 'elevation_source')
                outGeneric.text = elev_src

                is_landsat_4_7 = any(tile_id.startswith(x)
                                     for x in ('LT04', 'LT05', 'LE07'))
                if is_landsat_4_7:
                    outGeneric = ET.SubElement(outSceneGlobal, 'sensor_mode')
                    outGeneric.text = sensor_mode
                    outGeneric = ET.SubElement(outSceneGlobal,
                                               'ephemeris_type')
                    outGeneric.text = ephemeris_type

                outGeneric = ET.SubElement(outSceneGlobal, 'cpf_name')
                outGeneric.text = cpf_name

            if (newTag == 'lpgs_metadata_file'):

                if geometric_rmse_model.find("not found") == -1:
                    outGeneric = ET.SubElement(outSceneGlobal,
                                               'geometric_rmse_model')
                    outGeneric.text = geometric_rmse_model

                if geometric_rmse_model_x.find("not found") == -1:
                    outGeneric = ET.SubElement(outSceneGlobal,
                                               'geometric_rmse_model_x')
                    outGeneric.text = geometric_rmse_model_x

                if geometric_rmse_model_y.find("not found") == -1:
                    outGeneric = ET.SubElement(outSceneGlobal,
                                               'geometric_rmse_model_y')
                    outGeneric.text = geometric_rmse_model_y

        outSceneBands = ET.SubElement(outSceneMetadata, 'bands')

        # The scene bands
        for bandTag in sceneRoot.find(namespace + 'bands'):
            if bandTag.attrib.get('name') not in included_newnames:
                logger.debug('Skipping band not in current XML group')
                continue
            newTag = (bandTag.tag).replace(namespace, '')
            bandElement = ET.SubElement(outSceneBands, newTag, bandTag.attrib)
            bandElement.text = bandTag.text

            for child in bandTag:
                newTag2 = (child.tag).replace(namespace, '')
                childElement = ET.SubElement(bandElement, newTag2,
                                             child.attrib)
                childElement.text = child.text
                if (newTag2 in ["bitmap_description", "class_values"]):
                    for bitmapChild in child:
                        bitmapTag = (bitmapChild.tag).replace(namespace, '')
                        bands_band_bitmap = ET.SubElement(childElement,
                                                          bitmapTag,
                                                          bitmapChild.attrib)
                        bands_band_bitmap.text = bitmapChild.text

    logger.debug('Buildmetadata: Ready to write')

    namespace1Prefix = "xmlns"
    namespace2Prefix = "xmlns:xsi"
    namespace3Prefix = "xsi:schemaLocation"

    # TODO: these should come from the XSD
    namespace1URI = "https://landsat.usgs.gov/ard/v1"
    namespace2URI = "http://www.w3.org/2001/XMLSchema-instance"
    namespace3URI = ("https://landsat.usgs.gov/ard/v1 "
                     "https://landsat.usgs.gov/ard/ard_metadata_v1_1.xsd")

    outRoot.attrib[namespace3Prefix] = namespace3URI
    outRoot.attrib[namespace2Prefix] = namespace2URI
    outRoot.attrib[namespace1Prefix] = namespace1URI
    outRoot.attrib["version"] = "1.1"

    # Add string indentation - Unfortunately,
    # this function produces extra carriage returns
    # after some elements...

    prettyString = (
        minidom.parseString(ET.tostring(outRoot)
                            ).toprettyxml(encoding="utf-8", indent="    ")
    )

    # Write to temp file
    uglyFullName = metadata_filename.replace(".xml", "_ugly.xml")
    with open(uglyFullName, "w") as f:
        f.write(prettyString.encode('utf-8'))

    # Looks like the minidom pretty print added some
    # blank lines followed by CRLF.  The blank lines are
    # of more than one length in our case.  Remove any
    # blank lines.
    inMetafile = open(uglyFullName, "r")
    outMetafile = open(metadata_filename, "w")

    for curLine in inMetafile:

        allSpaces = True
        for curChar in curLine:
            if ((curChar != '\x20')
                    and (curChar != '\x0D')
                    and (curChar != '\x0A')):
                allSpaces = False
                continue

        if allSpaces is False:
            outMetafile.write(curLine)
        # else:
        #    print 'Found blank line'

    inMetafile.close()
    outMetafile.close()

    # Validate metafile that was just created
    tile_metadata = Metadata(xml_filename=metadata_filename)
    tile_metadata.validate()