예제 #1
0
    def _fill_algorithm_information(self, dataset, alg_src_info):
        alg_meta = ptype.AlgorithmMetadata(
            name=self.subset_name,
            version=str(alg_src_info['algorithm_version']))
        if self.subset_name == 'brdf':
            alg_meta.doi = alg_src_info['nbar_doi']
        else:
            alg_meta.doi = alg_src_info['nbar_terrain_corrected_doi']

        dataset.lineage.algorithm = alg_meta
예제 #2
0
def _populate_lineage(md, mtl_):
    product_md = _get(mtl_, 'PRODUCT_METADATA')
    if not md.lineage:
        md.lineage = ptype.LineageMetadata()
    if not md.lineage.algorithm:
        md.lineage.algorithm = ptype.AlgorithmMetadata()
    # Example "LPGS_2.3.0"
    soft_v = _get(mtl_, 'METADATA_FILE_INFO', 'processing_software_version')
    md.lineage.algorithm.name, md.lineage.algorithm.version = soft_v.split('_')
    if not md.lineage.algorithm.parameters:
        md.lineage.algorithm.parameters = {}
    if not md.lineage.ancillary:
        md.lineage.ancillary = {}
    md.lineage.ancillary_quality = _get(product_md, 'ephemeris_type')
예제 #3
0
        satellite_ref_point_start=ptype.Point(x=114, y=73),
        cloud_cover_percentage=0.0,
        sun_azimuth=102.37071009,
        sun_elevation=58.08261077,
        # sun_earth_distance=0.998137,
        # ground_control_points_version=2,
        # ground_control_points_model=47,
        # geometric_rmse_model=4.582,
        # geometric_rmse_model_x=3.370,
        # geometric_rmse_model_y=3.104,
        bands={}
    ),
    lineage=ptype.LineageMetadata(
        algorithm=ptype.AlgorithmMetadata(
            name='LPGS',
            version='12.5.0',
            parameters={}
        ),
        ancillary_quality='DEFINITIVE',
        ancillary={
            'cpf': ptype.AncillaryMetadata(
                name='L7CPF20050101_20050331.09'
            ),
            # We have the properties (quality) of the ancillary but not the file.
            'ephemeris': ptype.AncillaryMetadata(
                properties={'type': 'DEFINITIVE'}
            )
        }
    )
)
예제 #4
0
    def fill_metadata(self, dataset, path, additional_files=()):
        """
        :type additional_files: tuple[Path]
        :type dataset: ptype.DatasetMetadata
        :type path: Path
        :rtype: ptype.DatasetMetadata
        """
        dataset.ga_level = 'P55'

        # Copy relevant fields from source nbar.
        if 'nbar' in dataset.lineage.source_datasets:
            source_ortho = dataset.lineage.source_datasets['nbar']
            borrow_single_sourced_fields(dataset, source_ortho)

            # TODO, it'd be better to grab this from the images, but they're generated after
            # this code is run. Copying from Source will do for now
            dataset.grid_spatial = deepcopy(
                dataset.lineage.source_datasets['nbar'].grid_spatial)

            contiguous_data_bit = 0b100000000

            dataset.grid_spatial.projection.valid_data = self.calculate_valid_data_region(
                path, contiguous_data_bit)

        dataset.format_ = ptype.FormatMetadata('GeoTIFF')

        with open(str(path.joinpath(self.METADATA_FILE))) as f:
            pq_metadata = yaml.load(f, Loader=Loader)

        if not dataset.lineage:
            dataset.lineage = ptype.LineageMetadata()

        dataset.lineage.algorithm = ptype.AlgorithmMetadata(
            name='pqa',
            version=str(
                pq_metadata['algorithm_information']['software_version']),
            doi=pq_metadata['algorithm_information']['pq_doi'])

        # Add ancillary files
        ancils = pq_metadata['ancillary']
        ancil_files = {}
        for name, values in ancils.items():
            ancil_files[name] = ptype.AncillaryMetadata(
                type_=name,
                name=values['data_source'],
                uri=values['data_file'],
                file_owner=values['user'],
                # PyYAML parses these as datetimes already.
                access_dt=values['accessed'],
                modification_dt=values['modified'])

        if ancil_files:
            dataset.lineage.ancillary = ancil_files

        product_flags = {}
        # Record which tests where run in 'product_flags'
        for test_name, val in pq_metadata['tests_run'].items():
            product_flags['tested_%s' % test_name] = val

        dataset.product_flags = product_flags

        return dataset
예제 #5
0
 extent=ptype.ExtentMetadata(
     coord=ptype.CoordPolygon(
         ul=ptype.Coord(lat=-24.98805, lon=133.97954),
         ur=ptype.Coord(lat=-24.9864, lon=136.23866),
         ll=ptype.Coord(lat=-26.99236, lon=133.96208),
         lr=ptype.Coord(lat=-26.99055, lon=136.25985)
     ),
     center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785)
 ),
 image=ptype.ImageMetadata(
     satellite_ref_point_start=ptype.Point(x=101, y=78),
     bands={}
 ),
 lineage=ptype.LineageMetadata(
     source_datasets={'level1': test_ls8.EXPECTED_OUT},
     algorithm=ptype.AlgorithmMetadata(name='terrain', version='1.0'),
 ),
 grid_spatial=ptype.GridSpatialMetadata(
     projection=ptype.ProjectionMetadata(
         geo_ref_points=ptype.PointPolygon(
             ul=ptype.Point(
                 x=397012.5,
                 y=7235987.5
             ),
             ur=ptype.Point(
                 x=625012.5,
                 y=7235987.5
             ),
             ll=ptype.Point(
                 x=397012.5,
                 y=7013987.5
예제 #6
0
def _build_ls8_ortho():
    _reset_runtime_id()
    return ptype.DatasetMetadata(
        id_=uuid.UUID('17b92c16-51d3-11e4-909d-005056bb6972'),
        ga_label='LS8_OLITIRS_OTH_P51_GALPGS01-002_101_078_20141012',
        product_type='GAORTHO01',
        creation_dt=dateutil.parser.parse('2014-10-12 05:46:20'),
        size_bytes=2386550 * 1024,
        platform=ptype.PlatformMetadata(code='LANDSAT-8'),
        instrument=ptype.InstrumentMetadata(name='OLI_TIRS',
                                            type_="Multi-Spectral",
                                            operation_mode='PUSH-BROOM'),
        format_=ptype.FormatMetadata(name='GeoTiff', version=1),
        extent=ptype.ExtentMetadata(
            reference_system='WGS84',
            coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969),
                                     ur=ptype.Coord(lat=-24.96826,
                                                    lon=136.24838),
                                     lr=ptype.Coord(lat=-26.96338,
                                                    lon=136.26962),
                                     ll=ptype.Coord(lat=-26.96528,
                                                    lon=133.96233)),
            from_dt=dateutil.parser.parse("2014-10-12T00:55:54"),
            center_dt=dateutil.parser.parse("2014-10-12T00:56:06"),
            to_dt=dateutil.parser.parse("2014-10-12T00:56:18"),
        ),
        grid_spatial=ptype.GridSpatialMetadata(
            dimensions=[
                ptype.DimensionMetadata(name='sample',
                                        resolution=25.0,
                                        size=9161),
                ptype.DimensionMetadata(name='line',
                                        resolution=25.0,
                                        size=9161)
            ],
            projection=ptype.ProjectionMetadata(
                centre_point=ptype.Point(511512.500000, 7127487.500000),
                geo_ref_points=ptype.PointPolygon(
                    ul=ptype.Point(397012.5, 7237987.5),
                    ur=ptype.Point(626012.5, 7237987.5),
                    ll=ptype.Point(397012.5, 7016987.5),
                    lr=ptype.Point(626012.5, 7016987.5)),
                datum='GDA94',
                ellipsoid='GRS80',
                point_in_pixel='UL',
                map_projection='UTM',
                resampling_option='CUBIC_CONVOLUTION',
                zone=-53)),
        browse={
            'medium':
            ptype.BrowseMetadata(path=Path(
                'product/LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012.jpg'
            ),
                                 file_type='image/jpg',
                                 cell_size=219.75,
                                 red_band=7,
                                 green_band=5,
                                 blue_band=1),
            'full':
            ptype.BrowseMetadata(path=Path(
                'LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012_FR.jpg'),
                                 file_type='image/jpg',
                                 cell_size=25.0,
                                 red_band=7,
                                 green_band=5,
                                 blue_band=1)
        },
        image=ptype.ImageMetadata(
            satellite_ref_point_start=ptype.Point(101, 78),
            cloud_cover_percentage=0,
            cloud_cover_details=None,
            sun_elevation=58.00268508,
            sun_azimuth=59.41814014,
            ground_control_points_model=420,
            geometric_rmse_model=4.610,
            geometric_rmse_model_x=3.527,
            geometric_rmse_model_y=2.968,

            # TODO: What are these two?
            viewing_incidence_angle_long_track=0,
            viewing_incidence_angle_x_track=0,
            bands={
                'coastal_aerosol':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B1.TIF'),
                    number=1,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'visible_blue':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B2.TIF'),
                    number=2,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'visible_green':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B3.TIF'),
                    number=3,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'visible_red':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B4.TIF'),
                    number=4,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'near_infrared':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B5.TIF'),
                    number=5,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'short_wave_infrared1':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B6.TIF'),
                    number=6,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'short_wave_infrared2':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B7.TIF'),
                    number=7,
                    type_='reflective',
                    cell_size=25.0,
                ),
                'panchromatic':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B8.TIF'),
                    number=8,
                    type_='panchromatic',
                    cell_size=12.50,
                    shape=ptype.Point(17761, 18241),
                ),
                'cirrus':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B9.TIF'),
                    number=9,
                    type_='atmosphere',
                ),
                'thermal_infrared1':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B10.TIF'),
                    number=10,
                    type_='thermal',
                    cell_size=25.0,
                    shape=ptype.Point(8881, 9121),
                ),
                'thermal_infrared2':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_B11.TIF'),
                    number=11,
                    type_='thermal',
                    cell_size=25.0,
                    shape=ptype.Point(8881, 9121),
                ),
                'quality':
                ptype.BandMetadata(
                    path=Path('product/LC81010782014285LGN00_BQA.TIF'),
                    number='QA',
                    type_='quality',
                )
            }),
        lineage=ptype.LineageMetadata(
            algorithm=ptype.AlgorithmMetadata(
                name='Pinkmatter Landsat Processor',
                version='3.3.3104',
                parameters={
                    'resampling': 'CC',
                    'radiometric_correction': 'CPF',
                    'orientation': 'NUP',
                    'hemisphere': 'S',
                }),
            machine=ptype.MachineMetadata(
                hostname='rhe-jm-prod08.prod.lan',
                type_id='jobmanager',
                uname=
                'Linux rhe-jm-dev08.dev.lan 2.6.32-279.22.1.el6.x86_64 #1 SMP Sun Oct '
                '12 '
                '09:21:40 EST 2014 x86_64 x86_64 x86_64 GNU/Linux'),
            ancillary={
                'cpf':
                ptype.AncillaryMetadata(
                    name='L8CPF20141001_20141231.01',
                    uri=
                    '/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile'
                    '/L8CPF20141001_20141231.01'),
                'bpf_tirs':
                ptype.AncillaryMetadata(
                    name='LT8BPF20141012002432_20141012020301.01',
                    uri=
                    '/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10'
                    '/LT8BPF20141012002432_20141012020301.01'),
                'bpf_oli':
                ptype.AncillaryMetadata(
                    name='LO8BPF20141012002825_20141012011100.01',
                    uri=
                    '/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10'
                    '/LT8BPF20141012002432_20141012020301.01'),
                'rlut':
                ptype.AncillaryMetadata(name='L8RLUT20130211_20431231v09.h5')
            },
            source_datasets={'satellite_telemetry_data': _build_ls8_raw()}))
예제 #7
0
def _build_ls8_nbar():
    _reset_runtime_id()
    nbar = ptype.DatasetMetadata(
        id_=uuid.UUID("249ae098-bd88-11e4-beaa-1040f381a756"),
        size_bytes=622208 * 1024,
        ga_label='LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012',
        product_type='GANBAR01',
        platform=ptype.PlatformMetadata(code='LANDSAT-8'),
        instrument=ptype.InstrumentMetadata(name='OLI_TIRS',
                                            type_="Multi-Spectral",
                                            operation_mode='PUSH-BROOM'),
        # acquisition=ptype.AcquisitionMetadata(),
        format_=ptype.FormatMetadata(name='GeoTiff', version=1),
        extent=ptype.ExtentMetadata(
            reference_system='WGS84',
            coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969),
                                     ur=ptype.Coord(lat=-24.96826,
                                                    lon=136.24838),
                                     lr=ptype.Coord(lat=-26.96338,
                                                    lon=136.26962),
                                     ll=ptype.Coord(lat=-26.96528,
                                                    lon=133.96233)),
            from_dt=dateutil.parser.parse("2014-10-12T00:55:54"),
            to_dt=dateutil.parser.parse("2014-10-12T00:56:18"),
        ),
        grid_spatial=ptype.GridSpatialMetadata(
            dimensions=[
                ptype.DimensionMetadata(name='sample',
                                        resolution=25.0,
                                        size=9161),
                ptype.DimensionMetadata(name='line',
                                        resolution=25.0,
                                        size=9161)
            ],
            projection=ptype.ProjectionMetadata(
                centre_point=ptype.Point(511512.500000, 7127487.500000),
                geo_ref_points=ptype.PointPolygon(
                    ul=ptype.Point(397012.5, 7237987.5),
                    ur=ptype.Point(626012.5, 7237987.5),
                    ll=ptype.Point(397012.5, 7016987.5),
                    lr=ptype.Point(626012.5, 7016987.5)),
                datum='GDA94',
                ellipsoid='GRS80',
                point_in_pixel='UL',
                map_projection='UTM',
                resampling_option='CUBIC_CONVOLUTION',
                zone=-53)),
        browse={
            'medium':
            ptype.BrowseMetadata(path=Path(
                'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.tif'),
                                 file_type='image/jpg',
                                 cell_size=219.75,
                                 red_band=7,
                                 green_band=5,
                                 blue_band=2),
            'full':
            ptype.BrowseMetadata(path=Path(
                'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_FR.tif'),
                                 file_type='image/jpg',
                                 cell_size=25.0,
                                 red_band=7,
                                 green_band=5,
                                 blue_band=2)
        },
        image=ptype.ImageMetadata(
            satellite_ref_point_start=ptype.Point(101, 78),
            cloud_cover_percentage=0.01,
            cloud_cover_details=None,

            # TODO: What are these two?
            viewing_incidence_angle_long_track=0,
            viewing_incidence_angle_x_track=0,
            bands={
                '1':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B1.tif'
                ), ),
                '2':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B2.tif'
                ), ),
                '3':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B3.tif'
                ), ),
                '4':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B4.tif'
                ), ),
                '5':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B5.tif'
                ), ),
                '6':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B6.tif'
                ), ),
                '7':
                ptype.BandMetadata(path=Path(
                    'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B7.tif'
                ), )
            }),
        lineage=ptype.LineageMetadata(
            algorithm=ptype.AlgorithmMetadata(name='GANBAR',
                                              version='3.2.1',
                                              parameters={}),
            machine=ptype.MachineMetadata(),
            source_datasets={'level1': _build_ls8_ortho()},
            ancillary={}))
    return nbar
예제 #8
0
def _build_ls7_wofs():
    return ptype.DatasetMetadata(
        ga_label='LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897',
        product_type='GAWATER',
        size_bytes=616 * 1024,
        platform=ptype.PlatformMetadata(code='LS7'),
        instrument=ptype.InstrumentMetadata(name='ETM',
                                            type_='Multi-Spectral'),
        format_=ptype.FormatMetadata('GeoTIFF', version=1),
        extent=ptype.ExtentMetadata(
            reference_system='WGS84',
            coord=ptype.CoordPolygon(ul=ptype.Coord(140.0000000, -26.0000000),
                                     ll=ptype.Coord(140.0000000, -27.0000000),
                                     ur=ptype.Coord(141.0000000, -26.0000000),
                                     lr=ptype.Coord(141.0000000, -27.0000000)),

            # TODO: Should we store the center coordinate?
            from_dt=dateutil.parser.parse('2013-07-24 00:32:27.952897'),
            to_dt=dateutil.parser.parse('2013-07-24 00:33:15.899670')),
        grid_spatial=ptype.GridSpatialMetadata(
            dimensions=[
                ptype.DimensionMetadata(name='x',
                                        resolution=27.1030749476,
                                        size=4000),
                ptype.DimensionMetadata(name='y',
                                        resolution=27.1030749476,
                                        size=4000)
            ],
            # TODO: Should WOfS have tile coordinates here?
            # georectified=ptype.GeoRectifiedSpacialMetadata(
            # geo_ref_points=PointPolygon(
            # ul=ptype.Point(255012.500, 7229987.500),
            # ur=ptype.Point(497012.500, 7229987.500),
            # ll=ptype.Point(255012.500, 7019987.500),
            # lr=ptype.Point(497012.500, 7229987.500)
            # ),
            # checkpoint_availability=0,
            # datum='GDA94',
            #     ellipsoid='GRS80',
            #     point_in_pixel='UL',
            #     projection='UTM',
            #     zone=-54
            # )
        ),
        image=ptype.ImageMetadata(
            satellite_ref_point_start=ptype.Point(98, 78),
            satellite_ref_point_end=ptype.Point(98, 79),
            cloud_cover_percentage=0.76494375,
            cloud_cover_details='122391 count',
            sun_elevation=33.0061002772,
            sun_azimuth=38.2433049177,
            bands={
                'W':
                ptype.BandMetadata(path=Path(
                    'LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897.tif'),
                                   # TODO: Nodata value?
                                   )
            }),
        lineage=ptype.LineageMetadata(
            algorithm=ptype.AlgorithmMetadata(name='WOfS',
                                              version='1.3',
                                              parameters={}),
            machine=ptype.MachineMetadata(),
            source_datasets={
                # TODO: LS7 dataset?
            }))