def _build_ls8_raw(): _reset_runtime_id() raw = ptype.DatasetMetadata( id_=uuid.UUID('1c76a8ca-51ae-11e4-8644-0050568d59ac'), creation_dt=dateutil.parser.parse("2014-10-12 04:18:01"), size_bytes=5680940 * 1024, ga_label= 'MD_P00_LC81010700832014285LGN00_101_070-083_20141012T032336Z20141012T032910_1', product_type='satellite_telemetry_data', usgs=ptype.UsgsMetadata(interval_id='LC81010782014285LGN00'), platform=ptype.PlatformMetadata(code='LANDSAT-8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='MD'), acquisition=ptype.AcquisitionMetadata( aos=dateutil.parser.parse('2014-10-12T00:52:52'), los=dateutil.parser.parse('2014-10-12T00:58:37'), groundstation=ptype.GroundstationMetadata( code='ASA', antenna_coord=ptype.Coord(lat=-23.759, lon=133.8824, height=579.312)), heading='D', platform_orbit=8846), extent=None, grid_spatial=None, browse=None, image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point( 101, 70), satellite_ref_point_end=ptype.Point(101, 83)), lineage=ptype.LineageMetadata(machine=ptype.MachineMetadata())) return raw
def _populate_grid_spatial(md, mtl_): product_md = _get(mtl_, 'PRODUCT_METADATA') # We don't have a single set of dimensions. Depends on the band? # md.grid_spatial.dimensions = [] if not md.grid_spatial: md.grid_spatial = ptype.GridSpatialMetadata() if not md.grid_spatial.projection: md.grid_spatial.projection = ptype.ProjectionMetadata() md.grid_spatial.projection.geo_ref_points = ptype.PointPolygon( ul=ptype.Point(x=_get(product_md, 'corner_ul_projection_x_product'), y=_get(product_md, 'corner_ul_projection_y_product')), ur=ptype.Point(x=_get(product_md, 'corner_ur_projection_x_product'), y=_get(product_md, 'corner_ur_projection_y_product')), ll=ptype.Point(x=_get(product_md, 'corner_ll_projection_x_product'), y=_get(product_md, 'corner_ll_projection_y_product')), lr=ptype.Point(x=_get(product_md, 'corner_lr_projection_x_product'), y=_get(product_md, 'corner_lr_projection_y_product'))) # centre_point=None, projection_md = _get(mtl_, 'PROJECTION_PARAMETERS') md.grid_spatial.projection.datum = _get(projection_md, 'datum') md.grid_spatial.projection.ellipsoid = _get(projection_md, 'ellipsoid') # Where does this come from? 'ul' etc. # point_in_pixel=None, md.grid_spatial.projection.map_projection = _get(projection_md, 'map_projection') md.grid_spatial.projection.resampling_option = _get( projection_md, 'resampling_option') md.grid_spatial.projection.datum = _get(projection_md, 'datum') md.grid_spatial.projection.ellipsoid = _get(projection_md, 'ellipsoid') md.grid_spatial.projection.zone = _get(projection_md, 'utm_zone') md.grid_spatial.projection.orientation = _get(projection_md, 'orientation')
def _extract_mdf_id_fields(base_md, mdf_usgs_id): """ V I N ppp RRR rrr YYYY ddd GSI vv :type base_md: ptype.DatasetMetadata :type mdf_usgs_id: str :rtype: ptype.DatasetMetadata """ m = re.search((r"(?P<vehicle>L)" r"(?P<instrument>[OTC])" r"(?P<vehicle_number>\d)" r"(?P<path>\d{3})" r"(?P<row_start>\d{3})" r"(?P<row_end>\d{3})" r"(?P<acq_date>\d{7})" r"(?P<gsi>\w{3})" r"(?P<version>\d{2})"), mdf_usgs_id) fields = m.groupdict() if not base_md.usgs: base_md.usgs = ptype.UsgsMetadata() base_md.usgs.interval_id = mdf_usgs_id if not base_md.platform: base_md.platform = ptype.PlatformMetadata() base_md.platform.code = "LANDSAT_" + fields["vehicle_number"] if not base_md.instrument: base_md.instrument = ptype.InstrumentMetadata() base_md.instrument.name = LS8_SENSORS[fields["instrument"]] path = int(fields["path"]) if not base_md.image: base_md.image = ptype.ImageMetadata() base_md.image.satellite_ref_point_start = ptype.Point( path, int(fields["row_start"])) base_md.image.satellite_ref_point_end = ptype.Point( path, int(fields["row_end"])) # base_md.version = int(fields["version"]) or base_md.version # Probably less than useful without time. # if not base_md.extent: # base_md.extent = ptype.ExtentMetadata() # # base_md.extent.center_dt = base_md.extent.center_dt or datetime.strptime(fields["acq_date"], "%Y%j").date() if not base_md.acquisition: base_md.acquisition = ptype.AcquisitionMetadata() base_md.acquisition.groundstation = ptype.GroundstationMetadata( code=fields["gsi"]) return base_md
def populate_from_image_metadata(md): """ Populate by extracting metadata from existing band files. :type md: eodatasets.type.DatasetMetadata :rtype: eodatasets.type.DatasetMetadata """ for band_id, band in md.image.bands.items(): i = gdal.Open(str(band.path)) if not i: # TODO: log? throw? continue spacial_ref = osr.SpatialReference(i.GetProjectionRef()) # Extract actual image coords # md.grid_spatial.projection. band.shape = ptype.Point(i.RasterXSize, i.RasterYSize) band.cell_size = ptype.Point(abs(i.GetGeoTransform()[1]), abs(i.GetGeoTransform()[5])) # TODO separately: create standardised WGS84 coords. for md.extent # wkt_contents = spacial_ref.ExportToPrettyWkt() # TODO: if srs IsGeographic()? Otherwise srs IsProjected()? if not md.grid_spatial: md.grid_spatial = ptype.GridSpatialMetadata() if not md.grid_spatial.projection: md.grid_spatial.projection = ptype.ProjectionMetadata() md.grid_spatial.projection.geo_ref_points = _get_gdal_image_coords(i) md.grid_spatial.projection.unit = spacial_ref.GetLinearUnitsName() md.grid_spatial.projection.zone = spacial_ref.GetUTMZone() # ? md.grid_spatial.projection.datum = 'GDA94' md.grid_spatial.projection.ellipsoid = 'GRS80' # TODO: DATUM/Reference system etc. if not md.extent: md.extent = ptype.ExtentMetadata() md.extent.coord = reproject_coords( md.grid_spatial.projection.geo_ref_points, spacial_ref) # Get positional info, projection etc. # Is projection/etc same as previous? # -- If all match, set on wider image. i = None return md
def test_eods_fill_metadata(self): dataset_folder = "LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012" bandname = '10' bandfile = dataset_folder + '_B' + bandname + '.tif' input_folder = write_files({ dataset_folder: { 'metadata.xml': """<EODS_DATASET> <ACQUISITIONINFORMATION> <EVENT> <AOS>20141012T03:23:36</AOS> <LOS>20141012T03:29:10</LOS> </EVENT> </ACQUISITIONINFORMATION> <EXEXTENT> <TEMPORALEXTENTFROM>20141012 00:55:54</TEMPORALEXTENTFROM> <TEMPORALEXTENTTO>20141012 00:56:18</TEMPORALEXTENTTO> </EXEXTENT> </EODS_DATASET>""", 'scene01': { bandfile: '' } } }) expected = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_, ga_label=dataset_folder, ga_level='P54', product_type='EODS_NBAR', platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='GeoTiff'), acquisition=ptype.AcquisitionMetadata(aos=datetime.datetime(2014, 10, 12, 3, 23, 36), los=datetime.datetime(2014, 10, 12, 3, 29, 10), groundstation=ptype.GroundstationMetadata(code='LGS')), extent=ptype.ExtentMetadata( center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6), from_dt=datetime.datetime(2014, 10, 12, 0, 55, 54), to_dt=datetime.datetime(2014, 10, 12, 0, 56, 18) ), image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78), satellite_ref_point_end=ptype.Point(x=101, y=78), bands={bandname: ptype.BandMetadata(number=bandname, path=Path(input_folder, dataset_folder, 'scene01', bandfile))}) ) dataset = ptype.DatasetMetadata( id_=_EXPECTED_NBAR.id_ ) received = drivers.EODSDriver().fill_metadata(dataset, input_folder.joinpath(dataset_folder)) self.assert_same(expected, received)
def test_directory_parse(self): dataset_id = 'LC80880750762013254ASA00' metadata = mdf._extract_mdf_id_fields(ptype.DatasetMetadata(), dataset_id) self.assertEqual(metadata.usgs.interval_id, dataset_id) self.assertEqual(metadata.platform.code, 'LANDSAT_8') self.assertEqual(metadata.instrument.name, 'OLI_TIRS') self.assertEqual(metadata.image.satellite_ref_point_start, ptype.Point(88, 75)) self.assertEqual(metadata.image.satellite_ref_point_end, ptype.Point(88, 76)) self.assertEqual(metadata.acquisition.groundstation.code, 'ASA')
def test_create_typical_browse_metadata(): class TestDriver(drivers.DatasetDriver): def browse_image_bands(self, d): return '5', '1', '3' d = write_files({}) dataset = browseimage.create_typical_browse_metadata( TestDriver(), ptype.DatasetMetadata(), d) expected = ptype.DatasetMetadata( browse={ 'full': ptype.BrowseMetadata(path=d.joinpath('browse.fr.jpg'), file_type='image/jpg', red_band='5', green_band='1', blue_band='3'), 'medium': ptype.BrowseMetadata( path=d.joinpath('browse.jpg'), # Default medium size. shape=ptype.Point(1024, None), file_type='image/jpg', red_band='5', green_band='1', blue_band='3') }) expected.id_, dataset.id_ = None, None assert_same(expected, dataset)
def _test_mdf_output(metadata): self.assertEqual(metadata.usgs.interval_id, 'LC80880750762013254ASA00') self.assertEqual(metadata.platform.code, 'LANDSAT_8') self.assertEqual(metadata.instrument.name, 'OLI_TIRS') self.assertEqual(metadata.format_.name, 'MD') self.assertEqual(metadata.ga_level, 'P00') self.assertEqual(metadata.image.satellite_ref_point_start, ptype.Point(88, 75)) self.assertEqual(metadata.image.satellite_ref_point_end, ptype.Point(88, 76)) self.assertEqual(metadata.acquisition.groundstation.code, 'ASA') # No longer bother with vague center date. # self.assertEqual(metadata.extent.center_dt, datetime.date(2013, 9, 11)) self.assertEqual( metadata.acquisition.aos, datetime.datetime(2013, 9, 11, 23, 36, 11, 482000)) self.assertEqual( metadata.acquisition.los, datetime.datetime(2013, 9, 11, 23, 37, 14, 881000))
def create_dataset_browse_images( dataset_driver, dataset, target_directory, after_file_creation=lambda file_path: None): """ :type dataset_driver: drivers.DatasetDriver :type dataset: ptype.DatasetMetadata :type target_directory: Path :type after_file_creation: Path -> None :rtype: ptype.DatasetMetadata """ if not dataset.image or not dataset.image.bands: # A dataset without defined bands doesn't get a browse image (eg. raw file) _LOG.info('No bands defined. Skipping browse image.') return dataset # Create browse image metadata if missing. if not dataset.browse: create_typical_browse_metadata(dataset_driver, dataset, target_directory) # Create browse images based on the metadata. for browse_id, browse_metadata in dataset.browse.items(): x_constraint = None if browse_metadata.shape: x_constraint = browse_metadata.shape.x bands = dataset.image.bands necessary_bands = (browse_metadata.red_band, browse_metadata.green_band, browse_metadata.blue_band) if not all([bands.get(band) for band in necessary_bands]): raise ValueError( 'Some browse bands missing. Requires {!r}, has {!r}' ''.format(necessary_bands, bands.keys()) ) r_path, g_path, b_path = [bands[p].path for p in necessary_bands] cols, rows, output_res = _create_thumbnail( r_path, g_path, b_path, browse_metadata.path, x_constraint=x_constraint ) # Update with the exact shape information. browse_metadata.shape = ptype.Point(cols, rows) browse_metadata.cell_size = output_res after_file_creation(browse_metadata.path) return dataset
def _populate_image(md, mtl_): product_md = _get(mtl_, 'PRODUCT_METADATA') image_md = _get(mtl_, 'IMAGE_ATTRIBUTES') if not md.image: md.image = ptype.ImageMetadata() md.image.satellite_ref_point_start = ptype.Point( _get(product_md, 'wrs_path'), _get(product_md, 'wrs_row')) md.image.cloud_cover_percentage = _get(image_md, 'cloud_cover') md.image.sun_elevation = _get(image_md, 'sun_elevation') md.image.sun_azimuth = _get(image_md, 'sun_azimuth') md.image.sun_earth_distance = _get(image_md, 'earth_sun_distance') md.image.ground_control_points_model = _get(image_md, 'ground_control_points_model') # md.image. = _get(image_md, 'earth_sun_distance') md.image.geometric_rmse_model = _get(image_md, 'geometric_rmse_model') md.image.geometric_rmse_model_y = _get(image_md, 'geometric_rmse_model_y') md.image.geometric_rmse_model_x = _get(image_md, 'geometric_rmse_model_x') if not md.image.bands: md.image.bands = {}
def create_typical_browse_metadata(dataset_driver, dataset, destination_directory): """ Create browse metadata. :type dataset_driver: eodatasets.package.DatasetDriver :type dataset: ptype.DatasetMetadata :type destination_directory: Path :return: """ rgb_bands = dataset_driver.browse_image_bands(dataset) if len(rgb_bands) == 3: r, g, b = rgb_bands elif len(rgb_bands) == 1: band = rgb_bands[0] r, g, b = band, band, band else: raise ValueError('Unexpected number of bands (%s). Received %r' % (len(rgb_bands), rgb_bands)) dataset.browse = { 'medium': ptype.BrowseMetadata( path=destination_directory.joinpath('browse.jpg'), file_type='image/jpg', # cell_size=output_res, shape=ptype.Point(1024, None), red_band=r, green_band=g, blue_band=b), 'full': ptype.BrowseMetadata( path=destination_directory.joinpath('browse.fr.jpg'), file_type='image/jpg', # cell_size=output_res, red_band=r, green_band=g, blue_band=b) } return dataset
lat=-19.72798, lon=115.56872 ), lr=ptype.Coord( lat=-19.73177, lon=117.83040 ) ), # TODO: python dt is one digit less precise than mtl (02:03:36.9270519Z). Does this matter? center_dt=datetime.datetime(2005, 1, 7, 2, 3, 36, 927051) ), grid_spatial=ptype.GridSpatialMetadata( projection=ptype.ProjectionMetadata( geo_ref_points=ptype.PointPolygon( ul=ptype.Point( x=350012.500, y=8028987.500 ), ur=ptype.Point( x=587012.500, y=8028987.500 ), ll=ptype.Point( x=350012.500, y=7817987.500 ), lr=ptype.Point( x=587012.500, y=7817987.500 ) ), datum='GDA94',
def _get_point(gt, px, py): x = gt[0] + (px * gt[1]) + (py * gt[2]) y = gt[3] + (px * gt[4]) + (py * gt[5]) return ptype.Point(x, y)
def _build_ls7_wofs(): return ptype.DatasetMetadata( ga_label='LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897', product_type='GAWATER', size_bytes=616 * 1024, platform=ptype.PlatformMetadata(code='LS7'), instrument=ptype.InstrumentMetadata(name='ETM', type_='Multi-Spectral'), format_=ptype.FormatMetadata('GeoTIFF', version=1), extent=ptype.ExtentMetadata( reference_system='WGS84', coord=ptype.CoordPolygon(ul=ptype.Coord(140.0000000, -26.0000000), ll=ptype.Coord(140.0000000, -27.0000000), ur=ptype.Coord(141.0000000, -26.0000000), lr=ptype.Coord(141.0000000, -27.0000000)), # TODO: Should we store the center coordinate? from_dt=dateutil.parser.parse('2013-07-24 00:32:27.952897'), to_dt=dateutil.parser.parse('2013-07-24 00:33:15.899670')), grid_spatial=ptype.GridSpatialMetadata( dimensions=[ ptype.DimensionMetadata(name='x', resolution=27.1030749476, size=4000), ptype.DimensionMetadata(name='y', resolution=27.1030749476, size=4000) ], # TODO: Should WOfS have tile coordinates here? # georectified=ptype.GeoRectifiedSpacialMetadata( # geo_ref_points=PointPolygon( # ul=ptype.Point(255012.500, 7229987.500), # ur=ptype.Point(497012.500, 7229987.500), # ll=ptype.Point(255012.500, 7019987.500), # lr=ptype.Point(497012.500, 7229987.500) # ), # checkpoint_availability=0, # datum='GDA94', # ellipsoid='GRS80', # point_in_pixel='UL', # projection='UTM', # zone=-54 # ) ), image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(98, 78), satellite_ref_point_end=ptype.Point(98, 79), cloud_cover_percentage=0.76494375, cloud_cover_details='122391 count', sun_elevation=33.0061002772, sun_azimuth=38.2433049177, bands={ 'W': ptype.BandMetadata(path=Path( 'LS7_ETM_WATER_140_-027_2013-07-24T00-32-27.952897.tif'), # TODO: Nodata value? ) }), lineage=ptype.LineageMetadata( algorithm=ptype.AlgorithmMetadata(name='WOfS', version='1.3', parameters={}), machine=ptype.MachineMetadata(), source_datasets={ # TODO: LS7 dataset? }))
def _build_ls8_nbar(): _reset_runtime_id() nbar = ptype.DatasetMetadata( id_=uuid.UUID("249ae098-bd88-11e4-beaa-1040f381a756"), size_bytes=622208 * 1024, ga_label='LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012', product_type='GANBAR01', platform=ptype.PlatformMetadata(code='LANDSAT-8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS', type_="Multi-Spectral", operation_mode='PUSH-BROOM'), # acquisition=ptype.AcquisitionMetadata(), format_=ptype.FormatMetadata(name='GeoTiff', version=1), extent=ptype.ExtentMetadata( reference_system='WGS84', coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969), ur=ptype.Coord(lat=-24.96826, lon=136.24838), lr=ptype.Coord(lat=-26.96338, lon=136.26962), ll=ptype.Coord(lat=-26.96528, lon=133.96233)), from_dt=dateutil.parser.parse("2014-10-12T00:55:54"), to_dt=dateutil.parser.parse("2014-10-12T00:56:18"), ), grid_spatial=ptype.GridSpatialMetadata( dimensions=[ ptype.DimensionMetadata(name='sample', resolution=25.0, size=9161), ptype.DimensionMetadata(name='line', resolution=25.0, size=9161) ], projection=ptype.ProjectionMetadata( centre_point=ptype.Point(511512.500000, 7127487.500000), geo_ref_points=ptype.PointPolygon( ul=ptype.Point(397012.5, 7237987.5), ur=ptype.Point(626012.5, 7237987.5), ll=ptype.Point(397012.5, 7016987.5), lr=ptype.Point(626012.5, 7016987.5)), datum='GDA94', ellipsoid='GRS80', point_in_pixel='UL', map_projection='UTM', resampling_option='CUBIC_CONVOLUTION', zone=-53)), browse={ 'medium': ptype.BrowseMetadata(path=Path( 'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.tif'), file_type='image/jpg', cell_size=219.75, red_band=7, green_band=5, blue_band=2), 'full': ptype.BrowseMetadata(path=Path( 'LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_FR.tif'), file_type='image/jpg', cell_size=25.0, red_band=7, green_band=5, blue_band=2) }, image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(101, 78), cloud_cover_percentage=0.01, cloud_cover_details=None, # TODO: What are these two? viewing_incidence_angle_long_track=0, viewing_incidence_angle_x_track=0, bands={ '1': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B1.tif' ), ), '2': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B2.tif' ), ), '3': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B3.tif' ), ), '4': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B4.tif' ), ), '5': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B5.tif' ), ), '6': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B6.tif' ), ), '7': ptype.BandMetadata(path=Path( 'product/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012_B7.tif' ), ) }), lineage=ptype.LineageMetadata( algorithm=ptype.AlgorithmMetadata(name='GANBAR', version='3.2.1', parameters={}), machine=ptype.MachineMetadata(), source_datasets={'level1': _build_ls8_ortho()}, ancillary={})) return nbar
ga_level='P54', platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='GeoTIFF'), acquisition=ptype.AcquisitionMetadata(groundstation=ptype.GroundstationMetadata(code='LGN')), extent=ptype.ExtentMetadata( coord=ptype.CoordPolygon( ul=ptype.Coord(lat=-24.98805, lon=133.97954), ur=ptype.Coord(lat=-24.9864, lon=136.23866), ll=ptype.Coord(lat=-26.99236, lon=133.96208), lr=ptype.Coord(lat=-26.99055, lon=136.25985) ), center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785) ), image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(x=101, y=78), bands={} ), lineage=ptype.LineageMetadata( source_datasets={'level1': test_ls8.EXPECTED_OUT}, algorithm=ptype.AlgorithmMetadata(name='terrain', version='1.0'), ), grid_spatial=ptype.GridSpatialMetadata( projection=ptype.ProjectionMetadata( geo_ref_points=ptype.PointPolygon( ul=ptype.Point( x=397012.5, y=7235987.5 ), ur=ptype.Point( x=625012.5,
def _build_ls8_ortho(): _reset_runtime_id() return ptype.DatasetMetadata( id_=uuid.UUID('17b92c16-51d3-11e4-909d-005056bb6972'), ga_label='LS8_OLITIRS_OTH_P51_GALPGS01-002_101_078_20141012', product_type='GAORTHO01', creation_dt=dateutil.parser.parse('2014-10-12 05:46:20'), size_bytes=2386550 * 1024, platform=ptype.PlatformMetadata(code='LANDSAT-8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS', type_="Multi-Spectral", operation_mode='PUSH-BROOM'), format_=ptype.FormatMetadata(name='GeoTiff', version=1), extent=ptype.ExtentMetadata( reference_system='WGS84', coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.97, lon=133.97969), ur=ptype.Coord(lat=-24.96826, lon=136.24838), lr=ptype.Coord(lat=-26.96338, lon=136.26962), ll=ptype.Coord(lat=-26.96528, lon=133.96233)), from_dt=dateutil.parser.parse("2014-10-12T00:55:54"), center_dt=dateutil.parser.parse("2014-10-12T00:56:06"), to_dt=dateutil.parser.parse("2014-10-12T00:56:18"), ), grid_spatial=ptype.GridSpatialMetadata( dimensions=[ ptype.DimensionMetadata(name='sample', resolution=25.0, size=9161), ptype.DimensionMetadata(name='line', resolution=25.0, size=9161) ], projection=ptype.ProjectionMetadata( centre_point=ptype.Point(511512.500000, 7127487.500000), geo_ref_points=ptype.PointPolygon( ul=ptype.Point(397012.5, 7237987.5), ur=ptype.Point(626012.5, 7237987.5), ll=ptype.Point(397012.5, 7016987.5), lr=ptype.Point(626012.5, 7016987.5)), datum='GDA94', ellipsoid='GRS80', point_in_pixel='UL', map_projection='UTM', resampling_option='CUBIC_CONVOLUTION', zone=-53)), browse={ 'medium': ptype.BrowseMetadata(path=Path( 'product/LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012.jpg' ), file_type='image/jpg', cell_size=219.75, red_band=7, green_band=5, blue_band=1), 'full': ptype.BrowseMetadata(path=Path( 'LS8_OLITIRS_OTH_P51_GALPGS01-032_101_078_20141012_FR.jpg'), file_type='image/jpg', cell_size=25.0, red_band=7, green_band=5, blue_band=1) }, image=ptype.ImageMetadata( satellite_ref_point_start=ptype.Point(101, 78), cloud_cover_percentage=0, cloud_cover_details=None, sun_elevation=58.00268508, sun_azimuth=59.41814014, ground_control_points_model=420, geometric_rmse_model=4.610, geometric_rmse_model_x=3.527, geometric_rmse_model_y=2.968, # TODO: What are these two? viewing_incidence_angle_long_track=0, viewing_incidence_angle_x_track=0, bands={ 'coastal_aerosol': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B1.TIF'), number=1, type_='reflective', cell_size=25.0, ), 'visible_blue': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B2.TIF'), number=2, type_='reflective', cell_size=25.0, ), 'visible_green': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B3.TIF'), number=3, type_='reflective', cell_size=25.0, ), 'visible_red': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B4.TIF'), number=4, type_='reflective', cell_size=25.0, ), 'near_infrared': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B5.TIF'), number=5, type_='reflective', cell_size=25.0, ), 'short_wave_infrared1': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B6.TIF'), number=6, type_='reflective', cell_size=25.0, ), 'short_wave_infrared2': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B7.TIF'), number=7, type_='reflective', cell_size=25.0, ), 'panchromatic': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B8.TIF'), number=8, type_='panchromatic', cell_size=12.50, shape=ptype.Point(17761, 18241), ), 'cirrus': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B9.TIF'), number=9, type_='atmosphere', ), 'thermal_infrared1': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B10.TIF'), number=10, type_='thermal', cell_size=25.0, shape=ptype.Point(8881, 9121), ), 'thermal_infrared2': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_B11.TIF'), number=11, type_='thermal', cell_size=25.0, shape=ptype.Point(8881, 9121), ), 'quality': ptype.BandMetadata( path=Path('product/LC81010782014285LGN00_BQA.TIF'), number='QA', type_='quality', ) }), lineage=ptype.LineageMetadata( algorithm=ptype.AlgorithmMetadata( name='Pinkmatter Landsat Processor', version='3.3.3104', parameters={ 'resampling': 'CC', 'radiometric_correction': 'CPF', 'orientation': 'NUP', 'hemisphere': 'S', }), machine=ptype.MachineMetadata( hostname='rhe-jm-prod08.prod.lan', type_id='jobmanager', uname= 'Linux rhe-jm-dev08.dev.lan 2.6.32-279.22.1.el6.x86_64 #1 SMP Sun Oct ' '12 ' '09:21:40 EST 2014 x86_64 x86_64 x86_64 GNU/Linux'), ancillary={ 'cpf': ptype.AncillaryMetadata( name='L8CPF20141001_20141231.01', uri= '/eoancillarydata/sensor-specific/LANDSAT8/CalibrationParameterFile' '/L8CPF20141001_20141231.01'), 'bpf_tirs': ptype.AncillaryMetadata( name='LT8BPF20141012002432_20141012020301.01', uri= '/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10' '/LT8BPF20141012002432_20141012020301.01'), 'bpf_oli': ptype.AncillaryMetadata( name='LO8BPF20141012002825_20141012011100.01', uri= '/eoancillarydata/sensor-specific/LANDSAT8/BiasParameterFile/2014/10' '/LT8BPF20141012002432_20141012020301.01'), 'rlut': ptype.AncillaryMetadata(name='L8RLUT20130211_20431231v09.h5') }, source_datasets={'satellite_telemetry_data': _build_ls8_raw()}))
def fill_metadata(self, dataset, path, additional_files=()): """ :type additional_files: tuple[Path] :type dataset: ptype.DatasetMetadata :type path: Path :rtype: ptype.DatasetMetadata """ fields = re.match((r"(?P<vehicle>LS[578])" r"_(?P<instrument>OLI_TIRS|OLI|TIRS|TM|ETM)" r"_(?P<type>NBAR|PQ|FC)" r"_(?P<level>[^-_]*)" r"_(?P<product>[^-_]*)" r"-(?P<groundstation>[0-9]{3})" r"_(?P<path>[0-9]{3})" r"_(?P<row>[0-9]{3})" r"_(?P<date>[12][0-9]{7})" r"(_(?P<version>[0-9]+))?" "$"), path.stem).groupdict() dataset.product_type = "EODS_" + fields["type"] dataset.ga_level = fields["level"] dataset.ga_label = path.stem dataset.format_ = ptype.FormatMetadata(name='GeoTiff') if not dataset.platform: dataset.platform = ptype.PlatformMetadata() dataset.platform.code = "LANDSAT_" + fields["vehicle"][2] if not dataset.instrument: dataset.instrument = ptype.InstrumentMetadata() dataset.instrument.name = fields["instrument"] if not dataset.image: dataset.image = ptype.ImageMetadata(bands={}) dataset.image.satellite_ref_point_start = ptype.Point( int(fields["path"]), int(fields["row"])) dataset.image.satellite_ref_point_end = ptype.Point( int(fields["path"]), int(fields["row"])) for image_path in path.joinpath("scene01").iterdir(): band = self.to_band(dataset, image_path) if band: dataset.image.bands[band.number] = band md_image.populate_from_image_metadata(dataset) if not dataset.acquisition: dataset.acquisition = ptype.AcquisitionMetadata() for _station in _GROUNDSTATION_LIST: if _station["eods_domain_code"] == fields["groundstation"]: dataset.acquisition.groundstation = ptype.GroundstationMetadata( code=_station["code"]) break if not dataset.extent: dataset.extent = ptype.ExtentMetadata() def els2date(els): if not els: return None return parse(els[0].text) doc = ElementTree.parse(str(path.joinpath('metadata.xml'))) start_time = els2date(doc.findall("./EXEXTENT/TEMPORALEXTENTFROM")) end_time = els2date(doc.findall("./EXEXTENT/TEMPORALEXTENTTO")) # check if the dates in the metadata file are at least as accurate as what we have filename_time = datetime.datetime.strptime(fields["date"], "%Y%m%d") time_diff = start_time - filename_time # Is the EODS metadata extremely off? if abs(time_diff).days != 0: raise ValueError( 'EODS time information differs too much from source files: %s' % time_diff) dataset.acquisition.aos = els2date( doc.findall("./ACQUISITIONINFORMATION/EVENT/AOS")) dataset.acquisition.los = els2date( doc.findall("./ACQUISITIONINFORMATION/EVENT/LOS")) dataset.extent.center_dt = start_time + (end_time - start_time) / 2 dataset.extent.from_dt = start_time dataset.extent.to_dt = end_time return dataset
creation_dt=datetime.datetime(2014, 11, 12, 15, 8, 35), platform=ptype.PlatformMetadata(code='LANDSAT_8'), instrument=ptype.InstrumentMetadata(name='OLI_TIRS'), format_=ptype.FormatMetadata(name='GeoTIFF'), acquisition=ptype.AcquisitionMetadata( groundstation=ptype.GroundstationMetadata(code='LGN')), usgs=ptype.UsgsMetadata(scene_id='LC81010782014285LGN00'), extent=ptype.ExtentMetadata( coord=ptype.CoordPolygon(ul=ptype.Coord(lat=-24.98805, lon=133.97954), ur=ptype.Coord(lat=-24.9864, lon=136.23866), ll=ptype.Coord(lat=-26.99236, lon=133.96208), lr=ptype.Coord(lat=-26.99055, lon=136.25985)), center_dt=datetime.datetime(2014, 10, 12, 0, 56, 6, 5785)), grid_spatial=ptype.GridSpatialMetadata(projection=ptype.ProjectionMetadata( geo_ref_points=ptype. PointPolygon(ul=ptype.Point(x=397012.5, y=7235987.5), ur=ptype.Point(x=625012.5, y=7235987.5), ll=ptype.Point(x=397012.5, y=7013987.5), lr=ptype.Point(x=625012.5, y=7013987.5)), datum='GDA94', ellipsoid='GRS80', map_projection='UTM', orientation='NORTH_UP', resampling_option='CUBIC_CONVOLUTION', zone=-53)), image=ptype.ImageMetadata(satellite_ref_point_start=ptype.Point(x=101, y=78), cloud_cover_percentage=0.01, sun_azimuth=59.57807899, sun_elevation=57.89670734, sun_earth_distance=0.998137,