def create_scene(owner, prefix, landsat_id, config, datasource): logger.info('Creating scene for landsat id {}'.format(landsat_id)) gcs_prefix = io.gcs_path_for_landsat_id(landsat_id) metadata_resp = requests.get(io.make_path_for_mtl(gcs_prefix, landsat_id)) if metadata_resp.status_code == 404: logger.error('Landsat scene %s is not available yet in GCS', landsat_id) raise Exception('Could not find landsat scene %s', landsat_id) filter_metadata = extract_metadata(metadata_resp.content) (filename, cog_fname) = process_to_cog(prefix, gcs_prefix, landsat_id, config) s3_location = upload_file(owner, filename, cog_fname) logger.info('Creating image') ingest_location = 's3://{}/{}'.format(data_bucket, urllib.quote(s3_location)) scene = Scene('PRIVATE', [], datasource, {}, landsat_id, 'SUCCESS', 'SUCCESS', 'INGESTED', [io.make_path_for_mtl(gcs_prefix, landsat_id)], ingestLocation=ingest_location, cloudCover=filter_metadata['cloud_cover'], acquisitionDate=filter_metadata['acquisition_date'], sceneType='COG', owner=owner) image = create_geotiff_image( filename, ingest_location, filename=cog_fname, owner=owner, scene=scene.id, band_create_function=lambda x: config.bands.values()) scene.images = [image] return scene
def create_scene(owner, prefix, landsat_id, config, datasource): logger.info('Creating scene for landsat id {}'.format(landsat_id)) gcs_prefix = io.gcs_path_for_landsat_id(landsat_id) metadata_resp = requests.get(io.make_path_for_mtl(gcs_prefix, landsat_id)) if metadata_resp.status_code == 404: logger.error('Landsat scene %s is not available yet in GCS', landsat_id) raise Exception('Could not find landsat scene %s', landsat_id) filter_metadata = extract_metadata(metadata_resp.content) (filename, cog_fname) = process_to_cog(prefix, gcs_prefix, landsat_id, config) s3_location = upload_file(owner, filename, cog_fname) logger.info('Creating image') ingest_location = 's3://{}/{}'.format(data_bucket, urllib.quote(s3_location)) scene = Scene( 'PRIVATE', [], datasource, {}, landsat_id, 'SUCCESS', 'SUCCESS', 'INGESTED', [io.make_path_for_mtl(gcs_prefix, landsat_id)], ingestLocation=ingest_location, cloudCover=filter_metadata['cloud_cover'], acquisitionDate=filter_metadata['acquisition_date'], sceneType='COG', owner=owner) image = create_geotiff_image( filename, ingest_location, filename=cog_fname, owner=owner, scene=scene.id, band_create_function=lambda x: config.bands.values()) scene.images = [image] return scene
def create_scene(hdf_url, temp_directory, user_id, datasource): """Create a MODIS scene Args: hdf_url (str): URL for MODIS scene to download temp_directory (str): directory to use as scratch space when creating scene user_id (str): ID of owner for new MODIS scene datasource (str): ID of datasource for new MODIS scene """ config = modis_configs[datasource] granule_parts = os.path.basename(hdf_url).split('.') acquisition_datetime = datetime.strptime(granule_parts[1][1:], '%Y%j') name = '.'.join(granule_parts[:-1]) id = str(uuid.uuid4()) scene = Scene(0, Visibility.PRIVATE, [], datasource, {}, name, JobStatus.SUCCESS, JobStatus.SUCCESS, IngestStatus.INGESTED, [], owner=user_id, id=id, acquisitionDate=acquisition_datetime.isoformat() + 'Z', cloudCover=0) hdf_directory = os.path.join(temp_directory, 'hdf') os.mkdir(hdf_directory) hdf_filepath = download_hdf(hdf_url, hdf_directory) tiff_directory = os.path.join(temp_directory, 'tiffs') os.mkdir(tiff_directory) tifs = create_geotiffs(hdf_filepath, tiff_directory) s3_uris = upload_tifs(tifs, user_id, scene.id) images = [] get_band_func = partial(get_image_band, modis_config=config) for local_path, remote_path in zip(tifs, s3_uris): image = create_geotiff_image(local_path, remote_path, scene=scene.id, owner=user_id, band_create_function=get_band_func) images.append(image) scene.images = images scene.ingestLocation = s3_uris[0] scene.sceneType = 'COG' return scene
def create_scene(owner, prefix, landsat_id, config, datasource): logger.info('Creating scene for landsat id {}'.format(landsat_id)) gcs_prefix = io.gcs_path_for_landsat_id(landsat_id) logger.info('Fetching all bands') for band in config.bands.keys(): fetch_band(prefix, gcs_prefix, band, landsat_id) filter_metadata = extract_metadata( requests.get(io.make_path_for_mtl(gcs_prefix, landsat_id)).content) cog_fname = '{}_COG.tif'.format(landsat_id) stacked_fname = '{}_STACKED.tif'.format(landsat_id) filenames = { 'COG': os.path.join(prefix, cog_fname), 'STACKED': os.path.join(prefix, stacked_fname) } convert_to_cog(prefix, filenames['STACKED'], filenames['COG'], config, landsat_id) s3_location = upload_file(owner, filenames['COG'], cog_fname) logger.info('Creating image') ingest_location = 's3://{}/{}'.format(data_bucket, urllib.quote(s3_location)) scene = Scene(0, 'PRIVATE', [], datasource, {}, landsat_id, 'SUCCESS', 'SUCCESS', 'INGESTED', [io.make_path_for_mtl(gcs_prefix, landsat_id)], ingestLocation=ingest_location, cloudCover=filter_metadata['cloud_cover'], acquisitionDate=filter_metadata['acquisition_date'], sceneType='COG', owner=owner) image = create_geotiff_image( filenames['COG'], ingest_location, filename=cog_fname, owner=owner, scene=scene.id, band_create_function=lambda x: config.bands.values()) scene.images = [image] return scene
def create_scene(hdf_url, temp_directory, user_id, datasource): """Create a MODIS scene Args: hdf_url (str): URL for MODIS scene to download temp_directory (str): directory to use as scratch space when creating scene user_id (str): ID of owner for new MODIS scene datasource (str): ID of datasource for new MODIS scene """ config = modis_configs[datasource] granule_parts = os.path.basename(hdf_url).split('.') acquisition_datetime = datetime.strptime(granule_parts[1][1:], '%Y%j') name = '.'.join(granule_parts[:-1]) id = str(uuid.uuid4()) scene = Scene(Visibility.PRIVATE, [], datasource, {}, name, JobStatus.SUCCESS, JobStatus.SUCCESS, IngestStatus.INGESTED, [], owner=user_id, id=id, acquisitionDate=acquisition_datetime.isoformat() + 'Z', cloudCover=0) hdf_filepath = download_hdf(hdf_url, temp_directory) tifs = create_geotiffs(hdf_filepath, temp_directory) s3_uris = upload_tifs(tifs, user_id, scene.id) images = [] get_band_func = partial(get_image_band, modis_config=config) for local_path, remote_path in zip(tifs, s3_uris): image = create_geotiff_image(local_path, urllib.unquote(s3_uris[0]), scene=scene.id, owner=user_id, band_create_function=get_band_func) images.append(image) scene.images = images scene.ingestLocation = s3_uris[0] scene.sceneType = 'COG' return scene