def create_scene(owner, prefix, landsat_id, config, datasource):
logger.info('Creating scene for landsat id {}'.format(landsat_id))
gcs_prefix = io.gcs_path_for_landsat_id(landsat_id)
metadata_resp = requests.get(io.make_path_for_mtl(gcs_prefix, landsat_id))
if metadata_resp.status_code == 404:
logger.error('Landsat scene %s is not available yet in GCS',
landsat_id)
raise Exception('Could not find landsat scene %s', landsat_id)
filter_metadata = extract_metadata(metadata_resp.content)
(filename, cog_fname) = process_to_cog(prefix, gcs_prefix, landsat_id,
config)
s3_location = upload_file(owner, filename, cog_fname)
logger.info('Creating image')
ingest_location = 's3://{}/{}'.format(data_bucket,
urllib.quote(s3_location))
scene = Scene('PRIVATE', [],
datasource, {},
landsat_id,
'SUCCESS',
'SUCCESS',
'INGESTED', [io.make_path_for_mtl(gcs_prefix, landsat_id)],
ingestLocation=ingest_location,
cloudCover=filter_metadata['cloud_cover'],
acquisitionDate=filter_metadata['acquisition_date'],
sceneType='COG',
owner=owner)
image = create_geotiff_image(
filename,
ingest_location,
filename=cog_fname,
owner=owner,
scene=scene.id,
band_create_function=lambda x: config.bands.values())
scene.images = [image]
return scene
def create_scene(owner, prefix, landsat_id, config, datasource):
logger.info('Creating scene for landsat id {}'.format(landsat_id))
gcs_prefix = io.gcs_path_for_landsat_id(landsat_id)
metadata_resp = requests.get(io.make_path_for_mtl(gcs_prefix, landsat_id))
if metadata_resp.status_code == 404:
logger.error('Landsat scene %s is not available yet in GCS',
landsat_id)
raise Exception('Could not find landsat scene %s', landsat_id)
filter_metadata = extract_metadata(metadata_resp.content)
(filename, cog_fname) = process_to_cog(prefix, gcs_prefix, landsat_id, config)
s3_location = upload_file(owner, filename, cog_fname)
logger.info('Creating image')
ingest_location = 's3://{}/{}'.format(data_bucket,
urllib.quote(s3_location))
scene = Scene(
'PRIVATE', [],
datasource, {},
landsat_id,
'SUCCESS',
'SUCCESS',
'INGESTED', [io.make_path_for_mtl(gcs_prefix, landsat_id)],
ingestLocation=ingest_location,
cloudCover=filter_metadata['cloud_cover'],
acquisitionDate=filter_metadata['acquisition_date'],
sceneType='COG',
owner=owner)
image = create_geotiff_image(
filename,
ingest_location,
filename=cog_fname,
owner=owner,
scene=scene.id,
band_create_function=lambda x: config.bands.values())
scene.images = [image]
return scene
def create_scene(hdf_url, temp_directory, user_id, datasource):
"""Create a MODIS scene
Args:
hdf_url (str): URL for MODIS scene to download
temp_directory (str): directory to use as scratch space when creating scene
user_id (str): ID of owner for new MODIS scene
datasource (str): ID of datasource for new MODIS scene
"""
config = modis_configs[datasource]
granule_parts = os.path.basename(hdf_url).split('.')
acquisition_datetime = datetime.strptime(granule_parts[1][1:], '%Y%j')
name = '.'.join(granule_parts[:-1])
id = str(uuid.uuid4())
scene = Scene(0,
Visibility.PRIVATE, [],
datasource, {},
name,
JobStatus.SUCCESS,
JobStatus.SUCCESS,
IngestStatus.INGESTED, [],
owner=user_id,
id=id,
acquisitionDate=acquisition_datetime.isoformat() + 'Z',
cloudCover=0)
hdf_directory = os.path.join(temp_directory, 'hdf')
os.mkdir(hdf_directory)
hdf_filepath = download_hdf(hdf_url, hdf_directory)
tiff_directory = os.path.join(temp_directory, 'tiffs')
os.mkdir(tiff_directory)
tifs = create_geotiffs(hdf_filepath, tiff_directory)
s3_uris = upload_tifs(tifs, user_id, scene.id)
images = []
get_band_func = partial(get_image_band, modis_config=config)
for local_path, remote_path in zip(tifs, s3_uris):
image = create_geotiff_image(local_path,
remote_path,
scene=scene.id,
owner=user_id,
band_create_function=get_band_func)
images.append(image)
scene.images = images
scene.ingestLocation = s3_uris[0]
scene.sceneType = 'COG'
return scene
def create_scene(owner, prefix, landsat_id, config, datasource):
logger.info('Creating scene for landsat id {}'.format(landsat_id))
gcs_prefix = io.gcs_path_for_landsat_id(landsat_id)
logger.info('Fetching all bands')
for band in config.bands.keys():
fetch_band(prefix, gcs_prefix, band, landsat_id)
filter_metadata = extract_metadata(
requests.get(io.make_path_for_mtl(gcs_prefix, landsat_id)).content)
cog_fname = '{}_COG.tif'.format(landsat_id)
stacked_fname = '{}_STACKED.tif'.format(landsat_id)
filenames = {
'COG': os.path.join(prefix, cog_fname),
'STACKED': os.path.join(prefix, stacked_fname)
}
convert_to_cog(prefix, filenames['STACKED'], filenames['COG'], config,
landsat_id)
s3_location = upload_file(owner, filenames['COG'], cog_fname)
logger.info('Creating image')
ingest_location = 's3://{}/{}'.format(data_bucket,
urllib.quote(s3_location))
scene = Scene(0,
'PRIVATE', [],
datasource, {},
landsat_id,
'SUCCESS',
'SUCCESS',
'INGESTED', [io.make_path_for_mtl(gcs_prefix, landsat_id)],
ingestLocation=ingest_location,
cloudCover=filter_metadata['cloud_cover'],
acquisitionDate=filter_metadata['acquisition_date'],
sceneType='COG',
owner=owner)
image = create_geotiff_image(
filenames['COG'],
ingest_location,
filename=cog_fname,
owner=owner,
scene=scene.id,
band_create_function=lambda x: config.bands.values())
scene.images = [image]
return scene
def create_scene(hdf_url, temp_directory, user_id, datasource):
"""Create a MODIS scene
Args:
hdf_url (str): URL for MODIS scene to download
temp_directory (str): directory to use as scratch space when creating scene
user_id (str): ID of owner for new MODIS scene
datasource (str): ID of datasource for new MODIS scene
"""
config = modis_configs[datasource]
granule_parts = os.path.basename(hdf_url).split('.')
acquisition_datetime = datetime.strptime(granule_parts[1][1:], '%Y%j')
name = '.'.join(granule_parts[:-1])
id = str(uuid.uuid4())
scene = Scene(Visibility.PRIVATE, [], datasource, {}, name,
JobStatus.SUCCESS, JobStatus.SUCCESS, IngestStatus.INGESTED, [], owner=user_id, id=id,
acquisitionDate=acquisition_datetime.isoformat() + 'Z', cloudCover=0)
hdf_filepath = download_hdf(hdf_url, temp_directory)
tifs = create_geotiffs(hdf_filepath, temp_directory)
s3_uris = upload_tifs(tifs, user_id, scene.id)
images = []
get_band_func = partial(get_image_band, modis_config=config)
for local_path, remote_path in zip(tifs, s3_uris):
image = create_geotiff_image(local_path, urllib.unquote(s3_uris[0]),
scene=scene.id, owner=user_id, band_create_function=get_band_func)
images.append(image)
scene.images = images
scene.ingestLocation = s3_uris[0]
scene.sceneType = 'COG'
return scene