def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = FractionalCoverTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "band_math.png")
task.mosaic_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['pv', 'npv', 'bs']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset.astype('int32'),
bands=bands,
no_data=task.satellite.no_data_value)
write_png_from_xr(task.mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_png_from_xr(task.result_path, dataset, bands=['bs', 'pv', 'npv'])
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def generate_animation(index, combined_data):
base_index = (task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1) * index
for index in range((task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1)):
path = os.path.join(task.get_temp_path(),
"animation_{}.nc".format(base_index + index))
if os.path.exists(path):
animated_data = xr.open_dataset(path, autoclose=True)
if task.animated_product.animation_id == "cumulative":
animated_data = xr.concat([animated_data], 'time')
animated_data['time'] = [0]
clear_mask = create_cfmask_clean_mask(
animated_data.cf_mask
) if 'cf_mask' in animated_data else create_bit_mask(
animated_data.pixel_qa, [1, 2])
animated_data = task.get_processing_method()(
animated_data,
clean_mask=clear_mask,
intermediate_product=combined_data)
path = os.path.join(
task.get_temp_path(),
"animation_{}.png".format(base_index + index))
write_png_from_xr(path,
animated_data,
bands=[
task.query_type.red,
task.query_type.green,
task.query_type.blue
],
scale=(0, 4096))
def generate_animation(index, combined_data):
base_index = (task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1) * index
for index in range((task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1)):
path = os.path.join(task.get_temp_path(),
"animation_{}.nc".format(base_index + index))
if os.path.exists(path):
animated_data = xr.open_dataset(path)
if task.animated_product.animation_id == "cumulative":
animated_data = xr.concat([animated_data], 'time')
animated_data['time'] = [0]
clear_mask = task.satellite.get_clean_mask_func()(
animated_data)
animated_data = task.get_processing_method()(
animated_data,
clean_mask=clear_mask,
intermediate_product=combined_data,
no_data=task.satellite.no_data_value)
path = os.path.join(
task.get_temp_path(),
"animation_{}.png".format(base_index + index))
write_png_from_xr(path,
animated_data,
bands=[
task.query_type.red,
task.query_type.green,
task.query_type.blue
],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = SlipTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0])
task.result_path = os.path.join(task.get_result_path(), "slip_result.png")
task.result_mosaic_path = os.path.join(task.get_result_path(), "mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['slip']
export_xarray_to_netcdf(dataset, task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
write_png_from_xr(
task.result_path,
mask_mosaic_with_slip(dataset),
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_png_from_xr(
task.result_mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=[
task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
task._get_field_as_list('slip_pixels_per_acquisition')
],
data_labels=["Clean Pixel Percentage (%)", "SLIP Pixel Count (#)"],
titles=["Clean Pixel Percentage Per Acquisition", "SLIP Pixels Percentage Per Acquisition"])
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
return True
def recombine_geographic_chunks(chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
logger.info("RECOMBINE_GEO")
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
task = CoastalChangeTask.objects.get(pk=task_id)
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0], autoclose=True))
combined_data = combine_geographic_chunks(chunk_data)
if task.animated_product.animation_id != "none":
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(time_chunk_id))
animated_data = mask_mosaic_with_coastlines(
combined_data
) if task.animated_product.animation_id == "coastline_change" else mask_mosaic_with_coastal_change(
combined_data)
write_png_from_xr(path,
animated_data,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
path = os.path.join(task.get_temp_path(),
"recombined_geo_{}.nc".format(time_chunk_id))
combined_data.to_netcdf(path)
logger.info("Done combining geographic chunks for time: " +
str(time_chunk_id))
return path, metadata, {
'geo_chunk_id': geo_chunk_id,
'time_chunk_id': time_chunk_id
}
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CoastalChangeTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "coastline_change.png")
task.result_coastal_change_path = os.path.join(task.get_result_path(), "coastal_change.png")
task.result_mosaic_path = os.path.join(task.get_result_path(), "mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'cf_mask', 'coastal_change', 'coastline_old', 'coastline_new'
] if 'cf_mask' in dataset else [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa', 'coastal_change', 'coastline_old', 'coastline_new'
]
png_bands = ['red', 'green', 'blue']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands)
write_png_from_xr(task.result_path, mask_mosaic_with_coastlines(dataset), bands=png_bands, scale=(0, 4096))
write_png_from_xr(
task.result_coastal_change_path, mask_mosaic_with_coastal_change(dataset), bands=png_bands, scale=(0, 4096))
write_png_from_xr(task.result_mosaic_path, dataset, bands=png_bands, scale=(0, 4096))
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
for index in range(task.time_end - task.time_start):
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = AppNameTask.objects.get(pk=task_id)
# TODO: Add any paths that you've added in your models.py Result model and remove the ones that aren't there.
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.result_filled_path = os.path.join(task.get_result_path(),
"filled_png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(
), "animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
# TODO: Set the bands that should be written to the final products
bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'cf_mask'
] if 'cf_mask' in dataset else [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'
]
# TODO: If you're creating pngs, specify the RGB bands
png_bands = [
task.query_type.red, task.query_type.green, task.query_type.blue
]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands)
write_png_from_xr(task.result_path,
dataset,
bands=png_bands,
png_filled_path=task.result_filled_path,
fill_color=task.query_type.fill,
scale=(0, 4096))
# TODO: if there is no animation, remove this. Otherwise, open each time iteration slice and write to disk.
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I',
duration=1.0) as writer:
valid_range = reversed(
range(len(full_metadata))
) if task.animated_product.animation_id == "scene" and task.get_reverse_time(
) else range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
# TODO: if you're capturing more tabular metadata, plot it here by converting these to lists.
# an example of this is the current water detection app.
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CustomMosaicToolTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.result_filled_path = os.path.join(task.get_result_path(), "filled_png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements()
png_bands = [task.query_type.red, task.query_type.green, task.query_type.blue]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
write_png_from_xr(
task.result_path,
dataset,
bands=png_bands,
png_filled_path=task.result_filled_path,
fill_color=task.query_type.fill,
scale=task.satellite.get_scale(),
low_res=True,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = reversed(
range(len(full_metadata))) if task.animated_product.animation_id == "scene" and task.get_reverse_time(
) else range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
# print(max_)
# ndviR, ndviG, ndviB = ndvi
# ndviR = [255 if element<0.5 else 0 for element in ndvi]
# ndviG = [0 if element<0 else 255 for element in ndvi]
# ndviB = [0 if element> -10 for element in ndvi]
# for i in range(len(ndvi)):
# for j in range(len(ndvi[i])):
# if ndvi[i][j]
# for x in ndviR:
# if x<-0.25
# print(" MAX " + np.amax(np.amax(ndvi, axis=0)), axis=1)
write_png_from_xr('static/assets/results/urbanization/false_color.png',
ds_ndvi, ["NDVI", "NDVI", "NDVI"],
scale=[(min_, max_), (min_, max_), (min_, max_)])
elif sys.argv[9] == "NDBI":
ndbi = NDBI(landsat_dataset)[0] # Urbanization
ds_ndbi = ndbi.to_dataset(name="NDBI")
max_ = ndbi.max(axis=0).max(axis=0).values.item()
min_ = ndbi.min(axis=0).min(axis=0).values.item()
write_png_from_xr('static/assets/results/urbanization/false_color.png',
ds_ndbi, ["NDBI", "NDBI", "NDBI"],
scale=[(min_, max_), (min_, max_), (min_, max_)])
elif sys.argv[9] == "NDWI":
ndwi = NDWI(landsat_dataset)[0] # High Concentrations of Water
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = SpectralIndicesTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0])
task.result_path = os.path.join(task.get_result_path(), "band_math.png")
task.mosaic_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['band_math']
export_xarray_to_netcdf(dataset, task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
# Ensure data variables have the range of Landsat 7 Collection 1 Level 2
# since the color scales are tailored for that dataset.
platform = task.satellite.platform
collection = task.satellite.collection
level = task.satellite.level
if (platform, collection) != ('LANDSAT_7', 'c1'):
dataset = \
convert_range(dataset, from_platform=platform,
from_collection=collection, from_level=level,
to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
write_png_from_xr(
task.mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_single_band_png_from_xr(
task.result_path,
dataset,
band='band_math',
color_scale=task.color_scale_path.get(task.query_type.result_id),
no_data=task.satellite.no_data_value)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
return True
else:
desired_bands = ['red', 'green', 'nir', 'swir2', 'pixel_qa'
] # needed by ndvi, ndwi, ndbi and cloud masking
desired_bands = desired_bands + [
'blue'
] # blue is needed for a true color visualization purposes
landsat_dataset = dc.load(product=sys.argv[5],
platform=sys.argv[6],
lat=(sys.argv[1], sys.argv[2]),
lon=(sys.argv[3], sys.argv[4]),
time=(sys.argv[7], sys.argv[8]),
measurements=desired_bands)
if product == 'ls8_ingested':
landsat_mosaic = landsat_dataset
else:
landsat_mosaic = median_mosaic(landsat_dataset)
# print(landsat_mosaic)
# <br>
# > **Saving your data**
# > A .tiff or png is a great way to represent true color mosaics. The image below is a saved .png representation of of a landsat mosaic.
# In[12]:
write_png_from_xr('static/assets/results/cloud_free/cloud_free.png',
landsat_mosaic, ["red", "green", "blue"],
scale=[(0, 2000), (0, 2000), (0, 2000)])
# <br>
# In[11]:
landsat_mosaic = median_mosaic(landsat_dataset)
print(landsat_mosaic)
# <br>
# > **Saving your data**
# > A .tiff or png is a great way to represent true color mosaics. The image below is a saved .png representation of of a landsat mosaic.
# In[12]:
from utils.data_cube_utilities.dc_utilities import write_png_from_xr
write_png_from_xr('diagrams/urbanization/cloud_free_mosaic.png',
landsat_mosaic, ["red", "green", "blue"],
scale=[(0, 2000), (0, 2000), (0, 2000)])
# <br>
#
# 
#
# <br>
# <br>
# # Urbanization Analysis
# <br>
#
# > **NDWI, NDVI, NDBI**
# You will very rarely have urban classification and water classifications apply to the same pixel. For urban analysis, it may make sense to compute not just urban classes, but classes that are unlikely to co-occur with urbanization.
#
