def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CloudCoverageTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "cloud_coverage.png")
task.mosaic_path = os.path.join(task.get_result_path(), "mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = ['blue', 'green', 'red', 'cf_mask', 'total_pixels', 'total_clear',
'clear_percentage'] if 'cf_mask' in dataset else [
'blue', 'green', 'red', 'pixel_qa', 'total_pixels', 'total_clear', 'clear_percentage'
]
png_bands = ['red', 'green', 'blue']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('float64'), bands=bands)
write_png_from_xr(task.mosaic_path, dataset, bands=png_bands, scale=(0, 4096))
write_single_band_png_from_xr(task.result_path, dataset, band='clear_percentage', color_scale=task.color_scale_path)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
def generate_animation(index, combined_data):
base_index = (task.get_chunk_size()['time'] if task.get_chunk_size()['time'] is not None else 1) * index
for index in range((task.get_chunk_size()['time'] if task.get_chunk_size()['time'] is not None else 1)):
path = os.path.join(task.get_temp_path(), "animation_{}.nc".format(base_index + index))
if os.path.exists(path):
animated_data = xr.open_dataset(path, autoclose=True)
if task.animated_product.animation_id != "scene" and combined_data:
combine_intermediates(combined_data, animated_data)
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(base_index + index))
write_single_band_png_from_xr(
path,
animated_data,
task.animated_product.data_variable,
color_scale=task.color_scales[task.animated_product.data_variable],
fill_color=task.query_type.fill,
interpolate=False,
no_data=task.satellite.no_data_value)
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = TsmTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0]).astype('float64')
dataset['variability'] = dataset['max'] - dataset['normalized_data']
dataset['wofs'] = dataset.wofs / dataset.wofs_total_clean
nan_to_num(dataset, 0)
dataset_masked = mask_water_quality(dataset, dataset.wofs)
task.result_path = os.path.join(task.get_result_path(), "tsm.png")
task.clear_observations_path = os.path.join(task.get_result_path(), "clear_observations.png")
task.water_percentage_path = os.path.join(task.get_result_path(), "water_percentage.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset_masked)
task.metadata_from_dict(full_metadata)
bands = [task.query_type.data_variable, 'total_clean', 'wofs']
band_paths = [task.result_path, task.clear_observations_path, task.water_percentage_path]
dataset_masked.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset_masked, bands=bands, no_data=task.satellite.no_data_value)
for band, band_path in zip(bands, band_paths):
write_single_band_png_from_xr(
band_path,
dataset_masked,
band,
color_scale=task.color_scales[band],
fill_color='black',
interpolate=False,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_final_{}.nc".format(index))
if os.path.exists(path):
png_path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
animated_data = mask_water_quality(
xr.open_dataset(path).astype('float64'),
dataset.wofs) if task.animated_product.animation_id != "scene" else xr.open_dataset(
path)
write_single_band_png_from_xr(
png_path,
animated_data,
task.animated_product.data_variable,
color_scale=task.color_scales[task.animated_product.data_variable],
fill_color='black',
interpolate=False,
no_data=task.satellite.no_data_value)
image = imageio.imread(png_path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
task.update_bounds_from_dataset(dataset_masked)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
return True
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = SpectralIndicesTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0])
task.result_path = os.path.join(task.get_result_path(), "band_math.png")
task.mosaic_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['band_math']
export_xarray_to_netcdf(dataset, task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset.astype('int32'),
bands=bands,
no_data=task.satellite.no_data_value)
write_png_from_xr(task.mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_single_band_png_from_xr(task.result_path,
dataset,
band='band_math',
color_scale=task.color_scale_path.get(
task.query_type.result_id),
no_data=task.satellite.no_data_value)
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True).astype('float64')
task = WaterDetectionTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "water_percentage.png")
task.water_observations_path = os.path.join(task.get_result_path(), "water_observations.png")
task.clear_observations_path = os.path.join(task.get_result_path(), "clear_observations.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = ['normalized_data', 'total_data', 'total_clean']
band_paths = [task.result_path, task.water_observations_path, task.clear_observations_path]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset, bands=bands, no_data=task.satellite.no_data_value)
for band, band_path in zip(bands, band_paths):
write_single_band_png_from_xr(
band_path,
dataset,
band,
color_scale=task.color_scales[band],
fill_color=task.query_type.fill,
interpolate=False,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=[
task._get_field_as_list('clean_pixel_percentages_per_acquisition'), [
int(x) / max(int(y), 1)
for x, y in zip(
task._get_field_as_list('water_pixels_per_acquisition'),
task._get_field_as_list('clean_pixels_per_acquisition'))
]
],
data_labels=["Clean Pixel Percentage (%)", "Water Pixel Percentage (%)"],
titles=["Clean Pixel Percentage Per Acquisition", "Water Pixels Percentage Per Acquisition"])
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
