def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True).astype('float64')
dataset['variability'] = dataset['max'] - dataset['normalized_data']
dataset['wofs'] = dataset.wofs / dataset.wofs_total_clean
nan_to_num(dataset, 0)
dataset_masked = mask_water_quality(dataset, dataset.wofs)
task = TsmTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "tsm.png")
task.clear_observations_path = os.path.join(task.get_result_path(),
"clear_observations.png")
task.water_percentage_path = os.path.join(task.get_result_path(),
"water_percentage.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(
), "animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset_masked)
task.metadata_from_dict(full_metadata)
bands = [task.query_type.data_variable, 'total_clean', 'wofs']
band_paths = [
task.result_path, task.clear_observations_path,
task.water_percentage_path
]
dataset_masked.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset_masked,
bands=bands,
no_data=task.satellite.no_data_value)
for band, band_path in zip(bands, band_paths):
write_single_band_png_from_xr(band_path,
dataset_masked,
band,
color_scale=task.color_scales[band],
fill_color='black',
interpolate=False,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I',
duration=1.0) as writer:
valid_range = range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(),
"animation_final_{}.nc".format(index))
if os.path.exists(path):
png_path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(index))
animated_data = mask_water_quality(
xr.open_dataset(path,
autoclose=True).astype('float64'),
dataset.wofs
) if task.animated_product.animation_id != "scene" else xr.open_dataset(
path, autoclose=True)
write_single_band_png_from_xr(
png_path,
animated_data,
task.animated_product.data_variable,
color_scale=task.color_scales[
task.animated_product.data_variable],
fill_color='black',
interpolate=False,
no_data=task.satellite.no_data_value)
image = imageio.imread(png_path)
writer.append_data(image)
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
task.rewrite_pathnames()
task.update_bounds_from_dataset(dataset_masked)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CoastalChangeTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(),
"coastline_change.png")
task.result_coastal_change_path = os.path.join(task.get_result_path(),
"coastal_change.png")
task.result_mosaic_path = os.path.join(task.get_result_path(),
"mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(
), "animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + [
'coastal_change', 'coastline_old', 'coastline_new'
]
png_bands = ['red', 'green', 'blue']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset.astype('int32'),
bands=bands,
no_data=task.satellite.no_data_value)
write_png_from_xr(task.result_path,
mask_mosaic_with_coastlines(dataset),
bands=png_bands,
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_png_from_xr(task.result_coastal_change_path,
mask_mosaic_with_coastal_change(dataset),
bands=png_bands,
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_png_from_xr(task.result_mosaic_path,
dataset,
bands=png_bands,
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I',
duration=1.0) as writer:
for index in range(task.time_end - task.time_start):
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
logger.info("All products created.")
task.rewrite_pathnames()
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = SpectralIndicesTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "band_math.png")
task.mosaic_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['band_math']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset.astype('int32'),
bands=bands,
no_data=task.satellite.no_data_value)
write_png_from_xr(task.mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_single_band_png_from_xr(task.result_path,
dataset,
band='band_math',
color_scale=task.color_scale_path.get(
task.query_type.result_id),
no_data=task.satellite.no_data_value)
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
task.rewrite_pathnames()
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CustomMosaicToolTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.result_filled_path = os.path.join(task.get_result_path(),
"filled_png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(
), "animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements()
png_bands = [
task.query_type.red, task.query_type.green, task.query_type.blue
]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset.astype('int32'),
bands=bands,
no_data=task.satellite.no_data_value)
write_png_from_xr(task.result_path,
dataset,
bands=png_bands,
png_filled_path=task.result_filled_path,
fill_color=task.query_type.fill,
scale=task.satellite.get_scale(),
low_res=True,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I',
duration=1.0) as writer:
valid_range = reversed(
range(len(full_metadata))
) if task.animated_product.animation_id == "scene" and task.get_reverse_time(
) else range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
task.rewrite_pathnames()
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True