def generate_animation(index, combined_data):
base_index = (task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1) * index
for index in range((task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1)):
path = os.path.join(task.get_temp_path(),
"animation_{}.nc".format(base_index + index))
if os.path.exists(path):
animated_data = xr.open_dataset(path, autoclose=True)
if task.animated_product.animation_id == "cumulative":
animated_data = xr.concat([animated_data], 'time')
animated_data['time'] = [0]
clear_mask = create_cfmask_clean_mask(
animated_data.cf_mask
) if 'cf_mask' in animated_data else create_bit_mask(
animated_data.pixel_qa, [1, 2])
animated_data = task.get_processing_method()(
animated_data,
clean_mask=clear_mask,
intermediate_product=combined_data)
path = os.path.join(
task.get_temp_path(),
"animation_{}.png".format(base_index + index))
write_png_from_xr(path,
animated_data,
bands=[
task.query_type.red,
task.query_type.green,
task.query_type.blue
],
scale=(0, 4096))
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = SlipTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "slip_result.png")
task.result_mosaic_path = os.path.join(task.get_result_path(), "mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'cf_mask',
'slip'] if 'cf_mask' in dataset else ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa', 'slip']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands)
write_png_from_xr(task.result_path, mask_mosaic_with_slip(dataset), bands=['red', 'green', 'blue'], scale=(0, 4096))
write_png_from_xr(task.result_mosaic_path, dataset, bands=['red', 'green', 'blue'], scale=(0, 4096))
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=[
task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
task._get_field_as_list('slip_pixels_per_acquisition')
],
data_labels=["Clean Pixel Percentage (%)", "SLIP Pixel Count (#)"],
titles=["Clean Pixel Percentage Per Acquisition", "SLIP Pixels Percentage Per Acquisition"])
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
def recombine_geographic_chunks(chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
logger.info("RECOMBINE_GEO")
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
task = CoastalChangeTask.objects.get(pk=task_id)
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0], autoclose=True))
combined_data = combine_geographic_chunks(chunk_data)
if task.animated_product.animation_id != "none":
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(time_chunk_id))
animated_data = mask_mosaic_with_coastlines(
combined_data
) if task.animated_product.animation_id == "coastline_change" else mask_mosaic_with_coastal_change(
combined_data)
write_png_from_xr(path,
animated_data,
bands=['red', 'green', 'blue'],
scale=(0, 4096))
path = os.path.join(task.get_temp_path(),
"recombined_geo_{}.nc".format(time_chunk_id))
combined_data.to_netcdf(path)
logger.info("Done combining geographic chunks for time: " +
str(time_chunk_id))
return path, metadata, {
'geo_chunk_id': geo_chunk_id,
'time_chunk_id': time_chunk_id
}
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CustomMosaicToolTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.result_filled_path = os.path.join(task.get_result_path(), "filled_png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2',
'cf_mask'] if 'cf_mask' in dataset else ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']
png_bands = [task.query_type.red, task.query_type.green, task.query_type.blue]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands)
write_png_from_xr(
task.result_path,
dataset,
bands=png_bands,
png_filled_path=task.result_filled_path,
fill_color=task.query_type.fill,
scale=(0, 4096),
low_res=True)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = reversed(
range(len(full_metadata))) if task.animated_product.animation_id == "scene" and task.get_reverse_time(
) else range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CoastalChangeTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(),
"coastline_change.png")
task.result_coastal_change_path = os.path.join(task.get_result_path(),
"coastal_change.png")
task.result_mosaic_path = os.path.join(task.get_result_path(),
"mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(
), "animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'cf_mask',
'coastal_change', 'coastline_old', 'coastline_new'
] if 'cf_mask' in dataset else [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa',
'coastal_change', 'coastline_old', 'coastline_new'
]
png_bands = ['red', 'green', 'blue']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands)
write_png_from_xr(task.result_path,
mask_mosaic_with_coastlines(dataset),
bands=png_bands,
scale=(0, 4096))
write_png_from_xr(task.result_coastal_change_path,
mask_mosaic_with_coastal_change(dataset),
bands=png_bands,
scale=(0, 4096))
write_png_from_xr(task.result_mosaic_path,
dataset,
bands=png_bands,
scale=(0, 4096))
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I',
duration=1.0) as writer:
for index in range(task.time_end - task.time_start):
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
