def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = CustomMosaicToolTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters).isel(latitude=0, longitude=0)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
exclusion_list = ['satellite', 'pixel_qa']
plot_measurements = [band for band in single_pixel.data_vars if band not in exclusion_list]
datasets = [single_pixel[band].values.transpose() for band in plot_measurements] + [clear_mask]
data_labels = [stringcase.titlecase("{} Units".format(band)) for band in plot_measurements] + ["Clear"]
titles = [stringcase.titlecase("{} Band".format(band)) for band in plot_measurements] + ["Clear Mask"]
style = ['r-o', 'g-o', 'b-o', 'c-o', 'm-o', 'y-o', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = WaterDetectionTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel.isel(latitude=0, longitude=0))
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
wofs_data = task.get_processing_method()(single_pixel,
clean_mask=clear_mask,
enforce_float64=True,
no_data=task.satellite.no_data_value)
wofs_data = wofs_data.where(wofs_data != task.satellite.no_data_value).isel(latitude=0, longitude=0)
datasets = [wofs_data.wofs.values.transpose()] + [clear_mask]
data_labels = ["Water/Non Water"] + ["Clear"]
titles = ["Water/Non Water"] + ["Clear Mask"]
style = ['.', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = UrbanizationTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_dataset_by_extent(**parameters).isel(latitude=0, longitude=0)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
datasets = [data_array.values.transpose() for data_array in _apply_band_math(single_pixel)] + [clear_mask]
data_labels = ["NDVI", "NDWI", "NDBI"] + ["Clear"]
titles = ["Dense Vegetatin (NDVI)", "Water Concentration (NDWI)", "Urbanization (NDBI)", 'Clear Mask']
style = ['go', 'bo', 'ro', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = TsmTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
# Ensure data variables have the range of Landsat 7 Collection 1 Level 2
# since the color scales are tailored for that dataset.
platform = task.satellite.platform
collection = task.satellite.collection
level = task.satellite.level
if (platform, collection) != ('LANDSAT_7', 'c1'):
single_pixel = \
convert_range(single_pixel, from_platform=platform,
from_collection=collection, from_level=level,
to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
wofs_data = task.get_processing_method()(single_pixel,
clean_mask=clear_mask,
no_data=task.satellite.no_data_value)
wofs_data = \
wofs_data.where(wofs_data != task.satellite.no_data_value)
wofs_data = wofs_data.squeeze()
tsm_data = \
tsm(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value)
tsm_data = \
tsm_data.where(tsm_data != task.satellite.no_data_value)\
.squeeze().where(wofs_data.wofs.values == 1)
# Remove NaNs to avoid errors and yield a nicer plot.
water_non_nan_times = ~np.isnan(wofs_data.wofs.values)
wofs_data = wofs_data.isel(time=water_non_nan_times)
tsm_non_nan_times = ~np.isnan(tsm_data.tsm.values)
tsm_data = tsm_data.isel(time=tsm_non_nan_times)
datasets = [wofs_data.wofs.values.transpose().squeeze(),
tsm_data.tsm.values.transpose().squeeze()] + \
[clear_mask.squeeze()]
dates = [dates[water_non_nan_times], dates[tsm_non_nan_times]] + [dates]
data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"]
titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"]
style = ['.', 'ro', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = FractionalCoverTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "band_math.png")
task.mosaic_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['pv', 'npv', 'bs']
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
dataset.astype('int32'),
bands=bands,
no_data=task.satellite.no_data_value)
write_png_from_xr(task.mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_png_from_xr(task.result_path, dataset, bands=['bs', 'pv', 'npv'])
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = SlipTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0])
task.result_path = os.path.join(task.get_result_path(), "slip_result.png")
task.result_mosaic_path = os.path.join(task.get_result_path(), "mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['slip']
export_xarray_to_netcdf(dataset, task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
write_png_from_xr(
task.result_path,
mask_mosaic_with_slip(dataset),
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_png_from_xr(
task.result_mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=[
task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
task._get_field_as_list('slip_pixels_per_acquisition')
],
data_labels=["Clean Pixel Percentage (%)", "SLIP Pixel Count (#)"],
titles=["Clean Pixel Percentage Per Acquisition", "SLIP Pixels Percentage Per Acquisition"])
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
return True
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = FractionalCoverTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel.isel(latitude=0, longitude=0))
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
def _apply_band_math(ds, idx):
# mask out water manually. Necessary for frac. cover.
wofs = wofs_classify(ds, clean_mask=clear_mask[idx], mosaic=True)
clear_mask[idx] = False if wofs.wofs.values[0] == 1 else clear_mask[idx]
fractional_cover = frac_coverage_classify(ds, clean_mask=clear_mask[idx], no_data=task.satellite.no_data_value)
return fractional_cover
fractional_cover = xr.concat(
[
_apply_band_math(single_pixel.isel(time=data_point, drop=True), data_point)
for data_point in range(len(dates))
],
dim='time')
fractional_cover = fractional_cover.where(fractional_cover != task.satellite.no_data_value).isel(
latitude=0, longitude=0)
exclusion_list = []
plot_measurements = [band for band in fractional_cover.data_vars if band not in exclusion_list]
datasets = [fractional_cover[band].values.transpose() for band in plot_measurements] + [clear_mask]
data_labels = [stringcase.titlecase("%{}".format(band)) for band in plot_measurements] + ["Clear"]
titles = [
'Bare Soil Percentage', 'Photosynthetic Vegetation Percentage', 'Non-Photosynthetic Vegetation Percentage',
'Clear Mask'
]
style = ['ro', 'go', 'bo', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = TsmTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
wofs_data = task.get_processing_method()(single_pixel,
clean_mask=clear_mask,
no_data=task.satellite.no_data_value)
wofs_data = \
wofs_data.where(wofs_data != task.satellite.no_data_value)
wofs_data = wofs_data.squeeze()
tsm_data = \
tsm(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value)
tsm_data = \
tsm_data.where(tsm_data != task.satellite.no_data_value)\
.squeeze().where(wofs_data.wofs.values == 1)
# Remove NaNs to avoid errors and yield a nicer plot.
water_non_nan_times = ~np.isnan(wofs_data.wofs.values)
wofs_data = wofs_data.isel(time=water_non_nan_times)
tsm_non_nan_times = ~np.isnan(tsm_data.tsm.values)
tsm_data = tsm_data.isel(time=tsm_non_nan_times)
datasets = [wofs_data.wofs.values.transpose().squeeze(),
tsm_data.tsm.values.transpose().squeeze()] + \
[clear_mask.squeeze()]
dates = [dates[water_non_nan_times], dates[tsm_non_nan_times]] + [dates]
data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"]
titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"]
style = ['.', 'ro', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = BandMathTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_dataset_by_extent(**parameters).isel(latitude=0,
longitude=0)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(
single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status(
"ERROR",
"There is only a single acquisition for your parameter set.")
return None
def _apply_band_math(dataset):
#TODO: apply your band math here!
return (dataset.nir - dataset.red) / (dataset.nir + dataset.red)
datasets = [_apply_band_math(single_pixel).values.transpose()
] + [clear_mask]
data_labels = ["Band Math Result"] + ["Clear"]
titles = ["Band Math"] + ["Clear Mask"]
style = ['ro', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=datasets,
data_labels=data_labels,
titles=titles,
style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = TsmTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0]).astype('float64')
dataset['variability'] = dataset['max'] - dataset['normalized_data']
dataset['wofs'] = dataset.wofs / dataset.wofs_total_clean
nan_to_num(dataset, 0)
dataset_masked = mask_water_quality(dataset, dataset.wofs)
task.result_path = os.path.join(task.get_result_path(), "tsm.png")
task.clear_observations_path = os.path.join(task.get_result_path(), "clear_observations.png")
task.water_percentage_path = os.path.join(task.get_result_path(), "water_percentage.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset_masked)
task.metadata_from_dict(full_metadata)
bands = [task.query_type.data_variable, 'total_clean', 'wofs']
band_paths = [task.result_path, task.clear_observations_path, task.water_percentage_path]
dataset_masked.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset_masked, bands=bands, no_data=task.satellite.no_data_value)
for band, band_path in zip(bands, band_paths):
write_single_band_png_from_xr(
band_path,
dataset_masked,
band,
color_scale=task.color_scales[band],
fill_color='black',
interpolate=False,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_final_{}.nc".format(index))
if os.path.exists(path):
png_path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
animated_data = mask_water_quality(
xr.open_dataset(path).astype('float64'),
dataset.wofs) if task.animated_product.animation_id != "scene" else xr.open_dataset(
path)
write_single_band_png_from_xr(
png_path,
animated_data,
task.animated_product.data_variable,
color_scale=task.color_scales[task.animated_product.data_variable],
fill_color='black',
interpolate=False,
no_data=task.satellite.no_data_value)
image = imageio.imread(png_path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
task.update_bounds_from_dataset(dataset_masked)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = AppNameTask.objects.get(pk=task_id)
# TODO: Add any paths that you've added in your models.py Result model and remove the ones that aren't there.
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.result_filled_path = os.path.join(task.get_result_path(),
"filled_png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(
), "animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
# TODO: Set the bands that should be written to the final products
bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'cf_mask'
] if 'cf_mask' in dataset else [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'
]
# TODO: If you're creating pngs, specify the RGB bands
png_bands = [
task.query_type.red, task.query_type.green, task.query_type.blue
]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands)
write_png_from_xr(task.result_path,
dataset,
bands=png_bands,
png_filled_path=task.result_filled_path,
fill_color=task.query_type.fill,
scale=(0, 4096))
# TODO: if there is no animation, remove this. Otherwise, open each time iteration slice and write to disk.
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I',
duration=1.0) as writer:
valid_range = reversed(
range(len(full_metadata))
) if task.animated_product.animation_id == "scene" and task.get_reverse_time(
) else range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
# TODO: if you're capturing more tabular metadata, plot it here by converting these to lists.
# an example of this is the current water detection app.
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
shutil.rmtree(task.get_temp_path())
return True
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = SpectralIndicesTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_dataset_by_extent(**parameters).isel(latitude=0,
longitude=0)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(
single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status(
"ERROR",
"There is only a single acquisition for your parameter set.")
return None
# spectral_indices_map = {
# 'ndvi': lambda ds: (ds.nir - ds.red) / (ds.nir + ds.red),
# 'evi': lambda ds: 2.5 * (ds.nir - ds.red) / (ds.nir + 6 * ds.red - 7.5 * ds.blue + 1),
# 'savi': lambda ds: (ds.nir - ds.red) / (ds.nir + ds.red + 0.5) * (1.5),
# 'nbr': lambda ds: (ds.nir - ds.swir2) / (ds.nir + ds.swir2),
# 'nbr2': lambda ds: (ds.swir1 - ds.swir2) / (ds.swir1 + ds.swir2),
# 'ndwi': lambda ds: (ds.nir - ds.swir1) / (ds.nir + ds.swir1),
# 'ndbi': lambda ds: (ds.swir1 - ds.nir) / (ds.nir + ds.swir1),
# }
for spectral_index in spectral_indices_map:
single_pixel[spectral_index] = spectral_indices_map[spectral_index](
single_pixel)
exclusion_list = task.satellite.get_measurements()
plot_measurements = [
band for band in single_pixel.data_vars if band not in exclusion_list
]
datasets = [
single_pixel[band].values.transpose() for band in plot_measurements
] + [clear_mask]
data_labels = [
stringcase.uppercase("{}".format(band)) for band in plot_measurements
] + ["Clear"]
titles = [
stringcase.uppercase("{}".format(band)) for band in plot_measurements
] + ["Clear Mask"]
style = ['ro', 'go', 'bo', 'co', 'mo', 'yo', 'ko', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=datasets,
data_labels=data_labels,
titles=titles,
style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True)
task = CustomMosaicToolTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.result_filled_path = os.path.join(task.get_result_path(), "filled_png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements()
png_bands = [task.query_type.red, task.query_type.green, task.query_type.blue]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
write_png_from_xr(
task.result_path,
dataset,
bands=png_bands,
png_filled_path=task.result_filled_path,
fill_color=task.query_type.fill,
scale=task.satellite.get_scale(),
low_res=True,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = reversed(
range(len(full_metadata))) if task.animated_product.animation_id == "scene" and task.get_reverse_time(
) else range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
logger.info("CREATE_OUTPUT")
full_metadata = data[1]
dataset = xr.open_dataset(data[0], autoclose=True).astype('float64')
task = WaterDetectionTask.objects.get(pk=task_id)
task.result_path = os.path.join(task.get_result_path(), "water_percentage.png")
task.water_observations_path = os.path.join(task.get_result_path(), "water_observations.png")
task.clear_observations_path = os.path.join(task.get_result_path(), "clear_observations.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.animation_path = os.path.join(task.get_result_path(),
"animation.gif") if task.animated_product.animation_id != 'none' else ""
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = ['normalized_data', 'total_data', 'total_clean']
band_paths = [task.result_path, task.water_observations_path, task.clear_observations_path]
dataset.to_netcdf(task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset, bands=bands, no_data=task.satellite.no_data_value)
for band, band_path in zip(bands, band_paths):
write_single_band_png_from_xr(
band_path,
dataset,
band,
color_scale=task.color_scales[band],
fill_color=task.query_type.fill,
interpolate=False,
no_data=task.satellite.no_data_value)
if task.animated_product.animation_id != "none":
with imageio.get_writer(task.animation_path, mode='I', duration=1.0) as writer:
valid_range = range(len(full_metadata))
for index in valid_range:
path = os.path.join(task.get_temp_path(), "animation_{}.png".format(index))
if os.path.exists(path):
image = imageio.imread(path)
writer.append_data(image)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=[
task._get_field_as_list('clean_pixel_percentages_per_acquisition'), [
int(x) / max(int(y), 1)
for x, y in zip(
task._get_field_as_list('water_pixels_per_acquisition'),
task._get_field_as_list('clean_pixels_per_acquisition'))
]
],
data_labels=["Clean Pixel Percentage (%)", "Water Pixel Percentage (%)"],
titles=["Clean Pixel Percentage Per Acquisition", "Water Pixels Percentage Per Acquisition"])
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
shutil.rmtree(task.get_temp_path())
return True
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to variuos formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
task = SpectralIndicesTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
full_metadata = data[1]
dataset = xr.open_dataset(data[0])
task.result_path = os.path.join(task.get_result_path(), "band_math.png")
task.mosaic_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.data_netcdf_path = os.path.join(task.get_result_path(), "data_netcdf.nc")
task.final_metadata_from_dataset(dataset)
task.metadata_from_dict(full_metadata)
bands = task.satellite.get_measurements() + ['band_math']
export_xarray_to_netcdf(dataset, task.data_netcdf_path)
write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
# Ensure data variables have the range of Landsat 7 Collection 1 Level 2
# since the color scales are tailored for that dataset.
platform = task.satellite.platform
collection = task.satellite.collection
level = task.satellite.level
if (platform, collection) != ('LANDSAT_7', 'c1'):
dataset = \
convert_range(dataset, from_platform=platform,
from_collection=collection, from_level=level,
to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
write_png_from_xr(
task.mosaic_path,
dataset,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
write_single_band_png_from_xr(
task.result_path,
dataset,
band='band_math',
color_scale=task.color_scale_path.get(task.query_type.result_id),
no_data=task.satellite.no_data_value)
dates = list(map(lambda x: datetime.strptime(x, "%m/%d/%Y"), task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(
task.plot_path,
dates=dates,
datasets=task._get_field_as_list('clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
logger.info("All products created.")
# task.update_bounds_from_dataset(dataset)
task.complete = True
task.execution_end = datetime.now()
task.update_status("OK", "All products have been generated. Your result will be loaded on the map.")
return True
def create_output_products(self, data, task_id=None):
"""Create the final output products for this algorithm.
Open the final dataset and metadata and generate all remaining metadata.
Convert and write the dataset to various formats and register all values in the task model
Update status and exit.
Args:
data: tuple in the format of processing_task function - path, metadata, and {chunk ids}
"""
if data is None: return None
task = SpectralAnomalyTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
spectral_index = task.query_type.result_id
full_metadata = data[3]
# This is the difference (or "change") composite.
diff_composite = xr.open_dataset(data[0])
# This indicates where either the baseline or analysis composite
# was outside the corresponding user-specified range.
orig_composite_out_of_range = xr.open_dataset(data[1]) \
[spectral_index].astype(np.bool).values
# This indicates where either the baseline or analysis composite
# was the no_data value.
composite_no_data = xr.open_dataset(data[2]) \
[spectral_index].astype(np.bool).values
# Obtain a NumPy array of the data to create a plot later.
if spectral_index in ['ndvi', 'ndbi', 'ndwi', 'evi']:
diff_comp_np_arr = diff_composite[spectral_index].values
else: # Fractional Cover
diff_comp_np_arr = diff_composite['pv'].values
diff_comp_np_arr[composite_no_data] = np.nan
task.data_netcdf_path = os.path.join(task.get_result_path(),
"data_netcdf.nc")
task.data_path = os.path.join(task.get_result_path(), "data_tif.tif")
task.result_path = os.path.join(task.get_result_path(), "png_mosaic.png")
task.final_metadata_from_dataset(diff_composite)
task.metadata_from_dict(full_metadata)
# 1. Prepare to save the spectral index net change as a GeoTIFF and NetCDF.
if spectral_index in ['ndvi', 'ndbi', 'ndwi', 'evi']:
bands = [spectral_index]
else: # Fractional Coverage
bands = ['bs', 'pv', 'npv']
# 2. Prepare to create a PNG of the spectral index change composite.
# 2.1. Find the min and max possible difference for the selected spectral index.
spec_ind_min, spec_ind_max = spectral_indices_range_map[spectral_index]
diff_min_possible, diff_max_possible = spec_ind_min - spec_ind_max, spec_ind_max - spec_ind_min
# 2.2. Scale the difference composite to the range [0, 1] for plotting.
image_data = np.interp(diff_comp_np_arr,
(diff_min_possible, diff_max_possible), (0, 1))
# 2.3. Color by region.
# 2.3.1. First, color by change.
# If the user specified a change value range, the product is binary -
# denoting which pixels fall within the net change threshold.
cng_min, cng_max = task.change_threshold_min, task.change_threshold_max
if cng_min is not None and cng_max is not None:
image_data = np.empty((*image_data.shape, 4), dtype=image_data.dtype)
image_data[:, :] = mpl.colors.to_rgba('red')
else: # otherwise, use a red-green gradient.
cmap = plt.get_cmap('RdYlGn')
image_data = cmap(image_data)
# 2.3.2. Second, color regions in which the change was outside
# the optional user-specified change value range.
change_out_of_range_color = mpl.colors.to_rgba('black')
if cng_min is not None and cng_max is not None:
diff_composite_out_of_range = (diff_comp_np_arr <
cng_min) | (cng_max < diff_comp_np_arr)
image_data[diff_composite_out_of_range] = change_out_of_range_color
# 2.3.3. Third, color regions in which either the baseline or analysis
# composite was outside the user-specified composite value range.
composite_out_of_range_color = mpl.colors.to_rgba('white')
image_data[orig_composite_out_of_range] = composite_out_of_range_color
# 2.3.4. Fourth, color regions in which either the baseline or analysis
# composite was the no_data value as transparent.
composite_no_data_color = np.array([0., 0., 0., 0.])
image_data[composite_no_data] = composite_no_data_color
# Create output products (NetCDF, GeoTIFF, PNG).
export_xarray_to_netcdf(diff_composite, task.data_netcdf_path)
write_geotiff_from_xr(task.data_path,
diff_composite.astype('float32'),
bands=bands,
no_data=task.satellite.no_data_value)
plt.imsave(task.result_path, image_data)
# Plot metadata.
dates = list(
map(lambda x: datetime.strptime(x, "%m/%d/%Y"),
task._get_field_as_list('acquisition_list')))
if len(dates) > 1:
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=task._get_field_as_list(
'clean_pixel_percentages_per_acquisition'),
data_labels="Clean Pixel Percentage (%)",
titles="Clean Pixel Percentage Per Acquisition")
task.complete = True
task.execution_end = datetime.now()
task.update_status(
"OK",
"All products have been generated. Your result will be loaded on the map."
)
return True
