def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = TsmTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
# Ensure data variables have the range of Landsat 7 Collection 1 Level 2
# since the color scales are tailored for that dataset.
platform = task.satellite.platform
collection = task.satellite.collection
level = task.satellite.level
if (platform, collection) != ('LANDSAT_7', 'c1'):
single_pixel = \
convert_range(single_pixel, from_platform=platform,
from_collection=collection, from_level=level,
to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
wofs_data = task.get_processing_method()(single_pixel,
clean_mask=clear_mask,
no_data=task.satellite.no_data_value)
wofs_data = \
wofs_data.where(wofs_data != task.satellite.no_data_value)
wofs_data = wofs_data.squeeze()
tsm_data = \
tsm(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value)
tsm_data = \
tsm_data.where(tsm_data != task.satellite.no_data_value)\
.squeeze().where(wofs_data.wofs.values == 1)
# Remove NaNs to avoid errors and yield a nicer plot.
water_non_nan_times = ~np.isnan(wofs_data.wofs.values)
wofs_data = wofs_data.isel(time=water_non_nan_times)
tsm_non_nan_times = ~np.isnan(tsm_data.tsm.values)
tsm_data = tsm_data.isel(time=tsm_non_nan_times)
datasets = [wofs_data.wofs.values.transpose().squeeze(),
tsm_data.tsm.values.transpose().squeeze()] + \
[clear_mask.squeeze()]
dates = [dates[water_non_nan_times], dates[tsm_non_nan_times]] + [dates]
data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"]
titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"]
style = ['.', 'ro', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = TsmTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel.isel(
latitude=0, longitude=0))
single_pixel = single_pixel.where(
single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status(
"ERROR",
"There is only a single acquisition for your parameter set.")
return None
wofs_data = task.get_processing_method()(
single_pixel,
clean_mask=clear_mask,
enforce_float64=True,
no_data=task.satellite.no_data_value)
wofs_data = wofs_data.where(
wofs_data != task.satellite.no_data_value).isel(latitude=0,
longitude=0)
tsm_data = tsm(single_pixel,
clean_mask=clear_mask,
no_data=task.satellite.no_data_value)
tsm_data = tsm_data.where(tsm_data != task.satellite.no_data_value).isel(
latitude=0, longitude=0).where((wofs_data.wofs.values == 1))
datasets = [
wofs_data.wofs.values.transpose(),
tsm_data.tsm.values.transpose()
] + [clear_mask]
data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"]
titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"]
style = ['.', 'r-o', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path,
dates=dates,
datasets=datasets,
data_labels=data_labels,
titles=titles,
style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None):
parameters = parse_parameters_from_task(task_id=task_id)
validate_parameters(parameters, task_id=task_id)
task = TsmTask.objects.get(pk=task_id)
if task.status == "ERROR":
return None
dc = DataAccessApi(config=task.config_path)
single_pixel = dc.get_stacked_datasets_by_extent(**parameters)
clear_mask = task.satellite.get_clean_mask_func()(single_pixel)
single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value)
dates = single_pixel.time.values
if len(dates) < 2:
task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
return None
wofs_data = task.get_processing_method()(single_pixel,
clean_mask=clear_mask,
no_data=task.satellite.no_data_value)
wofs_data = \
wofs_data.where(wofs_data != task.satellite.no_data_value)
wofs_data = wofs_data.squeeze()
tsm_data = \
tsm(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value)
tsm_data = \
tsm_data.where(tsm_data != task.satellite.no_data_value)\
.squeeze().where(wofs_data.wofs.values == 1)
# Remove NaNs to avoid errors and yield a nicer plot.
water_non_nan_times = ~np.isnan(wofs_data.wofs.values)
wofs_data = wofs_data.isel(time=water_non_nan_times)
tsm_non_nan_times = ~np.isnan(tsm_data.tsm.values)
tsm_data = tsm_data.isel(time=tsm_non_nan_times)
datasets = [wofs_data.wofs.values.transpose().squeeze(),
tsm_data.tsm.values.transpose().squeeze()] + \
[clear_mask.squeeze()]
dates = [dates[water_non_nan_times], dates[tsm_non_nan_times]] + [dates]
data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"]
titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"]
style = ['.', 'ro', '.']
task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)
task.complete = True
task.update_status("OK", "Done processing pixel drill.")
def processing_task(self,
task_id=None,
geo_chunk_id=None,
time_chunk_id=None,
geographic_chunk=None,
time_chunk=None,
**parameters):
"""Process a parameter set and save the results to disk.
Uses the geographic and time chunk id to identify output products.
**params is updated with time and geographic ranges then used to load data.
the task model holds the iterative property that signifies whether the algorithm
is iterative or if all data needs to be loaded at once.
Args:
task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing
geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude
time_chunk: list of acquisition dates
parameters: all required kwargs to load data.
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)])
task = TsmTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
logger.info("Starting chunk: " + chunk_id)
if not os.path.exists(task.get_temp_path()):
return None
metadata = {}
def _get_datetime_range_containing(*time_ranges):
return (min(time_ranges) - timedelta(microseconds=1), max(time_ranges) + timedelta(microseconds=1))
times = list(
map(_get_datetime_range_containing, time_chunk)
if task.get_iterative() else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])])
dc = DataAccessApi(config=task.config_path)
updated_params = parameters
updated_params.update(geographic_chunk)
#updated_params.update({'products': parameters['']})
water_analysis = None
tsm_analysis = None
combined_data = None
base_index = (task.get_chunk_size()['time'] if task.get_chunk_size()['time'] is not None else 1) * time_chunk_id
for time_index, time in enumerate(times):
updated_params.update({'time': time})
data = dc.get_stacked_datasets_by_extent(**updated_params)
if check_cancel_task(self, task): return
if data is None or 'time' not in data:
logger.info("Invalid chunk.")
continue
clear_mask = task.satellite.get_clean_mask_func()(data)
wofs_data = task.get_processing_method()(data,
clean_mask=clear_mask,
enforce_float64=True,
no_data=task.satellite.no_data_value)
water_analysis = perform_timeseries_analysis(
wofs_data, 'wofs', intermediate_product=water_analysis, no_data=task.satellite.no_data_value)
clear_mask[(data.swir2.values > 100) | (wofs_data.wofs.values == 0)] = False
tsm_data = tsm(data, clean_mask=clear_mask, no_data=task.satellite.no_data_value)
tsm_analysis = perform_timeseries_analysis(
tsm_data, 'tsm', intermediate_product=tsm_analysis, no_data=task.satellite.no_data_value)
if check_cancel_task(self, task): return
combined_data = tsm_analysis
combined_data['wofs'] = water_analysis.total_data
combined_data['wofs_total_clean'] = water_analysis.total_clean
metadata = task.metadata_from_dataset(metadata, tsm_data, clear_mask, updated_params)
if task.animated_product.animation_id != "none":
path = os.path.join(task.get_temp_path(),
"animation_{}_{}.nc".format(str(geo_chunk_id), str(base_index + time_index)))
animated_data = tsm_data.isel(
time=0, drop=True) if task.animated_product.animation_id == "scene" else combined_data
animated_data.to_netcdf(path)
task.scenes_processed = F('scenes_processed') + 1
task.save(update_fields=['scenes_processed'])
if combined_data is None:
return None
path = os.path.join(task.get_temp_path(), chunk_id + ".nc")
combined_data.to_netcdf(path)
dc.close()
logger.info("Done with chunk: " + chunk_id)
return path, metadata, {'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id}