def pixel_drill(task_id=None): parameters = parse_parameters_from_task(task_id=task_id) validate_parameters(parameters, task_id=task_id) task = CustomMosaicToolTask.objects.get(pk=task_id) if task.status == "ERROR": return None dc = DataAccessApi(config=task.config_path) single_pixel = dc.get_stacked_datasets_by_extent(**parameters).isel(latitude=0, longitude=0) clear_mask = task.satellite.get_clean_mask_func()(single_pixel) single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value) dates = single_pixel.time.values if len(dates) < 2: task.update_status("ERROR", "There is only a single acquisition for your parameter set.") return None exclusion_list = ['satellite', 'pixel_qa'] plot_measurements = [band for band in single_pixel.data_vars if band not in exclusion_list] datasets = [single_pixel[band].values.transpose() for band in plot_measurements] + [clear_mask] data_labels = [stringcase.titlecase("{} Units".format(band)) for band in plot_measurements] + ["Clear"] titles = [stringcase.titlecase("{} Band".format(band)) for band in plot_measurements] + ["Clear Mask"] style = ['r-o', 'g-o', 'b-o', 'c-o', 'm-o', 'y-o', '.'] task.plot_path = os.path.join(task.get_result_path(), "plot_path.png") create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style) task.complete = True task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None): parameters = parse_parameters_from_task(task_id=task_id) validate_parameters(parameters, task_id=task_id) task = WaterDetectionTask.objects.get(pk=task_id) if task.status == "ERROR": return None dc = DataAccessApi(config=task.config_path) single_pixel = dc.get_stacked_datasets_by_extent(**parameters) clear_mask = task.satellite.get_clean_mask_func()(single_pixel.isel(latitude=0, longitude=0)) single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value) dates = single_pixel.time.values if len(dates) < 2: task.update_status("ERROR", "There is only a single acquisition for your parameter set.") return None wofs_data = task.get_processing_method()(single_pixel, clean_mask=clear_mask, enforce_float64=True, no_data=task.satellite.no_data_value) wofs_data = wofs_data.where(wofs_data != task.satellite.no_data_value).isel(latitude=0, longitude=0) datasets = [wofs_data.wofs.values.transpose()] + [clear_mask] data_labels = ["Water/Non Water"] + ["Clear"] titles = ["Water/Non Water"] + ["Clear Mask"] style = ['.', '.'] task.plot_path = os.path.join(task.get_result_path(), "plot_path.png") create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style) task.complete = True task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None): parameters = parse_parameters_from_task(task_id=task_id) validate_parameters(parameters, task_id=task_id) task = TsmTask.objects.get(pk=task_id) if task.status == "ERROR": return None dc = DataAccessApi(config=task.config_path) single_pixel = dc.get_stacked_datasets_by_extent(**parameters) clear_mask = task.satellite.get_clean_mask_func()(single_pixel) single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value) dates = single_pixel.time.values if len(dates) < 2: task.update_status("ERROR", "There is only a single acquisition for your parameter set.") return None # Ensure data variables have the range of Landsat 7 Collection 1 Level 2 # since the color scales are tailored for that dataset. platform = task.satellite.platform collection = task.satellite.collection level = task.satellite.level if (platform, collection) != ('LANDSAT_7', 'c1'): single_pixel = \ convert_range(single_pixel, from_platform=platform, from_collection=collection, from_level=level, to_platform='LANDSAT_7', to_collection='c1', to_level='l2') wofs_data = task.get_processing_method()(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value) wofs_data = \ wofs_data.where(wofs_data != task.satellite.no_data_value) wofs_data = wofs_data.squeeze() tsm_data = \ tsm(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value) tsm_data = \ tsm_data.where(tsm_data != task.satellite.no_data_value)\ .squeeze().where(wofs_data.wofs.values == 1) # Remove NaNs to avoid errors and yield a nicer plot. water_non_nan_times = ~np.isnan(wofs_data.wofs.values) wofs_data = wofs_data.isel(time=water_non_nan_times) tsm_non_nan_times = ~np.isnan(tsm_data.tsm.values) tsm_data = tsm_data.isel(time=tsm_non_nan_times) datasets = [wofs_data.wofs.values.transpose().squeeze(), tsm_data.tsm.values.transpose().squeeze()] + \ [clear_mask.squeeze()] dates = [dates[water_non_nan_times], dates[tsm_non_nan_times]] + [dates] data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"] titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"] style = ['.', 'ro', '.'] task.plot_path = os.path.join(task.get_result_path(), "plot_path.png") create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style) task.complete = True task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None): parameters = parse_parameters_from_task(task_id=task_id) validate_parameters(parameters, task_id=task_id) task = FractionalCoverTask.objects.get(pk=task_id) if task.status == "ERROR": return None dc = DataAccessApi(config=task.config_path) single_pixel = dc.get_stacked_datasets_by_extent(**parameters) clear_mask = task.satellite.get_clean_mask_func()(single_pixel.isel(latitude=0, longitude=0)) single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value) dates = single_pixel.time.values if len(dates) < 2: task.update_status("ERROR", "There is only a single acquisition for your parameter set.") return None def _apply_band_math(ds, idx): # mask out water manually. Necessary for frac. cover. wofs = wofs_classify(ds, clean_mask=clear_mask[idx], mosaic=True) clear_mask[idx] = False if wofs.wofs.values[0] == 1 else clear_mask[idx] fractional_cover = frac_coverage_classify(ds, clean_mask=clear_mask[idx], no_data=task.satellite.no_data_value) return fractional_cover fractional_cover = xr.concat( [ _apply_band_math(single_pixel.isel(time=data_point, drop=True), data_point) for data_point in range(len(dates)) ], dim='time') fractional_cover = fractional_cover.where(fractional_cover != task.satellite.no_data_value).isel( latitude=0, longitude=0) exclusion_list = [] plot_measurements = [band for band in fractional_cover.data_vars if band not in exclusion_list] datasets = [fractional_cover[band].values.transpose() for band in plot_measurements] + [clear_mask] data_labels = [stringcase.titlecase("%{}".format(band)) for band in plot_measurements] + ["Clear"] titles = [ 'Bare Soil Percentage', 'Photosynthetic Vegetation Percentage', 'Non-Photosynthetic Vegetation Percentage', 'Clear Mask' ] style = ['ro', 'go', 'bo', '.'] task.plot_path = os.path.join(task.get_result_path(), "plot_path.png") create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style) task.complete = True task.update_status("OK", "Done processing pixel drill.")
def pixel_drill(task_id=None): parameters = parse_parameters_from_task(task_id=task_id) validate_parameters(parameters, task_id=task_id) task = TsmTask.objects.get(pk=task_id) if task.status == "ERROR": return None dc = DataAccessApi(config=task.config_path) single_pixel = dc.get_stacked_datasets_by_extent(**parameters) clear_mask = task.satellite.get_clean_mask_func()(single_pixel) single_pixel = single_pixel.where(single_pixel != task.satellite.no_data_value) dates = single_pixel.time.values if len(dates) < 2: task.update_status("ERROR", "There is only a single acquisition for your parameter set.") return None wofs_data = task.get_processing_method()(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value) wofs_data = \ wofs_data.where(wofs_data != task.satellite.no_data_value) wofs_data = wofs_data.squeeze() tsm_data = \ tsm(single_pixel, clean_mask=clear_mask, no_data=task.satellite.no_data_value) tsm_data = \ tsm_data.where(tsm_data != task.satellite.no_data_value)\ .squeeze().where(wofs_data.wofs.values == 1) # Remove NaNs to avoid errors and yield a nicer plot. water_non_nan_times = ~np.isnan(wofs_data.wofs.values) wofs_data = wofs_data.isel(time=water_non_nan_times) tsm_non_nan_times = ~np.isnan(tsm_data.tsm.values) tsm_data = tsm_data.isel(time=tsm_non_nan_times) datasets = [wofs_data.wofs.values.transpose().squeeze(), tsm_data.tsm.values.transpose().squeeze()] + \ [clear_mask.squeeze()] dates = [dates[water_non_nan_times], dates[tsm_non_nan_times]] + [dates] data_labels = ["Water/Non Water", "TSM (g/L)"] + ["Clear"] titles = ["Water/Non Water", "TSM Values"] + ["Clear Mask"] style = ['.', 'ro', '.'] task.plot_path = os.path.join(task.get_result_path(), "plot_path.png") create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style) task.complete = True task.update_status("OK", "Done processing pixel drill.")
def processing_task(self, task_id=None, geo_chunk_id=None, time_chunk_id=None, geographic_chunk=None, time_chunk=None, **parameters): """Process a parameter set and save the results to disk. Uses the geographic and time chunk id to identify output products. **params is updated with time and geographic ranges then used to load data. the task model holds the iterative property that signifies whether the algorithm is iterative or if all data needs to be loaded at once. Args: task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude time_chunk: list of acquisition dates parameters: all required kwargs to load data. Returns: path to the output product, metadata dict, and a dict containing the geo/time ids """ chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)]) task = TsmTask.objects.get(pk=task_id) if check_cancel_task(self, task): return logger.info("Starting chunk: " + chunk_id) if not os.path.exists(task.get_temp_path()): return None metadata = {} def _get_datetime_range_containing(*time_ranges): return (min(time_ranges) - timedelta(microseconds=1), max(time_ranges) + timedelta(microseconds=1)) times = list( map(_get_datetime_range_containing, time_chunk) if task.get_iterative() else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])]) dc = DataAccessApi(config=task.config_path) updated_params = parameters updated_params.update(geographic_chunk) #updated_params.update({'products': parameters['']}) water_analysis = None tsm_analysis = None combined_data = None base_index = (task.get_chunk_size()['time'] if task.get_chunk_size()['time'] is not None else 1) * time_chunk_id for time_index, time in enumerate(times): updated_params.update({'time': time}) data = dc.get_stacked_datasets_by_extent(**updated_params) if check_cancel_task(self, task): return if data is None or 'time' not in data: logger.info("Invalid chunk.") continue clear_mask = task.satellite.get_clean_mask_func()(data) wofs_data = task.get_processing_method()(data, clean_mask=clear_mask, enforce_float64=True, no_data=task.satellite.no_data_value) water_analysis = perform_timeseries_analysis( wofs_data, 'wofs', intermediate_product=water_analysis, no_data=task.satellite.no_data_value) clear_mask[(data.swir2.values > 100) | (wofs_data.wofs.values == 0)] = False tsm_data = tsm(data, clean_mask=clear_mask, no_data=task.satellite.no_data_value) tsm_analysis = perform_timeseries_analysis( tsm_data, 'tsm', intermediate_product=tsm_analysis, no_data=task.satellite.no_data_value) if check_cancel_task(self, task): return combined_data = tsm_analysis combined_data['wofs'] = water_analysis.total_data combined_data['wofs_total_clean'] = water_analysis.total_clean metadata = task.metadata_from_dataset(metadata, tsm_data, clear_mask, updated_params) if task.animated_product.animation_id != "none": path = os.path.join(task.get_temp_path(), "animation_{}_{}.nc".format(str(geo_chunk_id), str(base_index + time_index))) animated_data = tsm_data.isel( time=0, drop=True) if task.animated_product.animation_id == "scene" else combined_data animated_data.to_netcdf(path) task.scenes_processed = F('scenes_processed') + 1 task.save(update_fields=['scenes_processed']) if combined_data is None: return None path = os.path.join(task.get_temp_path(), chunk_id + ".nc") combined_data.to_netcdf(path) dc.close() logger.info("Done with chunk: " + chunk_id) return path, metadata, {'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id}
def processing_task(task_id=None, geo_chunk_id=None, time_chunk_id=None, geographic_chunk=None, time_chunk=None, **parameters): """Process a parameter set and save the results to disk. Uses the geographic and time chunk id to identify output products. **params is updated with time and geographic ranges then used to load data. the task model holds the iterative property that signifies whether the algorithm is iterative or if all data needs to be loaded at once. Args: task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude time_chunk: list of acquisition dates parameters: all required kwargs to load data. Returns: path to the output product, metadata dict, and a dict containing the geo/time ids """ chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)]) task = AppNameTask.objects.get(pk=task_id) logger.info("Starting chunk: " + chunk_id) if not os.path.exists(task.get_temp_path()): return None iteration_data = None metadata = {} def _get_datetime_range_containing(*time_ranges): return (min(time_ranges) - timedelta(microseconds=1), max(time_ranges) + timedelta(microseconds=1)) times = list( map(_get_datetime_range_containing, time_chunk) if task.get_iterative( ) else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])]) dc = DataAccessApi(config=task.config_path) updated_params = parameters updated_params.update(geographic_chunk) #updated_params.update({'products': parameters['']}) iteration_data = None base_index = (task.get_chunk_size()['time'] if task.get_chunk_size() ['time'] is not None else 1) * time_chunk_id for time_index, time in enumerate(times): updated_params.update({'time': time}) # TODO: If this is not a multisensory app replace get_stacked_datasets_by_extent with get_dataset_by_extent data = dc.get_stacked_datasets_by_extent(**updated_params) if data is None or 'time' not in data: logger.info("Invalid chunk.") continue # TODO: Replace anything here with your processing - do you need to create additional masks? Apply bandmaths? etc. clear_mask = create_cfmask_clean_mask( data.cf_mask) if 'cf_mask' in data else create_bit_mask( data.pixel_qa, [1, 2]) add_timestamp_data_to_xr(data) metadata = task.metadata_from_dataset(metadata, data, clear_mask, updated_params) # TODO: Make sure you're producing everything required for your algorithm. iteration_data = task.get_processing_method()( data, clean_mask=clear_mask, intermediate_product=iteration_data) # TODO: If there is no animation you can remove this block. Otherwise, save off the data that you need. if task.animated_product.animation_id != "none": path = os.path.join( task.get_temp_path(), "animation_{}_{}.nc".format(str(geo_chunk_id), str(base_index + time_index))) if task.animated_product.animation_id == "scene": #need to clear out all the metadata.. clear_attrs(data) #can't reindex on time - weird? data.isel(time=0).drop('time').to_netcdf(path) elif task.animated_product.animation_id == "cumulative": iteration_data.to_netcdf(path) task.scenes_processed = F('scenes_processed') + 1 task.save() if iteration_data is None: return None path = os.path.join(task.get_temp_path(), chunk_id + ".nc") iteration_data.to_netcdf(path) dc.close() logger.info("Done with chunk: " + chunk_id) return path, metadata, { 'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id }
def processing_task(task_id=None, geo_chunk_id=None, time_chunk_id=None, geographic_chunk=None, time_chunk=None, **parameters): """Process a parameter set and save the results to disk. Uses the geographic and time chunk id to identify output products. **params is updated with time and geographic ranges then used to load data. the task model holds the iterative property that signifies whether the algorithm is iterative or if all data needs to be loaded at once. Args: task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude time_chunk: list of acquisition dates parameters: all required kwargs to load data. Returns: path to the output product, metadata dict, and a dict containing the geo/time ids """ chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)]) task = FractionalCoverTask.objects.get(pk=task_id) logger.info("Starting chunk: " + chunk_id) if not os.path.exists(task.get_temp_path()): return None iteration_data = None metadata = {} def _get_datetime_range_containing(*time_ranges): return (min(time_ranges) - timedelta(microseconds=1), max(time_ranges) + timedelta(microseconds=1)) times = list( map(_get_datetime_range_containing, time_chunk) if task.get_iterative( ) else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])]) dc = DataAccessApi(config=task.config_path) updated_params = parameters updated_params.update(geographic_chunk) #updated_params.update({'products': parameters['']}) iteration_data = None base_index = (task.get_chunk_size()['time'] if task.get_chunk_size() ['time'] is not None else 1) * time_chunk_id for time_index, time in enumerate(times): updated_params.update({'time': time}) data = dc.get_stacked_datasets_by_extent(**updated_params) if data is None or 'time' not in data: logger.info("Invalid chunk.") continue clear_mask = create_cfmask_clean_mask( data.cf_mask) if 'cf_mask' in data else create_bit_mask( data.pixel_qa, [1, 2]) add_timestamp_data_to_xr(data) metadata = task.metadata_from_dataset(metadata, data, clear_mask, updated_params) iteration_data = task.get_processing_method()( data, clean_mask=clear_mask, intermediate_product=iteration_data) task.scenes_processed = F('scenes_processed') + 1 task.save() if iteration_data is None: return None path = os.path.join(task.get_temp_path(), chunk_id + ".nc") iteration_data.to_netcdf(path) dc.close() logger.info("Done with chunk: " + chunk_id) return path, metadata, { 'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id }
def processing_task(self, task_id=None, geo_chunk_id=None, time_chunk_id=None, geographic_chunk=None, time_chunk=None, **parameters): """Process a parameter set and save the results to disk. Uses the geographic and time chunk id to identify output products. **params is updated with time and geographic ranges then used to load data. the task model holds the iterative property that signifies whether the algorithm is iterative or if all data needs to be loaded at once. Args: task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude time_chunk: list of acquisition dates parameters: all required kwargs to load data. Returns: path to the output product, metadata dict, and a dict containing the geo/time ids """ chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)]) task = FractionalCoverTask.objects.get(pk=task_id) if check_cancel_task(self, task): return logger.info("Starting chunk: " + chunk_id) if not os.path.exists(task.get_temp_path()): return None iteration_data = None metadata = {} times = list( map(_get_datetime_range_containing, time_chunk) if task.get_iterative( ) else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])]) dc = DataAccessApi(config=task.config_path) updated_params = parameters updated_params.update(geographic_chunk) iteration_data = None for time_index, time in enumerate(times): updated_params.update({'time': time}) data = dc.get_stacked_datasets_by_extent(**updated_params) if check_cancel_task(self, task): return if data is None: logger.info("Empty chunk.") continue if 'time' not in data: logger.info("Invalid chunk.") continue clear_mask = task.satellite.get_clean_mask_func()(data) add_timestamp_data_to_xr(data) metadata = task.metadata_from_dataset(metadata, data, clear_mask, updated_params) iteration_data = task.get_processing_method()( data, clean_mask=clear_mask, intermediate_product=iteration_data, no_data=task.satellite.no_data_value, reverse_time=task.get_reverse_time()) if check_cancel_task(self, task): return task.scenes_processed = F('scenes_processed') + 1 # Avoid overwriting the task's status if it is cancelled. task.save(update_fields=['scenes_processed']) if iteration_data is None: return None path = os.path.join(task.get_temp_path(), chunk_id + ".nc") export_xarray_to_netcdf(iteration_data, path) dc.close() logger.info("Done with chunk: " + chunk_id) return path, metadata, { 'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id }
def processing_task(self, task_id=None, geo_chunk_id=None, time_chunk_id=None, geographic_chunk=None, time_chunk=None, **parameters): """Process a parameter set and save the results to disk. Uses the geographic and time chunk id to identify output products. **params is updated with time and geographic ranges then used to load data. the task model holds the iterative property that signifies whether the algorithm is iterative or if all data needs to be loaded at once. Args: task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude time_chunk: list of acquisition dates parameters: all required kwargs to load data. Returns: path to the output product, metadata dict, and a dict containing the geo/time ids """ chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)]) task = CustomMosaicToolTask.objects.get(pk=task_id) if check_cancel_task(self, task): return logger.info("Starting chunk: " + chunk_id) if not os.path.exists(task.get_temp_path()): return None iteration_data = None metadata = {} def _get_datetime_range_containing(*time_ranges): return (min(time_ranges) - timedelta(microseconds=1), max(time_ranges) + timedelta(microseconds=1)) times = list( map(_get_datetime_range_containing, time_chunk) if task.get_iterative( ) else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])]) dc = DataAccessApi(config=task.config_path) updated_params = parameters updated_params.update(geographic_chunk) #updated_params.update({'products': parameters['']}) iteration_data = None base_index = (task.get_chunk_size()['time'] if task.get_chunk_size() ['time'] is not None else 1) * time_chunk_id for time_index, time in enumerate(times): updated_params.update({'time': time}) data = dc.get_stacked_datasets_by_extent(**updated_params) if check_cancel_task(self, task): return if data is None or 'time' not in data: logger.info("Invalid chunk.") continue clear_mask = task.satellite.get_clean_mask_func()(data) add_timestamp_data_to_xr(data) metadata = task.metadata_from_dataset(metadata, data, clear_mask, updated_params) iteration_data = task.get_processing_method()( data, clean_mask=clear_mask, intermediate_product=iteration_data, no_data=task.satellite.no_data_value, reverse_time=task.get_reverse_time()) if check_cancel_task(self, task): return if task.animated_product.animation_id != "none": path = os.path.join( task.get_temp_path(), "animation_{}_{}.nc".format(str(geo_chunk_id), str(base_index + time_index))) if task.animated_product.animation_id == "scene": #need to clear out all the metadata.. clear_attrs(data) #can't reindex on time - weird? export_xarray_to_netcdf(data.isel(time=0).drop('time'), path) elif task.animated_product.animation_id == "cumulative": export_xarray_to_netcdf(iteration_data, path) task.scenes_processed = F('scenes_processed') + 1 # Avoid overwriting the task's status if it is cancelled. task.save(update_fields=['scenes_processed']) if iteration_data is None: return None path = os.path.join(task.get_temp_path(), chunk_id + ".nc") export_xarray_to_netcdf(iteration_data, path) dc.close() logger.info("Done with chunk: " + chunk_id) return path, metadata, { 'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id }
def processing_task(self, task_id=None, geo_chunk_id=None, time_chunk_id=None, geographic_chunk=None, time_chunk=None, **parameters): """Process a parameter set and save the results to disk. Uses the geographic and time chunk id to identify output products. **params is updated with time and geographic ranges then used to load data. the task model holds the iterative property that signifies whether the algorithm is iterative or if all data needs to be loaded at once. Args: task_id, geo_chunk_id, time_chunk_id: identification for the main task and what chunk this is processing geographic_chunk: range of latitude and longitude to load - dict with keys latitude, longitude time_chunk: list of acquisition dates parameters: all required kwargs to load data. Returns: path to the output product, metadata dict, and a dict containing the geo/time ids """ chunk_id = "_".join([str(geo_chunk_id), str(time_chunk_id)]) task = WaterDetectionTask.objects.get(pk=task_id) if check_cancel_task(self, task): return logger.info("Starting chunk: " + chunk_id) if not os.path.exists(task.get_temp_path()): return None metadata = {} times = list( map(_get_datetime_range_containing, time_chunk) if task.get_iterative( ) else [_get_datetime_range_containing(time_chunk[0], time_chunk[-1])]) dc = DataAccessApi(config=task.config_path) updated_params = parameters updated_params.update(geographic_chunk) water_analysis = None base_index = (task.get_chunk_size()['time'] if task.get_chunk_size() ['time'] is not None else 1) * time_chunk_id for time_index, time in enumerate(times): updated_params.update({'time': time}) data = dc.get_stacked_datasets_by_extent(**updated_params) if check_cancel_task(self, task): return if data is None: logger.info("Empty chunk.") continue if 'time' not in data: logger.info("Invalid chunk.") continue clear_mask = task.satellite.get_clean_mask_func()(data) # Ensure data variables have the range of Landsat 7 Collection 1 Level 2 # since the color scales are tailored for that dataset. platform = task.satellite.platform collection = task.satellite.collection level = task.satellite.level if (platform, collection) != ('LANDSAT_7', 'c1'): data = \ convert_range(data, from_platform=platform, from_collection=collection, from_level=level, to_platform='LANDSAT_7', to_collection='c1', to_level='l2') wofs_data = task.get_processing_method()( data, clean_mask=clear_mask, no_data=task.satellite.no_data_value) water_analysis = perform_timeseries_analysis( wofs_data, 'wofs', intermediate_product=water_analysis, no_data=task.satellite.no_data_value) metadata = task.metadata_from_dataset(metadata, wofs_data, clear_mask.data, updated_params) if task.animated_product.animation_id != "none": path = os.path.join( task.get_temp_path(), "animation_{}_{}.nc".format(str(geo_chunk_id), str(base_index + time_index))) animated_data = wofs_data.isel( time=0, drop=True ) if task.animated_product.animation_id == "scene" else water_analysis export_xarray_to_netcdf(animated_data, path) if check_cancel_task(self, task): return task.scenes_processed = F('scenes_processed') + 1 task.save(update_fields=['scenes_processed']) if water_analysis is None: return None path = os.path.join(task.get_temp_path(), chunk_id + ".nc") export_xarray_to_netcdf(water_analysis, path) dc.close() logger.info("Done with chunk: " + chunk_id) return path, metadata, { 'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id }