def validate_parameters(self, parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = NdviAnomalyTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
dc = DataAccessApi(config=task.config_path)
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
# the actual acquisitino exists, lets try the baseline:
validation_params = {**parameters}
# there were no acquisitions in the year 1000, hopefully
validation_params.update({
'time': (task.time_start.replace(year=task.time_start.year - 5),
task.time_start - timedelta(microseconds=1))
})
acquisitions = dc.list_acquisition_dates(**validation_params)
# list/map/int chain required to cast int to each baseline month, it won't work if they're strings.
grouped_dates = group_datetimes_by_month(
acquisitions,
months=list(map(int, task.baseline_selection.split(","))))
if check_cancel_task(self, task): return
if not grouped_dates:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
task.complete = True
task.update_status(
"ERROR",
"The provided Satellite model measurements aren't valid for the product. Please check the measurements listed in the {} model."
.format(task.satellite.name))
return None
dc.close()
return parameters
def validate_parameters(self, parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = SlipTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
dc = DataAccessApi(config=task.config_path)
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
if len(acquisitions) < task.baseline_length + 1:
task.complete = True
task.update_status(
"ERROR",
"There are an insufficient number of acquisitions for your baseline length."
)
return None
validation_parameters = {**parameters}
validation_parameters.pop('time')
validation_parameters.pop('measurements')
validation_parameters.update({
'product': 'terra_aster_gdm_' + task.area_id,
'platform': 'TERRA'
})
if len(dc.list_acquisition_dates(**validation_parameters)) < 1:
task.complete = True
task.update_status(
"ERROR", "There is no elevation data for this parameter set.")
return None
if check_cancel_task(self, task): return
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
task.complete = True
task.update_status(
"ERROR",
"The provided Satellite model measurements aren't valid for the product. Please check the measurements listed in the {} model."
.format(task.satellite.name))
return None
dc.close()
return parameters
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = SlipTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
if len(acquisitions) < task.baseline_length + 1:
task.complete = True
task.update_status(
"ERROR",
"There are an insufficient number of acquisitions for your baseline length."
)
return None
validation_parameters = {**parameters}
validation_parameters.pop('time')
validation_parameters.pop('measurements')
validation_parameters.update({
'product': 'terra_aster_gdm_' + task.area_id,
'platform': 'TERRA'
})
if len(dc.list_acquisition_dates(**validation_parameters)) < 1:
task.complete = True
task.update_status(
"ERROR", "There is no elevation data for this parameter set.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
parameters['measurements'] = [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'
]
dc.close()
return parameters
def perform_task_chunking(parameters, task_id=None):
"""Chunk parameter sets into more manageable sizes
Uses functions provided by the task model to create a group of
parameter sets that make up the arg.
Args:
parameters: parameter stream containing all kwargs to load data
Returns:
parameters with a list of geographic and time ranges
"""
if parameters is None:
return None
task = NdviAnomalyTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
dates = dc.list_acquisition_dates(**parameters)
task_chunk_sizing = task.get_chunk_size()
geographic_chunks = create_geographic_chunks(
longitude=parameters['longitude'],
latitude=parameters['latitude'],
geographic_chunk_size=task_chunk_sizing['geographic'])
grouped_dates_params = {**parameters}
grouped_dates_params.update({
'time': (datetime(1000, 1,
1), task.time_start - timedelta(microseconds=1))
})
acquisitions = dc.list_acquisition_dates(**grouped_dates_params)
grouped_dates = group_datetimes_by_month(
acquisitions,
months=list(map(int, task.baseline_selection.split(","))))
# create a single monolithic list of all acq. dates - there should be only one.
time_chunks = []
for date_group in grouped_dates:
time_chunks.extend(grouped_dates[date_group])
# time chunks casted to a list, essnetially.
time_chunks = [time_chunks]
logger.info("Time chunks: {}, Geo chunks: {}".format(
len(time_chunks), len(geographic_chunks)))
dc.close()
task.update_status("WAIT", "Chunked parameter set.")
return {
'parameters': parameters,
'geographic_chunks': geographic_chunks,
'time_chunks': time_chunks
}
def validate_parameters(self, parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = SpectralAnomalyTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
dc = DataAccessApi(config=task.config_path)
baseline_parameters = parameters.copy()
baseline_parameters['time'] = parameters['baseline_time']
baseline_acquisitions = dc.list_acquisition_dates(**baseline_parameters)
analysis_parameters = parameters.copy()
analysis_parameters['time'] = parameters['analysis_time']
analysis_acquisitions = dc.list_acquisition_dates(**analysis_parameters)
if len(baseline_acquisitions) < 1:
task.complete = True
task.update_status(
"ERROR", "There are no acquisitions for this parameter set "
"for the baseline time period.")
return None
if len(analysis_acquisitions) < 1:
task.complete = True
task.update_status(
"ERROR", "There are no acquisitions for this parameter set "
"for the analysis time period.")
return None
if check_cancel_task(self, task): return
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
task.complete = True
task.update_status(
"ERROR",
"The provided Satellite model measurements aren't valid for the product. Please check the measurements listed in the {} model."
.format(task.satellite.name))
return None
dc.close()
return parameters
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = SpectralIndicesTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
#validate for any number of criteria here - num acquisitions, etc.
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR", "There are no acquistions for this parameter set.")
return None
if not task.compositor.is_iterative() and (task.time_end - task.time_start).days > 367:
task.complete = True
task.update_status("ERROR", "Median pixel operations are only supported for single year time periods.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'], parameters['measurements']):
parameters['measurements'] = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']
dc.close()
return parameters
def perform_task_chunking(parameters, task_id=None):
"""Chunk parameter sets into more manageable sizes
Uses functions provided by the task model to create a group of
parameter sets that make up the arg.
Args:
parameters: parameter stream containing all kwargs to load data
Returns:
parameters with a list of geographic and time ranges
"""
if parameters is None:
return None
task = SpectralIndicesTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
dates = dc.list_acquisition_dates(**parameters)
task_chunk_sizing = task.get_chunk_size()
geographic_chunks = create_geographic_chunks(
longitude=parameters['longitude'],
latitude=parameters['latitude'],
geographic_chunk_size=task_chunk_sizing['geographic'])
time_chunks = create_time_chunks(
dates, _reversed=task.get_reverse_time(), time_chunk_size=task_chunk_sizing['time'])
logger.info("Time chunks: {}, Geo chunks: {}".format(len(time_chunks), len(geographic_chunks)))
dc.close()
task.update_status("WAIT", "Chunked parameter set.")
return {'parameters': parameters, 'geographic_chunks': geographic_chunks, 'time_chunks': time_chunks}
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = CloudCoverageTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
#validate for any number of criteria here - num acquisitions, etc.
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR", "There are no acquistions for this parameter set.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'], parameters['measurements']):
parameters['measurements'] = ['blue', 'green', 'red', 'pixel_qa']
dc.close()
return parameters
def get_acquisition_list(task, area_id, satellite, date):
dc = DataAccessApi(config=task.config_path)
# lists all acquisition dates for use in single tmeslice queries.
# satellite.product_prefix +
product = area_id
acquisitions = dc.list_acquisition_dates(product, satellite.datacube_platform, time=(datetime(1900, 1, 1), date))
return acquisitions
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = CoastalChangeTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
validation_params = dict(parameters)
# verify that both the start and end year have acquisitions
for year in parameters['time']:
validation_params.update({'time': (year, year.replace(year=year.year + 1))})
acquisitions = dc.list_acquisition_dates(**validation_params)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR", "There must be at least one acquisition in both the start and ending year.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'], parameters['measurements']):
parameters['measurements'] = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']
dc.close()
return parameters
def perform_task_chunking(parameters, task_id=None):
"""Chunk parameter sets into more manageable sizes
Uses functions provided by the task model to create a group of
parameter sets that make up the arg.
Args:
parameters: parameter stream containing all kwargs to load data
Returns:
parameters with a list of geographic and time ranges
"""
if parameters is None:
return None
task = CoastalChangeTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
dates = dc.list_acquisition_dates(**parameters)
task_chunk_sizing = task.get_chunk_size()
geographic_chunks = create_geographic_chunks(
longitude=parameters['longitude'],
latitude=parameters['latitude'],
geographic_chunk_size=task_chunk_sizing['geographic'])
grouped_dates = group_datetimes_by_year(dates)
# we need to pair these with the first year - subsequent years.
time_chunks = None
if task.animated_product.animation_id == 'none':
# first and last only
time_chunks = [[
grouped_dates[task.time_start], grouped_dates[task.time_end]
]]
else:
initial_year = grouped_dates.pop(task.time_start)
time_chunks = [[initial_year, grouped_dates[year]]
for year in grouped_dates]
logger.info("Time chunks: {}, Geo chunks: {}".format(
len(time_chunks), len(geographic_chunks)))
dc.close()
task.update_status("WAIT", "Chunked parameter set.")
return {
'parameters': parameters,
'geographic_chunks': geographic_chunks,
'time_chunks': time_chunks
}
def validate_parameters(self, parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = SpectralIndicesTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
dc = DataAccessApi(config=task.config_path)
#validate for any number of criteria here - num acquisitions, etc.
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
if not task.compositor.is_iterative() and (task.time_end -
task.time_start).days > 367:
task.complete = True
task.update_status(
"ERROR",
"Median pixel operations are only supported for single year time periods."
)
return None
if check_cancel_task(self, task): return
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
task.complete = True
task.update_status(
"ERROR",
"The provided Satellite model measurements aren't valid for the product. Please check the measurements listed in the {} model."
.format(task.satellite.name))
return None
dc.close()
return parameters
def validate_parameters(self, parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = CoastalChangeTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
dc = DataAccessApi(config=task.config_path)
validation_params = dict(parameters)
# verify that both the start and end year have acquisitions
for year in parameters['time']:
validation_params.update(
{'time': (year, year.replace(year=year.year + 1))})
acquisitions = dc.list_acquisition_dates(**validation_params)
if len(acquisitions) < 1:
task.complete = True
task.update_status(
"ERROR",
"There must be at least one acquisition in both the start and ending year."
)
return None
if check_cancel_task(self, task): return
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
task.complete = True
task.update_status(
"ERROR",
"The provided Satellite model measurements aren't valid for the product. Please check the measurements listed in the {} model."
.format(task.satellite.name))
return None
dc.close()
return parameters
