def perform_task_chunking(parameters, task_id=None):
"""Chunk parameter sets into more manageable sizes
Uses functions provided by the task model to create a group of
parameter sets that make up the arg.
Args:
parameters: parameter stream containing all kwargs to load data
Returns:
parameters with a list of geographic and time ranges
"""
if parameters is None:
return None
task = NdviAnomalyTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
dates = dc.list_acquisition_dates(**parameters)
task_chunk_sizing = task.get_chunk_size()
geographic_chunks = create_geographic_chunks(
longitude=parameters['longitude'],
latitude=parameters['latitude'],
geographic_chunk_size=task_chunk_sizing['geographic'])
grouped_dates_params = {**parameters}
grouped_dates_params.update({
'time': (datetime(1000, 1,
1), task.time_start - timedelta(microseconds=1))
})
acquisitions = dc.list_acquisition_dates(**grouped_dates_params)
grouped_dates = group_datetimes_by_month(
acquisitions,
months=list(map(int, task.baseline_selection.split(","))))
# create a single monolithic list of all acq. dates - there should be only one.
time_chunks = []
for date_group in grouped_dates:
time_chunks.extend(grouped_dates[date_group])
# time chunks casted to a list, essnetially.
time_chunks = [time_chunks]
logger.info("Time chunks: {}, Geo chunks: {}".format(
len(time_chunks), len(geographic_chunks)))
dc.close()
task.update_status("WAIT", "Chunked parameter set.")
return {
'parameters': parameters,
'geographic_chunks': geographic_chunks,
'time_chunks': time_chunks
}
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = NdviAnomalyTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
# the actual acquisitino exists, lets try the baseline:
validation_params = {**parameters}
# there were no acquisitions in the year 1000, hopefully
validation_params.update({
'time': (task.time_start.replace(year=task.time_start.year - 5),
task.time_start - timedelta(microseconds=1))
})
acquisitions = dc.list_acquisition_dates(**validation_params)
# list/map/int chain required to cast int to each baseline month, it won't work if they're strings.
grouped_dates = group_datetimes_by_month(
acquisitions,
months=list(map(int, task.baseline_selection.split(","))))
if not grouped_dates:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
parameters['measurements'] = [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'
]
dc.close()
return parameters
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = CloudCoverageTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
#validate for any number of criteria here - num acquisitions, etc.
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
parameters['measurements'] = ['blue', 'green', 'red', 'pixel_qa']
dc.close()
return parameters
def perform_task_chunking(parameters, task_id=None):
"""Chunk parameter sets into more manageable sizes
Uses functions provided by the task model to create a group of
parameter sets that make up the arg.
Args:
parameters: parameter stream containing all kwargs to load data
Returns:
parameters with a list of geographic and time ranges
"""
if parameters is None:
return None
task = SlipTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
dates = dc.list_acquisition_dates(**parameters)
task_chunk_sizing = task.get_chunk_size()
geographic_chunks = create_geographic_chunks(
longitude=parameters['longitude'],
latitude=parameters['latitude'],
geographic_chunk_size=task_chunk_sizing['geographic'])
time_chunks = generate_baseline(dates, task.baseline_length)
logger.info("Time chunks: {}, Geo chunks: {}".format(len(time_chunks), len(geographic_chunks)))
dc.close()
task.update_status("WAIT", "Chunked parameter set.")
return {'parameters': parameters, 'geographic_chunks': geographic_chunks, 'time_chunks': time_chunks}
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = SlipTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR", "There are no acquistions for this parameter set.")
return None
if len(acquisitions) < task.baseline_length + 1:
task.complete = True
task.update_status("ERROR", "There are an insufficient number of acquisitions for your baseline length.")
return None
validation_parameters = {**parameters}
validation_parameters.pop('time')
validation_parameters.pop('measurements')
validation_parameters.update({'product': 'terra_aster_gdm_' + task.area_id, 'platform': 'TERRA'})
if len(dc.list_acquisition_dates(**validation_parameters)) < 1:
task.complete = True
task.update_status("ERROR", "There is no elevation data for this parameter set.")
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'], parameters['measurements']):
parameters['measurements'] = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']
dc.close()
return parameters
def perform_task_chunking(parameters, task_id=None):
"""Chunk parameter sets into more manageable sizes
Uses functions provided by the task model to create a group of
parameter sets that make up the arg.
Args:
parameters: parameter stream containing all kwargs to load data
Returns:
parameters with a list of geographic and time ranges
"""
if parameters is None:
return None
task = CoastalChangeTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
dates = dc.list_acquisition_dates(**parameters)
task_chunk_sizing = task.get_chunk_size()
geographic_chunks = create_geographic_chunks(
longitude=parameters['longitude'],
latitude=parameters['latitude'],
geographic_chunk_size=task_chunk_sizing['geographic'])
grouped_dates = group_datetimes_by_year(dates)
# we need to pair these with the first year - subsequent years.
time_chunks = None
if task.animated_product.animation_id == 'none':
# first and last only
time_chunks = [[
grouped_dates[task.time_start], grouped_dates[task.time_end]
]]
else:
initial_year = grouped_dates.pop(task.time_start)
time_chunks = [[initial_year, grouped_dates[year]]
for year in grouped_dates]
logger.info("Time chunks: {}, Geo chunks: {}".format(
len(time_chunks), len(geographic_chunks)))
dc.close()
task.update_status("WAIT", "Chunked parameter set.")
return {
'parameters': parameters,
'geographic_chunks': geographic_chunks,
'time_chunks': time_chunks
}
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = BandMathTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
#validate for any number of criteria here - num acquisitions, etc.
acquisitions = dc.list_acquisition_dates(**parameters)
if len(acquisitions) < 1:
task.complete = True
task.update_status("ERROR",
"There are no acquistions for this parameter set.")
return None
if task.compositor.id == "median_pixel" and (task.time_end -
task.time_start).days > 367:
task.complete = True
task.update_status(
"ERROR",
"Median pixel operations are only supported for single year time periods."
)
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
parameters['measurements'] = [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'
]
dc.close()
return parameters
def validate_parameters(parameters, task_id=None):
"""Validate parameters generated by the parameter parsing task
All validation should be done here - are there data restrictions?
Combinations that aren't allowed? etc.
Returns:
parameter dict with all keyword args required to load data.
-or-
updates the task with ERROR and a message, returning None
"""
task = CoastalChangeTask.objects.get(pk=task_id)
dc = DataAccessApi(config=task.config_path)
validation_params = dict(parameters)
# verify that both the start and end year have acquisitions
for year in parameters['time']:
validation_params.update(
{'time': (year, year.replace(year=year.year + 1))})
acquisitions = dc.list_acquisition_dates(**validation_params)
if len(acquisitions) < 1:
task.complete = True
task.update_status(
"ERROR",
"There must be at least one acquisition in both the start and ending year."
)
return None
task.update_status("WAIT", "Validated parameters.")
if not dc.validate_measurements(parameters['product'],
parameters['measurements']):
parameters['measurements'] = [
'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'
]
dc.close()
return parameters
def get_acquisition_list(task, area_id, platform, date):
dc = DataAccessApi(config=task.config_path)
# lists all acquisition dates for use in single tmeslice queries.
product = Satellite.objects.get(datacube_platform=platform).product_prefix + area_id
acquisitions = dc.list_acquisition_dates(product, platform, time=(datetime(1900, 1, 1), date))
return acquisitions