def perform_task_chunking(parameters, task_id=None): """Chunk parameter sets into more manageable sizes Uses functions provided by the task model to create a group of parameter sets that make up the arg. Args: parameters: parameter stream containing all kwargs to load data Returns: parameters with a list of geographic and time ranges """ if parameters is None: return None task = NdviAnomalyTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) dates = dc.list_acquisition_dates(**parameters) task_chunk_sizing = task.get_chunk_size() geographic_chunks = create_geographic_chunks( longitude=parameters['longitude'], latitude=parameters['latitude'], geographic_chunk_size=task_chunk_sizing['geographic']) grouped_dates_params = {**parameters} grouped_dates_params.update({ 'time': (datetime(1000, 1, 1), task.time_start - timedelta(microseconds=1)) }) acquisitions = dc.list_acquisition_dates(**grouped_dates_params) grouped_dates = group_datetimes_by_month( acquisitions, months=list(map(int, task.baseline_selection.split(",")))) # create a single monolithic list of all acq. dates - there should be only one. time_chunks = [] for date_group in grouped_dates: time_chunks.extend(grouped_dates[date_group]) # time chunks casted to a list, essnetially. time_chunks = [time_chunks] logger.info("Time chunks: {}, Geo chunks: {}".format( len(time_chunks), len(geographic_chunks))) dc.close() task.update_status("WAIT", "Chunked parameter set.") return { 'parameters': parameters, 'geographic_chunks': geographic_chunks, 'time_chunks': time_chunks }
def validate_parameters(parameters, task_id=None): """Validate parameters generated by the parameter parsing task All validation should be done here - are there data restrictions? Combinations that aren't allowed? etc. Returns: parameter dict with all keyword args required to load data. -or- updates the task with ERROR and a message, returning None """ task = NdviAnomalyTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) acquisitions = dc.list_acquisition_dates(**parameters) if len(acquisitions) < 1: task.complete = True task.update_status("ERROR", "There are no acquistions for this parameter set.") return None # the actual acquisitino exists, lets try the baseline: validation_params = {**parameters} # there were no acquisitions in the year 1000, hopefully validation_params.update({ 'time': (task.time_start.replace(year=task.time_start.year - 5), task.time_start - timedelta(microseconds=1)) }) acquisitions = dc.list_acquisition_dates(**validation_params) # list/map/int chain required to cast int to each baseline month, it won't work if they're strings. grouped_dates = group_datetimes_by_month( acquisitions, months=list(map(int, task.baseline_selection.split(",")))) if not grouped_dates: task.complete = True task.update_status("ERROR", "There are no acquistions for this parameter set.") return None task.update_status("WAIT", "Validated parameters.") if not dc.validate_measurements(parameters['product'], parameters['measurements']): parameters['measurements'] = [ 'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa' ] dc.close() return parameters
def validate_parameters(parameters, task_id=None): """Validate parameters generated by the parameter parsing task All validation should be done here - are there data restrictions? Combinations that aren't allowed? etc. Returns: parameter dict with all keyword args required to load data. -or- updates the task with ERROR and a message, returning None """ task = CloudCoverageTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) #validate for any number of criteria here - num acquisitions, etc. acquisitions = dc.list_acquisition_dates(**parameters) if len(acquisitions) < 1: task.complete = True task.update_status("ERROR", "There are no acquistions for this parameter set.") return None task.update_status("WAIT", "Validated parameters.") if not dc.validate_measurements(parameters['product'], parameters['measurements']): parameters['measurements'] = ['blue', 'green', 'red', 'pixel_qa'] dc.close() return parameters
def perform_task_chunking(parameters, task_id=None): """Chunk parameter sets into more manageable sizes Uses functions provided by the task model to create a group of parameter sets that make up the arg. Args: parameters: parameter stream containing all kwargs to load data Returns: parameters with a list of geographic and time ranges """ if parameters is None: return None task = SlipTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) dates = dc.list_acquisition_dates(**parameters) task_chunk_sizing = task.get_chunk_size() geographic_chunks = create_geographic_chunks( longitude=parameters['longitude'], latitude=parameters['latitude'], geographic_chunk_size=task_chunk_sizing['geographic']) time_chunks = generate_baseline(dates, task.baseline_length) logger.info("Time chunks: {}, Geo chunks: {}".format(len(time_chunks), len(geographic_chunks))) dc.close() task.update_status("WAIT", "Chunked parameter set.") return {'parameters': parameters, 'geographic_chunks': geographic_chunks, 'time_chunks': time_chunks}
def validate_parameters(parameters, task_id=None): """Validate parameters generated by the parameter parsing task All validation should be done here - are there data restrictions? Combinations that aren't allowed? etc. Returns: parameter dict with all keyword args required to load data. -or- updates the task with ERROR and a message, returning None """ task = SlipTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) acquisitions = dc.list_acquisition_dates(**parameters) if len(acquisitions) < 1: task.complete = True task.update_status("ERROR", "There are no acquistions for this parameter set.") return None if len(acquisitions) < task.baseline_length + 1: task.complete = True task.update_status("ERROR", "There are an insufficient number of acquisitions for your baseline length.") return None validation_parameters = {**parameters} validation_parameters.pop('time') validation_parameters.pop('measurements') validation_parameters.update({'product': 'terra_aster_gdm_' + task.area_id, 'platform': 'TERRA'}) if len(dc.list_acquisition_dates(**validation_parameters)) < 1: task.complete = True task.update_status("ERROR", "There is no elevation data for this parameter set.") return None task.update_status("WAIT", "Validated parameters.") if not dc.validate_measurements(parameters['product'], parameters['measurements']): parameters['measurements'] = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa'] dc.close() return parameters
def perform_task_chunking(parameters, task_id=None): """Chunk parameter sets into more manageable sizes Uses functions provided by the task model to create a group of parameter sets that make up the arg. Args: parameters: parameter stream containing all kwargs to load data Returns: parameters with a list of geographic and time ranges """ if parameters is None: return None task = CoastalChangeTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) dates = dc.list_acquisition_dates(**parameters) task_chunk_sizing = task.get_chunk_size() geographic_chunks = create_geographic_chunks( longitude=parameters['longitude'], latitude=parameters['latitude'], geographic_chunk_size=task_chunk_sizing['geographic']) grouped_dates = group_datetimes_by_year(dates) # we need to pair these with the first year - subsequent years. time_chunks = None if task.animated_product.animation_id == 'none': # first and last only time_chunks = [[ grouped_dates[task.time_start], grouped_dates[task.time_end] ]] else: initial_year = grouped_dates.pop(task.time_start) time_chunks = [[initial_year, grouped_dates[year]] for year in grouped_dates] logger.info("Time chunks: {}, Geo chunks: {}".format( len(time_chunks), len(geographic_chunks))) dc.close() task.update_status("WAIT", "Chunked parameter set.") return { 'parameters': parameters, 'geographic_chunks': geographic_chunks, 'time_chunks': time_chunks }
def validate_parameters(parameters, task_id=None): """Validate parameters generated by the parameter parsing task All validation should be done here - are there data restrictions? Combinations that aren't allowed? etc. Returns: parameter dict with all keyword args required to load data. -or- updates the task with ERROR and a message, returning None """ task = BandMathTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) #validate for any number of criteria here - num acquisitions, etc. acquisitions = dc.list_acquisition_dates(**parameters) if len(acquisitions) < 1: task.complete = True task.update_status("ERROR", "There are no acquistions for this parameter set.") return None if task.compositor.id == "median_pixel" and (task.time_end - task.time_start).days > 367: task.complete = True task.update_status( "ERROR", "Median pixel operations are only supported for single year time periods." ) return None task.update_status("WAIT", "Validated parameters.") if not dc.validate_measurements(parameters['product'], parameters['measurements']): parameters['measurements'] = [ 'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa' ] dc.close() return parameters
def validate_parameters(parameters, task_id=None): """Validate parameters generated by the parameter parsing task All validation should be done here - are there data restrictions? Combinations that aren't allowed? etc. Returns: parameter dict with all keyword args required to load data. -or- updates the task with ERROR and a message, returning None """ task = CoastalChangeTask.objects.get(pk=task_id) dc = DataAccessApi(config=task.config_path) validation_params = dict(parameters) # verify that both the start and end year have acquisitions for year in parameters['time']: validation_params.update( {'time': (year, year.replace(year=year.year + 1))}) acquisitions = dc.list_acquisition_dates(**validation_params) if len(acquisitions) < 1: task.complete = True task.update_status( "ERROR", "There must be at least one acquisition in both the start and ending year." ) return None task.update_status("WAIT", "Validated parameters.") if not dc.validate_measurements(parameters['product'], parameters['measurements']): parameters['measurements'] = [ 'blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa' ] dc.close() return parameters
def get_acquisition_list(task, area_id, platform, date): dc = DataAccessApi(config=task.config_path) # lists all acquisition dates for use in single tmeslice queries. product = Satellite.objects.get(datacube_platform=platform).product_prefix + area_id acquisitions = dc.list_acquisition_dates(product, platform, time=(datetime(1900, 1, 1), date)) return acquisitions