def nlcd_streams(result): """ From a dictionary mapping NLCD codes to the count of stream pixels on each, return a dictionary with keys 'ag_stream_pct', 'low_urban_stream_pct' and 'med_high_urban_stream_pct' which indicate the percent of streams in agricultural areas (namely NLCD 81 Pasture/Hay and 82 Cultivated Crops), low density urban areas (NLCD 21 and 22), and medium and high density urban areas (NLCD 23 and 24) respectively. In addition, we inspect the result to see if it includes an 'error' key. If so, it would indicate that a preceeding task has thrown an exception, and thus we throw an exception with that message. We throw the actual exception in this final task in the chain, rather than one of the preceeding ones, because when creating a chain the resulting AsyncResult points to the last task in the chain. If task in the middle of the chain fails, and the last task doesn't run, the AsyncResult is never marked as Ready. In the case of pure chains this can be addressed with a simple link_error, but in the case of chords the entire set of tasks in the chord's header needs to be Ready before the body can execute. Thus, if the final task never runs, Celery repeatedly calls chord_unlock infinitely causing overflows. By throwing the exception in the final task instead of an intermediate one, we ensure that the group is marked as Ready, and the chord is notified of the failure and suspends execution gracefully. Furthermore, this task is decorated with 'throws=Exception' so that the exception is logged as INFO, rather than ERROR. This is because we have already logged it as an ERROR the first time it was thrown up the chain, and this will reduce noise in the logs. This task should be used as a template for making other geoprocessing post-processing tasks, to be used in geop_tasks. """ if 'error' in result: raise Exception(f'[nlcd_streams] {result["error"]}') # This can't be done in geoprocessing.run because the keys may be tuples, # which are not JSON serializable and thus can't be shared between tasks result = parse(result) ag_count = sum(result.get(nlcd, 0) for nlcd in AG_NLCD_CODES) low_urban_count = sum(result.get(nlcd, 0) for nlcd in [21, 22]) med_high_urban_count = sum(result.get(nlcd, 0) for nlcd in [23, 24]) total = sum(result.values()) ag, low, med_high = (float(count) / total if total > 0 else 0 for count in (ag_count, low_urban_count, med_high_urban_count)) lu_stream_pct = [0.0] * NLU for nlcd, stream_count in result.items(): lu = get_lu_index(nlcd) if lu is not None: lu_stream_pct[lu] += float(stream_count) / total return { 'ag_stream_pct': ag, 'low_urban_stream_pct': low, 'med_high_urban_stream_pct': med_high, 'lu_stream_pct': lu_stream_pct }
def nlcd_soil(result): """ Results are expected to be in the format: { (NLCD ID, Soil Group ID): Count, } We calculate a number of values relying on various combinations of these raster datasets. """ if 'error' in result: raise Exception('[nlcd_soil] {}'.format(result['error'])) ng_count = parse(result) # Raise exception if no NLCD values if len(ng_count.values()) == 0: raise Exception(NO_LAND_COVER) # Split combined counts into separate ones for processing # Reduce [(n, g, t): c] to n_count = {} # [n: sum(c)] ng2_count = {} # [(n, g): sum(c)] for (n, g), count in ng_count.iteritems(): n_count[n] = count + n_count.get(n, 0) # Map soil group values to usable subset g2 = settings.SOIL_GROUP[g] ng2_count[(n, g2)] = count + ng2_count.get((n, g2), 0) return { 'cn': curve_number(n_count, ng2_count), 'landuse_pcts': landuse_pcts(n_count), }
def nlcd_kfactor(result): if 'error' in result: raise Exception('[nlcd_kfactor] {}'.format(result['error'])) result = parse(result) # average kfactor for each land use # see Class1.vb#6431 kf = [0.0] * NLU for nlcd_code, kfactor in result.iteritems(): lu_ind = get_lu_index(nlcd_code) if lu_ind is not None: kf[lu_ind] = kfactor # average kfactor across all land uses, ignoring zero values # see Class1.vb#4151 num_nonzero_kf = len([k for k in kf if k != 0.0]) avg_kf = 0.0 if num_nonzero_kf != 0: avg_kf = sum(kf) / num_nonzero_kf output = { 'kf': kf, 'avg_kf': avg_kf } return output
def analyze_protected_lands(result, area_of_interest=None): if 'error' in result: raise Exception('[analyze_protected_lands] {}'.format(result['error'])) pixel_width = aoi_resolution(area_of_interest) if area_of_interest else 1 result = parse(result) histogram = {} total_count = 0 categories = [] for key, count in result.iteritems(): total_count += count histogram[key] = count + histogram.get(key, 0) for class_id, (code, name) in layer_classmaps.PROTECTED_LANDS.iteritems(): categories.append({ 'area': histogram.get(class_id, 0) * pixel_width * pixel_width, 'class_id': class_id, 'code': code, 'coverage': float(histogram.get(class_id, 0)) / total_count, 'type': name, }) return { 'survey': { 'name': 'protected_lands', 'displayName': 'Protected Lands', 'categories': categories, } }
def gwn(result): """ Derive Groundwater Nitrogen and Phosphorus """ if 'error' in result: raise Exception(f'[gwn] {result["error"]}') result = parse(result) gr_nitr_conc, gr_phos_conc = groundwater_nitrogen_conc(result) return {'gr_nitr_conc': gr_nitr_conc, 'gr_phos_conc': gr_phos_conc}
def gwn(sjs_result): """ Derive Groundwater Nitrogen and Phosphorus """ if 'error' in sjs_result: raise Exception('[gwn] {}'.format(sjs_result['error'])) result = parse(sjs_result) gr_nitr_conc, gr_phos_conc = groundwater_nitrogen_conc(result) return {'gr_nitr_conc': gr_nitr_conc, 'gr_phos_conc': gr_phos_conc}
def avg_awc(result): """ Get `AvgAwc` from MMW-Geoprocessing endpoint Original at [email protected]:4150 """ if 'error' in result: raise Exception(f'[awc] {result["error"]}') result = parse(result) return {'avg_awc': list(result.values())[0]}
def avg_awc(result): """ Get `AvgAwc` from MMW-Geoprocessing endpoint Original at [email protected]:4150 """ if 'error' in result: raise Exception('[awc] {}'.format(result['error'])) result = parse(result) return {'avg_awc': result.values()[0]}
def slope(result): if 'error' in result: raise Exception('[slope] {}'.format(result['error'])) result = parse(result) # average slope over the AOI # see Class1.vb#6252 avg_slope = result[0] output = {'avg_slope': avg_slope} return output
def slope(result): if 'error' in result: raise Exception(f'[slope] {result["error"]}') result = parse(result) # average slope over the AOI # see Class1.vb#6252 avg_slope = result[0] output = {'avg_slope': avg_slope} return output
def soiln(result): """ Get `SoilN` from MMW-Geoprocessing endpoint Originally a static value of 2000 at [email protected]:9587 """ if 'error' in result: raise Exception(f'[soiln] {result["error"]}') result = parse(result) soiln = list(result.values())[0] * 9.0 return {'soiln': soiln}
def soiln(result): """ Get `SoilN` from MMW-Geoprocessing endpoint Originally a static value of 2000 at [email protected]:9587 """ if 'error' in result: raise Exception('[soiln] {}'.format(result['error'])) result = parse(result) soiln = result.values()[0] * 4.0 return {'soiln': soiln}
def soilp(result): """ Get `SoilP` from MMW-Geoprocessing endpoint Originally calculated via lookup table at [email protected]:8975-8988 """ if 'error' in result: raise Exception(f'[soilp] {result["error"]}') result = parse(result) soilp = list(result.values())[0] * 1.6 return {'soilp': soilp}
def analyze_climate(result, category, month): """ Given a climate category ('ppt' or 'tmean') and a month (1 - 12), tags the resulting value with that category and month and returns it as a dictionary. """ if 'error' in result: raise Exception('[analyze_climate_{category}_{month}] {error}'.format( category=category, month=month, error=result['error'])) result = parse(result) key = '{}__{}'.format(category, month) return {key: result[0]}
def recess_coef(result): """ Get `RecessCoef` from MMW-Geoprocessing endpoint Originally a static value 0.06 [email protected]:10333 """ if 'error' in result: raise Exception(f'[recess_coef] {result["error"]}') result = parse(result) recess_coef = list(result.values())[0] * -0.0015 + 0.1103 recess_coef = recess_coef if recess_coef >= 0 else 0.01 return {'recess_coef': recess_coef}
def nlcd_streams_drb(result): """ This callback is run when the geometry falls within the DRB. We calculate the percentage of DRB streams in each land use type. """ if 'error' in result: raise Exception(f'[nlcd_streams_drb] {result["error"]}') result = parse(result) total = sum(result.values()) lu_stream_pct_drb = [0.0] * NLU for nlcd, stream_count in result.items(): lu = get_lu_index(nlcd) if lu is not None: lu_stream_pct_drb[lu] += float(stream_count) / total return {'lu_stream_pct_drb': lu_stream_pct_drb}
def analyze_nlcd(result, area_of_interest=None, nlcd_year='2011_2011'): if 'error' in result: raise Exception(f'[analyze_nlcd_{nlcd_year}] {result["error"]}') pixel_width = aoi_resolution(area_of_interest) if area_of_interest else 1 result = parse(result) histogram = {} total_ara = 0 total_count = 0 categories = [] def area(dictionary, key, default=0): return dictionary.get(key, default) * pixel_width * pixel_width # Convert results to histogram, calculate total for key, count in result.items(): nlcd, ara = key total_count += count total_ara += count if ara == 1 else 0 histogram[nlcd] = count + histogram.get(nlcd, 0) has_ara = total_ara > 0 for nlcd, (code, name) in layer_classmaps.NLCD.items(): categories.append({ 'area': area(histogram, nlcd), 'active_river_area': area(result, (nlcd, 1)) if has_ara else None, 'code': code, 'coverage': float(histogram.get(nlcd, 0)) / total_count, 'nlcd': nlcd, 'type': name, }) return { 'survey': { 'name': f'land_{nlcd_year}', 'displayName': f'Land Use/Cover {nlcd_year[5:]} (NLCD{nlcd_year[2:4]})', 'categories': categories, } }
def nlcd_slope(result): if 'error' in result: raise Exception('[nlcd_slope] {}'.format(result['error'])) result = parse(result) ag_slope_3_count = 0 ag_slope_3_8_count = 0 ag_count = 0 total_count = 0 for (nlcd_code, slope), count in result.iteritems(): if nlcd_code in AG_NLCD_CODES: if slope > 3: ag_slope_3_count += count if 3 < slope < 8: ag_slope_3_8_count += count ag_count += count total_count += count # percent of AOI that is agricultural with slope > 3% # see Class1.vb#7223 ag_slope_3_pct = (float(ag_slope_3_count) / total_count if total_count > 0 else 0.0) # percent of AOI that is agricultural with 3% < slope < 8% ag_slope_3_8_pct = (float(ag_slope_3_8_count) / total_count if total_count > 0 else 0.0) # percent of agricultural parts of AOI with slope > 3% # see Class1.vb#9864 n41 = float(ag_slope_3_count) / ag_count if ag_count > 0 else 0.0 output = { 'ag_slope_3_pct': ag_slope_3_pct, 'ag_slope_3_8_pct': ag_slope_3_8_pct, 'n41': n41 } return output
def collect_worksheet_aois(result, shapes): """ Given a geoprocessing result of NLCD and NLCD+Streams for every area of interest within every HUC-12, processes the raw results and returns a dictionary a area of interest IDs corresponding to their processed results. """ if 'error' in result: raise Exception(f'[collect_worksheet_aois] {result["error"]}') NULL_RESULT = {'nlcd_streams': {}, 'nlcd': {}} collection = {} for shape in shapes: output = result.get(shape['id'], NULL_RESULT) nlcd = collect_nlcd(parse(output['nlcd']), shape['shape']) streams = stream_data(nlcd_streams(output['nlcd_streams']), shape['shape']) collection[shape['id']] = {'nlcd': nlcd, 'streams': streams} return collection