def nlcd_streams(result):
"""
From a dictionary mapping NLCD codes to the count of stream pixels on
each, return a dictionary with keys 'ag_stream_pct', 'low_urban_stream_pct'
and 'med_high_urban_stream_pct' which indicate the percent of streams in
agricultural areas (namely NLCD 81 Pasture/Hay and 82 Cultivated Crops),
low density urban areas (NLCD 21 and 22), and medium and high density urban
areas (NLCD 23 and 24) respectively.
In addition, we inspect the result to see if it includes an 'error' key.
If so, it would indicate that a preceeding task has thrown an exception,
and thus we throw an exception with that message.
We throw the actual exception in this final task in the chain, rather than
one of the preceeding ones, because when creating a chain the resulting
AsyncResult points to the last task in the chain. If task in the middle of
the chain fails, and the last task doesn't run, the AsyncResult is never
marked as Ready. In the case of pure chains this can be addressed with a
simple link_error, but in the case of chords the entire set of tasks in
the chord's header needs to be Ready before the body can execute. Thus, if
the final task never runs, Celery repeatedly calls chord_unlock infinitely
causing overflows. By throwing the exception in the final task instead of
an intermediate one, we ensure that the group is marked as Ready, and the
chord is notified of the failure and suspends execution gracefully.
Furthermore, this task is decorated with 'throws=Exception' so that the
exception is logged as INFO, rather than ERROR. This is because we have
already logged it as an ERROR the first time it was thrown up the chain,
and this will reduce noise in the logs.
This task should be used as a template for making other geoprocessing
post-processing tasks, to be used in geop_tasks.
"""
if 'error' in result:
raise Exception(f'[nlcd_streams] {result["error"]}')
# This can't be done in geoprocessing.run because the keys may be tuples,
# which are not JSON serializable and thus can't be shared between tasks
result = parse(result)
ag_count = sum(result.get(nlcd, 0) for nlcd in AG_NLCD_CODES)
low_urban_count = sum(result.get(nlcd, 0) for nlcd in [21, 22])
med_high_urban_count = sum(result.get(nlcd, 0) for nlcd in [23, 24])
total = sum(result.values())
ag, low, med_high = (float(count) / total if total > 0 else 0
for count in (ag_count, low_urban_count,
med_high_urban_count))
lu_stream_pct = [0.0] * NLU
for nlcd, stream_count in result.items():
lu = get_lu_index(nlcd)
if lu is not None:
lu_stream_pct[lu] += float(stream_count) / total
return {
'ag_stream_pct': ag,
'low_urban_stream_pct': low,
'med_high_urban_stream_pct': med_high,
'lu_stream_pct': lu_stream_pct
}
def nlcd_soil(result):
"""
Results are expected to be in the format:
{
(NLCD ID, Soil Group ID): Count,
}
We calculate a number of values relying on various combinations
of these raster datasets.
"""
if 'error' in result:
raise Exception('[nlcd_soil] {}'.format(result['error']))
ng_count = parse(result)
# Raise exception if no NLCD values
if len(ng_count.values()) == 0:
raise Exception(NO_LAND_COVER)
# Split combined counts into separate ones for processing
# Reduce [(n, g, t): c] to
n_count = {} # [n: sum(c)]
ng2_count = {} # [(n, g): sum(c)]
for (n, g), count in ng_count.iteritems():
n_count[n] = count + n_count.get(n, 0)
# Map soil group values to usable subset
g2 = settings.SOIL_GROUP[g]
ng2_count[(n, g2)] = count + ng2_count.get((n, g2), 0)
return {
'cn': curve_number(n_count, ng2_count),
'landuse_pcts': landuse_pcts(n_count),
}
def nlcd_kfactor(result):
if 'error' in result:
raise Exception('[nlcd_kfactor] {}'.format(result['error']))
result = parse(result)
# average kfactor for each land use
# see Class1.vb#6431
kf = [0.0] * NLU
for nlcd_code, kfactor in result.iteritems():
lu_ind = get_lu_index(nlcd_code)
if lu_ind is not None:
kf[lu_ind] = kfactor
# average kfactor across all land uses, ignoring zero values
# see Class1.vb#4151
num_nonzero_kf = len([k for k in kf if k != 0.0])
avg_kf = 0.0
if num_nonzero_kf != 0:
avg_kf = sum(kf) / num_nonzero_kf
output = {
'kf': kf,
'avg_kf': avg_kf
}
return output
def analyze_protected_lands(result, area_of_interest=None):
if 'error' in result:
raise Exception('[analyze_protected_lands] {}'.format(result['error']))
pixel_width = aoi_resolution(area_of_interest) if area_of_interest else 1
result = parse(result)
histogram = {}
total_count = 0
categories = []
for key, count in result.iteritems():
total_count += count
histogram[key] = count + histogram.get(key, 0)
for class_id, (code, name) in layer_classmaps.PROTECTED_LANDS.iteritems():
categories.append({
'area': histogram.get(class_id, 0) * pixel_width * pixel_width,
'class_id': class_id,
'code': code,
'coverage': float(histogram.get(class_id, 0)) / total_count,
'type': name,
})
return {
'survey': {
'name': 'protected_lands',
'displayName': 'Protected Lands',
'categories': categories,
}
}
def gwn(result):
"""
Derive Groundwater Nitrogen and Phosphorus
"""
if 'error' in result:
raise Exception(f'[gwn] {result["error"]}')
result = parse(result)
gr_nitr_conc, gr_phos_conc = groundwater_nitrogen_conc(result)
return {'gr_nitr_conc': gr_nitr_conc, 'gr_phos_conc': gr_phos_conc}
def gwn(sjs_result):
"""
Derive Groundwater Nitrogen and Phosphorus
"""
if 'error' in sjs_result:
raise Exception('[gwn] {}'.format(sjs_result['error']))
result = parse(sjs_result)
gr_nitr_conc, gr_phos_conc = groundwater_nitrogen_conc(result)
return {'gr_nitr_conc': gr_nitr_conc, 'gr_phos_conc': gr_phos_conc}
def avg_awc(result):
"""
Get `AvgAwc` from MMW-Geoprocessing endpoint
Original at [email protected]:4150
"""
if 'error' in result:
raise Exception(f'[awc] {result["error"]}')
result = parse(result)
return {'avg_awc': list(result.values())[0]}
def avg_awc(result):
"""
Get `AvgAwc` from MMW-Geoprocessing endpoint
Original at [email protected]:4150
"""
if 'error' in result:
raise Exception('[awc] {}'.format(result['error']))
result = parse(result)
return {'avg_awc': result.values()[0]}
def slope(result):
if 'error' in result:
raise Exception('[slope] {}'.format(result['error']))
result = parse(result)
# average slope over the AOI
# see Class1.vb#6252
avg_slope = result[0]
output = {'avg_slope': avg_slope}
return output
def slope(result):
if 'error' in result:
raise Exception(f'[slope] {result["error"]}')
result = parse(result)
# average slope over the AOI
# see Class1.vb#6252
avg_slope = result[0]
output = {'avg_slope': avg_slope}
return output
def soiln(result):
"""
Get `SoilN` from MMW-Geoprocessing endpoint
Originally a static value of 2000 at [email protected]:9587
"""
if 'error' in result:
raise Exception(f'[soiln] {result["error"]}')
result = parse(result)
soiln = list(result.values())[0] * 9.0
return {'soiln': soiln}
def soiln(result):
"""
Get `SoilN` from MMW-Geoprocessing endpoint
Originally a static value of 2000 at [email protected]:9587
"""
if 'error' in result:
raise Exception('[soiln] {}'.format(result['error']))
result = parse(result)
soiln = result.values()[0] * 4.0
return {'soiln': soiln}
def soilp(result):
"""
Get `SoilP` from MMW-Geoprocessing endpoint
Originally calculated via lookup table at [email protected]:8975-8988
"""
if 'error' in result:
raise Exception(f'[soilp] {result["error"]}')
result = parse(result)
soilp = list(result.values())[0] * 1.6
return {'soilp': soilp}
def analyze_climate(result, category, month):
"""
Given a climate category ('ppt' or 'tmean') and a month (1 - 12), tags
the resulting value with that category and month and returns it as a
dictionary.
"""
if 'error' in result:
raise Exception('[analyze_climate_{category}_{month}] {error}'.format(
category=category, month=month, error=result['error']))
result = parse(result)
key = '{}__{}'.format(category, month)
return {key: result[0]}
def recess_coef(result):
"""
Get `RecessCoef` from MMW-Geoprocessing endpoint
Originally a static value 0.06 [email protected]:10333
"""
if 'error' in result:
raise Exception(f'[recess_coef] {result["error"]}')
result = parse(result)
recess_coef = list(result.values())[0] * -0.0015 + 0.1103
recess_coef = recess_coef if recess_coef >= 0 else 0.01
return {'recess_coef': recess_coef}
def nlcd_streams_drb(result):
"""
This callback is run when the geometry falls within the DRB. We calculate
the percentage of DRB streams in each land use type.
"""
if 'error' in result:
raise Exception(f'[nlcd_streams_drb] {result["error"]}')
result = parse(result)
total = sum(result.values())
lu_stream_pct_drb = [0.0] * NLU
for nlcd, stream_count in result.items():
lu = get_lu_index(nlcd)
if lu is not None:
lu_stream_pct_drb[lu] += float(stream_count) / total
return {'lu_stream_pct_drb': lu_stream_pct_drb}
def analyze_nlcd(result, area_of_interest=None, nlcd_year='2011_2011'):
if 'error' in result:
raise Exception(f'[analyze_nlcd_{nlcd_year}] {result["error"]}')
pixel_width = aoi_resolution(area_of_interest) if area_of_interest else 1
result = parse(result)
histogram = {}
total_ara = 0
total_count = 0
categories = []
def area(dictionary, key, default=0):
return dictionary.get(key, default) * pixel_width * pixel_width
# Convert results to histogram, calculate total
for key, count in result.items():
nlcd, ara = key
total_count += count
total_ara += count if ara == 1 else 0
histogram[nlcd] = count + histogram.get(nlcd, 0)
has_ara = total_ara > 0
for nlcd, (code, name) in layer_classmaps.NLCD.items():
categories.append({
'area': area(histogram, nlcd),
'active_river_area': area(result, (nlcd, 1)) if has_ara else None,
'code': code,
'coverage': float(histogram.get(nlcd, 0)) / total_count,
'nlcd': nlcd,
'type': name,
})
return {
'survey': {
'name': f'land_{nlcd_year}',
'displayName':
f'Land Use/Cover {nlcd_year[5:]} (NLCD{nlcd_year[2:4]})',
'categories': categories,
}
}
def nlcd_slope(result):
if 'error' in result:
raise Exception('[nlcd_slope] {}'.format(result['error']))
result = parse(result)
ag_slope_3_count = 0
ag_slope_3_8_count = 0
ag_count = 0
total_count = 0
for (nlcd_code, slope), count in result.iteritems():
if nlcd_code in AG_NLCD_CODES:
if slope > 3:
ag_slope_3_count += count
if 3 < slope < 8:
ag_slope_3_8_count += count
ag_count += count
total_count += count
# percent of AOI that is agricultural with slope > 3%
# see Class1.vb#7223
ag_slope_3_pct = (float(ag_slope_3_count) / total_count
if total_count > 0 else 0.0)
# percent of AOI that is agricultural with 3% < slope < 8%
ag_slope_3_8_pct = (float(ag_slope_3_8_count) / total_count
if total_count > 0 else 0.0)
# percent of agricultural parts of AOI with slope > 3%
# see Class1.vb#9864
n41 = float(ag_slope_3_count) / ag_count if ag_count > 0 else 0.0
output = {
'ag_slope_3_pct': ag_slope_3_pct,
'ag_slope_3_8_pct': ag_slope_3_8_pct,
'n41': n41
}
return output
def collect_worksheet_aois(result, shapes):
"""
Given a geoprocessing result of NLCD and NLCD+Streams for every
area of interest within every HUC-12, processes the raw results
and returns a dictionary a area of interest IDs corresponding to
their processed results.
"""
if 'error' in result:
raise Exception(f'[collect_worksheet_aois] {result["error"]}')
NULL_RESULT = {'nlcd_streams': {}, 'nlcd': {}}
collection = {}
for shape in shapes:
output = result.get(shape['id'], NULL_RESULT)
nlcd = collect_nlcd(parse(output['nlcd']),
shape['shape'])
streams = stream_data(nlcd_streams(output['nlcd_streams']),
shape['shape'])
collection[shape['id']] = {'nlcd': nlcd, 'streams': streams}
return collection
