def test_term_frequency_untiled_one_shape(self):
dataset_catalog_file = get_path("../catalog/cdl_2014_untiled.json")
rd = read_catalog(dataset_catalog_file)
df = self.make_dataframe(rd.query(self.vl_one_shape))
# Compare against expected output computed via single-tile computation
# read from a file.
df_distance = (self.df_expected_one_shape - df.loc[self.one_shape_id])\
.apply(np.abs)
assert(df_distance[1] < 0.05)
assert(df_distance[5] < 0.05)
def test_term_frequency_untiled_one_shape(self):
dataset_catalog_file = get_path("../catalog/cdl_2014_untiled.json")
rd = read_catalog(dataset_catalog_file)
df = self.make_dataframe(rd.query(self.vl_one_shape))
# Compare against expected output computed via single-tile computation
# read from a file.
df_distance = (self.df_expected_one_shape - df.loc[self.one_shape_id])\
.apply(np.abs)
assert (df_distance[1] < 0.05)
assert (df_distance[5] < 0.05)
def test_co_ne_border(self):
p = get_path("vector/clu/clu_public_a_co095.shp")
vl, _ = read_layer(p)
p = get_path("vector/co_ne_border.geojson")
co_ne_border, df = read_geojson(p)
vl = vl.within(co_ne_border.bbox())
rd = read_catalog(get_path("../catalog/co_soil.json"))
for r in rd.query(vl):
r
#compute_stats(r.values, r.weights)
rd = read_catalog(get_path("../catalog/co_soil_bad.json"))
failed = False
try:
for r in rd.query(vl):
r
#compute_stats(r.values, r.weights)
except IndexError:
failed = True
assert failed
def test_co_ne_border(self):
p = get_path("vector/clu/clu_public_a_co095.shp")
vl, _ = read_layer(p)
p = get_path("vector/co_ne_border.geojson")
co_ne_border, df = read_geojson(p)
vl = vl.within(co_ne_border.bbox())
rd = read_catalog(get_path("../catalog/co_soil.json"))
for r in rd.query(vl):
r
#compute_stats(r.values, r.weights)
rd = read_catalog(get_path("../catalog/co_soil_bad.json"))
failed = False
try:
for r in rd.query(vl):
r
#compute_stats(r.values, r.weights)
except IndexError:
failed = True
assert failed
def test_shapes_outside_raster_should_be_filtered(self):
# 2 shapes in IL that should be in range,
# plus 2 shapes in CA that should be out of range.
p = get_path("vector/clu/four_shapes_2il_2ca.geojson")
vl_outside, vl_df = read_geojson(p)
assert (len(vl_outside.keys()) == 4)
dataset_catalog_file = get_path("../catalog/cdl_2014.json")
rd = read_catalog(dataset_catalog_file)
df = self.make_dataframe(rd.query(vl_outside))
assert (len(df.index) == 4)
sums = df.sum(axis=1).map(int)
assert df[sums < 1e-6].shape[0] == 2
assert df[(sums - 1).map(np.abs) < 1e-6].shape[0] == 2
def test_term_frequency_untiled_all_shapes(self):
dataset_catalog_file = get_path("../catalog/cdl_2014_untiled.json")
rd = read_catalog(dataset_catalog_file)
df = self.make_dataframe(rd.query(self.vl))
assert (len(df.index) == 23403)
# Compare against expected output computed via single-tile computation
# read from a file.
df_distance = (self.df_expected - df).applymap(np.abs)
corn_error = (df_distance[1] * self.areas).sum()
soy_error = (df_distance[5] * self.areas).sum()
assert (corn_error < 0.02), corn_error
assert (soy_error < 0.02), soy_error
def test_shapes_outside_raster_should_be_filtered(self):
# 2 shapes in IL that should be in range,
# plus 2 shapes in CA that should be out of range.
p = get_path("vector/clu/four_shapes_2il_2ca.geojson")
vl_outside, vl_df = read_geojson(p)
assert (len(vl_outside.keys()) == 4)
dataset_catalog_file = get_path("../catalog/cdl_2014.json")
rd = read_catalog(dataset_catalog_file)
df = self.make_dataframe(rd.query(vl_outside))
assert (len(df.index) == 4)
sums = df.sum(axis=1).map(int)
assert df[sums < 1e-6].shape[0] == 2
assert df[(sums - 1).map(np.abs) < 1e-6].shape[0] == 2
def test_term_frequency_untiled_all_shapes(self):
dataset_catalog_file = get_path("../catalog/cdl_2014_untiled.json")
rd = read_catalog(dataset_catalog_file)
df = self.make_dataframe(rd.query(self.vl))
assert (len(df.index) == 23403)
# Compare against expected output computed via single-tile computation
# read from a file.
df_distance = (self.df_expected - df).applymap(np.abs)
corn_error = (df_distance[1]*self.areas).sum()
soy_error = (df_distance[5]*self.areas).sum()
assert(corn_error < 0.02), corn_error
assert(soy_error < 0.02), soy_error
def setup_class(cls):
os.chdir(base)
# Let's add some CLUs for Washington County, IL
cls.vl, _ = read_layer(get_path("vector/clu/clu_public_a_il189.shp"),
index="uniqueid")
# Figure out which raster files we'll need.
cls.dataset = read_catalog(get_path("../catalog/cdl_2014.json"))
# Create a RasterBand for the raster. Raster data is stored
# and read from there.
cls.rb = RasterBand(get_path("raster/95000_45000.tif"))
# Convert CLUs to pixels.
cls.px_shps = cls.dataset.to_pixels(cls.vl)
def processplace(placeid, urbanextenttable, uidata):
BASERASTERPATH = os.environ.get("HIRASTERBASE",
'/data/rasterstorage')
RASTERSETS = {
'grump2000': 'grump/population2000.json',
'grump2005': 'grump/population2005.json',
'grump2010': 'grump/population2010.json',
'grump2015': 'grump/population2015.json',
'grump2020': 'grump/population2020.json',
'landscanpopulation': 'landscan/landscan.json',
'impervious': 'nlcd/impervious/nlcd_impervious_2011.json',
'nlcd': 'nlcd/landcover/nlcd_landcover_2011.json'
}
# 'nlcd/impervious/nlcd_impervious_2001.json',
# 'nlcd/impervious/nlcd_impervious_2006.json',
results = get_geom_from_postgis(urbanextenttable, placeid)
urbanextentgeom = from_series(pd.Series([results[0]]))
urbanextentgeomjson = results[1]
urbanextentarea = results[2]
bufferextentgeom = None
bufferextentgeomjson = None
if uidata['extentbuffer']:
results = get_geom_from_postgis_buffer(
urbanextenttable, placeid,
float(uidata['extentbuffer'][0]))
bufferextentgeom = from_series(pd.Series([results[0]]))
bufferextentgeomjson = results[1]
bufferextentarea = results[2]
elif uidata['fixedbuffer']:
results = get_geom_from_postgis_buffer(
urbanextenttable, placeid, int(uidata['fixedbuffer'][0]),
True)
bufferextentgeom = from_series(pd.Series([results[0]]))
bufferextentgeomjson = results[1]
bufferextentarea = results[2]
rasterresults = {}
print bufferextentgeom
for rasterkey in RASTERSETS.keys():
if uidata.get(rasterkey, None):
try:
tempr = read_catalog(
op.join(BASERASTERPATH, RASTERSETS[rasterkey]))
rasterresults[rasterkey] = {}
result = tempr.query(urbanextentgeom).next()
rasterresults[rasterkey]['urbanextent'] = {
'sum':
float(result.values.sum()),
'mean':
float(result.values.mean()),
'weightedmean':
float((result.values * result.weights).sum() /
result.weights.sum()),
'std':
float(result.values.std()),
'weightedsum':
float((result.values * result.weights).sum()),
'min':
float(result.values.min()),
'max':
float(result.values.max())
# 'valuecounts': dict(result.value_counts())
}
if bufferextentgeomjson:
result = tempr.query(bufferextentgeom).next()
rasterresults[rasterkey]['bufferextent'] = {
'sum':
float(result.values.sum()),
'mean':
float(result.values.mean()),
'weightedmean':
float((result.values * result.weights).sum() /
result.weights.sum()),
'std':
float(result.values.std()),
'weightedsum':
float((result.values * result.weights).sum()),
'min':
float(result.values.min()),
'max':
float(result.values.max())
# 'valuecounts': dict(result.value_counts())
}
except Exception, e:
traceback.print_exc()
print e
pass
def test_unconventional_tilepath(self):
dataset_catalog_file = get_path("../catalog/cdl_2014_tilepath.json")
rd = read_catalog(dataset_catalog_file, workdir='data/raster')
df = self.make_dataframe(rd.query(self.vl_one_shape))
assert (len(df.index) == 1)
'ruralgeomwgs84': wkb.loads(str(firstitem[2])),
}
except Exception, e:
print e
print placeid
print s['usgsplacename']
print s['gnisid']
print "loading the dem.tif file"
demfile = "/Volumes/UrbisBackup/rasterstorage/dem/dem.tif"
demsrc = rasterio.open(demfile)
profile = demsrc.profile
print "loading the dem.json file"
dempyspatialpath = "/Volumes/UrbisBackup/rasterstorage/dem/dem.json"
dempyspatial = read_catalog(dempyspatialpath)
print "doing", s[u'usgsplacename']
if s['censusurb'].get('processbuffer', False):
return (
placeid,
s,
)
try:
buffergeom = mapping(s['censusurb']['ruralgeom'])
try:
os.mkdir('scratch')
except:
pass
def test_unconventional_tilepath(self):
dataset_catalog_file = get_path("../catalog/cdl_2014_tilepath.json")
rd = read_catalog(dataset_catalog_file, workdir='data/raster')
df = self.make_dataframe(rd.query(self.vl_one_shape))
assert (len(df.index) == 1)
