Extract, reproject, resample, and recalculate Landscan data set.
Created on Thu Mar 5 08:28:07 2020
@author: twillia2
"""
import numpy as np
import subprocess as sp
import rasterio
import xarray as xr
from dask.distributed import Client
from gdalmethods import Data_Path, translate, warp, gdal_options
dpp = Data_Path("/scratch/twillia2/weto/populations/data")
dpc = Data_Path("/projects/rev/data/conus/")
# These come in ESRI grids, we can translate them to geotiffs
sample_src = '~/Downloads/landscan2016_cyprus/cyprus_pop'
sample_dst = dpp.join("test_translation.tif")
translate(sample_src, sample_dst, overwrite=True, format="GTiff")
# We want our geometries to match the exclusion datasets, like this one
template = rasterio.open(dpc.join("_windready_conus.tif"))
# Warp new sample to albers with nearest neighbor first, don't resample
src = dpp.join("rasters/wgs/landscan_2017/landscan_night_2017.tif")
dst = dpp.join("rasters/albers/landscan_near.tif")
# After a few resampling methods, nearest neighbor looks good enough
"""
Build overlapping wildlife range tiffs
"""
import dask.array as da
from itertools import combinations
import numpy as np
import os
import rasterio
import xarray as xr
from numpy.random import randint
from dask.distributed import Client
from gdalmethods import Data_Path, to_raster
DP = Data_Path("/projects/rev/data/conus/wildlife")
DP2 = Data_Path("/scratch/twillia2/weto/wildlife")
RANGE_CATEGORIES = {
1: {
"eastern_red": "bat_ranges/eastern_red_bat_range.tif",
"hoary": "bat_ranges/hoary_range.tif",
"silver": "bat_ranges/silver_range.tif",
# "big_brown": "bat_ranges/.tif",
"tricolor": "bat_ranges/tricolored_bat_extent.tif",
"little_brown": "bat_ranges/little_brown_bat_extent.tif",
"brazilian": "bat_ranges/brazilian_free_tail_range.tif"
},
2: {
"indiana": "bat_ranges/spext_indiana_bat.tif",
"long_eared": "bat_ranges/northern_long_eared_bat_extent.tif",
Created on Fri Feb 14 09:18:28 2020
@author: twillia2
"""
import dask.array as da
import numpy as np
import xarray as xr
from dask.distributed import Client
from gdalmethods import Data_Path, to_raster, warp
# Data Paths
# dp = Data_Path("~/Box/WETO 1.2/data")
DP = Data_Path("/scratch/twillia2/weto/data")
EXL_PATH = DP.join("rasters", "core_exclusions_raster",
"alopez_core_exclusions.tif")
ROAD_PATH = DP.join("rasters", "core_exclusions_raster", "conus_roads.tif")
CONUS_PATH = DP.join("rasters", "albers", "90m", "conus.tif")
RAIL_PATH = DP.join("rasters", "core_exclusions_raster", "conus_rail.tif")
# Best chunk size?
CHUNKS = {'band': 1, 'x': 5000, 'y': 5000}
def build_exclusions():
""" Build exclusion file for the WETO rent map."""
# Using the 'core exclusions raster' that antyhony put together.
excl = xr.open_rasterio(EXL_PATH, chunks=CHUNKS)
"""
import fiona
import itertools
import multiprocessing as mp
import numpy as np
import rasterio
import sys
import p_tqdm
from gdalmethods import Data_Path
from rasterstats import zonal_stats
from tqdm import tqdm
# Paths
DPA = Data_Path("/scratch/twillia2/weto/populations/albers")
TIF = DPA.join("landscan_night_2017.tif")
SHP = DPA.join("sc_circle_single_buffer.gpkg")
NA = rasterio.open(TIF).nodata
def get_chunks(data, n):
"""Yield successive n-sized chunks from a slice-able iterable."""
chunks = []
for i in range(0, len(data), n):
chunks.append(data[i:i + n])
return chunks
def zonal_stats_partial(feats):
Create a code raster of categories from the original.
Created on Thu Mar 26 12:40:25 2020
@author: twillia2
"""
import numpy as np
import pandas as pd
import rasterio
import xarray as xr
from dask.distributed import Client
from gdalmethods import Data_Path, to_raster
DP = Data_Path("/scratch/twillia2/weto/data")
CATEGORIES = {1: "BLM", 2: "Tribal", 3: "State", 4: "Private"}
CHUNKS = {'band': 1, 'x': 5000, 'y': 5000}
def categorize(row):
if "BLM" in row["type"]:
return 1
if "Tribal" in row["type"]:
return 2
if "State Land" in row["type"]:
return 3
else:
return 4
"""
import os
from tqdm import tqdm
import dask.array as da
import geopandas as gpd
import pandas as pd
import rasterio
import xarray as xr
from dask.distributed import Client
from gdalmethods import Data_Path, rasterize
DP = Data_Path("/scratch/twillia2/weto/data")
TEMPLATE = DP.join("rasters", "albers", "acre", "rent_map.tif")
DIVISIONS_PATH = DP.join("rasters", "albers", "acre", "census_divisions.tif")
# Get a shapefile of the census blocks
def make_divisions():
"""Download reproject and rasterize the US Census divisions."""
# Paths
shp_path = DP.join("shapefiles", "USA", "cb_2018_us_division_5m.shp")
tproj = rasterio.open(TEMPLATE).crs.to_proj4()
if not os.path.exists(shp_path):
# Read
"""
import os
import dask.array as da
import numpy as np
import xarray as xr
from osgeo import gdal
from dask.distributed import Client
from gdalmethods import Data_Path, to_raster
from tqdm import tqdm
# Data Paths
DP = Data_Path("/scratch/twillia2/weto/data")
DPM = Data_Path(DP.join("rasters/albers/acre/masks"))
# Set the chunk sizes
CHUNKS = {'band': 1, 'x': 5000, 'y': 5000}
# Reference geometry
REFERENCE = gdal.Open(DP.join("rasters/albers/acre/blm_codes.tif"))
PROJ = REFERENCE.GetProjection()
GEOM = REFERENCE.GetGeoTransform()
@da.as_gufunc(signature="(i)->(i)",
output_dtypes=int,
vectorize=True,
allow_rechunk=True)
Created on Mon Feb 3 09:30:06 2020
@author: twillia2
"""
import camelot
import geopandas as gpd
import os
import pandas as pd
import requests
from osgeo import gdal
from gdalmethods import Data_Path, rasterize
# Data Path
dp = Data_Path("/scratch/twillia2/weto/data")
# Here's everything that needs o be reformatted
def fixit(x):
x = x.replace("*", "").replace("\n", "").replace("$", "").replace(",", "")
return x
# Lookup table
lookup = pd.read_csv(dp.join("tables/conus_cbe_lookup.csv"))
lookup.columns = ['code', 'type', 'dollar_ac']
zone_lu = lookup[lookup["type"].str.contains("BLM Zone")]
zone_lu["zone"] = zone_lu["type"].apply(lambda x: int(x[-2:]))
zone_lu["dollar_ac"] = zone_lu["dollar_ac"].apply(fixit)
zone_lu["dollar_ac"] = zone_lu["dollar_ac"].astype(float)
Created on Thu Feb 27 08:56:48 2020
@author: twillia2
"""
import geopandas as gpd
import numpy as np
import pandas as pd
import subprocess as sp
import matplotlib.pyplot as plt
import os
import rasterio
from gdalmethods import Data_Path, to_geo, warp
# Data Paths
DP = Data_Path("/scratch/twillia2/weto/populations")
DPA = Data_Path(DP.join("albers"))
DPSC = Data_Path("/projects/rev/new_projects/ipm_wind/outputs")
DPTMP = Data_Path("/projects/rev/data/conus")
PROJ = ("+proj=aea +lat_0=40 +lon_0=-96 +lat_1=20 +lat_2=60 +x_0=0 +y_0=0 "
"+ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs")
# Reproject Supply Curve Table
if not os.path.exists(DPA.join("sc_gids_singles.gpkg")):
sc = pd.read_csv(DPSC.join("outputs_sc.csv"))
sc = sc[["sc_gid", "latitude", "longitude"]]
sc = sc.drop_duplicates(keep="first", subset=["latitude", "longitude"])
sc = sc.reset_index(drop=True)
sc = to_geo(sc, loncol="longitude", latcol="latitude", epsg=4326)
sc.to_file(DP.join("sc_gids_singles.gpkg"), driver="GPKG")
src = DP.join("sc_gids_singles.gpkg")
which land-values from the 30m grid to assign to acre grid cell?
"""
import geopandas as gpd
import numpy as np
import pandas as pd
import rasterio
import subprocess as sp
from gdalmethods import warp, rasterize, Data_Path, reproject_polygon
from gdalmethods import Map_Values, tile_raster
from osgeo import ogr
# paths
# dp = Data_Path("~/Box/WETO 1.2/data")
dp = Data_Path("/scratch/twillia2/weto/data")
template_path = dp.join("rasters/albers/acre/rent_exclusions.tif")
nlcd_img_path = dp.join("rasters/NLCD/NLCD_2016_Land_Cover_L48_20190424.img")
nlcd_tif_path = dp.join("rasters/NLCD/NLCD_2016_Land_Cover_L48_20190424.tif")
nlcd_acre_path = dp.join("rasters/albers/acre/nlcd.tif")
nlcd_acre_ag_path = dp.join("rasters/albers/acre/nlcd_ag.tif")
county_path = dp.join("shapefiles/USA/tl_2017_us_county.shp")
county_acre_path = dp.join("rasters/albers/acre/county_gids.tif")
county_wgs_path = dp.join("shapefiles/USA/wgs/tl_2017_us_county.shp")
county_albers_path = dp.join("shapefiles/USA/albers/tl_2017_us_county.shp")
ag_product_path = dp.join("rasters/albers/acre/agcounty_product.tif")
# Template for geometries
template = rasterio.open(template_path)
# Translate .img format to .tif
# -*- coding: utf-8 -*-
"""
Rasterize tribal land.
Created on Mon Feb 3 09:30:06 2020
@author: twillia2
"""
import geopandas as gpd
import pandas as pd
from osgeo import gdal
from gdalmethods import Data_Path, rasterize, reproject_polygon
# Data Path
dp = Data_Path("/scratch/twillia2/weto/data")
# Lookup table
lookup = pd.read_csv(dp.join("tables/conus_cbe_lookup.csv"))
lookup.columns = ['code', 'type', 'dollar_ac']
# First add the codes from the lookup table to the shapefiles and rewrite
tribal = gpd.read_file(dp.join("shapefiles/tribal/tl_2016_us_aiannh.shp"))
# This one only has one value
code = lookup["code"][lookup["type"] == "Tribal Land"].values[0]
# Add this code into the shapefile and reproject
tribal["code"] = code
shp_path = dp.join("shapefiles/tribal/tribal_codes.shp")
shp_path_albers = dp.join("shapefiles/tribal/albers/tribal_codes.shp")
"""
Rasterize State Land Codes.
Created on Mon Feb 3 10:35:24 2020
@author: twillia2
"""
import geopandas as gpd
import os
import pandas as pd
from osgeo import gdal
from gdalmethods import Data_Path, rasterize, reproject_polygon
# Data Path
dp = Data_Path("/scratch/twillia2/weto/data")
# Lookup table
lookup = pd.read_csv(dp.join("tables/conus_cbe_lookup.csv"))
lookup.columns = ['code', 'type', 'dollar_ac']
# Get just the state names
state_lu = lookup[["type", "code"]][lookup["type"].str.contains("State Land")]
get_state = lambda x: x[:x.index("State Land") - 1]
state_lu["state_nm"] = state_lu["type"].apply(get_state)
state_lu["state_nm"] = state_lu["state_nm"].astype(str)
# First add the codes from the lookup table to the shapefiles and rewrite
state = gpd.read_file(dp.join("shapefiles/USA/conus_padus_state.shp"))
# Join state_lu with state