# assign coordinates to the matrix that holds response values (minimatrix)
for i in range(0, n, 1):
for j in range(0, n, 1):
if count >= len(responselist):
break
else:
outdict['latitude'].append(
(lat_init + i * pixel_size))
outdict['longitude'].append(
(lon_init + j * pixel_size))
outdict[question].append(minimatrix[i, j])
count = count + 1
newdf = pd.DataFrame.from_dict(outdict)
newdf["geometry"] = newdf.apply(
lambda x: Point(x["longitude"], x["latitude"]), axis=1)
newgdf = gpd.GeoDataFrame(newdf)
name = "round_{0}_{1}.tif".format(rd, question)
output = os.path.join(fpath, name)
cat_raster, _ = rasterize(newgdf,
affine=affine,
shape=out_shape,
attribute=question,
nodata=nan_val,
fill=255,
output=output)
#pixel_size = 0.00026949999
# canal_path = os.path.expanduser("~/git/afghanistan_gie/canal_data/canal_lines.geojson")
border = path+"/afghanistan_settlements.geojson"
#with fiona.open(canal_path) as canal_src:
# bounds = canal_src.bounds
#grid_extent = fiona.open(path+"/afg_border.geojson")
#grid_feature = grid_extent[0]
#grid_shape = shape(grid_feature['geometry'])
#bounds = grid_shape.bounds
bounds = (60, 29, 75, 39)
rv_array, affine = rasterize(border, pixel_size=pixel_size, bounds=bounds)
binary_raster_path = path+"/binary_starts.tif"
export_raster(rv_array, affine, binary_raster_path)
distance_raster_path = path+"/distance_starts_settlements.tif"
def raster_conditional(rarray):
return (rarray == 1)
dist = build_distance_array(rv_array, affine=affine,
output=distance_raster_path,
conditional=raster_conditional)
print "Rasterizing"
# initialize output array
output = np.zeros(shape=(shape[0], shape[1]))
mixed_val = 4
i = 0
for f in src_files:
i += 1
print "Processing: ", f
features = fiona.open(f)
rv_array, _ = rasterize(vectors=features, pixel_size=pixel_size,
affine=affine, shape=shape)
output += rv_array * i
# any cell with value > i must have multiple features
output = np.where(output > i, mixed_val, output)
export_raster(output, affine=affine, nodata=255, path=categorical_output_raster_path)
# ------------------------
# distance raster
print "Generating distance raster"
output_base = "/sciclone/aiddata10/REU/data/rasters/external/global/ambient_air_pollution_2013"
# -----------------------------------------------------------------------------
field_list = ["{0}_{1}".format(*i) for i in itertools.product(col_prefixes, years)]
df = pd.read_csv(input_csv, delimiter=",", encoding='utf-8')
df = df[field_list + ['x', 'y']]
df['geometry'] = df.apply(lambda z: Point(z['x'], z['y']), axis=1)
gdf = gpd.GeoDataFrame(df)
for field in field_list:
print "rasterizing {0}".format(field)
rasterize(
gdf,
attribute=field,
pixel_size=pixel_size,
bounds=gdf.geometry.total_bounds,
output="{0}/{1}/{2}.tif".format(output_base, field[:-5], field),
fill=-1,
nodata=-1)
xmin = -180
xmax = 180
ymin = -90
ymax = 90
shape = (int((ymax-ymin)/pixel_size), int((xmax-xmin)/pixel_size))
affine = Affine(pixel_size, 0, xmin,
0, -pixel_size, ymax)
print "Rasterizing"
petroleum, _ = rasterize(vectors=features, output=binary_output_raster_path,
pixel_size=pixel_size, affine=affine, shape=shape)
# --------------------------------------
# distance to gem
print "Generating distance raster"
def raster_conditional(rarray):
return (rarray == 1)
dist = build_distance_array(petroleum, affine=affine,
output=distance_output_raster_path,
conditional=raster_conditional)
shorelines_path = "/sciclone/aiddata10/REU/geo/raw/gshhg/gshhg-shp-2.3.6/GSHHS_shp/f/GSHHS_f_L1.shp" pixel_size = 0.01 xmin = -180 xmax = 180 ymin = -90 ymax = 90 affine = Affine(pixel_size, 0, xmin, 0, -pixel_size, ymax) shape = (int((ymax - ymin) / pixel_size), int((xmax - xmin) / pixel_size)) print "Rasterizing" shorelines, _ = rasterize(shorelines_path, affine=affine, shape=shape) shorelines = np.logical_not(shorelines).astype(int) water_output_raster_path = "/sciclone/aiddata10/REU/geo/data/rasters/distance_to/coast_236/binary/coast_binary.tif" export_raster(shorelines, affine, water_output_raster_path) # ----------------------------------------------------------------------------- # import rasterio # water_src = rasterio.open(water_output_raster_path) # water = water_src.read()[0] # affine = water_src.affine distance_output_raster_path = "/sciclone/aiddata10/REU/geo/data/rasters/distance_to/coast_236/coast_distance.tif"
pixel_size = 0.01
xmin = -180
xmax = 180
ymin = -90
ymax = 90
affine = Affine(pixel_size, 0, xmin,
0, -pixel_size, ymax)
shape = (int((ymax-ymin)/pixel_size), int((xmax-xmin)/pixel_size))
borders, _ = rasterize(borders_path, affine=affine, shape=shape)
borders_output_raster_path = "/sciclone/aiddata10/REU/geo/data/rasters/distance_to/gadm28_borders/binary/gadm28_borders_binary.tif"
export_raster(borders, affine, borders_output_raster_path)
# -----------------------------------------------------------------------------
# import rasterio
# borders_src = rasterio.open(borders_output_raster_path)
# borders = borders_src.read()[0]
# affine = borders_src.affine
]
# build intermediary rasters for individual categories/years
print "selected {0} features for field: {1}, and year: {2}".format(
len(features_filtered), cat, year)
if len(features_filtered) == 0:
print "\tno feature selected for violence type {0} and year {1}".format(
cat, year)
pass
else:
cat_raster, _ = rasterize(features_filtered,
affine=affine,
shape=out_shape)
output_file = "ucdp_" + field_values[cat] + "_" + str(year) + ".tif"
output_path = os.path.join(final_output, output_file)
export_raster(cat_raster,
affine=affine,
path=output_path,
out_dtype='float64',
nodata=None)
t2 = time.time()
sum_time = t2 - t1
gdf = gpd.read_file(inf) count = len(gdf) speedlist = [10.0,60.0,120.0] * (count/3) # not replicable by other shapefile layer gdf['randomspeed'] = speedlist psize = 0.005 xmin = gdf.bounds['minx'].min() xmax = gdf.bounds['maxx'].max() ymin = gdf.bounds['miny'].min() ymax = gdf.bounds['maxy'].max() bounds = (xmin,ymin,xmax,ymax) rst,_= rasterize(gdf,pixel_size=psize,bounds=bounds,output=output, attribute='randomspeed') # ------------------------------ # processing railroad shapefile downloaded from osm railroad = fiona.open(inf1) railbound = railroad.bounds psize1 = 0.5 print type(railroad) #railroad = railroad * 40.0 # 40 is the speed 40km/hr of using railroad rst,_= rasterize(railroad,pixel_size=psize1,bounds=railbound,output=output1)
pixel_size = 0.01
xmin = -180
xmax = 180
ymin = -90
ymax = 90
shape = (int((ymax - ymin) / pixel_size), int((xmax - xmin) / pixel_size))
affine = Affine(pixel_size, 0, xmin, 0, -pixel_size, ymax)
print "Rasterizing"
gem, _ = rasterize(features,
output=binary_output_raster_path,
pixel_size=pixel_size,
affine=affine,
shape=shape)
# --------------------------------------
# distance to gem
print "Generating distance raster"
def raster_conditional(rarray):
return (rarray == 1)
dist = build_distance_array(gem,
affine=affine,
rasterized_feature_output_path = "{0}/data/{1}_binary.tif".format(
base, out_name)
output_raster_path = "{0}/data/{1}_distance.tif".format(base, out_name)
# 0.1 is probably too coarse for quality
# 0.001 might be more than we need for quality
# test with central america rivers @ 30s res
# run time for rasterization was reasonable
# distance processes may be too slow at this fine scale though
# testing with 0.01 for now
pixel_size = 0.01
shp = fiona.open(shp_path, "r")
rv_array, affine = rasterize(shp,
pixel_size=pixel_size,
bounds=shp.bounds,
output=rasterized_feature_output_path)
# rv_array = fake_rasterize()
# max distance in cells
# for actual distance: max_dist * pixel_size
max_dist = 100
nrows, ncols = rv_array.shape
# print rv_array
print rv_array.shape
print nrows * ncols
# raise
df['geometry'] = df.apply(lambda z: Point(z['longitude'], z['latitude']),
axis=1)
gdf = gpd.GeoDataFrame(df)
for year in range(1989, 2017):
print "Rasterizing year: {0}".format(year)
filtered = df[df["year"] == year]
grouped = filtered.groupby(["newid"]).agg({
'longitude': 'last',
'latitude': 'last',
'best': 'sum'
}).reset_index()
grouped['geometry'] = grouped.apply(
lambda z: Point(z['longitude'], z['latitude']), axis=1)
tmp_gdf = gpd.GeoDataFrame(grouped)
array, affine = rasterize(tmp_gdf,
attribute="best",
pixel_size=pixel_size,
bounds=gdf.geometry.total_bounds,
output="{0}/ucdp_deaths_{1}.tif".format(
output_dir, year),
fill=0,
nodata=-1)
xmin = -180
xmax = 180
ymin = -90
ymax = 90
affine = Affine(pixel_size, 0, xmin,
0, -pixel_size, ymax)
shape = (int((ymax-ymin)/pixel_size), int((xmax-xmin)/pixel_size))
roads = np.zeros(shape, dtype='byte')
for n in names:
path = "{0}/{1}".format("/sciclone/aiddata10/REU/geo/raw/groads",
"groads-v1-{0}-shp/gROADS-v1-{0}.shp".format(n))
rv_array, _ = rasterize(path, affine=affine, shape=shape)
roads = roads | rv_array
roads_output_raster_path = "/sciclone/aiddata10/REU/geo/data/rasters/distance_to/groads/binary/groads_binary.tif"
export_raster(roads, affine, roads_output_raster_path)
# -----------------------------------------------------------------------------
# import rasterio
# roads_src = rasterio.open(roads_output_raster_path)
# roads = roads_src.read()[0]
# affine = roads_src.affine
from shapely.geometry import shape
pixel_size = 0.00026949999
# canal_path = os.path.expanduser("~/git/afghanistan_gie/canal_data/canal_lines.geojson")
canal_path = path + "/mine_incidents.geojson"
#with fiona.open(canal_path) as canal_src:
# bounds = canal_src.bounds
grid_extent = fiona.open(path + "/grid_extent.geojson")
grid_feature = grid_extent[0]
grid_shape = shape(grid_feature['geometry'])
bounds = grid_shape.bounds
rv_array, affine = rasterize(canal_path, pixel_size=pixel_size, bounds=bounds)
# binary_raster_path = "/sciclone/aiddata10/REU/projects/afghanistan_gie/distance_to_canals/binary_canals.tif"
# binary_raster_path = "/sciclone/aiddata10/REU/projects/afghanistan_gie/distance_to_canals/binary_starts.tif"
binary_raster_path = path + "/binary_starts.tif"
export_raster(rv_array, affine, binary_raster_path)
# import tarfile
# def make_tarfile(dst, src):
# with tarfile.open(dst, "w:gz") as tar:
# tar.add(src, arcname=os.path.basename(src))
# make_tarfile(dst=binary_raster_path + ".tar.gz" , src=binary_raster_path)
gdf = gpd.GeoDataFrame(df)
pixel_size = 0.1
data_field = 'best' # best estimate of deaths
field_values = {1: "State-based", 2: "Non-state", 3: "One-sided"}
# output_dir = "/Users/miranda/Documents/AidData/sciclone/datasets/UCDP_GED_conflict/output"
output_dir = "/sciclone/data20/zlv/ucdp/fatality"
for field in field_values.keys():
field_gdf = gdf[gdf["type_of_violence"] == field]
for year in range(1989, 2016):
rst_gdf = field_gdf[field_gdf["year"] == year]
print "rasterize year", year
rasterize(rst_gdf,
attribute=data_field,
pixel_size=pixel_size,
bounds=rst_gdf.geometry.total_bounds,
output="{0}/{1}_{2}_{3}.tif".format(output_dir,
"ucdp_fatality",
field_values[field],
year),
fill=-1,
nodata=-1)
shorelines_path = "gshhg-shp-2.3.6/GSHHS_shp/f/GSHHS_f_L1.shp" lakes_path = "natural-earth-vector/10m_physical/ne_10m_lakes.shp" rivers_path = "natural-earth-vector/10m_physical/ne_10m_rivers_lake_centerlines.shp" pixel_size = 0.01 xmin = -180 xmax = 180 ymin = -90 ymax = 90 affine = Affine(pixel_size, 0, xmin, 0, -pixel_size, ymax) shape = (int((ymax - ymin) / pixel_size), int((xmax - xmin) / pixel_size)) shorelines, _ = rasterize(shorelines_path, affine=affine, shape=shape) shorelines = np.logical_not(shorelines).astype(int) lakes, _ = rasterize(lakes_path, affine=affine, shape=shape) rivers, _ = rasterize(rivers_path, affine=affine, shape=shape) water = shorelines + lakes + rivers water_output_raster_path = "water_binary.tif" export_raster(water, affine, water_output_raster_path) # ----------------------------------------------------------------------------- # import rasterio # water_src = rasterio.open(water_output_raster_path)
import os
import pandas as pd
import geopandas as gpd
from shapely.geometry import Point
from distancerasters import rasterize, export_raster
input_path = "/sciclone/aiddata10/REU/geo/raw/africa_child_mortality/ChildMortality5m0Estimates_Burke-HeftNeal-Bendavid_v1.txt"
df = pd.read_table(input_path, sep=' ')
df['geometry'] = df.apply(lambda z: Point(z['lon'], z['lat']), axis=1)
gdf = gpd.GeoDataFrame(df)
pixel_size = 0.1
data_field = 'est5m0'
years = [1980, 1990, 2000]
output_dir = "/sciclone/aiddata10/REU/geo/data/rasters/external/global/africa_child_mortality"
for year in years:
rst_gdf = gdf[gdf["decade"] == year]
print "rasterize year", year
rasterize(rst_gdf,
attribute=data_field,
pixel_size=pixel_size,
bounds=rst_gdf.geometry.total_bounds,
output="{0}/{1}_{2}.tif".format(output_dir,
"africa_child_mortality", year),
fill=-1,
nodata=-1)
