def merge_shapes(
inputfile,
outputfile=None,
overwrite=False,
verbose=True,
vverbose=False,
):
"""
Merges all the shapes in a shapefile into a single shape.
"""
if outputfile is None: output = '{}/merged'.format(os.getcwd())
if os.path.isfile(outputfile + '.shp') and not overwrite:
if verbose:
print('combined watershed shapefile {} exists'.format(outputfile))
return
if verbose:
print('combining shapes from {}\n'.format(inputfile) +
'this may take a while...\n')
# start by copying the projection files
shutil.copy(inputfile + '.prj', outputfile + '.prj')
# load the catchment and flowline shapefiles
r = Reader(inputfile, shapeType=5)
try:
combined = combine_shapes(r.shapes(), verbose=vverbose)
except:
print('error: unable to combine shapes')
raise
# create the new file with the merged shapes
w = Writer(outputfile, shapeType=5)
w.poly(shapeType=5, parts=[combined])
# copy the fields from the original and then the first record; note this
# can be adapted as needed
for field in r.fields:
w.field(*field)
w.record(*r.record(0))
w.close()
if verbose:
its = inputfile, outputfile
print('successfully combined shapes from {} to {}\n'.format(*its))
def clip_value(in_file, ot_dir, min_height, max_height):
"""
オンライン学習４ ベクタデータのフィルタリング
浸水・土砂崩れベクタデータをGISデータの属性値(値)を使用してフィルタリングするプログラムを実行します。
関数 : clip_value
引数1 : 浸水・土砂崩れベクタデータ(*.shp)
引数2 : 出力ディレクトリ名
引数3 : 出力対象となる値の最小値
引数4 : 出力対象となる値の最大値
"""
# Get actual file path
in_file = path.join(DATA_PATH_BASE, in_file)
ot_dir = path.join(DATA_PATH_BASE, ot_dir)
makedirs(ot_dir, exist_ok=True)
ot_file = path.join(
ot_dir, "{0}v.tif".format(path.splitext(path.basename(in_file))[0]))
reader = ShpReader(in_file, encoding='cp932')
writer = ShpWriter(ot_file, encoding='cp932')
# Create DBF schema
height_col_id = None
for i, col in enumerate(
(col for col in reader.fields if col[0] != "DeletionFlag")):
if col[0] != "DeletionFlag":
writer.field(col[0], col[1], col[2], col[3])
if col[0] == "height":
height_col_id = i
if height_col_id is None:
print("height column not found in polygon shapefile")
return
# Filtering
n_mesh = reader.numRecords
cnt_mesh = 0
for data in reader.iterShapeRecords():
height = data.record[height_col_id]
if (height is not None) and (min_height <= height <= max_height):
# This polygon is output target.
writer.shape(data.shape)
writer.record(*data.record)
cnt_mesh = cnt_mesh + 1
if cnt_mesh % 100000 == 0:
print("{0}K / {1}K".format(cnt_mesh / 1000, n_mesh / 1000))
writer.close()
def clip_shape(in_mesh, in_mask, ot_dir, flg_mask):
"""
オンライン学習４ ベクタデータのフィルタリング
浸水・土砂崩れベクタデータをGISデータの形状を利用してフィルタリングします。
関数 : clip_shape
引数1 : 浸水・土砂崩れベクタデータ(*.shp)
引数2 : GISデータ(*.shp)
引数3 : 出力ディレクトリ名
引数4 : 出力フラグ(True：GISデータと重なる部分を出力、False：GISデータと重ならない部分を出力)
"""
# Get actual file path
in_mesh = path.join(DATA_PATH_BASE, in_mesh)
in_mask = path.join(DATA_PATH_BASE, in_mask)
ot_dir = path.join(DATA_PATH_BASE, ot_dir)
makedirs(ot_dir, exist_ok=True)
ot_file = path.join(ot_dir, "{0}s.tif".format(path.splitext(path.basename(in_mesh))[0]))
reader_mesh = ShpReader(in_mesh, encoding='cp932')
reader_mask = ShpReader(in_mask, encoding='cp932')
writer = ShpWriter(ot_file, encoding='cp932')
# Create DBF schema
for col in reader_mesh.fields:
if col[0] != "DeletionFlag":
writer.field(col[0], col[1], col[2], col[3])
# Create set of mask polygon
maskdata = []
for data in reader_mask.iterShapes():
points = data.points
points_split = list(data.parts) + [len(points)]
poly_list = []
for i in range(len(points_split) - 1):
points_part = points[points_split[i]:points_split[i + 1]]
poly_list.append(Polygon(points_part))
# Use as mask polygon only when all key conditions are satisfied.
# Memorize shape and bbox of polygon.
x_range = min(points, key=lambda x: x[0])[0], max(points, key=lambda x: x[0])[0]
y_range = min(points, key=lambda x: x[1])[1], max(points, key=lambda x: x[1])[1]
maskdata.append((x_range, y_range, poly_list))
# Filtering
n_mesh = reader_mesh.numRecords
cnt_mesh = 0
for data in reader_mesh.iterShapeRecords():
center = Polygon(data.shape.points).centroid
x = center.x
y = center.y
masked = False
for x_range, y_range, mask_polys in maskdata:
# Primary screening by mask polygon bbox.
if x < x_range[0] or x > x_range[1] or y < y_range[0] or y > y_range[1]:
continue
mask_count = sum(poly.contains(center) for poly in mask_polys)
if mask_count % 2 == 1:
masked = True
break
if masked == flg_mask:
# This polygon is output target.
writer.shape(data.shape)
writer.record(*data.record)
cnt_mesh = cnt_mesh + 1
if cnt_mesh % 100000 == 0:
print("{0}K / {1}K".format(cnt_mesh/1000, n_mesh/1000))
writer.close()
def add_height_vector(in_polys, in_hpoint, dst_fn):
"""
オンライン学習3 被害領域の抽出、ラスタベクタ変換
メッシュデータに標高値を付与します。
関数 ： add_height_vector
引数1 ： 入力メッシュデータ名(.tif)
引数2 ： 数値標高ﾓﾃﾞﾙ名(.shp)
引数3 ： 出力ファイル名(.shp)
"""
# Read DEM data
print("loading DEM data ...")
dem = GridData()
dem_reader = ShpReader(in_hpoint, encoding='cp932')
n_p = dem_reader.numRecords
cnt_p = 0
for data in dem_reader.iterShapeRecords():
point = Point(data.shape.points[0])
p_val = data.record
dem.add_data(point.x, point.y, p_val)
cnt_p = cnt_p + 1
if cnt_p % 100000 == 0:
print("{0}K / {1}K".format(cnt_p/1000, n_p/1000))
print("loaded DEM data .")
print()
# Process each polygon shapefile
for in_poly in in_polys:
print("processing {0} ...".format(in_poly))
poly_reader = ShpReader(in_poly)
poly_writer = ShpWriter(target=dst_fn)
# Create DBF schema
for col in poly_reader.fields:
if col[0] != "DeletionFlag":
poly_writer.field(col[0], col[1], col[2], col[3])
poly_writer.field("height", "N", 18, 9)
# Attach elevation value
n_poly = poly_reader.numRecords
cnt_poly = 0
for data in poly_reader.iterShapeRecords():
center = Polygon(data.shape.points).centroid
key_x = dem.search_nearest_x(center.coords[0][0])
key_y = dem.search_nearest_y(center.coords[0][1])
dem_record = dem.get_data(key_x, key_y)
if dem_record:
# Nearest grid point has elevation value
record = data.record + dem_record
else:
# Nearest grid point doesn't have elevation value
record = data.record + [None]
poly_writer.shape(data.shape)
poly_writer.record(*record)
cnt_poly = cnt_poly + 1
if cnt_poly % 100000 == 0:
print("{0}K / {1}K".format(cnt_poly/1000, n_poly/1000))
poly_writer.close()
print("processed {0} .".format(in_poly))
print()
def extract_bbox(self, bbox, output, verbose=True):
"""Extracts the NID dam locations for a watershed from the dam
shapefile and the 8-digit hydrologic unit code of interest.
"""
self.download_compressed()
xmin, ymin, xmax, ymax = bbox
# copy the projection files
if verbose: print('copying the projections from the NID source\n')
projection = self.source + '.prj'
shutil.copy(projection, output + '.prj')
# get the dams within the watershed
if verbose: print('reading the dam file\n')
sf = Reader(self.source, shapeType=1)
# work around for issues with pyshp
damrecords = []
for i in range(len(sf.shapes())):
try:
damrecords.append(sf.record(i))
except:
damrecords.append([-100 for i in range(len(sf.fields))])
name_index = sf.fields.index(['DAM_NAME', 'C', 65, 0]) - 1
nid_index = sf.fields.index(['NIDID', 'C', 7, 0]) - 1
long_index = sf.fields.index(['LONGITUDE', 'N', 19, 11]) - 1
lat_index = sf.fields.index(['LATITUDE', 'N', 19, 11]) - 1
river_index = sf.fields.index(['RIVER', 'C', 65, 0]) - 1
owner_index = sf.fields.index(['OWN_NAME', 'C', 65, 0]) - 1
type_index = sf.fields.index(['DAM_TYPE', 'C', 10, 0]) - 1
purp_index = sf.fields.index(['PURPOSES', 'C', 254, 0]) - 1
year_index = sf.fields.index(['YR_COMPL', 'C', 10, 0]) - 1
high_index = sf.fields.index(['NID_HEIGHT', 'N', 19, 11]) - 1
mstor_index = sf.fields.index(['MAX_STOR', 'N', 19, 11]) - 1
nstor_index = sf.fields.index(['NORMAL_STO', 'N', 19, 11]) - 1
area_index = sf.fields.index(['SURF_AREA', 'N', 19, 11]) - 1
# iterate through the fields and determine which points are in the box
if verbose: print('extracting dams into new file\n')
dam_indices = []
i = 0
for record in damrecords:
lat = record[lat_index]
lon = record[long_index]
if self.inside_box([xmin, ymin], [xmax, ymax], [lon, lat]):
dam_indices.append(i)
i += 1
# write the data from the bbox to a new shapefile
w = Writer(output, shapeType=1)
for field in sf.fields:
w.field(*field)
for i in dam_indices:
point = sf.shape(i).points[0]
w.point(*point)
values = damrecords[i]
rs = []
for value in values:
if isinstance(value, bytes): value = value.decode('utf-8')
rs.append(value)
w.record(*rs)
w.close()
if verbose:
print('successfully extracted NID dam locations to new file\n')
def extract_HUC8(
self,
HUC8,
output,
gagefile='gagestations',
verbose=True,
):
"""
Extracts the USGS gage stations for a watershed from the gage
station shapefile into a shapefile for the 8-digit hydrologic unit
code of interest.
"""
# make sure the metadata exist locally
self.download_metadata()
# make sure the output destination exists
if not os.path.isdir(output): os.mkdir(output)
sfile = '{}/{}'.format(output, gagefile)
if not os.path.isfile(sfile + '.shp'):
# copy the projection
shutil.copy(self.NWIS + '.prj', sfile + '.prj')
# read the file
gagereader = Reader(self.NWIS, shapeType=1)
gagerecords = gagereader.records()
# pull out the HUC8 record to parse the dataset
HUC8_index = gagereader.fields.index(['HUC', 'C', 8, 0]) - 1
# iterate through the field and find gages in the watershed
its = HUC8, sfile
print('extracting gage stations in {} to {}\n'.format(*its))
gage_indices = []
i = 0
for record in gagerecords:
if record[HUC8_index] == HUC8: gage_indices.append(i)
i += 1
# write the data from the HUC8 to a new shapefile
w = Writer(sfile, shapeType=1)
for field in gagereader.fields:
w.field(*field)
for i in gage_indices:
point = gagereader.shape(i).points[0]
w.point(*point)
w.record(*gagerecords[i])
w.close()
if verbose:
print('successfully extracted NWIS gage stations\n')
elif verbose:
print('gage station file {} exists\n'.format(sfile))
self.set_metadata(sfile)
def extract_aquifers(directory, HUC8, aquifers, pad=0.2, verbose=True):
"""Extracts aquifers from the source datafile to the destination using
the HUC8 boundaries for the query."""
start = time.time()
# open up the HUC8 boundary shapefile and use it to get the bounding box
shapefile = Reader(directory + '/%s/%scatchments' % (HUC8, HUC8))
xmin, ymin, xmax, ymax = get_boundaries(shapefile.shapes())
# convert to bounding corners for testing
p1 = [xmin - pad * (xmax - xmin), ymin - pad * (ymax - ymin)]
p2 = [xmax + pad * (xmax - xmin), ymax + pad * (ymax - ymin)]
shapefile = None
# start by copying the projection files
if verbose: print('\ncopying the projections\n')
shutil.copy(directory + '/%s/%scatchments.prj' % (HUC8, HUC8),
directory + '/%s/%saquifers.prj' % (HUC8, HUC8))
# open the flowline file
if verbose: print('reading the aquifer file\n')
shapefile = Reader(aquifers, shapeType=5)
# work around for issues with pyshp
records = []
for i in range(len(shapefile.shapes())):
try:
records.append(shapefile.record(i))
except:
records.append('')
# use the bounding boxes to see if the shapes are within the watershed area
if verbose: print('searching for aquifers in the watershed\n')
bboxes = [shapefile.shape(i).bbox for i in range(len(records))]
corners = [[[b[0], b[1]], [b[0], b[3]], [b[2], b[1]], [b[2], b[3]]]
for b in bboxes]
indices = [
i for i, c in zip(range(len(corners)), corners)
if any([inside_box(p1, p2, p) for p in c])
or all([inside_box(c[0], c[3], p1),
inside_box(c[0], c[3], p2)])
]
# remove any non aquifers
indices = [i for i in indices if shapefile.record(i)[4] != 999]
# find a record for the non aquifer
i = 0
while shapefile.record(i)[4] != 999:
i += 1
nonrecord = shapefile.record(i)
nonrecord[1] = nonrecord[1].decode('utf-8')
nonrecord[5] = 0
nonrecord[6] = 0
if len(indices) == 0:
if verbose: print('query returned no values, returning\n')
return
# write the data from the HUC8 to a new shapefile
w = Writer(directory + '/%s/%saquifers' % (HUC8, HUC8), shapeType=5)
for field in shapefile.fields:
w.field(*field)
for i in indices:
shape = shapefile.shape(i)
# check for multiple parts
if len(shape.parts) > 1:
parts = [
shape.points[i:j]
for i, j in zip(shape.parts[:-1], shape.parts[1:])
]
else:
parts = [shape.points]
record = records[i]
# little work around for blank binary values
if isinstance(record[1], bytes):
record[1] = record[1].decode('utf-8')
w.poly(shapeType=5, parts=parts)
w.record(*record)
# add a shape for the bounding box showing no aquifer locations
part = [p1, [p1[0], p2[1]], p2, [p2[0], p1[1]]]
w.poly(shapeType=5, parts=[part])
w.record(*nonrecord)
w.close()
end = time.time()
if verbose:
print('successfully queried data in %.2f seconds\n' % (end - start))
def trans_vector(in_file, ot_dir, output_flg, dem_path, flood_flg):
"""
オンライン学習3 被害領域の抽出、ラスタベクタ変換
二値画像からポリゴンを生成します
関数 ： trans_vector
引数1 ： 入力ファイル名(.tif)
引数2 ： 出力ディレクトリ名
引数3 ： 出力フラグ(0：被災領域、1：非被災領域)
引数4 ： 数値標高ﾓﾃﾞﾙ名(.shp)
引数5 ： 災害フラグ(True：浸水、False：土砂崩れ)
"""
# Get destination file name
filename = path.splitext(path.basename(in_file))[0]
if filename.lower().startswith("sendai"):
basename = "Sendai"
elif filename.lower().startswith("kumamoto"):
basename = "Kumamoto"
else:
basename = filename
# Get actual file path
in_file = path.join(DATA_PATH_BASE, in_file)
ot_dir = path.join(DATA_PATH_BASE, ot_dir)
dem_path = path.join(DATA_PATH_BASE, dem_path)
makedirs(ot_dir, exist_ok=True)
print("creating shapefile ...")
# Create shapefile information of output area
fn_tmp = path.join(ot_dir, "tmp.shp")
writer = ShpWriter(target=fn_tmp, shapeType=POLYGON)
writer.field("id", "C", "20", 0)
writer.field("type", "C", "10", 0)
writer.field("format", "C", "10", 0)
writer.field("dis_tf", "C", "8", 0)
writer.field("dis_tt", "C", "8", 0)
writer.field("proc", "C", "8", 0)
writer.field("pre_dn", "C", "10", 0)
writer.field("pre_st", "C", "10", 0)
writer.field("post_dn", "C", "10", 0)
writer.field("post_st", "C", "10", 0)
"""
オンライン学習3 被害領域の抽出、ラスタベクタ変換
ポリゴンに付与する属性情報を定義するプログラムを実行します
関数 ： get_flood_record
関数 ： get_land_slide_record
"""
if flood_flg:
# flood processing
record = get_flood_record()
else:
# landslide processing
record = get_land_slide_record()
# Read binary image and get coordinate information
bin = imread(in_file)
rect_tiff = RectGeoTiffCoordinateCalculator(in_file)
# Create rectangle polygon of output area and output to shapefile
n_shape = bin.shape[0] * bin.shape[1]
cnt = 0
for x_index, y_index in itertools.product(range(bin.shape[1]),
range(bin.shape[0])):
"""
オンライン学習3 被害領域の抽出、ラスタベクタ変換
二値画像の各ピクセル四隅座標(緯度、経度)を計算するプログラムを実行します
関数 ： create_polygon_points
引数1 ： 対象ピクセルのx番号
引数2 ： 対象ピクセルのy番号
引数3 ： 二値画像の図形情報(ファイル名、画像サイズ等)を持つインスタンス
"""
points = create_polygon_points(x_index, y_index, rect_tiff)
"""
オンライン学習3 被害領域の抽出、ラスタベクタ変換
二値画像からメッシュを作成します
bin[y_index, x_index] == 255)：ピクセル値が255の場合
output_flg == "0"：被災領域のメッシュを作成
output_flg == "1"：非被災領域のメッシュを作成
"""
if (bin[y_index, x_index] == 255) == (output_flg == "0"):
# This pixel is output target.
writer.poly([points])
writer.record(*record)
cnt = cnt + 1
if cnt % 100000 == 0:
print("{0}K / {1}K".format(cnt / 1000, n_shape / 1000))
writer.close()
print("created shapefile .")
if output_flg == "0":
fn_out = path.join(
ot_dir, "{0}_{1}cbp.shp".format(basename,
time.strftime("%Y%m%d%H%M%S")))
else:
fn_out = path.join(
ot_dir, "{0}_{1}cbpr.shp".format(basename,
time.strftime("%Y%m%d%H%M%S")))
# Attach elevation value
"""
オンライン学習3 被害領域の抽出、ラスタベクタ変換
メッシュデータに標高値を付与するプログラムを実行します
関数 ： add_height_vector
引数1 ： 入力メッシュデータ名(.tif)
引数2 ： 数値標高ﾓﾃﾞﾙ名(.shp)
引数3 ： 出力ファイル名(.shp)
"""
add_height_vector([fn_tmp], dem_path, fn_out)
if not DEV_FLAG:
# Delete temporary file.
remove("{0}.shp".format(path.splitext(fn_tmp)[0]))
remove("{0}.shx".format(path.splitext(fn_tmp)[0]))
remove("{0}.dbf".format(path.splitext(fn_tmp)[0]))