def main(args): 'Runs calculation chain' try: action = args[1] config = args[2] except IndexError: action = config = None inm_log = InmarsatLog.from_csv('data', 'inmarsat-su-log-redacted.csv') time_step = timedelta(seconds=10) bin_log = inm_log.bin_data(time_step) traj_time_step = time_step / 2 traj = Trajectory.from_csv('data', acars='acars.csv', adsb='all-combined.csv', radar='route.csv') int_traj = traj.int_data(bin_log.data[0].time - traj_time_step, traj_time_step) r_btos = RadialDistance.from_bto(bin_log.data) r_known = RadialDistance.from_traj(int_traj.data) r_flight = r_known.append(r_btos).take_after(T0['take-off']) trend_times = [(2014, 3, 7, 16, 42, 0), (2014, 3, 7, 17, 23, 0), (2014, 3, 7, 18, 28, 15), (2014, 3, 7, 19, 41, 5), (2014, 3, 7, 20, 41, 5), (2014, 3, 7, 21, 41, 25), (2014, 3, 7, 22, 41, 25), (2014, 3, 8, 0, 19, 35)] r_trend = r_flight.filter_by_list( [datetime(*item) for item in trend_times]) r_interp = r_flight.interpolate(r_trend, T0['off-radar']) if action == 'polygon': with open(config, 'rt') as cfg_file: cfg = load(cfg_file) from shapefile import Writer, POLYGON shp_file = Writer(POLYGON) shp_file.field('TIME', 'C', '20') for contour in cfg['times']: contour_pts = [] ind_a = 0 ind_b = 1 contour_time = datetime(*contour['time']) if contour_time > r_interp.data[-1][0]: contour_time = r_interp.data[-1][0] for offset in cfg['time_delta']: time_offset = timedelta(minutes=offset) cur_time = contour_time + time_offset contour_pts += [ r_interp.find_loc(cur_time, lat) for lat in frange( contour['lat_bounds'][ind_a], contour['lat_bounds'] [ind_b], cfg['lat_step'] * (ind_b - ind_a)) ] ind_a, ind_b = ind_b, ind_a shp_file.poly(parts=[contour_pts]) poly_name = '{}_{}_{}'.format(*contour['time'][2:5]) shp_file.record(poly_name) shp_file.save(cfg['save_to_file'])
def extract_catchments( self, source, destination, flowlinefile, verbose=True, ): """ Extracts the catchments from the source data file to the destination using the list of comids for the query. """ # make a list of the comids comids = self.get_comids(flowlinefile) # open the catchment shapefile if verbose: print('reading the catchment shapefile\n') shapefile = Reader(source) # get the index of the feature id, which links to the flowline comid featureid_index = shapefile.fields.index(['FEATUREID', 'N', 9, 0]) - 1 # go through the comids from the flowlines and add the corresponding # catchment to the catchment list if verbose: print('searching the catchments in the watershed\n') records = shapefile.records() indices = [] i = 0 for record in records: if record[featureid_index] in comids: indices.append(i) i += 1 if len(indices) == 0: print('query returned no values, returning\n') raise # create the new shapefile if verbose: print('writing the new catchment shapefile\n') w = Writer() for field in shapefile.fields: w.field(*field) for i in indices: shape = shapefile.shape(i) w.poly(shapeType=5, parts=[shape.points]) w.record(*records[i]) w.save(destination)
def merge_shapes( inputfile, outputfile=None, overwrite=False, verbose=True, vverbose=False, ): """ Merges all the shapes in a shapefile into a single shape. """ if outputfile is None: output = '{}/merged'.format(os.getcwd()) if os.path.isfile(outputfile + '.shp') and not overwrite: if verbose: print('combined watershed shapefile {} exists'.format(outputfile)) return if verbose: print('combining shapes from {}\n'.format(inputfile) + 'this may take a while...\n') # start by copying the projection files shutil.copy(inputfile + '.prj', outputfile + '.prj') # load the catchment and flowline shapefiles r = Reader(inputfile, shapeType=5) try: combined = combine_shapes(r.shapes(), verbose=vverbose) except: print('error: unable to combine shapes') raise # create the new file with the merged shapes w = Writer(shapeType=5) w.poly(shapeType=5, parts=[combined]) # copy the fields from the original and then the first record; note this # can be adapted as needed for field in r.fields: w.field(*field) w.record(*r.record(0)) w.save(outputfile) if verbose: its = inputfile, outputfile print('successfully combined shapes from {} to {}\n'.format(*its))
def extract_flowlines(self, source, destination, HUC8, verbose = True): """Extracts flowlines from the source datafile to the destination using the HUC8 for the query.""" # open the flowline file if verbose: print('reading the flowline file\n') shapefile = Reader(source, shapeType = 3) records = shapefile.records() # figure out which field codes are the Reach code and comid reach_index = shapefile.fields.index(['REACHCODE', 'C', 14, 0]) - 1 # go through the reach indices, add add them to the list of flowlines # if in the watershed; also make a list of the corresponding comids if verbose: print('searching for flowlines in the watershed\n') indices = [] i = 0 for record in records: if record[reach_index][:8] == HUC8: indices.append(i) i+=1 if len(indices) == 0: if verbose: print('error: query returned no values') raise # write the data from the HUC8 to a new shapefile w = Writer(shapeType = 3) for field in shapefile.fields: w.field(*field) for i in indices: shape = shapefile.shape(i) w.poly(shapeType = 3, parts = [shape.points]) record = records[i] # little work around for blank GNIS_ID and GNIS_NAME values if isinstance(record[3], bytes): record[3] = record[3].decode('utf-8') if isinstance(record[4], bytes): record[4] = record[4].decode('utf-8') w.record(*record) w.save(destination) if verbose: l = len(indices) print('queried {} flowlines from original shapefile\n'.format(l))
def extract_catchments(self, source, destination, flowlinefile, verbose = True, ): """ Extracts the catchments from the source data file to the destination using the list of comids for the query. """ # make a list of the comids comids = self.get_comids(flowlinefile) # open the catchment shapefile if verbose: print('reading the catchment shapefile\n') shapefile = Reader(source) # get the index of the feature id, which links to the flowline comid featureid_index = shapefile.fields.index(['FEATUREID', 'N', 9, 0]) - 1 # go through the comids from the flowlines and add the corresponding # catchment to the catchment list if verbose: print('searching the catchments in the watershed\n') records = shapefile.records() indices = [] i = 0 for record in records: if record[featureid_index] in comids: indices.append(i) i+=1 if len(indices) == 0: print('query returned no values, returning\n') raise # create the new shapefile if verbose: print('writing the new catchment shapefile\n') w = Writer() for field in shapefile.fields: w.field(*field) for i in indices: shape = shapefile.shape(i) w.poly(shapeType = 5, parts = [shape.points]) w.record(*records[i]) w.save(destination)
def merge_shapes(inputfile, outputfile = None, overwrite = False, verbose = True, vverbose = False, ): """ Merges all the shapes in a shapefile into a single shape. """ if outputfile is None: output = '{}/merged'.format(os.getcwd()) if os.path.isfile(outputfile + '.shp') and not overwrite: if verbose: print('combined watershed shapefile {} exists'.format(outputfile)) return if verbose: print('combining shapes from {}\n'.format(inputfile) + 'this may take a while...\n') # start by copying the projection files shutil.copy(inputfile + '.prj', outputfile + '.prj') # load the catchment and flowline shapefiles r = Reader(inputfile, shapeType = 5) try: combined = combine_shapes(r.shapes(), verbose = vverbose) except: print('error: unable to combine shapes') raise # create the new file with the merged shapes w = Writer(shapeType = 5) w.poly(shapeType = 5, parts = [combined]) # copy the fields from the original and then the first record; note this # can be adapted as needed for field in r.fields: w.field(*field) w.record(*r.record(0)) w.save(outputfile) if verbose: its = inputfile, outputfile print('successfully combined shapes from {} to {}\n'.format(*its))
def plot_gage_subbasin(self, hspfmodel, folder): """Makes a plot of the subbasin area.""" subbasinfile = '{}/subbasins'.format(folder) boundaryfile = '{}/boundary'.format(folder) flowfile = '{}/flowlines'.format(folder) combinedfile = '{}/combined'.format(folder) watershedplot = '{}/watershed.png'.format(folder) # make a shapefile of the subbasins for the watershed f = '{0}/{1}/{1}subbasins'.format(self.directory, self.HUC8) for out in (subbasinfile, boundaryfile, flowfile, combinedfile): if not os.path.isfile(out + '.prj'): shutil.copy(f + '.prj', out + '.prj') if not os.path.isfile(subbasinfile + '.shp'): subshapes = [] subrecords = [] for subbasin in hspfmodel.subbasins: f = '{0}/{1}/{2}/combined'.format(self.directory, self.HUC8, subbasin) s = Reader(f, shapeType = 5) subshapes.append(s.shape(0).points) subrecords.append(s.record(0)) w = Writer(shapeType = 5) for field in s.fields: w.field(*field) for record in subrecords: w.record(*record) for shape in subshapes: w.poly(shapeType = 5, parts = [shape]) w.save(subbasinfile) if not os.path.isfile(combinedfile + '.shp'): fshapes = [] frecords = [] for subbasin in hspfmodel.subbasins: f = '{0}/{1}/{2}/combined_flowline'.format(self.directory, self.HUC8, subbasin) r = Reader(f, shapeType = 3) fshapes.append(r.shape(0).points) frecords.append(r.record(0)) w = Writer(shapeType = 3) for field in r.fields: w.field(*field) for record in frecords: w.record(*record) for shape in fshapes: w.poly(shapeType = 3, parts = [shape]) w.save(combinedfile) # merge the shapes into a watershed if not os.path.exists(boundaryfile + '.shp'): merge_shapes(subbasinfile, outputfile = boundaryfile) # make a flowline file for the subbasins for the watershed if not os.path.isfile(flowfile + '.shp'): shapes = [] records = [] for subbasin in hspfmodel.subbasins: f = '{0}/{1}/{2}/flowlines'.format(self.directory, self.HUC8, subbasin) r = Reader(f, shapeType = 3) for shape in r.shapes(): shapes.append(shape.points) for record in r.records(): records.append(record) w = Writer(shapeType = 3) for field in r.fields: w.field(*field) for record in records: w.record(*record) for shape in shapes: w.poly(shapeType = 3, parts = [shape]) w.save(flowfile) if not os.path.isfile(watershedplot): plot_gage_subbasin(folder, self.HUC8, self.gageid, hspfmodel, output = watershedplot)
def merge_shapes(inputfile, outputfile = None, overwrite = False, verbose = True, vverbose = False): """Merges all the shapes in a shapefile into a single shape.""" if outputfile is None: output = '{}/merged'.format(os.getcwd()) if os.path.isfile(outputfile + '.shp') and not overwrite: if verbose: print('combined watershed shapefile %s exists' % outputfile) return if verbose: print('combining shapes from %s\n' % inputfile) # start by copying the projection files shutil.copy(inputfile + '.prj', outputfile + '.prj') # load the catchment and flowline shapefiles r = Reader(inputfile, shapeType = 5) n = len(r.records()) try: shapes = [] records = [] bboxes = [] for i in range(n): shape = r.shape(i) record = r.record(i) shape_list = format_shape(shape.points) for sh in shape_list: shapes.append(sh) records.append(record) bboxes.append(shape.bbox) try: combined = combine_shapes(shapes, bboxes, verbose = vverbose) except: if verbose: print('trying alternate trace method') combined = combine_shapes(shapes, bboxes, skip = True, verbose = vverbose) except: if verbose: print('trying alternate trace method') shapes = [] records = [] bboxes = [] for i in range(n): shape = r.shape(i) record = r.record(i) shape_list = format_shape(shape.points, omit = True) for sh in shape_list: shapes.append(sh) records.append(record) bboxes.append(shape.bbox) try: combined = combine_shapes(shapes, bboxes, verbose = vverbose) except: if verbose: print('trying alternate trace method') combined = combine_shapes(shapes, bboxes, skip = True, verbose = vverbose) # create the new file with the merged shapes w = Writer(shapeType = 5) w.poly(shapeType = 5, parts = [combined]) # copy the fields from the original and then the first record; note this # can be adapted as needed for field in r.fields: w.field(*field) w.record(*r.record(0)) w.save(outputfile) if verbose: print('successfully combined shapes from %s to %s\n' % (inputfile, outputfile))
def extract_aquifers(directory, HUC8, aquifers, pad=0.2, verbose=True): """Extracts aquifers from the source datafile to the destination using the HUC8 boundaries for the query.""" start = time.time() # open up the HUC8 boundary shapefile and use it to get the bounding box shapefile = Reader(directory + '/%s/%scatchments' % (HUC8, HUC8)) xmin, ymin, xmax, ymax = get_boundaries(shapefile.shapes()) # convert to bounding corners for testing p1 = [xmin - pad * (xmax - xmin), ymin - pad * (ymax - ymin)] p2 = [xmax + pad * (xmax - xmin), ymax + pad * (ymax - ymin)] shapefile = None # start by copying the projection files if verbose: print('\ncopying the projections\n') shutil.copy(directory + '/%s/%scatchments.prj' % (HUC8, HUC8), directory + '/%s/%saquifers.prj' % (HUC8, HUC8)) # open the flowline file if verbose: print('reading the aquifer file\n') shapefile = Reader(aquifers, shapeType=5) # work around for issues with pyshp records = [] for i in range(len(shapefile.shapes())): try: records.append(shapefile.record(i)) except: records.append('') # use the bounding boxes to see if the shapes are within the watershed area if verbose: print('searching for aquifers in the watershed\n') bboxes = [shapefile.shape(i).bbox for i in range(len(records))] corners = [[[b[0], b[1]], [b[0], b[3]], [b[2], b[1]], [b[2], b[3]]] for b in bboxes] indices = [ i for i, c in zip(range(len(corners)), corners) if any([inside_box(p1, p2, p) for p in c]) or all([inside_box(c[0], c[3], p1), inside_box(c[0], c[3], p2)]) ] # remove any non aquifers indices = [i for i in indices if shapefile.record(i)[4] != 999] # find a record for the non aquifer i = 0 while shapefile.record(i)[4] != 999: i += 1 nonrecord = shapefile.record(i) nonrecord[1] = nonrecord[1].decode('utf-8') nonrecord[5] = 0 nonrecord[6] = 0 if len(indices) == 0: if verbose: print('query returned no values, returning\n') return # write the data from the HUC8 to a new shapefile w = Writer(directory + '/%s/%saquifers' % (HUC8, HUC8), shapeType=5) for field in shapefile.fields: w.field(*field) for i in indices: shape = shapefile.shape(i) # check for multiple parts if len(shape.parts) > 1: parts = [ shape.points[i:j] for i, j in zip(shape.parts[:-1], shape.parts[1:]) ] else: parts = [shape.points] record = records[i] # little work around for blank binary values if isinstance(record[1], bytes): record[1] = record[1].decode('utf-8') w.poly(shapeType=5, parts=parts) w.record(*record) # add a shape for the bounding box showing no aquifer locations part = [p1, [p1[0], p2[1]], p2, [p2[0], p1[1]]] w.poly(shapeType=5, parts=[part]) w.record(*nonrecord) w.close() end = time.time() if verbose: print('successfully queried data in %.2f seconds\n' % (end - start))
def plot_gage_subbasin(self, hspfmodel, folder): """Makes a plot of the subbasin area.""" subbasinfile = '{}/subbasins'.format(folder) boundaryfile = '{}/boundary'.format(folder) flowfile = '{}/flowlines'.format(folder) combinedfile = '{}/combined'.format(folder) watershedplot = '{}/watershed.png'.format(folder) # make a shapefile of the subbasins for the watershed f = '{0}/{1}/{1}subbasins'.format(self.directory, self.HUC8) for out in (subbasinfile, boundaryfile, flowfile, combinedfile): if not os.path.isfile(out + '.prj'): shutil.copy(f + '.prj', out + '.prj') if not os.path.isfile(subbasinfile + '.shp'): subshapes = [] subrecords = [] for subbasin in hspfmodel.subbasins: f = '{0}/{1}/{2}/combined'.format(self.directory, self.HUC8, subbasin) s = Reader(f, shapeType=5) subshapes.append(s.shape(0).points) subrecords.append(s.record(0)) w = Writer(shapeType=5) for field in s.fields: w.field(*field) for record in subrecords: w.record(*record) for shape in subshapes: w.poly(shapeType=5, parts=[shape]) w.save(subbasinfile) if not os.path.isfile(combinedfile + '.shp'): fshapes = [] frecords = [] for subbasin in hspfmodel.subbasins: f = '{0}/{1}/{2}/combined_flowline'.format( self.directory, self.HUC8, subbasin) r = Reader(f, shapeType=3) fshapes.append(r.shape(0).points) frecords.append(r.record(0)) w = Writer(shapeType=3) for field in r.fields: w.field(*field) for record in frecords: w.record(*record) for shape in fshapes: w.poly(shapeType=3, parts=[shape]) w.save(combinedfile) # merge the shapes into a watershed if not os.path.exists(boundaryfile + '.shp'): merge_shapes(subbasinfile, outputfile=boundaryfile) # make a flowline file for the subbasins for the watershed if not os.path.isfile(flowfile + '.shp'): shapes = [] records = [] for subbasin in hspfmodel.subbasins: f = '{0}/{1}/{2}/flowlines'.format(self.directory, self.HUC8, subbasin) r = Reader(f, shapeType=3) for shape in r.shapes(): shapes.append(shape.points) for record in r.records(): records.append(record) w = Writer(shapeType=3) for field in r.fields: w.field(*field) for record in records: w.record(*record) for shape in shapes: w.poly(shapeType=3, parts=[shape]) w.save(flowfile) if not os.path.isfile(watershedplot): plot_gage_subbasin(folder, self.HUC8, self.gageid, hspfmodel, output=watershedplot)
def combine_catchments(catchmentfile, flowfile, elevationfile, comid, output=None, overwrite=False, verbose=True): """Combines together all the catchments in a basin catchment shapefile. Creates a new shapefile called "combined" in the same directory as the original file. Uses the elevation data from the raster file and the flow data file to estimate the length and average slope of the overland flow plane. """ t0 = time.time() numpy.seterr(all='raise') if output is None: output = os.getcwd() + r'\combined' if os.path.isfile(output + '.shp') and not overwrite: if verbose: print('combined catchment shapefile %s exists' % output) return if verbose: print('combining catchments from %s\n' % catchmentfile) # start by copying the projection files shutil.copy(catchmentfile + '.prj', output + '.prj') # load the catchment and flowline shapefiles c = Reader(catchmentfile, shapeType=5) f = Reader(flowfile, shapeType=3) # make lists of the comids and featureids featureid_index = c.fields.index(['FEATUREID', 'N', 9, 0]) - 1 comid_index = f.fields.index(['COMID', 'N', 9, 0]) - 1 featureids = [r[featureid_index] for r in c.records()] comids = [r[comid_index] for r in f.records()] # check that shapes are traceable--don't have multiple points and start # and end at the same place--then make an appropriate list of shapes # and records--note it's more memory efficient to read one at a time n = len(c.records()) shapes = [] records = [] bboxes = [] try: for i in range(n): catchment = c.shape(i) record = c.record(i) shape_list = format_shape(catchment.points) for s in shape_list: shapes.append(s) records.append(record) bboxes.append(catchment.bbox) try: combined = combine_shapes(shapes, bboxes, verbose=verbose) except: combined = combine_shapes(shapes, bboxes, skip=True, verbose=verbose) except: shapes = [] records = [] bboxes = [] for i in range(n): catchment = c.shape(i) record = c.record(i) shape_list = format_shape(catchment.points, omit=True) for s in shape_list: shapes.append(s) records.append(record) bboxes.append(catchment.bbox) try: combined = combine_shapes(shapes, bboxes, verbose=verbose) except: combined = combine_shapes(shapes, bboxes, skip=True, verbose=verbose) # iterate through the catchments and get the elevation data from NED # then estimate the value of the overland flow plane length and slope lengths = numpy.empty((n), dtype='float') slopes = numpy.empty((n), dtype='float') for i in range(n): catchment = c.shape(i) flowline = f.shape(comids.index(featureids[i])) catchpoints = get_raster_on_poly(elevationfile, catchment.points, verbose=verbose) catchpoints = numpy.array([p for p in catchpoints]) zs = get_raster(elevationfile, flowline.points) flowpoints = numpy.array([[p[0], p[1], z] for p, z in zip(flowline.points, zs)]) # iterate through the raster values and find the closest flow point closest = numpy.empty((len(catchpoints), 3), dtype='float') for point, j in zip(catchpoints, range(len(catchpoints))): closest[j] = flowpoints[numpy.dot(flowpoints[:, :2], point[:2]).argmin()] # estimate the slope and overland flow plane length length, slope = get_overland_vector(catchpoints, closest) if verbose: print('avg slope and length =', slope.mean(), length.mean()) lengths[i], slopes[i] = length.mean(), slope.mean() if verbose: print('\nfinished overland flow plane calculations\n') # get area of the subbasin from the catchment metadata areasq_index = c.fields.index(['AreaSqKM', 'N', 19, 6]) - 1 areas = numpy.array([r[areasq_index] for r in c.records()]) # take the area weighted average of the slopes and flow lengths tot_area = round(areas.sum(), 2) avg_length = round(1000 * numpy.sum(areas * lengths) / tot_area, 1) avg_slope = round(numpy.sum(areas * slopes) / tot_area, 4) # get the centroid and the average elevation combined = [[float(x), float(y)] for x, y in combined] centroid = get_centroid(numpy.array(combined)) Cx, Cy = round(centroid[0], 4), round(centroid[1], 4) elev_matrix, origin = get_raster_in_poly(elevationfile, combined, verbose=verbose) elev_matrix = elev_matrix.flatten() elev_matrix = elev_matrix[elev_matrix.nonzero()] avg_elev = round(elev_matrix.mean() / 100., 2) # write the data to the shapefile w = Writer(shapeType=5) fields = [['ComID', 'N', 9, 0], ['PlaneLenM', 'N', 8, 2], ['PlaneSlope', 'N', 9, 6], ['AreaSqKm', 'N', 10, 2], ['CenX', 'N', 12, 6], ['CenY', 'N', 12, 6], ['AvgElevM', 'N', 8, 2]] record = [comid, avg_length, avg_slope, tot_area, Cx, Cy, avg_elev] for field in fields: w.field(*field) w.record(*record) w.poly(shapeType=5, parts=[combined]) w.save(output) if verbose: print('\ncompleted catchment combination in %.1f seconds\n' % (time.time() - t0))
def export_data(self, query): def get_label(item): label = item.descriptor if label is None: return None return label.label def abbrev_to(name, chars, columns_abbrev): if len(name) > chars: n = 1 while True: name_new = name[:chars - len(str(n))] + str(n) if not name_new in columns_abbrev.values(): return name_new n += 1 return name path = as_path(self.url, check_if_exists = False) if path is None: return geometries = [] # [[coords, geometry_type], ...] row_idxs = [] # [row_idx, ...] for row_idx, row in enumerate(query): for column in row: label = get_label(row[column]) if label.__class__.__name__ == "DGeometry": geometries.append(label.coords) row_idxs.append(row_idx) break if not row_idxs: return columns_abbrev = {} # {column: column_abbrev, ...}; abbreviated column names for column in query.columns: column_abbrev = column if len(column_abbrev) > 10: if "." in column_abbrev: column_abbrev = column_abbrev.split(".") column_abbrev = "_".join([abbrev_to(column_abbrev[0], 4, columns_abbrev), abbrev_to(column_abbrev[1], 5, columns_abbrev)]) else: column_abbrev = abbrev_to(column_abbrev, 10, columns_abbrev) column_abbrev = column_abbrev.replace(".", "_") columns_abbrev[column] = column_abbrev shapeType = -1 shape_types = { "POINT": 1, "LINESTRING": 3, "POLYGON": 5, "MULTIPOINT": 8, "POINTZ": 11, "LINESTRINGZ": 13, "POLYGONZ": 15, "MULTIPOINTZ": 18, "POINTM": 21, "LINESTRINGM": 23, "POLYGONM": 25, "MULTIPOINTM": 28, } for _, geometry_type in geometries: if geometry_type not in shape_types: raise Exception("Unknown geometry type") if shapeType > -1: if shape_types[geometry_type] != shapeType: raise Exception("Geometries must be of the same type") else: shapeType = shape_types[geometry_type] sf = Writer(shapeType = shapeType) types = {} # {column: type, ...} shp_types = {bool: "C", int: "N", float: "N", str: "C"} conv_order = ["N", "C"] for row in query: for column in row: label = get_label(row[column]) if label.__class__.__name__ != "DString": continue value = label.try_numeric typ = type(value) typ = shp_types[typ] if typ in shp_types else "C" if (not column in types) or ((typ != types[column]) and (conv_order.index(typ) > conv_order.index(types[column]))): types[column] = typ for column in types: sf.field(columns_abbrev[column], fieldType = types[column], size = "128") for i in range(len(geometries)): row = query[row_idxs[i]] coords = geometries[i][0] if shapeType in [1, 11, 21]: # point types sf.point(*coords[0], shapeType = shapeType) else: sf.poly(shapeType = shapeType, parts = [coords]) if types: record = [] for column in types: label = get_label(row[column]) if label is not None: label = label.value record.append(label) sf.record(*record) sf.save(path)
def trans_vector(in_file, ot_dir, output_flg, dem_path, flood_flg): """ オンライン学習3 被害領域の抽出、ラスタベクタ変換 二値画像からポリゴンを生成します 関数 : trans_vector 引数1 : 入力ファイル名(.tif) 引数2 : 出力ディレクトリ名 引数3 : 出力フラグ(0:被災領域、1:非被災領域) 引数4 : 数値標高モデル名(.shp) 引数5 : 災害フラグ(True:浸水、False:土砂崩れ) """ # Get destination file name filename = path.splitext(path.basename(in_file))[0] if filename.lower().startswith("sendai"): basename = "Sendai" elif filename.lower().startswith("kumamoto"): basename = "Kumamoto" else: basename = filename # Get actual file path in_file = path.join(DATA_PATH_BASE, in_file) ot_dir = path.join(DATA_PATH_BASE, ot_dir) dem_path = path.join(DATA_PATH_BASE, dem_path) makedirs(ot_dir, exist_ok=True) print("creating shapefile ...") # Create shapefile information of output area fn_tmp = path.join(ot_dir, "tmp.shp") writer = ShpWriter(target=fn_tmp, shapeType=POLYGON) writer.field("id", "C", "20", 0) writer.field("type", "C", "10", 0) writer.field("format", "C", "10", 0) writer.field("dis_tf", "C", "8", 0) writer.field("dis_tt", "C", "8", 0) writer.field("proc", "C", "8", 0) writer.field("pre_dn", "C", "10", 0) writer.field("pre_st", "C", "10", 0) writer.field("post_dn", "C", "10", 0) writer.field("post_st", "C", "10", 0) """ オンライン学習3 被害領域の抽出、ラスタベクタ変換 ポリゴンに付与する属性情報を定義するプログラムを実行します 関数 : get_flood_record 関数 : get_land_slide_record """ if flood_flg: # flood processing record = get_flood_record() else: # landslide processing record = get_land_slide_record() # Read binary image and get coordinate information bin = imread(in_file) rect_tiff = RectGeoTiffCoordinateCalculator(in_file) # Create rectangle polygon of output area and output to shapefile n_shape = bin.shape[0] * bin.shape[1] cnt = 0 for x_index, y_index in itertools.product(range(bin.shape[1]), range(bin.shape[0])): """ オンライン学習3 被害領域の抽出、ラスタベクタ変換 二値画像の各ピクセル四隅座標(緯度、経度)を計算するプログラムを実行します 関数 : create_polygon_points 引数1 : 対象ピクセルのx番号 引数2 : 対象ピクセルのy番号 引数3 : 二値画像の図形情報(ファイル名、画像サイズ等)を持つインスタンス """ points = create_polygon_points(x_index, y_index, rect_tiff) """ オンライン学習3 被害領域の抽出、ラスタベクタ変換 二値画像からメッシュを作成します bin[y_index, x_index] == 255):ピクセル値が255の場合 output_flg == "0":被災領域のメッシュを作成 output_flg == "1":非被災領域のメッシュを作成 """ if (bin[y_index, x_index] == 255) == (output_flg == "0"): # This pixel is output target. writer.poly([points]) writer.record(*record) cnt = cnt + 1 if cnt % 100000 == 0: print("{0}K / {1}K".format(cnt / 1000, n_shape / 1000)) writer.close() print("created shapefile .") if output_flg == "0": fn_out = path.join( ot_dir, "{0}_{1}cbp.shp".format(basename, time.strftime("%Y%m%d%H%M%S"))) else: fn_out = path.join( ot_dir, "{0}_{1}cbpr.shp".format(basename, time.strftime("%Y%m%d%H%M%S"))) # Attach elevation value """ オンライン学習3 被害領域の抽出、ラスタベクタ変換 メッシュデータに標高値を付与するプログラムを実行します 関数 : add_height_vector 引数1 : 入力メッシュデータ名(.tif) 引数2 : 数値標高モデル名(.shp) 引数3 : 出力ファイル名(.shp) """ add_height_vector([fn_tmp], dem_path, fn_out) if not DEV_FLAG: # Delete temporary file. remove("{0}.shp".format(path.splitext(fn_tmp)[0])) remove("{0}.shx".format(path.splitext(fn_tmp)[0])) remove("{0}.dbf".format(path.splitext(fn_tmp)[0]))
def merge_shapes(inputfile, outputfile = None, overwrite = False, verbose = True, vverbose = False): """Merges all the shapes in a shapefile into a single shape.""" if outputfile is None: output = '{}/merged'.format(os.getcwd()) if os.path.isfile(outputfile + '.shp') and not overwrite: if verbose: print('combined watershed shapefile %s exists' % outputfile) return if verbose: print('combining shapes from {}\n'.format(inputfile) + 'this may take a while...\n') # start by copying the projection files shutil.copy(inputfile + '.prj', outputfile + '.prj') # load the catchment and flowline shapefiles r = Reader(inputfile, shapeType = 5) n = len(r.records()) try: shapes = [] records = [] bboxes = [] for i in range(n): shape = r.shape(i) record = r.record(i) shape_list = format_shape(shape.points) for sh in shape_list: shapes.append(sh) records.append(record) bboxes.append(shape.bbox) try: combined = combine_shapes(shapes, bboxes, verbose = vverbose) except: if verbose: print('trying alternate trace method') combined = combine_shapes(shapes, bboxes, skip = True, verbose = vverbose) except: if verbose: print('trying alternate trace method') shapes = [] records = [] bboxes = [] for i in range(n): shape = r.shape(i) record = r.record(i) shape_list = format_shape(shape.points, omit = True) for sh in shape_list: shapes.append(sh) records.append(record) bboxes.append(shape.bbox) try: combined = combine_shapes(shapes, bboxes, verbose = vverbose) except: if verbose: print('trying alternate trace method') combined = combine_shapes(shapes, bboxes, skip = True, verbose = vverbose) # create the new file with the merged shapes w = Writer(shapeType = 5) w.poly(shapeType = 5, parts = [combined]) # copy the fields from the original and then the first record; note this # can be adapted as needed for field in r.fields: w.field(*field) w.record(*r.record(0)) w.save(outputfile) if verbose: print('successfully combined shapes from %s to %s\n' % (inputfile, outputfile))
def extract_aquifers(directory, HUC8, aquifers, pad = 0.2, verbose = True): """Extracts aquifers from the source datafile to the destination using the HUC8 boundaries for the query.""" start = time.time() # open up the HUC8 boundary shapefile and use it to get the bounding box shapefile = Reader(directory + '/%s/%scatchments' % (HUC8, HUC8)) xmin, ymin, xmax, ymax = get_boundaries(shapefile.shapes()) # convert to bounding corners for testing p1 = [xmin - pad * (xmax - xmin), ymin - pad * (ymax - ymin)] p2 = [xmax + pad * (xmax - xmin), ymax + pad * (ymax - ymin)] shapefile = None # start by copying the projection files if verbose: print('\ncopying the projections\n') shutil.copy(directory + '/%s/%scatchments.prj' % (HUC8, HUC8), directory + '/%s/%saquifers.prj' % (HUC8, HUC8)) # open the flowline file if verbose: print('reading the aquifer file\n') shapefile = Reader(aquifers, shapeType = 5) # work around for issues with pyshp records = [] for i in range(len(shapefile.shapes())): try: records.append(shapefile.record(i)) except: records.append('') # use the bounding boxes to see if the shapes are within the watershed area if verbose: print('searching for aquifers in the watershed\n') bboxes = [shapefile.shape(i).bbox for i in range(len(records))] corners = [[[b[0], b[1]], [b[0], b[3]], [b[2], b[1]], [b[2], b[3]]] for b in bboxes] indices = [i for i, c in zip(range(len(corners)), corners) if any([inside_box(p1, p2, p) for p in c]) or all([inside_box(c[0], c[3], p1), inside_box(c[0], c[3], p2)])] # remove any non aquifers indices = [i for i in indices if shapefile.record(i)[4] != 999] # find a record for the non aquifer i = 0 while shapefile.record(i)[4] != 999: i+=1 nonrecord = shapefile.record(i) nonrecord[1] = nonrecord[1].decode('utf-8') nonrecord[5] = 0 nonrecord[6] = 0 if len(indices) == 0: if verbose: print('query returned no values, returning\n') return # write the data from the HUC8 to a new shapefile w = Writer(shapeType = 5) for field in shapefile.fields: w.field(*field) for i in indices: shape = shapefile.shape(i) # check for multiple parts if len(shape.parts) > 1: parts = [shape.points[i:j] for i, j in zip(shape.parts[:-1], shape.parts[1:])] else: parts = [shape.points] record = records[i] # little work around for blank binary values if isinstance(record[1], bytes): record[1] = record[1].decode('utf-8') w.poly(shapeType = 5, parts = parts) w.record(*record) # add a shape for the bounding box showing no aquifer locations part = [p1, [p1[0], p2[1]], p2, [p2[0], p1[1]]] w.poly(shapeType = 5, parts = [part]) w.record(*nonrecord) w.save(directory + '/%s/%saquifers' % (HUC8, HUC8)) end = time.time() if verbose: print('successfully queried data in %.2f seconds\n' % (end - start))