def main(argv): """Go Main Go.""" scenario = int(argv[1]) pgconn = get_dbconn("idep") cursor = pgconn.cursor() cursor2 = pgconn.cursor() cursor.execute( """ SELECT st_x(st_pointn(st_transform(geom, 4326), 1)), st_y(st_pointn(st_transform(geom, 4326), 1)), fid, climate_file from flowpaths WHERE scenario = %s """, (scenario, ), ) ok = wrong = updated = 0 for row in cursor: fn = get_cli_fname(row[0], row[1]) if row[3] == fn: ok += 1 continue if row[3] is not None and row[3] != fn: wrong += 1 cursor2.execute( """ UPDATE flowpaths SET climate_file = %s where fid = %s """, (fn, row[2]), ) updated += 1 print("%s rows updated: %s ok: %s wrong: %s" % (cursor.rowcount, updated, ok, wrong)) cursor2.close() pgconn.commit()
def finder(lon, lat): for deltax in range(25): for multi in [-1 * deltax, 1 * deltax]: for multi2 in [-1 * deltax, 1 * deltax]: newlon = lon - multi * deltax / 100. newlat = lat - multi2 * deltax / 100. newfn = get_cli_fname(newlon, newlat) if os.path.isfile(newfn): return newfn
def missing_logic(scenario, fn): """Figure out what to do when this filename is missing""" print("Searching for replacement for '%s'" % (fn, )) lon = float(fn[17:23]) lat = float(fn[24:30]) if not os.path.isdir(os.path.dirname(fn)): os.makedirs(os.path.dirname(fn)) # So there should be a file at an interval of 0.25 lon2 = lon - (lon * 100 % 25) / 100.0 lat2 = lat - (lat * 100 % 25) / 100.0 testfn = get_cli_fname(lon2, lat2, scenario) if not os.path.isfile(testfn): print("Whoa, why doesn't %s exist?" % (testfn, )) sys.exit() print("%s->%s" % (testfn, fn)) shutil.copyfile(testfn, fn)
def missing_logic(scenario, fn): """Figure out what to do when this filename is missing""" print("Searching for replacement for '%s'" % (fn,)) lon = float(fn[17:23]) lat = float(fn[24:30]) if not os.path.isdir(os.path.dirname(fn)): os.makedirs(os.path.dirname(fn)) # So there should be a file at an interval of 0.25 lon2 = lon - (lon * 100 % 25) / 100. lat2 = lat - (lat * 100 % 25) / 100. testfn = get_cli_fname(lon2, lat2, scenario) if not os.path.isfile(testfn): print("Whoa, why doesn't %s exist?" % (testfn,)) sys.exit() print("%s->%s" % (testfn, fn)) shutil.copyfile(testfn, fn)
def myjob(row): """ Thread job, yo """ [xidx, yidx] = row lon = MYWEST + xidx * 0.01 lat = MYSOUTH + yidx * 0.01 fn = get_cli_fname(lon, lat, SCENARIO) if not os.path.isfile(fn): return False # Okay we have work to do data = open(fn, "r").read() pos = data.find(VALID.strftime("%-d\t%-m\t%Y")) if pos == -1: print("Date find failure for %s" % (fn, )) return False pos2 = data[pos:].find( (VALID + datetime.timedelta(days=1)).strftime("%-d\t%-m\t%Y")) if pos2 == -1: print("Date2 find failure for %s" % (fn, )) return False bpdata = compute_breakpoint(PRECIP[yidx, xidx, :]) if bpdata is None: bpdata = [] thisday = ( "%s\t%s\t%s\t%s\t%3.1f\t%3.1f\t%4.0f\t%4.1f\t%s\t%4.1f\n%s%s") % ( VALID.day, VALID.month, VALID.year, len(bpdata), HIGH_TEMP[yidx, xidx], LOW_TEMP[yidx, xidx], SOLAR[yidx, xidx], WIND[yidx, xidx], 0, DEWPOINT[yidx, xidx], "\n".join(bpdata), "\n" if bpdata else "", ) fp = open(fn, "w") fp.write(data[:pos] + thisday + data[(pos + pos2):]) fp.close() return True
def spiral(lon, lat): """https://stackoverflow.com/questions/398299/looping-in-a-spiral""" x = y = 0 dx = 0 dy = -1 # points near the domain edge need to seach a bit further than 0.25deg X = 40 Y = 40 for _ in range(40**2): if (-X / 2 < x <= X / 2) and (-Y / 2 < y <= Y / 2): newfn = get_cli_fname(lon + x * 0.01, lat + y * 0.01) if os.path.isfile(newfn): return newfn if x == y or (x < 0 and x == -y) or (x > 0 and x == 1 - y): dx, dy = -dy, dx x, y = x + dx, y + dy return None
def myjob(row): """ Thread job, yo """ [xidx, yidx] = row lon = MYWEST + xidx * 0.01 lat = MYSOUTH + yidx * 0.01 fn = get_cli_fname(lon, lat, SCENARIO) if not os.path.isfile(fn): return False # Okay we have work to do data = open(fn, 'r').read() pos = data.find(VALID.strftime("%-d\t%-m\t%Y")) if pos == -1: print('Date find failure for %s' % (fn,)) return False pos2 = data[pos:].find( (VALID + datetime.timedelta(days=1)).strftime("%-d\t%-m\t%Y")) if pos2 == -1: print('Date2 find failure for %s' % (fn,)) return False bpdata = compute_breakpoint(PRECIP[yidx, xidx, :]) if bpdata is None: bpdata = [] thisday = ("%s\t%s\t%s\t%s\t%3.1f\t%3.1f\t%4.0f\t%4.1f\t%s\t%4.1f\n%s%s" ) % (VALID.day, VALID.month, VALID.year, len(bpdata), HIGH_TEMP[yidx, xidx], LOW_TEMP[yidx, xidx], SOLAR[yidx, xidx], WIND[yidx, xidx], 0, DEWPOINT[yidx, xidx], "\n".join(bpdata), "\n" if bpdata else "") fp = open(fn, 'w') fp.write(data[:pos] + thisday + data[(pos+pos2):]) fp.close() return True
def main(): """Go Main Go.""" # We shall use this file, no mater what SRC = "/i/0/cli/095x038/095.17x038.13.cli" SCENARIO = 0 pgconn = get_dbconn("postgis") cursor = pgconn.cursor() created = 0 removed = 0 for lon in np.arange(WEST, EAST + 0.25, 0.25): for lat in np.arange(SOUTH, NORTH + 0.25, 0.25): fn = get_cli_fname(lon, lat, SCENARIO) # Ensure this point is on land, in CONUS cursor.execute( """ SELECT ugc from ugcs where ST_Contains(geom, ST_GeomFromText('Point(%s %s)',4326)) and substr(ugc, 3, 1) = 'C' and end_ts is null """, (lon, lat), ) if cursor.rowcount == 0: # print("lon: %s lat: %s is not onland" % (lon, lat)) if os.path.isfile(fn): print("Removing %s" % (fn, )) os.unlink(fn) removed += 1 continue mydir = fn.rsplit("/", 1)[0] if not os.path.isdir(mydir): os.makedirs(mydir) if not os.path.isfile(fn): created += 1 shutil.copyfile(SRC, fn) print("We just created %s new files, removed %s files!" % (created, removed))
def do_flowpath(zone, huc_12, fid, fpath): """ Process a given flowpathid """ # slope = compute_slope(fid) # I need bad soilfiles so that the length can be computed cursor2.execute( """SELECT segid, elevation, length, f.surgo, slope, management, 'MWDEP_'||surgo||'.SOL' as soilfile, landuse, round(ST_X(ST_Transform(geom,4326))::numeric,2) as x, round(ST_Y(ST_Transform(geom,4326))::numeric,2) as y from flowpath_points f WHERE flowpath = %s and length < 9999 and scenario = 0 ORDER by segid ASC""", (fid, ), ) rows = [] x = None y = None maxmanagement = 0 for row in cursor2: if row["slope"] < 0: print("%s,%s had a negative slope, deleting!" % (huc_12, fpath)) return None if row["soilfile"] == "MWDEP_9999.SOL": continue if not os.path.isfile("/i/0/sol_input/%s" % (row["soilfile"], )): if row["soilfile"] not in MISSED_SOILS: print("Missing soilfile: %s" % (row["soilfile"], )) MISSED_SOILS[row["soilfile"]] = 0 MISSED_SOILS[row["soilfile"]] += 1 continue if x is None: x = row["x"] y = row["y"] if row["management"] > maxmanagement: maxmanagement = row["management"] if row["slope"] < 0.00001: row["slope"] = 0.00001 # hard coded... # row['slope'] = slope rows.append(row) if x is None: print("%s,%s had no valid soils, deleting" % (huc_12, fpath)) return None if len(rows) < 2: print("%s,%s had only 1 row of data, deleting" % (huc_12, fpath)) return None if len(rows) > 19: rows = simplify(rows) maxslope = 0 for row in rows: if row["slope"] > maxslope: maxslope = row["slope"] if maxslope > 1: s = compute_slope(fid) print("%s %3i %4.1f %5.1f %5.1f" % (huc_12, fpath, maxslope, rows[-1]["length"], s)) # SLP.write("%s,%.6f\n" % (fid, maxslope)) if rows[-1]["length"] < 1: print("%s,%s has zero length, deleting" % (huc_12, fpath)) return None res = {} res["clifile"] = get_cli_fname(x, y, 0) cursor3.execute( """ UPDATE flowpaths SET climate_file = %s WHERE fpath = %s and huc_12 = %s and scenario = %s """, (res["clifile"], fpath, huc_12, SCENARIO), ) # return res["huc8"] = huc_12[:8] res["huc12"] = huc_12 res["envfn"] = "/i/%s/env/%s/%s_%s.env" % ( SCENARIO, res["huc8"], huc_12, fpath, ) res["date"] = datetime.datetime.now() res["aspect"] = compute_aspect(rows[0]["x"], rows[0]["y"], rows[-1]["x"], rows[-1]["y"]) res["prj_fn"] = "/i/%s/prj/%s/%s/%s_%s.prj" % ( SCENARIO, res["huc8"], huc_12[-4:], huc_12, fpath, ) res["length"] = rows[-1]["length"] res["slope_points"] = len(rows) # Iterate over the rows and figure out the slopes and distance fracs slpdata = "" if rows[0]["length"] != 0: print(("WARNING: HUC12:%s FPATH:%s had missing soil at top of slope") % (huc_12, fpath)) slpdata = " 0.000,0.00001" res["slope_points"] = len(rows) + 1 prevsoil = None lsoilstart = 0 soils = [] soillengths = [] for row in rows: if prevsoil != row["surgo"]: if prevsoil is not None: soils.append(prevsoil) soillengths.append(row["length"] - lsoilstart) prevsoil = row["surgo"] lsoilstart = row["length"] if res["length"] == 0: continue slpdata += " %.3f,%.5f" % (row["length"] / res["length"], row["slope"]) soils.append(prevsoil) soillengths.append(res["length"] - lsoilstart) res["soilbreaks"] = len(soillengths) - 1 res["soils"] = "" res["slpdata"] = slpdata for d, s in zip(soillengths, soils): res["soils"] += """ %s { Distance = %.3f File = "/i/0/sol_input/MWDEP_%s.SOL" }\n""" % ( s, d, s, ) prevman = None lmanstart = 0 mans = [] manlengths = [] for row in rows: if row["landuse"] is None: continue if prevman is None or prevman != row["landuse"]: if prevman is not None: mans.append(get_rotation(prevman)) manlengths.append(row["length"] - lmanstart) prevman = row["landuse"] lmanstart = row["length"] if prevman is None: print("%s,%s has no managements, skipping" % (huc_12, fpath)) return mans.append(get_rotation(prevman)) manlengths.append(res["length"] - lmanstart) res["manbreaks"] = len(manlengths) - 1 res["managements"] = "" for d, s in zip(manlengths, mans): res["managements"] += """ %s { Distance = %.3f File = "%s" }\n""" % ( s, d, s, ) return res
def test_cli_fname(): """Do we get the right climate file names?""" res = dep.get_cli_fname(-95.5, 42.5, 0) assert res == "/i/0/cli/095x042/095.50x042.50.cli" res = dep.get_cli_fname(-97.9999, 42.0, 0) assert res == "/i/0/cli/098x042/098.00x042.00.cli"
def do_flowpath(zone, huc_12, fid, fpath): """ Process a given flowpathid """ # slope = compute_slope(fid) # I need bad soilfiles so that the length can be computed cursor2.execute("""SELECT segid, elevation, length, f.surgo, slope, management, 'MWDEP_'||surgo||'.SOL' as soilfile, lu2007 || lu2008 || lu2009 || lu2010 || lu2011 || lu2012 || lu2013 || lu2014 || lu2015 || lu2016 || lu2017 || lu2018 || lu2019 as lstring, round(ST_X(ST_Transform(geom,4326))::numeric,2) as x, round(ST_Y(ST_Transform(geom,4326))::numeric,2) as y from flowpath_points f WHERE flowpath = %s and length < 9999 and scenario = %s ORDER by segid ASC""", (fid, SCENARIO)) rows = [] x = None y = None maxmanagement = 0 for row in cursor2: if row['slope'] < 0: print('%s,%s had a negative slope, deleting!' % (huc_12, fpath)) delete_flowpath(fid) return None if row['soilfile'] == 'MWDEP_9999.SOL': continue if not os.path.isfile("/i/%s/sol_input/%s" % (SCENARIO, row['soilfile'])): if row['soilfile'] not in MISSED_SOILS: print('Missing soilfile: %s' % (row['soilfile'],)) MISSED_SOILS[row['soilfile']] = 0 MISSED_SOILS[row['soilfile']] += 1 continue if x is None: x = row['x'] y = row['y'] if row['management'] > maxmanagement: maxmanagement = row['management'] if row['slope'] < 0.00001: row['slope'] = 0.00001 # hard coded... # row['slope'] = slope rows.append(row) if x is None: print('%s,%s had no valid soils, deleting' % (huc_12, fpath)) delete_flowpath(fid) return None if len(rows) < 2: print('%s,%s had only 1 row of data, deleting' % (huc_12, fpath)) delete_flowpath(fid) return None if len(rows) > 19: rows = simplify(rows) maxslope = 0 for row in rows: if row['slope'] > maxslope: maxslope = row['slope'] if maxslope > 1: s = compute_slope(fid) print("%s %3i %4.1f %5.1f %5.1f" % (huc_12, fpath, maxslope, rows[-1]['length'], s)) # SLP.write("%s,%.6f\n" % (fid, maxslope)) if rows[-1]['length'] < 1: print('%s,%s has zero length, deleting' % (huc_12, fpath)) delete_flowpath(fid) return None res = {} # These scenarios use one climate file if SCENARIO in [5, 0]: res['clifile'] = get_cli_fname(-96.44, 43.28, SCENARIO) else: res['clifile'] = get_cli_fname(x, y, SCENARIO) # Store climate_file name with flowpath to make life easier cursor3.execute("""UPDATE flowpaths SET climate_file = %s WHERE fid = %s """, (res['clifile'], fid)) if cursor3.rowcount != 1: print('ERROR Updating climate_file for FID: %s' % (fid,)) # return res['huc8'] = huc_12[:8] res['huc12'] = huc_12 res['envfn'] = "/i/%s/env/%s/%s_%s.env" % (SCENARIO, res['huc8'], huc_12, fpath) res['date'] = datetime.datetime.now() res['aspect'] = compute_aspect(rows[0]['x'], rows[0]['y'], rows[-1]['x'], rows[-1]['y']) res['prj_fn'] = "/i/%s/prj/%s/%s/%s_%s.prj" % (SCENARIO, res['huc8'], huc_12[-4:], huc_12, fpath) res['length'] = rows[-1]['length'] res['slope_points'] = len(rows) # Iterate over the rows and figure out the slopes and distance fracs slpdata = "" if rows[0]['length'] != 0: print(('WARNING: HUC12:%s FPATH:%s had missing soil at top of slope' ) % (huc_12, fpath)) slpdata = " 0.000,0.00001" res['slope_points'] = len(rows) + 1 prevsoil = None lsoilstart = 0 soils = [] soillengths = [] for row in rows: if prevsoil != row['surgo']: if prevsoil is not None: soils.append(prevsoil) soillengths.append(row['length'] - lsoilstart) prevsoil = row['surgo'] lsoilstart = row['length'] if res['length'] == 0: continue slpdata += " %.3f,%.5f" % (row['length'] / res['length'], row['slope']) soils.append(prevsoil) soillengths.append(res['length'] - lsoilstart) res['soilbreaks'] = len(soillengths) - 1 res['soils'] = "" res['slpdata'] = slpdata for d, s in zip(soillengths, soils): res['soils'] += """ %s { Distance = %.3f File = "/i/%s/sol_input/MWDEP_%s.SOL" }\n""" % (s, d, SCENARIO, s) prevman = None lmanstart = 0 mans = [] manlengths = [] for row in rows: if row['lstring'] is None: continue if prevman is None or prevman != row['lstring']: if prevman is not None: mans.append(get_rotation(zone, prevman, maxmanagement)) manlengths.append(row['length'] - lmanstart) prevman = row['lstring'] lmanstart = row['length'] if prevman is None: print('%s,%s has no managements, skipping' % (huc_12, fpath)) return mans.append(get_rotation(zone, prevman, maxmanagement)) manlengths.append(res['length'] - lmanstart) res['manbreaks'] = len(manlengths) - 1 res['managements'] = "" for d, s in zip(manlengths, mans): res['managements'] += """ %s { Distance = %.3f File = "%s" }\n""" % (s, d, s) return res
"""When DEP expands, we have a problem that we have no close by precip files. So lets ensure we have one file per 0.25 degree lat and lon, we can then use these to fast-start newly expanded areas... """ import os import shutil import numpy as np from pyiem.dep import SOUTH, EAST, NORTH, WEST, get_cli_fname # We shall use this file, no mater what SRC = "/i/0/cli/095x038/095.17x038.13.cli" SCENARIO = 0 WANTS = ['.00', '.25', '.50', '.75'] created = 0 for lon in np.arange(WEST, EAST, 0.05): for lat in np.arange(SOUTH, NORTH, 0.05): if (("%.2f" % (lon, ))[-3:] in WANTS and ("%.2f" % (lat, ))[-3:] in WANTS): fn = get_cli_fname(lon, lat, SCENARIO) mydir = fn.rsplit("/", 1)[0] if not os.path.isdir(mydir): os.makedirs(mydir) if not os.path.isfile(fn): created += 1 shutil.copyfile(SRC, fn) print("We just created %s new files!" % (created, ))
def test_cli_fname(): """Do we get the right climate file names?""" res = dep.get_cli_fname(-95.5, 42.5, 0) assert res == "/i/0/cli/095x042/095.50x042.50.cli" res = dep.get_cli_fname(-97.9999, 42.0, 0) assert res == "/i/0/cli/098x042/098.00x042.00.cli"
def do_flowpath(zone, huc_12, fid, fpath): """ Process a given flowpathid """ # slope = compute_slope(fid) # I need bad soilfiles so that the length can be computed cursor2.execute("""SELECT segid, elevation, length, f.surgo, slope, management, 'DEP_'||surgo||'.SOL' as soilfile, lu2007 || lu2008 || lu2009 || lu2010 || lu2011 || lu2012 || lu2013 || lu2014 || lu2015 || lu2016 || lu2017 || lu2018 || lu2019 as lstring, round(ST_X(ST_Transform(geom,4326))::numeric,2) as x, round(ST_Y(ST_Transform(geom,4326))::numeric,2) as y from flowpath_points f WHERE flowpath = %s and length < 9999 and scenario = %s ORDER by segid ASC""", (fid, SCENARIO)) rows = [] x = None y = None maxmanagement = 0 for row in cursor2: if row['slope'] < 0: print('%s,%s had a negative slope, deleting!' % (huc_12, fpath)) delete_flowpath(fid) return None if row['soilfile'] == 'DEP_9999.SOL': continue if not os.path.isfile("/i/%s/sol_input/%s" % (SCENARIO, row['soilfile'])): if row['soilfile'] not in MISSED_SOILS: print('Missing soilfile: %s' % (row['soilfile'],)) MISSED_SOILS[row['soilfile']] = 0 MISSED_SOILS[row['soilfile']] += 1 continue if x is None: x = row['x'] y = row['y'] if row['management'] > maxmanagement: maxmanagement = row['management'] if row['slope'] < 0.00001: row['slope'] = 0.00001 # hard coded... # row['slope'] = slope rows.append(row) if x is None: print('%s,%s had no valid soils, deleting' % (huc_12, fpath)) delete_flowpath(fid) return None if len(rows) < 2: print('%s,%s had only 1 row of data, deleting' % (huc_12, fpath)) delete_flowpath(fid) return None if len(rows) > 19: rows = simplify(rows) maxslope = 0 for row in rows: if row['slope'] > maxslope: maxslope = row['slope'] if maxslope > MAX_SLOPE_RATIO: s = compute_slope(fid) print("Error max-slope>0.3 %s[%3i] max:%4.1f len:%5.1f bulk:%5.1f" % ( huc_12, fpath, maxslope, rows[-1]['length'], s)) delete_flowpath(fid) return None # SLP.write("%s,%.6f\n" % (fid, maxslope)) if rows[-1]['length'] < 1: print('%s,%s has zero length, deleting' % (huc_12, fpath)) delete_flowpath(fid) return None res = {} # These scenarios use one climate file if SCENARIO in [5, ]: res['clifile'] = get_cli_fname(-96.44, 43.28, SCENARIO) else: res['clifile'] = get_cli_fname(x, y, SCENARIO) # Store climate_file name with flowpath to make life easier cursor3.execute(""" UPDATE flowpaths SET climate_file = %s WHERE fid = %s """, (res['clifile'], fid)) if cursor3.rowcount != 1: print('ERROR Updating climate_file for FID: %s' % (fid,)) # return res['huc8'] = huc_12[:8] res['huc12'] = huc_12 res['envfn'] = "/i/%s/env/%s/%s_%s.env" % (SCENARIO, res['huc8'], huc_12, fpath) res['date'] = datetime.datetime.now() res['aspect'] = compute_aspect(rows[0]['x'], rows[0]['y'], rows[-1]['x'], rows[-1]['y']) res['prj_fn'] = "/i/%s/prj/%s/%s/%s_%s.prj" % (SCENARIO, res['huc8'], huc_12[-4:], huc_12, fpath) res['length'] = rows[-1]['length'] res['slope_points'] = len(rows) # Iterate over the rows and figure out the slopes and distance fracs slpdata = "" if rows[0]['length'] != 0: print(('WARNING: HUC12:%s FPATH:%s had missing soil at top of slope' ) % (huc_12, fpath)) slpdata = " 0.000,0.00001" res['slope_points'] = len(rows) + 1 prevsoil = None lsoilstart = 0 soils = [] soillengths = [] for row in rows: if prevsoil != row['surgo']: if prevsoil is not None: soils.append(prevsoil) soillengths.append(row['length'] - lsoilstart) prevsoil = row['surgo'] lsoilstart = row['length'] if res['length'] == 0: continue slpdata += " %.3f,%.5f" % (row['length'] / res['length'], row['slope']) soils.append(prevsoil) soillengths.append(res['length'] - lsoilstart) res['soilbreaks'] = len(soillengths) - 1 res['soils'] = "" res['slpdata'] = slpdata for d, s in zip(soillengths, soils): res['soils'] += """ %s { Distance = %.3f File = "/i/%s/sol_input/DEP_%s.SOL" }\n""" % (s, d, SCENARIO, s) prevman = None lmanstart = 0 mans = [] manlengths = [] for row in rows: if row['lstring'] is None: continue if prevman is None or prevman != row['lstring']: if prevman is not None: mans.append(get_rotation(zone, prevman, maxmanagement)) manlengths.append(row['length'] - lmanstart) prevman = row['lstring'] lmanstart = row['length'] if prevman is None: print('%s,%s has no managements, skipping' % (huc_12, fpath)) return mans.append(get_rotation(zone, prevman, maxmanagement)) manlengths.append(res['length'] - lmanstart) res['manbreaks'] = len(manlengths) - 1 res['managements'] = "" for d, s in zip(manlengths, mans): res['managements'] += """ %s { Distance = %.3f File = "%s" }\n""" % (s, d, s) return res