def show_report(the_id, csv=False, showall=False):
dbc.execute("SELECT * FROM runs where id = ?", (the_id, ))
run_rows = dbc.fetchall()
print(tabulate.tabulate(run_rows))
dbc.execute("SELECT * FROM timings WHERE run_id = {} "
"ORDER BY batch, count, numrecs".format(the_id))
rows = dbc.fetchall()
headers = ['id', 'batch', 'backend', 'numrecs', 'count',
'total_time', 'avg_time', 'mem_inc', 'load_inc', 'disk_inc',
'image', 'runid']
if csv:
fmt = tabulate.simple_separated_format(",")
else:
fmt = 'simple'
if not showall:
print(tabulate.tabulate(rows, headers=headers, tablefmt=fmt,
floatfmt='.3g'))
return
for row in rows:
print("\n\n")
print(tabulate.tabulate([row], headers=headers, tablefmt=fmt,
floatfmt='.3g'))
id = row[0]
dbc.execute("SELECT * FROM recs WHERE timings_id = ?", (id,))
print("\n")
print(tabulate.tabulate(dbc.fetchall(), headers=['cmd', 'dur', 'id'],
floatfmt='.3g'))
def test_simple_separated_format():
"Regression: simple_separated_format() accepts any separator (issue #12)"
from tabulate import simple_separated_format
fmt = simple_separated_format("!")
expected = 'spam!eggs'
formatted = tabulate([["spam", "eggs"]], tablefmt=fmt)
print("expected: %r\n\ngot: %r\n" % (expected, formatted))
assert_equal(expected, formatted)
def pretty_print(self):
"""
Give the string representation of the grid for HUMAN.
Using ratio symbol as column separator.
"""
separator = simple_separated_format(NlgSymbols.ratio)
s = tabulate(self, tablefmt=separator)
return s + '\n'
def test_simple_separated_format_with_headers():
"Regression: simple_separated_format() on tables with headers (issue #15)"
from tabulate import simple_separated_format
expected = ' a| b\n 1| 2'
formatted = tabulate([[1, 2]],
headers=["a", "b"],
tablefmt=simple_separated_format("|"))
assert_equal(expected, formatted)
def test_simple_separated_format():
"Regression: simple_separated_format() accepts any separator (issue #12)"
from tabulate import simple_separated_format
fmt = simple_separated_format("!")
expected = 'spam!eggs'
formatted = tabulate([["spam", "eggs"]], tablefmt=fmt)
print("expected: %r\n\ngot: %r\n" % (expected, formatted))
assert_equal(expected, formatted)
def test_unaligned_separated():
"Output: non-aligned data columns"
expected = '\n'.join(['name|score', 'Alice|1', 'Bob|333'])
fmt = simple_separated_format("|")
result = tabulate([['Alice', 1], ['Bob', 333]], ["name", "score"],
tablefmt=fmt,
stralign=None,
numalign=None)
assert_equal(expected, result)
def test_unaligned_separated():
"Output: non-aligned data columns"
expected = '\n'.join(['name|score',
'Alice|1',
'Bob|333'])
fmt = simple_separated_format("|")
result = tabulate([['Alice', 1],['Bob', 333]],
["name", "score"],
tablefmt=fmt, stralign=None, numalign=None)
assert_equal(expected, result)
def test_unaligned_separated():
"Output: non-aligned data columns"
expected = "\n".join(["name|score", "Alice|1", "Bob|333"])
fmt = simple_separated_format("|")
result = tabulate(
[["Alice", 1], ["Bob", 333]],
["name", "score"],
tablefmt=fmt,
stralign=None,
numalign=None,
)
assert_equal(expected, result)
def details_price_textual(self) -> str:
sd = self.details_price()
si_volume = si_format(sd.volume, precision=2)
si_cap = si_format(round_currency_scalar(sd.market_capitalization),
precision=2)
si_volume_f = float(si_volume[:-1])
si_cap_f = float(si_cap[:-1])
data = [{
conf.LOCAL['currency']: round_currency_scalar(sd.price),
'24h H': round_currency_scalar(sd.price_24h_high),
'24h L': round_currency_scalar(sd.price_24h_low),
'52w H': round_currency_scalar(sd.price_52w_high),
'52w L': round_currency_scalar(sd.price_52w_low),
'% 1h': round_percent(sd.percent_change_1h),
'% 24h': round_percent(sd.percent_change_24h),
'% 7d': round_percent(sd.percent_change_7d),
'% 30d': round_percent(sd.percent_change_30d),
'% 52w': round_percent(sd.percent_change_52w),
'% YTD': round_percent(sd.percent_change_ytd),
'Vol': si_volume_f,
'Cap': si_cap_f,
# 'Recom': sd.recommendation
}]
# Creates pandas DataFrame by passing
# Lists of dictionaries and row index.
df = pd.DataFrame(data, index=[self.symbol])
df_T = df.T
table_fmt = simple_separated_format(': ')
text = tabulate(df_T,
tablefmt=table_fmt,
colalign=('right', 'decimal'))
text = text.replace(str(si_volume_f), si_volume)
text = text.replace(str(si_cap_f), si_cap)
text = ''.join((f' {self.symbol}\n', f'{text}\n',
f'Recom: {sd.recommendation}'))
return text
def test_simple_separated_format_with_headers():
"Regression: simple_separated_format() on tables with headers (issue #15)"
from tabulate import simple_separated_format
expected = ' a| b\n 1| 2'
formatted = tabulate([[1,2]], headers=["a", "b"], tablefmt=simple_separated_format("|"))
assert_equal(expected, formatted)
def _printiso(tsd,
date_format=None,
sep=',',
float_format='%g',
showindex="never",
headers="keys",
tablefmt="csv"):
"""Separate so can use in tests."""
sys.tracebacklimit = 1000
if isinstance(tsd, (pd.DataFrame, pd.Series)):
if isinstance(tsd, pd.Series):
tsd = pd.DataFrame(tsd)
if len(tsd.columns) == 0:
tsd = pd.DataFrame(index=tsd.index)
# Not perfectly true, but likely will use showindex for indices
# that are not time stamps.
if showindex is True:
if not tsd.index.name:
tsd.index.name = 'UniqueID'
else:
if not tsd.index.name:
tsd.index.name = 'Datetime'
print_index = True
if tsd.index.is_all_dates is True:
if tsd.index.name is None:
tsd.index.name = 'Datetime'
# Someone made the decision about the name
# This is how I include time zone info by tacking on to the
# index.name.
elif 'datetime' not in tsd.index.name.lower():
tsd.index.name = 'Datetime'
else:
# This might be overkill, but tstoolbox is for time-series.
# Revisit if necessary.
print_index = False
if tsd.index.name == 'UniqueID':
print_index = False
if showindex in ['always', 'default']:
print_index = True
elif isinstance(tsd, (int, float, tuple, pd.np.ndarray)):
tablefmt = None
if tablefmt in ["csv", "tsv", "csv_nos", "tsv_nos"]:
sep = {
"csv": ",",
"tsv": "\\t",
"csv_nos": ",",
"tsv_nos": "\\t"
}[tablefmt]
if isinstance(tsd, pd.DataFrame):
try:
tsd.to_csv(sys.stdout,
float_format=float_format,
date_format=date_format,
sep=sep,
index=print_index)
return
except IOError:
return
else:
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if fmt is None:
print(str(list(tsd))[1:-1])
elif tablefmt in ['csv_nos', 'tsv_nos']:
print(
tb(tsd, tablefmt=fmt, showindex=showindex,
headers=headers).replace(' ', ''))
else:
print(tb(tsd, tablefmt=fmt, showindex=showindex, headers=headers))
def process(uci,
hbn,
pwbe,
year,
ofilename,
modulus,
tablefmt,
float_format=".2f"):
from hspfbintoolbox.hspfbintoolbox import extract
if ofilename:
sys.stdout = open(ofilename, "w")
try:
year = int(year)
except TypeError:
pass
lcnames = dict(zip(range(modulus + 1, 1), zip(range(modulus + 1, 1))))
inverse_lcnames = dict(
zip(range(modulus + 1, 1), zip(range(modulus + 1, 1))))
inverse_lc = {}
lnds = {}
if uci is not None:
with open(uci) as fp:
content = fp.readlines()
if not os.path.exists(hbn):
raise ValueError(f"""
*
* File {hbn} does not exist.
*
""")
content = [i[:80] for i in content]
content = [i.rstrip() for i in content]
schematic_start = content.index("SCHEMATIC")
schematic_end = content.index("END SCHEMATIC")
schematic = content[schematic_start:schematic_end + 1]
perlnd_start = content.index("PERLND")
perlnd_end = content.index("END PERLND")
perlnd = content[perlnd_start:perlnd_end + 1]
pgeninfo_start = perlnd.index(" GEN-INFO")
pgeninfo_end = perlnd.index(" END GEN-INFO")
pgeninfo = perlnd[pgeninfo_start:pgeninfo_end + 1]
masslink_start = content.index("MASS-LINK")
masslink_end = content.index("END MASS-LINK")
masslink = content[masslink_start:masslink_end + 1]
lcnames = {}
inverse_lcnames = {}
inverse_lc = {}
for line in pgeninfo[1:-1]:
if "***" in line:
continue
if "" == line.strip():
continue
try:
_ = int(line[5:10])
continue
except ValueError:
pass
lcnames.setdefault(line[10:30].strip(), []).append(int(line[:5]))
inverse_lcnames[int(line[:5])] = line[10:30].strip()
inverse_lc[int(line[:5]) % modulus] = line[10:30].strip()
masslink = [i for i in masslink if "***" not in i]
masslink = [i for i in masslink if len(i.strip()) > 0]
masslink = " ".join(masslink)
mlgroups = re.findall(
r" MASS-LINK +?([0-9]+).*?LND [PI]WATER [PS][EU]RO.*? END MASS-LINK +?\1 ",
masslink,
)
for line in schematic[3:-1]:
if "***" in line:
continue
if "" == line:
continue
words = line.split()
if words[0] in ["PERLND", "IMPLND"] and words[5] in mlgroups:
lnds[(words[0], int(words[1]))] = lnds.setdefault(
(words[0], int(words[1])), 0.0) + float(words[2])
try:
pdf = extract(hbn, "yearly", ",,,")
except ValueError:
raise ValueError(
tsutils.error_wrapper(f"""
The binary file "{hbn}" does not have consistent ending months between PERLND and
IMPLND. This could be caused by the BYREND (Binary YeaR END) being set
differently in the PERLND:BINARY-INFO and IMPLND:BINARY-INFO, or you could
have the PRINT-INFO bug. To work around the PRINT-INFO bug, add a PERLND
PRINT-INFO block, setting the PYREND here will actually work in the
BINARY-INFO block.
"""))
if year is not None:
pdf = pd.DataFrame(pdf.loc[f"{year}-01-01", :]).T
pdf = pdf[[i for i in pdf.columns if "PERLND" in i or "IMPLND" in i]]
mindex = [i.split("_") for i in pdf.columns]
mindex = [(i[0], int(i[1]), i[2], int(i[1]) % modulus) for i in mindex]
mindex = pd.MultiIndex.from_tuples(mindex,
names=["op", "number", "wbt", "lc"])
pdf.columns = mindex
pdf = pdf.sort_index(axis="columns")
mindex = pdf.columns
aindex = [(i[0], i[1]) for i in pdf.columns]
mindex = [(
i[0],
int(i[1]),
i[2],
int(i[1]) % modulus,
float(lnds.setdefault(j, 0.0)),
str(inverse_lcnames.setdefault(int(i[1]), "")),
) for i, j in zip(mindex, aindex)]
mindex = pd.MultiIndex.from_tuples(
mindex, names=["op", "number", "wbt", "lc", "area", "lcname"])
pdf.columns = mindex
nsum = {}
areas = {}
namelist = {}
setl = [i[1] for i in pwbe]
setl = [item for sublist in setl for item in sublist]
for lue in ["PERLND", "IMPLND"]:
for wbterm in [i[0] for i in setl if i[0]]:
for lc in list(range(1, modulus + 1)):
try:
subset = pdf.loc[:, (lue, slice(None), wbterm, lc,
slice(None), slice(None))]
except KeyError:
continue
_give_negative_warning(subset)
if uci is None:
if subset.empty is True:
nsum[(lue, lc, wbterm)] = 0.0
if (lue, lc) not in namelist:
namelist[(lue, lc)] = ""
else:
nsum[(lue, lc,
wbterm)] = subset.mean(axis="columns").mean()
namelist[(lue, lc)] = inverse_lc.setdefault(lc, lc)
else:
sareas = subset.columns.get_level_values("area")
ssareas = sum(sareas)
if (lue, lc) not in areas:
areas[(lue, lc)] = ssareas
if subset.empty is True or ssareas == 0:
nsum[(lue, lc, wbterm)] = 0.0
if (lue, lc) not in namelist:
namelist[(lue, lc)] = ""
else:
fa = sareas / areas[(lue, lc)]
nsum[(lue, lc,
wbterm)] = ((subset *
fa).sum(axis="columns").mean())
namelist[(lue, lc)] = inverse_lc.setdefault(lc, lc)
newnamelist = []
for key, value in sorted(namelist.items()):
if key[0] != "PERLND":
continue
if key[1] == value:
newnamelist.append(f"{key[1]}")
else:
newnamelist.append(f"{key[1]}-{value}")
printlist = []
printlist.append([" "] + newnamelist + ["ALL"])
mapipratio = {}
mapipratio["PERLND"] = 1.0
mapipratio["IMPLND"] = 1.0
if uci is not None:
pareas = []
pnl = []
iareas = []
for nloper, nllc in namelist:
if nloper == "PERLND":
pnl.append((nloper, nllc))
pareas.append(areas[("PERLND", nllc)])
# If there is a PERLND there must be a IMPLND.
for ploper, pllc in pnl:
try:
iareas.append(areas[("IMPLND", pllc)])
except KeyError:
iareas.append(0.0)
ipratio = np.array(iareas) / (np.array(pareas) + np.array(iareas))
ipratio = np.nan_to_num(ipratio)
ipratio = np.pad(ipratio, (0, len(pareas) - len(iareas)), "constant")
sumareas = sum(pareas) + sum(iareas)
percent_areas = {}
percent_areas["PERLND"] = np.array(pareas) / sumareas * 100
percent_areas["IMPLND"] = np.array(iareas) / sumareas * 100
percent_areas[
"COMBINED"] = percent_areas["PERLND"] + percent_areas["IMPLND"]
printlist.append(["PERVIOUS"])
printlist.append(["AREA(acres)"] +
[str(i) if i > 0 else ""
for i in pareas] + [str(sum(pareas))])
printlist.append(
["AREA(%)"] +
[str(i) if i > 0 else "" for i in percent_areas["PERLND"]] +
[str(sum(percent_areas["PERLND"]))])
printlist.append([])
printlist.append(["IMPERVIOUS"])
printlist.append(["AREA(acres)"] +
[str(i) if i > 0 else ""
for i in iareas] + [str(sum(iareas))])
printlist.append(
["AREA(%)"] +
[str(i) if i > 0 else "" for i in percent_areas["IMPLND"]] +
[str(sum(percent_areas["IMPLND"]))])
printlist.append([])
mapipratio["PERLND"] = 1.0 - ipratio
mapipratio["IMPLND"] = ipratio
mapr = {}
mapr["PERLND"] = 1.0
mapr["IMPLND"] = 1.0
for term, op in pwbe:
if not term:
printlist.append([])
continue
test = [i[1] for i in op]
if "IMPLND" in test and "PERLND" in test:
maprat = mapipratio
sumop = "COMBINED"
else:
maprat = mapr
sumop = test[0]
te = [0.0]
for sterm, operation in op:
try:
tmp = np.array([
nsum[(*i, sterm)] for i in sorted(namelist)
if i[0] == operation
])
if uci is not None:
tmp = (np.pad(tmp,
(0, len(pareas) - len(tmp)), "constant") *
maprat[operation])
te = te + tmp
except KeyError:
pass
if uci is None:
te = ([term] + [str(i) if i > 0 else ""
for i in te] + [str(sum(te) / len(te))])
else:
nte = np.pad(te, (0, len(iareas) - len(te)), "constant")
te = ([term] + [str(i) if i > 0 else "" for i in nte] +
[str(sum(nte * percent_areas[sumop]) / 100)])
printlist.append(te)
if tablefmt in ["csv", "tsv", "csv_nos", "tsv_nos"]:
sep = {
"csv": ",",
"tsv": "\\t",
"csv_nos": ",",
"tsv_nos": "\\t"
}[tablefmt]
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if tablefmt in ["csv_nos", "tsv_nos"]:
print(
re.sub(" *, *", ",",
tabulate(printlist, tablefmt=fmt, floatfmt=float_format)))
else:
print(tabulate(printlist, tablefmt=fmt, floatfmt=float_format))
def parameters(
uci,
index_prefix="",
index_delimiter="",
modulus=20,
tablefmt="csv_nos",
float_format="g",
):
"""Develops a table of parameter values.
Parameters
----------
${uci}
${index_prefix}
${index_delimiter}
${modulus}
${tablefmt}
${float_format}
"""
blocklist = ["PWAT-PARM2", "PWAT-PARM3", "PWAT-PARM4"] # , 'PWAT-STATE1']
params = {}
params["PWAT-PARM2"] = [
"FOREST",
"LZSN",
"INFILT",
"LSUR",
"SLSUR",
"KVARY",
"AGWRC",
]
params["PWAT-PARM3"] = [
"PETMAX",
"PETMIN",
"INFEXP",
"INFILD",
"DEEPFR",
"BASETP",
"AGWETP",
]
params["PWAT-PARM4"] = ["CEPSC", "UZSN", "NSUR", "INTFW", "IRC", "LZETP"]
# params['PWAT-STATE1'] = ['CEPS', 'SURS', 'UZS', 'IFWS', 'LZS', 'AGWS', 'GWVS']
defaults = {}
defaults["FOREST"] = 0.0
defaults["KVARY"] = 0.0
defaults["PETMAX"] = 40.0
defaults["PETMIN"] = 35.0
defaults["INFEXP"] = 2.0
defaults["INFILD"] = 2.0
defaults["DEEPFR"] = 0.0
defaults["BASETP"] = 0.0
defaults["AGWETP"] = 0.0
defaults["CEPSC"] = 0.0
defaults["NSUR"] = 0.1
defaults["LZETP"] = 0.0
# defaults['CEPS'] = 0.0
# defaults['SURS'] = 0.0
# defaults['UZS'] = 0.001
# defaults['IFWS'] = 0.0
# defaults['LZS'] = 0.001
# defaults['AGWS'] = 0.0
# defaults['GWVS'] = 0.0
with open(uci) as fp:
content = fp.readlines()
content = [i[:81].rstrip() for i in content if "***" not in i]
content = [i.rstrip() for i in content if i]
files_start = content.index("FILES")
files_end = content.index("END FILES")
files = content[files_start + 1:files_end]
supfname = ""
for line in files:
words = line.split()
if words[0] == "PESTSU":
supfname = words[2]
if supfname:
ucipath = os.path.dirname(uci)
with open(os.path.join(ucipath, supfname)) as sfp:
supfname = sfp.readlines()
supfname = [i.strip() for i in supfname if "***" not in i]
supfname = [i.strip() for i in supfname if i]
supfname = {
key.split()[0]: [float(i) for i in value.split()]
for key, value in zip(supfname[:-1:2], supfname[1::2])
}
rngdata = []
order = []
for blk in blocklist:
start = content.index(f" {blk}")
end = content.index(f" END {blk}")
block_lines = content[start + 1:end]
order.extend(params[blk])
for line in block_lines:
rngstrt = int(line[:5])
try:
rngend = int(line[5:10]) + 1
except ValueError:
rngend = rngstrt + 1
tilde = re.match("~([0-9][0-9]*)~", line[10:])
if tilde:
tilde = tilde[0][1:-1]
for rng in list(range(rngstrt, rngend)):
for index, par in enumerate(params[blk]):
if tilde:
rngdata.append([
index_prefix + index_delimiter + str(rng),
par,
supfname[tilde][index],
])
else:
start = (index + 1) * 10
rngdata.append([
index_prefix + index_delimiter + str(rng),
par,
float(line[start:start + 10]),
])
df = pd.DataFrame(rngdata)
df.columns = ["lue", "term", "val"]
df = df.pivot(index="lue", columns="term")
df.columns = [i[1] for i in df.columns]
df = df.loc[:, order]
if index_prefix:
spliton = index_prefix[-1]
if index_delimiter:
spliton = index_delimiter
if index_prefix or index_delimiter:
df = df.reindex(index=df.index.to_series().str.rsplit(
spliton).str[-1].astype(int).sort_values().index)
else:
df.index = df.index.astype(int)
df = df.sort_index()
if tablefmt in ["csv", "tsv", "csv_nos", "tsv_nos"]:
sep = {
"csv": ",",
"tsv": "\t",
"csv_nos": ",",
"tsv_nos": "\t"
}[tablefmt]
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if tablefmt == "csv_nos":
print(
re.sub(
" *, *",
",",
tabulate(df,
headers="keys",
tablefmt=fmt,
floatfmt=float_format),
))
elif tablefmt == "tsv_nos":
print(
re.sub(
" *\t *",
"\t",
tabulate(df,
headers="keys",
tablefmt=fmt,
floatfmt=float_format),
))
else:
print(tabulate(df, headers="keys", tablefmt=fmt,
floatfmt=float_format))
def mapping(hbn, year=None, ofilename='', tablefmt='csv_nos', index_prefix=''):
"""Develops a csv file appropriate for joining to a GIS layer.
Parameters
----------
hbn : str
This is the binary output file containing PERLND and IMPLND
information. This should be the binary output file created by the
`uci` file.
year
[optional, defaults to None]
If None the water balance would cover the period of simulation.
Otherwise the year for the water balance.
ofilename
[optional, defaults to '']
If empty string '', then prints to stdout, else prints to `ofilename`.
tablefmt : str
[optional, default is 'cvs_nos']
The table format. Can be one of 'csv', 'tsv', 'csv_nos', 'tsv_nos',
'plain', 'simple', 'github', 'grid', 'fancy_grid', 'pipe', 'orgtbl',
'jira', 'presto', 'psql', 'rst', 'mediawiki', 'moinmoin', 'youtrack',
'html', 'latex', 'latex_raw', 'latex_booktabs' and 'textile'.
index_prefix : str
[optional, default is '']
This is the string to prefix the land use element number.
"""
from hspfbintoolbox.hspfbintoolbox import extract
if ofilename:
sys.stdout = open(ofilename, 'w')
try:
pdf = extract(hbn, 'yearly', ',,,')
except ValueError:
raise ValueError("""
*
* The binary file does not have consistent ending months between PERLND and
* IMPLND. This could be caused by the BYREND (Binary YeaR END) being set
* differently in the PERLND:BINARY-INFO and IMPLND:BINARY-INFO, or you could
* have the PRINT-INFO bug. To work around the PRINT-INFO bug, add a PERLND
* PRINT-INFO block, setting the PYREND here will actually work in the
* BINARY-INFO block.
*
""")
if year is not None:
pdf = pd.DataFrame(pdf.ix['{0}-01-01'.format(year), :]).T
pdf = pdf[[i for i in pdf.columns if 'PERLND' in i or 'IMPLND' in i]]
mindex = [i.split('_') for i in pdf.columns]
mindex = [(i[0][0], int(i[1]), i[2]) for i in mindex]
mindex = pd.MultiIndex.from_tuples(mindex, names=['op', 'number', 'wbt'])
pdf.columns = mindex
if pd.np.any(pdf < 0):
warnings.warn(f'There is a negative value.')
pdf = pdf.mean(axis='index').to_frame()
mindex = [('_'.join([i[0], i[2]]), i[1]) for i in pdf.index]
mindex = pd.MultiIndex.from_tuples(mindex, names=['wbt', 'number'])
pdf.index = mindex
pdf = pdf.unstack('wbt')
mindex = [i[1] for i in pdf.columns]
pdf.columns = mindex
pdf.index.name = 'lue'
if index_prefix:
pdf.index = [index_prefix + str(i) for i in pdf.index]
if tablefmt in ['csv', 'tsv', 'csv_nos', 'tsv_nos']:
sep = {
'csv': ',',
'tsv': '\\t',
'csv_nos': ',',
'tsv_nos': '\\t'
}[tablefmt]
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if tablefmt in ['csv_nos', 'tsv_nos']:
print(
re.sub(
' *, *', ',',
tabulate(pdf, tablefmt=fmt, headers='keys').replace('nan',
'')))
else:
print(tabulate(pdf, tablefmt=fmt, headers='keys').replace('nan', ''))
def process(uci, hbn, pwbe, year, ofilename, modulus, tablefmt):
from hspfbintoolbox.hspfbintoolbox import extract
if ofilename:
sys.stdout = open(ofilename, 'w')
try:
year = int(year)
except TypeError:
pass
lcnames = dict(zip(range(modulus + 1, 1), zip(range(modulus + 1, 1))))
inverse_lcnames = dict(
zip(range(modulus + 1, 1), zip(range(modulus + 1, 1))))
inverse_lc = {}
lnds = {}
if uci is not None:
with open(uci) as fp:
content = fp.readlines()
if not os.path.exists(hbn):
raise ValueError("""
*
* File {0} does not exist.
*
""".format(hbn))
content = [i[:80] for i in content]
content = [i.rstrip() for i in content]
schematic_start = content.index('SCHEMATIC')
schematic_end = content.index('END SCHEMATIC')
schematic = content[schematic_start:schematic_end + 1]
perlnd_start = content.index('PERLND')
perlnd_end = content.index('END PERLND')
perlnd = content[perlnd_start:perlnd_end + 1]
geninfo_start = perlnd.index(' GEN-INFO')
geninfo_end = perlnd.index(' END GEN-INFO')
geninfo = perlnd[geninfo_start:geninfo_end + 1]
masslink_start = content.index('MASS-LINK')
masslink_end = content.index('END MASS-LINK')
masslink = content[masslink_start:masslink_end + 1]
lcnames = {}
inverse_lcnames = {}
inverse_lc = {}
for line in geninfo[1:-1]:
if '***' in line:
continue
if '' == line.strip():
continue
try:
_ = int(line[5:10])
continue
except ValueError:
pass
lcnames.setdefault(line[10:30].strip(), []).append(int(line[:5]))
inverse_lcnames[int(line[:5])] = line[10:30].strip()
inverse_lc[int(line[:5]) % modulus] = line[10:30].strip()
masslink = [i for i in masslink if '***' not in i]
masslink = [i for i in masslink if len(i.strip()) > 0]
masslink = ' '.join(masslink)
mlgroups = re.findall(
r' MASS-LINK +?([0-9]+).*?LND [PI]WATER.*? END MASS-LINK +?\1 ',
masslink)
for line in schematic[3:-1]:
if '***' in line:
continue
if '' == line:
continue
words = line.split()
if words[0] in ['PERLND', 'IMPLND'] and words[5] in mlgroups:
lnds[(words[0], int(words[1]))] = lnds.setdefault(
(words[0], int(words[1])), 0.0) + float(words[2])
try:
pdf = extract(hbn, 'yearly', ',,,')
except ValueError:
raise ValueError("""
*
* The binary file does not have consistent ending months between PERLND and
* IMPLND. This could be caused by the BYREND (Binary YeaR END) being set
* differently in the PERLND:BINARY-INFO and IMPLND:BINARY-INFO, or you could
* have the PRINT-INFO bug. To work around the PRINT-INFO bug, add a PERLND
* PRINT-INFO block, setting the PYREND here will actually work in the
* BINARY-INFO block.
*
""")
if year is not None:
pdf = pd.DataFrame(pdf.ix['{0}-01-01'.format(year), :]).T
pdf = pdf[[i for i in pdf.columns if 'PERLND' in i or 'IMPLND' in i]]
mindex = [i.split('_') for i in pdf.columns]
mindex = [(i[0], int(i[1]), i[2], int(i[1]) % modulus) for i in mindex]
mindex = pd.MultiIndex.from_tuples(mindex,
names=['op', 'number', 'wbt', 'lc'])
pdf.columns = mindex
pdf = pdf.sort_index(axis='columns')
mindex = pdf.columns
aindex = [(i[0], i[1]) for i in pdf.columns]
mindex = [(i[0], int(i[1]), i[2], int(i[1]) % modulus,
float(lnds.setdefault(j, 0.0)),
str(inverse_lcnames.setdefault(int(i[1]), '')))
for i, j in zip(mindex, aindex)]
mindex = pd.MultiIndex.from_tuples(
mindex, names=['op', 'number', 'wbt', 'lc', 'area', 'lcname'])
pdf.columns = mindex
nsum = {}
areas = {}
namelist = {}
setl = [i[1] for i in pwbe]
setl = [item for sublist in setl for item in sublist]
for lue in ['PERLND', 'IMPLND']:
for wbterm in [i[0] for i in setl if i[0]]:
for lc in list(range(1, 21)):
try:
subset = pdf.loc[:, (lue, slice(None), wbterm, lc,
slice(None), slice(None))]
except KeyError:
continue
if pd.np.any(subset < 0):
warnings.warn(
f'There is a negative value for {lue}, {wbterm}, '
f'with land cover {lc}.')
if uci is None:
if subset.empty is True:
nsum[(lue, lc, wbterm)] = 0.0
if (lue, lc) not in namelist:
namelist[(lue, lc)] = ''
else:
nsum[(lue, lc,
wbterm)] = subset.mean(axis='columns').mean()
namelist[(lue, lc)] = inverse_lc.setdefault(lc, lc)
else:
sareas = subset.columns.get_level_values('area')
ssareas = sum(sareas)
if (lue, lc) not in areas:
areas[(lue, lc)] = ssareas
if subset.empty is True or ssareas == 0:
nsum[(lue, lc, wbterm)] = 0.0
if (lue, lc) not in namelist:
namelist[(lue, lc)] = ''
else:
fa = sareas / areas[(lue, lc)]
nsum[(lue, lc,
wbterm)] = (subset *
fa).sum(axis='columns').mean()
namelist[(lue, lc)] = inverse_lc.setdefault(lc, lc)
newnamelist = []
for key, value in sorted(namelist.items()):
if key[0] != 'PERLND':
continue
if key[1] == value:
newnamelist.append('{0}'.format(key[1]))
else:
newnamelist.append('{0}-{1}'.format(key[1], value))
printlist = []
printlist.append([' '] + newnamelist + ['ALL'])
mapipratio = {}
mapipratio['PERLND'] = 1.0
mapipratio['IMPLND'] = 1.0
if uci is not None:
pareas = [areas[i] for i in sorted(areas) if i[0] == 'PERLND']
iareas = [areas[i] for i in sorted(areas) if i[0] == 'IMPLND']
ipratio = pd.np.array(iareas) / (pd.np.array(pareas) +
pd.np.array(iareas))
sumareas = sum(pareas) + sum(iareas)
percent_areas = {}
percent_areas['PERLND'] = pd.np.array(pareas) / sumareas * 100
percent_areas['IMPLND'] = pd.np.array(iareas) / sumareas * 100
percent_areas[
'COMBINED'] = percent_areas['PERLND'] + percent_areas['IMPLND']
printlist.append(['PERVIOUS'])
printlist.append(['AREA(acres)'] +
[str(i) if i > 0 else ''
for i in pareas] + [str(sum(pareas))])
printlist.append(
['AREA(%)'] +
[str(i) if i > 0 else '' for i in percent_areas['PERLND']] +
[str(sum(percent_areas['PERLND']))])
printlist.append([])
printlist.append(['IMPERVIOUS'])
printlist.append(['AREA(acres)'] +
[str(i) if i > 0 else ''
for i in iareas] + [str(sum(iareas))])
printlist.append(
['AREA(%)'] +
[str(i) if i > 0 else '' for i in percent_areas['IMPLND']] +
[str(sum(percent_areas['IMPLND']))])
printlist.append([])
mapipratio['PERLND'] = 1.0 - ipratio
mapipratio['IMPLND'] = ipratio
mapr = {}
mapr['PERLND'] = 1.0
mapr['IMPLND'] = 1.0
for term, op in pwbe:
if not term:
printlist.append([])
continue
test = [i[1] for i in op]
if 'IMPLND' in test and 'PERLND' in test:
maprat = mapipratio
sumop = 'COMBINED'
else:
maprat = mapr
sumop = test[0]
te = 0.0
for sterm, operation in op:
te = te + pd.np.array([
nsum[(*i, sterm)]
for i in sorted(namelist) if i[0] == operation
]) * maprat[operation]
if uci is None:
te = [term] + [str(i) if i > 0 else ''
for i in te] + [str(sum(te) / len(te))]
else:
te = [term] + [str(i) if i > 0 else '' for i in te
] + [str(sum(te * percent_areas[sumop]) / 100)]
printlist.append(te)
if tablefmt in ['csv', 'tsv', 'csv_nos', 'tsv_nos']:
sep = {
'csv': ',',
'tsv': '\\t',
'csv_nos': ',',
'tsv_nos': '\\t'
}[tablefmt]
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if tablefmt in ['csv_nos', 'tsv_nos']:
print(re.sub(' *, *', ',', tabulate(printlist, tablefmt=fmt)))
else:
print(tabulate(printlist, tablefmt=fmt))
def mapping(hbn,
year=None,
ofilename="",
tablefmt="csv_nos",
index_prefix="",
float_format="g"):
"""Develops a csv file appropriate for joining to a GIS layer.
Parameters
----------
${hbn}
${year}
${ofilename}
${tablefmt}
${index_prefix}
[optional, defaults to '']
A string prepended to the PERLND code, which would allow being
run on different models and collected into one dataset by
creating a unique ID.
${float_format}
"""
from hspfbintoolbox.hspfbintoolbox import extract
if ofilename:
sys.stdout = open(ofilename, "w")
try:
pdf = extract(hbn, "yearly", ",,,")
except ValueError:
raise ValueError("""
*
* The binary file does not have consistent ending months between PERLND and
* IMPLND. This could be caused by the BYREND (Binary YeaR END) being set
* differently in the PERLND:BINARY-INFO and IMPLND:BINARY-INFO, or you could
* have the PRINT-INFO bug. To work around the PRINT-INFO bug, add a PERLND
* PRINT-INFO block, setting the PYREND here will actually work in the
* BINARY-INFO block.
*
""")
if year is not None:
pdf = pd.DataFrame(pdf.loc[f"{year}-01-01", :]).T
pdf = pdf[[i for i in pdf.columns if "PERLND" in i or "IMPLND" in i]]
mindex = [i.split("_") for i in pdf.columns]
mindex = [(i[0][0], int(i[1]), i[2]) for i in mindex]
mindex = pd.MultiIndex.from_tuples(mindex, names=["op", "number", "wbt"])
pdf.columns = mindex
_give_negative_warning(pdf)
pdf = pdf.mean(axis="index").to_frame()
mindex = [("_".join([i[0], i[2]]), i[1]) for i in pdf.index]
mindex = pd.MultiIndex.from_tuples(mindex, names=["wbt", "number"])
pdf.index = mindex
pdf = pdf.unstack("wbt")
mindex = [i[1] for i in pdf.columns]
pdf.columns = mindex
pdf.index.name = "lue"
if index_prefix:
pdf.index = [index_prefix + str(i) for i in pdf.index]
if tablefmt in ["csv", "tsv", "csv_nos", "tsv_nos"]:
sep = {
"csv": ",",
"tsv": "\\t",
"csv_nos": ",",
"tsv_nos": "\\t"
}[tablefmt]
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if tablefmt in ["csv_nos", "tsv_nos"]:
print(
re.sub(
" *, *",
",",
tabulate(pdf,
tablefmt=fmt,
headers="keys",
floatfmt=float_format).replace("nan", ""),
))
else:
print(
tabulate(pdf, tablefmt=fmt, headers="keys",
floatfmt=float_format).replace("nan", ""))
def _printiso(
tsd,
date_format=None,
sep=',',
float_format='%g',
showindex='never',
headers='keys',
tablefmt='csv',
):
"""Separate so can use in tests."""
sys.tracebacklimit = 1000
if isinstance(tsd, (pd.DataFrame, pd.Series)):
if isinstance(tsd, pd.Series):
tsd = pd.DataFrame(tsd)
if tsd.columns.empty:
tsd = pd.DataFrame(index=tsd.index)
# Not perfectly true, but likely will use showindex for indices
# that are not time stamps.
if showindex is True:
if not tsd.index.name:
tsd.index.name = 'UniqueID'
else:
if not tsd.index.name:
tsd.index.name = 'Datetime'
print_index = True
if tsd.index.is_all_dates is True:
if not tsd.index.name:
tsd.index.name = 'Datetime'
# Someone made the decision about the name
# This is how I include time zone info by tacking on to the
# index.name.
elif 'datetime' not in tsd.index.name.lower():
tsd.index.name = 'Datetime'
else:
# This might be overkill, but tstoolbox is for time-series.
# Revisit if necessary.
print_index = False
if tsd.index.name == 'UniqueID':
print_index = False
if showindex in ['always', 'default']:
print_index = True
elif isinstance(tsd, (int, float, tuple, pd.np.ndarray)):
tablefmt = None
if tablefmt in ['csv', 'tsv', 'csv_nos', 'tsv_nos']:
sep = {'csv': ',',
'tsv': '\t',
'csv_nos': ',',
'tsv_nos': '\t'}[tablefmt]
if isinstance(tsd, pd.DataFrame):
try:
tsd.to_csv(sys.stdout,
float_format=float_format,
date_format=date_format,
sep=sep,
index=print_index)
return
except IOError:
return
else:
fmt = simple_separated_format(sep)
else:
fmt = tablefmt
if fmt is None:
print(str(list(tsd))[1:-1])
elif tablefmt in ['csv_nos', 'tsv_nos']:
print(tb(tsd,
tablefmt=fmt,
showindex=showindex,
headers=headers).replace(' ', ''))
else:
print(tb(tsd,
tablefmt=fmt,
showindex=showindex,
headers=headers))