def conduits(self):
"""
collect all useful and available data related model conduits and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._conduits_df is not None:
return self._conduits_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
conduits_df = dataframe_from_inp(inp.path, "CONDUITS")
xsections_df = dataframe_from_inp(inp.path, "XSECTIONS")
conduits_df = conduits_df.join(xsections_df)
if rpt:
# create a dictionary holding data from an rpt file, if provided
link_flow_df = dataframe_from_rpt(rpt.path, "Link Flow Summary")
conduits_df = conduits_df.join(link_flow_df)
# add conduit coordinates
xys = conduits_df.apply(lambda r: get_link_coords(
r, self.inp.coordinates, self.inp.vertices),
axis=1)
df = conduits_df.assign(coords=xys.map(lambda x: x[0]))
# add conduit up/down inverts and calculate slope
elevs = self.nodes()[['InvertElev']]
df = pd.merge(df,
elevs,
left_on='InletNode',
right_index=True,
how='left')
df = df.rename(index=str, columns={"InvertElev": "InletNodeInvert"})
df = pd.merge(df,
elevs,
left_on='OutletNode',
right_index=True,
how='left')
df = df.rename(index=str, columns={"InvertElev": "OutletNodeInvert"})
df['UpstreamInvert'] = df.InletNodeInvert + df.InOffset
df['DownstreamInvert'] = df.OutletNodeInvert + df.OutOffset
df['SlopeFtPerFt'] = (df.UpstreamInvert -
df.DownstreamInvert) / df.Length
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._conduits_df = df
return df
def junctions(self):
"""
Get/set junctions section of the INP file.
:return: junctions section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.junctions
InvertElev MaxDepth InitDepth SurchargeDepth PondedArea
Name
J3 6.547 15 0 0 0
1 17.000 0 0 0 0
2 17.000 0 0 0 0
3 16.500 0 0 0 0
4 16.000 0 0 0 0
5 15.000 0 0 0 0
J2 13.000 15 0 0 0
"""
if self._junctions_df is None:
self._junctions_df = dataframe_from_inp(self.path, "JUNCTIONS")
return self._junctions_df
def weirs(self):
"""
Get/set weirs section of the INP file.
"""
if self._weirs_df is None:
self._weirs_df = dataframe_from_inp(self.path, "[WEIRS]")
return self._weirs_df
def orifices(self):
"""
Get/set orifices section of the INP file.
"""
if self._orifices_df is None:
self._orifices_df = dataframe_from_inp(self.path, "[ORIFICES]")
return self._orifices_df
def pumps(self):
"""
Get/set pumps section of the INP file.
"""
if self._pumps_df is None:
self._pumps_df = dataframe_from_inp(self.path, "[PUMPS]")
return self._pumps_df
def xsections(self):
"""
Get/set pumps section of the INP file.
"""
if self._xsections_df is None:
self._xsections_df = dataframe_from_inp(self.path, "[XSECTIONS]")
return self._xsections_df
def conduits(self):
"""
Get/set conduits section of the INP file.
:return: Conduits section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.conduits[['InletNode', 'OutletNode', 'Length', 'ManningN']]
InletNode OutletNode Length ManningN
Name
C1:C2 J1 J2 244.63 0.01
C2.1 J2 J3 666.00 0.01
1 1 4 400.00 0.01
2 4 5 400.00 0.01
3 5 J1 400.00 0.01
4 3 4 400.00 0.01
5 2 5 400.00 0.01
"""
if self._conduits_df is None:
self._conduits_df = dataframe_from_inp(self.path, "[CONDUITS]")
return self._conduits_df
def test_modify_model():
from swmmio.utils.modify_model import replace_inp_section
from swmmio import Model
import pandas as pd
# initialize a baseline model object
baseline = Model(MODEL_FULL_FEATURES_XY)
of_test = pd.read_csv(OUTFALLS_MODIFIED, index_col=0)
rise = 10.0 # set the starting sea level rise condition
# create a dataframe of the model's outfalls
outfalls = dataframe_from_inp(baseline.inp.path, '[OUTFALLS]')
# add the current rise to the outfalls' stage elevation
outfalls['OutfallType'] = 'FIXED'
outfalls.loc[:, 'InvertElev'] = pd.to_numeric(
outfalls.loc[:, 'InvertElev']) + rise
of_test.loc[:, 'InvertElev'] = pd.to_numeric(of_test.loc[:, 'InvertElev'])
with tempfile.TemporaryDirectory() as tempdir:
# copy the base model into a new directory
newdir = os.path.join(tempdir, str(rise))
makedirs(newdir)
newfilepath = os.path.join(
newdir, baseline.inp.name + "_" + str(rise) + '_SLR.inp')
shutil.copyfile(baseline.inp.path, newfilepath)
# Overwrite the OUTFALLS section of the new model with the adjusted data
replace_inp_section(newfilepath, '[OUTFALLS]', outfalls)
m2 = Model(newfilepath)
of2 = m2.inp.outfalls
assert (of2.loc['J4', 'InvertElev'].round(1) == of_test.loc[
'J4', 'InvertElev'].round(1))
def search_for_duplicates(inp_path, verbose=False):
"""
scan an inp file and determine if any element IDs are duplicated in
any section. Method: count the uniques and compare to total length
"""
headers = swmmio.utils.text.get_inp_sections_details(inp_path)['headers']
dups_found = False
for header, cols, in headers.items():
if cols != 'blob':
df = dataframe_from_inp(inp_path, section=header)
elements = df.index
n_unique = len(elements.unique()) #number of unique elements
n_total = len(elements) #total number of elements
if verbose:
print('{} -> (uniques, total) -> ({}, {})'.format(
header, n_unique, n_total))
if n_unique != n_total:
dups = ', '.join(
df[df.index.duplicated()].index.unique().tolist())
print('duplicate found in {}\nsection: {}\n{}'.format(
inp_path, header, dups))
dups_found = True
return dups_found
def subareas(self):
"""
Get/set subareas section of the INP file.
"""
if self._subareas_df is None:
self._subareas_df = dataframe_from_inp(self.path, "[SUBAREAS]")
return self._subareas_df
def test_inflow_dwf_dataframe():
m = swmmio.Model(MODEL_XSECTION_ALT_03)
dwf = dataframe_from_inp(m.inp.path, 'dwf')
assert (dwf.loc['dummy_node2',
'AverageValue'] == pytest.approx(0.000275, 0.0001))
inf = m.inp.inflows
assert (inf.loc['dummy_node2', 'Time Series'] == 'my_time_series')
assert (pd.isna(inf.loc['dummy_node6', 'Time Series']))
def polygons(self):
"""
get/set polygons section of model
:return: dataframe of model coordinates
"""
if self._polygons_df is not None:
return self._polygons_df
self._polygons_df = dataframe_from_inp(self.path, '[Polygons]')
return self._polygons_df
def vertices(self):
"""
get/set vertices section of model
:return: dataframe of model coordinates
"""
if self._vertices_df is not None:
return self._vertices_df
self._vertices_df = dataframe_from_inp(self.path, 'VERTICES')
return self._vertices_df
def coordinates(self):
"""
Get/set coordinates section of model
:return: dataframe of model coordinates
"""
if self._coordinates_df is not None:
return self._coordinates_df
self._coordinates_df = dataframe_from_inp(self.path, "COORDINATES")
return self._coordinates_df
def inflows(self):
"""
get/set inflows section of model
:return: dataframe of model coordinates
"""
if self._inflows_df is not None:
return self._inflows_df
inf = dataframe_from_inp(self.path, 'INFLOWS', quote_replace='_!!!!_')
self._inflows_df = inf.replace('_!!!!_', pd.np.nan)
return self._inflows_df
def drop_invalid_model_elements(inp):
"""
Identify references to elements in the model that are undefined and remove them from the
model. These should coincide with warnings/errors produced by SWMM5 when undefined elements
are referenced in links, subcatchments, and controls.
:param model: swmmio.Model
:return:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES_INVALID
>>> m = swmmio.Model(MODEL_FULL_FEATURES_INVALID)
>>> drop_invalid_model_elements(m.inp)
['InvalidLink2', 'InvalidLink1']
>>> m.inp
Index(['C1:C2', 'C2.1', '1', '2', '4', '5'], dtype='object', name='Name')
"""
juncs = dataframe_from_inp(inp.path, "[JUNCTIONS]").index.tolist()
outfs = dataframe_from_inp(inp.path, "[OUTFALLS]").index.tolist()
stors = dataframe_from_inp(inp.path, "[STORAGE]").index.tolist()
nids = juncs + outfs + stors
# drop links with bad refs to inlet/outlet nodes
from swmmio.utils.functions import find_invalid_links
inv_conds = find_invalid_links(inp, nids, 'conduits', drop=True)
inv_pumps = find_invalid_links(inp, nids, 'pumps', drop=True)
inv_orifs = find_invalid_links(inp, nids, 'orifices', drop=True)
inv_weirs = find_invalid_links(inp, nids, 'weirs', drop=True)
# drop other parts of bad links
invalid_links = inv_conds + inv_pumps + inv_orifs + inv_weirs
inp.xsections = inp.xsections.loc[~inp.xsections.index.isin(invalid_links)]
# drop invalid subcats and their related components
invalid_subcats = inp.subcatchments.index[~inp.subcatchments['Outlet'].
isin(nids)]
inp.subcatchments = inp.subcatchments.loc[~inp.subcatchments.index.
isin(invalid_subcats)]
inp.subareas = inp.subareas.loc[~inp.subareas.index.isin(invalid_subcats)]
inp.infiltration = inp.infiltration.loc[~inp.infiltration.index.
isin(invalid_subcats)]
return invalid_links + invalid_subcats
def subcatchments(self):
"""
Get/set subcatchments section of the INP file.
:return: subcatchments section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._subcatchments_df is None:
self._subcatchments_df = dataframe_from_inp(self.path, "[SUBCATCHMENTS]")
return self._subcatchments_df
def files(self):
"""
Get/set files section of the INP file.
:return: files section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._files_df is None:
self._files_df = dataframe_from_inp(self.path, "[FILES]")
return self._files_df.reset_index()
def storage(self):
"""
Get/set storage section of the INP file.
:return: storage section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._storage_df is None:
self._storage_df = dataframe_from_inp(self.path, "[STORAGE]")
return self._storage_df
def infiltration(self):
"""
Get/set infiltration section of the INP file.
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> m.inp.infiltration
MaxRate MinRate Decay DryTime MaxInfil
Subcatchment
S1 3.0 0.5 4 7 0
S2 3.0 0.5 4 7 0
S3 3.0 0.5 4 7 0
S4 3.0 0.5 4 7 0
"""
if self._infiltration_df is None:
self._infiltration_df = dataframe_from_inp(self.path, "infiltration")
return self._infiltration_df
def outfalls(self):
"""
Get/set outfalls section of the INP file.
:return: outfalls section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.outfalls
InvertElev OutfallType StageOrTimeseries TideGate
Name
J4 0 FREE NO NaN
"""
if self._outfalls_df is None:
self._outfalls_df = dataframe_from_inp(self.path, "[OUTFALLS]")
return self._outfalls_df
def __call__(self):
"""
collect all useful and available data related to the conduits and
organize in one dataframe.
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> conduits_section = ModelSection(model, 'conduits')
>>> conduits_section()
"""
# concat inp sections with unique element IDs
headers = get_inp_sections_details(self.inp.path)
dfs = [
dataframe_from_inp(self.inp.path, sect)
for sect in self.inp_sections if sect.upper() in headers
]
# return empty df if no inp sections found
if len(dfs) == 0:
return pd.DataFrame()
df = pd.concat(dfs, axis=0, sort=False)
# join to this any sections with matching IDs (e.g. XSECTIONS)
for sect in self.join_sections:
rsuffix = f"_{sect.replace(' ', '_')}"
df = df.join(dataframe_from_inp(self.inp.path, sect),
rsuffix=rsuffix)
if df.empty:
return df
# if there is an RPT available, grab relevant sections
if self.rpt:
for rpt_sect in self.rpt_sections:
df = df.join(dataframe_from_rpt(self.rpt.path, rpt_sect))
# add coordinates
if self.geomtype == 'point':
df = df.join(self.inp.coordinates[['X', 'Y']])
xys = df.apply(lambda r: nodexy(r), axis=1)
df = df.assign(coords=xys)
elif self.geomtype == 'linestring':
# add conduit coordinates
xys = df.apply(lambda r: get_link_coords(r, self.inp.coordinates,
self.inp.vertices),
axis=1)
df = df.assign(coords=xys.map(lambda x: x[0]))
# make inlet/outlet node IDs string type
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
elif self.geomtype == 'polygon':
p = self.inp.polygons
# take stacked coordinates and orient in list of tuples,
xys = p.groupby(by=p.index).apply(
lambda r: [(xy['X'], xy['Y']) for ix, xy in r.iterrows()])
# copy the first point to the last position
xys = xys.apply(lambda r: r + [r[0]])
df = df.assign(coords=xys)
# confirm index name is string
df = df.rename(index=str)
# trim to desired columns
if self.columns is not None:
df = df[[c for c in self.columns if c in df.columns]]
self._df = df
return df
def merge_models(inp1, inp2, target='merged_model.inp'):
"""
Merge two separate swmm models into one model. This creates a diff, ignores
removed sections, and uses inp1 settings where conflicts exist (altered sections in diff)
:param inp1: swmmio.Model.inp object to be combined with inp2
:param inp2: swmmio.Model.inp object to be combined with inp1
:param target: path of new model
:return: path to target
"""
# model object to store resulting merged model
m3 = swmmio.Model(inp1)
inp_diff = INPDiff(inp1, inp2)
with open(target, 'w') as newf:
for section, _ in inp_diff.all_inp_objects.items():
# don't consider the "removed" parts of the diff
# print('{}: {}'.format(section,inp_diff.all_inp_objects[section]['columns']))
# check if the section is not in problem_sections and there are changes
# in self.instructions and commit changes to it from baseline accordingly
col_order = []
if (section not in problem_sections and
inp_diff.all_inp_objects[section]['columns'] != ['blob']
and section in inp_diff.diffs):
# df of baseline model section
basedf = dataframe_from_inp(
m3.inp.path,
section,
additional_cols=[';', 'Comment', 'Origin'])
basedf[';'] = ';'
col_order = basedf.columns
# grab the changes to
changes = inp_diff.diffs[section]
# remove elements that have alterations keep ones tagged for removal
# (unchanged, but not present in m2)
remove_ids = changes.altered.index
new_section = basedf.drop(remove_ids)
# add elements
new_section = pd.concat(
[new_section, changes.altered, changes.added],
axis=0,
sort=False)
else:
# section is not well understood or is problematic, just blindly copy
new_section = dataframe_from_inp(
m3.inp.path,
section,
additional_cols=[';', 'Comment', 'Origin'])
new_section[';'] = ';'
# print ('dealing with confusing section: {}\n{}'.format(section, new_section))
# print(new_section.head())
# write the section
new_section = new_section[col_order]
new_section[';'] = ';'
vc_utils.write_inp_section(newf,
inp_diff.all_inp_objects,
section,
new_section,
pad_top=True)
return target
def __init__(self,
model1=None,
model2=None,
section='JUNCTIONS',
build_instr_file=None):
self.model1 = model1 if model1 else ""
self.model2 = model2 if model2 else ""
if model1 and model2:
df1 = dataframe_from_inp(model1.inp.path, section)
df2 = dataframe_from_inp(model2.inp.path, section)
df1[';'] = ';'
df2[';'] = ';'
col_order = list(df2.columns) + ['Comment', 'Origin']
m2_origin_string = os.path.basename(model2.inp.path).replace(
' ', '-')
# BUG -> this fails if a df1 or df2 is None i.e. if a section doesn't exist in one model
added_ids = df2.index.difference(df1.index)
removed_ids = df1.index.difference(df2.index)
# find where elements were changed (but kept with same ID)
common_ids = df1.index.difference(
removed_ids) # original - removed = in common
# both dfs concatenated, with matched indices for each element
full_set = pd.concat([df1.loc[common_ids], df2.loc[common_ids]],
sort=False)
# remove whitespace
full_set = full_set.apply(lambda x: x.astype(str).str.strip()
if x.dtype == "object" else x)
# drop dupes on the set, all things that did not changed should have 1 row
changes_with_dupes = full_set.drop_duplicates()
# duplicate indicies are rows that have changes, isolate these
# idx[idx.duplicated()].unique()
changed_ids = changes_with_dupes.index[
changes_with_dupes.index.duplicated()].unique(
) # .get_duplicates()
added = df2.loc[added_ids].copy()
added[
'Comment'] = 'Added' # from model {}'.format(model2.inp.path)
added['Origin'] = m2_origin_string
altered = df2.loc[changed_ids].copy()
altered[
'Comment'] = 'Altered' # in model {}'.format(model2.inp.path)
altered['Origin'] = m2_origin_string
removed = df1.loc[removed_ids].copy()
removed[
'Comment'] = 'Removed' # in model {}'.format(model2.inp.path)
removed['Origin'] = m2_origin_string
# removed = removed[col_order]
self.old = df1
self.new = df2
self.added = added
self.removed = removed
self.altered = altered
if build_instr_file:
# if generating from a build instructions file, do this (more efficient)
df = dataframe_from_bi(build_instr_file, section=section)
self.added = df.loc[df['Comment'] == 'Added']
self.removed = df.loc[df['Comment'] == 'Removed']
self.altered = df.loc[df['Comment'] == 'Altered']