def build(self, baseline_dir, target_path):
"""
build a complete INP file with the build instructions committed to a
baseline model.
"""
basemodel = swmmio.Model(baseline_dir)
allheaders = funcs.complete_inp_headers(basemodel.inp.path)
#new_inp = os.path.join(target_dir, 'model.inp')
with open (target_path, 'w') as f:
for section in allheaders['order']:
#check if the section is not in problem_sections and there are changes
#in self.instructions and commit changes to it from baseline accordingly
if (section not in problem_sections
and allheaders['headers'][section] != 'blob'
and section in self.instructions):
#df of baseline model section
basedf = create_dataframeINP(basemodel.inp.path, section)
#grab the changes to
changes = self.instructions[section]
#remove elements that have alterations and or tagged for removal
remove_ids = changes.removed.index | changes.altered.index
new_section = basedf.drop(remove_ids)
#add elements
new_section = pd.concat([new_section, changes.altered, changes.added])
else:
#section is not well understood or is problematic, just blindly copy
new_section = create_dataframeINP(basemodel.inp.path, section=section)
#write the section
vc_utils.write_inp_section(f, allheaders, section, new_section)
def __call__(self):
"""
collect all useful and available data related to the conduits and
organize in one dataframe.
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> conduits_section = ModelSection(model, 'conduits')
>>> conduits_section()
"""
# create dataframes of relevant sections from the INP
for ix, sect in enumerate(self.config['inp_sections']):
if ix == 0:
df = create_dataframeINP(self.inp.path, sect, comment_cols=False)
else:
df_other = create_dataframeINP(self.inp.path, sect, comment_cols=False)
df = df.join(df_other)
if self.rpt:
for rpt_sect in self.config['rpt_sections']:
df = df.join(create_dataframeRPT(self.rpt.path, rpt_sect))
# add conduit coordinates
xys = df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1)
df = df.assign(coords=xys.map(lambda x: x[0]))
# make inlet/outlet node IDs string type
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
return df
def __call__(self):
"""
collect all useful and available data related to the conduits and
organize in one dataframe.
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> conduits_section = ModelSection(model, 'conduits')
>>> conduits_section()
"""
# create dataframes of relevant sections from the INP
for ix, sect in enumerate(self.config['inp_sections']):
if ix == 0:
df = create_dataframeINP(self.inp.path, sect, comment_cols=False)
else:
df_other = create_dataframeINP(self.inp.path, sect, comment_cols=False)
df = df.join(df_other)
if df.empty:
return df
# if there is an RPT available, grab relevant sections
if self.rpt:
for rpt_sect in self.config['rpt_sections']:
df = df.join(create_dataframeRPT(self.rpt.path, rpt_sect))
# add conduit coordinates
xys = df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1)
df = df.assign(coords=xys.map(lambda x: x[0]))
# make inlet/outlet node IDs string type
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
return df
def conduits(self):
"""
collect all useful and available data related model conduits and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._conduits_df is not None:
return self._conduits_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
conduits_df = create_dataframeINP(inp.path,
"[CONDUITS]",
comment_cols=False)
xsections_df = create_dataframeINP(inp.path,
"[XSECTIONS]",
comment_cols=False)
conduits_df = conduits_df.join(xsections_df)
if rpt:
# create a dictionary holding data from an rpt file, if provided
link_flow_df = create_dataframeRPT(rpt.path, "Link Flow Summary")
conduits_df = conduits_df.join(link_flow_df)
# add conduit coordinates
xys = conduits_df.apply(lambda r: get_link_coords(
r, self.inp.coordinates, self.inp.vertices),
axis=1)
df = conduits_df.assign(coords=xys.map(lambda x: x[0]))
# add conduit up/down inverts and calculate slope
elevs = self.nodes()[['InvertElev']]
df = pd.merge(df,
elevs,
left_on='InletNode',
right_index=True,
how='left')
df = df.rename(index=str, columns={"InvertElev": "InletNodeInvert"})
df = pd.merge(df,
elevs,
left_on='OutletNode',
right_index=True,
how='left')
df = df.rename(index=str, columns={"InvertElev": "OutletNodeInvert"})
df['UpstreamInvert'] = df.InletNodeInvert + df.InletOffset
df['DownstreamInvert'] = df.OutletNodeInvert + df.OutletOffset
df['SlopeFtPerFt'] = (df.UpstreamInvert -
df.DownstreamInvert) / df.Length
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._conduits_df = df
return df
def __init__(self,
model1=None,
model2=None,
section='[JUNCTIONS]',
build_instr_file=None):
if model1 and model2:
df1 = create_dataframeINP(model1.inp, section)
df2 = create_dataframeINP(model2.inp, section)
#BUG -> this fails if a df1 or df2 is None i.e. if a section doesn't exist in one model
added_ids = df2.index.difference(df1.index)
removed_ids = df1.index.difference(df2.index)
#find where elements were changed (but kept with same ID)
common_ids = df1.index.difference(
removed_ids) #original - removed = in common
#both dfs concatenated, with matched indices for each element
full_set = pd.concat([df1.ix[common_ids], df2.ix[common_ids]])
#drop dupes on the set, all things that did not changed should have 1 row
changes_with_dupes = full_set.drop_duplicates()
#duplicate indicies are rows that have changes, isolate these
changed_ids = changes_with_dupes.index.get_duplicates()
added = df2.ix[added_ids]
added[
'Comment'] = 'Added' # from model {}'.format(model2.inp.filePath)
added['Origin'] = model2.inp.filePath
altered = df2.ix[changed_ids]
altered[
'Comment'] = 'Altered' # in model {}'.format(model2.inp.filePath)
altered['Origin'] = model2.inp.filePath
removed = df1.ix[removed_ids]
#comment out the removed elements
#removed.index = ["; " + str(x) for x in removed.index]
removed[
'Comment'] = 'Removed' # in model {}'.format(model2.inp.filePath)
removed['Origin'] = model2.inp.filePath
self.old = df1
self.new = df2
self.added = added
self.removed = removed
self.altered = altered
if build_instr_file:
#if generating from a build instructions file, do this (more efficient)
df = create_dataframeBI(build_instr_file, section=section)
self.added = df.loc[df['Comment'] == 'Added']
self.removed = df.loc[df['Comment'] == 'Removed']
self.altered = df.loc[df['Comment'] == 'Altered']
def nodes(self, bbox=None, subset=None):
"""
collect all useful and available data related model nodes and organize
in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._nodes_df is not None and bbox == self.bbox:
return self._nodes_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
juncs_df = create_dataframeINP(inp.path, "[JUNCTIONS]")
outfalls_df = create_dataframeINP(inp.path, "[OUTFALLS]")
storage_df = create_dataframeINP(inp.path, "[STORAGE]")
# concatenate the DFs and keep only relevant cols
all_nodes = pd.concat([juncs_df, outfalls_df, storage_df])
cols = ['InvertElev', 'MaxDepth', 'SurchargeDepth', 'PondedArea']
all_nodes = all_nodes[cols]
if rpt:
# add results data if a rpt file was found
depth_summ = create_dataframeRPT(rpt.path, "Node Depth Summary")
flood_summ = create_dataframeRPT(rpt.path, "Node Flooding Summary")
# join the rpt data (index on depth df, suffixes for common cols)
rpt_df = depth_summ.join(flood_summ,
lsuffix='_depth',
rsuffix='_flood')
all_nodes = all_nodes.join(rpt_df) # join to the all_nodes df
all_nodes = all_nodes.join(self.inp.coordinates[['X', 'Y']])
def nodexy(row):
if math.isnan(row.X) or math.isnan(row.Y):
return None
else:
return [(row.X, row.Y)]
xys = all_nodes.apply(lambda r: nodexy(r), axis=1)
all_nodes = all_nodes.assign(coords=xys)
all_nodes = all_nodes.rename(index=str)
self._nodes_df = all_nodes
return all_nodes
def nodes(self, bbox=None, subset=None):
"""
collect all useful and available data related model nodes and organize
in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._nodes_df is not None and bbox == self.bbox:
return self._nodes_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
juncs_df = create_dataframeINP(inp.path, "[JUNCTIONS]")
outfalls_df = create_dataframeINP(inp.path, "[OUTFALLS]")
storage_df = create_dataframeINP(inp.path, "[STORAGE]")
# concatenate the DFs and keep only relevant cols
all_nodes = pd.concat([juncs_df, outfalls_df, storage_df])
cols = ['InvertElev', 'MaxDepth', 'SurchargeDepth', 'PondedArea']
all_nodes = all_nodes[cols]
if rpt:
# add results data if a rpt file was found
depth_summ = create_dataframeRPT(rpt.path, "Node Depth Summary")
flood_summ = create_dataframeRPT(rpt.path, "Node Flooding Summary")
# join the rpt data (index on depth df, suffixes for common cols)
rpt_df = depth_summ.join(
flood_summ, lsuffix='_depth', rsuffix='_flood')
all_nodes = all_nodes.join(rpt_df) # join to the all_nodes df
all_nodes = all_nodes.join(self.inp.coordinates[['X', 'Y']])
def nodexy(row):
if math.isnan(row.X) or math.isnan(row.Y):
return None
else:
return [(row.X, row.Y)]
xys = all_nodes.apply(lambda r: nodexy(r), axis=1)
all_nodes = all_nodes.assign(coords=xys)
all_nodes = all_nodes.rename(index=str)
self._nodes_df = all_nodes
return all_nodes
def conduits(self):
"""
Get/set conduits section of the INP file.
:return: Conduits section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.conduits[['InletNode', 'OutletNode', 'Length', 'ManningN']]
InletNode OutletNode Length ManningN
Name
C1:C2 J1 J2 244.63 0.01
C2.1 J2 J3 666.00 0.01
1 1 4 400.00 0.01
2 4 5 400.00 0.01
3 5 J1 400.00 0.01
4 3 4 400.00 0.01
5 2 5 400.00 0.01
"""
if self._conduits_df is None:
self._conduits_df = create_dataframeINP(self.path,
"[CONDUITS]",
comment_cols=False)
return self._conduits_df
def weirs(self):
"""
collect all useful and available data related model weirs and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._weirs_df is not None:
return self._weirs_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
weirs_df = create_dataframeINP(inp.path, "[WEIRS]")
if weirs_df.empty:
return pd.DataFrame()
weirs_df = weirs_df[['InletNode', 'OutletNode', 'WeirType', 'CrestHeight']]
# add conduit coordinates
xys = weirs_df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1)
df = weirs_df.assign(coords=xys.map(lambda x: x[0]))
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._weirs_df = df
return df
def junctions(self):
"""
Get/set junctions section of the INP file.
:return: junctions section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.junctions
InvertElev MaxDepth InitDepth SurchargeDepth PondedArea
Name
J3 6.547 15 0 0 0
1 17.000 0 0 0 0
2 17.000 0 0 0 0
3 16.500 0 0 0 0
4 16.000 0 0 0 0
5 15.000 0 0 0 0
J2 13.000 15 0 0 0
"""
if self._junctions_df is None:
self._junctions_df = create_dataframeINP(self.path,
"[JUNCTIONS]",
comment_cols=False)
return self._junctions_df
def search_for_duplicates(inp_path, verbose=False):
"""
scan an inp file and determine if any element IDs are duplicated in
any section. Method: count the uniques and compare to total length
"""
headers = funcs.complete_inp_headers(inp_path)['headers']
dups_found = False
for header, cols, in headers.items():
if cols != 'blob':
df = dataframes.create_dataframeINP(inp_path, section=header)
elements = df.index
n_unique = len(elements.unique()) #number of unique elements
n_total = len(elements) #total number of elements
if verbose:
print('{} -> (uniques, total) -> ({}, {})'.format(
header, n_unique, n_total))
if n_unique != n_total:
dups = ', '.join(
df[df.index.duplicated()].index.unique().tolist())
print('duplicate found in {}\nsection: {}\n{}'.format(
inp_path, header, dups))
dups_found = True
return dups_found
def weirs(self):
"""
collect all useful and available data related model weirs and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._weirs_df is not None:
return self._weirs_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
weirs_df = create_dataframeINP(inp.path, "[WEIRS]")
if weirs_df.empty:
return pd.DataFrame()
weirs_df = weirs_df[[
'InletNode', 'OutletNode', 'WeirType', 'CrestHeight'
]]
# add conduit coordinates
xys = weirs_df.apply(lambda r: get_link_coords(r, self.inp.coordinates,
self.inp.vertices),
axis=1)
df = weirs_df.assign(coords=xys.map(lambda x: x[0]))
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._weirs_df = df
return df
def orifices(self):
"""
collect all useful and available data related model orifices and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._orifices_df is not None:
return self._orifices_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
orifices_df = create_dataframeINP(inp.path,
"[ORIFICES]",
comment_cols=False)
if orifices_df.empty:
return pd.DataFrame()
# add conduit coordinates
xys = orifices_df.apply(lambda r: get_link_coords(
r, self.inp.coordinates, self.inp.vertices),
axis=1)
df = orifices_df.assign(coords=xys.map(lambda x: x[0]))
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._orifices_df = df
return df
def orifices(self):
"""
collect all useful and available data related model orifices and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._orifices_df is not None:
return self._orifices_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
orifices_df = create_dataframeINP(inp.path, "[ORIFICES]", comment_cols=False)
if orifices_df.empty:
return pd.DataFrame()
# add conduit coordinates
xys = orifices_df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1)
df = orifices_df.assign(coords=xys.map(lambda x: x[0]))
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._orifices_df = df
return df
def junctions(self):
"""
Get/set junctions section of the INP file.
:return: junctions section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.junctions
InvertElev MaxDepth InitDepth SurchargeDepth PondedArea
Name
J3 6.547 15 0 0 0
1 17.000 0 0 0 0
2 17.000 0 0 0 0
3 16.500 0 0 0 0
4 16.000 0 0 0 0
5 15.000 0 0 0 0
J2 13.000 15 0 0 0
"""
if self._junctions_df is None:
self._junctions_df = create_dataframeINP(self.path, "[JUNCTIONS]", comment_cols=False)
return self._junctions_df
def conduits(self):
"""
Get/set conduits section of the INP file.
:return: Conduits section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.conduits[['InletNode', 'OutletNode', 'Length', 'ManningN']]
InletNode OutletNode Length ManningN
Name
C1:C2 J1 J2 244.63 0.01
C2.1 J2 J3 666.00 0.01
1 1 4 400.00 0.01
2 4 5 400.00 0.01
3 5 J1 400.00 0.01
4 3 4 400.00 0.01
5 2 5 400.00 0.01
"""
if self._conduits_df is None:
self._conduits_df = create_dataframeINP(self.path, "[CONDUITS]", comment_cols=False)
return self._conduits_df
def __init__(self, model1=None, model2=None, section='[JUNCTIONS]', build_instr_file=None):
if model1 and model2:
df1 = create_dataframeINP(model1.inp.path, section)
df2 = create_dataframeINP(model2.inp.path, section)
#BUG -> this fails if a df1 or df2 is None i.e. if a section doesn't exist in one model
added_ids = df2.index.difference(df1.index)
removed_ids = df1.index.difference(df2.index)
#find where elements were changed (but kept with same ID)
common_ids = df1.index.difference(removed_ids) #original - removed = in common
#both dfs concatenated, with matched indices for each element
full_set = pd.concat([df1.loc[common_ids], df2.loc[common_ids]])
#drop dupes on the set, all things that did not changed should have 1 row
changes_with_dupes = full_set.drop_duplicates()
#duplicate indicies are rows that have changes, isolate these
changed_ids = changes_with_dupes.index.get_duplicates()
added = df2.loc[added_ids].copy()
added['Comment'] = 'Added'# from model {}'.format(model2.inp.path)
added['Origin'] = model2.inp.path
altered = df2.loc[changed_ids].copy()
altered['Comment'] = 'Altered'# in model {}'.format(model2.inp.path)
altered['Origin'] = model2.inp.path
removed = df1.loc[removed_ids].copy()
#comment out the removed elements
#removed.index = ["; " + str(x) for x in removed.index]
removed['Comment'] = 'Removed'# in model {}'.format(model2.inp.path)
removed['Origin'] = model2.inp.path
self.old = df1
self.new = df2
self.added = added
self.removed = removed
self.altered = altered
if build_instr_file:
#if generating from a build instructions file, do this (more efficient)
df = create_dataframeBI(build_instr_file, section = section)
self.added = df.loc[df['Comment'] == 'Added']
self.removed = df.loc[df['Comment'] == 'Removed']
self.altered = df.loc[df['Comment'] == 'Altered']
def orifices(self):
"""
Get/set orifices section of the INP file.
"""
if self._orifices_df is None:
self._orifices_df = create_dataframeINP(self.path,
"[ORIFICES]",
comment_cols=False)
return self._orifices_df
def polygons(self):
"""
get/set polygons section of model
:return: dataframe of model coordinates
"""
if self._polygons_df is not None:
return self._polygons_df
self._polygons_df = create_dataframeINP(self.path, '[Polygons]', comment_cols=False)
return self._polygons_df
def weirs(self):
"""
Get/set weirs section of the INP file.
"""
if self._weirs_df is None:
self._weirs_df = create_dataframeINP(self.path,
"[WEIRS]",
comment_cols=False)
return self._weirs_df
def vertices(self):
"""
get/set vertices section of model
:return: dataframe of model coordinates
"""
if self._vertices_df is not None:
return self._vertices_df
self._vertices_df = create_dataframeINP(self.path, '[VERTICES]', comment_cols=False)
return self._vertices_df
def xsections(self):
"""
Get/set pumps section of the INP file.
"""
if self._xsections_df is None:
self._xsections_df = create_dataframeINP(self.path,
"[XSECTIONS]",
comment_cols=False)
return self._xsections_df
def coordinates(self):
"""
Get/set coordinates section of model
:return: dataframe of model coordinates
"""
if self._coordinates_df is not None:
return self._coordinates_df
self._coordinates_df = create_dataframeINP(self.path, "[COORDINATES]", comment_cols=False)
return self._coordinates_df
def pumps(self):
"""
Get/set pumps section of the INP file.
"""
if self._pumps_df is None:
self._pumps_df = create_dataframeINP(self.path,
"[PUMPS]",
comment_cols=False)
return self._pumps_df
def conduits(self):
"""
collect all useful and available data related model conduits and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if self._conduits_df is not None:
return self._conduits_df
# parse out the main objects of this model
inp = self.inp
rpt = self.rpt
# create dataframes of relevant sections from the INP
conduits_df = create_dataframeINP(inp.path, "[CONDUITS]", comment_cols=False)
xsections_df = create_dataframeINP(inp.path, "[XSECTIONS]", comment_cols=False)
conduits_df = conduits_df.join(xsections_df)
if rpt:
# create a dictionary holding data from an rpt file, if provided
link_flow_df = create_dataframeRPT(rpt.path, "Link Flow Summary")
conduits_df = conduits_df.join(link_flow_df)
# add conduit coordinates
xys = conduits_df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1)
df = conduits_df.assign(coords=xys.map(lambda x: x[0]))
# add conduit up/down inverts and calculate slope
elevs = self.nodes()[['InvertElev']]
df = pd.merge(df, elevs, left_on='InletNode', right_index=True, how='left')
df = df.rename(index=str, columns={"InvertElev": "InletNodeInvert"})
df = pd.merge(df, elevs, left_on='OutletNode', right_index=True, how='left')
df = df.rename(index=str, columns={"InvertElev": "OutletNodeInvert"})
df['UpstreamInvert'] = df.InletNodeInvert + df.InletOffset
df['DownstreamInvert'] = df.OutletNodeInvert + df.OutletOffset
df['SlopeFtPerFt'] = (df.UpstreamInvert - df.DownstreamInvert) / df.Length
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
self._conduits_df = df
return df
def infiltration(self):
"""
Get/set infiltration section of the INP file.
"""
if self._infiltration_df is None:
self._infiltration_df = create_dataframeINP(self.path,
"[INFILTRATION]",
comment_cols=False,
headers=self._headers)
return self._infiltration_df
def subareas(self):
"""
Get/set subareas section of the INP file.
"""
if self._subareas_df is None:
self._subareas_df = create_dataframeINP(self.path,
"[SUBAREAS]",
comment_cols=False,
headers=self._headers)
return self._subareas_df
def redefine_time_series(inp):
# Redefining timeseries if these are not in the right directory.
try:
with HiddenPrints():
#Changing the path of the Time series to run models.
from swmmio.utils import modify_model
from swmmio.utils.modify_model import replace_inp_section
from swmmio.utils.dataframes import create_dataframeINP
# Extracting a section (Timeseries) to a pandas DataFrame
path = inp.model_dir
inp_file = inp.model_name
baseline = path + '/' + inp_file + '.inp'
# Create a dataframe of the model's time series
Timeseries = create_dataframeINP(baseline, '[TIMESERIES]')
New_Timeseries = Timeseries.copy()
# Modify the path containing the timeseries
for i in range(len(Timeseries)):
string = Timeseries.iloc[i][0]
if '"' in string: # Check if string is an external file if not nothing is changed.
# This might be slow of the timeseries are long
Rainfile_old = string.split('"')[-2]
if '\\' in Rainfile_old: # If the rain file is in an absolute path this is changed (Also if is in another directory than the model)
Rainfile_old = Rainfile_old.split('/')[-1]
Rain_name = string.split('"')[-3]
Rainfile_new = path + '/' + Rainfile_old
New_Timeseries.iloc[i][
0] = Rain_name + '"' + Rainfile_new + '"'
print('Rainfile_new: ' + Rainfile_new)
print('Rainfile_old: ' + Rainfile_old)
print('Rain_name:' + Rain_name)
else:
New_Timeseries.iloc[i][0] = np.nan
New_Timeseries.dropna(inplace=True)
# Create a temporary file with the adjusted path
new_file = inp_file + '_tmp.inp'
shutil.copyfile(baseline, path + '/' + new_file)
# print('path:' + path )
#Overwrite the TIMESERIES section of the new model with the adjusted data
with pd.option_context(
'display.max_colwidth', 400
): # set the maximum length of string to prit the full string into .inp
replace_inp_section(path + '/' + new_file, '[TIMESERIES]',
New_Timeseries)
model_inp = inp.model_dir + '/' + inp.model_name + '_tmp.inp'
except KeyError:
model_inp = inp.model_dir + '/' + inp.model_name + '.inp'
return model_inp
def drop_invalid_model_elements(inp):
"""
Identify references to elements in the model that are undefined and remove them from the
model. These should coincide with warnings/errors produced by SWMM5 when undefined elements
are referenced in links, subcatchments, and controls.
:param model: swmmio.Model
:return:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES_INVALID
>>> m = swmmio.Model(MODEL_FULL_FEATURES_INVALID)
>>> drop_invalid_model_elements(m.inp)
['InvalidLink2', 'InvalidLink1']
>>> m.inp.conduits.index
Index(['C1:C2', 'C2.1', '1', '2', '4', '5'], dtype='object', name='Name')
"""
from swmmio.utils.dataframes import create_dataframeINP
juncs = create_dataframeINP(inp.path, "[JUNCTIONS]").index.tolist()
outfs = create_dataframeINP(inp.path, "[OUTFALLS]").index.tolist()
stors = create_dataframeINP(inp.path, "[STORAGE]").index.tolist()
nids = juncs + outfs + stors
# drop links with bad refs to inlet/outlet nodes
from swmmio.utils.functions import find_invalid_links
inv_conds = find_invalid_links(inp, nids, 'conduits', drop=True)
inv_pumps = find_invalid_links(inp, nids, 'pumps', drop=True)
inv_orifs = find_invalid_links(inp, nids, 'orifices', drop=True)
inv_weirs = find_invalid_links(inp, nids, 'weirs', drop=True)
# drop other parts of bad links
invalid_links = inv_conds + inv_pumps + inv_orifs + inv_weirs
inp.xsections = inp.xsections.loc[~inp.xsections.index.isin(invalid_links)]
# drop invalid subcats and their related components
invalid_subcats = inp.subcatchments.index[~inp.subcatchments['Outlet'].
isin(nids)]
inp.subcatchments = inp.subcatchments.loc[~inp.subcatchments.index.
isin(invalid_subcats)]
inp.subareas = inp.subareas.loc[~inp.subareas.index.isin(invalid_subcats)]
inp.infiltration = inp.infiltration.loc[~inp.infiltration.index.
isin(invalid_subcats)]
return invalid_links + invalid_subcats
def coordinates(self):
"""
Get/set coordinates section of model
:return: dataframe of model coordinates
"""
if self._coordinates_df is not None:
return self._coordinates_df
self._coordinates_df = create_dataframeINP(self.path,
"[COORDINATES]",
comment_cols=False)
return self._coordinates_df
def polygons(self):
"""
get/set polygons section of model
:return: dataframe of model coordinates
"""
if self._polygons_df is not None:
return self._polygons_df
self._polygons_df = create_dataframeINP(self.path,
'[Polygons]',
comment_cols=False)
return self._polygons_df
def vertices(self):
"""
get/set vertices section of model
:return: dataframe of model coordinates
"""
if self._vertices_df is not None:
return self._vertices_df
self._vertices_df = create_dataframeINP(self.path,
'[VERTICES]',
comment_cols=False)
return self._vertices_df
def storage(self):
"""
Get/set storage section of the INP file.
:return: storage section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._storage_df is None:
self._storage_df = create_dataframeINP(self.path,
"[STORAGE]",
comment_cols=False)
return self._storage_df
def files(self):
"""
Get/set files section of the INP file.
:return: files section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._files_df is None:
self._files_df = create_dataframeINP(self.path,
"[FILES]",
comment_cols=False)
return self._files_df.reset_index()
def build(self, baseline_dir, target_path):
"""
build a complete INP file with the build instructions committed to a
baseline model.
"""
basemodel = swmmio.Model(baseline_dir)
allheaders = funcs.complete_inp_headers(basemodel.inp.filePath)
#new_inp = os.path.join(target_dir, 'model.inp')
with open(target_path, 'w') as f:
for section in allheaders['order']:
#check if the section is not in problem_sections and there are changes
#in self.instructions and commit changes to it from baseline accordingly
if (section not in problem_sections
and allheaders['headers'][section] != 'blob'
and section in self.instructions):
#df of baseline model section
basedf = create_dataframeINP(basemodel.inp, section)
#grab the changes to
changes = self.instructions[section]
#remove elements that have alterations and or tagged for removal
remove_ids = changes.removed.index | changes.altered.index
new_section = basedf.drop(remove_ids)
#add elements
new_section = pd.concat(
[new_section, changes.altered, changes.added])
else:
#section is not well understood or is problematic, just blindly copy
new_section = create_dataframeINP(basemodel.inp,
section=section)
#write the section
vc_utils.write_inp_section(f, allheaders, section, new_section)
def options(self):
"""
Get/set options section of the INP file.
:return: options section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._options_df is None:
self._options_df = create_dataframeINP(self.path,
"[OPTIONS]",
comment_cols=False,
headers=self._headers)
return self._options_df
def subcatchments(self):
"""
Get/set subcatchments section of the INP file.
:return: subcatchments section of the INP file
:rtype: pandas.DataFrame
Examples:
"""
if self._subcatchments_df is None:
self._subcatchments_df = create_dataframeINP(self.path,
"[SUBCATCHMENTS]",
comment_cols=False,
headers=self._headers)
return self._subcatchments_df
def outfalls(self):
"""
Get/set outfalls section of the INP file.
:return: outfalls section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.outfalls
InvertElev OutfallType StageOrTimeseries TideGate
Name
J4 0 FREE NO NaN
"""
if self._outfalls_df is None:
self._outfalls_df = create_dataframeINP(self.path, "[OUTFALLS]", comment_cols=False)
return self._outfalls_df
def subcatchments(self):
"""
collect all useful and available data related subcatchments and organize
in one dataframe.
"""
subs = create_dataframeINP(self.inp.path, "[SUBCATCHMENTS]")
subs = subs.drop([';', 'Comment', 'Origin'], axis=1)
polygons_df = self.inp.polygons
if self.rpt:
flw = create_dataframeRPT(self.rpt.path,
'Subcatchment Runoff Summary')
subs = subs.join(flw)
# more accurate runoff calculations
subs['RunoffAcFt'] = subs.TotalRunoffIn / 12.0 * subs.Area
subs['RunoffMGAccurate'] = subs.RunoffAcFt / 3.06888785
self._subcatchments_df = subs
return subs
def subcatchments(self):
"""
collect all useful and available data related subcatchments and organize
in one dataframe.
"""
subs = create_dataframeINP(self.inp.path, "[SUBCATCHMENTS]")
subs = subs.drop([';', 'Comment', 'Origin'], axis=1)
polygons_df = self.inp.polygons
if self.rpt:
flw = create_dataframeRPT(
self.rpt.path, 'Subcatchment Runoff Summary')
subs = subs.join(flw)
# more accurate runoff calculations
subs['RunoffAcFt'] = subs.TotalRunoffIn / 12.0 * subs.Area
subs['RunoffMGAccurate'] = subs.RunoffAcFt / 3.06888785
self._subcatchments_df = subs
return subs
def outfalls(self):
"""
Get/set outfalls section of the INP file.
:return: outfalls section of the INP file
:rtype: pandas.DataFrame
Examples:
>>> import swmmio
>>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> model.inp.outfalls
InvertElev OutfallType StageOrTimeseries TideGate
Name
J4 0 FREE NO NaN
"""
if self._outfalls_df is None:
self._outfalls_df = create_dataframeINP(self.path,
"[OUTFALLS]",
comment_cols=False)
return self._outfalls_df
def search_for_duplicates(inp_path, verbose = False):
"""
scan an inp file and determine if any element IDs are duplicated in
any section. Method: count the uniques and compare to total length
"""
headers = funcs.complete_inp_headers(inp_path)['headers']
dups_found = False
for header, cols, in headers.items():
if cols != 'blob':
df = dataframes.create_dataframeINP(inp_path, section=header)
elements = df.index
n_unique = len(elements.unique()) #number of unique elements
n_total = len(elements) #total number of elements
if verbose:
print('{} -> (uniques, total) -> ({}, {})'.format(header, n_unique , n_total))
if n_unique != n_total:
dups = ', '.join(df[df.index.duplicated()].index.unique().tolist())
print('duplicate found in {}\nsection: {}\n{}'.format(inp_path, header, dups))
dups_found = True
return dups_found
def generate_inp_from_diffs(basemodel, inpdiffs, target_dir):
"""
create a new inp with respect to a baseline inp and changes instructed
with a list of inp diff files (build instructions). This saves having to
recalculate the differences of each model from the baseline whenever we want
to combine versions.
"""
#step 1 --> combine the diff/build instructions
allheaders = funcs.complete_inp_headers(basemodel.inp.filePath)
combi_build_instr_file = os.path.join(target_dir, 'build_instructions.txt')
newinp = os.path.join(target_dir, 'new.inp')
with open(combi_build_instr_file, 'w') as f:
for header in allheaders['order']:
s = ''
section_header_written = False
for inp in inpdiffs:
sect_s = None
if not section_header_written:
sect_s = text.extract_section_from_inp(inp,
header,
cleanheaders=False,
return_string=True,
skipheaders=False)
section_header_written = True
else:
sect_s = text.extract_section_from_inp(inp,
header,
cleanheaders=False,
return_string=True,
skipheaders=True)
if sect_s:
#remove the extra space between data in the same table
#coming from diffrent models.
if sect_s[-2:] == '\n\n': #NOTE Check this section...
s += sect_s[:-1]
else:
s += sect_s
f.write(s + '\n')
#step 2 --> clean up the new combined diff instructions
df_dict = clean_inp_diff_formatting(
combi_build_instr_file) #makes more human readable
#step 3 --> create a new inp based on the baseline, with the inp_diff
#instructions applied
with open(newinp, 'w') as f:
for section in allheaders['order']:
print section
if section not in problem_sections and allheaders['headers'][
section] != 'blob':
#check if a changes from baseline spreadheet exists, and use this
#information if available to create the changes array
df = create_dataframeINP(basemodel.inp, section)
df['Origin'] = '' #add the origin column if not there
if section in df_dict:
df_change = df_dict[section]
ids_to_drop = df_change.loc[df_change['Comment'].isin(
['Removed', 'Altered'])].index
df = df.drop(ids_to_drop)
df = df.append(df_change.loc[df_change['Comment'].isin(
['Added', 'Altered'])])
new_section = df
else:
#blindly copy this section from the base model
new_section = create_dataframeINP(basemodel.inp,
section=section)
#write the section into the inp file and the excel file
vc_utils.write_inp_section(f, allheaders, section, new_section)
def generate_inp_from_diffs(basemodel, inpdiffs, target_dir):
"""
create a new inp with respect to a baseline inp and changes instructed
with a list of inp diff files (build instructions). This saves having to
recalculate the differences of each model from the baseline whenever we want
to combine versions.
NOTE THIS ISN'T USED ANYWHERE. DELETE ????
"""
#step 1 --> combine the diff/build instructions
allheaders = funcs.complete_inp_headers(basemodel.inp.path)
combi_build_instr_file = os.path.join(target_dir, 'build_instructions.txt')
newinp = os.path.join(target_dir, 'new.inp')
with open (combi_build_instr_file, 'w') as f:
for header in allheaders['order']:
s = ''
section_header_written = False
for inp in inpdiffs:
sect_s = None
if not section_header_written:
sect_s = text.extract_section_from_inp(inp, header,
cleanheaders=False,
return_string=True,
skipheaders=False)
section_header_written = True
else:
sect_s = text.extract_section_from_inp(inp, header,
cleanheaders=False,
return_string=True,
skipheaders=True)
if sect_s:
#remove the extra space between data in the same table
#coming from diffrent models.
if sect_s[-2:] == '\n\n': #NOTE Check this section...
s += sect_s[:-1]
else:
s += sect_s
f.write(s + '\n')
#step 2 --> clean up the new combined diff instructions
# df_dict = clean_inp_diff_formatting(combi_build_instr_file) #makes more human readable
#step 3 --> create a new inp based on the baseline, with the inp_diff
#instructions applied
with open (newinp, 'w') as f:
for section in allheaders['order']:
print(section)
if section not in problem_sections and allheaders['headers'][section] != 'blob':
#check if a changes from baseline spreadheet exists, and use this
#information if available to create the changes array
df = create_dataframeINP(basemodel.inp.path, section)
df['Origin'] = '' #add the origin column if not there
if section in df_dict:
df_change = df_dict[section]
ids_to_drop = df_change.loc[df_change['Comment'].isin(['Removed', 'Altered'])].index
df = df.drop(ids_to_drop)
df = df.append(df_change.loc[df_change['Comment'].isin(['Added', 'Altered'])])
new_section = df
else:
#blindly copy this section from the base model
new_section = create_dataframeINP(basemodel.inp.path, section=section)
#write the section into the inp file and the excel file
vc_utils.write_inp_section(f, allheaders, section, new_section)
def __init__(self,
model1=None,
model2=None,
section='[JUNCTIONS]',
build_instr_file=None):
if model1 and model2:
df1 = create_dataframeINP(model1.inp.path, section)
df2 = create_dataframeINP(model2.inp.path, section)
m2_origin_string = os.path.basename(model2.inp.path).replace(
' ', '-')
# BUG -> this fails if a df1 or df2 is None i.e. if a section doesn't exist in one model
added_ids = df2.index.difference(df1.index)
removed_ids = df1.index.difference(df2.index)
# find where elements were changed (but kept with same ID)
common_ids = df1.index.difference(
removed_ids) # original - removed = in common
# both dfs concatenated, with matched indices for each element
full_set = pd.concat([df1.loc[common_ids], df2.loc[common_ids]])
# remove whitespace
full_set = full_set.apply(lambda x: x.str.strip()
if x.dtype == "object" else x)
# drop dupes on the set, all things that did not changed should have 1 row
changes_with_dupes = full_set.drop_duplicates()
# duplicate indicies are rows that have changes, isolate these
# idx[idx.duplicated()].unique()
changed_ids = changes_with_dupes.index[
changes_with_dupes.index.duplicated()].unique(
) # .get_duplicates()
added = df2.loc[added_ids].copy()
added[
'Comment'] = 'Added' # from model {}'.format(model2.inp.path)
added['Origin'] = m2_origin_string
altered = df2.loc[changed_ids].copy()
altered[
'Comment'] = 'Altered' # in model {}'.format(model2.inp.path)
altered['Origin'] = m2_origin_string
removed = df1.loc[removed_ids].copy()
# comment out the removed elements
# removed.index = ["; " + str(x) for x in removed.index]
removed[
'Comment'] = 'Removed' # in model {}'.format(model2.inp.path)
removed['Origin'] = m2_origin_string
self.old = df1
self.new = df2
self.added = added
self.removed = removed
self.altered = altered
if build_instr_file:
# if generating from a build instructions file, do this (more efficient)
df = create_dataframeBI(build_instr_file, section=section)
self.added = df.loc[df['Comment'] == 'Added']
self.removed = df.loc[df['Comment'] == 'Removed']
self.altered = df.loc[df['Comment'] == 'Altered']
