def create_inp_build_instructions(inpA, inpB, path, filename, comments=''):
"""
pass in two inp file paths and produce a spreadsheet showing the differences
found in each of the INP sections. These differences should then be used
whenever we need to rebuild this model from the baseline reference model.
Note: this should be split into a func that creates a overall model "diff"
that can then be written as a BI file or used programmatically
"""
allsections_a = get_inp_sections_details(inpA)
modela = swmmio.Model(inpA)
modelb = swmmio.Model(inpB)
# create build insructions folder
if not os.path.exists(path):
os.makedirs(path)
filepath = os.path.join(path, filename) + '.txt'
problem_sections = ['TITLE', 'CURVES', 'TIMESERIES', 'RDII', 'HYDROGRAPHS']
with open(filepath, 'w') as newf:
# write meta data
metadata = {
# 'Baseline Model':modela.inp.path,
# 'ID':filename,
'Parent Models': {
'Baseline': {
inpA: vc_utils.modification_date(inpA)
},
'Alternatives': {
inpB: vc_utils.modification_date(inpB)
}
},
'Log': {
filename: comments
}
}
# print metadata
vc_utils.write_meta_data(newf, metadata)
for section, _ in allsections_a.items():
if section not in problem_sections:
# calculate the changes in the current section
changes = INPSectionDiff(modela, modelb, section)
data = pd.concat(
[changes.removed, changes.added, changes.altered],
axis=0,
sort=False)
# vc_utils.write_excel_inp_section(excelwriter, allsections_a, section, data)
vc_utils.write_inp_section(
newf,
allsections_a,
section,
data,
pad_top=False,
na_fill='NaN'
) # na fill fixes SNOWPACK blanks spaces issue
return BuildInstructions(filepath)
def test_remove_model_section():
with tempfile.TemporaryDirectory() as tempdir:
m1 = swmmio.Model(MODEL_A_PATH)
# create a copy of the model without subcatchments
# m1.inp.infiltration = m1.inp.infiltration.iloc[0:0]
m1.inp.subcatchments = m1.inp.subcatchments.iloc[0:0]
# m1.inp.subareas = m1.inp.subareas.iloc[0:0]
# m1.inp.polygons = m1.inp.polygons.iloc[0:0]
# save to temp location
temp_inp = os.path.join(tempdir, f'{m1.inp.name}.inp')
m1.inp.save(temp_inp)
m2 = swmmio.Model(temp_inp)
sects1 = get_inp_sections_details(m1.inp.path)
sects2 = get_inp_sections_details(m2.inp.path)
# confirm saving a copy retains all sections except those removed
assert ['SUBCATCHMENTS'] == [x for x in sects1 if x not in sects2]
# confirm subcatchments returns an empty df
assert m2.subcatchments.dataframe.empty
def test_infiltration_section():
# horton
m = swmmio.Model(MODEL_FULL_FEATURES_XY)
inf = m.inp.infiltration
assert (inf.columns.tolist() == [
'MaxRate', 'MinRate', 'Decay', 'DryTime', 'MaxInfil'
])
# curve number
m = swmmio.Model(MODEL_CURVE_NUMBER)
inf = m.inp.infiltration
assert m.inp.options.loc['INFILTRATION', 'Value'] == 'CURVE_NUMBER'
assert (inf.columns.tolist() == [
'CurveNum', 'Conductivity (depreciated)', 'DryTime'
])
# mod horton
m = swmmio.Model(MODEL_MOD_HORTON)
inf = m.inp.infiltration
assert m.inp.options.loc['INFILTRATION', 'Value'] == 'MODIFIED_HORTON'
assert (inf.columns.tolist() == [
'MaxRate', 'MinRate', 'Decay', 'DryTime', 'MaxInfil'
])
# green ampt
m = swmmio.Model(MODEL_GREEN_AMPT)
inf = m.inp.infiltration
assert m.inp.options.loc['INFILTRATION', 'Value'] == 'GREEN_AMPT'
assert (inf.columns.tolist() == ['Suction', 'HydCon', 'IMDmax'])
# mod green ampt
m = swmmio.Model(MODEL_MOD_GREEN_AMPT)
inf = m.inp.infiltration
assert m.inp.options.loc['INFILTRATION', 'Value'] == 'MODIFIED_GREEN_AMPT'
assert (inf.columns.tolist() == ['Suction', 'Ksat', 'IMD'])
def test_example_1():
model = swmmio.Model(MODEL_EX_1)
element_types = ['nodes', 'links', 'subcatchments']
elem_dict = {
element: model.__getattribute__(element).geojson
for element in element_types
}
swmm_version = model.rpt.swmm_version
assert (swmm_version['major'] == 5)
assert (swmm_version['minor'] == 1)
assert (swmm_version['patch'] == 12)
model_b = swmmio.Model(MODEL_EX_1B)
swmm_version = model_b.rpt.swmm_version
assert (swmm_version['patch'] == 13)
elem_dict = {
element: model_b.__getattribute__(element).geojson
for element in element_types
}
subs = model.subcatchments.dataframe
assert subs['TotalInfil'].sum() == pytest.approx(12.59, rel=0.001)
assert subs['TotalRunoffMG'].sum() == pytest.approx(2.05, rel=0.001)
# access lower level api
peak_runoff = model.rpt.subcatchment_runoff_summary['PeakRunoff']
assert peak_runoff.values == pytest.approx(
[4.66, 4.52, 2.45, 2.45, 6.56, 1.5, 0.79, 1.33], rel=0.001)
assert peak_runoff.values == pytest.approx(subs['PeakRunoff'].values,
rel=0.001)
def __init__(self, model1=None, model2=None):
m1 = model1
m2 = model2
if isinstance(m1, str):
m1 = swmmio.Model(m1)
if isinstance(m2, str):
m2 = swmmio.Model(m2)
self.m1 = m1
self.m2 = m2
self.diffs = OrderedDict()
m1_sects = get_inp_sections_details(m1.inp.path)
m2_sects = get_inp_sections_details(m2.inp.path)
# get union of sections found, maintain order
sects = list(m1_sects.keys()) + list(m2_sects.keys())
seen = set()
self.all_sections = [
x for x in sects if not (x in seen or seen.add(x))
]
self.all_inp_objects = OrderedDict(m1_sects)
self.all_inp_objects.update(m2_sects)
for section in self.all_sections:
if section not in problem_sections:
# calculate the changes in the current section
changes = INPSectionDiff(m1, m2, section)
self.diffs[section] = changes
def replace_inp_section(inp_path, modified_section_header, new_data):
"""
modify an existing inp file by passing in new data (Pandas Dataframe)
and the section header that should be modified. This function will overwrite
all data in the old section with the passed data
:param inp_path: path to inp file to be changed
:param modified_section_header: section for which data should be change
:param new_data: pd.DataFrame of data to overwrite data in the modified section
:return: swmmio.Model instantiated with modified inp file
"""
sections = get_inp_sections_details(inp_path)
m = swmmio.Model(inp_path)
with tempfile.TemporaryDirectory() as tempdir:
with open(inp_path) as oldf:
tmp_inp_path = os.path.join(tempdir, f'{m.inp.name}.inp')
with open(tmp_inp_path, 'w') as new:
# write each line as is from the original model until we find the
# header of the section we wish to overwrite
found_section = False
found_next_section = False
for line in oldf:
if modified_section_header in line:
# write the replacement data in the new file now
write_inp_section(new,
sections,
modified_section_header,
new_data,
pad_top=False)
found_section = True
if (found_section
and any(es in line for es in sections.keys())
and modified_section_header not in line):
found_next_section = True
if found_next_section or not found_section:
# write the lines from the original file
# if we haven't found the section to modify.
# if we have found the section and we've found the NEXT section
# continue writing original file's lines
new.write(line)
if not found_section:
# the header doesn't exist in the old model
# so we should append it to the bottom of file
write_inp_section(new, sections, modified_section_header,
new_data)
# rename files and remove old if we should overwrite
os.remove(inp_path)
shutil.copy2(tmp_inp_path, inp_path)
return swmmio.Model(inp_path)
def test_draw_model():
m = swmmio.Model(MODEL_FULL_FEATURES_XY)
target_img_pth = os.path.join(DATA_PATH, 'test-draw-model.png')
sg.draw_model(m, file_path=target_img_pth)
assert os.path.exists(target_img_pth)
os.remove(target_img_pth)
def test_coordinates():
m = swmmio.Model(MODEL_FULL_FEATURES_XY)
coordinates = m.inp.coordinates
# coordinates.to_csv(df_test_coordinates_csv)
test_coords = pd.read_csv(df_test_coordinates_csv, index_col=0)
assert (coordinates.equals(test_coords))
def test_web_map_01():
m = swmmio.Model(MODEL_A_PATH, crs="+init=EPSG:2817")
with tempfile.TemporaryDirectory() as tempdir:
fname = os.path.join(tempdir, 'test-map.html')
sg.create_map(m, filename=fname)
assert os.path.exists(fname)
def __init__(self, model1=None, model2=None):
"""
diff of all INP sections between two models
:param model1:
:param model2:
>>> from swmmio.tests.data import MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B
>>> mydiff = INPDiff(MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B)
>>> mydiff.diffs['[XSECTIONS]']
Shape Geom1 Geom2 Geom3 Geom4 Barrels ; Comment Origin
Link
1:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
2:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
3:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
4:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
5:J1 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
<BLANKLINE>
>>> print(mydiff)
"""
m1 = model1
m2 = model2
if isinstance(m1, str):
m1 = swmmio.Model(m1)
if isinstance(m2, str):
m2 = swmmio.Model(m2)
self.m1 = m1
self.m2 = m2
self.diffs = OrderedDict()
m1_sects = get_inp_sections_details(m1.inp.path)
m2_sects = get_inp_sections_details(m2.inp.path)
# get union of sections found, maintain order
sects = list(m1_sects.keys()) + list(m2_sects.keys())
seen = set()
self.all_sections = [
x for x in sects if not (x in seen or seen.add(x))
]
self.all_inp_objects = OrderedDict(m1_sects)
self.all_inp_objects.update(m2_sects)
for section in self.all_sections:
if section not in problem_sections:
# calculate the changes in the current section
changes = INPSectionDiff(m1, m2, section)
self.diffs[section] = changes
def test_inflow_dwf_dataframe():
m = swmmio.Model(MODEL_XSECTION_ALT_03)
dwf = dataframe_from_inp(m.inp.path, 'dwf')
assert (dwf.loc['dummy_node2',
'AverageValue'] == pytest.approx(0.000275, 0.0001))
inf = m.inp.inflows
assert (inf.loc['dummy_node2', 'Time Series'] == 'my_time_series')
assert (pd.isna(inf.loc['dummy_node6', 'Time Series']))
def test_model_to_networkx():
m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
G = m.network
assert (G['J2']['J3']['C2.1']['Length'] == 666)
assert (G['J1']['J2']['C1:C2']['Length'] == 244.63)
assert (round(G.node['J2']['InvertElev'], 3) == 13.0)
links = m.links()
assert (len(links) == len(G.edges()))
def test_draw_red_and_grey_nodes():
m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
target_img_pth = os.path.join(DATA_PATH, 'test-draw-model.png')
nodes = m.nodes()
nodes['draw_color'] = '#787882'
nodes.loc[['J1', 'J2', 'J3'], 'draw_color'] = '#ff0000'
nodes['draw_size'] = nodes['InvertElev'] * 3
sg.draw_model(conduits=m.conduits(), nodes=nodes, file_path=target_img_pth)
assert os.path.exists(target_img_pth)
os.remove(target_img_pth)
def test_centroid_and_bbox_from_coords():
m = swmmio.Model(MODEL_A_PATH, crs="+init=EPSG:2817")
m.to_crs("+init=EPSG:4326")
c, bbox = centroid_and_bbox_from_coords(m.nodes.dataframe['coords'])
assert c == (-70.97068150884797, 43.74695249578866)
assert bbox == [-70.97068150884797, 43.74695249578866, -70.97068150884797, 43.74695249578866]
c, bbox = centroid_and_bbox_from_coords([(0, 0), (0, 10), (10, 10), (10, 0)])
assert c == (5, 5)
assert bbox == [0, 0, 10, 10]
def test_centroid_and_bbox_from_coords():
m = swmmio.Model(MODEL_A_PATH, crs="EPSG:2817")
m.to_crs("EPSG:4326")
c, bbox = centroid_and_bbox_from_coords(m.nodes.dataframe['coords'])
assert c == pytest.approx((-70.97068, 43.74695), rel=1e-3)
assert bbox == pytest.approx((-70.97068, 43.74695, -70.97068, 43.74695),
rel=1e-3)
c, bbox = centroid_and_bbox_from_coords([(0, 0), (0, 10), (10, 10),
(10, 0)])
assert c == (5, 5)
assert bbox == [0, 0, 10, 10]
def test_change_crs():
m = swmmio.Model(MODEL_A_PATH, crs="+init=EPSG:2817")
v1 = m.inp.vertices
v2 = change_crs(m.inp.vertices, m.crs, "+init=EPSG:4326")
assert v1.shape == v2.shape
s = """
Name X Y
J4-001.1 -70.959386 43.732851
J4-001.1 -70.958415 43.732578
J4-001.1 -70.959423 43.730452
J2-095.1 -70.951378 43.767796
"""
v2_test = pd.read_csv(StringIO(s), index_col=0, delim_whitespace=True, skiprows=[0])
assert v2.to_string() == v2_test.to_string()
def test_nodes_dataframe():
m = swmmio.Model(MODEL_XSECTION_ALT_01)
nodes = m.nodes()
node_ids_01 = [
'dummy_node1', 'dummy_node2', 'dummy_node3', 'dummy_node4',
'dummy_node5', 'dummy_node6', 'dummy_outfall'
]
assert (list(nodes.index) == node_ids_01)
assert (nodes.loc['dummy_node1', 'InvertElev'] == -10.99)
assert (nodes.loc['dummy_node2', 'MaxDepth'] == 20)
assert (nodes.loc['dummy_node3', 'X'] == -4205.457)
assert (nodes.loc['dummy_node4', 'MaxDepth'] == 12.59314)
assert (nodes.loc['dummy_node5', 'PondedArea'] == 73511)
def __init__(self, inp_file_path, out_file_path):
if not inp_file_path:
self.inp_file_path = config['inp_file_path']
else:
self.inp_file_path = inp_file_path
if not out_file_path:
self.out_file_path = config['out_file_path']
else:
self.out_file_path = out_file_path
#initialize some baseline dataframes
self.baseline = swmmio.core.inp(inp_file_path)
self.baselineModel = swmmio.Model(inp_file_path)
self.baselineLinks = self.baselineModel.links.dataframe
self.baselineNodes = self.baselineModel.nodes.dataframe
self.timeseries = self.baseline.timeseries # any changes should be made on deep copy of self.timeseries
self.inflows = self.baseline.inflows
def test_change_crs():
m = swmmio.Model(MODEL_A_PATH, crs="EPSG:2817")
v1 = m.inp.vertices
v2 = change_crs(m.inp.vertices, m.crs, "WGS84")
assert v1.shape == v2.shape
s = """
Name X Y
J4-001.1 -70.959386 43.732851
J4-001.1 -70.958415 43.732578
J4-001.1 -70.959423 43.730452
J2-095.1 -70.951378 43.767796
"""
v2_test = pd.read_csv(StringIO(s),
index_col=0,
delim_whitespace=True,
skiprows=[0])
assert v2['X'].values == pytest.approx(v2_test['X'].values, rel=1e-3)
assert v2['Y'].values == pytest.approx(v2_test['Y'].values, rel=1e-3)
def get_sim_duration_and_report_step():
# initialize a model model object
model = swmmio.Model(data_dir + 'scenario.inp')
inp_start_date = model.inp.options.loc["START_DATE"].values[0]
inp_start_time = model.inp.options.loc["START_TIME"].values[0]
inp_end_date = model.inp.options.loc["END_DATE"].values[0]
inp_end_time = model.inp.options.loc["END_TIME"].values[0]
start_time = datetime.datetime.strptime(
inp_start_date + ' ' + inp_start_time, '%d/%m/%Y %H:%M:%S')
end_time = datetime.datetime.strptime(inp_end_date + ' ' + inp_end_time,
'%d/%m/%Y %H:%M:%S')
report_step = datetime.datetime.strptime(
model.inp.options.loc["REPORT_STEP"].values[0], '%H:%M:%S').minute
simulation_duration = int((end_time - start_time).total_seconds() / 60)
return simulation_duration, report_step
def test_create_dataframeRPT():
m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
depth_summ = swmmio.create_dataframeRPT(m.rpt.path, "Node Depth Summary")
flood_summ = swmmio.create_dataframeRPT(m.rpt.path,
"Node Flooding Summary")
inflo_summ = swmmio.create_dataframeRPT(m.rpt.path, "Node Inflow Summary")
print('\n', depth_summ)
print(inflo_summ)
print(flood_summ)
assert (inflo_summ.loc['J3', 'TotalInflowV'] == 6.1)
assert (inflo_summ.loc['J1', 'MaxTotalInflow'] == 3.52)
assert (depth_summ.loc['J3', 'MaxNodeDepth'] == 1.64)
assert (depth_summ.loc['4', 'MaxNodeDepth'] == 0.87)
# need to ensure indices are strings always
assert (flood_summ.loc[5, 'TotalFloodVol'] == 0)
def make_inp_file(user_input):
# initialize a baseline model object
baseline = swmmio.Model(data_dir + 'baseline.inp')
# create a dataframe of the model's subcatchments
subs = baseline.inp.subcatchments
# reads updates to model from user input and updates the swmmio model
for update in user_input["model_updates"]:
# update the outlet_id in the row of subcatchment_id
subs.loc[update["subcatchment_id"], ['Outlet']] = update['outlet_id']
baseline.inp.subcatchments = subs
# set the rain gage from user input as raingage for all subcatchments
scenario_rain_gage_name = 'RainGage' + '_' + str(user_input["rain_event"]["return_period"])
for i, row in subs.iterrows():
subs.at[i, 'Raingage'] = scenario_rain_gage_name
# Save the new model with the adjusted data
new_file_path = data_dir + 'scenario.inp'
baseline.inp.save(new_file_path)
def build(self, baseline_dir, target_path):
"""
build a complete INP file with the build instructions committed to a
baseline model.
"""
basemodel = swmmio.Model(baseline_dir)
allheaders = get_inp_sections_details(basemodel.inp.path)
# new_inp = os.path.join(target_dir, 'model.inp')
with open(target_path, 'w') as f:
for section, _ in allheaders.items():
# check if the section is not in problem_sections and there are changes
# in self.instructions and commit changes to it from baseline accordingly
if (section not in problem_sections
and allheaders[section]['columns'] != ['blob']
and section in self.instructions):
# df of baseline model section
basedf = dataframe_from_bi(basemodel.inp.path, section)
basedf[';'] = ';'
# grab the changes to
changes = self.instructions[section]
# remove elements that have alterations and or tagged for removal
remove_ids = changes.removed.index | changes.altered.index
new_section = basedf.drop(remove_ids)
# add elements
new_section = pd.concat(
[new_section, changes.altered, changes.added])
else:
# section is not well understood or is problematic, just blindly copy
new_section = dataframe_from_bi(basemodel.inp.path,
section=section)
new_section[';'] = ';'
# write the section
vc_utils.write_inp_section(f, allheaders, section, new_section)
def test_model_section():
m = swmmio.Model(MODEL_FULL_FEATURES_XY)
def pumps_old_method(model):
"""
collect all useful and available data related model pumps and
organize in one dataframe.
"""
# check if this has been done already and return that data accordingly
if model._pumps_df is not None:
return model._pumps_df
# parse out the main objects of this model
inp = model.inp
rpt = model.rpt
# create dataframes of relevant sections from the INP
pumps_df = create_dataframeINP(inp.path, "[PUMPS]", comment_cols=False)
if pumps_df.empty:
return pd.DataFrame()
# add conduit coordinates
xys = pumps_df.apply(
lambda r: swmmio.get_link_coords(r, inp.coordinates, inp.vertices),
axis=1)
df = pumps_df.assign(coords=xys.map(lambda x: x[0]))
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
model._pumps_df = df
return df
pumps_old_method = pumps_old_method(m)
pumps = m.pumps()
assert (pumps_old_method.equals(pumps))
def test_conduits_dataframe(test_model_01):
m = swmmio.Model(MODEL_FULL_FEATURES_PATH)
conduits = m.conduits()
assert (list(conduits.index) == ['C1:C2'])
def test_model_02():
return swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
def test_model_01():
return swmmio.Model(MODEL_FULL_FEATURES_XY)
def merge_models(inp1, inp2, target='merged_model.inp'):
"""
Merge two separate swmm models into one model. This creates a diff, ignores
removed sections, and uses inp1 settings where conflicts exist (altered sections in diff)
:param inp1: swmmio.Model.inp object to be combined with inp2
:param inp2: swmmio.Model.inp object to be combined with inp1
:param target: path of new model
:return: path to target
"""
# model object to store resulting merged model
m3 = swmmio.Model(inp1)
inp_diff = INPDiff(inp1, inp2)
with open(target, 'w') as newf:
for section, _ in inp_diff.all_inp_objects.items():
# don't consider the "removed" parts of the diff
# print('{}: {}'.format(section,inp_diff.all_inp_objects[section]['columns']))
# check if the section is not in problem_sections and there are changes
# in self.instructions and commit changes to it from baseline accordingly
col_order = []
if (section not in problem_sections and
inp_diff.all_inp_objects[section]['columns'] != ['blob']
and section in inp_diff.diffs):
# df of baseline model section
basedf = dataframe_from_inp(
m3.inp.path,
section,
additional_cols=[';', 'Comment', 'Origin'])
basedf[';'] = ';'
col_order = basedf.columns
# grab the changes to
changes = inp_diff.diffs[section]
# remove elements that have alterations keep ones tagged for removal
# (unchanged, but not present in m2)
remove_ids = changes.altered.index
new_section = basedf.drop(remove_ids)
# add elements
new_section = pd.concat(
[new_section, changes.altered, changes.added],
axis=0,
sort=False)
else:
# section is not well understood or is problematic, just blindly copy
new_section = dataframe_from_inp(
m3.inp.path,
section,
additional_cols=[';', 'Comment', 'Origin'])
new_section[';'] = ';'
# print ('dealing with confusing section: {}\n{}'.format(section, new_section))
# print(new_section.head())
# write the section
new_section = new_section[col_order]
new_section[';'] = ';'
vc_utils.write_inp_section(newf,
inp_diff.all_inp_objects,
section,
new_section,
pad_top=True)
return target
