def test_remove_model_section():
with tempfile.TemporaryDirectory() as tempdir:
m1 = swmmio.Model(MODEL_A_PATH)
# create a copy of the model without subcatchments
# m1.inp.infiltration = m1.inp.infiltration.iloc[0:0]
m1.inp.subcatchments = m1.inp.subcatchments.iloc[0:0]
# m1.inp.subareas = m1.inp.subareas.iloc[0:0]
# m1.inp.polygons = m1.inp.polygons.iloc[0:0]
# save to temp location
temp_inp = os.path.join(tempdir, f'{m1.inp.name}.inp')
m1.inp.save(temp_inp)
m2 = swmmio.Model(temp_inp)
sects1 = get_inp_sections_details(m1.inp.path)
sects2 = get_inp_sections_details(m2.inp.path)
# confirm saving a copy retains all sections except those removed
assert ['SUBCATCHMENTS'] == [x for x in sects1 if x not in sects2]
# confirm subcatchments returns an empty df
assert m2.subcatchments.dataframe.empty
def __init__(self, model1=None, model2=None):
m1 = model1
m2 = model2
if isinstance(m1, str):
m1 = swmmio.Model(m1)
if isinstance(m2, str):
m2 = swmmio.Model(m2)
self.m1 = m1
self.m2 = m2
self.diffs = OrderedDict()
m1_sects = get_inp_sections_details(m1.inp.path)
m2_sects = get_inp_sections_details(m2.inp.path)
# get union of sections found, maintain order
sects = list(m1_sects.keys()) + list(m2_sects.keys())
seen = set()
self.all_sections = [
x for x in sects if not (x in seen or seen.add(x))
]
self.all_inp_objects = OrderedDict(m1_sects)
self.all_inp_objects.update(m2_sects)
for section in self.all_sections:
if section not in problem_sections:
# calculate the changes in the current section
changes = INPSectionDiff(m1, m2, section)
self.diffs[section] = changes
def create_inp_build_instructions(inpA, inpB, path, filename, comments=''):
"""
pass in two inp file paths and produce a spreadsheet showing the differences
found in each of the INP sections. These differences should then be used
whenever we need to rebuild this model from the baseline reference model.
Note: this should be split into a func that creates a overall model "diff"
that can then be written as a BI file or used programmatically
"""
allsections_a = get_inp_sections_details(inpA)
modela = swmmio.Model(inpA)
modelb = swmmio.Model(inpB)
# create build insructions folder
if not os.path.exists(path):
os.makedirs(path)
filepath = os.path.join(path, filename) + '.txt'
problem_sections = ['TITLE', 'CURVES', 'TIMESERIES', 'RDII', 'HYDROGRAPHS']
with open(filepath, 'w') as newf:
# write meta data
metadata = {
# 'Baseline Model':modela.inp.path,
# 'ID':filename,
'Parent Models': {
'Baseline': {
inpA: vc_utils.modification_date(inpA)
},
'Alternatives': {
inpB: vc_utils.modification_date(inpB)
}
},
'Log': {
filename: comments
}
}
# print metadata
vc_utils.write_meta_data(newf, metadata)
for section, _ in allsections_a.items():
if section not in problem_sections:
# calculate the changes in the current section
changes = INPSectionDiff(modela, modelb, section)
data = pd.concat(
[changes.removed, changes.added, changes.altered],
axis=0,
sort=False)
# vc_utils.write_excel_inp_section(excelwriter, allsections_a, section, data)
vc_utils.write_inp_section(
newf,
allsections_a,
section,
data,
pad_top=False,
na_fill='NaN'
) # na fill fixes SNOWPACK blanks spaces issue
return BuildInstructions(filepath)
def create_dataframe_multi_index(inp_path, section='CURVES'):
# format the section header for look up in headers OrderedDict
sect = remove_braces(section).upper()
# get list of all section headers in inp to use as section ending flags
headers = get_inp_sections_details(inp_path, include_brackets=False)
if sect not in headers:
warnings.warn(f'{sect} section not found in {inp_path}')
return pd.DataFrame()
# extract the string and read into a dataframe
start_string = format_inp_section_header(section)
end_strings = [format_inp_section_header(h) for h in headers.keys()]
s = extract_section_of_file(inp_path, start_string, end_strings)
cols = headers[sect]['columns']
f = StringIO(s)
data = []
for line in f.readlines():
items = line.strip().split()
if len(items) == 3:
items = [items[0], None, items[1], items[2]]
if len(items) == 4:
data.append(items)
df = pd.DataFrame(data=data, columns=cols)
df = df.set_index(['Name', 'Type'])
return df
def __init__(self, model1=None, model2=None):
"""
diff of all INP sections between two models
:param model1:
:param model2:
>>> from swmmio.tests.data import MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B
>>> mydiff = INPDiff(MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B)
>>> mydiff.diffs['[XSECTIONS]']
Shape Geom1 Geom2 Geom3 Geom4 Barrels ; Comment Origin
Link
1:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
2:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
3:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
4:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
5:J1 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp
<BLANKLINE>
>>> print(mydiff)
"""
m1 = model1
m2 = model2
if isinstance(m1, str):
m1 = swmmio.Model(m1)
if isinstance(m2, str):
m2 = swmmio.Model(m2)
self.m1 = m1
self.m2 = m2
self.diffs = OrderedDict()
m1_sects = get_inp_sections_details(m1.inp.path)
m2_sects = get_inp_sections_details(m2.inp.path)
# get union of sections found, maintain order
sects = list(m1_sects.keys()) + list(m2_sects.keys())
seen = set()
self.all_sections = [
x for x in sects if not (x in seen or seen.add(x))
]
self.all_inp_objects = OrderedDict(m1_sects)
self.all_inp_objects.update(m2_sects)
for section in self.all_sections:
if section not in problem_sections:
# calculate the changes in the current section
changes = INPSectionDiff(m1, m2, section)
self.diffs[section] = changes
def replace_inp_section(inp_path, modified_section_header, new_data):
"""
modify an existing inp file by passing in new data (Pandas Dataframe)
and the section header that should be modified. This function will overwrite
all data in the old section with the passed data
:param inp_path: path to inp file to be changed
:param modified_section_header: section for which data should be change
:param new_data: pd.DataFrame of data to overwrite data in the modified section
:return: swmmio.Model instantiated with modified inp file
"""
sections = get_inp_sections_details(inp_path)
m = swmmio.Model(inp_path)
with tempfile.TemporaryDirectory() as tempdir:
with open(inp_path) as oldf:
tmp_inp_path = os.path.join(tempdir, f'{m.inp.name}.inp')
with open(tmp_inp_path, 'w') as new:
# write each line as is from the original model until we find the
# header of the section we wish to overwrite
found_section = False
found_next_section = False
for line in oldf:
if modified_section_header in line:
# write the replacement data in the new file now
write_inp_section(new,
sections,
modified_section_header,
new_data,
pad_top=False)
found_section = True
if (found_section
and any(es in line for es in sections.keys())
and modified_section_header not in line):
found_next_section = True
if found_next_section or not found_section:
# write the lines from the original file
# if we haven't found the section to modify.
# if we have found the section and we've found the NEXT section
# continue writing original file's lines
new.write(line)
if not found_section:
# the header doesn't exist in the old model
# so we should append it to the bottom of file
write_inp_section(new, sections, modified_section_header,
new_data)
# rename files and remove old if we should overwrite
os.remove(inp_path)
shutil.copy2(tmp_inp_path, inp_path)
return swmmio.Model(inp_path)
def dataframe_from_inp(inp_path,
section,
additional_cols=None,
quote_replace=' ',
**kwargs):
"""
create a dataframe from a section of an INP file
:param inp_path:
:param section:
:param additional_cols:
:param skip_headers:
:param quote_replace:
:return:
"""
# format the section header for look up in headers OrderedDict
sect = remove_braces(section).upper()
# get list of all section headers in inp to use as section ending flags
headers = get_inp_sections_details(inp_path, include_brackets=False)
if sect not in headers:
warnings.warn(f'{sect} section not found in {inp_path}')
return pd.DataFrame()
# extract the string and read into a dataframe
start_string = format_inp_section_header(section)
end_strings = [format_inp_section_header(h) for h in headers.keys()]
s = extract_section_of_file(inp_path, start_string, end_strings, **kwargs)
# replace occurrences of double quotes ""
s = s.replace('""', quote_replace)
# and get the list of columns to use for parsing this section
# add any additional columns needed for special cases (build instructions)
additional_cols = [] if additional_cols is None else additional_cols
cols = headers[sect]['columns'] + additional_cols
if headers[sect]['columns'][0] == 'blob':
# return the whole row, without specific col headers
return pd.read_csv(StringIO(s), delim_whitespace=False)
else:
try:
df = pd.read_csv(StringIO(s),
header=None,
delim_whitespace=True,
skiprows=[0],
index_col=0,
names=cols)
except IndexError:
print(
f'failed to parse {section} with cols: {cols}. head:\n{s[:500]}'
)
raise
return df
def test_modify_inp_sections():
m1 = swmmio.Model(MODEL_A_PATH)
with tempfile.TemporaryDirectory() as tempdir:
cp_path = os.path.join(tempdir, f'{m1.inp.name}.inp')
m1.inp.save(cp_path)
# first confirm saving a copy produces same content
assert filecmp.cmp(MODEL_A_PATH, cp_path)
# change a section
m2_path = os.path.join(tempdir, f'{m1.inp.name}_nodes_99.inp')
m1.inp.junctions['InvertElev'] = 99
m1.inp.save(m2_path)
m2 = swmmio.Model(m2_path)
# confirm the change worked
assert m2.inp.junctions['InvertElev'].min() == 99
assert m2.inp.junctions['InvertElev'].max() == 99
# confirm all sections are in the new model
sects1 = get_inp_sections_details(m1.inp.path)
sects2 = get_inp_sections_details(m2.inp.path)
assert all([x in sects1 for x in sects2])
def headers(self):
"""
Return all headers and associated column names found in the INP file.
"""
if self._inp_section_details is None:
self._inp_section_details = get_inp_sections_details(self.path,
include_brackets=True,
)
# select the correct infiltration column names
infil_type = self.options.loc['INFILTRATION', 'Value']
infil_cols = INFILTRATION_COLS[infil_type]
# overwrite the dynamic sections with proper header cols
self._inp_section_details['[INFILTRATION]'] = list(infil_cols)
return self._inp_section_details
def __init__(self, build_instr_file=None):
# create a change object for each section that is different from baseline
self.instructions = {}
self.metadata = {}
if build_instr_file:
# read the instructions and create a dictionary of Change objects
allheaders = get_inp_sections_details(build_instr_file)
instructions = {}
for section, _ in allheaders.items():
change = INPSectionDiff(build_instr_file=build_instr_file,
section=section)
instructions.update({section: change})
self.instructions = instructions
# read the meta data
self.metadata = vc_utils.read_meta_data(build_instr_file)
def build(self, baseline_dir, target_path):
"""
build a complete INP file with the build instructions committed to a
baseline model.
"""
basemodel = swmmio.Model(baseline_dir)
allheaders = get_inp_sections_details(basemodel.inp.path)
# new_inp = os.path.join(target_dir, 'model.inp')
with open(target_path, 'w') as f:
for section, _ in allheaders.items():
# check if the section is not in problem_sections and there are changes
# in self.instructions and commit changes to it from baseline accordingly
if (section not in problem_sections
and allheaders[section]['columns'] != ['blob']
and section in self.instructions):
# df of baseline model section
basedf = dataframe_from_bi(basemodel.inp.path, section)
basedf[';'] = ';'
# grab the changes to
changes = self.instructions[section]
# remove elements that have alterations and or tagged for removal
remove_ids = changes.removed.index | changes.altered.index
new_section = basedf.drop(remove_ids)
# add elements
new_section = pd.concat(
[new_section, changes.altered, changes.added])
else:
# section is not well understood or is problematic, just blindly copy
new_section = dataframe_from_bi(basemodel.inp.path,
section=section)
new_section[';'] = ';'
# write the section
vc_utils.write_inp_section(f, allheaders, section, new_section)
def __call__(self):
"""
collect all useful and available data related to the conduits and
organize in one dataframe.
>>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
>>> conduits_section = ModelSection(model, 'conduits')
>>> conduits_section()
"""
# concat inp sections with unique element IDs
headers = get_inp_sections_details(self.inp.path)
dfs = [
dataframe_from_inp(self.inp.path, sect)
for sect in self.inp_sections if sect.upper() in headers
]
# return empty df if no inp sections found
if len(dfs) == 0:
return pd.DataFrame()
df = pd.concat(dfs, axis=0, sort=False)
# join to this any sections with matching IDs (e.g. XSECTIONS)
for sect in self.join_sections:
rsuffix = f"_{sect.replace(' ', '_')}"
df = df.join(dataframe_from_inp(self.inp.path, sect),
rsuffix=rsuffix)
if df.empty:
return df
# if there is an RPT available, grab relevant sections
if self.rpt:
for rpt_sect in self.rpt_sections:
df = df.join(dataframe_from_rpt(self.rpt.path, rpt_sect))
# add coordinates
if self.geomtype == 'point':
df = df.join(self.inp.coordinates[['X', 'Y']])
xys = df.apply(lambda r: nodexy(r), axis=1)
df = df.assign(coords=xys)
elif self.geomtype == 'linestring':
# add conduit coordinates
xys = df.apply(lambda r: get_link_coords(r, self.inp.coordinates,
self.inp.vertices),
axis=1)
df = df.assign(coords=xys.map(lambda x: x[0]))
# make inlet/outlet node IDs string type
df.InletNode = df.InletNode.astype(str)
df.OutletNode = df.OutletNode.astype(str)
elif self.geomtype == 'polygon':
p = self.inp.polygons
# take stacked coordinates and orient in list of tuples,
xys = p.groupby(by=p.index).apply(
lambda r: [(xy['X'], xy['Y']) for ix, xy in r.iterrows()])
# copy the first point to the last position
xys = xys.apply(lambda r: r + [r[0]])
df = df.assign(coords=xys)
# confirm index name is string
df = df.rename(index=str)
# trim to desired columns
if self.columns is not None:
df = df[[c for c in self.columns if c in df.columns]]
self._df = df
return df
