def create_inp_build_instructions(inpA, inpB, path, filename, comments=''): """ pass in two inp file paths and produce a spreadsheet showing the differences found in each of the INP sections. These differences should then be used whenever we need to rebuild this model from the baseline reference model. Note: this should be split into a func that creates a overall model "diff" that can then be written as a BI file or used programmatically """ allsections_a = get_inp_sections_details(inpA) modela = swmmio.Model(inpA) modelb = swmmio.Model(inpB) # create build insructions folder if not os.path.exists(path): os.makedirs(path) filepath = os.path.join(path, filename) + '.txt' problem_sections = ['TITLE', 'CURVES', 'TIMESERIES', 'RDII', 'HYDROGRAPHS'] with open(filepath, 'w') as newf: # write meta data metadata = { # 'Baseline Model':modela.inp.path, # 'ID':filename, 'Parent Models': { 'Baseline': { inpA: vc_utils.modification_date(inpA) }, 'Alternatives': { inpB: vc_utils.modification_date(inpB) } }, 'Log': { filename: comments } } # print metadata vc_utils.write_meta_data(newf, metadata) for section, _ in allsections_a.items(): if section not in problem_sections: # calculate the changes in the current section changes = INPSectionDiff(modela, modelb, section) data = pd.concat( [changes.removed, changes.added, changes.altered], axis=0, sort=False) # vc_utils.write_excel_inp_section(excelwriter, allsections_a, section, data) vc_utils.write_inp_section( newf, allsections_a, section, data, pad_top=False, na_fill='NaN' ) # na fill fixes SNOWPACK blanks spaces issue return BuildInstructions(filepath)
def test_remove_model_section(): with tempfile.TemporaryDirectory() as tempdir: m1 = swmmio.Model(MODEL_A_PATH) # create a copy of the model without subcatchments # m1.inp.infiltration = m1.inp.infiltration.iloc[0:0] m1.inp.subcatchments = m1.inp.subcatchments.iloc[0:0] # m1.inp.subareas = m1.inp.subareas.iloc[0:0] # m1.inp.polygons = m1.inp.polygons.iloc[0:0] # save to temp location temp_inp = os.path.join(tempdir, f'{m1.inp.name}.inp') m1.inp.save(temp_inp) m2 = swmmio.Model(temp_inp) sects1 = get_inp_sections_details(m1.inp.path) sects2 = get_inp_sections_details(m2.inp.path) # confirm saving a copy retains all sections except those removed assert ['SUBCATCHMENTS'] == [x for x in sects1 if x not in sects2] # confirm subcatchments returns an empty df assert m2.subcatchments.dataframe.empty
def test_infiltration_section(): # horton m = swmmio.Model(MODEL_FULL_FEATURES_XY) inf = m.inp.infiltration assert (inf.columns.tolist() == [ 'MaxRate', 'MinRate', 'Decay', 'DryTime', 'MaxInfil' ]) # curve number m = swmmio.Model(MODEL_CURVE_NUMBER) inf = m.inp.infiltration assert m.inp.options.loc['INFILTRATION', 'Value'] == 'CURVE_NUMBER' assert (inf.columns.tolist() == [ 'CurveNum', 'Conductivity (depreciated)', 'DryTime' ]) # mod horton m = swmmio.Model(MODEL_MOD_HORTON) inf = m.inp.infiltration assert m.inp.options.loc['INFILTRATION', 'Value'] == 'MODIFIED_HORTON' assert (inf.columns.tolist() == [ 'MaxRate', 'MinRate', 'Decay', 'DryTime', 'MaxInfil' ]) # green ampt m = swmmio.Model(MODEL_GREEN_AMPT) inf = m.inp.infiltration assert m.inp.options.loc['INFILTRATION', 'Value'] == 'GREEN_AMPT' assert (inf.columns.tolist() == ['Suction', 'HydCon', 'IMDmax']) # mod green ampt m = swmmio.Model(MODEL_MOD_GREEN_AMPT) inf = m.inp.infiltration assert m.inp.options.loc['INFILTRATION', 'Value'] == 'MODIFIED_GREEN_AMPT' assert (inf.columns.tolist() == ['Suction', 'Ksat', 'IMD'])
def test_example_1(): model = swmmio.Model(MODEL_EX_1) element_types = ['nodes', 'links', 'subcatchments'] elem_dict = { element: model.__getattribute__(element).geojson for element in element_types } swmm_version = model.rpt.swmm_version assert (swmm_version['major'] == 5) assert (swmm_version['minor'] == 1) assert (swmm_version['patch'] == 12) model_b = swmmio.Model(MODEL_EX_1B) swmm_version = model_b.rpt.swmm_version assert (swmm_version['patch'] == 13) elem_dict = { element: model_b.__getattribute__(element).geojson for element in element_types } subs = model.subcatchments.dataframe assert subs['TotalInfil'].sum() == pytest.approx(12.59, rel=0.001) assert subs['TotalRunoffMG'].sum() == pytest.approx(2.05, rel=0.001) # access lower level api peak_runoff = model.rpt.subcatchment_runoff_summary['PeakRunoff'] assert peak_runoff.values == pytest.approx( [4.66, 4.52, 2.45, 2.45, 6.56, 1.5, 0.79, 1.33], rel=0.001) assert peak_runoff.values == pytest.approx(subs['PeakRunoff'].values, rel=0.001)
def __init__(self, model1=None, model2=None): m1 = model1 m2 = model2 if isinstance(m1, str): m1 = swmmio.Model(m1) if isinstance(m2, str): m2 = swmmio.Model(m2) self.m1 = m1 self.m2 = m2 self.diffs = OrderedDict() m1_sects = get_inp_sections_details(m1.inp.path) m2_sects = get_inp_sections_details(m2.inp.path) # get union of sections found, maintain order sects = list(m1_sects.keys()) + list(m2_sects.keys()) seen = set() self.all_sections = [ x for x in sects if not (x in seen or seen.add(x)) ] self.all_inp_objects = OrderedDict(m1_sects) self.all_inp_objects.update(m2_sects) for section in self.all_sections: if section not in problem_sections: # calculate the changes in the current section changes = INPSectionDiff(m1, m2, section) self.diffs[section] = changes
def replace_inp_section(inp_path, modified_section_header, new_data): """ modify an existing inp file by passing in new data (Pandas Dataframe) and the section header that should be modified. This function will overwrite all data in the old section with the passed data :param inp_path: path to inp file to be changed :param modified_section_header: section for which data should be change :param new_data: pd.DataFrame of data to overwrite data in the modified section :return: swmmio.Model instantiated with modified inp file """ sections = get_inp_sections_details(inp_path) m = swmmio.Model(inp_path) with tempfile.TemporaryDirectory() as tempdir: with open(inp_path) as oldf: tmp_inp_path = os.path.join(tempdir, f'{m.inp.name}.inp') with open(tmp_inp_path, 'w') as new: # write each line as is from the original model until we find the # header of the section we wish to overwrite found_section = False found_next_section = False for line in oldf: if modified_section_header in line: # write the replacement data in the new file now write_inp_section(new, sections, modified_section_header, new_data, pad_top=False) found_section = True if (found_section and any(es in line for es in sections.keys()) and modified_section_header not in line): found_next_section = True if found_next_section or not found_section: # write the lines from the original file # if we haven't found the section to modify. # if we have found the section and we've found the NEXT section # continue writing original file's lines new.write(line) if not found_section: # the header doesn't exist in the old model # so we should append it to the bottom of file write_inp_section(new, sections, modified_section_header, new_data) # rename files and remove old if we should overwrite os.remove(inp_path) shutil.copy2(tmp_inp_path, inp_path) return swmmio.Model(inp_path)
def test_draw_model(): m = swmmio.Model(MODEL_FULL_FEATURES_XY) target_img_pth = os.path.join(DATA_PATH, 'test-draw-model.png') sg.draw_model(m, file_path=target_img_pth) assert os.path.exists(target_img_pth) os.remove(target_img_pth)
def test_coordinates(): m = swmmio.Model(MODEL_FULL_FEATURES_XY) coordinates = m.inp.coordinates # coordinates.to_csv(df_test_coordinates_csv) test_coords = pd.read_csv(df_test_coordinates_csv, index_col=0) assert (coordinates.equals(test_coords))
def test_web_map_01(): m = swmmio.Model(MODEL_A_PATH, crs="+init=EPSG:2817") with tempfile.TemporaryDirectory() as tempdir: fname = os.path.join(tempdir, 'test-map.html') sg.create_map(m, filename=fname) assert os.path.exists(fname)
def __init__(self, model1=None, model2=None): """ diff of all INP sections between two models :param model1: :param model2: >>> from swmmio.tests.data import MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B >>> mydiff = INPDiff(MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B) >>> mydiff.diffs['[XSECTIONS]'] Shape Geom1 Geom2 Geom3 Geom4 Barrels ; Comment Origin Link 1:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 2:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 3:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 4:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 5:J1 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp <BLANKLINE> >>> print(mydiff) """ m1 = model1 m2 = model2 if isinstance(m1, str): m1 = swmmio.Model(m1) if isinstance(m2, str): m2 = swmmio.Model(m2) self.m1 = m1 self.m2 = m2 self.diffs = OrderedDict() m1_sects = get_inp_sections_details(m1.inp.path) m2_sects = get_inp_sections_details(m2.inp.path) # get union of sections found, maintain order sects = list(m1_sects.keys()) + list(m2_sects.keys()) seen = set() self.all_sections = [ x for x in sects if not (x in seen or seen.add(x)) ] self.all_inp_objects = OrderedDict(m1_sects) self.all_inp_objects.update(m2_sects) for section in self.all_sections: if section not in problem_sections: # calculate the changes in the current section changes = INPSectionDiff(m1, m2, section) self.diffs[section] = changes
def test_inflow_dwf_dataframe(): m = swmmio.Model(MODEL_XSECTION_ALT_03) dwf = dataframe_from_inp(m.inp.path, 'dwf') assert (dwf.loc['dummy_node2', 'AverageValue'] == pytest.approx(0.000275, 0.0001)) inf = m.inp.inflows assert (inf.loc['dummy_node2', 'Time Series'] == 'my_time_series') assert (pd.isna(inf.loc['dummy_node6', 'Time Series']))
def test_model_to_networkx(): m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) G = m.network assert (G['J2']['J3']['C2.1']['Length'] == 666) assert (G['J1']['J2']['C1:C2']['Length'] == 244.63) assert (round(G.node['J2']['InvertElev'], 3) == 13.0) links = m.links() assert (len(links) == len(G.edges()))
def test_draw_red_and_grey_nodes(): m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) target_img_pth = os.path.join(DATA_PATH, 'test-draw-model.png') nodes = m.nodes() nodes['draw_color'] = '#787882' nodes.loc[['J1', 'J2', 'J3'], 'draw_color'] = '#ff0000' nodes['draw_size'] = nodes['InvertElev'] * 3 sg.draw_model(conduits=m.conduits(), nodes=nodes, file_path=target_img_pth) assert os.path.exists(target_img_pth) os.remove(target_img_pth)
def test_centroid_and_bbox_from_coords(): m = swmmio.Model(MODEL_A_PATH, crs="+init=EPSG:2817") m.to_crs("+init=EPSG:4326") c, bbox = centroid_and_bbox_from_coords(m.nodes.dataframe['coords']) assert c == (-70.97068150884797, 43.74695249578866) assert bbox == [-70.97068150884797, 43.74695249578866, -70.97068150884797, 43.74695249578866] c, bbox = centroid_and_bbox_from_coords([(0, 0), (0, 10), (10, 10), (10, 0)]) assert c == (5, 5) assert bbox == [0, 0, 10, 10]
def test_centroid_and_bbox_from_coords(): m = swmmio.Model(MODEL_A_PATH, crs="EPSG:2817") m.to_crs("EPSG:4326") c, bbox = centroid_and_bbox_from_coords(m.nodes.dataframe['coords']) assert c == pytest.approx((-70.97068, 43.74695), rel=1e-3) assert bbox == pytest.approx((-70.97068, 43.74695, -70.97068, 43.74695), rel=1e-3) c, bbox = centroid_and_bbox_from_coords([(0, 0), (0, 10), (10, 10), (10, 0)]) assert c == (5, 5) assert bbox == [0, 0, 10, 10]
def test_change_crs(): m = swmmio.Model(MODEL_A_PATH, crs="+init=EPSG:2817") v1 = m.inp.vertices v2 = change_crs(m.inp.vertices, m.crs, "+init=EPSG:4326") assert v1.shape == v2.shape s = """ Name X Y J4-001.1 -70.959386 43.732851 J4-001.1 -70.958415 43.732578 J4-001.1 -70.959423 43.730452 J2-095.1 -70.951378 43.767796 """ v2_test = pd.read_csv(StringIO(s), index_col=0, delim_whitespace=True, skiprows=[0]) assert v2.to_string() == v2_test.to_string()
def test_nodes_dataframe(): m = swmmio.Model(MODEL_XSECTION_ALT_01) nodes = m.nodes() node_ids_01 = [ 'dummy_node1', 'dummy_node2', 'dummy_node3', 'dummy_node4', 'dummy_node5', 'dummy_node6', 'dummy_outfall' ] assert (list(nodes.index) == node_ids_01) assert (nodes.loc['dummy_node1', 'InvertElev'] == -10.99) assert (nodes.loc['dummy_node2', 'MaxDepth'] == 20) assert (nodes.loc['dummy_node3', 'X'] == -4205.457) assert (nodes.loc['dummy_node4', 'MaxDepth'] == 12.59314) assert (nodes.loc['dummy_node5', 'PondedArea'] == 73511)
def __init__(self, inp_file_path, out_file_path): if not inp_file_path: self.inp_file_path = config['inp_file_path'] else: self.inp_file_path = inp_file_path if not out_file_path: self.out_file_path = config['out_file_path'] else: self.out_file_path = out_file_path #initialize some baseline dataframes self.baseline = swmmio.core.inp(inp_file_path) self.baselineModel = swmmio.Model(inp_file_path) self.baselineLinks = self.baselineModel.links.dataframe self.baselineNodes = self.baselineModel.nodes.dataframe self.timeseries = self.baseline.timeseries # any changes should be made on deep copy of self.timeseries self.inflows = self.baseline.inflows
def test_change_crs(): m = swmmio.Model(MODEL_A_PATH, crs="EPSG:2817") v1 = m.inp.vertices v2 = change_crs(m.inp.vertices, m.crs, "WGS84") assert v1.shape == v2.shape s = """ Name X Y J4-001.1 -70.959386 43.732851 J4-001.1 -70.958415 43.732578 J4-001.1 -70.959423 43.730452 J2-095.1 -70.951378 43.767796 """ v2_test = pd.read_csv(StringIO(s), index_col=0, delim_whitespace=True, skiprows=[0]) assert v2['X'].values == pytest.approx(v2_test['X'].values, rel=1e-3) assert v2['Y'].values == pytest.approx(v2_test['Y'].values, rel=1e-3)
def get_sim_duration_and_report_step(): # initialize a model model object model = swmmio.Model(data_dir + 'scenario.inp') inp_start_date = model.inp.options.loc["START_DATE"].values[0] inp_start_time = model.inp.options.loc["START_TIME"].values[0] inp_end_date = model.inp.options.loc["END_DATE"].values[0] inp_end_time = model.inp.options.loc["END_TIME"].values[0] start_time = datetime.datetime.strptime( inp_start_date + ' ' + inp_start_time, '%d/%m/%Y %H:%M:%S') end_time = datetime.datetime.strptime(inp_end_date + ' ' + inp_end_time, '%d/%m/%Y %H:%M:%S') report_step = datetime.datetime.strptime( model.inp.options.loc["REPORT_STEP"].values[0], '%H:%M:%S').minute simulation_duration = int((end_time - start_time).total_seconds() / 60) return simulation_duration, report_step
def test_create_dataframeRPT(): m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) depth_summ = swmmio.create_dataframeRPT(m.rpt.path, "Node Depth Summary") flood_summ = swmmio.create_dataframeRPT(m.rpt.path, "Node Flooding Summary") inflo_summ = swmmio.create_dataframeRPT(m.rpt.path, "Node Inflow Summary") print('\n', depth_summ) print(inflo_summ) print(flood_summ) assert (inflo_summ.loc['J3', 'TotalInflowV'] == 6.1) assert (inflo_summ.loc['J1', 'MaxTotalInflow'] == 3.52) assert (depth_summ.loc['J3', 'MaxNodeDepth'] == 1.64) assert (depth_summ.loc['4', 'MaxNodeDepth'] == 0.87) # need to ensure indices are strings always assert (flood_summ.loc[5, 'TotalFloodVol'] == 0)
def make_inp_file(user_input): # initialize a baseline model object baseline = swmmio.Model(data_dir + 'baseline.inp') # create a dataframe of the model's subcatchments subs = baseline.inp.subcatchments # reads updates to model from user input and updates the swmmio model for update in user_input["model_updates"]: # update the outlet_id in the row of subcatchment_id subs.loc[update["subcatchment_id"], ['Outlet']] = update['outlet_id'] baseline.inp.subcatchments = subs # set the rain gage from user input as raingage for all subcatchments scenario_rain_gage_name = 'RainGage' + '_' + str(user_input["rain_event"]["return_period"]) for i, row in subs.iterrows(): subs.at[i, 'Raingage'] = scenario_rain_gage_name # Save the new model with the adjusted data new_file_path = data_dir + 'scenario.inp' baseline.inp.save(new_file_path)
def build(self, baseline_dir, target_path): """ build a complete INP file with the build instructions committed to a baseline model. """ basemodel = swmmio.Model(baseline_dir) allheaders = get_inp_sections_details(basemodel.inp.path) # new_inp = os.path.join(target_dir, 'model.inp') with open(target_path, 'w') as f: for section, _ in allheaders.items(): # check if the section is not in problem_sections and there are changes # in self.instructions and commit changes to it from baseline accordingly if (section not in problem_sections and allheaders[section]['columns'] != ['blob'] and section in self.instructions): # df of baseline model section basedf = dataframe_from_bi(basemodel.inp.path, section) basedf[';'] = ';' # grab the changes to changes = self.instructions[section] # remove elements that have alterations and or tagged for removal remove_ids = changes.removed.index | changes.altered.index new_section = basedf.drop(remove_ids) # add elements new_section = pd.concat( [new_section, changes.altered, changes.added]) else: # section is not well understood or is problematic, just blindly copy new_section = dataframe_from_bi(basemodel.inp.path, section=section) new_section[';'] = ';' # write the section vc_utils.write_inp_section(f, allheaders, section, new_section)
def test_model_section(): m = swmmio.Model(MODEL_FULL_FEATURES_XY) def pumps_old_method(model): """ collect all useful and available data related model pumps and organize in one dataframe. """ # check if this has been done already and return that data accordingly if model._pumps_df is not None: return model._pumps_df # parse out the main objects of this model inp = model.inp rpt = model.rpt # create dataframes of relevant sections from the INP pumps_df = create_dataframeINP(inp.path, "[PUMPS]", comment_cols=False) if pumps_df.empty: return pd.DataFrame() # add conduit coordinates xys = pumps_df.apply( lambda r: swmmio.get_link_coords(r, inp.coordinates, inp.vertices), axis=1) df = pumps_df.assign(coords=xys.map(lambda x: x[0])) df.InletNode = df.InletNode.astype(str) df.OutletNode = df.OutletNode.astype(str) model._pumps_df = df return df pumps_old_method = pumps_old_method(m) pumps = m.pumps() assert (pumps_old_method.equals(pumps))
def test_conduits_dataframe(test_model_01): m = swmmio.Model(MODEL_FULL_FEATURES_PATH) conduits = m.conduits() assert (list(conduits.index) == ['C1:C2'])
def test_model_02(): return swmmio.Model(MODEL_FULL_FEATURES__NET_PATH)
def test_model_01(): return swmmio.Model(MODEL_FULL_FEATURES_XY)
def merge_models(inp1, inp2, target='merged_model.inp'): """ Merge two separate swmm models into one model. This creates a diff, ignores removed sections, and uses inp1 settings where conflicts exist (altered sections in diff) :param inp1: swmmio.Model.inp object to be combined with inp2 :param inp2: swmmio.Model.inp object to be combined with inp1 :param target: path of new model :return: path to target """ # model object to store resulting merged model m3 = swmmio.Model(inp1) inp_diff = INPDiff(inp1, inp2) with open(target, 'w') as newf: for section, _ in inp_diff.all_inp_objects.items(): # don't consider the "removed" parts of the diff # print('{}: {}'.format(section,inp_diff.all_inp_objects[section]['columns'])) # check if the section is not in problem_sections and there are changes # in self.instructions and commit changes to it from baseline accordingly col_order = [] if (section not in problem_sections and inp_diff.all_inp_objects[section]['columns'] != ['blob'] and section in inp_diff.diffs): # df of baseline model section basedf = dataframe_from_inp( m3.inp.path, section, additional_cols=[';', 'Comment', 'Origin']) basedf[';'] = ';' col_order = basedf.columns # grab the changes to changes = inp_diff.diffs[section] # remove elements that have alterations keep ones tagged for removal # (unchanged, but not present in m2) remove_ids = changes.altered.index new_section = basedf.drop(remove_ids) # add elements new_section = pd.concat( [new_section, changes.altered, changes.added], axis=0, sort=False) else: # section is not well understood or is problematic, just blindly copy new_section = dataframe_from_inp( m3.inp.path, section, additional_cols=[';', 'Comment', 'Origin']) new_section[';'] = ';' # print ('dealing with confusing section: {}\n{}'.format(section, new_section)) # print(new_section.head()) # write the section new_section = new_section[col_order] new_section[';'] = ';' vc_utils.write_inp_section(newf, inp_diff.all_inp_objects, section, new_section, pad_top=True) return target