def conduits(self): """ collect all useful and available data related model conduits and organize in one dataframe. """ # check if this has been done already and return that data accordingly if self._conduits_df is not None: return self._conduits_df # parse out the main objects of this model inp = self.inp rpt = self.rpt # create dataframes of relevant sections from the INP conduits_df = dataframe_from_inp(inp.path, "CONDUITS") xsections_df = dataframe_from_inp(inp.path, "XSECTIONS") conduits_df = conduits_df.join(xsections_df) if rpt: # create a dictionary holding data from an rpt file, if provided link_flow_df = dataframe_from_rpt(rpt.path, "Link Flow Summary") conduits_df = conduits_df.join(link_flow_df) # add conduit coordinates xys = conduits_df.apply(lambda r: get_link_coords( r, self.inp.coordinates, self.inp.vertices), axis=1) df = conduits_df.assign(coords=xys.map(lambda x: x[0])) # add conduit up/down inverts and calculate slope elevs = self.nodes()[['InvertElev']] df = pd.merge(df, elevs, left_on='InletNode', right_index=True, how='left') df = df.rename(index=str, columns={"InvertElev": "InletNodeInvert"}) df = pd.merge(df, elevs, left_on='OutletNode', right_index=True, how='left') df = df.rename(index=str, columns={"InvertElev": "OutletNodeInvert"}) df['UpstreamInvert'] = df.InletNodeInvert + df.InOffset df['DownstreamInvert'] = df.OutletNodeInvert + df.OutOffset df['SlopeFtPerFt'] = (df.UpstreamInvert - df.DownstreamInvert) / df.Length df.InletNode = df.InletNode.astype(str) df.OutletNode = df.OutletNode.astype(str) self._conduits_df = df return df
def junctions(self): """ Get/set junctions section of the INP file. :return: junctions section of the INP file :rtype: pandas.DataFrame Examples: >>> import swmmio >>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH >>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) >>> model.inp.junctions InvertElev MaxDepth InitDepth SurchargeDepth PondedArea Name J3 6.547 15 0 0 0 1 17.000 0 0 0 0 2 17.000 0 0 0 0 3 16.500 0 0 0 0 4 16.000 0 0 0 0 5 15.000 0 0 0 0 J2 13.000 15 0 0 0 """ if self._junctions_df is None: self._junctions_df = dataframe_from_inp(self.path, "JUNCTIONS") return self._junctions_df
def weirs(self): """ Get/set weirs section of the INP file. """ if self._weirs_df is None: self._weirs_df = dataframe_from_inp(self.path, "[WEIRS]") return self._weirs_df
def orifices(self): """ Get/set orifices section of the INP file. """ if self._orifices_df is None: self._orifices_df = dataframe_from_inp(self.path, "[ORIFICES]") return self._orifices_df
def pumps(self): """ Get/set pumps section of the INP file. """ if self._pumps_df is None: self._pumps_df = dataframe_from_inp(self.path, "[PUMPS]") return self._pumps_df
def xsections(self): """ Get/set pumps section of the INP file. """ if self._xsections_df is None: self._xsections_df = dataframe_from_inp(self.path, "[XSECTIONS]") return self._xsections_df
def conduits(self): """ Get/set conduits section of the INP file. :return: Conduits section of the INP file :rtype: pandas.DataFrame Examples: >>> import swmmio >>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH >>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) >>> model.inp.conduits[['InletNode', 'OutletNode', 'Length', 'ManningN']] InletNode OutletNode Length ManningN Name C1:C2 J1 J2 244.63 0.01 C2.1 J2 J3 666.00 0.01 1 1 4 400.00 0.01 2 4 5 400.00 0.01 3 5 J1 400.00 0.01 4 3 4 400.00 0.01 5 2 5 400.00 0.01 """ if self._conduits_df is None: self._conduits_df = dataframe_from_inp(self.path, "[CONDUITS]") return self._conduits_df
def test_modify_model(): from swmmio.utils.modify_model import replace_inp_section from swmmio import Model import pandas as pd # initialize a baseline model object baseline = Model(MODEL_FULL_FEATURES_XY) of_test = pd.read_csv(OUTFALLS_MODIFIED, index_col=0) rise = 10.0 # set the starting sea level rise condition # create a dataframe of the model's outfalls outfalls = dataframe_from_inp(baseline.inp.path, '[OUTFALLS]') # add the current rise to the outfalls' stage elevation outfalls['OutfallType'] = 'FIXED' outfalls.loc[:, 'InvertElev'] = pd.to_numeric( outfalls.loc[:, 'InvertElev']) + rise of_test.loc[:, 'InvertElev'] = pd.to_numeric(of_test.loc[:, 'InvertElev']) with tempfile.TemporaryDirectory() as tempdir: # copy the base model into a new directory newdir = os.path.join(tempdir, str(rise)) makedirs(newdir) newfilepath = os.path.join( newdir, baseline.inp.name + "_" + str(rise) + '_SLR.inp') shutil.copyfile(baseline.inp.path, newfilepath) # Overwrite the OUTFALLS section of the new model with the adjusted data replace_inp_section(newfilepath, '[OUTFALLS]', outfalls) m2 = Model(newfilepath) of2 = m2.inp.outfalls assert (of2.loc['J4', 'InvertElev'].round(1) == of_test.loc[ 'J4', 'InvertElev'].round(1))
def search_for_duplicates(inp_path, verbose=False): """ scan an inp file and determine if any element IDs are duplicated in any section. Method: count the uniques and compare to total length """ headers = swmmio.utils.text.get_inp_sections_details(inp_path)['headers'] dups_found = False for header, cols, in headers.items(): if cols != 'blob': df = dataframe_from_inp(inp_path, section=header) elements = df.index n_unique = len(elements.unique()) #number of unique elements n_total = len(elements) #total number of elements if verbose: print('{} -> (uniques, total) -> ({}, {})'.format( header, n_unique, n_total)) if n_unique != n_total: dups = ', '.join( df[df.index.duplicated()].index.unique().tolist()) print('duplicate found in {}\nsection: {}\n{}'.format( inp_path, header, dups)) dups_found = True return dups_found
def subareas(self): """ Get/set subareas section of the INP file. """ if self._subareas_df is None: self._subareas_df = dataframe_from_inp(self.path, "[SUBAREAS]") return self._subareas_df
def test_inflow_dwf_dataframe(): m = swmmio.Model(MODEL_XSECTION_ALT_03) dwf = dataframe_from_inp(m.inp.path, 'dwf') assert (dwf.loc['dummy_node2', 'AverageValue'] == pytest.approx(0.000275, 0.0001)) inf = m.inp.inflows assert (inf.loc['dummy_node2', 'Time Series'] == 'my_time_series') assert (pd.isna(inf.loc['dummy_node6', 'Time Series']))
def polygons(self): """ get/set polygons section of model :return: dataframe of model coordinates """ if self._polygons_df is not None: return self._polygons_df self._polygons_df = dataframe_from_inp(self.path, '[Polygons]') return self._polygons_df
def vertices(self): """ get/set vertices section of model :return: dataframe of model coordinates """ if self._vertices_df is not None: return self._vertices_df self._vertices_df = dataframe_from_inp(self.path, 'VERTICES') return self._vertices_df
def coordinates(self): """ Get/set coordinates section of model :return: dataframe of model coordinates """ if self._coordinates_df is not None: return self._coordinates_df self._coordinates_df = dataframe_from_inp(self.path, "COORDINATES") return self._coordinates_df
def inflows(self): """ get/set inflows section of model :return: dataframe of model coordinates """ if self._inflows_df is not None: return self._inflows_df inf = dataframe_from_inp(self.path, 'INFLOWS', quote_replace='_!!!!_') self._inflows_df = inf.replace('_!!!!_', pd.np.nan) return self._inflows_df
def drop_invalid_model_elements(inp): """ Identify references to elements in the model that are undefined and remove them from the model. These should coincide with warnings/errors produced by SWMM5 when undefined elements are referenced in links, subcatchments, and controls. :param model: swmmio.Model :return: >>> import swmmio >>> from swmmio.tests.data import MODEL_FULL_FEATURES_INVALID >>> m = swmmio.Model(MODEL_FULL_FEATURES_INVALID) >>> drop_invalid_model_elements(m.inp) ['InvalidLink2', 'InvalidLink1'] >>> m.inp Index(['C1:C2', 'C2.1', '1', '2', '4', '5'], dtype='object', name='Name') """ juncs = dataframe_from_inp(inp.path, "[JUNCTIONS]").index.tolist() outfs = dataframe_from_inp(inp.path, "[OUTFALLS]").index.tolist() stors = dataframe_from_inp(inp.path, "[STORAGE]").index.tolist() nids = juncs + outfs + stors # drop links with bad refs to inlet/outlet nodes from swmmio.utils.functions import find_invalid_links inv_conds = find_invalid_links(inp, nids, 'conduits', drop=True) inv_pumps = find_invalid_links(inp, nids, 'pumps', drop=True) inv_orifs = find_invalid_links(inp, nids, 'orifices', drop=True) inv_weirs = find_invalid_links(inp, nids, 'weirs', drop=True) # drop other parts of bad links invalid_links = inv_conds + inv_pumps + inv_orifs + inv_weirs inp.xsections = inp.xsections.loc[~inp.xsections.index.isin(invalid_links)] # drop invalid subcats and their related components invalid_subcats = inp.subcatchments.index[~inp.subcatchments['Outlet']. isin(nids)] inp.subcatchments = inp.subcatchments.loc[~inp.subcatchments.index. isin(invalid_subcats)] inp.subareas = inp.subareas.loc[~inp.subareas.index.isin(invalid_subcats)] inp.infiltration = inp.infiltration.loc[~inp.infiltration.index. isin(invalid_subcats)] return invalid_links + invalid_subcats
def subcatchments(self): """ Get/set subcatchments section of the INP file. :return: subcatchments section of the INP file :rtype: pandas.DataFrame Examples: """ if self._subcatchments_df is None: self._subcatchments_df = dataframe_from_inp(self.path, "[SUBCATCHMENTS]") return self._subcatchments_df
def files(self): """ Get/set files section of the INP file. :return: files section of the INP file :rtype: pandas.DataFrame Examples: """ if self._files_df is None: self._files_df = dataframe_from_inp(self.path, "[FILES]") return self._files_df.reset_index()
def storage(self): """ Get/set storage section of the INP file. :return: storage section of the INP file :rtype: pandas.DataFrame Examples: """ if self._storage_df is None: self._storage_df = dataframe_from_inp(self.path, "[STORAGE]") return self._storage_df
def infiltration(self): """ Get/set infiltration section of the INP file. >>> import swmmio >>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH >>> m = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) >>> m.inp.infiltration MaxRate MinRate Decay DryTime MaxInfil Subcatchment S1 3.0 0.5 4 7 0 S2 3.0 0.5 4 7 0 S3 3.0 0.5 4 7 0 S4 3.0 0.5 4 7 0 """ if self._infiltration_df is None: self._infiltration_df = dataframe_from_inp(self.path, "infiltration") return self._infiltration_df
def outfalls(self): """ Get/set outfalls section of the INP file. :return: outfalls section of the INP file :rtype: pandas.DataFrame Examples: >>> import swmmio >>> from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH >>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) >>> model.inp.outfalls InvertElev OutfallType StageOrTimeseries TideGate Name J4 0 FREE NO NaN """ if self._outfalls_df is None: self._outfalls_df = dataframe_from_inp(self.path, "[OUTFALLS]") return self._outfalls_df
def __call__(self): """ collect all useful and available data related to the conduits and organize in one dataframe. >>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) >>> conduits_section = ModelSection(model, 'conduits') >>> conduits_section() """ # concat inp sections with unique element IDs headers = get_inp_sections_details(self.inp.path) dfs = [ dataframe_from_inp(self.inp.path, sect) for sect in self.inp_sections if sect.upper() in headers ] # return empty df if no inp sections found if len(dfs) == 0: return pd.DataFrame() df = pd.concat(dfs, axis=0, sort=False) # join to this any sections with matching IDs (e.g. XSECTIONS) for sect in self.join_sections: rsuffix = f"_{sect.replace(' ', '_')}" df = df.join(dataframe_from_inp(self.inp.path, sect), rsuffix=rsuffix) if df.empty: return df # if there is an RPT available, grab relevant sections if self.rpt: for rpt_sect in self.rpt_sections: df = df.join(dataframe_from_rpt(self.rpt.path, rpt_sect)) # add coordinates if self.geomtype == 'point': df = df.join(self.inp.coordinates[['X', 'Y']]) xys = df.apply(lambda r: nodexy(r), axis=1) df = df.assign(coords=xys) elif self.geomtype == 'linestring': # add conduit coordinates xys = df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1) df = df.assign(coords=xys.map(lambda x: x[0])) # make inlet/outlet node IDs string type df.InletNode = df.InletNode.astype(str) df.OutletNode = df.OutletNode.astype(str) elif self.geomtype == 'polygon': p = self.inp.polygons # take stacked coordinates and orient in list of tuples, xys = p.groupby(by=p.index).apply( lambda r: [(xy['X'], xy['Y']) for ix, xy in r.iterrows()]) # copy the first point to the last position xys = xys.apply(lambda r: r + [r[0]]) df = df.assign(coords=xys) # confirm index name is string df = df.rename(index=str) # trim to desired columns if self.columns is not None: df = df[[c for c in self.columns if c in df.columns]] self._df = df return df
def merge_models(inp1, inp2, target='merged_model.inp'): """ Merge two separate swmm models into one model. This creates a diff, ignores removed sections, and uses inp1 settings where conflicts exist (altered sections in diff) :param inp1: swmmio.Model.inp object to be combined with inp2 :param inp2: swmmio.Model.inp object to be combined with inp1 :param target: path of new model :return: path to target """ # model object to store resulting merged model m3 = swmmio.Model(inp1) inp_diff = INPDiff(inp1, inp2) with open(target, 'w') as newf: for section, _ in inp_diff.all_inp_objects.items(): # don't consider the "removed" parts of the diff # print('{}: {}'.format(section,inp_diff.all_inp_objects[section]['columns'])) # check if the section is not in problem_sections and there are changes # in self.instructions and commit changes to it from baseline accordingly col_order = [] if (section not in problem_sections and inp_diff.all_inp_objects[section]['columns'] != ['blob'] and section in inp_diff.diffs): # df of baseline model section basedf = dataframe_from_inp( m3.inp.path, section, additional_cols=[';', 'Comment', 'Origin']) basedf[';'] = ';' col_order = basedf.columns # grab the changes to changes = inp_diff.diffs[section] # remove elements that have alterations keep ones tagged for removal # (unchanged, but not present in m2) remove_ids = changes.altered.index new_section = basedf.drop(remove_ids) # add elements new_section = pd.concat( [new_section, changes.altered, changes.added], axis=0, sort=False) else: # section is not well understood or is problematic, just blindly copy new_section = dataframe_from_inp( m3.inp.path, section, additional_cols=[';', 'Comment', 'Origin']) new_section[';'] = ';' # print ('dealing with confusing section: {}\n{}'.format(section, new_section)) # print(new_section.head()) # write the section new_section = new_section[col_order] new_section[';'] = ';' vc_utils.write_inp_section(newf, inp_diff.all_inp_objects, section, new_section, pad_top=True) return target
def __init__(self, model1=None, model2=None, section='JUNCTIONS', build_instr_file=None): self.model1 = model1 if model1 else "" self.model2 = model2 if model2 else "" if model1 and model2: df1 = dataframe_from_inp(model1.inp.path, section) df2 = dataframe_from_inp(model2.inp.path, section) df1[';'] = ';' df2[';'] = ';' col_order = list(df2.columns) + ['Comment', 'Origin'] m2_origin_string = os.path.basename(model2.inp.path).replace( ' ', '-') # BUG -> this fails if a df1 or df2 is None i.e. if a section doesn't exist in one model added_ids = df2.index.difference(df1.index) removed_ids = df1.index.difference(df2.index) # find where elements were changed (but kept with same ID) common_ids = df1.index.difference( removed_ids) # original - removed = in common # both dfs concatenated, with matched indices for each element full_set = pd.concat([df1.loc[common_ids], df2.loc[common_ids]], sort=False) # remove whitespace full_set = full_set.apply(lambda x: x.astype(str).str.strip() if x.dtype == "object" else x) # drop dupes on the set, all things that did not changed should have 1 row changes_with_dupes = full_set.drop_duplicates() # duplicate indicies are rows that have changes, isolate these # idx[idx.duplicated()].unique() changed_ids = changes_with_dupes.index[ changes_with_dupes.index.duplicated()].unique( ) # .get_duplicates() added = df2.loc[added_ids].copy() added[ 'Comment'] = 'Added' # from model {}'.format(model2.inp.path) added['Origin'] = m2_origin_string altered = df2.loc[changed_ids].copy() altered[ 'Comment'] = 'Altered' # in model {}'.format(model2.inp.path) altered['Origin'] = m2_origin_string removed = df1.loc[removed_ids].copy() removed[ 'Comment'] = 'Removed' # in model {}'.format(model2.inp.path) removed['Origin'] = m2_origin_string # removed = removed[col_order] self.old = df1 self.new = df2 self.added = added self.removed = removed self.altered = altered if build_instr_file: # if generating from a build instructions file, do this (more efficient) df = dataframe_from_bi(build_instr_file, section=section) self.added = df.loc[df['Comment'] == 'Added'] self.removed = df.loc[df['Comment'] == 'Removed'] self.altered = df.loc[df['Comment'] == 'Altered']