def test_remove_model_section(): with tempfile.TemporaryDirectory() as tempdir: m1 = swmmio.Model(MODEL_A_PATH) # create a copy of the model without subcatchments # m1.inp.infiltration = m1.inp.infiltration.iloc[0:0] m1.inp.subcatchments = m1.inp.subcatchments.iloc[0:0] # m1.inp.subareas = m1.inp.subareas.iloc[0:0] # m1.inp.polygons = m1.inp.polygons.iloc[0:0] # save to temp location temp_inp = os.path.join(tempdir, f'{m1.inp.name}.inp') m1.inp.save(temp_inp) m2 = swmmio.Model(temp_inp) sects1 = get_inp_sections_details(m1.inp.path) sects2 = get_inp_sections_details(m2.inp.path) # confirm saving a copy retains all sections except those removed assert ['SUBCATCHMENTS'] == [x for x in sects1 if x not in sects2] # confirm subcatchments returns an empty df assert m2.subcatchments.dataframe.empty
def __init__(self, model1=None, model2=None): m1 = model1 m2 = model2 if isinstance(m1, str): m1 = swmmio.Model(m1) if isinstance(m2, str): m2 = swmmio.Model(m2) self.m1 = m1 self.m2 = m2 self.diffs = OrderedDict() m1_sects = get_inp_sections_details(m1.inp.path) m2_sects = get_inp_sections_details(m2.inp.path) # get union of sections found, maintain order sects = list(m1_sects.keys()) + list(m2_sects.keys()) seen = set() self.all_sections = [ x for x in sects if not (x in seen or seen.add(x)) ] self.all_inp_objects = OrderedDict(m1_sects) self.all_inp_objects.update(m2_sects) for section in self.all_sections: if section not in problem_sections: # calculate the changes in the current section changes = INPSectionDiff(m1, m2, section) self.diffs[section] = changes
def create_inp_build_instructions(inpA, inpB, path, filename, comments=''): """ pass in two inp file paths and produce a spreadsheet showing the differences found in each of the INP sections. These differences should then be used whenever we need to rebuild this model from the baseline reference model. Note: this should be split into a func that creates a overall model "diff" that can then be written as a BI file or used programmatically """ allsections_a = get_inp_sections_details(inpA) modela = swmmio.Model(inpA) modelb = swmmio.Model(inpB) # create build insructions folder if not os.path.exists(path): os.makedirs(path) filepath = os.path.join(path, filename) + '.txt' problem_sections = ['TITLE', 'CURVES', 'TIMESERIES', 'RDII', 'HYDROGRAPHS'] with open(filepath, 'w') as newf: # write meta data metadata = { # 'Baseline Model':modela.inp.path, # 'ID':filename, 'Parent Models': { 'Baseline': { inpA: vc_utils.modification_date(inpA) }, 'Alternatives': { inpB: vc_utils.modification_date(inpB) } }, 'Log': { filename: comments } } # print metadata vc_utils.write_meta_data(newf, metadata) for section, _ in allsections_a.items(): if section not in problem_sections: # calculate the changes in the current section changes = INPSectionDiff(modela, modelb, section) data = pd.concat( [changes.removed, changes.added, changes.altered], axis=0, sort=False) # vc_utils.write_excel_inp_section(excelwriter, allsections_a, section, data) vc_utils.write_inp_section( newf, allsections_a, section, data, pad_top=False, na_fill='NaN' ) # na fill fixes SNOWPACK blanks spaces issue return BuildInstructions(filepath)
def create_dataframe_multi_index(inp_path, section='CURVES'): # format the section header for look up in headers OrderedDict sect = remove_braces(section).upper() # get list of all section headers in inp to use as section ending flags headers = get_inp_sections_details(inp_path, include_brackets=False) if sect not in headers: warnings.warn(f'{sect} section not found in {inp_path}') return pd.DataFrame() # extract the string and read into a dataframe start_string = format_inp_section_header(section) end_strings = [format_inp_section_header(h) for h in headers.keys()] s = extract_section_of_file(inp_path, start_string, end_strings) cols = headers[sect]['columns'] f = StringIO(s) data = [] for line in f.readlines(): items = line.strip().split() if len(items) == 3: items = [items[0], None, items[1], items[2]] if len(items) == 4: data.append(items) df = pd.DataFrame(data=data, columns=cols) df = df.set_index(['Name', 'Type']) return df
def __init__(self, model1=None, model2=None): """ diff of all INP sections between two models :param model1: :param model2: >>> from swmmio.tests.data import MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B >>> mydiff = INPDiff(MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES_XY_B) >>> mydiff.diffs['[XSECTIONS]'] Shape Geom1 Geom2 Geom3 Geom4 Barrels ; Comment Origin Link 1:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 2:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 3:4 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 4:5 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp 5:J1 CIRCULAR 1 0 0 0 1.0 ; Removed model_full_features_b.inp <BLANKLINE> >>> print(mydiff) """ m1 = model1 m2 = model2 if isinstance(m1, str): m1 = swmmio.Model(m1) if isinstance(m2, str): m2 = swmmio.Model(m2) self.m1 = m1 self.m2 = m2 self.diffs = OrderedDict() m1_sects = get_inp_sections_details(m1.inp.path) m2_sects = get_inp_sections_details(m2.inp.path) # get union of sections found, maintain order sects = list(m1_sects.keys()) + list(m2_sects.keys()) seen = set() self.all_sections = [ x for x in sects if not (x in seen or seen.add(x)) ] self.all_inp_objects = OrderedDict(m1_sects) self.all_inp_objects.update(m2_sects) for section in self.all_sections: if section not in problem_sections: # calculate the changes in the current section changes = INPSectionDiff(m1, m2, section) self.diffs[section] = changes
def replace_inp_section(inp_path, modified_section_header, new_data): """ modify an existing inp file by passing in new data (Pandas Dataframe) and the section header that should be modified. This function will overwrite all data in the old section with the passed data :param inp_path: path to inp file to be changed :param modified_section_header: section for which data should be change :param new_data: pd.DataFrame of data to overwrite data in the modified section :return: swmmio.Model instantiated with modified inp file """ sections = get_inp_sections_details(inp_path) m = swmmio.Model(inp_path) with tempfile.TemporaryDirectory() as tempdir: with open(inp_path) as oldf: tmp_inp_path = os.path.join(tempdir, f'{m.inp.name}.inp') with open(tmp_inp_path, 'w') as new: # write each line as is from the original model until we find the # header of the section we wish to overwrite found_section = False found_next_section = False for line in oldf: if modified_section_header in line: # write the replacement data in the new file now write_inp_section(new, sections, modified_section_header, new_data, pad_top=False) found_section = True if (found_section and any(es in line for es in sections.keys()) and modified_section_header not in line): found_next_section = True if found_next_section or not found_section: # write the lines from the original file # if we haven't found the section to modify. # if we have found the section and we've found the NEXT section # continue writing original file's lines new.write(line) if not found_section: # the header doesn't exist in the old model # so we should append it to the bottom of file write_inp_section(new, sections, modified_section_header, new_data) # rename files and remove old if we should overwrite os.remove(inp_path) shutil.copy2(tmp_inp_path, inp_path) return swmmio.Model(inp_path)
def dataframe_from_inp(inp_path, section, additional_cols=None, quote_replace=' ', **kwargs): """ create a dataframe from a section of an INP file :param inp_path: :param section: :param additional_cols: :param skip_headers: :param quote_replace: :return: """ # format the section header for look up in headers OrderedDict sect = remove_braces(section).upper() # get list of all section headers in inp to use as section ending flags headers = get_inp_sections_details(inp_path, include_brackets=False) if sect not in headers: warnings.warn(f'{sect} section not found in {inp_path}') return pd.DataFrame() # extract the string and read into a dataframe start_string = format_inp_section_header(section) end_strings = [format_inp_section_header(h) for h in headers.keys()] s = extract_section_of_file(inp_path, start_string, end_strings, **kwargs) # replace occurrences of double quotes "" s = s.replace('""', quote_replace) # and get the list of columns to use for parsing this section # add any additional columns needed for special cases (build instructions) additional_cols = [] if additional_cols is None else additional_cols cols = headers[sect]['columns'] + additional_cols if headers[sect]['columns'][0] == 'blob': # return the whole row, without specific col headers return pd.read_csv(StringIO(s), delim_whitespace=False) else: try: df = pd.read_csv(StringIO(s), header=None, delim_whitespace=True, skiprows=[0], index_col=0, names=cols) except IndexError: print( f'failed to parse {section} with cols: {cols}. head:\n{s[:500]}' ) raise return df
def test_modify_inp_sections(): m1 = swmmio.Model(MODEL_A_PATH) with tempfile.TemporaryDirectory() as tempdir: cp_path = os.path.join(tempdir, f'{m1.inp.name}.inp') m1.inp.save(cp_path) # first confirm saving a copy produces same content assert filecmp.cmp(MODEL_A_PATH, cp_path) # change a section m2_path = os.path.join(tempdir, f'{m1.inp.name}_nodes_99.inp') m1.inp.junctions['InvertElev'] = 99 m1.inp.save(m2_path) m2 = swmmio.Model(m2_path) # confirm the change worked assert m2.inp.junctions['InvertElev'].min() == 99 assert m2.inp.junctions['InvertElev'].max() == 99 # confirm all sections are in the new model sects1 = get_inp_sections_details(m1.inp.path) sects2 = get_inp_sections_details(m2.inp.path) assert all([x in sects1 for x in sects2])
def headers(self): """ Return all headers and associated column names found in the INP file. """ if self._inp_section_details is None: self._inp_section_details = get_inp_sections_details(self.path, include_brackets=True, ) # select the correct infiltration column names infil_type = self.options.loc['INFILTRATION', 'Value'] infil_cols = INFILTRATION_COLS[infil_type] # overwrite the dynamic sections with proper header cols self._inp_section_details['[INFILTRATION]'] = list(infil_cols) return self._inp_section_details
def __init__(self, build_instr_file=None): # create a change object for each section that is different from baseline self.instructions = {} self.metadata = {} if build_instr_file: # read the instructions and create a dictionary of Change objects allheaders = get_inp_sections_details(build_instr_file) instructions = {} for section, _ in allheaders.items(): change = INPSectionDiff(build_instr_file=build_instr_file, section=section) instructions.update({section: change}) self.instructions = instructions # read the meta data self.metadata = vc_utils.read_meta_data(build_instr_file)
def build(self, baseline_dir, target_path): """ build a complete INP file with the build instructions committed to a baseline model. """ basemodel = swmmio.Model(baseline_dir) allheaders = get_inp_sections_details(basemodel.inp.path) # new_inp = os.path.join(target_dir, 'model.inp') with open(target_path, 'w') as f: for section, _ in allheaders.items(): # check if the section is not in problem_sections and there are changes # in self.instructions and commit changes to it from baseline accordingly if (section not in problem_sections and allheaders[section]['columns'] != ['blob'] and section in self.instructions): # df of baseline model section basedf = dataframe_from_bi(basemodel.inp.path, section) basedf[';'] = ';' # grab the changes to changes = self.instructions[section] # remove elements that have alterations and or tagged for removal remove_ids = changes.removed.index | changes.altered.index new_section = basedf.drop(remove_ids) # add elements new_section = pd.concat( [new_section, changes.altered, changes.added]) else: # section is not well understood or is problematic, just blindly copy new_section = dataframe_from_bi(basemodel.inp.path, section=section) new_section[';'] = ';' # write the section vc_utils.write_inp_section(f, allheaders, section, new_section)
def __call__(self): """ collect all useful and available data related to the conduits and organize in one dataframe. >>> model = swmmio.Model(MODEL_FULL_FEATURES__NET_PATH) >>> conduits_section = ModelSection(model, 'conduits') >>> conduits_section() """ # concat inp sections with unique element IDs headers = get_inp_sections_details(self.inp.path) dfs = [ dataframe_from_inp(self.inp.path, sect) for sect in self.inp_sections if sect.upper() in headers ] # return empty df if no inp sections found if len(dfs) == 0: return pd.DataFrame() df = pd.concat(dfs, axis=0, sort=False) # join to this any sections with matching IDs (e.g. XSECTIONS) for sect in self.join_sections: rsuffix = f"_{sect.replace(' ', '_')}" df = df.join(dataframe_from_inp(self.inp.path, sect), rsuffix=rsuffix) if df.empty: return df # if there is an RPT available, grab relevant sections if self.rpt: for rpt_sect in self.rpt_sections: df = df.join(dataframe_from_rpt(self.rpt.path, rpt_sect)) # add coordinates if self.geomtype == 'point': df = df.join(self.inp.coordinates[['X', 'Y']]) xys = df.apply(lambda r: nodexy(r), axis=1) df = df.assign(coords=xys) elif self.geomtype == 'linestring': # add conduit coordinates xys = df.apply(lambda r: get_link_coords(r, self.inp.coordinates, self.inp.vertices), axis=1) df = df.assign(coords=xys.map(lambda x: x[0])) # make inlet/outlet node IDs string type df.InletNode = df.InletNode.astype(str) df.OutletNode = df.OutletNode.astype(str) elif self.geomtype == 'polygon': p = self.inp.polygons # take stacked coordinates and orient in list of tuples, xys = p.groupby(by=p.index).apply( lambda r: [(xy['X'], xy['Y']) for ix, xy in r.iterrows()]) # copy the first point to the last position xys = xys.apply(lambda r: r + [r[0]]) df = df.assign(coords=xys) # confirm index name is string df = df.rename(index=str) # trim to desired columns if self.columns is not None: df = df[[c for c in self.columns if c in df.columns]] self._df = df return df