def build(self, baseline_dir, target_path): """ build a complete INP file with the build instructions committed to a baseline model. """ basemodel = swmmio.Model(baseline_dir) allheaders = funcs.complete_inp_headers(basemodel.inp.path) #new_inp = os.path.join(target_dir, 'model.inp') with open (target_path, 'w') as f: for section in allheaders['order']: #check if the section is not in problem_sections and there are changes #in self.instructions and commit changes to it from baseline accordingly if (section not in problem_sections and allheaders['headers'][section] != 'blob' and section in self.instructions): #df of baseline model section basedf = create_dataframeINP(basemodel.inp.path, section) #grab the changes to changes = self.instructions[section] #remove elements that have alterations and or tagged for removal remove_ids = changes.removed.index | changes.altered.index new_section = basedf.drop(remove_ids) #add elements new_section = pd.concat([new_section, changes.altered, changes.added]) else: #section is not well understood or is problematic, just blindly copy new_section = create_dataframeINP(basemodel.inp.path, section=section) #write the section vc_utils.write_inp_section(f, allheaders, section, new_section)
def create_inp_build_instructions(inpA, inpB, path, filename, comments=''): """ pass in two inp file paths and produce a spreadsheet showing the differences found in each of the INP sections. These differences should then be used whenever we need to rebuild this model from the baseline reference model. Note: this should be split into a func that creates a overall model "diff" that can then be written as a BI file or used programmatically """ allsections_a = funcs.complete_inp_headers(inpA) modela = swmmio.Model(inpA) modelb = swmmio.Model(inpB) #create build insructions folder if not os.path.exists(path): os.makedirs(path) filepath = os.path.join(path, filename) + '.txt' # xlpath = os.path.join(path, filename) + '.xlsx' # excelwriter = pd.ExcelWriter(xlpath) # vc_utils.create_change_info_sheet(excelwriter, modela, modelb) problem_sections = [ '[TITLE]', '[CURVES]', '[TIMESERIES]', '[RDII]', '[HYDROGRAPHS]' ] with open(filepath, 'w') as newf: #write meta data metadata = { #'Baseline Model':modela.inp.path, #'ID':filename, 'Parent Models': { 'Baseline': { inpA: vc_utils.modification_date(inpA) }, 'Alternatives': { inpB: vc_utils.modification_date(inpB) } }, 'Log': { filename: comments } } #print metadata vc_utils.write_meta_data(newf, metadata) for section in allsections_a['order']: if section not in problem_sections: #calculate the changes in the current section changes = INPDiff(modela, modelb, section) data = pd.concat( [changes.removed, changes.added, changes.altered]) #vc_utils.write_excel_inp_section(excelwriter, allsections_a, section, data) vc_utils.write_inp_section( newf, allsections_a, section, data, pad_top=False, na_fill='NaN') #na fill fixes SNOWPACK blanks spaces issue
def replace_inp_section(inp_path, modified_section_header, new_data): """ modify an existing inp file by passing in new data (Pandas Dataframe) and the section header that should be modified. This function will overwrite all data in the old section with the passed data :param inp_path: path to inp file to be changed :param modified_section_header: section for which data should be change :param new_data: pd.DataFrame of data to overwrite data in the modified section :return: swmmio.Model instantiated with modified inp file """ sections = get_inp_sections_details(inp_path) m = swmmio.Model(inp_path) with tempfile.TemporaryDirectory() as tempdir: with open(inp_path) as oldf: tmp_inp_path = os.path.join(tempdir, f'{m.inp.name}.inp') with open(tmp_inp_path, 'w') as new: # write each line as is from the original model until we find the # header of the section we wish to overwrite found_section = False found_next_section = False for line in oldf: if modified_section_header in line: # write the replacement data in the new file now write_inp_section(new, sections, modified_section_header, new_data, pad_top=False) found_section = True if (found_section and any(es in line for es in sections.keys()) and modified_section_header not in line): found_next_section = True if found_next_section or not found_section: # write the lines from the original file # if we haven't found the section to modify. # if we have found the section and we've found the NEXT section # continue writing original file's lines new.write(line) if not found_section: # the header doesn't exist in the old model # so we should append it to the bottom of file write_inp_section(new, sections, modified_section_header, new_data) # rename files and remove old if we should overwrite os.remove(inp_path) shutil.copy2(tmp_inp_path, inp_path) return swmmio.Model(inp_path)
def save(self, dir, filename): """ save the current BuildInstructions instance to file (human readable) """ if not os.path.exists(dir): os.makedirs(dir) filepath = os.path.join(dir, filename) with open (filepath, 'w') as f: vc_utils.write_meta_data(f, self.metadata) for section, change_obj in self.instructions.items(): section_df = pd.concat([change_obj.removed, change_obj.altered, change_obj.added]) vc_utils.write_inp_section(f, allheaders=None, sectionheader=section, section_data=section_df, pad_top=False, na_fill='NaN')
def create_inp_build_instructions(inpA, inpB, path, filename, comments=''): """ pass in two inp file paths and produce a spreadsheet showing the differences found in each of the INP sections. These differences should then be used whenever we need to rebuild this model from the baseline reference model. Note: this should be split into a func that creates a overall model "diff" that can then be written as a BI file or used programmatically """ allsections_a = funcs.complete_inp_headers(inpA) modela = swmmio.Model(inpA) modelb = swmmio.Model(inpB) #create build insructions folder if not os.path.exists(path): os.makedirs(path) filepath = os.path.join(path, filename) + '.txt' # xlpath = os.path.join(path, filename) + '.xlsx' # excelwriter = pd.ExcelWriter(xlpath) # vc_utils.create_change_info_sheet(excelwriter, modela, modelb) problem_sections = ['[TITLE]', '[CURVES]', '[TIMESERIES]', '[RDII]', '[HYDROGRAPHS]'] with open (filepath, 'w') as newf: #write meta data metadata = { #'Baseline Model':modela.inp.path, #'ID':filename, 'Parent Models':{ 'Baseline':{inpA:vc_utils.modification_date(inpA)}, 'Alternatives':{inpB:vc_utils.modification_date(inpB)} }, 'Log':{filename:comments} } #print metadata vc_utils.write_meta_data(newf, metadata) for section in allsections_a['order']: if section not in problem_sections: #calculate the changes in the current section changes = INPDiff(modela, modelb, section) data = pd.concat([changes.removed, changes.added, changes.altered]) #vc_utils.write_excel_inp_section(excelwriter, allsections_a, section, data) vc_utils.write_inp_section(newf, allsections_a, section, data, pad_top=False, na_fill='NaN') #na fill fixes SNOWPACK blanks spaces issue
def save(self, dir, filename): """ save the current BuildInstructions instance to file (human readable) """ if not os.path.exists(dir): os.makedirs(dir) filepath = os.path.join(dir, filename) with open(filepath, 'w') as f: vc_utils.write_meta_data(f, self.metadata) for section, change_obj in self.instructions.iteritems(): section_df = pd.concat( [change_obj.removed, change_obj.altered, change_obj.added]) vc_utils.write_inp_section(f, allheaders=None, sectionheader=section, section_data=section_df, pad_top=False, na_fill='NaN')
def build(self, baseline_dir, target_path): """ build a complete INP file with the build instructions committed to a baseline model. """ basemodel = swmmio.Model(baseline_dir) allheaders = get_inp_sections_details(basemodel.inp.path) # new_inp = os.path.join(target_dir, 'model.inp') with open(target_path, 'w') as f: for section, _ in allheaders.items(): # check if the section is not in problem_sections and there are changes # in self.instructions and commit changes to it from baseline accordingly if (section not in problem_sections and allheaders[section]['columns'] != ['blob'] and section in self.instructions): # df of baseline model section basedf = dataframe_from_bi(basemodel.inp.path, section) basedf[';'] = ';' # grab the changes to changes = self.instructions[section] # remove elements that have alterations and or tagged for removal remove_ids = changes.removed.index | changes.altered.index new_section = basedf.drop(remove_ids) # add elements new_section = pd.concat( [new_section, changes.altered, changes.added]) else: # section is not well understood or is problematic, just blindly copy new_section = dataframe_from_bi(basemodel.inp.path, section=section) new_section[';'] = ';' # write the section vc_utils.write_inp_section(f, allheaders, section, new_section)
def generate_inp_from_diffs(basemodel, inpdiffs, target_dir): """ create a new inp with respect to a baseline inp and changes instructed with a list of inp diff files (build instructions). This saves having to recalculate the differences of each model from the baseline whenever we want to combine versions. """ #step 1 --> combine the diff/build instructions allheaders = funcs.complete_inp_headers(basemodel.inp.filePath) combi_build_instr_file = os.path.join(target_dir, 'build_instructions.txt') newinp = os.path.join(target_dir, 'new.inp') with open(combi_build_instr_file, 'w') as f: for header in allheaders['order']: s = '' section_header_written = False for inp in inpdiffs: sect_s = None if not section_header_written: sect_s = text.extract_section_from_inp(inp, header, cleanheaders=False, return_string=True, skipheaders=False) section_header_written = True else: sect_s = text.extract_section_from_inp(inp, header, cleanheaders=False, return_string=True, skipheaders=True) if sect_s: #remove the extra space between data in the same table #coming from diffrent models. if sect_s[-2:] == '\n\n': #NOTE Check this section... s += sect_s[:-1] else: s += sect_s f.write(s + '\n') #step 2 --> clean up the new combined diff instructions df_dict = clean_inp_diff_formatting( combi_build_instr_file) #makes more human readable #step 3 --> create a new inp based on the baseline, with the inp_diff #instructions applied with open(newinp, 'w') as f: for section in allheaders['order']: print section if section not in problem_sections and allheaders['headers'][ section] != 'blob': #check if a changes from baseline spreadheet exists, and use this #information if available to create the changes array df = create_dataframeINP(basemodel.inp, section) df['Origin'] = '' #add the origin column if not there if section in df_dict: df_change = df_dict[section] ids_to_drop = df_change.loc[df_change['Comment'].isin( ['Removed', 'Altered'])].index df = df.drop(ids_to_drop) df = df.append(df_change.loc[df_change['Comment'].isin( ['Added', 'Altered'])]) new_section = df else: #blindly copy this section from the base model new_section = create_dataframeINP(basemodel.inp, section=section) #write the section into the inp file and the excel file vc_utils.write_inp_section(f, allheaders, section, new_section)
def merge_models(inp1, inp2, target='merged_model.inp'): """ Merge two separate swmm models into one model. This creates a diff, ignores removed sections, and uses inp1 settings where conflicts exist (altered sections in diff) :param inp1: swmmio.Model.inp object to be combined with inp2 :param inp2: swmmio.Model.inp object to be combined with inp1 :param target: path of new model :return: path to target """ # model object to store resulting merged model m3 = swmmio.Model(inp1) inp_diff = INPDiff(inp1, inp2) with open(target, 'w') as newf: for section, _ in inp_diff.all_inp_objects.items(): # don't consider the "removed" parts of the diff # print('{}: {}'.format(section,inp_diff.all_inp_objects[section]['columns'])) # check if the section is not in problem_sections and there are changes # in self.instructions and commit changes to it from baseline accordingly col_order = [] if (section not in problem_sections and inp_diff.all_inp_objects[section]['columns'] != ['blob'] and section in inp_diff.diffs): # df of baseline model section basedf = dataframe_from_inp( m3.inp.path, section, additional_cols=[';', 'Comment', 'Origin']) basedf[';'] = ';' col_order = basedf.columns # grab the changes to changes = inp_diff.diffs[section] # remove elements that have alterations keep ones tagged for removal # (unchanged, but not present in m2) remove_ids = changes.altered.index new_section = basedf.drop(remove_ids) # add elements new_section = pd.concat( [new_section, changes.altered, changes.added], axis=0, sort=False) else: # section is not well understood or is problematic, just blindly copy new_section = dataframe_from_inp( m3.inp.path, section, additional_cols=[';', 'Comment', 'Origin']) new_section[';'] = ';' # print ('dealing with confusing section: {}\n{}'.format(section, new_section)) # print(new_section.head()) # write the section new_section = new_section[col_order] new_section[';'] = ';' vc_utils.write_inp_section(newf, inp_diff.all_inp_objects, section, new_section, pad_top=True) return target
def replace_inp_section(inp_path, modified_section_header, new_data, overwrite=True): """ modify an existing model by passing in new data (Pandas Dataframe) and the section header that should be modified. This funciton will overwrite all data in the old section with the passed data """ tmpfilename = os.path.splitext(os.path.basename(inp_path))[0] + '_mod.inp' wd = os.path.dirname(inp_path) tmpfilepath = os.path.join(os.path.dirname(inp_path), tmpfilename) allheaders = complete_inp_headers(inp_path) basemodel = swmmio.Model(inp_path) with open(inp_path) as oldf: with open(tmpfilepath, 'w') as new: #create the companion excel file #create the MS Excel writer object # xlpath = os.path.join(wd, basemodel.inp.name + '_modified.xlsx') # excelwriter = pd.ExcelWriter(xlpath) # vc_utils.create_info_sheet(excelwriter, basemodel) #write each line as is from the original model until we find the #header of the section we wish to overwrite found_section = False found_next_section = False for line in oldf: if modified_section_header in line: #write the replacement data in the new file now vc_utils.write_inp_section(new, allheaders, modified_section_header, new_data, pad_top=False) found_section = True if (found_section and not found_next_section and line.strip() in allheaders['headers'] and modified_section_header != line.strip()): found_next_section = True new.write('\n\n') #add some space before the next section if found_next_section or not found_section: #write the lines from the original file #if we haven't found the section to modify. #if we have found the section and we've found the NEXT section #continue writing original file's lines new.write(line) if not found_section: #the header doesn't exist in the old model #so we should append it to the bottom of file vc_utils.write_inp_section(new, allheaders, modified_section_header, new_data) # excelwriter.save() #rename files and remove old if we should overwrite if overwrite: os.remove(inp_path) os.rename(tmpfilepath, inp_path) return swmmio.Model(inp_path)
def generate_inp_from_diffs(basemodel, inpdiffs, target_dir): """ create a new inp with respect to a baseline inp and changes instructed with a list of inp diff files (build instructions). This saves having to recalculate the differences of each model from the baseline whenever we want to combine versions. NOTE THIS ISN'T USED ANYWHERE. DELETE ???? """ #step 1 --> combine the diff/build instructions allheaders = funcs.complete_inp_headers(basemodel.inp.path) combi_build_instr_file = os.path.join(target_dir, 'build_instructions.txt') newinp = os.path.join(target_dir, 'new.inp') with open (combi_build_instr_file, 'w') as f: for header in allheaders['order']: s = '' section_header_written = False for inp in inpdiffs: sect_s = None if not section_header_written: sect_s = text.extract_section_from_inp(inp, header, cleanheaders=False, return_string=True, skipheaders=False) section_header_written = True else: sect_s = text.extract_section_from_inp(inp, header, cleanheaders=False, return_string=True, skipheaders=True) if sect_s: #remove the extra space between data in the same table #coming from diffrent models. if sect_s[-2:] == '\n\n': #NOTE Check this section... s += sect_s[:-1] else: s += sect_s f.write(s + '\n') #step 2 --> clean up the new combined diff instructions # df_dict = clean_inp_diff_formatting(combi_build_instr_file) #makes more human readable #step 3 --> create a new inp based on the baseline, with the inp_diff #instructions applied with open (newinp, 'w') as f: for section in allheaders['order']: print(section) if section not in problem_sections and allheaders['headers'][section] != 'blob': #check if a changes from baseline spreadheet exists, and use this #information if available to create the changes array df = create_dataframeINP(basemodel.inp.path, section) df['Origin'] = '' #add the origin column if not there if section in df_dict: df_change = df_dict[section] ids_to_drop = df_change.loc[df_change['Comment'].isin(['Removed', 'Altered'])].index df = df.drop(ids_to_drop) df = df.append(df_change.loc[df_change['Comment'].isin(['Added', 'Altered'])]) new_section = df else: #blindly copy this section from the base model new_section = create_dataframeINP(basemodel.inp.path, section=section) #write the section into the inp file and the excel file vc_utils.write_inp_section(f, allheaders, section, new_section)