def test_create_inp_build_instructions(): inp.create_inp_build_instructions(MODEL_XSECTION_BASELINE, MODEL_XSECTION_ALT_03, 'vc_dir', 'test_version_id', 'cool comments') latest_bi = vc_utils.newest_file('vc_dir') bi = inp.BuildInstructions(latest_bi) juncs = bi.instructions['[JUNCTIONS]'] assert (all(j in juncs.altered.index for j in ['dummy_node1', 'dummy_node5']))
def test_create_inp_build_instructions(): inp.create_inp_build_instructions(MODEL_XSECTION_BASELINE, MODEL_XSECTION_ALT_03, 'vc_dir', 'test_version_id', 'cool comments') latest_bi = vc_utils.newest_file('vc_dir') bi = inp.BuildInstructions(latest_bi) juncs = bi.instructions['[JUNCTIONS]'] assert(all(j in juncs.altered.index for j in [ 'dummy_node1', 'dummy_node5']))
def test_inp_diff_from_bi(): change = INPSectionDiff(build_instr_file=BUILD_INSTR_01, section='JUNCTIONS') alt_juncs = change.altered assert alt_juncs.loc['dummy_node1', 'InvertElev'] == pytest.approx(-15, 0.01) assert alt_juncs.loc['dummy_node5', 'InvertElev'] == pytest.approx(-6.96, 0.01) with tempfile.TemporaryDirectory() as tempdir: # test with spaces in path temp_dir_01 = os.path.join(tempdir, 'path with spaces') makedirs(temp_dir_01) shutil.copy(BUILD_INSTR_01, temp_dir_01) BUILD_INSTR_01_spaces = os.path.join(temp_dir_01, BUILD_INSTR_01) change = INPSectionDiff(build_instr_file=BUILD_INSTR_01_spaces, section='[JUNCTIONS]') alt_juncs = change.altered assert alt_juncs.loc['dummy_node1', 'InvertElev'] == pytest.approx(-15, 0.01) assert alt_juncs.loc['dummy_node5', 'InvertElev'] == pytest.approx(-6.96, 0.01) # test with parent models in directory structure with spaces in path temp_dir_02 = os.path.join(tempdir, 'root with spaces') temp_dir_03 = os.path.join(temp_dir_02, 'vc_dir') makedirs(temp_dir_02) shutil.copy(MODEL_XSECTION_BASELINE, temp_dir_02) shutil.copy(MODEL_XSECTION_ALT_03, temp_dir_02) MODEL_XSECTION_BASELINE_spaces = os.path.join(temp_dir_02, MODEL_XSECTION_BASELINE) MODEL_XSECTION_ALT_03_spaces = os.path.join(temp_dir_02, MODEL_XSECTION_ALT_03) inp.create_inp_build_instructions(MODEL_XSECTION_BASELINE_spaces, MODEL_XSECTION_ALT_03_spaces, temp_dir_03, 'test_version_id', 'cool comments') latest_bi_spaces = vc_utils.newest_file(temp_dir_03) change = INPSectionDiff(build_instr_file=latest_bi_spaces, section='[JUNCTIONS]') alt_juncs = change.altered assert alt_juncs.loc['dummy_node1', 'InvertElev'] == pytest.approx(-15, 0.01) assert alt_juncs.loc['dummy_node5', 'InvertElev'] == pytest.approx(-6.96, 0.01)
def test_add_models(): """ create build instructions for entire models with respect to a blank (empty) model such that BIs include additions for each model element. This is experimental. """ with tempfile.TemporaryDirectory() as tempdir: bi_b = inp.create_inp_build_instructions(MODEL_BLANK, MODEL_FULL_FEATURES_XY_B, tempdir, 'from_blank_model_b', 'cool comments') bi_a = inp.create_inp_build_instructions(MODEL_BLANK, MODEL_FULL_FEATURES_XY, tempdir, 'from_blank_model_a', 'should have all we need') bi_final = bi_a + bi_b bi_final.save(tempdir, 'merged-build-instr.txt') bi_final.build(MODEL_BLANK, os.path.join(tempdir, 'merged-model.inp')) assert os.path.exists(os.path.join(tempdir, 'merged-model.inp'))
def test_create_inp_build_instructions(): with tempfile.TemporaryDirectory() as tempdir: temp_vc_dir_01 = os.path.join(tempdir, 'vc_dir') temp_vc_dir_02 = os.path.join(tempdir, 'vc root with spaces') temp_vc_dir_03 = os.path.join(temp_vc_dir_02, 'vc_dir') inp.create_inp_build_instructions(MODEL_XSECTION_BASELINE, MODEL_XSECTION_ALT_03, temp_vc_dir_01, 'test_version_id', 'cool comments') latest_bi = vc_utils.newest_file(temp_vc_dir_01) bi = inp.BuildInstructions(latest_bi) juncs = bi.instructions['JUNCTIONS'] assert (all(j in juncs.altered.index for j in ['dummy_node1', 'dummy_node5'])) # reproduce test with same files in a directory structure with spaces in path makedirs(temp_vc_dir_02) shutil.copy(MODEL_XSECTION_BASELINE, temp_vc_dir_02) shutil.copy(MODEL_XSECTION_ALT_03, temp_vc_dir_02) MODEL_XSECTION_BASELINE_spaces = os.path.join(temp_vc_dir_02, MODEL_XSECTION_BASELINE) MODEL_XSECTION_ALT_03_spaces = os.path.join(temp_vc_dir_02, MODEL_XSECTION_ALT_03) inp.create_inp_build_instructions(MODEL_XSECTION_BASELINE_spaces, MODEL_XSECTION_ALT_03_spaces, temp_vc_dir_03, 'test_version_id', 'cool comments') latest_bi_spaces = vc_utils.newest_file(temp_vc_dir_03) bi_sp = inp.BuildInstructions(latest_bi_spaces) juncs_sp = bi_sp.instructions['JUNCTIONS'] print(juncs_sp.altered) assert (all(j in juncs_sp.altered.index for j in ['dummy_node1', 'dummy_node5']))
def create_combinations(baseline_dir, rsn_dir, combi_dir, version_id='', comments=''): """ Generate SWMM5 models of each logical combination of all implementation phases (IP) across all relief sewer networks (RSN). Inputs: baseline_dir -> path to directory containing the baseline SWMM5 model rsn_dir -> path to directory containing subdirectories for each RSN containing directories for each IP within the network combi_dir -> target directory in which child models will be created version_id -> identifier for a given version (optional) comments -> comments tracked within build instructions log for each model scenario (optional) Calling create_combinations will update child models if parent models have been changed. """ baseinp = Model(baseline_dir).inp.path version_id += '_' + datetime.now().strftime("%y%m%d%H%M%S") #create a list of directories pointing to each IP in each RSN RSN_dirs = [os.path.join(rsn_dir, rsn) for rsn in os.listdir(rsn_dir)] IP_dirs = [os.path.join(d, ip) for d in RSN_dirs for ip in os.listdir(d)] #list of lists of each IP within each RSN, including a 'None' phase. IPs = [[None] + os.listdir(d) for d in RSN_dirs] #identify all scenarios (cartesian product of sets of IPs between each RSN) #then isolate child scenarios with atleast 2 parents (sets with one parent #are already modeled as IPs within the RSNs) all_scenarios = [[_f for _f in s if _f] for s in itertools.product(*IPs)] child_scenarios = [s for s in all_scenarios if len(s) > 1] #notify user of what was initially found str_IPs = '\n'.join([', '.join([_f for _f in i if _f]) for i in IPs]) print(('Found {} implementation phases among {} networks:\n{}\n' 'This yeilds {} combined scenarios ({} total)'.format(len(IP_dirs), len(RSN_dirs),str_IPs,len(child_scenarios),len(all_scenarios) - 1))) # ========================================================================== # UPDATE/CREATE THE PARENT MODEL BUILD INSTRUCTIONS # ========================================================================== for ip_dir in IP_dirs: ip_model = Model(ip_dir) vc_dir = os.path.join(ip_dir, 'vc') if not os.path.exists(vc_dir): print('creating new build instructions for {}'.format(ip_model.name)) inp.create_inp_build_instructions(baseinp, ip_model.inp.path, vc_dir, version_id, comments) else: #check if the alternative model was changed since last run of this tool #--> compare the modification date to the BI's modification date meta data latest_bi = vc_utils.newest_file(vc_dir) if not vc_utils.bi_is_current(latest_bi): #revision date of the alt doesn't match the newest build #instructions for this 'imp_level', so we should refresh it print('updating build instructions for {}'.format(ip_model.name)) inp.create_inp_build_instructions(baseinp, ip_model.inp.path, vc_dir, version_id, comments) # ========================================================================== # UPDATE/CREATE THE CHILD MODELS AND CHILD BUILD INSTRUCTIONS # ========================================================================== for scen in child_scenarios: newcombi = '_'.join(sorted(scen)) new_dir = os.path.join(combi_dir, newcombi) vc_dir = os.path.join(combi_dir, newcombi, 'vc') #parent model build instr files #BUG (this breaks with model IDs with more than 1 char) parent_vc_dirs = [os.path.join(rsn_dir, f[0], f, 'vc') for f in scen] latest_parent_bis = [vc_utils.newest_file(d) for d in parent_vc_dirs] build_instrcts = [inp.BuildInstructions(bi) for bi in latest_parent_bis] if not os.path.exists(new_dir): os.mkdir(new_dir) newinppath = os.path.join(new_dir, newcombi + '.inp') print('creating new child model: {}'.format(newcombi)) new_build_instructions = sum(build_instrcts) new_build_instructions.save(vc_dir, version_id+'.txt') new_build_instructions.build(baseline_dir, newinppath) else: #check if the alternative model was changed since last run #of this tool --> compare the modification date to the BI's #modification date meta data latest_bi = vc_utils.newest_file(os.path.join(new_dir,'vc')) if not vc_utils.bi_is_current(latest_bi): #revision date of the alt doesn't match the newest build #instructions for this 'imp_level', so we should refresh it print('updating child build instructions for {}'.format(newcombi)) newinppath = os.path.join(new_dir, newcombi + '.inp') new_build_instructions = sum(build_instrcts) new_build_instructions.save(vc_dir, version_id+'.txt') new_build_instructions.build(baseline_dir, newinppath)
def create_combinations(baseline_dir, rsn_dir, combi_dir, version_id='', comments=''): """ Generate SWMM5 models of each logical combination of all implementation phases (IP) across all relief sewer networks (RSN). Inputs: baseline_dir -> path to directory containing the baseline SWMM5 model rsn_dir -> path to directory containing subdirectories for each RSN containing directories for each IP within the network combi_dir -> target directory in which child models will be created version_id -> identifier for a given version (optional) comments -> comments tracked within build instructions log for each model scenario (optional) Calling create_combinations will update child models if parent models have been changed. """ baseinp = Model(baseline_dir).inp.path version_id += '_' + datetime.now().strftime("%y%m%d%H%M%S") #create a list of directories pointing to each IP in each RSN RSN_dirs = [os.path.join(rsn_dir, rsn) for rsn in os.listdir(rsn_dir)] IP_dirs = [os.path.join(d, ip) for d in RSN_dirs for ip in os.listdir(d)] #list of lists of each IP within each RSN, including a 'None' phase. IPs = [[None] + os.listdir(d) for d in RSN_dirs] #identify all scenarios (cartesian product of sets of IPs between each RSN) #then isolate child scenarios with atleast 2 parents (sets with one parent #are already modeled as IPs within the RSNs) all_scenarios = [[_f for _f in s if _f] for s in itertools.product(*IPs)] child_scenarios = [s for s in all_scenarios if len(s) > 1] #notify user of what was initially found str_IPs = '\n'.join([', '.join([_f for _f in i if _f]) for i in IPs]) print(('Found {} implementation phases among {} networks:\n{}\n' 'This yeilds {} combined scenarios ({} total)'.format( len(IP_dirs), len(RSN_dirs), str_IPs, len(child_scenarios), len(all_scenarios) - 1))) # ========================================================================== # UPDATE/CREATE THE PARENT MODEL BUILD INSTRUCTIONS # ========================================================================== for ip_dir in IP_dirs: ip_model = Model(ip_dir) vc_dir = os.path.join(ip_dir, 'vc') if not os.path.exists(vc_dir): print('creating new build instructions for {}'.format( ip_model.name)) inp.create_inp_build_instructions(baseinp, ip_model.inp.path, vc_dir, version_id, comments) else: #check if the alternative model was changed since last run of this tool #--> compare the modification date to the BI's modification date meta data latest_bi = vc_utils.newest_file(vc_dir) if not vc_utils.bi_is_current(latest_bi): #revision date of the alt doesn't match the newest build #instructions for this 'imp_level', so we should refresh it print('updating build instructions for {}'.format( ip_model.name)) inp.create_inp_build_instructions(baseinp, ip_model.inp.path, vc_dir, version_id, comments) # ========================================================================== # UPDATE/CREATE THE CHILD MODELS AND CHILD BUILD INSTRUCTIONS # ========================================================================== for scen in child_scenarios: newcombi = '_'.join(sorted(scen)) new_dir = os.path.join(combi_dir, newcombi) vc_dir = os.path.join(combi_dir, newcombi, 'vc') #parent model build instr files #BUG (this breaks with model IDs with more than 1 char) parent_vc_dirs = [os.path.join(rsn_dir, f[0], f, 'vc') for f in scen] latest_parent_bis = [vc_utils.newest_file(d) for d in parent_vc_dirs] build_instrcts = [ inp.BuildInstructions(bi) for bi in latest_parent_bis ] if not os.path.exists(new_dir): os.mkdir(new_dir) newinppath = os.path.join(new_dir, newcombi + '.inp') print('creating new child model: {}'.format(newcombi)) new_build_instructions = sum(build_instrcts) new_build_instructions.save(vc_dir, version_id + '.txt') new_build_instructions.build(baseline_dir, newinppath) else: #check if the alternative model was changed since last run #of this tool --> compare the modification date to the BI's #modification date meta data latest_bi = vc_utils.newest_file(os.path.join(new_dir, 'vc')) if not vc_utils.bi_is_current(latest_bi): #revision date of the alt doesn't match the newest build #instructions for this 'imp_level', so we should refresh it print('updating child build instructions for {}'.format( newcombi)) newinppath = os.path.join(new_dir, newcombi + '.inp') new_build_instructions = sum(build_instrcts) new_build_instructions.save(vc_dir, version_id + '.txt') new_build_instructions.build(baseline_dir, newinppath)
def create_combinations(baseline_dir, alternatives_dir, combi_dir, version_id='', comments=''): """ given a set of main alternatives split into models varying levels of implementation, this function combines all implementation levels of all alternatives into all logical combinations. """ basemodel = Model(baseline_dir) baseinp = basemodel.inp.filePath alt_directories = os.listdir( alternatives_dir) #list of dirs holding alt models implementation_levels = [] newmodels = [] version_id += '_' + datetime.now().strftime("%y%m%d%H%M%S") for alt in alt_directories: #iterate through each implementation level of each alternative for imp_level in os.listdir(os.path.join(alternatives_dir, alt)): implementation_levels.append(os.path.join(alt, imp_level)) #create or refresh the build instructions file for the alternatives alt_imp_level_dir = os.path.join(alternatives_dir, alt, imp_level) alt_imp_inp = Model(alt_imp_level_dir).inp.filePath vc_directory = os.path.join(alt_imp_level_dir, 'vc') if not os.path.exists(vc_directory): print 'creating new build instructions for {}'.format( imp_level) inp.create_inp_build_instructions(baseinp, alt_imp_inp, vc_directory, version_id, comments) else: #check if the alternative model was changed since last run of this tool #--> compare the modification date to the BI's modification date meta data latest_bi = vc_utils.newest_file(vc_directory) if not vc_utils.bi_is_current(latest_bi): #revision date of the alt doesn't match the newest build #instructions for this 'imp_level', so we should refresh it print 'updating build instructions for {}'.format( imp_level) inp.create_inp_build_instructions(baseinp, alt_imp_inp, vc_directory, version_id, comments) #creat directories for new model combinations for L in range(1, len(implementation_levels) + 1): #break for subset in itertools.combinations(implementation_levels, L): #subset e.g. = 'A\A01' #newcombi = '_'.join(subset) newcombi = '_'.join([os.path.split(s)[1] for s in subset]) new_combi_dir = os.path.join(combi_dir, newcombi) vc_directory = os.path.join(new_combi_dir, 'vc') #create a list of the parent directories, use that to prevent #two or more from same alternative alternative_dirs = [os.path.split(s)[0] for s in subset] if len(alternative_dirs) == len( set(alternative_dirs)) and len(subset) > 1: #confirming the list length is equal to the set length (hashable) #confirms that there are not duplicates in the items list parent_vc_dirs = [ os.path.join(alternatives_dir, f, 'vc') for f in subset ] latest_parent_bis = [ vc_utils.newest_file(d) for d in parent_vc_dirs ] build_instrcts = [ inp.BuildInstructions(bi) for bi in latest_parent_bis ] if not os.path.exists( new_combi_dir): #and newcombi not in flavors: #check to make sure new model doesn't repeat two or more from #a particular genre. #print new_combi_dir os.mkdir(new_combi_dir) newinppath = os.path.join(new_combi_dir, newcombi + '.inp') #collect build instructions from each alt's implementation #level for this combination. Select those with the current #version id, or the latest version. print 'creating new child model: {}'.format(newcombi) new_build_instructions = sum(build_instrcts) new_build_instructions.save(vc_directory, version_id + '.txt') new_build_instructions.build(baseline_dir, newinppath) else: #check if the alternative model was changed since last run #of this tool --> compare the modification date to the BI's #modification date meta data latest_bi = vc_utils.newest_file( os.path.join(new_combi_dir, 'vc')) if not vc_utils.bi_is_current(latest_bi): #revision date of the alt doesn't match the newest build #instructions for this 'imp_level', so we should refresh it print 'updating child build instructions for {}'.format( newcombi) newinppath = os.path.join(new_combi_dir, newcombi + '.inp') new_build_instructions = sum(build_instrcts) new_build_instructions.save(vc_directory, version_id + '.txt') new_build_instructions.build(baseline_dir, newinppath)