def validateEPANET(self):
try:
enData = pyepanet.ENepanet()
except:
raise RuntimeError("EPANET DLL is missing or corrupt. Please reinstall PyEPANET.")
self.epanetOkay = True
return
def setUp(self):
self.origDir = os.getcwd()
os.chdir(testDir)
try:
self.enData = pyepanet.ENepanet()
self.enData.ENopen(join(dataDir, 'Net3.inp'), 'tmp.rpt')
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
def __init__(self):
pywst.common.problem.Problem.__init__(
self, 'booster_msx', ("network", "scenario", "impact",
"booster msx", "solver", "configure"))
self.epanetOkay = False
try:
enData = pyepanet.ENepanet()
except Exception as E:
print type(E), E
raise RuntimeError(
"EPANET DLL is missing or corrupt. Please reinstall PyEPANET.")
self.epanetOkay = True
def runSamplelocation(self):
logger = logging.getLogger('wst.grabsample.samplelocation')
metric_file = self.getSampleLocationOption('nodes metric')
if metric_file not in self.none_list:
print "WARNING: weighted distinguishability not supported with tsg or tsi. Use signals"
#print self.opts
out_prefix = ''
if self.getConfigureOption('output prefix') not in self.none_list:
out_prefix += self.getConfigureOption('output prefix') + '_'
#self.createInversionSimDat()
cmd = 'samplelocation '
is_inp_file = (self.getNetworkOption('epanet file')
not in self.none_list)
is_wqm_file = (self.getSampleLocationOption('wqm file')
not in self.none_list)
# Optional arguments are simulation duration and water quality timestep, which will
# override what is in the EPANET input file.
if is_inp_file and self.getNetworkOption(
'simulation duration') not in ['INP', 'Inp', 'inp']:
cmd += '--simulation-duration-minutes=' + str(
self.getNetworkOption('simulation duration')) + ' '
if is_inp_file and self.getNetworkOption(
'water quality timestep') not in ['INP', 'Inp', 'inp']:
cmd += '--quality-timestep-minutes=' + str(
self.getNetworkOption('water quality timestep')) + ' '
# substitude integers for node names in output files
#cmd += '--output-merlion-labels '
# the above command will produce the following file
tmpdir = os.path.dirname(self.opts['configure']['output prefix'])
prefix = os.path.basename(self.opts['configure']['output prefix'])
# if merlion is being used substitude integers for node names in output files
if self.getEventsOption('merlion'):
cmd += '--merlion '
cmd += '--output-merlion-labels '
label_map_file = wst_util.declare_tempfile(
os.path.join(tmpdir, prefix + "_MERLION_LABEL_MAP.txt"))
else:
cmd += '--output-epanet-labels '
label_map_file = wst_util.declare_tempfile(
os.path.join(tmpdir, prefix + "_EPANET_LABEL_MAP.txt"))
# Prepend all output file names with this
cmd += '--output-prefix=' + out_prefix + ' '
# Dissable merlion warnings
if self.getEventsOption('ignore merlion warnings'):
cmd += '--ignore-merlion-warnings '
# Prepend fixed sensorsname
logger.debug(self.getSampleLocationOption('fixed sensors'))
if self.getSampleLocationOption('fixed sensors') not in self.none_list:
try:
enData = pyepanet.ENepanet()
enData.ENopen(self.opts['network']['epanet file'], 'tmp.rpt')
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
fixed_sensor_names, feasible_node_indices = wst_util.feasible_nodes(\
self.getSampleLocationOption('fixed sensors'),\
[], \
True, enData)
enData.ENclose()
tmpdir = os.path.dirname(self.opts['configure']['output prefix'])
tmpprefix = 'tmp_' + os.path.basename(
self.opts['configure']['output prefix'])
tmpSensorFile = pyutilib.services.TempfileManager.create_tempfile(
prefix=tmpprefix, dir=tmpdir, suffix='.txt')
# write nodes file
fid = open(tmpSensorFile, 'w')
for n in fixed_sensor_names:
fid.write(n + '\n')
fid.close()
cmd += '--fixed-sensors=' + tmpSensorFile + ' '
self.opts['grabsample']['fixed sensors'] = tmpSensorFile
# Allowed location filename
if self.getSampleLocationOption(
'feasible nodes') not in self.none_list:
try:
enData = pyepanet.ENepanet()
enData.ENopen(self.opts['network']['epanet file'], 'tmp.rpt')
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
feasible_node_names, feasible_node_indices = wst_util.feasible_nodes(\
self.getSampleLocationOption('feasible nodes'),\
[], \
True, enData)
enData.ENclose()
tmpdir = os.path.dirname(self.opts['configure']['output prefix'])
tmpprefix = 'tmp_' + os.path.basename(
self.opts['configure']['output prefix'])
tmpNodesFile = pyutilib.services.TempfileManager.create_tempfile(
prefix=tmpprefix, dir=tmpdir, suffix='.txt')
# write nodes file
fid = open(tmpNodesFile, 'w')
for n in feasible_node_names:
fid.write(n + '\n')
fid.close()
cmd += '--allowed-nodes=' + tmpNodesFile + ' '
self.opts['grabsample']['feasible nodes'] = tmpNodesFile
#LOGIC VALIDATION
is_tsg_file = (self.getEventsOption('tsg file') not in self.none_list)
is_scn_file = (self.getEventsOption('scn file') not in self.none_list)
assert (is_inp_file != is_wqm_file) # one and only one is true
assert (is_tsg_file != is_scn_file) # one and only one is true
if self.getSampleLocationOption('model format') not in [
'AMPL', 'PYOMO'
]:
raise IOError("Invalid model format: " +
self.getSampleLocationOption('model format'))
#Check for the threshold
if self.getSampleLocationOption('threshold') not in self.none_list:
cmd += '--threshold=' + str(
self.getSampleLocationOption('threshold')) + ' '
#check for the number of samples
if self.getSampleLocationOption('num samples') not in self.none_list:
cmd += '--number-samples=' + str(
self.getSampleLocationOption('num samples')) + ' '
# Check for greedy selection algorithm
if self.getSampleLocationOption('greedy selection'):
cmd += '--greedy-selection '
# Check for greedy selection algorithm
if self.getSampleLocationOption('with weights'):
cmd += '--with-weights '
# The file defining the water quality model
if is_inp_file:
cmd += '--inp=' + self.getNetworkOption('epanet file') + ' '
elif is_wqm_file:
cmd += '--wqm=' + self.getSampleLocationOption('wqm file') + ' '
if is_tsg_file:
'''if len(open(self.getEventsOption('tsg file'),'r').readlines())<2:
print '\nError: The events file must have more than one event.'
exit(1)'''
cmd += '--tsg=' + self.getEventsOption('tsg file') + ' '
elif is_scn_file:
'''if int(open(self.getEventsOption('scn file'),'r').readline())<2:
print '\nError: The events file must have more than one event.'
exit(1)'''
cmd += '--scn=' + self.getEventsOption('scn file') + ' '
if self.getSampleLocationOption('sample time') in self.none_list:
raise IOError("A sample time must be specified")
else:
cmd += str(self.getSampleLocationOption('sample time'))
#print cmd
logger.info("Launching samplelocation executable ...")
logger.debug(cmd)
out = pyutilib.services.TempfileManager.create_tempfile(
dir=tmpdir, prefix='tmp_', suffix='samplelocation.out')
sim_timelimit = None
sub_logger = logging.getLogger('wst.grabsample.samplelocation.exec')
sub_logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(out, mode='w')
sub_logger.addHandler(fh)
p = pyutilib.subprocess.run(
cmd,
timelimit=sim_timelimit,
stdout=pywst.common.problem.LoggingFile(sub_logger))
if (p[0]):
print '\nAn error occured when running the samplelocation executable'
print 'Error Message: ', p[1]
print 'Command: ', cmd, '\n'
raise RuntimeError(
"An error occured when running the samplelocation executable")
nodemap = {}
if not self.getSampleLocationOption('greedy selection'):
f = open(label_map_file, 'r')
for line in f:
t = line.split()
nodemap[t[1]] = t[0]
f.close()
return nodemap
def run(self):
logger.info("WST grabsample subcommand")
logger.info("---------------------------")
# set start time
self.startTime = time.time()
# validate input
logger.info("Validating configuration file")
self.validate()
if self.getEventsOption('signals') not in self.none_list:
files_folder = self.getEventsOption('signals')
#TODO LIST SCENARIOS
optimal_locations, objective = self.runSIGNALSanalysis(
files_folder)
# Convert node ID's to node names in Solution
Solution = list()
Solution = [objective, optimal_locations]
else:
try:
enData = pyepanet.ENepanet()
enData.ENopen(self.opts['network']['epanet file'], 'tmp.rpt')
enData.ENclose()
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
# write tmp TSG file if ['scenario']['tsg file'] == none
if self.opts['scenario'][
'tsi file'] in pywst.common.problem.none_list and self.opts[
'scenario'][
'tsg file'] in pywst.common.problem.none_list:
tmpdir = os.path.dirname(
self.opts['configure']['output prefix'])
tmpprefix = 'tmp_' + os.path.basename(
self.opts['configure']['output prefix'])
tmpTSGFile = pyutilib.services.TempfileManager.create_tempfile(
prefix=tmpprefix, dir=tmpdir, suffix='.tsg')
# Check if scenario list contains one or less nodes.
wst_util.write_tsg(self.opts['scenario']['location'],\
self.opts['scenario']['type'],\
self.opts['scenario']['species'],\
self.opts['scenario']['strength'],\
self.opts['scenario']['start time'],\
self.opts['scenario']['end time'],\
tmpTSGFile)
self.opts['scenario']['tsg file'] = tmpTSGFile
# expand tsg file
extTSGfile = wst_util.expand_tsg(self.opts)
self.opts['scenario']['tsg file'] = extTSGfile
# Run samplelocation executable
nodemap = self.runSamplelocation()
Solution = ()
greedy_solution = {}
#
if self.getConfigureOption('output prefix') not in self.none_list:
json_result_file = self.getConfigureOption(
'output prefix') + '_grabsample.json'
else:
json_result_file = 'grabsample.json'
#
# For greedy algorithm get solution directly from the json file
if self.getSampleLocationOption('greedy selection'):
data_from_results = open(json_result_file).read()
greedy_solution = json.loads(data_from_results)
Solution = (greedy_solution['objective'],
[str(i['id']) for i in greedy_solution['Nodes']])
#else run optimization algorithms
else:
#Get fixed sensor list . Not to be included in final solution
fixed_sensor_ID = []
inv_nodemap = dict((v, k) for k, v in nodemap.iteritems())
if self.getSampleLocationOption(
'fixed sensors') not in self.none_list:
sensors_filename = self.getSampleLocationOption(
'fixed sensors')
sensor_file = open(sensors_filename)
fixed_sensor_list = [line.strip() for line in sensor_file]
sensor_file.close()
for node_name in fixed_sensor_list:
if len(node_name) > 0:
fixed_sensor_ID.append(inv_nodemap[node_name])
solve_timelimit = None
p = (
1,
"There was a problem with the 'model type' or 'model format' options"
)
cmd = None
not_allowed_nodes_set = set()
"""
if self.getSampleLocationOption('not feasible nodes') not in self.none_list \
and len(open(self.getSampleLocationOption('not feasible nodes'),'r').readlines())!=0:
label_map_file = "_MERLION_LABEL_MAP.txt"
name_to_id={}
f = open(self.getConfigureOption('output prefix')+label_map_file,'r')
for line in f:
t = line.split()
name_to_id[t[0]] = t[1]
f.close()
for line in open(self.getSampleLocationOption('not feasible nodes'),'r'):
l=line.split()
for n_ in l:
if name_to_id.has_key(n_)!=True:
print '\nERROR: Nodename ',n_,' specified in ',self.getSampleLocationOption('not feasible nodes')\
,' is not part of the network'
exit(1)
not_allowed_nodes_set.add(int(name_to_id[n_]))
"""
#run pyomo or ampl
if self.getSampleLocationOption('model format') == 'AMPL':
exe = self.getConfigureOption('ampl executable')
if self.getConfigureOption(
'output prefix') not in self.none_list:
inp = self.getConfigureOption(
'output prefix') + '_ampl.run'
out = self.getConfigureOption(
'output prefix') + '_ampl.out'
else:
inp = 'ampl.run'
out = 'ampl.out'
results_file = self.createAMPLRun(inp,
not_allowed_nodes_set)
print results_file
cmd = '%s %s' % (exe, inp)
logger.info("Launching AMPL ...")
sub_logger = logging.getLogger(
'wst.grabsample.models.ampl')
sub_logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(out, mode='w')
sub_logger.addHandler(fh)
p = pyutilib.subprocess.run(
cmd,
timelimit=solve_timelimit,
stdout=pywst.common.problem.LoggingFile(sub_logger))
if (p[0] or not os.path.isfile(results_file)):
message = 'An error occured when running the optimization problem.\n Error Message: ' + p[
1] + '\n Command: ' + cmd + '\n'
logger.error(message)
raise RuntimeError(message)
#try to load the results file
#print "results file: "+ results_file
filereader = open(results_file, 'r')
line1 = filereader.readline().split()
nodes = []
for l in xrange(0, len(line1)):
if l > 0 and line1[l] not in fixed_sensor_ID:
nodes.append(line1[l])
objective = float(filereader.readline().split()[1])
filereader.close()
Solution = (objective, nodes)
#print Solution
elif self.getSampleLocationOption('model format') == 'PYOMO':
logger.info("Launching PYOMO ...")
Solution = self.runPYOMOmodel(fixed_sensor_ID,
not_allowed_nodes_set)
# Convert node ID's to node names in Solution
for i in xrange(0, len(Solution[1])):
Solution[1][i] = nodemap[str(Solution[1][i])]
# Get information to print results to the screen
if self.getEventsOption('scn file') not in self.none_list:
events_filename = self.getEventsOption('scn file')
scnfile = open(events_filename)
content_scn = scnfile.read()
scnfile.close()
number_events = content_scn.count('scenario')
elif self.getEventsOption('tsg file') not in self.none_list:
events_filename = self.getEventsOption('tsg file')
number_events = len([line for line in open(events_filename, 'r')])
elif self.getEventsOption('signals') not in self.none_list:
events_filename = self.getEventsOption('signals')
number_events = self.signal_scenarios
# remove temporary files if debug = 0
if self.opts['configure']['debug'] == 0:
pyutilib.services.TempfileManager.clear_tempfiles()
# write output file
prefix = os.path.basename(self.opts['configure']['output prefix'])
logfilename = logger.parent.handlers[0].baseFilename
outfilename = logger.parent.handlers[0].baseFilename.replace(
'.log', '.yml')
visymlfilename = logger.parent.handlers[0].baseFilename.replace(
'.log', '_vis.yml')
# Get node list from Solution
node_list = Solution[1]
#for i in xrange(0,len(Solution[1])):
# if self.getSampleLocationOption('greedy selection'):
# node_list.append(Solution[1][i])
# else:
# node_list.append(nodemap[str(Solution[1][i])])
sample_time = self.getSampleLocationOption('sample time')
N_samples = self.getSampleLocationOption('num samples')
threshold = self.getSampleLocationOption('threshold')
# Write output yml file
config = wst_config.output_config()
module_blocks = ("general", "grabsample")
template_options = {
'general': {
'cpu time': round(time.time() - self.startTime, 3),
'directory': os.path.dirname(logfilename),
'log file': os.path.basename(logfilename)
},
'grabsample': {
'nodes': node_list,
'objective': Solution[0],
'threshold': threshold,
'count': N_samples,
'time': sample_time
}
}
if outfilename != None:
self.saveOutput(outfilename, config, module_blocks,
template_options)
# Write output visualization yml file
config = wst_config.master_config()
module_blocks = ("network", "visualization", "configure")
template_options = {
'network': {
'epanet file':
os.path.abspath(self.opts['network']['epanet file'])
},
'visualization': {
'layers': [{
'label': 'Optimal sample locations',
'locations': "['grabsample']['nodes'][i]",
'file': outfilename,
'location type': 'node',
'shape': 'circle',
'fill': {
'color': 'blue',
'size': 15
}
}]
},
'configure': {
'output prefix':
os.path.abspath(self.opts['configure']['output prefix'])
}
} #os.path.join('vis', os.path.basename(self.opts['configure']['output prefix']))}}
if visymlfilename != None:
self.saveVisOutput(visymlfilename, config, module_blocks,
template_options)
# Run visualization
cmd = ['wst', 'visualization', visymlfilename]
p = pyutilib.subprocess.run(
cmd) # logging information should not be printed to the screen
# print solution to screen
logger.info("\nWST normal termination")
logger.info("---------------------------")
logger.info("Directory: " + os.path.dirname(logfilename))
logger.info("Results file: " + os.path.basename(outfilename))
logger.info("Log file: " + os.path.basename(logfilename))
logger.info("Visualization configuration file: " +
os.path.basename(visymlfilename) + '\n')
return Solution
def expand_tsg(opts):
import itertools
# read in tsg file, output extended tsg file
# tsg format = <Src1><SrcN> <SrcType> <SrcSpecie> <SrcStrngth> <Start> <Stop>
# Srci = ALL, NZD, or EPANET node id [str]
# SrcType = MASS, CONCEN, FLOWPACED, or SETPOINT [str]
# SrcSpecies = The character ID of the water quality specie added by the source [str]
# SrcStrngth = contaminant strength, units depend on SrcType
# Start, Stop = time in seconds [integer]
fidIn = open(opts['scenario']['tsg file'], 'r')
prefix = opts['configure']['output prefix']
extTSGfile = get_tempfile(prefix, '.extended.tsg')
fidOut = open(extTSGfile, 'w')
fidOut.write('; Extended tsg file\n')
try:
enData = pyepanet.ENepanet()
enData.ENopen(opts['network']['epanet file'],
prefix + 'epanet_tmp.rpt')
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
nnodes = enData.ENgetcount(pyepanet.EN_NODECOUNT) - enData.ENgetcount(
pyepanet.EN_TANKCOUNT)
all_nodes = []
nzd_nodes = []
for i in xrange(enData.ENgetcount(pyepanet.EN_NODECOUNT)):
if enData.ENgetnodetype(i + 1) == pyepanet.EN_JUNCTION:
all_nodes.append(enData.ENgetnodeid(i + 1))
dem = enData.ENgetnodevalue(i + 1, pyepanet.EN_BASEDEMAND)
if dem > 0:
nzd_nodes.append(enData.ENgetnodeid(i + 1))
enData.ENclose()
nScenarios = 0
while True:
line = fidIn.readline()
if not line:
break
line = line.split()
if len(line) == 0 or line[0] == ';':
continue
source_nodes = []
for i in range(len(line)):
if line[i] not in [
'MASS', 'CONCEN', 'FLOWPACED', 'SETPOINT', 'mass',
'concen', 'flowpaced', 'setpoint'
]:
if line[i] == 'ALL':
source_nodes.append(all_nodes)
elif line[i] == 'NZD':
source_nodes.append(nzd_nodes)
else:
source_nodes.append([line[i]])
else:
scenarios = list(itertools.product(*source_nodes))
for j in range(len(scenarios)):
scenarios[j] = list(scenarios[j])
for k in scenarios[j]:
fidOut.write('%s ' % k)
for k in range(i, len(line)):
fidOut.write('%s ' % line[k])
fidOut.write('\n')
break
#nScenarios = nScenarios + len(scenarios)
fidIn.close()
fidOut.close()
return extTSGfile
def eventDetection_merlion(opts, detection, raw_options_string=''):
import time
start = time.time()
logger = logging.getLogger('wst.eventdetection')
logger.info(
"Calculating detection time for each scenario using MERLION water quality model..."
)
cmd = pyutilib.services.registered_executable('eventDetection')
if cmd is None:
raise RuntimeError(
"Cannot find the eventDetection executable on the system PATH")
else:
cmd = cmd.get_path() + ' '
if raw_options_string != '':
cmd += raw_options_string + ' '
# water quality model file
cmd += '--inp ' + opts['network']['epanet file'] + ' '
if opts['scenario']['ignore merlion warnings'] is True:
cmd += '--ignore-merlion-warnings '
if opts['scenario']['tsi file'] not in none_list:
cmd += '--tsi ' + opts['scenario']['tsi file'] + ' '
elif opts['scenario']['tsg file'] not in none_list:
cmd += '--tsg ' + opts['scenario']['tsg file'] + ' '
# output prefix
prefix = opts['configure']['output prefix']
if prefix is None:
prefix = ''
cmd += '--output-prefix=' + prefix + ' '
tmpDir = os.path.dirname(prefix)
# sensor file
if detection.__class__ is list:
# handle the case where we are given a list of sensor names
tmp_sensor_file = get_tempfile(prefix, '.boostersim.sensors')
f = open(tmp_sensor_file, 'w')
for s in detection:
print >> f, s
f.close()
cmd += '--sensors ' + tmp_sensor_file + ' '
else:
if detection.strip() != '':
# handle the case where we are given a sensor file
cmd += '--sensors ' + detection + ' '
out = prefix + 'eventDetection.out'
sim_timelimit = None
p = pyutilib.subprocess.run(cmd,
timelimit=sim_timelimit,
outfile=out,
cwd=tmpDir)
if (p[0]):
raise RuntimeError("An error occured when running the eventDetection executable\n" +\
"Error Message: "+ p[1]+"\nCommand: "+ cmd)
# open EventAnalysis.txt and load array into time_detect
try:
enData = pyepanet.ENepanet()
enData.ENopen(opts['network']['epanet file'],
prefix + 'epanet_tmp.rpt')
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
# convert time units for flushing app (integer minutes)
simulationDuration = int(enData.ENgettimeparam(pyepanet.EN_DURATION) / 60)
enData.ENclose()
fidIn = open(prefix + 'EventAnalysis.txt', 'r')
lines = fidIn.readlines()
fidIn.close()
time_detect = list()
for i in range(len(lines)):
if lines[i].strip('\n') in none_list:
time_detect.append(simulationDuration)
else:
time_detect.append(int(lines[i].strip('\n')) / 60)
return time_detect
def setFeasibleNodes(self):
# check that the give epanet file exists, otherwise epanet will just cause a segfault
try:
fp = open(self.getNetworkOption('epanet file'))
fp.close()
except IOError:
raise
try:
enData = pyepanet.ENepanet()
epanet_rpt_file = pyutilib.services.TempfileManager.create_tempfile(suffix = '.epanet.rpt')
enData.ENopen(self.getNetworkOption('epanet file'),epanet_rpt_file)
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
nnodes = enData.ENgetcount(pyepanet.EN_NODECOUNT)
# set feasible locations
list_feas = []
feasible = self.opts['booster']['feasible nodes']
if feasible == 'ALL':
for i in range(nnodes):
if enData.ENgetnodetype(i+1) != pyepanet.EN_TANK:
if enData.ENgetnodeid(i+1) not in self.getBoosterOption('source nodes'):
list_feas.append(enData.ENgetnodeid(i+1))
elif feasible == 'NZD':
for i in range(nnodes):
if enData.ENgetnodetype(i+1) != pyepanet.EN_TANK:
dem = enData.ENgetnodevalue(i+1,pyepanet.EN_BASEDEMAND)
if dem > 0:
list_feas.append(enData.ENgetnodeid(i+1))
elif feasible.__class__ is list:
for i in feasible:
list_feas.append(str(i))
elif feasible in self.none_list:
# prevents entering next 'elif' block
pass
elif feasible.__class__ is str:
try:
fid = open(feasible,'r')
except:
raise RuntimeError("feasible nodes file did not load")
list_feas = fid.read()
fid.close()
list_feas = list_feas.splitlines()
else:
print >> sys.stderr, "Unsupported feasible nodes, setting option to None"
self.setBoosterOption('feasible nodes',None)
# set infeasible locations
list_infeas = []
infeasible = self.opts['booster']['infeasible nodes']
if infeasible == 'ALL':
for i in range(nnodes):
list_infeas.append(enData.ENgetnodeid(i+1))
elif infeasible == 'NZD':
for i in range(nnodes):
dem = enData.ENgetnodevalue(i+1,pyepanet.EN_BASEDEMAND)
if dem > 0:
list_infeas.append(enData.ENgetnodeid(i+1))
elif infeasible.__class__ is list:
for i in infeasible:
list_infeas.append(str(i))
elif infeasible in self.none_list:
# prevents entering next 'elif' block
pass
elif infeasible.__class__ is str:
try:
fid = open(infeasible,'r')
except:
raise RuntimeError("Infeasible nodes file did not load")
list_infeas = fid.read()
fid.close()
list_infeas = list_infeas.splitlines()
else:
print "Unsupported infeasible nodes, setting option to None"
self.setBoosterOption('infeasible nodes',None)
# remove infeasible from feasible
final_list = []
for i in list_feas:
if i not in list_infeas:
final_list.append(i)
# make sure the list consists of strings
self.setBoosterOption('fixed nodes', [str(node) for node in self.getBoosterOption('fixed nodes')])
for node in self.getBoosterOption('fixed nodes'):
if str(node) not in final_list:
raise ValueError("Fixed node with name %s is not in the list of feasible nodes." % (node))
# assign to internal parameters
self.opts['internal']['nodeNames'] = list(final_list) #copy
if len(final_list) == 0:
raise RuntimeError("\nERROR: List of feasible booster node locations is empty.\n")
elif self.getBoosterOption('evaluate') is True:
print
print "Evaluation Mode: Fixing the booster station solution to the set of candidate booster station nodes."
print
self.setBoosterOption( 'max boosters', [len(self.opts['internal']['nodeNames'])] )
elif len(final_list) < max(self.getBoosterOption('max boosters')):
print >> sys.stderr, ' '
print >> sys.stderr, "WARNING: max boosters is larger than the number of candidate booster nodes.\n"
print >> sys.stderr, ' '
enData.ENclose()
return
def run(self):
# setup logger
logger = logging.getLogger('wst.booster_msx')
logger.info("WST booster_msx subcommand")
logger.info("---------------------------")
# set start time
self.startTime = time.time()
# validate input
logger.info("Validating configuration file")
self.validate()
# open inp file, set feasible nodes
try:
enData = pyepanet.ENepanet()
enData.ENopen(self.opts['network']['epanet file'], 'tmp.rpt')
except:
rmsg = "Error: EPANET inp file not loaded using pyepanet"
logger.error(msg)
raise RuntimeError(msg)
nlinks = enData.ENgetcount(pyepanet.EN_LINKCOUNT)
self.all_link_ids = [enData.ENgetlinkid(i + 1) for i in range(nlinks)]
self.all_link_endpoints = dict(
(i + 1, enData.ENgetlinknodes(i + 1)) for i in range(nlinks))
self.node_names, self.node_indices = wst_util.feasible_nodes(\
self.opts['booster msx']['feasible nodes'],\
self.opts['booster msx']['infeasible nodes'], \
self.opts['booster msx']['max boosters'], enData)
#if len(self.node_names) == 0:
# logger.warn('List of feasible node locations is empty. Booster msx will default to using all nzd junctions as feasible node locations.')
if len(self.node_names) < self.opts['booster msx']['max boosters']:
logger.warn(
'Max nodes reduced to match number of feasible locations.')
self.opts['booster msx']['max boosters'] = len(self.node_names)
enData.ENclose()
# write tmp TSG file if ['scenario']['tsg file'] == none
if self.opts['scenario']['tsi file'] in pywst.common.problem.none_list:
if self.opts['scenario'][
'tsg file'] in pywst.common.problem.none_list:
tmpdir = os.path.dirname(
self.opts['configure']['output prefix'])
tmpTSGFile = pyutilib.services.TempfileManager.create_tempfile(
dir=tmpdir, prefix='tmp_', suffix='.tsg')
wst_util.write_tsg(self.opts['scenario']['location'],\
self.opts['scenario']['type'],\
self.opts['scenario']['species'],\
self.opts['scenario']['strength'],\
self.opts['scenario']['start time'],\
self.opts['scenario']['end time'],\
tmpTSGFile)
self.opts['scenario']['tsg file'] = tmpTSGFile
# expand tsg file
extTSGfile = wst_util.expand_tsg(self.opts)
self.opts['scenario']['tsg file'] = extTSGfile
# get detection times
self.time_detect = wst_util.eventDetection_tevasim(
self.opts, self.opts['booster msx']['detection'])
self.results = {
'dateOfLastRun': '',
'nodesToBoost': [],
'finalMetric': -999,
'runTime': None
}
self.results['dateOfLastRun'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
# define tmp filenames for booster_msx
tmpdir = os.path.dirname(self.opts['configure']['output prefix'])
#ftevasim = pyutilib.services.TempfileManager.create_tempfile(dir=tmpdir, prefix='tmp_', suffix='_tevasim.yml')
#fsim2Impact = pyutilib.services.TempfileManager.create_tempfile(dir=tmpdir, prefix='tmp_', suffix='_sim2Impact.yml')
finp = pyutilib.services.TempfileManager.create_tempfile(dir=tmpdir,
prefix='tmp_',
suffix='.in')
fout = pyutilib.services.TempfileManager.create_tempfile(dir=tmpdir,
prefix='tmp_',
suffix='.out')
ffwd = pyutilib.services.TempfileManager.create_tempfile(
dir=tmpdir, prefix='tmp_', suffix='_fwd.py')
_solver_type = self.opts['solver']['type']
# coliny_ea solver (through DAKOTA)
if _solver_type == 'dakota:coliny_ea' or _solver_type == 'coliny_ea':
self.createDakotaInput(filename=finp, fwdRunFilename=ffwd)
self.createDriverScripts(filename=ffwd, driver='dakota')
logger.info('Launching Dakota ...')
dexe = self.opts['configure']['dakota executable']
if dexe is None:
msg = "Error: Cannot find the dakota executable on the system PATH"
logger.error(msg)
raise RuntimeError(msg)
cmd = ' '.join([dexe, '-input', finp, '-output', fout])
logger.debug(cmd)
sub_logger = logging.getLogger('wst.booster_msx.dakota')
sub_logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(fout, mode='w')
sub_logger.addHandler(fh)
p = pyutilib.subprocess.run(
cmd, stdout=pywst.common.problem.LoggingFile(sub_logger))
sub_logger.removeHandler(fh)
fh.close()
self.parseDakotaOutput(fout)
# coliny_ea solver (through COLINY)
elif _solver_type == 'coliny:ea':
self.createColinyInput('sco:ea', [
'max_iterations', 'max_function_evaluations',
'population_size', 'initialization_type', 'fitness_type',
'crossover_rate', 'crossover_type', 'mutation_rate',
'mutation_type', 'seed'
],
filename=finp,
fwdRunFilename=ffwd)
self.createDriverScripts(filename=ffwd, driver='coliny')
logger.info('Launching Coliny ...')
cexe = self.opts['configure']['coliny executable']
if cexe is None:
msg = "Error: Cannot find the coliny executable on the system PATH"
logger.error(msg)
raise RuntimeError(msg)
cmd = ' '.join([cexe, finp])
logger.debug(cmd)
sub_logger = logging.getLogger('wst.booster_msx.coliny')
sub_logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(fout, mode='w')
sub_logger.addHandler(fh)
p = pyutilib.subprocess.run(
cmd, stdout=pywst.common.problem.LoggingFile(sub_logger))
sub_logger.removeHandler(fh)
fh.close()
self.parseColinyOutput(fout)
# StateMachine LS (through COLINY)
elif _solver_type == 'coliny:StateMachineLS':
try:
enData = pyepanet.ENepanet()
enData.ENopen(self.opts['network']['epanet file'], 'tmp.rpt')
except:
msg = "Error: EPANET inp file not loaded using pyepanet"
logger.error(msg)
raise RuntimeError(msg)
nJunctions = enData.ENgetcount(pyepanet.EN_NODECOUNT) \
- enData.ENgetcount(pyepanet.EN_TANKCOUNT)
enData.ENclose()
init_pts = self.opts['solver']['initial points']
self.createStateMachineInput()
self.createColinyInput(
'sco:StateMachineLS',
['verbosity', 'max_iterations', 'max_fcn_evaluations'],
filename=finp,
fwdRunFilename=ffwd,
init_pts=init_pts,
repn='bin')
self.createDriverScripts(filename=ffwd,
driver='coliny',
repn='bin')
logger.info('Launching Coliny ...')
cexe = self.opts['configure']['coliny executable']
cmd = ' '.join([cexe, finp])
logger.debug(cmd)
sub_logger = logging.getLogger('wst.booster_msx.coliny')
sub_logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(fout, mode='w')
sub_logger.addHandler(fh)
p = pyutilib.subprocess.run(
cmd, stdout=pywst.common.problem.LoggingFile(sub_logger))
sub_logger.removeHandler(fh)
fh.close()
self.parseColinyOutput(fout, repn='bin')
elif _solver_type == 'EVALUATE':
self.createDakotaInput(filename=finp, fwdRunFilename=ffwd)
self.createDriverScripts(filename=ffwd, driver='dakota')
logger.info('Evaluate Placement ...')
# create dummy params.in
fid = open('params.in', 'w')
fid.write('%d variables\n' % len(self.node_names))
for i in range(len(self.node_names)):
fid.write('%s x%d\n' % (i + 1, i + 1))
#fid.write('1 functions\n')
#fid.write('1 ASV_1\n')
#fid.write('2 derivative_variables\n')
#fid.write('1 DVV_1\n')
#fid.write('2 DVV_2\n')
#fid.write('0 analysis_components\n')
fid.close()
eval_out_file = pyutilib.services.TempfileManager.create_tempfile(
dir=tmpdir, prefix='tmp_', suffix='_eval.out')
cmd = ' '.join(['python', ffwd, 'params.in', eval_out_file])
logger.debug(cmd)
sub_logger = logging.getLogger('wst.booster_msx.exec')
sub_logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(fout, mode='w')
sub_logger.addHandler(fh)
p = pyutilib.subprocess.run(
cmd, stdout=pywst.common.problem.LoggingFile(sub_logger))
sub_logger.removeHandler(fh)
fh.close()
fid = open(eval_out_file, 'r')
self.results['finalMetric'] = float(fid.read())
self.results['nodesToBoost'] = self.opts['booster msx'][
'feasible nodes']
fid.close()
else:
raise Exception("ERROR: Unknown or unsupported solver, '%s'" %
_solver_type)
# remove temporary files if debug = 0
if self.opts['configure']['debug'] == 0:
pyutilib.services.TempfileManager.clear_tempfiles()
if os.path.exists('./hydraulics.hyd'):
os.remove('./hydraulics.hyd')
if os.path.exists('./tmp.rpt'):
os.remove('./tmp.rpt')
# write output file
prefix = os.path.basename(self.opts['configure']['output prefix'])
logfilename = logger.parent.handlers[0].baseFilename
outfilename = logger.parent.handlers[0].baseFilename.replace(
'.log', '.yml')
visymlfilename = logger.parent.handlers[0].baseFilename.replace(
'.log', '_vis.yml')
# Write output yml file
config = wst_config.output_config()
module_blocks = ("general", "booster")
template_options = {
'general': {
'cpu time': round(time.time() - self.startTime, 3),
'directory': os.path.dirname(logfilename),
'log file': os.path.basename(logfilename)
},
'booster': {
'nodes': self.results['nodesToBoost'],
'objective': self.results['finalMetric']
}
}
self.saveOutput(outfilename, config, module_blocks, template_options)
# Write output visualization yml file
config = wst_config.master_config()
module_blocks = ("network", "visualization", "configure")
template_options = {
'network': {
'epanet file':
os.path.abspath(self.opts['network']['epanet file'])
},
'visualization': {
'layers': []
},
'configure': {
'output prefix':
os.path.abspath(self.opts['configure']['output prefix'])
}
}
if len(self.opts['booster msx']['detection']) > 0:
template_options['visualization']['layers'].append({
'label':
'Sensor locations',
'locations':
self.opts['booster msx']['detection'],
'location type':
'node',
'shape':
'square',
'fill': {
'color': '#000099"',
'size': 15,
'opacity': 0
},
'line': {
'color': '#000099',
'size': 2,
'opacity': 0.6
}
})
if len(self.results['nodesToBoost']) > 0:
template_options['visualization']['layers'].append({
'label':
'Booster locations',
'locations':
"['booster']['nodes'][i]",
'file':
outfilename,
'location type':
'node',
'shape':
'circle',
'fill': {
'color': '#aa0000',
'size': 15,
'opacity': 0.6
},
'line': {
'color': '#000099',
'size': 1,
'opacity': 0
}
})
if visymlfilename != None:
self.saveVisOutput(visymlfilename, config, module_blocks,
template_options)
# Run visualization
cmd = ['wst', 'visualization', visymlfilename]
p = pyutilib.subprocess.run(
cmd) # logging information should not be printed to the screen
# print solution to screen
logger.info("\nWST Normal Termination")
logger.info("---------------------------")
logger.info("Directory: " + os.path.dirname(logfilename))
logger.info("Results file: " + os.path.basename(outfilename))
logger.info("Log file: " + os.path.basename(logfilename))
logger.info("Visualization configuration file: " +
os.path.basename(visymlfilename) + '\n')
return
def createDriverScripts(self, filename=None, driver=None, repn='idx'):
try:
enData = pyepanet.ENepanet()
enData.ENopen(self.opts['network']['epanet file'], 'tmp.rpt')
except:
raise RuntimeError("EPANET inp file not loaded using pyepanet")
# convert time unit to pattern units for msx file
import math
patternTimestep = enData.ENgettimeparam(
pyepanet.EN_PATTERNSTEP) # returns timestep (in seconds)
simulationDuration = math.floor(
int(enData.ENgettimeparam(pyepanet.EN_DURATION) / patternTimestep))
enData.ENclose()
responseTime = math.floor(self.opts['booster msx']['response time'] /
(patternTimestep / 60))
duration = math.floor(self.opts['booster msx']['duration'] /
(patternTimestep / 60))
timeDetect = [
int(math.floor(x / (patternTimestep / 60)))
for x in self.time_detect
]
if filename is None:
filename = self.opts['configure']['output prefix'] + '_fwd.py'
fid = open(filename, 'w')
fid.write('#!' + sys.executable + '\n')
fid.write(python_template_a)
fid.write('verbose = %d\n' % (self.opts['solver']['verbose'], ))
# booster options
fid.write('timeDetect = %r\n' % (timeDetect, ))
fid.write('responseDelay = %d\n' % responseTime)
fid.write('boostConc = %r\n' %
(self.opts['booster msx']['strength'], ))
fid.write('boostLength = %d\n' % (duration, ))
fid.write('simDuration = %d\n' % (simulationDuration, ))
fid.write('toxinSpecies = %r\n' %
(self.opts['booster msx']['toxin species'], ))
fid.write('deconSpecies = %r\n' %
(self.opts['booster msx']['decon species'], ))
fid.write('maxNodes = %d\n' %
(self.opts['booster msx']['max boosters'], ))
fid.write('nodeMap = %r\n' % (self.node_names, ))
fid.write('prefix = %r\n' %
'tmp') # (self.opts['configure']['output prefix'],))
fid.write('prefixDir = r"%s"\n' %
os.path.dirname(self.opts['configure']['output prefix']))
# network options
fid.write('epanetInputFile = r"%s"\n' %
os.path.abspath(self.opts['network']['epanet file']))
# scenario options
if self.opts['scenario'][
'tsi file'] not in pywst.common.problem.none_list:
fid.write('scenarioFile = %r\n' %
(self.opts['scenario']['tsi file'], ))
else:
fid.write('scenarioFile = %r\n' %
(self.opts['scenario']['tsg file'], ))
fid.write('msxFile = r"%s"\n' %
os.path.abspath(self.opts['scenario']['msx file']))
fid.write('merlion = %r\n' % (self.opts['scenario']['merlion'], ))
fid.write('erdCompression = %r\n' %
(self.opts['scenario']['erd compression'], ))
fid.write('merlionNsims = %r\n' %
(self.opts['scenario']['merlion nsims'], ))
fid.write('ignoreMerlionWarnings = %r\n' %
(self.opts['scenario']['ignore merlion warnings'], ))
# impact options
fid.write('metric = %r\n' %
(self.opts['impact']['metric'][0].lower(), ))
if self.opts['impact'][
'tai file'] not in pywst.common.problem.none_list:
fid.write('healthImpactsFile = r"%s"\n' %
os.path.abspath(self.opts['impact']['tai file']))
else:
fid.write('healthImpactsFile = r"%s"\n' %
self.opts['impact']['tai file'])
fid.write('responseTime = %r\n' %
(self.opts['impact']['response time'], ))
fid.write('detectionLimit = %r\n' %
(self.opts['impact']['detection limit'], ))
fid.write('detectionConfidence = %r\n' %
(self.opts['impact']['detection confidence'], ))
# executables
fid.write('tevasimexe = %r\n' %
(self.opts['configure']['tevasim executable'], ))
fid.write('tso2Impactexe = %r\n' %
(self.opts['configure']['sim2Impact executable'], ))
fid.write('driver = %s_Problem("%s")\n' % (driver, repn))
fid.write('\n\n')
fid.write(python_template_b)
fid.close()
#cmdfilename = self.opts['configure']['output prefix']+'_dakota.cmd'
#fid = open(cmdfilename,'wt')
#fid.write('"%s" -input "%s" -output "%s"'%(self.opts['configure']['dakota executable'],
# self.opts['configure']['output prefix']+'_dakota.in',
# self.opts['configure']['output prefix']+'_dakota.out'))
#fid.close()
if os.name in ['nt', 'dos']:
batfilename = filename.replace('.py', '.bat')
fid = open(batfilename, 'wt')
fid.write('@echo off\n')
fid.write('"%s" "%s" %%*%%' % (sys.executable, filename))
fid.close()
else:
os.chmod(filename, 0755)
#os.chmod(cmdfilename,0755)
return