def validate(dump_dir, p4_file, log_file, config):
try:
result = validate_p4(p4_file, dump_dir, config["compiler_bin"],
log_file)
except TimeoutError:
log.error("Validation timed out.")
dump_file(TIMEOUT_DIR, p4_file)
dump_file(TIMEOUT_DIR, log_file)
# reset the dump directory
return util.EXIT_FAILURE
if result != util.EXIT_SUCCESS:
info_file = p4_file.with_suffix("").joinpath(
f"{p4_file.stem}_info.json")
bug_dir = None
if result == util.EXIT_UNDEF:
log.error("Found instance of unstable code!")
bug_dir = UNDEF_DIR
else:
log.error("Failed to validate the P4 code!")
bug_dir = VALIDATION_BUG_DIR
log.error("Rerun the example with:")
out_file = bug_dir.joinpath(p4_file.name)
log.error("python3 bin/validate_p4_translation -u -i %s", out_file)
dump_file(bug_dir, log_file)
dump_file(bug_dir, p4_file)
dump_file(bug_dir, info_file)
if config["do_prune"]:
info_file = bug_dir.joinpath(f"{p4_file.stem}_info.json")
p4_cmd = f"{PRUNER_BIN} "
p4_cmd += f"--config {info_file} "
p4_cmd += f" {bug_dir.joinpath(f'{p4_file.stem}.p4')} "
p4_cmd += f" --working-dir {bug_dir.joinpath(f'{p4_file.stem}')}"
log.info("Pruning P4 file with command %s ", p4_cmd)
util.start_process(p4_cmd)
return result
def check(idx, config):
test_id = generate_id()
test_name = f"{test_id}_{idx}"
dump_dir = OUTPUT_DIR.joinpath(f"dmp_{test_name}")
util.check_dir(dump_dir)
log_file = dump_dir.joinpath(f"{test_name}.log")
p4_file = dump_dir.joinpath(f"{test_name}.p4")
seed = int.from_bytes(os.getrandom(8), "big")
log.info("Testing P4 program: %s - Seed: %s", p4_file.name, seed)
# generate a random program
result, p4_file = generate_p4_prog(P4RANDOM_BIN, p4_file, config, seed)
if result.returncode != util.EXIT_SUCCESS:
log.error("Failed generate P4 code!")
dump_result(result, GENERATOR_BUG_DIR, p4_file)
# reset the dump directory
util.del_dir(dump_dir)
return result.returncode
# check compilation
result = compile_p4_prog(config["compiler_bin"], p4_file, dump_dir)
if result.returncode != util.EXIT_SUCCESS:
if not is_known_bug(result):
log.error("Failed to compile the P4 code!")
log.error("Found a new bug!")
dump_result(result, CRASH_BUG_DIR, p4_file)
dump_file(CRASH_BUG_DIR, p4_file)
if config["do_prune"]:
info_file = CRASH_BUG_DIR.joinpath(f"{p4_file.stem}_info.json")
info = validation.INFO
# customize the main info with the new information
info["compiler"] = str(config["compiler_bin"])
info["exit_code"] = result.returncode
info["p4z3_bin"] = str(P4Z3_BIN)
info["out_dir"] = str(CRASH_BUG_DIR)
info["input_file"] = str(p4_file)
info["allow_undef"] = False
info["err_string"] = result.stderr.decode("utf-8")
log.error("Dumping configuration to %s.", info_file)
with open(info_file, 'w') as json_file:
json.dump(info, json_file, indent=2, sort_keys=True)
p4_cmd = f"{PRUNER_BIN} "
p4_cmd += f"--config {info_file} "
p4_cmd += f" {CRASH_BUG_DIR.joinpath(f'{p4_file.stem}.p4')} "
log.error("Pruning P4 file with command %s ", p4_cmd)
util.start_process(p4_cmd)
# reset the dump directory
util.del_dir(dump_dir)
return result
# check validation
if config["do_validate"]:
result = validate(dump_dir, p4_file, log_file, config)
elif config["use_blackbox"]:
result = run_p4_test(dump_dir, p4_file, log_file, config)
# reset the dump directory
util.del_dir(dump_dir)
return result
def ovs_vsctl_is_ovs_bridge(bridge):
"""
This function verifies whether given port is an OVS bridge. If it is an
OVS bridge then it will return True.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "br-exists", bridge])
return ret == 0
def ovs_vsctl_add_port_to_bridge(bridge, iface):
"""
This function adds given interface to the bridge.
"""
ret, _out, _err = util.start_process(
["ovs-vsctl", "add-port", bridge, iface])
return ret
def ovs_vsctl_is_ovs_bridge(bridge):
"""
This function verifies whether given port is an OVS bridge. If it is an
OVS bridge then it will return True.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "br-exists", bridge])
return ret == 0
def ovs_vsctl_add_port_to_bridge(bridge, iface):
"""
This function adds given interface to the bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "add-port", bridge,
iface])
return ret
def ovs_get_physical_interface(bridge):
"""
This function tries to figure out which is the physical interface that
belongs to the bridge. If there are multiple physical interfaces assigned
to this bridge then it will return the first match.
"""
ret, out, _err = util.start_process(["ovs-vsctl", "list-ifaces", bridge])
if ret == 0:
ifaces = out.splitlines()
for iface in ifaces:
ret, out, _err = util.start_process(["ovs-vsctl", "get",
"Interface", iface, "type"])
if ret == 0:
if ('""' in out) or ('system' in out):
return iface # this should be the physical interface
return None
def ovs_get_physical_interface(bridge):
"""
This function tries to figure out which is the physical interface that
belongs to the bridge. If there are multiple physical interfaces assigned
to this bridge then it will return the first match.
"""
ret, out, _err = util.start_process(["ovs-vsctl", "list-ifaces", bridge])
if ret == 0:
ifaces = out.splitlines()
for iface in ifaces:
ret, out, _err = util.start_process(
["ovs-vsctl", "get", "Interface", iface, "type"])
if ret == 0:
if ('""' in out) or ('system' in out):
return iface # this should be the physical interface
return None
def start_grafana(grafana_home_path, grafana_port):
args = [
GRAFANA_BIN, '--homepath', grafana_home_path, '--config', 'custom.ini'
]
log_path = path.join(PROMTIMER_DIR, 'logs/grafana.log')
logging.info(
'starting grafana server (on localhost:{}; logging to {})'.format(
grafana_port, log_path))
# Don't specify a log file as it is done within the custom.ini file
# otherwise the output is duplicated.
return util.start_process(args, None, PROMTIMER_DIR)
def launch_prometheus():
if kube_env.check_kubernetes_status() != util.EXIT_SUCCESS:
log.error("Kubernetes is not set up."
" Did you run the deployment script?")
sys.exit(util.EXIT_FAILURE)
cmd = "kubectl get pods -n istio-system -lapp=prometheus "
cmd += " -o jsonpath={.items[0].metadata.name}"
prom_pod_name = util.get_output_from_proc(cmd).decode("utf-8")
cmd = f"kubectl port-forward -n istio-system {prom_pod_name} 9090"
prom_proc = util.start_process(cmd, preexec_fn=os.setsid)
time.sleep(2)
prom_api = PrometheusConnect(url="http://localhost:9090", disable_ssl=True)
return prom_proc, prom_api
def launch_storage_mon():
if kube_env.check_kubernetes_status() != util.EXIT_SUCCESS:
log.error("Kubernetes is not set up."
" Did you run the deployment script?")
sys.exit(util.EXIT_FAILURE)
cmd = "kubectl get pods -lapp=storage-upstream "
cmd += " -o jsonpath={.items[0].metadata.name}"
storage_pod_name = util.get_output_from_proc(cmd).decode("utf-8")
cmd = f"kubectl port-forward {storage_pod_name} 8090:8080"
storage_proc = util.start_process(cmd, preexec_fn=os.setsid)
# Let settle things in a bit
time.sleep(2)
return storage_proc
def ovs_vsctl_set(table, record, column, key, value):
"""
This function allows to alter the OVS database. If column is a map, then
caller should also set the key, otherwise the key should be left as an
empty string.
"""
if key is None:
index = column
else:
index = "%s:%s" % (column, key)
index_value = "%s=%s" % (index, value)
ret, _out, _err = util.start_process(
["ovs-vsctl", "set", table, record, index_value])
return ret
def ovs_vsctl_set(table, record, column, key, value):
"""
This function allows to alter the OVS database. If column is a map, then
caller should also set the key, otherwise the key should be left as an
empty string.
"""
if key is None:
index = column
else:
index = "%s:%s" % (column, key)
index_value = "%s=%s" % (index, value)
ret, _out, _err = util.start_process(["ovs-vsctl", "set", table, record,
index_value])
return ret
def runFire(scenario):
"""!
Run fire fore given scenario
@param scenario Scenario to run simulations for
@return stdout as a string
@return stderr as a string
"""
raster = None
if scenario.perim:
scenario.perim, raster = rasterize_perim(scenario.run_output,
scenario.perim,
scenario.year,
scenario.fire)
cmd = os.path.join(Settings.HOME_DIR, Settings.BINARY)
a = ['"' + scenario.run_output + '"', scenario.day, str(scenario.lat), str(scenario.lon), scenario.start_time]
if scenario.keep_intensity:
a = a + ["-i"]
args = ' '.join(a)
if scenario.actuals_only:
args = args + " -a"
if scenario.no_async:
args += " -s"
if scenario.current_size and 1 < int(scenario.current_size) and not raster:
args += " --size " + str(int(scenario.current_size))
if raster:
args = args + ' --perim "' + raster + '"'
if scenario.ffmc:
args += " --ffmc " + scenario.ffmc
if scenario.dmc:
args += " --dmc " + scenario.dmc
if scenario.dc:
args += " --dc " + scenario.dc
if scenario.apcp_0800:
args += " --apcp_0800 " + scenario.apcp_0800
if scenario.score:
args += " --score " + scenario.score
# run generated command for parsing data
run_what = [cmd] + shlex.split(args.replace('\\', '/'))
logging.info("Running: " + ' '.join(run_what))
t0 = timeit.default_timer()
stdout, stderr = finish_process(start_process(run_what, scenario.simulation_flags, Settings.HOME_DIR))
t1 = timeit.default_timer()
logging.info("Took {}s to run simulations".format(t1 - t0))
return stdout, stderr
def maybe_start(self, log_dir):
"""
Starts the Prometheus instance that serves stats for this source.
"""
log_path = path.join(log_dir, 'prom-{}.log'.format(self._short_name))
listen_addr = '0.0.0.0:{}'.format(self.port())
args = [
Source.PROMETHEUS_BIN, '--config.file',
path.join(util.get_root_dir(),
'noscrape.yml'), '--storage.tsdb.path',
path.join(self._cbcollect_dir,
'stats_snapshot'), '--storage.tsdb.no-lockfile',
'--storage.tsdb.retention.time', '10y', '--query.lookback-delta',
'600s', '--web.listen-address', listen_addr
]
logging.info(
'starting prometheus server on {} against {}; logging to {}'.
format(listen_addr, path.join(self._cbcollect_dir,
'stats_snapshot'), log_path))
return util.start_process(args, log_path)
def start_prometheuses(cbcollects, base_port, log_dir):
nodes = []
for i, cbcollect in enumerate(cbcollects):
log_path = path.join(log_dir, 'prom-{}.log'.format(i))
listen_addr = '0.0.0.0:{}'.format(base_port + i)
args = [
PROMETHEUS_BIN, '--config.file',
path.join(util.get_root_dir(),
'noscrape.yml'), '--storage.tsdb.path',
path.join(cbcollect,
'stats_snapshot'), '--storage.tsdb.retention.time',
'10y', '--web.listen-address', listen_addr
]
logging.info(
'starting prometheus server {} (on {}; logging to {})'.format(
i, listen_addr, log_path))
node = util.start_process(args, log_path)
nodes.append(node)
return nodes
def run_bmv2_test(out_dir, p4_input, use_psa=False):
cmd = "python3 "
cmd += f"{P4C_DIR}/backends/bmv2/run-bmv2-test.py "
cmd += f"{P4C_DIR} -v "
if use_psa:
cmd += "-p "
cmd += f"-bd {P4C_DIR}/build "
cmd += f"{out_dir}/{p4_input.name} "
test_proc = util.start_process(cmd, cwd=out_dir)
def signal_handler(sig, frame):
log.warning("run_bmv2_test: Caught Interrupt, exiting...")
os.kill(test_proc.pid, signal.SIGINT)
sys.exit(1)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
stdout, stderr = test_proc.communicate()
return test_proc, stdout, stderr
def ovs_vsctl_show():
ret, _out, _err = util.start_process(["ovs-vsctl", "show"])
print _out
def ovs_vsctl_add_trunk_port(port, trunk):
ret, _out, err = util.start_process(["set", "port", port, trunk])
return ret
def ovs_vsctl_del_port_from_bridge(port):
"""
This function removes given port from a OVS bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "del-port", port])
return ret
def ifconfig(ip, port):
ret, _out, err = util.start_process(["ip", "addr", "add", ip, "dev", port])
return err
def iplink(port, status):
ret, _out, err = util.start_process(["ip", "link", "set", port, status])
return err
def ovs_vsctl_del_bridge(bridge):
"""
This function deletes the OVS bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "del-br", bridge])
return ret
def ovs_vsctl_del_bridge(bridge):
"""
This function deletes the OVS bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "del-br", bridge])
return ret
def ovs_vsctl_admin_port(bridge, port, tag):
final_tag = "%s=%s" % ("tag", tag)
ret, _out, err = util.start_process(
["ovs-vsctl", "add-port", bridge, port, final_tag])
return ret
def ovs_vsctl_set_admin(port, type):
final_type = "type=" + type
ret, _out, err = util.start_process(
["ovs-vsctl", "set", "Interface", port, final_type])
def start_fortio(gateway_url):
cmd = f"{FILE_DIR}/bin/fortio "
cmd += "load -c 50 -qps 300 -jitter -t 0 -loglevel Warning "
cmd += f"http://{gateway_url}/productpage"
fortio_proc = util.start_process(cmd, preexec_fn=os.setsid)
return fortio_proc
def run_tofino_test(out_dir, p4_input, stf_file_name):
# we need to change the working directory
# tofino scripts make some assumptions where to dump files
prog_name = p4_input.stem
# we need to create a specific test dir in which we can run tests
test_dir = out_dir.joinpath("test_dir")
util.check_dir(test_dir)
util.copy_file(stf_file_name, test_dir)
template_name = test_dir.joinpath(f"{prog_name}.py")
# use a test template that runs stf tests
util.copy_file(f"{FILE_DIR}/tofino_test_template.py", template_name)
# initialize the target install
log.info("Building the tofino target...")
config_cmd = f"{TOFINO_DIR}/pkgsrc/p4-build/configure "
config_cmd += "--with-tofino --with-p4c=bf-p4c "
config_cmd += f"--prefix={TOFINO_DIR}/install "
config_cmd += f"--bindir={TOFINO_DIR}/install/bin "
config_cmd += f"P4_NAME={prog_name} "
config_cmd += f"P4_PATH={p4_input.resolve()} "
config_cmd += "P4_VERSION=p4-16 "
config_cmd += "P4_ARCHITECTURE=tna "
result = util.exec_process(config_cmd, cwd=out_dir)
if result.returncode != util.EXIT_SUCCESS:
return result, result.stdout, result.stderr
# create the target
make_cmd = f"make -C {out_dir} "
result = util.exec_process(make_cmd)
if result.returncode != util.EXIT_SUCCESS:
return result, result.stdout, result.stderr
# install the target in the tofino folder
make_cmd = f"make install -C {out_dir} "
result = util.exec_process(make_cmd)
if result.returncode != util.EXIT_SUCCESS:
return result, result.stdout, result.stderr
procs = []
test_proc = None
# start the target in the background
log.info("Starting the tofino model...")
os_env = os.environ.copy()
os_env["SDE"] = f"{TOFINO_DIR}"
os_env["SDE_INSTALL"] = f"{TOFINO_DIR}/install"
model_cmd = f"{TOFINO_DIR}/run_tofino_model.sh "
model_cmd += f"-p {prog_name} "
proc = util.start_process(model_cmd,
preexec_fn=os.setsid,
env=os_env,
cwd=out_dir)
procs.append(proc)
# start the binary for the target in the background
log.info("Launching switchd...")
os_env = os.environ.copy()
os_env["SDE"] = f"{TOFINO_DIR}"
os_env["SDE_INSTALL"] = f"{TOFINO_DIR}/install"
switch_cmd = f"{TOFINO_DIR}/run_switchd.sh "
switch_cmd += "--arch tofino "
switch_cmd += f"-p {prog_name} "
proc = util.start_process(switch_cmd,
preexec_fn=os.setsid,
env=os_env,
cwd=out_dir)
procs.append(proc)
# wait for a bit
time.sleep(2)
# finally we can run the test
log.info("Running the actual test...")
test_cmd = f"{TOFINO_DIR}/run_p4_tests.sh "
test_cmd += f"-t {test_dir} "
os_env = os.environ.copy()
os_env["SDE"] = f"{TOFINO_DIR}"
os_env["SDE_INSTALL"] = f"{TOFINO_DIR}/install"
# inserting this path is necessary for the tofino_test_template.py
os_env["PYTHONPATH"] = f"${{PYTHONPATH}}:{ROOT_DIR}"
test_proc = util.start_process(test_cmd, env=os_env, cwd=out_dir)
def signal_handler(sig, frame):
log.warning("run_tofino_test: Caught Interrupt, exiting...")
cleanup(procs)
os.kill(test_proc.pid, signal.SIGINT)
os.kill(test_proc.pid, signal.SIGTERM)
sys.exit(1)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
stdout, stderr = test_proc.communicate()
cleanup(procs)
return test_proc, stdout, stderr
def ovs_vsctl_add_bridge(bridge):
"""
This function creates an OVS bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "add-br", bridge])
return ret
def run(config_file, args):
"""!
Run fire based on configuartion file and command line arguments
@param config_file Configuration file to read Scenario from
@param args argparser to pass to Scenario
@return None
"""
scenario = Scenario(config_file, args)
had_output = scenario.is_current()
ensure_dir(os.path.dirname(scenario.run_output))
sizes = None
# try to find a perimeter and import it
changed = False
def clean_flag():
if os.path.exists(scenario.mapflag):
os.remove(scenario.mapflag)
def cleanup():
if os.path.exists(scenario.run_output):
logging.error("Removing output after run failed for " + scenario.fire)
shutil.rmtree(scenario.run_output, True)
clean_flag()
if scenario.force or not (had_output or os.path.exists(scenario.runflag)):
# HACK: do this right away so that running more than once shouldn't do the same fire in both processes
try:
ensure_dir(scenario.run_output)
write_file(scenario.run_output, "running", " ")
stdout, stderr = runFire(scenario)
write_file(scenario.run_output, "output.txt", '\n'.join(stdout.split('\r\n')))
os.remove(scenario.runflag)
changed = True
except KeyboardInterrupt:
# this doesn't work for some reason
cleanup()
except Exception as e:
logging.fatal("Error running " + scenario.fire)
print(e)
traceback.print_exc()
cleanup()
return
# don't delete output if maps fail
if (changed or scenario.force_maps or scenario.check_maps) and not (scenario.no_maps or os.path.exists(scenario.runflag)):
t0 = timeit.default_timer()
try:
scenario.save_point_shp()
from mxd import makeMaps
pdf_out = makeMaps(scenario, scenario.run_output, scenario.force_maps or changed, scenario.hide)
except Exception as e:
logging.fatal(e)
traceback.print_exc()
cleanup()
# run didn't work before so run it now
run(config_file, args)
try:
if scenario.outbase == Settings.OUTPUT_DEFAULT:
district_folder = os.path.join(Settings.FIRE_ROOT, FOLDER_BY_TLA[scenario.fire[:3]])
to_folder = os.path.join(district_folder, scenario.fire)
to_file = os.path.join(to_folder, 'FireSTARR', os.path.basename(pdf_out))
if not os.path.exists(to_folder):
print("*********************************************")
print("Need to make fire folder for {}".format(scenario.fire))
print("*********************************************")
cmd = r'C:\Windows\System32\cscript.exe'
run_what = [cmd, os.path.join(district_folder, "CreateFireFolder.vbe"), str(int(scenario.fire[3:]))]
logging.debug("Running: " + ' '.join(run_what))
finish_process(start_process(run_what, Settings.PROCESS_FLAGS, district_folder))
if not os.path.exists(to_folder):
logging.fatal("MAKING FOLDER FAILED")
sys.exit(-1)
if not os.path.exists(to_file):
try_copy(pdf_out, to_file)
csv_out = os.path.splitext(pdf_out)[0] + "_assets.csv"
csv_to = os.path.splitext(to_file)[0] + "_assets.csv"
if not os.path.exists(csv_to):
try_copy(csv_out, csv_to)
except Exception as e:
logging.fatal("Couldn't copy to file plan")
print(e)
traceback.print_exc()
clean_flag()
t1 = timeit.default_timer()
logging.info("Took {}s to make maps".format(t1 - t0))
def ovs_vsctl_add_bridge(bridge):
"""
This function creates an OVS bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "add-br", bridge])
return ret
def makeMaps(scenario, run_output, force_maps, hide):
"""!
@param scenario Scenario to use settings from
@param run_output Folder where simulation output resides
@param force_maps Whether or not to force making maps if they already exist
@param hide Whether or not to show perimeter closest to date on map
@return Path to final output pdf
"""
import pdf
from pdf import makePDF
perimeters = PerimeterList(scenario.year, scenario.fire)
sim_output = readSimOutput(run_output)
startup = find_lines(sim_output, 'Startup indices ')
startup = startup[0] if (
len(startup) > 0) else "Startup indices are not valid"
prefix = 'actuals_' if scenario.actuals_only else 'wxshield_'
fire_prefix = scenario.fire + "_" + ('actual_'
if scenario.actuals_only else '')
probs = [
x for x in os.listdir(os.path.join(Settings.HOME_DIR, run_output))
if x.startswith(prefix) and x[-3:] == "asc"
]
day0 = find_day(probs[0]) - 1
jds = map(find_day, probs)
dates = map(find_date, probs)
days = map(lambda x: x - day0, jds)
extent = None
perim = None
ensure_dir(scenario.outbase)
out_dir = os.path.join(scenario.outbase, scenario.fire[:3])
ensure_dir(out_dir)
for_time = os.path.basename(scenario.run_output)
pdf_output = os.path.abspath(
os.path.join(out_dir, fire_prefix + for_time + ".pdf"))
copied = os.path.join(scenario.outbase, os.path.basename(pdf_output))
# HACK: if any one map is required then make them all
if not (force_maps or not os.path.exists(pdf_output)):
logging.info("Maps already exist for " + scenario.fire)
return copied
for_time = os.path.basename(scenario.run_output)
mapflag = os.path.join(out_dir,
scenario.fire + "_" + for_time + "_mapsinprogress")
if os.path.exists(mapflag):
logging.info("Maps already being made for " + scenario.fire)
return copied
write_file(os.path.dirname(mapflag), os.path.basename(mapflag), " ")
map_output = getMapOutput(run_output)
logging.info("Making maps for " + scenario.fire)
# HACK: run in parallel but assume this works for now
wxshield = getWxSHIELDFile(dates[0], scenario.fire, map_output)
processes = []
run_what = r'python.exe firestarr\getWxshield.py {} {} {} {} {} "{}"'.format(
scenario.lat, scenario.lon, dates[0], days[-1], scenario.fire,
map_output)
if 'overridden' in startup:
startup_values = map(lambda x: x.strip(),
startup[startup.find('(') + 1:-1].split(','))
logging.debug(startup_values)
# HACK: just use known positions for now
#~ (0.0mm, FFMC 92.0, DMC 59.0, DC 318.0)
#~ print(startup_values[0][:-2], startup_values[1][5:].strip(), startup_values[2][4:].strip(), startup_values[0][3:].strip())
apcp = float(startup_values[0][:-2])
ffmc = float(startup_values[1][5:].strip())
dmc = float(startup_values[2][4:].strip())
dc = float(startup_values[3][3:].strip())
run_what += ' --apcp_0800 {} --ffmc {} --dmc {} --dc {}'.format(
apcp, ffmc, dmc, dc)
logging.debug(run_what)
processes.append(
start_process(run_what, Settings.PROCESS_FLAGS, Settings.HOME_DIR))
arcpy.env.overwriteOutput = True
ensure_dir(os.path.dirname(out_dir))
ensure_dir(out_dir)
# keep these until the end so they lock the file names
mxd_paths = []
mxd_names = []
risk_paths = []
risk_names = []
scores = []
txtFuelRaster = find_line(sim_output, 'Fuel raster is ', 'Fuel raster is ')
suffix = findSuffix(txtFuelRaster)
env_push()
png_processes = []
arcpy.env.scratchWorkspace = ensure_dir(
arcpy.CreateScratchName(scenario.fire + os.path.basename(run_output),
"", "Workspace",
arcpy.GetSystemEnvironment('TEMP')))
for i in reversed(xrange(len(days))):
f = os.path.join(
run_output, probs[i].replace(prefix,
'sizes_').replace('.asc', '.csv'))
run_what = r'python.exe firestarr\plotsize.py "{}" "{}"'.format(
f, days[i])
png_processes = [
start_process(run_what, Settings.PROCESS_FLAGS, Settings.HOME_DIR)
] + png_processes
for i in reversed(xrange(len(days))):
finish_process(png_processes[i])
arcpy.env.addOutputsToMap = False
prob_input = os.path.join(run_output, probs[i])
c_prob = arcpy.sa.Int(arcpy.sa.Raster(prob_input) * 10)
shp_class = os.path.join(
map_output, probs[i].replace(".asc",
"_class_poly.shp").replace("-", "_"))
# keep getting 'WARNING: Error of opening hash table for code page.' when we save to file plan
poly = "in_memory\poly"
logging.debug("Converting to polygon")
arcpy.RasterToPolygon_conversion(c_prob, poly, "SIMPLIFY")
del c_prob
#~ print(shp_class)
arcpy.CopyFeatures_management(poly, shp_class)
del poly
perim = None if hide else perimeters.find_perim(
scenario.fire, dates[i])
copyMXD = None
if len(days) - 1 == i:
# we need to get the extent from the last map
copyMXD, theMXD, extent = getProjectionMXD(i,
scenario.actuals_only,
scenario.run_output,
scenario.fire, extent,
perim)
run_what = r'python.exe firestarr\saveboth.py "{}" "{}"'.format(
copyMXD, fire_prefix + dates[i] + ".png")
processes.append(
start_process(run_what, Settings.PROCESS_FLAGS,
Settings.HOME_DIR))
del theMXD
run_what = r'python.exe firestarr\assets.py {} "{}" {} "{}" {}'.format(
i, scenario.run_output, scenario.fire, extent, prefix)
else:
copyMXD = getProjectionMXDName(i, scenario.actuals_only,
scenario.run_output, scenario.fire,
extent, perim)
run_what = r'python.exe firestarr\getProjectionMXD.py {} "{}" {} "{}"'.format(
i, scenario.run_output, scenario.fire, extent)
if scenario.actuals_only:
run_what += ' --actuals'
if perim:
run_what += ' --perim "{}"'.format(perim)
processes.append(
start_process(run_what, Settings.PROCESS_FLAGS, Settings.HOME_DIR))
mxd_paths = [copyMXD] + mxd_paths
mxd_names = [fire_prefix + dates[i] + ".png"] + mxd_names
start_raster = os.path.join(run_output, scenario.fire + '.tif')
fire_raster = None
if os.path.exists(start_raster):
fire_raster = arcpy.sa.Raster(start_raster)
# need to make sure the extent is the same for all rasters or they don't add properly
env_push()
setSnapAndExtent(prob_input)
def by_intensity(intensity):
letter = intensity.upper()[0]
prob_i = os.path.join(
run_output,
prob_input.replace(prefix, 'intensity_{}_'.format(letter)))
ra = Settings.RAMPART_MASK.format(intensity, suffix)
logging.debug(prob_i)
raster = arcpy.sa.Int(
arcpy.sa.Raster(prob_i) * arcpy.sa.Raster(ra))
if fire_raster is not None:
# don't count anything in the starting perimeter
# HACK: will not consider fires that start from just a size
raster = arcpy.sa.Con(arcpy.sa.IsNull(fire_raster), raster, 0)
raster = arcpy.sa.Con(arcpy.sa.IsNull(raster), 0, raster)
return raster
low_raster = by_intensity('low')
moderate_raster = by_intensity('moderate')
high_raster = by_intensity('high')
total_raster = low_raster + moderate_raster + high_raster
total_raster = arcpy.sa.SetNull(0 == total_raster, total_raster)
total_path = os.path.join(
map_output,
prob_input.replace(prefix, 'RA_').replace('.asc', '.tif'))
total_raster.save(total_path)
del low_raster
del moderate_raster
del high_raster
score = arcpy.RasterToNumPyArray(total_raster, nodata_to_value=0).sum()
# .58 so that 10 for social & economic gives a 10 total score
score = fixK(score / 1000000.0 / 0.58)
env_pop()
run_what = r'python.exe firestarr\getRiskMXD.py {} "{}" {} "{}" "{}"'.format(
i, scenario.run_output, scenario.fire, extent, score)
if scenario.actuals_only:
run_what += ' --actuals'
if perim:
run_what += ' --perim "{}"'.format(perim)
processes.append(
start_process(run_what, Settings.PROCESS_FLAGS, Settings.HOME_DIR))
copyMXD = getRiskMXDName(i, scenario.actuals_only, scenario.run_output,
scenario.fire, extent, perim)
risk_paths = [copyMXD] + risk_paths
risk_names = [
os.path.join(os.path.dirname(copyMXD),
fire_prefix + dates[i] + "_risk.png")
] + risk_names
scores = [score] + scores
env_pop()
copyMXD = getFuelMXDName(fire_prefix, scenario.run_output, scenario.fire,
extent, perim)
run_what = r'python.exe firestarr\getFuelMXD.py {} "{}" {} "{}"'.format(
fire_prefix, scenario.run_output, scenario.fire, extent)
if perim:
run_what += ' --perim "{}"'.format(perim)
processes.append(
start_process(run_what, Settings.PROCESS_FLAGS, Settings.HOME_DIR))
mxd_paths = [copyMXD] + mxd_paths
mxd_names = [fire_prefix + "_fuels.png"] + mxd_names
mxd_names = map(lambda x: os.path.abspath(os.path.join(map_output, x)),
mxd_names)
copyMXD = getImpactMXDName(fire_prefix, scenario.run_output, scenario.fire,
extent, perim)
run_what = r'python.exe firestarr\getImpactMXD.py {} "{}" {} "{}"'.format(
fire_prefix, scenario.run_output, scenario.fire, extent)
if perim:
run_what += ' --perim "{}"'.format(perim)
processes.append(
start_process(run_what, Settings.PROCESS_FLAGS, Settings.HOME_DIR))
risk_paths = [copyMXD] + risk_paths
risk_names = [
os.path.join(os.path.dirname(copyMXD), fire_prefix + "_impact.png")
] + risk_names
for process in processes:
finish_process(process)
# HACK: put in not generated images for any missing maps
if len(mxd_names) < 6:
mxd_names = (
mxd_names +
[os.path.join(Settings.HOME_DIR, 'not_generated.png')] * 6)[:6]
if len(risk_names) < 6:
risk_names = (
risk_names +
[os.path.join(Settings.HOME_DIR, 'not_generated.png')] * 6)[:6]
logging.debug(mxd_names + [wxshield] + risk_names)
makePDF(scenario.fire, days, dates, mxd_names, wxshield, risk_names,
sim_output, pdf_output, scores)
try_copy(pdf_output, copied)
# HACK: use known file name for assets
csv_orig = os.path.abspath(
os.path.join(run_output, fire_prefix + for_time + "_assets.csv"))
csv_output = os.path.abspath(
os.path.join(out_dir, os.path.basename(csv_orig)))
csv_copied = os.path.join(scenario.outbase, os.path.basename(csv_orig))
try_copy(csv_orig, csv_output)
try_copy(csv_orig, csv_copied)
fixtime(scenario.fire, parse(for_time.replace('_', ' ')),
[pdf_output, copied, csv_orig, csv_copied])
try:
tryForceRemove(mapflag)
except:
pass
# shouldn't need any of these intermediary outputs
shutil.rmtree(map_output, True)
return copied
def ovs_vsctl_del_port_from_bridge(port):
"""
This function removes given port from a OVS bridge.
"""
ret, _out, _err = util.start_process(["ovs-vsctl", "del-port", port])
return ret
