def parse_oniontrace_logs(args):
otracetools_exe = which('oniontracetools')
if otracetools_exe == None:
logging.warning(
"Cannot find oniontracetools in your PATH. Is your python venv active? Do you have oniontracetools installed?"
)
logging.warning("Unable to parse oniontrace simulation data.")
return
cmd_str = f"{otracetools_exe} parse -m {args.nprocesses} -e '.*\.oniontrace\.[0-9]+.stdout' shadow.data/hosts"
cmd = cmdsplit(cmd_str)
datestr = datetime.datetime.now().strftime("%Y-%m-%d.%H:%M:%S")
with open_writeable_file(
f"{args.prefix}/oniontracetools.parse.{datestr}.log") as outf:
logging.info("Parsing oniontrace log data with oniontracetools now...")
comproc = subprocess.run(cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
logging.info(f"oniontracetools returned code {comproc.returncode}")
return comproc.returncode == 0
def plot_oniontrace(args):
oniontracetools_exe = which('oniontracetools')
if oniontracetools_exe == None:
logging.warning(
"Cannot find oniontracetools in your PATH. Is your python venv active? Do you have oniontracetools installed?"
)
logging.warning("Unable to plot oniontrace data.")
return
# plot the tgen simulation data for each tgen json file in the tornet path
cmd_prefix_str = f"{oniontracetools_exe} plot --expression 'relay|4uthority' --prefix 'relays'"
for collection in args.tornet_collection_path:
for json_path in find_matching_files_in_dir(
collection, "oniontrace.analysis.json"):
dir_path = os.path.dirname(json_path)
dir_name = os.path.basename(dir_path)
cmd_str = f"{cmd_prefix_str} --data {json_path} {dir_name}"
cmd = cmdsplit(cmd_str)
datestr = datetime.datetime.now().strftime("%Y-%m-%d.%H:%M:%S")
with open_writeable_file(
f"{dir_path}/oniontracetools.plot.{datestr}.log") as outf:
logging.info(
f"Using oniontracetools to plot data from {json_path} now..."
)
comproc = subprocess.run(cmd,
cwd=dir_path,
stdout=outf,
stderr=subprocess.STDOUT)
logging.info(
f"oniontracetools returned code {comproc.returncode}")
def parse_tgen_logs(args):
tgentools_exe = which('tgentools')
if tgentools_exe == None:
logging.warning(
"Cannot find tgentools in your PATH. Is your python venv active? Do you have tgentools installed?"
)
logging.warning("Unable to parse tgen simulation data.")
return
cmd_str = f"{tgentools_exe} parse -m {args.nprocesses} -e 'perfclient[0-9]+\.tgen\.[0-9]+.stdout' --complete shadow.data/hosts"
cmd = cmdsplit(cmd_str)
datestr = datetime.datetime.now().strftime("%Y-%m-%d.%H:%M:%S")
with open_writeable_file(
f"{args.prefix}/tgentools.parse.{datestr}.log") as outf:
logging.info("Parsing tgen log data with tgentools now...")
comproc = subprocess.run(cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
logging.info(f"tgentools returned code {comproc.returncode}")
return comproc.returncode == 0
def run(args):
logging.info("Starting to archive simulation results now.")
if which('xz') == None or which('tar') == None or which('dd') == None:
logging.warning(
"We require the tar, xz, and dd tools to archive the results.")
logging.critical("Unable to archive with missing tools.")
return
shutil.copy2(
f"{args.prefix}/shadow.data/hosts/4uthority1/cached-consensus",
f"{args.prefix}/consensus")
logging.info("Compressing consensus.")
__xz_parallel(args, "consensus")
logging.info("Compressing shadow config.")
__xz_parallel(args, "shadow.config.xml")
logging.info("Compressing dstat log.")
__xz_parallel(args, "dstat.log")
logging.info("Compressing free log.")
__xz_parallel(args, "free.log")
logging.info("Compressing shadow log.")
__xz_parallel(args, "shadow.log")
logging.info("Compressing conf dir.")
if __tar_xz_parallel(args, "conf"):
shutil.rmtree(f"{args.prefix}/conf")
logging.info("Compressing shadow template dir.")
if __tar_xz_parallel(args, "shadow.data.template"):
shutil.rmtree(f"{args.prefix}/shadow.data.template")
logging.info("Compressing shadow data dir.")
if __tar_xz_parallel(args,
"shadow.data",
excludes=['cached-*', 'diff-cache', 'keys', 'lock']):
shutil.rmtree(f"{args.prefix}/shadow.data")
logging.info("Compressing remaining log files.")
for name in os.listdir(args.prefix):
if name.endswith(".log"):
__xz_parallel(args, name)
def set_plot_options():
options = {
#'backend': 'PDF',
'font.size': 12,
'figure.figsize': (4, 3),
'figure.dpi': 100.0,
'grid.color': '0.1',
'grid.linestyle': ':',
'grid.linewidth': 0.5,
'axes.grid': True,
#'axes.grid.axis' : 'y',
#'axes.axisbelow': True,
'axes.titlesize': 14,
'axes.labelsize': 10,
'axes.formatter.limits': (-4, 4),
'xtick.labelsize': 8,
'ytick.labelsize': 8,
'lines.linewidth': 2.0,
'lines.markeredgewidth': 0.5,
'lines.markersize': 10,
'legend.fontsize': 10,
'legend.fancybox': False,
'legend.shadow': False,
'legend.borderaxespad': 0.5,
'legend.columnspacing': 1.0,
'legend.numpoints': 1,
'legend.handletextpad': 0.25,
'legend.handlelength': 2.0,
'legend.labelspacing': 0.25,
'legend.markerscale': 1.0,
}
options_latex = {
# turn on the following to embedd fonts; requires latex
'ps.useafm': True,
'pdf.use14corefonts': True,
'text.usetex': True,
# 'text.latex.preamble': r'\boldmath',
'text.latex.preamble': r'\usepackage{amsmath}',
}
for option_key in options:
rcParams[option_key] = options[option_key]
if which("latex") != None:
for option_key in options_latex:
rcParams[option_key] = options_latex[option_key]
if 'figure.max_num_figures' in rcParams:
rcParams['figure.max_num_figures'] = 50
if 'figure.max_open_warning' in rcParams:
rcParams['figure.max_open_warning'] = 50
if 'legend.ncol' in rcParams:
rcParams['legend.ncol'] = 50
def __start_dstat(args):
dstat_exe_path = which('dstat')
if dstat_exe_path == None:
return None
dstat_cmd = cmdsplit(f"{dstat_exe_path} -cmstTy --fs --output dstat.log")
dstat_subp = subprocess.Popen(dstat_cmd,
cwd=args.prefix,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
return dstat_subp
def __run_free_loop(args, stop_event):
date_exe_path = which('date')
free_exe_path = which('free')
with open(f"{args.prefix}/free.log", 'w') as outf:
while not stop_event.is_set():
if date_exe_path != None:
date_cmd = cmdsplit(
f"{date_exe_path} --utc '+%s.%N %Z seconds since epoch'")
comproc = subprocess.run(date_cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
if free_exe_path != None:
free_cmd = cmdsplit(f"{free_exe_path} -w -b -l")
comproc = subprocess.run(free_cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
sleep(1)
def __run_shadow(args):
shadow_exe_path = which('shadow')
if shadow_exe_path == None:
logging.warning(
"Cannot find shadow in your PATH. Do you have shadow installed (e.g., in ~/.shadow/bin)? Did you update your PATH?"
)
logging.warning("Unable to run simulation without shadow.")
return None
with open_writeable_file(f"{args.prefix}/shadow.log",
compress=args.do_compress) as outf:
shadow_cmd = cmdsplit(
f"{shadow_exe_path} {args.shadow_args} shadow.config.xml")
comproc = subprocess.run(shadow_cmd, cwd=args.prefix, stdout=outf)
return comproc