def parse_oniontrace_logs(args):
otracetools_exe = which('oniontracetools')
if otracetools_exe == None:
logging.warning(
"Cannot find oniontracetools in your PATH. Is your python venv active? Do you have oniontracetools installed?"
)
logging.warning("Unable to parse oniontrace simulation data.")
return
cmd_str = f"{otracetools_exe} parse -m {args.nprocesses} -e '.*\.oniontrace\.[0-9]+.stdout' shadow.data/hosts"
cmd = cmdsplit(cmd_str)
datestr = datetime.datetime.now().strftime("%Y-%m-%d.%H:%M:%S")
with open_writeable_file(
f"{args.prefix}/oniontracetools.parse.{datestr}.log") as outf:
logging.info("Parsing oniontrace log data with oniontracetools now...")
comproc = subprocess.run(cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
logging.info(f"oniontracetools returned code {comproc.returncode}")
return comproc.returncode == 0
def plot_oniontrace(args):
oniontracetools_exe = which('oniontracetools')
if oniontracetools_exe == None:
logging.warning(
"Cannot find oniontracetools in your PATH. Is your python venv active? Do you have oniontracetools installed?"
)
logging.warning("Unable to plot oniontrace data.")
return
# plot the tgen simulation data for each tgen json file in the tornet path
cmd_prefix_str = f"{oniontracetools_exe} plot --expression 'relay|4uthority' --prefix 'relays'"
for collection in args.tornet_collection_path:
for json_path in find_matching_files_in_dir(
collection, "oniontrace.analysis.json"):
dir_path = os.path.dirname(json_path)
dir_name = os.path.basename(dir_path)
cmd_str = f"{cmd_prefix_str} --data {json_path} {dir_name}"
cmd = cmdsplit(cmd_str)
datestr = datetime.datetime.now().strftime("%Y-%m-%d.%H:%M:%S")
with open_writeable_file(
f"{dir_path}/oniontracetools.plot.{datestr}.log") as outf:
logging.info(
f"Using oniontracetools to plot data from {json_path} now..."
)
comproc = subprocess.run(cmd,
cwd=dir_path,
stdout=outf,
stderr=subprocess.STDOUT)
logging.info(
f"oniontracetools returned code {comproc.returncode}")
def parse_tgen_logs(args):
tgentools_exe = which('tgentools')
if tgentools_exe == None:
logging.warning(
"Cannot find tgentools in your PATH. Is your python venv active? Do you have tgentools installed?"
)
logging.warning("Unable to parse tgen simulation data.")
return
cmd_str = f"{tgentools_exe} parse -m {args.nprocesses} -e 'perfclient[0-9]+\.tgen\.[0-9]+.stdout' --complete shadow.data/hosts"
cmd = cmdsplit(cmd_str)
datestr = datetime.datetime.now().strftime("%Y-%m-%d.%H:%M:%S")
with open_writeable_file(
f"{args.prefix}/tgentools.parse.{datestr}.log") as outf:
logging.info("Parsing tgen log data with tgentools now...")
comproc = subprocess.run(cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
logging.info(f"tgentools returned code {comproc.returncode}")
return comproc.returncode == 0
def __xz_parallel(args, filename):
path = f"{args.prefix}/{filename}"
if os.path.exists(path):
xz_cmd = cmdsplit(f"xz -9 --threads={args.nprocesses} {path}")
comproc = subprocess.run(xz_cmd,
cwd=args.prefix,
stdout=subprocess.DEVNULL)
if comproc.returncode == 0:
return True
return False
def __start_dstat(args):
dstat_exe_path = which('dstat')
if dstat_exe_path == None:
return None
dstat_cmd = cmdsplit(f"{dstat_exe_path} -cmstTy --fs --output dstat.log")
dstat_subp = subprocess.Popen(dstat_cmd,
cwd=args.prefix,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
return dstat_subp
def __run_free_loop(args, stop_event):
date_exe_path = which('date')
free_exe_path = which('free')
with open(f"{args.prefix}/free.log", 'w') as outf:
while not stop_event.is_set():
if date_exe_path != None:
date_cmd = cmdsplit(
f"{date_exe_path} --utc '+%s.%N %Z seconds since epoch'")
comproc = subprocess.run(date_cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
if free_exe_path != None:
free_cmd = cmdsplit(f"{free_exe_path} -w -b -l")
comproc = subprocess.run(free_cmd,
cwd=args.prefix,
stdout=outf,
stderr=subprocess.STDOUT)
sleep(1)
def __tar_xz_parallel(args, dirname, excludes=[]):
dirpath = f"{args.prefix}/{dirname}"
if not os.path.exists(dirpath):
return False
# we are basically trying to do something like:
# tar cf - FLAGS dirname | xz -8e --threads=N > dirname.tar.xz
flags = [f"--exclude='{e}'" for e in excludes]
flag_str = ' '.join(flags)
tar_cmd = cmdsplit(f"tar cf - {flag_str} {dirname}")
tarproc = subprocess.Popen(tar_cmd,
cwd=args.prefix,
stdout=subprocess.PIPE)
xz_cmd = cmdsplit(f"xz -9 --threads={args.nprocesses} -")
xzproc = subprocess.Popen(xz_cmd,
cwd=args.prefix,
stdin=tarproc.stdout,
stdout=subprocess.PIPE)
dd_cmd = cmdsplit(f"dd of={dirname}.tar.xz")
ddproc = subprocess.Popen(dd_cmd,
cwd=args.prefix,
stdin=xzproc.stdout,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
# wait for the above and collec the return codes
tar_rc = tarproc.wait()
xz_rc = xzproc.wait()
dd_rc = ddproc.wait()
if tar_rc == 0 and xz_rc == 0 and dd_rc == 0:
return True
else:
return False
def __run_shadow(args):
shadow_exe_path = which('shadow')
if shadow_exe_path == None:
logging.warning(
"Cannot find shadow in your PATH. Do you have shadow installed (e.g., in ~/.shadow/bin)? Did you update your PATH?"
)
logging.warning("Unable to run simulation without shadow.")
return None
with open_writeable_file(f"{args.prefix}/shadow.log",
compress=args.do_compress) as outf:
shadow_cmd = cmdsplit(
f"{shadow_exe_path} {args.shadow_args} shadow.config.xml")
comproc = subprocess.run(shadow_cmd, cwd=args.prefix, stdout=outf)
return comproc
def __run_shadow(args):
shadow_exe_path = args.shadowexe
if shadow_exe_path == None:
logging.warning(
"Cannot find shadow in your PATH. Do you have shadow installed (e.g., in ~/.shadow/bin)? Did you update your PATH?"
)
logging.warning("Unable to run simulation without shadow.")
return None
cmd_prefix = "/usr/bin/chrt -f 1 " if args.use_realtime else ""
args_suffix = " --template-directory=shadow.data.template" if '--template-directory' not in args.shadow_args else ""
shadow_args = args.shadow_args
with open_writeable_file(f"{args.prefix}/shadow.log",
compress=args.do_compress) as outf:
shadow_cmd = cmdsplit(
f"{cmd_prefix}{shadow_exe_path} {shadow_args}{args_suffix} shadow.config.yaml"
)
comproc = subprocess.run(shadow_cmd, cwd=args.prefix, stdout=outf)
return comproc