def add_config_algo_env_from_dir(self, path):
assert is_training_progress_file(path)
rlscope_dir = _d(path)
# Path looks like:
# '$HOME/clone/rlscope/output/rlscope_bench/all.debug/config_uninstrumented/a2c/PongNoFrameskip-v4/process/a2c_PongNoFrameskip-v4/phase/default_phase/training_progress.trace_3.proto'
# - $HOME
# - clone
# - rlscope
# - output
# - rlscope_bench
# - all.debug
# -7 - config_uninstrumented
# -6 - a2c
# -5 - PongNoFrameskip-v4
# -4 - process
# -3 - a2c_PongNoFrameskip-v4
# -2 - phase
# -1 - default_phase
norm_path = os.path.normpath(rlscope_dir)
components = norm_path.split(os.sep)
env_id = components[-5]
algo = components[-6]
fields = {
'algo': algo,
'env_id': env_id,
}
assert is_config_dir(os.sep.join(components[:-7 + 1]))
# if is_config_dir(os.sep.join(components[:-7 + 1])):
fields['config'] = components[-7]
# else:
# # Default to 'instrumented'
# fields['config'] = 'instrumented'
return fields
def new_bkup_file(self, path):
bkup_files = self._bkup_files(_d(path))
# import ipdb; ipdb.set_trace()
if len(bkup_files) == 0:
return "{path}.bkup".format(**locals())
new_id = max(self.bkup_file_idx(p) for p in bkup_files) + 1
return "{path}.{new_id}.bkup".format(**locals())
def recover(self, path):
if not self.is_bkup_file(path):
path = self.guess_bkup_file(_d(path))
assert self.is_bkup_file(path)
new_path = self.recov_file(path)
if not self.check_paths(path, new_path):
return False
self.move_file(path, new_path)
def dump_plot_index_py(self):
cmd = sys.argv
if not self.dry_run:
src = _j(py_config.ROOT, 'rlscope/scripts/rlscope_plot_index.py')
dst = _j(self.out_dir, 'rlscope_plot_index.py')
logger.info("cp {src} -> {dst}".format(src=src, dst=dst))
os.makedirs(_d(dst), exist_ok=True)
shutil.copyfile(src, dst)
os.makedirs(_d(self.plot_index_path), exist_ok=True)
if _e(self.plot_index_path) and not self.replace:
logger.warning(
"{path} exists; skipping".format(path=self.plot_index_path))
return
with open(self.plot_index_path, 'w') as f:
contents = textwrap.dedent("""\
#!/usr/bin/env python3
### GENERATED FILE; do NOT modify!
### generated using:
### CMD:
### {cmd}
### PWD:
### {pwd}
DIRECTORY = "{dir}"
INDEX = \\
{index}
""").format(
dir=os.path.realpath(self.directory),
index=textwrap.indent(pprint.pformat(self.index),
prefix=" "),
cmd=" ".join(cmd),
pwd=os.getcwd(),
)
if self.debug:
logger.info("> Generated file: {path}".format(
path=self.plot_index_path))
logger.info(contents)
if not self.dry_run:
f.write(contents)
def maybe_add_algo_env(self, path):
assert is_training_progress_file(path)
rlscope_directory = _d(path)
if self.algo_env_from_dir:
return self.add_config_algo_env_from_dir(path)
if not _e(experiment.experiment_config_path(rlscope_directory)):
return self.add_experiment_config(path)
# Not sure what (algo, env) is; don't add those columns.
return None
def add_experiment_config(self, path):
"""
add_fields(path)
We expect to find experiment_config.json where the machine_util.*.proto files live.
:param path:
:return:
"""
assert is_training_progress_file(path)
directory = _d(path)
path = experiment.experiment_config_path(directory)
if not _e(path):
if self.debug:
logger.info(
"Didn't find {path}; skip adding experiment columns to csv"
.format(path=path))
return None
data = experiment.load_experiment_config(directory)
return data
def __enter__(self):
if self._is_path:
# if self.append:
# self.mode = 'ab'
# else:
# self.mode = 'wb'
if self.append:
self.mode = 'a'
else:
self.mode = 'w'
if self.makedirs:
# logger.info("mkdirs {path}".format(path=_d(self.file)))
os.makedirs(_d(self.file), exist_ok=True)
# logger.info("ScopedLogFile.file = {path}".format(path=self.file))
self.f = open(self.file, self.mode)
return self.f
else:
# We can only append to a stream.
self.f = self.file
return self.f
import subprocess as _S
from os.path import dirname as _d
_d = _d(__file__)
def _git_call(*args):
return _S.call(("git", ) + args, cwd=_d)
def _git_output(*args):
p = _S.Popen(("git", ) + args, cwd=_d, stdout=_S.PIPE, stderr=_S.PIPE)
out, err = p.communicate()
if p.returncode:
raise _S.CalledProcessError(p.returncode, "git", err)
return out.strip()
_git_call("update-index", "-q", "--refresh")
dirty = _git_call("diff-index", "--quiet", "HEAD", "--")
if dirty not in (0, 1):
raise _S.CalledProcessError(dirty, "git")
try:
revision = int(_git_output("rev-list", "--count", "HEAD"))
except _S.CalledProcessError: # BBB: Git too old
revision = len(_git_output("rev-list", "HEAD").split())
short = _git_output("rev-parse", "--short", "HEAD")
version = "0-%s.g%s" % (revision, short)
if dirty:
version += ".dirty"
def get_variable_path(ParserKlass, src_files, bench_name):
pretty_base = "{base}.variable.pretty.txt".format(base=_b(src_files.get('profile_path', bench_name)))
return _j(_d(src_files.get('profile_path', bench_name)), pretty_base)
def _microbench_path(self, bench_name):
return get_microbench_path(_d(self._pretty_profile_path(bench_name)))
:py:class:`_EnvVars` is for reading environment variables. """ from os.path import join as _j, abspath as _a, exists as _e, dirname as _d, basename as _b from os import environ as ENV import shutil import re import ctypes.util import sys import os import textwrap # NOTE: Don't import logger here since rlscope_logging needs py_config. # from rlscope.profiler.rlscope_logging import logger import rlscope INSTALL_ROOT = _d(os.path.realpath(rlscope.__file__)) CPP_LIB = _j(INSTALL_ROOT, 'cpp', 'lib') CPP_BIN = _j(INSTALL_ROOT, 'cpp', 'bin') CPP_INCLUDE = _j(INSTALL_ROOT, 'cpp', 'include') USE_NUMBA = False nb = None if USE_NUMBA: import numba as nb ROOT = _d(_d(_a(__file__))) RLSCOPE_DIR = ROOT GIT_REPO_URL = "https://github.com/UofT-EcoSystem/rlscope"
import os
import subprocess
import sys
import types
from os.path import expandvars as _e
from os.path import join as _j, abspath as _P, dirname as _d, realpath as _r
from glob import glob
import dot_config
import dot_util
#
# Configuration
#
ROOT = _d(_d(_d(_r(__file__))))
if not os.path.exists(ROOT):
print ("ERROR: ROOT ({ROOT}) doesn't exist".format(ROOT=ROOT))
sys.exit(1)
# Time to wait to connect to various things.
MAX_TIME_SEC = 60
COMMON_SH = _j(ROOT, "src/sh/common.sh")
EXPORTS_SH = _j(ROOT, "src/sh/exports.sh")
#
# End of configuration
#
def config_vars():
import subprocess as _S
from os.path import dirname as _d
_d = _d(__file__)
def _git_call(*args):
return _S.call(("git",) + args, cwd=_d)
def _git_output(*args):
p = _S.Popen(("git",) + args, cwd=_d, stdout=_S.PIPE, stderr=_S.PIPE)
out, err = p.communicate()
if p.returncode:
raise _S.CalledProcessError(p.returncode, "git", err)
return out.strip()
_git_call("update-index", "-q", "--refresh")
dirty = _git_call("diff-index", "--quiet", "HEAD", "--")
if dirty not in (0, 1):
raise _S.CalledProcessError(dirty, "git")
try:
revision = int(_git_output("rev-list", "--count", "HEAD"))
except _S.CalledProcessError: # BBB: Git too old
revision = len(_git_output("rev-list", "HEAD").split())
short = _git_output("rev-parse", "--short", "HEAD")
version = "0-%s.g%s" % (revision, short)
if dirty:
version += ".dirty"
# Because the software could be forked or have local changes/commits, above
# properties can't be used to decide whether a peer runs an appropriate version: