def test_smart_path():
rel_path = 'test/lib/test_util.py'
fake_rel_path = 'test/lib/test_util.py_fake'
abs_path = os.path.abspath(__file__)
assert util.smart_path(rel_path) == abs_path
assert util.smart_path(fake_rel_path) == abs_path + '_fake'
assert util.smart_path(abs_path) == abs_path
assert util.smart_path(abs_path, as_dir=True) == os.path.dirname(abs_path)
def save_image(figure, filepath=None):
if os.environ['PY_ENV'] == 'test':
return
if filepath is None:
filepath = f'{PLOT_FILEDIR}/{ps.get(figure, "layout.title")}.png'
filepath = util.smart_path(filepath)
dirname, filename = os.path.split(filepath)
try:
cmd = f'orca graph -o {filename} \'{json.dumps(figure)}\''
if 'linux' in sys.platform:
cmd = 'xvfb-run -a -s "-screen 0 1400x900x24" -- ' + cmd
Popen(cmd, cwd=dirname, shell=True, stderr=DEVNULL, stdout=DEVNULL)
logger.info(f'Graph saved to {dirname}/{filename}')
except Exception as e:
logger.exception(
'Please install orca for plotly and run retro-analysis to generate graphs.')
def test_write_read_as_plain_list(test_str, filename, dtype):
data_path = f'test/fixture/lib/util/{filename}'
util.write(test_str, util.smart_path(data_path))
assert os.path.exists(data_path)
data_dict = util.read(util.smart_path(data_path))
assert isinstance(data_dict, dtype)
def test_write_read_as_df(test_df, filename, dtype):
data_path = f'test/fixture/lib/util/{filename}'
util.write(test_df, util.smart_path(data_path))
assert os.path.exists(data_path)
data_df = util.read(util.smart_path(data_path))
assert isinstance(data_df, dtype)
def load(net, model_path):
'''Save model weights from a path into a net module'''
device = None if torch.cuda.is_available() else 'cpu'
net.load_state_dict(torch.load(util.smart_path(model_path), map_location=device))
logger.info(f'Loaded model from {model_path}')
def save(net, model_path):
'''Save model weights to path'''
torch.save(net.state_dict(), util.smart_path(model_path))
logger.info(f'Saved model to {model_path}')
def get_env_path(env_name):
'''Get the path to Unity env binaries distributed via npm'''
env_path = util.smart_path(f'node_modules/slm-env-{env_name}/build/{env_name}')
env_dir = os.path.dirname(env_path)
assert os.path.exists(env_dir), f'Missing {env_path}. See README to install from yarn.'
return env_path
def save(net, model_path):
'''Save model weights to path'''
torch.save(net.state_dict(), util.smart_path(model_path))
def test_write_read_as_plain_list(test_str, filename, dtype):
data_path = f'test/fixture/lib/util/{filename}'
util.write(test_str, util.smart_path(data_path))
assert os.path.exists(data_path)
data_dict = util.read(util.smart_path(data_path))
assert isinstance(data_dict, dtype)
def test_write_read_as_df(test_df, filename, dtype):
data_path = f'test/fixture/lib/util/{filename}'
util.write(test_df, util.smart_path(data_path))
assert os.path.exists(data_path)
data_df = util.read(util.smart_path(data_path))
assert isinstance(data_df, dtype)
def load(net, model_path):
'''Save model weights from a path into a net module'''
net.load_state_dict(torch.load(util.smart_path(model_path)))
logger.info(f'Loaded model from {model_path}')
The data visualization module
TODO pie, swarm, box plots
'''
from plotly import (
graph_objs as go,
offline as py,
tools,
)
from slm_lab import config
from slm_lab.lib import logger, util
import os
import plotly
import pydash as ps
PLOT_FILEDIR = util.smart_path('data')
os.makedirs(PLOT_FILEDIR, exist_ok=True)
if util.is_jupyter():
py.init_notebook_mode(connected=True)
logger = logger.get_logger(__name__)
def plot(*args, **kwargs):
if util.is_jupyter():
return py.iplot(*args, **kwargs)
else:
kwargs.update({'auto_open': ps.get(kwargs, 'auto_open', False)})
return py.plot(*args, **kwargs)
def save_image(figure, filepath=None):
from slm_lab.lib import util
import colorlog
import logging
import os
import sys
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # mute tf warnings on optimized setup
LOG_FILEPATH = util.smart_path(f'log/{util.get_ts()}_slm_lab.log')
LOG_FORMAT = '[%(asctime)s %(levelname)s] %(message)s'
LOG_LEVEL = logging.DEBUG if bool(os.environ.get('DEBUG')) else logging.INFO
class DedentFormatter(logging.Formatter):
'''The formatter to dedent broken python multiline string'''
def format(self, record):
record.msg = util.dedent(record.msg)
return super(DedentFormatter, self).format(record)
os.makedirs(os.path.dirname(LOG_FILEPATH), exist_ok=True)
color_formatter = colorlog.ColoredFormatter(
'%(log_color)s[%(asctime)s %(levelname)s]%(reset)s %(message)s')
fh = logging.FileHandler(LOG_FILEPATH)
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(color_formatter)
lab_logger = logging.getLogger('slm')
lab_logger.setLevel(LOG_LEVEL)
lab_logger.addHandler(fh)
lab_logger.addHandler(sh)
lab_logger.propagate = False
showlegend=False)
if idx == len(envs) - 1:
# plot extra to crop legend out
plot_env(algos,
env,
data_folder,
legend_list=legend_list,
frame_scales=frame_scales,
showlegend=True)
except Exception as e:
logger.warning(f'Cant plot for env: {env}. Error: {e}')
# Discrete
# LunarLander + Small Atari + Unity
data_folder = util.smart_path('../Desktop/benchmark/discrete')
algos = [
'dqn',
'ddqn_per',
'a2c_gae',
'a2c_nstep',
'ppo',
'*sac',
]
legend_list = [
'DQN',
'DDQN+PER',
'A2C (GAE)',
'A2C (n-step)',
'PPO',
from slm_lab.lib import util
import colorlog
import logging
import os
import sys
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # mute tf warnings on optimized setup
LOG_FILEPATH = util.smart_path(
f'log/{os.environ.get("PY_ENV")}_{util.get_timestamp()}_slm_lab.log')
LOG_FORMAT = '[%(asctime)s %(levelname)s] %(message)s'
LOG_LEVEL = logging.DEBUG if bool(os.environ.get('DEBUG')) else logging.INFO
class DedentFormatter(logging.Formatter):
'''The formatter to dedent broken python multiline string'''
def format(self, record):
record.msg = util.dedent(record.msg)
return super(DedentFormatter, self).format(record)
os.makedirs(os.path.dirname(LOG_FILEPATH), exist_ok=True)
color_formatter = colorlog.ColoredFormatter(
'%(log_color)s[%(asctime)s %(levelname)s]%(reset)s %(message)s')
fh = logging.FileHandler(LOG_FILEPATH)
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(color_formatter)
lab_logger = logging.getLogger('slm')
lab_logger.setLevel(LOG_LEVEL)
lab_logger.addHandler(fh)
'''
from plotly import (
graph_objs as go,
offline as py,
tools,
)
from slm_lab import config
from slm_lab.lib import logger, util
from subprocess import Popen, DEVNULL
import os
import plotly
import pydash as ps
import sys
import ujson as json
PLOT_FILEDIR = util.smart_path('data')
os.makedirs(PLOT_FILEDIR, exist_ok=True)
if util.is_jupyter():
py.init_notebook_mode(connected=True)
logger = logger.get_logger(__name__)
def plot(*args, **kwargs):
if util.is_jupyter():
return py.iplot(*args, **kwargs)
else:
kwargs.update({'auto_open': ps.get(kwargs, 'auto_open', False)})
return py.plot(*args, **kwargs)
def save_image(figure, filepath=None):
