def config_charts(config_yaml, path=".charts.yml"):
from textwrap import dedent
from ml_logger import logger
if not config_yaml:
try:
caller_script = abspath(
inspect.getmodule(inspect.stack()[1][0]).__file__)
cwd = dirname(caller_script)
except:
cwd = os.getcwd()
logger.log_text(dedent(config_yaml).lstrip(), path)
def thunk(*args, **kwargs):
import traceback
from ml_logger import logger
assert not (args and ARGS), \
f"can not use position argument at both thunk creation as well as run.\n" \
f"_args: {args}\n" \
f"ARGS: {ARGS}\n"
logger.configure(root_dir=RUN.server,
prefix=PREFIX,
register_experiment=False,
max_workers=10)
logger.log_params(host=dict(hostname=logger.hostname),
run=dict(status="running",
startTime=logger.now(),
job_id=logger.job_id))
import time
try:
_KWARGS = {**KWARGS}
_KWARGS.update(**kwargs)
results = fn(*(args or ARGS), **_KWARGS)
logger.log_line("========== execution is complete ==========")
logger.log_params(
run=dict(status="completed", completeTime=logger.now()))
logger.flush()
time.sleep(3)
except Exception as e:
tb = traceback.format_exc()
with logger.SyncContext(
): # Make sure uploaded finished before termination.
logger.print(tb, color="red")
logger.log_text(tb, filename="traceback.err")
logger.log_params(
run=dict(status="error", exitTime=logger.now()))
logger.flush()
time.sleep(3)
raise e
return results
def config_charts(config_yaml="", path=None):
from textwrap import dedent
from ml_logger import logger
if not config_yaml:
caller_script = abspath(
inspect.getmodule(inspect.stack()[1][0]).__file__)
if path is None:
path = logger.stem(caller_script) + ".charts.yml"
try: # first try the namesake chart file
with open(os.path.join(os.path.dirname(caller_script), path),
'r') as s:
config_yaml = s.read()
cprint(f"Found ml-dash config file \n{path}", 'green')
except: # do not upload when can not find
path = ".charts.yml"
with open(os.path.join(os.path.dirname(caller_script), path),
'r') as s:
config_yaml = s.read()
cprint(f"Found ml-dash config file \n{path}", 'green')
logger.log_text(dedent(config_yaml), ".charts.yml")
def _(*args, **kwargs):
import traceback
from ml_logger import logger
assert not (args and ARGS), f"can not use position argument at both thunk creation as well as " \
f"run.\n_args: {args}\nARGS: {ARGS}"
logger.configure(log_directory=RUN.server,
prefix=PREFIX,
register_experiment=False,
max_workers=10)
logger.log_params(host=dict(hostname=logger.hostname),
run=dict(status="running", startTime=logger.now()))
try:
_KWARGS = KWARGS.copy()
_KWARGS.update(kwargs)
fn(*(args or ARGS), **_KWARGS)
logger.log_line("========= execution is complete ==========")
logger.log_params(
run=dict(status="completed", completeTime=logger.now()))
except Exception as e:
import time
time.sleep(1)
tb = traceback.format_exc()
with logger.SyncContext(
): # Make sure uploaded finished before termination.
logger.log_text(tb, filename="traceback.err")
logger.log_params(
run=dict(status="error", exitTime=logger.now()))
logger.log_line(tb)
logger.flush()
time.sleep(30)
raise e
import time
time.sleep(30)
from ml_logger import logger
from dmc_gen_analysis import RUN
for i in range(9):
logger.configure(
root_dir=RUN.server,
prefix=f"/geyang/dmc_gen/2021/03-05/00_setup/train/01.17.36/{i}")
logger.log_text("""
keys:
- Args.seed
- Args.algorithm
charts:
- yKey: episode_reward/mean
xKey: step
- yKey: train/episode_reward/mean
xKey: step
""",
filename=".charts.yml",
overwrite=True)
Args.task = ['walk', 'swingup', 'catch', 'spin']
Args.algo = ['pad', 'soda', 'curl', 'rad']
Args.seed = [100, 300, 400]
for i, args in enumerate(sweep):
# jaynes.config("vision", launch=dict(ip=ms[i]))
RUN.job_postfix = f"{Args.domain}-{Args.task}/{Args.algo}/{Args.seed}"
thunk = instr(train, args)
logger.log_text("""
keys:
- Args.domain
- Args.task
- Args.algo
- Args.seed
charts:
- yKey: "episode_reward/mean"
xKey: step
- yKey: "train/episode_reward/mean"
xKey: step
- type: video
glob: videos/*_train.mp4
- type: video
glob: videos/*_test.mp4
""",
".charts.yml",
overwrite=True,
dedent=True)
jaynes.run(thunk)
jaynes.listen()
def instr(fn, *ARGS, __file=False, __silent=False, **KWARGS):
"""
thunk for configuring the logger. The reason why this is not a decorator is
:param fn: function to be called
:param *ARGS: position arguments for the call
:param __file__: console mode, by-pass file related logging
:param __silent: do not print
:param **KWARGS: keyword arguments for the call
:return: a thunk that can be called without parameters
"""
from ml_logger import logger
if __file:
caller_script = pJoin(os.getcwd(), __file)
else:
launch_module = inspect.getmodule(inspect.stack()[1][0])
__file = launch_module.__file__
caller_script = abspath(__file)
# note: for scripts in the `plan2vec` module this also works -- b/c we truncate fixed depth.
script_path = logger.truncate(caller_script,
depth=len(__file__.split('/')) - 1)
file_stem = logger.stem(script_path)
file_name = basename(file_stem)
RUN(file_name=file_name, file_stem=file_stem, now=logger.now())
PREFIX = RUN.PREFIX
# todo: there should be a better way to log these.
# todo: we shouldn't need to log to the same directory, and the directory for the run shouldn't be fixed.
logger.configure(
root_dir=RUN.server,
prefix=PREFIX,
asynchronous=False, # use sync logger
max_workers=4,
register_experiment=False)
if RUN.restart:
with logger.Sync():
logger.remove(".")
logger.upload_file(caller_script)
# the tension is in between creation vs run. Code snapshot are shared, but runs need to be unique.
_ = dict()
if ARGS:
_['args'] = ARGS
if KWARGS:
_['kwargs'] = KWARGS
logger.log_params(run=logger.run_info(status="created",
script_path=script_path),
revision=logger.rev_info(),
fn=logger.fn_info(fn),
**_,
silent=__silent)
logger.print(
'taking diff, if this step takes too long, check if your '
'uncommitted changes are too large.',
color="green")
logger.diff()
if RUN.readme:
logger.log_text(RUN.readme, "README.md", dedent=True)
import jaynes # now set the job name to prefix
if jaynes.RUN.config and jaynes.RUN.mode != "local":
runner_class, runner_args = jaynes.RUN.config['runner']
if 'name' in runner_args: # ssh mode does not have 'name'.
runner_args['name'] = pJoin(file_name, RUN.JOB_NAME)
del logger, jaynes, runner_args, runner_class
if not __file:
cprint(f'Set up job name', "green")
def thunk(*args, **kwargs):
import traceback
from ml_logger import logger
assert not (args and ARGS), \
f"can not use position argument at both thunk creation as well as run.\n" \
f"_args: {args}\n" \
f"ARGS: {ARGS}\n"
logger.configure(root_dir=RUN.server,
prefix=PREFIX,
register_experiment=False,
max_workers=10)
logger.log_params(host=dict(hostname=logger.hostname),
run=dict(status="running",
startTime=logger.now(),
job_id=logger.job_id))
import time
try:
_KWARGS = {**KWARGS}
_KWARGS.update(**kwargs)
results = fn(*(args or ARGS), **_KWARGS)
logger.log_line("========== execution is complete ==========")
logger.log_params(
run=dict(status="completed", completeTime=logger.now()))
logger.flush()
time.sleep(3)
except Exception as e:
tb = traceback.format_exc()
with logger.SyncContext(
): # Make sure uploaded finished before termination.
logger.print(tb, color="red")
logger.log_text(tb, filename="traceback.err")
logger.log_params(
run=dict(status="error", exitTime=logger.now()))
logger.flush()
time.sleep(3)
raise e
return results
return thunk
import os
from ml_logger import logger
logger.configure(log_directory=os.path.expanduser("~/ml-logger-debug"),
prefix='episodeyang/demo-project/first-run')
for i in range(100):
logger.log(loss=0.9**i, step=i, flush=True)
logger.log_text('charts: [{"yKey": "loss", "xKey": "step"}]', ".charts.yml")
with sweep.product:
with sweep.zip:
Args.env_name = env_names
Args.seed = [100, 200, 300, 400, 500, 600]
# jaynes.config('local' if Args.debug else "gpu-vaughan", launch=dict(timeout=10000))
jaynes.config('local' if Args.debug else "cpu-mars")
# jaynes.config('local')
for i, deps in sweep.items():
thunk = instr(main,
deps,
_job_postfix=f"{Args.env_name.split(':')[-1]}")
logger.log_text("""
keys:
- run.status
- Args.env_name
- host.hostname
charts:
- yKey: test/success
xKey: epoch
- yKey: test/success
xKey: timesteps
- yKey: train/success/mean
xKey: time
""",
".charts.yml",
dedent=True)
jaynes.run(thunk)
import pickle
from cmx import doc
from ml_logger import logger
doc @ """
# Debug Logger Overwrite Bug
Reading from metrics file:
"""
logger.configure("http://54.71.92.65:8080", prefix='geyang/project/debug_logs')
logger.remove('debug_logs')
doc.print(logger.root)
logger.log_text("""
charts:
- i
""", dedent=True, filename=".charts.yml")
for i in range(3):
logger.log_key_value(i=i)
logger.flush()
import time
time.sleep(1)
doc @ "```ansi"
doc @ logger.load_text("outputs.log")
doc @ "```"
with doc:
data = logger.read_metrics()