s = """
# The Awesome ML-Logger
You can run the following code with ml-logger:
```python
from ml_logger import logger
logger.log(lr=0, clip range=0.200, step=0, timestamp='2018-11-16T00:09:27.198142', reward=-109.43)
logger.flush()
```
╒════════════════════╤════════════════════════════╕
│ lr │ 0.000 │
├────────────────────┼────────────────────────────┤
│ clip range │ 0.200 │
├────────────────────┼────────────────────────────┤
│ step │ 0 │
├────────────────────┼────────────────────────────┤
│ timestamp │'2018-11-16T00:09:27.198142'│
├────────────────────┼────────────────────────────┤
│ reward │ -109.43 │
╘════════════════════╧════════════════════════════╛
"""
print(s)
if __name__ == "__main__":
jaynes.config(mode='fair')
jaynes.run(launch)
jaynes.listen()
def work_fn():
print('working')
# for i in range(100):
# print('this is running', i)
# sleep(0.5)
# sys.stdout.flush()
if __name__ == '__main__':
import jaynes
# from dmc_gen.train import train
for i in range(60):
jaynes.config("vision", launch=dict(ip=f"visiongpu{i:02d}"))
jaynes.run(work_fn)
# for i in range(1):
# jaynes.config("vision-gpu")
# jaynes.run(work_fn)
# for i in range(60):
# jaynes.config("visiongpu", launch=dict(ip=f"visiongpu{i:02d}"),
# runner=dict(pypath="$HOME/jaynes-debug", work_dir="$HOME/jaynes-debug"), mounts=[], )
# jaynes.run(work_fn, aug_data_prefix="/afs/csail.mit.edu/u/g/geyang/mit/dmc_gen/custom_vendor/data")
# jaynes.config("supercloud", runner=dict(n_cpu=1, n_gpu=0))
# jaynes.run(train_fn)
jaynes.listen()
# highly non-rectangular
import jaynes
from launch_entry import train_fn
if __name__ == "__main__":
hosts = [
'visiongpu001',
'visiongpu002',
'visiongpu003',
]
for host in hosts:
jaynes.config(verbose=False, launch=dict(ip=host))
jaynes.run(train_fn)
jaynes.listen(200)
Args.task = ['walk', 'swingup', 'catch', 'spin']
Args.algo = ['pad', 'soda', 'curl', 'rad']
Args.seed = [100, 300, 400]
for i, args in enumerate(sweep):
# jaynes.config("vision", launch=dict(ip=ms[i]))
RUN.job_postfix = f"{Args.domain}-{Args.task}/{Args.algo}/{Args.seed}"
thunk = instr(train, args)
logger.log_text("""
keys:
- Args.domain
- Args.task
- Args.algo
- Args.seed
charts:
- yKey: "episode_reward/mean"
xKey: step
- yKey: "train/episode_reward/mean"
xKey: step
- type: video
glob: videos/*_train.mp4
- type: video
glob: videos/*_test.mp4
""",
".charts.yml",
overwrite=True,
dedent=True)
jaynes.run(thunk)
jaynes.listen()
# logger.remove(".")
# a = np.ones([1, 1, 100_000_000 // 4])
# logger.print(f"the size of the tensor is {a.size}")
# data = dict(key="ok", large=a)
# logger.torch_save(data, f"save/data-{logger.now('%H.%M.%S')}.pkl")
logger.print('done')
if __name__ == '__main__':
import jaynes
# jaynes.config("supercloud", launch=dict(n_gpu=0))
jaynes.config("local", launch=dict(n_gpu=0))
# ip = "visiongpu50"
# ip = "improbable005"
for ip in [
"improbable005",
# "improbable006",
# "improbable007",
# "improbable008",
# "improbable010"
]:
# jaynes.config("vision", launch=dict(ip=ip))
# jaynes.run(dmc_debug)
# jaynes.run(torch_cuda)
# jaynes.run(gym_render)
jaynes.run(torch_upload)
jaynes.listen(200)
def launch(lr, model_name="LeNet"):
print(f"training model {model_name} with {lr}")
print('...')
print('This is working!!')
if __name__ == "__main__":
import jaynes
jaynes.config()
jaynes.run(launch, lr=1e-3)
# this line allows you to keep the pipe open and hear back from the remote instance.
jaynes.listen(200)
"""Minimal Exasmple for E-MAML running on HalfCheetahGoalDir-v0
"""
from e_maml_tf.config import RUN, G, Reporting, DEBUG
from e_maml_tf.train import run_e_maml
if __name__ == '__main__':
G.env_name = "HalfCheetahGoalDir-v0"
G.n_tasks = 20
G.n_graphs = 1
G.n_grad_steps = 5
G.meta_n_grad_steps = 1
# to debug this locally
run_e_maml()
# to launch with Jaynes on a SLURM cluster
import jaynes
jaynes.config('default')
jaynes.run(run_e_maml, _G=vars(G))
import jaynes
def launch(root, seed=None):
from ml_logger import logger
logger.configure(root_dir=root,
prefix=f"geyang/jaynes-demo/seed-{seed}",
register_experiment=True)
logger.print("this has be ran")
if __name__ == "__main__":
import os
logger_server = os.environ.get("ML_LOGGER_ROOT")
for seed in range(4):
# set the verbose to True to see everything
jaynes.config(verbose=False,
launch=dict(name=f"test-jaynes-launch-{seed}"))
jaynes.run(launch, root=logger_server, seed=seed)
jaynes.listen(100)
def train_fn(root, prefix=None):
from ml_logger import logger
logger.configure(root_dir=root, prefix=prefix)
logger.log_line(prefix, "launch is working.", file="o_hey")
if __name__ == '__main__':
import os
import jaynes
from ml_logger import logger, ROOT
jaynes.config('local')
jaynes.run(train_fn, root=ROOT, prefix=os.getcwd())
jaynes.listen()
import jaynes
green = lambda s: f"\x1b[32m{s}\x1b[0m"
def train_fn(seed=100):
from time import sleep
print(f'[seed: {seed}] See real-time pipe-back from the server:')
for i in range(3):
print(f"[seed: {seed}] step: {i}")
sleep(0.1)
print(green(f'[seed: {seed}] Finished!'))
if __name__ == "__main__":
jaynes.config(verbose=False)
for i in range(4):
jaynes.run(train_fn, seed=i * 100)
jaynes.listen(200)
"""A simple demo for launching ML jobs with Jaynes.
"""
def train_fn(some_variable=0):
import tensorflow as tf
print(f"tensorflow version: {tf.__version__}")
print('training is happening!')
print("some_variable is", some_variable)
if __name__ == "__main__":
import jaynes
jaynes.config('default')
jaynes.run(train_fn, some_variable=5)
jaynes.listen(timeout=60)