s = """
# The Awesome ML-Logger
You can run the following code with ml-logger:
```python
from ml_logger import logger
logger.log(lr=0, clip range=0.200, step=0, timestamp='2018-11-16T00:09:27.198142', reward=-109.43)
logger.flush()
```
╒════════════════════╤════════════════════════════╕
│ lr │ 0.000 │
├────────────────────┼────────────────────────────┤
│ clip range │ 0.200 │
├────────────────────┼────────────────────────────┤
│ step │ 0 │
├────────────────────┼────────────────────────────┤
│ timestamp │'2018-11-16T00:09:27.198142'│
├────────────────────┼────────────────────────────┤
│ reward │ -109.43 │
╘════════════════════╧════════════════════════════╛
"""
print(s)
if __name__ == "__main__":
jaynes.config(mode='fair')
jaynes.run(launch)
jaynes.listen()
def work_fn():
print('working')
# for i in range(100):
# print('this is running', i)
# sleep(0.5)
# sys.stdout.flush()
if __name__ == '__main__':
import jaynes
# from dmc_gen.train import train
for i in range(60):
jaynes.config("vision", launch=dict(ip=f"visiongpu{i:02d}"))
jaynes.run(work_fn)
# for i in range(1):
# jaynes.config("vision-gpu")
# jaynes.run(work_fn)
# for i in range(60):
# jaynes.config("visiongpu", launch=dict(ip=f"visiongpu{i:02d}"),
# runner=dict(pypath="$HOME/jaynes-debug", work_dir="$HOME/jaynes-debug"), mounts=[], )
# jaynes.run(work_fn, aug_data_prefix="/afs/csail.mit.edu/u/g/geyang/mit/dmc_gen/custom_vendor/data")
# jaynes.config("supercloud", runner=dict(n_cpu=1, n_gpu=0))
# jaynes.run(train_fn)
jaynes.listen()
# highly non-rectangular
import jaynes
from launch_entry import train_fn
if __name__ == "__main__":
hosts = [
'visiongpu001',
'visiongpu002',
'visiongpu003',
]
for host in hosts:
jaynes.config(verbose=False, launch=dict(ip=host))
jaynes.run(train_fn)
jaynes.listen(200)
RUN.restart = True
RUN.prefix = "{username}/{project}/{file_stem}/{job_name}"
RUN.job_name = "{job_postfix}"
ms = [
# "visiongpu54",
# "improbable005",
# "improbable006",
# "improbable007",
# "improbable008",
# "improbable009",
# "improbable010",
]
if 'pydevd' in sys.modules:
jaynes.config("local")
else:
jaynes.config("supercloud", )
with Sweep(Args) as sweep, sweep.product:
with sweep.zip:
Args.domain = ['walker', 'cartpole', 'ball_in_cup', 'finger']
Args.task = ['walk', 'swingup', 'catch', 'spin']
Args.algo = ['pad', 'soda', 'curl', 'rad']
Args.seed = [100, 300, 400]
for i, args in enumerate(sweep):
# jaynes.config("vision", launch=dict(ip=ms[i]))
RUN.job_postfix = f"{Args.domain}-{Args.task}/{Args.algo}/{Args.seed}"
thunk = instr(train, args)
logger.log_text("""
logger.print('done')
# logger.remove(".")
# a = np.ones([1, 1, 100_000_000 // 4])
# logger.print(f"the size of the tensor is {a.size}")
# data = dict(key="ok", large=a)
# logger.torch_save(data, f"save/data-{logger.now('%H.%M.%S')}.pkl")
logger.print('done')
if __name__ == '__main__':
import jaynes
# jaynes.config("supercloud", launch=dict(n_gpu=0))
jaynes.config("local", launch=dict(n_gpu=0))
# ip = "visiongpu50"
# ip = "improbable005"
for ip in [
"improbable005",
# "improbable006",
# "improbable007",
# "improbable008",
# "improbable010"
]:
# jaynes.config("vision", launch=dict(ip=ip))
# jaynes.run(dmc_debug)
# jaynes.run(torch_cuda)
# jaynes.run(gym_render)
jaynes.run(torch_upload)
jaynes.listen(200)
def launch(lr, model_name="LeNet"):
print(f"training model {model_name} with {lr}")
print('...')
print('This is working!!')
if __name__ == "__main__":
import jaynes
jaynes.config()
jaynes.run(launch, lr=1e-3)
# this line allows you to keep the pipe open and hear back from the remote instance.
jaynes.listen(200)
with Sweep(Args) as sweep:
Args.gamma = 0.99
Args.clip_inputs = True
Args.normalize_inputs = True
Args.n_workers = 12
Args.n_epochs = 500
with sweep.product:
with sweep.zip:
Args.env_name = env_names
Args.seed = [100, 200, 300, 400, 500, 600]
# jaynes.config('local' if Args.debug else "gpu-vaughan", launch=dict(timeout=10000))
jaynes.config('local' if Args.debug else "cpu-mars")
# jaynes.config('local')
for i, deps in sweep.items():
thunk = instr(main,
deps,
_job_postfix=f"{Args.env_name.split(':')[-1]}")
logger.log_text("""
keys:
- run.status
- Args.env_name
- host.hostname
charts:
- yKey: test/success
xKey: epoch
- yKey: test/success
xKey: timesteps
from dmc_gen.config import Args
from dmc_gen_analysis import instr, RUN
with Sweep(Args) as sweep:
Args.load_checkpoint = "/geyang/dmc_gen/01_baselines/train/ball_in_cup-catch/rad/100"
Args.domain = 'ball_in_cup'
Args.task = 'catch'
Args.algo = 'rad'
Args.seed = 100
for i, args in enumerate(sweep):
if 'pydevd' in sys.modules:
jaynes.config('local')
else:
# jaynes.config("vision", launch=dict(ip=f"visiongpu{ms[i]:02d}"))
jaynes.config("supercloud", )
RUN.job_postfix = f"{Args.domain}-{Args.task}/{Args.algo}/{Args.seed}"
thunk = instr(train, args, _job_counter=False)
logger.log_text("""
keys:
- Args.domain
- Args.task
- Args.algo
- Args.seed
charts:
"""Minimal Exasmple for E-MAML running on HalfCheetahGoalDir-v0
"""
from e_maml_tf.config import RUN, G, Reporting, DEBUG
from e_maml_tf.train import run_e_maml
if __name__ == '__main__':
G.env_name = "HalfCheetahGoalDir-v0"
G.n_tasks = 20
G.n_graphs = 1
G.n_grad_steps = 5
G.meta_n_grad_steps = 1
# to debug this locally
run_e_maml()
# to launch with Jaynes on a SLURM cluster
import jaynes
jaynes.config('default')
jaynes.run(run_e_maml, _G=vars(G))
import jaynes
def launch(root, seed=None):
from ml_logger import logger
logger.configure(root_dir=root,
prefix=f"geyang/jaynes-demo/seed-{seed}",
register_experiment=True)
logger.print("this has be ran")
if __name__ == "__main__":
import os
logger_server = os.environ.get("ML_LOGGER_ROOT")
for seed in range(4):
# set the verbose to True to see everything
jaynes.config(verbose=False,
launch=dict(name=f"test-jaynes-launch-{seed}"))
jaynes.run(launch, root=logger_server, seed=seed)
jaynes.listen(100)
import jaynes
def launch():
from time import sleep
print("this is running")
sleep(10)
print("this is done")
if __name__ == "__main__":
jaynes.config(launch=dict(timeout=0))
for i in range(64):
jaynes.run(launch)
jaynes.listen()
def train_fn(root, prefix=None):
from ml_logger import logger
logger.configure(root_dir=root, prefix=prefix)
logger.log_line(prefix, "launch is working.", file="o_hey")
if __name__ == '__main__':
import os
import jaynes
from ml_logger import logger, ROOT
jaynes.config('local')
jaynes.run(train_fn, root=ROOT, prefix=os.getcwd())
jaynes.listen()
```
╒════════════════════╤════════════════════════════╕
│ lr │ 0.000 │
├────────────────────┼────────────────────────────┤
│ clip range │ 0.200 │
├────────────────────┼────────────────────────────┤
│ step │ 0 │
├────────────────────┼────────────────────────────┤
│ timestamp │'2018-11-16T00:09:27.198142'│
├────────────────────┼────────────────────────────┤
│ reward │ -109.43 │
╘════════════════════╧════════════════════════════╛
"""
print(s)
if __name__ == "__main__":
jaynes.config(verbose=False)
# try below
# jaynes.config(mode='oberyn', verbose=True)
# try run locally!
# jaynes.config(mode='local', verbose=True)
jaynes.run(launch)
# this line allows you to keep the pipe open and hear back from the remote instance.
jaynes.listen()