# sleep(0.5)
# sys.stdout.flush()
if __name__ == '__main__':
import jaynes
# from dmc_gen.train import train
for i in range(60):
jaynes.config("vision", launch=dict(ip=f"visiongpu{i:02d}"))
jaynes.run(work_fn)
# for i in range(1):
# jaynes.config("vision-gpu")
# jaynes.run(work_fn)
# for i in range(60):
# jaynes.config("visiongpu", launch=dict(ip=f"visiongpu{i:02d}"),
# runner=dict(pypath="$HOME/jaynes-debug", work_dir="$HOME/jaynes-debug"), mounts=[], )
# jaynes.run(work_fn, aug_data_prefix="/afs/csail.mit.edu/u/g/geyang/mit/dmc_gen/custom_vendor/data")
# jaynes.config("supercloud", runner=dict(n_cpu=1, n_gpu=0))
# jaynes.run(train_fn)
jaynes.listen()
# highly non-rectangular
# custom cuda kernels
# how do fully general tensor product
#
# Does it make it harder to optimize. Higher order
import jaynes
def launch(root, seed=None):
from ml_logger import logger
logger.configure(root_dir=root,
prefix=f"geyang/jaynes-demo/seed-{seed}",
register_experiment=True)
logger.print("this has be ran")
if __name__ == "__main__":
import os
logger_server = os.environ.get("ML_LOGGER_ROOT")
for seed in range(4):
# set the verbose to True to see everything
jaynes.config(verbose=False,
launch=dict(name=f"test-jaynes-launch-{seed}"))
jaynes.run(launch, root=logger_server, seed=seed)
jaynes.listen(100)
import jaynes
from launch_entry import train_fn
if __name__ == "__main__":
hosts = [
'visiongpu001',
'visiongpu002',
'visiongpu003',
]
for host in hosts:
jaynes.config(verbose=False, launch=dict(ip=host))
jaynes.run(train_fn)
jaynes.listen(200)
"""A simple demo for launching ML jobs with Jaynes.
"""
def train_fn(some_variable=0):
import tensorflow as tf
print(f"tensorflow version: {tf.__version__}")
print('training is happening!')
print("some_variable is", some_variable)
if __name__ == "__main__":
import jaynes
jaynes.config('default')
jaynes.run(train_fn, some_variable=5)
jaynes.listen(timeout=60)
