def ls_cli_main(*args, **kwargs):
user_path = pathlib.Path(__file__).parent.joinpath("fs_modules")
sys.argv.extend(["--user-dir", str(user_path)])
main(*args, **kwargs)
def test_num_plus_parser():
''' Ensure we catch errors relating to num_nodes/num_gpus + -i/-e being mutually exclusive'''
# inclusion
with pytest.raises(ValueError):
dsrun.main(args="--num_nodes 1 -i localhost foo.py".split())
with pytest.raises(ValueError):
dsrun.main(args="--num_nodes 1 --num_gpus 1 -i localhost foo.py".split())
with pytest.raises(ValueError):
dsrun.main(args="--num_gpus 1 -i localhost foo.py".split())
# exclusion
with pytest.raises(ValueError):
dsrun.main(args="--num_nodes 1 -e localhost foo.py".split())
with pytest.raises(ValueError):
dsrun.main(args="--num_nodes 1 --num_gpus 1 -e localhost foo.py".split())
with pytest.raises(ValueError):
dsrun.main(args="--num_gpus 1 -e localhost foo.py".split())
import logging
import os
import deepspeed
from deepspeed.launcher.runner import main
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
from megatron.neox_arguments import NeoXArgs
from megatron.utils import get_wandb_api_key
neox_args = NeoXArgs.consume_deepy_args()
if neox_args.wandb_group is not None:
# concat the wandb group name with a uid to make sure it's unique
import wandb
neox_args.wandb_group += "_" + wandb.util.generate_id()
neox_args.print()
deepspeed_main_args = neox_args.get_deepspeed_main_args()
# Extract wandb API key and inject into worker environments
wandb_token = get_wandb_api_key(neox_args=neox_args)
if wandb_token is not None:
deepspeed.launcher.runner.EXPORT_ENVS.append('WANDB_API_KEY')
os.environ['WANDB_API_KEY'] = wandb_token
if __name__ == '__main__':
main(deepspeed_main_args)
""" Gets the git commit hash of your current repo (if it exists) """
try:
git_hash = git_hash = subprocess.check_output(
["git", "describe", "--always"]).strip()
git_hash = git_hash.decode()
except subprocess.CalledProcessError:
git_hash = None
return git_hash
# Generate unique run group name
wandb_group = shortuuid.uuid()
extra_conf = {'wandb_group': wandb_group, 'git_hash': get_git_commit_hash()}
# Extract wandb API key and inject into worker environments
wandb_token = get_wandb_api_key()
if wandb_token is not None:
deepspeed.launcher.runner.EXPORT_ENVS.append('WANDB_API_KEY')
os.environ['WANDB_API_KEY'] = wandb_token
old_style_args, conf = ConfigMonster().consume_args(extra_conf=extra_conf)
if 'log-dir' in conf:
os.makedirs(conf['log-dir'], exist_ok=True)
file_prefix = os.path.join(conf['log-dir'], '0-deepy')
Tee(file_prefix + '_stdout.txt', err=False)
Tee(file_prefix + '_stderr.txt', err=True)
if __name__ == '__main__':
main(old_style_args)
import sys
import os
import deepspeed
from deepspeed.launcher.runner import main
import requests
def get_wandb_api_key():
""" Get Weights and Biases API key from ENV or .netrc file. Otherwise return None """
if 'WANDB_API_KEY' in os.environ:
return os.environ['WANDB_API_KEY']
wandb_token = requests.utils.get_netrc_auth('https://api.wandb.ai')
if wandb_token is not None:
return wandb_token[1]
# Generate unique run group name
wandb_group = shortuuid.uuid()
sys.argv.extend(['--group_name', wandb_group])
# Extract wandb API key and inject into worker environments
wandb_token = get_wandb_api_key()
if wandb_token is not None:
deepspeed.launcher.runner.EXPORT_ENVS.append('WANDB_API_KEY')
os.environ['WANDB_API_KEY'] = wandb_token
if __name__ == '__main__':
main()