Python main Examples

Example #1

def ls_cli_main(*args, **kwargs):
    user_path = pathlib.Path(__file__).parent.joinpath("fs_modules")
    sys.argv.extend(["--user-dir", str(user_path)])
    main(*args, **kwargs)

Example #2

def test_num_plus_parser():
    ''' Ensure we catch errors relating to num_nodes/num_gpus + -i/-e being mutually exclusive'''

    # inclusion
    with pytest.raises(ValueError):
        dsrun.main(args="--num_nodes 1 -i localhost foo.py".split())
    with pytest.raises(ValueError):
        dsrun.main(args="--num_nodes 1 --num_gpus 1 -i localhost foo.py".split())
    with pytest.raises(ValueError):
        dsrun.main(args="--num_gpus 1 -i localhost foo.py".split())

    # exclusion
    with pytest.raises(ValueError):
        dsrun.main(args="--num_nodes 1 -e localhost foo.py".split())
    with pytest.raises(ValueError):
        dsrun.main(args="--num_nodes 1 --num_gpus 1 -e localhost foo.py".split())
    with pytest.raises(ValueError):
        dsrun.main(args="--num_gpus 1 -e localhost foo.py".split())

Example #3

File: deepy.py Project: ShivanshuPurohit/gpt-neox

import logging
import os

import deepspeed
from deepspeed.launcher.runner import main

logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))

from megatron.neox_arguments import NeoXArgs
from megatron.utils import get_wandb_api_key



neox_args = NeoXArgs.consume_deepy_args()
if neox_args.wandb_group is not None:
    # concat the wandb group name with a uid to make sure it's unique
    import wandb
    neox_args.wandb_group += "_" + wandb.util.generate_id()
neox_args.print()
deepspeed_main_args = neox_args.get_deepspeed_main_args()

# Extract wandb API key and inject into worker environments
wandb_token = get_wandb_api_key(neox_args=neox_args)
if wandb_token is not None:
    deepspeed.launcher.runner.EXPORT_ENVS.append('WANDB_API_KEY')
    os.environ['WANDB_API_KEY'] = wandb_token

if __name__ == '__main__':
    main(deepspeed_main_args)

Example #4

File: deepy.py Project: yingnengd/gpt-neox

    """ Gets the git commit hash of your current repo (if it exists) """
    try:
        git_hash = git_hash = subprocess.check_output(
            ["git", "describe", "--always"]).strip()
        git_hash = git_hash.decode()
    except subprocess.CalledProcessError:
        git_hash = None
    return git_hash


# Generate unique run group name
wandb_group = shortuuid.uuid()
extra_conf = {'wandb_group': wandb_group, 'git_hash': get_git_commit_hash()}

# Extract wandb API key and inject into worker environments
wandb_token = get_wandb_api_key()
if wandb_token is not None:
    deepspeed.launcher.runner.EXPORT_ENVS.append('WANDB_API_KEY')
    os.environ['WANDB_API_KEY'] = wandb_token

old_style_args, conf = ConfigMonster().consume_args(extra_conf=extra_conf)

if 'log-dir' in conf:
    os.makedirs(conf['log-dir'], exist_ok=True)
    file_prefix = os.path.join(conf['log-dir'], '0-deepy')
    Tee(file_prefix + '_stdout.txt', err=False)
    Tee(file_prefix + '_stderr.txt', err=True)

if __name__ == '__main__':
    main(old_style_args)

Example #5

File: deepy.py Project: chimjoHA/gpt-neox

import sys
import os
import deepspeed
from deepspeed.launcher.runner import main
import requests


def get_wandb_api_key():
    """ Get Weights and Biases API key from ENV or .netrc file. Otherwise return None """
    if 'WANDB_API_KEY' in os.environ:
        return os.environ['WANDB_API_KEY']

    wandb_token = requests.utils.get_netrc_auth('https://api.wandb.ai')

    if wandb_token is not None:
        return wandb_token[1]


# Generate unique run group name
wandb_group = shortuuid.uuid()
sys.argv.extend(['--group_name', wandb_group])

# Extract wandb API key and inject into worker environments
wandb_token = get_wandb_api_key()
if wandb_token is not None:
    deepspeed.launcher.runner.EXPORT_ENVS.append('WANDB_API_KEY')
    os.environ['WANDB_API_KEY'] = wandb_token

if __name__ == '__main__':
    main()