def configure(self, parser: ArgumentParser):
# parser.add_argument(
# '--name', default=self.image.name, type=str
# )
parser.add_argument('--profile', default='default', type=str)
parser.add_argument('--path', default=self.image.path, type=str)
parser.add_argument('--tag', default=self.image.tag, type=str)
parser.add_argument('--accounts',
default=",".join(self.image.accounts),
type=str)
bool_argument(parser,
'--force',
default=True,
help="If image already exists in ECR, do nothing")
bool_argument(parser,
'--pull',
default=True,
help="Pull latest version of FROM images if building")
bool_argument(parser,
'--push',
default=True,
help="Push results to ECR")
bool_argument(parser,
'--cache',
default=True,
help='Use cached layers during build')
bool_argument(parser, '--wsl', default=False, help='WSL path fix')
for k, v in self.image.download_files.items():
flag = "--{}".format(k.replace("_", "-"))
parser.add_argument(flag,
default=v,
type=str,
help=f"Download auxiliary file (default: {v})")
return super().configure(parser)
def train_args(parser,
max_epochs=10,
n_saved=10,
save_event='EPOCH_COMPLETED'):
parser.add_argument(
'--max-epochs',
type=int,
default=max_epochs,
metavar='N',
help='number of epochs to train (default: {})'.format(max_epochs))
parser.add_argument(
'--n-saved',
default=n_saved,
type=int,
help='Number of checkpoints to keep (default: ``{}``)'.format(n_saved))
parser.add_argument('--save-event', default=save_event, help='save event')
bool_argument(parser,
'--train-pbar',
default=True,
help='Enable train progress bar')
parser.add_argument('--train-print-event',
default=None,
help='training print event')
parser.add_argument('--train-log-event',
default=None,
help='training log event')
def mlflow_args(parser,
mlflow_enable=True,
mlflow_tracking_uri=None,
mlflow_tracking_password=None,
mlflow_tracking_username=None,
mlflow_tracking_secret_name=None,
mlflow_tracking_secret_region=None,
mlflow_tracking_secret_profile=None,
mlflow_experiment_name='default',
mlflow_run_name=None):
bool_argument(
parser,
'--mlflow-enable',
default=mlflow_enable,
help='Enable logging to MLflow (default: {})'.format(mlflow_enable))
parser.add_argument('--mlflow-experiment-name',
default=mlflow_experiment_name,
help='Experiment name in MLflow (default: {})'.format(
mlflow_experiment_name))
parser.add_argument(
'--mlflow-run-name',
default=mlflow_run_name,
help='Run name in MLflow (default: {})'.format(mlflow_run_name))
parser.add_argument(
'--mlflow-tracking-uri',
default=mlflow_tracking_uri,
help='URI of MLflow tracking server (default: ``{}``)'.format(
mlflow_tracking_uri))
parser.add_argument(
'--mlflow-tracking-username',
default=mlflow_tracking_username,
type=str,
help='Username for MLflow tracking server (default: ``{}``)'.format(
mlflow_tracking_username))
parser.add_argument(
'--mlflow-tracking-password',
default=mlflow_tracking_password,
type=str,
help='Password for MLflow tracking server (default: ``{}``)'.format(
mlflow_tracking_password))
parser.add_argument(
'--mlflow-tracking-secret-name',
default=mlflow_tracking_secret_name,
type=str,
help='Secret for accessing MLflow (default: ``{}``)'.format(
mlflow_tracking_secret_name))
parser.add_argument(
'--mlflow-tracking-secret-profile',
default=mlflow_tracking_secret_profile,
type=str,
help=
'Profile for accessing secret for accessing MLflow (default: ``{}``)'.
format(mlflow_tracking_secret_profile))
parser.add_argument(
'--mlflow-tracking-secret-region',
default=mlflow_tracking_secret_region,
type=str,
help='Region for accessing secret for accessing MLflow (default: ``{}``)'
.format(mlflow_tracking_secret_region))
def train_args(parser: argparse.ArgumentParser,
batch_size=64,
eval_batch_size=64,
max_epochs=100,
learning_rate=0.0003,
device=None,
mlflow_enable=True,
mlflow_tracking_uri=None,
mlflow_experiment_name='default',
mlflow_run_name=None,
arguments='all',
n_saved=5):
r"""
Configure arguments for training
Parameters
----------
parser : argparse.ArgumentParser
Argument parser to configure.
batch_size : int
Batch size.
arguments : list
List of arguments to include (e.g. ``['batch_size``]) or the string ``"all"`` to include all arguments.
(default: "all")
sagemaker_args : dict
Keyword arguments to ``aws-sagemaker-remote.training.main.sagemaker_training_main``.
See `aws_sagemaker_remote <https://aws-sagemaker-remote.readthedocs.io/en/latest/>`_.
"""
if arguments == 'all' or 'workers' in arguments:
parser.add_argument("--workers",
type=int,
default=2,
help="number of data loading workers (default: 2)")
if arguments == 'all' or 'batch_size' in arguments:
parser.add_argument(
'--batch-size',
type=int,
default=batch_size,
metavar='N',
help='input batch size for training (default: {})'.format(
batch_size))
if arguments == 'all' or 'test_batch_size' in arguments:
parser.add_argument(
'--test-batch-size',
type=int,
default=eval_batch_size,
metavar='N',
help='input batch size for testing (default: {})'.format(
eval_batch_size))
if arguments == 'all' or 'max_epochs' in arguments:
parser.add_argument(
'--max-epochs',
type=int,
default=max_epochs,
metavar='N',
help='number of epochs to train (default: {})'.format(max_epochs))
if arguments == 'all' or 'learning_rate' in arguments:
parser.add_argument(
'--learning-rate',
type=float,
default=learning_rate,
metavar='LR',
help='learning rate (default: {})'.format(learning_rate))
if arguments == 'all' or 'seed' in arguments:
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
if arguments == 'all' or 'mlflow_enable' in arguments:
bool_argument(parser,
'--mlflow-enable',
default=mlflow_enable,
help='Enable logging to MLflow (default: {})'.format(
mlflow_enable))
if arguments == 'all' or 'mlflow_experiment_name' in arguments:
parser.add_argument(
'--mlflow-experiment-name',
default=mlflow_experiment_name,
help='Experiment name in MLflow (default: {})'.format(
mlflow_experiment_name))
if arguments == 'all' or 'mlflow_run_name' in arguments:
parser.add_argument(
'--mlflow-run-name',
default=mlflow_run_name,
help='Run name in MLflow (default: {})'.format(mlflow_run_name))
if arguments == 'all' or 'mlflow_tracking_uri' in arguments:
parser.add_argument(
'--mlflow-tracking-uri',
default=mlflow_tracking_uri,
help='URI of MLflow tracking server (default: ``{}``)'.format(
mlflow_tracking_uri))
if arguments == 'all' or 'n_saved' in arguments:
parser.add_argument(
'--n-saved',
default=n_saved,
type=int,
help='Number of checkpoints to keep (default: ``{}``)'.format(
n_saved))
if arguments == 'all' or 'device' in arguments:
if device is None:
device = "cpu" if not torch.cuda.is_available() else "cuda"
parser.add_argument("--device",
type=str,
default=device,
help="device to use (default: {})".format(device))
"""
def batch_argparse_callback(parser: argparse.ArgumentParser,
config: BatchConfig):
parser.add_argument('--profile',
type=str,
default=config.profile,
help='AWS profile name')
parser.add_argument('--output-json',
type=str,
default=None,
help='Output job information to JSON file')
parser.add_argument(
'--stack-name',
type=str,
default=config.stack_name,
help=
f'AWS CloudFormation stack name to which resources are deployed (default: {config.stack_name})',
required=not config.stack_name)
parser.add_argument(
'--code-dir',
type=str,
default=config.code_dir,
help=f'Directory of Lambda code (default: {config.code_dir})',
required=not config.code_dir)
bool_argument(
parser,
'--deploy',
default=False,
help='Force Lambda deployment even if function already exists')
bool_argument(
parser,
'--deploy-only',
default=False,
help=
'Deploy and exit. Use `--deploy yes --deploy-only yes` to force deployment and exit'
)
bool_argument(parser,
'--confirmation-required',
default=True,
help='Require confirmation in console to run job')
bool_argument(parser,
'--development',
default=config.development,
help='Webpack in development mode')
parser.add_argument(
'--manifest',
type=str,
default=config.manifest,
help=
'File manifest to process. Must be a CSV with first column containing an S3 bucket and second column containing an S3 key.',
required=not config.manifest)
parser.add_argument('--report',
type=str,
default=config.report,
help='S3 path to store report')
parser.add_argument('--description',
type=str,
default=config.description,
help='Description of batch job')
parser.add_argument('--timeout',
type=int,
default=config.timeout,
help='Hard timeout of Lambda in seconds')
parser.add_argument(
'--ignore',
type=int,
default=0,
help=
'Number of columns of input CSV to ignore. Job will fail if CSV does not have 2+ignore columns. For example, if your CSV has bucket, key, and 5 more columns set ignore to 5.'
)
parser.add_argument('--memory',
type=int,
default=128,
help='Memory to allocate')
if config.support_soft:
parser.add_argument('--soft-timeout',
type=int,
default=config.soft_timeout,
help='Soft timeout in seconds')
# parser.add_argument(
# '--role-name', type=str, default=config.role_name, help='S3 path to store report'
# )
if config.argparse_callback:
config.argparse_callback(parser)
def eval_args(parser: argparse.ArgumentParser):
bool_argument(parser,
'--eval-pbar',
default=True,
help='Enable eval progress bar')
def batch_argparse_callback(parser: argparse.ArgumentParser,
config: BatchConfig):
parser.add_argument('--profile',
type=str,
default=config.profile,
help='AWS profile name')
parser.add_argument('--output-json',
type=str,
default=None,
help='Output job information to JSON file')
parser.add_argument('--stack-name',
type=str,
default=config.stack_name,
help='Stack name for deploying Lambda',
required=not config.stack_name)
parser.add_argument('--code-dir',
type=str,
default=config.code_dir,
help='Directory of Lambda code',
required=not config.code_dir)
bool_argument(
parser,
'--deploy',
default=False,
help='Force Lambda deployment even if function already exists')
bool_argument(
parser,
'--deploy-only',
default=False,
help=
'Deploy and exit. Use `--deploy yes --deploy-only yes` to force deployment and exit'
)
bool_argument(parser,
'--confirmation-required',
default=True,
help='Require confirmation in console to run job')
bool_argument(parser,
'--development',
default=config.development,
help='Require confirmation in console to run job')
parser.add_argument('--manifest',
type=str,
default=config.manifest,
help='File manifest to process',
required=not config.manifest)
parser.add_argument('--report',
type=str,
default=config.report,
help='S3 path to store report')
parser.add_argument('--description',
type=str,
default=config.description,
help='S3 path to store report')
parser.add_argument('--timeout',
type=int,
default=config.timeout,
help='S3 path to store report')
parser.add_argument('--ignore',
type=int,
default=0,
help='Columns to ignore')
parser.add_argument('--memory',
type=int,
default=128,
help='Memory to allocate')
parser.add_argument('--soft-timeout',
type=int,
default=config.soft_timeout,
help='S3 path to store report')
# parser.add_argument(
# '--role-name', type=str, default=config.role_name, help='S3 path to store report'
# )
if config.argparse_callback:
config.argparse_callback(parser)